OXYGENHUB
◆ main openmsx / src / input / MsxChar2Unicode.cc
6.3 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw
1 
#include "MsxChar2Unicode.hh"
2
3 
#include "File.hh"
4 
#include "FileContext.hh"
5 
#include "FileException.hh"
6 
#include "MSXException.hh"
7
8 
#include "StringOp.hh"
9 
#include "one_of.hh"
10 
#include "ranges.hh"
11 
#include "strCat.hh"
12 
#include "utf8_unchecked.hh"
13
14 
#include <algorithm>
15 
#include <bit>
16
17 
namespace openmsx {
18
19 
MsxChar2Unicode::MsxChar2Unicode(std::string_view mappingName)
20 
{
21 
	std::ranges::fill(msx2unicode, uint32_t(-1));
22
23 
	std::string filename;
24 
	try {
25 
		filename = systemFileContext().resolve(
26 
			tmpStrCat("unicodemaps/character_set_mappings/", mappingName));
27 
	} catch (FileException& e) {
28 
		throw MSXException("Couldn't find MSX character mapping file that was specified in unicodemap: ", mappingName, " (", e.getMessage(), ")");
29 
	}
30 
	try {
31 
		auto buf = File(filename).mmap<const char>();
32 
		parseVid(std::string_view(buf.data(), buf.size()));
33 
	} catch (FileException&) {
34 
		throw MSXException("Couldn't load MSX character mapping file that was specified in unicodemap: ", filename);
35 
	} catch (MSXException& e) {
36 
		throw MSXException(e.getMessage(), " in ", filename);
37 
	}
38 
}
39
40 
/* Remove the next line from 'text' and return it.
41 
 * A line is everything upto the first newline character. The character is
42 
 * removed from 'text' but not included in the return value.
43 
 */
44 
static constexpr std::string_view getLine(std::string_view& text)
45 
{
46 
	if (auto pos = text.find_first_of('\n'); pos != std::string_view::npos) {
47 
		// handle both 'LF' and 'CR LF'
48 
		auto pos2 = ((pos != 0) && (text[pos - 1] == '\r')) ? pos - 1 : pos;
49 
		auto result = text.substr(0, pos2);
50 
		text.remove_prefix(pos + 1);
51 
		return result;
52 
	}
53 
	return std::exchange(text, {});
54 
}
55
56 
/* Return the given line with comments at the end of the line removed.
57 
 * Comments start at the first '#' character and continue till the end of the
58 
 * line.
59 
 */
60 
[[nodiscard]] static constexpr std::string_view stripComments(std::string_view line)
61 
{
62 
	if (auto pos = line.find_first_of('#'); pos != std::string_view::npos) {
63 
		line = line.substr(0, pos);
64 
	}
65 
	return line;
66 
}
67
68 
/* Returns true iff the given character is a separator (whitespace).
69 
 * Newline and hash-mark are handled (already removed) by other functions.
70 
 */
71 
[[nodiscard]] static constexpr bool isSep(char c)
72 
{
73 
	return c == one_of(' ', '\t', '\r'); // whitespace
74 
}
75
76 
/* Remove one token from 'line' and return it.
77 
 * Tokens are separated by one or more separator character as defined by 'isSep()'.
78 
 * Those separators are removed from 'line' but not included in the return value.
79 
 * The assumption is that there are no leading separator characters before calling this function.
80 
 */
81 
static constexpr std::string_view getToken(std::string_view& line)
82 
{
83 
	size_t s = line.size();
84 
	size_t i = 0;
85 
	while ((i < s) && !isSep(line[i])) {
86 
		++i;
87 
	}
88 
	auto result = line.substr(0, i);
89 
	while ((i < s) && isSep(line[i])) {
90 
		++i;
91 
	}
92 
	line.remove_prefix(i);
93 
	return result;
94 
}
95
96 
void MsxChar2Unicode::parseVid(std::string_view file)
97 
{
98 
	// The general syntax of this file is
99 
	//     <msx-char> <unicode> # comments
100 
	// Fields are separated via whitespace (tabs or spaces).
101 
	// For example:
102 
	//     0x2A	0x002A	# ASTERISK
103
104 
	// Usually each msx-char only has a single corresponding unicode, and
105 
	// then 'unicode2msx.size()' will be (close to) 256. But for example
106 
	// this is not the case in 'MSXVIDAR.TXT'.
107 
	unicode2msx.reserve(256);
108
109 
	while (!file.empty()) {
110 
		auto origLine = getLine(file);
111 
		auto line = stripComments(origLine);
112
113 
		auto msxTok = getToken(line);
114 
		if (msxTok.empty()) continue; // empty line (or only whitespace / comments)
115 
		auto msx = StringOp::stringTo<uint8_t>(msxTok);
116 
		if (!msx) {
117 
			throw MSXException("Invalid msx character value, expected an "
118 
			                   "integer in range 0x00..0xff, but got: ", msxTok);
119 
		}
120
121 
		auto unicodeTok = getToken(line);
122 
		if ((unicodeTok.size() >= 5) && (unicodeTok[0] == '<') &&
123 
		    (unicodeTok[3] == '>')   && (unicodeTok[4] == '+')) {
124 
			// In some files the <unicode> field is preceded with an annotation like:
125 
			//   <LR>+0x0020     left-to-right
126 
			//   <RL>+0x0020     right-to-left
127 
			//   <RV>+0x0020     reverse-video
128 
			// Just strip out that annotation and ignore. Current
129 
			// implementation assumes the code 'LR', 'RL', 'RV' is
130 
			// exactly two characters long.
131 
			unicodeTok.remove_prefix(5);
132 
		}
133 
		auto unicode = StringOp::stringTo<uint32_t>(unicodeTok);
134 
		if (!unicode || *unicode > 0x10ffff) {
135 
			throw MSXException("Invalid unicode character value, expected an "
136 
			                   "integer in range 0..0x10ffff, but got: ", unicodeTok);
137 
		}
138
139 
		if (!line.empty()) {
140 
			throw MSXException("Syntax error, expected \"<msx-char> <unicode>\", "
141 
			                   "but got: ", origLine);
142 
		}
143
144 
		// There can be duplicates (e.g. in 'MSXVIDAR.TXT'), in that
145 
		// case only keep the first entry ...
146 
		if (msx2unicode[*msx] == uint32_t(-1)) {
147 
			msx2unicode[*msx] = *unicode;
148 
		}
149 
		// ... but (for now) keep all unicode->msx mappings (duplicates are removed below).
150 
		unicode2msx.emplace_back(*unicode, *msx);
151 
	}
152
153 
	// Sort on unicode (for later binary-search). If there are duplicate
154 
	// unicodes (with different msx-code), then keep the first entry (hence
155 
	// use stable_sort).
156 
	std::ranges::stable_sort(unicode2msx, {}, &Entry::unicode);
157 
	auto u = std::ranges::unique(unicode2msx, {}, &Entry::unicode);
158 
	unicode2msx.erase(u.begin(), u.end());
159 
}
160
161 
std::string MsxChar2Unicode::msxToUtf8(
162 
	std::span<const uint8_t> msx, const std::function<uint32_t(uint8_t)>& fallback) const
163 
{
164 
	std::string utf8;
165 
	utf8.reserve(msx.size()); // possibly underestimation, but that's fine
166 
	auto out = std::back_inserter(utf8);
167 
	for (auto m : msx) {
168 
		auto u = msx2unicode[m];
169 
		auto u2 = (u != uint32_t(-1)) ? u : fallback(m);
170 
		out = utf8::unchecked::append(u2, out);
171 
	}
172 
	return utf8;
173 
}
174
175 
std::vector<uint8_t> MsxChar2Unicode::utf8ToMsx(
176 
	std::string_view utf8, const std::function<uint8_t(uint32_t)>& fallback) const
177 
{
178 
	std::vector<uint8_t> msx;
179 
	auto it = utf8.begin(), et = utf8.end();
180 
	while (it != et) {
181 
		auto u = utf8::unchecked::next(it);
182 
		auto m = binary_find(unicode2msx, u, {}, &Entry::unicode);
183 
		msx.push_back(m ? m->msx : fallback(u));
184 
	}
185 
	return msx;
186 
}
187
188 
std::string MsxChar2Unicode::msxToUtf8(std::span<const uint8_t> msx, char fallback) const
189 
{
190 
	return msxToUtf8(msx, [&](uint32_t) { return fallback; });
191 
}
192
193 
std::vector<uint8_t> MsxChar2Unicode::utf8ToMsx(std::string_view utf8, char fallback) const
194 
{
195 
	return utf8ToMsx(utf8, [&](uint8_t) { return fallback; });
196 
}
197
198
199 
} // namespace openmsx