src/input/MsxChar2Unicode.cc at main · doomsdayone/openmsx

◆ main openmsx / src / input / MsxChar2Unicode.cc

6.3 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

#include "MsxChar2Unicode.hh"

#include "File.hh"
#include "FileContext.hh"
#include "FileException.hh"
#include "MSXException.hh"

#include "StringOp.hh"
#include "one_of.hh"
#include "ranges.hh"
#include "strCat.hh"
#include "utf8_unchecked.hh"

#include <algorithm>
#include <bit>

namespace openmsx {

MsxChar2Unicode::MsxChar2Unicode(std::string_view mappingName)
{
	std::ranges::fill(msx2unicode, uint32_t(-1));

	std::string filename;
	try {
		filename = systemFileContext().resolve(
			tmpStrCat("unicodemaps/character_set_mappings/", mappingName));
	} catch (FileException& e) {
		throw MSXException("Couldn't find MSX character mapping file that was specified in unicodemap: ", mappingName, " (", e.getMessage(), ")");
	}
	try {
		auto buf = File(filename).mmap<const char>();
		parseVid(std::string_view(buf.data(), buf.size()));
	} catch (FileException&) {
		throw MSXException("Couldn't load MSX character mapping file that was specified in unicodemap: ", filename);
	} catch (MSXException& e) {
		throw MSXException(e.getMessage(), " in ", filename);
	}
}

/* Remove the next line from 'text' and return it.
 * A line is everything upto the first newline character. The character is
 * removed from 'text' but not included in the return value.
 */
static constexpr std::string_view getLine(std::string_view& text)
{
	if (auto pos = text.find_first_of('\n'); pos != std::string_view::npos) {
		// handle both 'LF' and 'CR LF'
		auto pos2 = ((pos != 0) && (text[pos - 1] == '\r')) ? pos - 1 : pos;
		auto result = text.substr(0, pos2);
		text.remove_prefix(pos + 1);
		return result;
	}
	return std::exchange(text, {});
}

/* Return the given line with comments at the end of the line removed.
 * Comments start at the first '#' character and continue till the end of the
 * line.
 */
[[nodiscard]] static constexpr std::string_view stripComments(std::string_view line)
{
	if (auto pos = line.find_first_of('#'); pos != std::string_view::npos) {
		line = line.substr(0, pos);
	}
	return line;
}

/* Returns true iff the given character is a separator (whitespace).
 * Newline and hash-mark are handled (already removed) by other functions.
 */
[[nodiscard]] static constexpr bool isSep(char c)
{
	return c == one_of(' ', '\t', '\r'); // whitespace
}

/* Remove one token from 'line' and return it.
 * Tokens are separated by one or more separator character as defined by 'isSep()'.
 * Those separators are removed from 'line' but not included in the return value.
 * The assumption is that there are no leading separator characters before calling this function.
 */
static constexpr std::string_view getToken(std::string_view& line)
{
	size_t s = line.size();
	size_t i = 0;
	while ((i < s) && !isSep(line[i])) {
		++i;
	}
	auto result = line.substr(0, i);
	while ((i < s) && isSep(line[i])) {
		++i;
	}
	line.remove_prefix(i);
	return result;
}

void MsxChar2Unicode::parseVid(std::string_view file)
{
	// The general syntax of this file is
	//     <msx-char> <unicode> # comments
	// Fields are separated via whitespace (tabs or spaces).
	// For example:
	//     0x2A	0x002A	# ASTERISK

	// Usually each msx-char only has a single corresponding unicode, and
	// then 'unicode2msx.size()' will be (close to) 256. But for example
	// this is not the case in 'MSXVIDAR.TXT'.
	unicode2msx.reserve(256);

	while (!file.empty()) {
		auto origLine = getLine(file);
		auto line = stripComments(origLine);

		auto msxTok = getToken(line);
		if (msxTok.empty()) continue; // empty line (or only whitespace / comments)
		auto msx = StringOp::stringTo<uint8_t>(msxTok);
		if (!msx) {
			throw MSXException("Invalid msx character value, expected an "
			                   "integer in range 0x00..0xff, but got: ", msxTok);
		}

		auto unicodeTok = getToken(line);
		if ((unicodeTok.size() >= 5) && (unicodeTok[0] == '<') &&
		    (unicodeTok[3] == '>')   && (unicodeTok[4] == '+')) {
			// In some files the <unicode> field is preceded with an annotation like:
			//   <LR>+0x0020     left-to-right
			//   <RL>+0x0020     right-to-left
			//   <RV>+0x0020     reverse-video
			// Just strip out that annotation and ignore. Current
			// implementation assumes the code 'LR', 'RL', 'RV' is
			// exactly two characters long.
			unicodeTok.remove_prefix(5);
		}
		auto unicode = StringOp::stringTo<uint32_t>(unicodeTok);
		if (!unicode || *unicode > 0x10ffff) {
			throw MSXException("Invalid unicode character value, expected an "
			                   "integer in range 0..0x10ffff, but got: ", unicodeTok);
		}

		if (!line.empty()) {
			throw MSXException("Syntax error, expected \"<msx-char> <unicode>\", "
			                   "but got: ", origLine);
		}

		// There can be duplicates (e.g. in 'MSXVIDAR.TXT'), in that
		// case only keep the first entry ...
		if (msx2unicode[*msx] == uint32_t(-1)) {
			msx2unicode[*msx] = *unicode;
		}
		// ... but (for now) keep all unicode->msx mappings (duplicates are removed below).
		unicode2msx.emplace_back(*unicode, *msx);
	}

	// Sort on unicode (for later binary-search). If there are duplicate
	// unicodes (with different msx-code), then keep the first entry (hence
	// use stable_sort).
	std::ranges::stable_sort(unicode2msx, {}, &Entry::unicode);
	auto u = std::ranges::unique(unicode2msx, {}, &Entry::unicode);
	unicode2msx.erase(u.begin(), u.end());
}

std::string MsxChar2Unicode::msxToUtf8(
	std::span<const uint8_t> msx, const std::function<uint32_t(uint8_t)>& fallback) const
{
	std::string utf8;
	utf8.reserve(msx.size()); // possibly underestimation, but that's fine
	auto out = std::back_inserter(utf8);
	for (auto m : msx) {
		auto u = msx2unicode[m];
		auto u2 = (u != uint32_t(-1)) ? u : fallback(m);
		out = utf8::unchecked::append(u2, out);
	}
	return utf8;
}

std::vector<uint8_t> MsxChar2Unicode::utf8ToMsx(
	std::string_view utf8, const std::function<uint8_t(uint32_t)>& fallback) const
{
	std::vector<uint8_t> msx;
	auto it = utf8.begin(), et = utf8.end();
	while (it != et) {
		auto u = utf8::unchecked::next(it);
		auto m = binary_find(unicode2msx, u, {}, &Entry::unicode);
		msx.push_back(m ? m->msx : fallback(u));
	}
	return msx;
}

std::string MsxChar2Unicode::msxToUtf8(std::span<const uint8_t> msx, char fallback) const
{
	return msxToUtf8(msx, [&](uint32_t) { return fallback; });
}

std::vector<uint8_t> MsxChar2Unicode::utf8ToMsx(std::string_view utf8, char fallback) const
{
	return utf8ToMsx(utf8, [&](uint8_t) { return fallback; });
}


} // namespace openmsx

e8555c89 Andy McCall 1 month ago	1	#include "MsxChar2Unicode.hh"
	2
	3	#include "File.hh"
	4	#include "FileContext.hh"
	5	#include "FileException.hh"
	6	#include "MSXException.hh"
	7
	8	#include "StringOp.hh"
	9	#include "one_of.hh"
	10	#include "ranges.hh"
	11	#include "strCat.hh"
	12	#include "utf8_unchecked.hh"
	13
	14	#include <algorithm>
	15	#include <bit>
	16
	17	namespace openmsx {
	18
	19	MsxChar2Unicode::MsxChar2Unicode(std::string_view mappingName)
	20	{
	21	std::ranges::fill(msx2unicode, uint32_t(-1));
	22
	23	std::string filename;
	24	try {
	25	filename = systemFileContext().resolve(
	26	tmpStrCat("unicodemaps/character_set_mappings/", mappingName));
	27	} catch (FileException& e) {
	28	throw MSXException("Couldn't find MSX character mapping file that was specified in unicodemap: ", mappingName, " (", e.getMessage(), ")");
	29	}
	30	try {
	31	auto buf = File(filename).mmap<const char>();
	32	parseVid(std::string_view(buf.data(), buf.size()));
	33	} catch (FileException&) {
	34	throw MSXException("Couldn't load MSX character mapping file that was specified in unicodemap: ", filename);
	35	} catch (MSXException& e) {
	36	throw MSXException(e.getMessage(), " in ", filename);
	37	}
	38	}
	39
	40	/* Remove the next line from 'text' and return it.
	41	* A line is everything upto the first newline character. The character is
	42	* removed from 'text' but not included in the return value.
	43	*/
	44	static constexpr std::string_view getLine(std::string_view& text)
	45	{
	46	if (auto pos = text.find_first_of('\n'); pos != std::string_view::npos) {
	47	// handle both 'LF' and 'CR LF'
	48	auto pos2 = ((pos != 0) && (text[pos - 1] == '\r')) ? pos - 1 : pos;
	49	auto result = text.substr(0, pos2);
	50	text.remove_prefix(pos + 1);
	51	return result;
	52	}
	53	return std::exchange(text, {});
	54	}
	55
	56	/* Return the given line with comments at the end of the line removed.
	57	* Comments start at the first '#' character and continue till the end of the
	58	* line.
	59	*/
	60	[[nodiscard]] static constexpr std::string_view stripComments(std::string_view line)
	61	{
	62	if (auto pos = line.find_first_of('#'); pos != std::string_view::npos) {
	63	line = line.substr(0, pos);
	64	}
65	return line;
66	}
67
68	/* Returns true iff the given character is a separator (whitespace).
69	* Newline and hash-mark are handled (already removed) by other functions.
70	*/
71	[[nodiscard]] static constexpr bool isSep(char c)
72	{
73	return c == one_of(' ', '\t', '\r'); // whitespace
74	}
75
76	/* Remove one token from 'line' and return it.
77	* Tokens are separated by one or more separator character as defined by 'isSep()'.
78	* Those separators are removed from 'line' but not included in the return value.
79	* The assumption is that there are no leading separator characters before calling this function.
80	*/
81	static constexpr std::string_view getToken(std::string_view& line)
82	{
83	size_t s = line.size();
84	size_t i = 0;
85	while ((i < s) && !isSep(line[i])) {
86	++i;
87	}
88	auto result = line.substr(0, i);
89	while ((i < s) && isSep(line[i])) {
90	++i;
91	}
92	line.remove_prefix(i);
93	return result;
94	}
95
96	void MsxChar2Unicode::parseVid(std::string_view file)
97	{
98	// The general syntax of this file is
99	// <msx-char> <unicode> # comments
100	// Fields are separated via whitespace (tabs or spaces).
101	// For example:
102	// 0x2A 0x002A # ASTERISK
103
104	// Usually each msx-char only has a single corresponding unicode, and
105	// then 'unicode2msx.size()' will be (close to) 256. But for example
106	// this is not the case in 'MSXVIDAR.TXT'.
107	unicode2msx.reserve(256);
108
109	while (!file.empty()) {
110	auto origLine = getLine(file);
111	auto line = stripComments(origLine);
112
113	auto msxTok = getToken(line);
114	if (msxTok.empty()) continue; // empty line (or only whitespace / comments)
115	auto msx = StringOp::stringTo<uint8_t>(msxTok);
116	if (!msx) {
117	throw MSXException("Invalid msx character value, expected an "
118	"integer in range 0x00..0xff, but got: ", msxTok);
119	}
120
121	auto unicodeTok = getToken(line);
122	if ((unicodeTok.size() >= 5) && (unicodeTok[0] == '<') &&
123	(unicodeTok[3] == '>') && (unicodeTok[4] == '+')) {
124	// In some files the <unicode> field is preceded with an annotation like:
125	// <LR>+0x0020 left-to-right
126	// <RL>+0x0020 right-to-left
127	// <RV>+0x0020 reverse-video
128	// Just strip out that annotation and ignore. Current
129	// implementation assumes the code 'LR', 'RL', 'RV' is
130	// exactly two characters long.
131	unicodeTok.remove_prefix(5);
132	}
133	auto unicode = StringOp::stringTo<uint32_t>(unicodeTok);
134	if (!unicode \|\| *unicode > 0x10ffff) {
135	throw MSXException("Invalid unicode character value, expected an "
136	"integer in range 0..0x10ffff, but got: ", unicodeTok);
137	}
138
139	if (!line.empty()) {
140	throw MSXException("Syntax error, expected \"<msx-char> <unicode>\", "
141	"but got: ", origLine);
142	}
143
144	// There can be duplicates (e.g. in 'MSXVIDAR.TXT'), in that
145	// case only keep the first entry ...
146	if (msx2unicode[*msx] == uint32_t(-1)) {
147	msx2unicode[msx] = unicode;
148	}
149	// ... but (for now) keep all unicode->msx mappings (duplicates are removed below).
150	unicode2msx.emplace_back(unicode, msx);
151	}
152
153	// Sort on unicode (for later binary-search). If there are duplicate
154	// unicodes (with different msx-code), then keep the first entry (hence
155	// use stable_sort).
156	std::ranges::stable_sort(unicode2msx, {}, &Entry::unicode);
157	auto u = std::ranges::unique(unicode2msx, {}, &Entry::unicode);
158	unicode2msx.erase(u.begin(), u.end());
159	}
160
161	std::string MsxChar2Unicode::msxToUtf8(
162	std::span<const uint8_t> msx, const std::function<uint32_t(uint8_t)>& fallback) const
163	{
164	std::string utf8;
165	utf8.reserve(msx.size()); // possibly underestimation, but that's fine
166	auto out = std::back_inserter(utf8);
167	for (auto m : msx) {
168	auto u = msx2unicode[m];
169	auto u2 = (u != uint32_t(-1)) ? u : fallback(m);
170	out = utf8::unchecked::append(u2, out);
171	}
172	return utf8;
173	}
174
175	std::vector<uint8_t> MsxChar2Unicode::utf8ToMsx(
176	std::string_view utf8, const std::function<uint8_t(uint32_t)>& fallback) const
177	{
178	std::vector<uint8_t> msx;
179	auto it = utf8.begin(), et = utf8.end();
180	while (it != et) {
181	auto u = utf8::unchecked::next(it);
182	auto m = binary_find(unicode2msx, u, {}, &Entry::unicode);
183	msx.push_back(m ? m->msx : fallback(u));
184	}
185	return msx;
186	}
187
188	std::string MsxChar2Unicode::msxToUtf8(std::span<const uint8_t> msx, char fallback) const
189	{
190	return msxToUtf8(msx, [&](uint32_t) { return fallback; });
191	}
192
193	std::vector<uint8_t> MsxChar2Unicode::utf8ToMsx(std::string_view utf8, char fallback) const
194	{
195	return utf8ToMsx(utf8, [&](uint8_t) { return fallback; });
196	}
197
198
199	} // namespace openmsx

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

Keyboard