src/utils/utf8_core.hh at main · doomsdayone/openmsx

◆ main openmsx / src / utils / utf8_core.hh

6.8 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

// UTF8-CPP   http://utfcpp.sourceforge.net/
//   Slightly simplified (and reformatted) to fit openMSX coding style.

// Copyright 2006 Nemanja Trifunovic

/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:

The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/


#ifndef UTF8_CORE_HH
#define UTF8_CORE_HH

#include "narrow.hh"
#include "one_of.hh"

#include <array>
#include <cstdint>
#include <iterator>

namespace utf8 {

// Helper code - not intended to be directly called by the library users.
// May be changed at any time
namespace internal {

// Unicode constants
// Leading (high) surrogates: 0xd800 - 0xdbff
// Trailing (low) surrogates: 0xdc00 - 0xdfff
inline constexpr uint16_t LEAD_SURROGATE_MIN  = 0xd800u;
inline constexpr uint16_t LEAD_SURROGATE_MAX  = 0xdbffu;
inline constexpr uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
inline constexpr uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
inline constexpr uint16_t LEAD_OFFSET         = LEAD_SURROGATE_MIN - (0x10000 >> 10);
inline constexpr uint32_t SURROGATE_OFFSET    = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;

// Maximum valid value for a Unicode code point
inline constexpr uint32_t CODE_POINT_MAX      = 0x0010ffffu;

[[nodiscard]] constexpr bool is_trail(uint8_t oc)
{
	return (oc >> 6) == 0x2;
}

[[nodiscard]] constexpr bool is_surrogate(uint32_t cp)
{
	return (cp >= LEAD_SURROGATE_MIN) && (cp <= TRAIL_SURROGATE_MAX);
}

[[nodiscard]] constexpr bool is_code_point_valid(uint32_t cp)
{
	return (cp <= CODE_POINT_MAX) && !is_surrogate(cp) &&
	       (cp != one_of(0xfffeu, 0xffffu));
}

[[nodiscard]] constexpr unsigned sequence_length(uint8_t lead)
{
	if (lead < 0x80) {
		return 1;
	} else if ((lead >> 5) == 0x06) {
		return 2;
	} else if ((lead >> 4) == 0x0e) {
		return 3;
	} else if ((lead >> 3) == 0x1e) {
		return 4;
	} else {
		return 0;
	}
}

enum class utf_error : uint8_t {
	OK,
	NOT_ENOUGH_ROOM,
	INVALID_LEAD,
	INCOMPLETE_SEQUENCE,
	OVERLONG_SEQUENCE,
	INVALID_CODE_POINT
};

template<typename octet_iterator>
[[nodiscard]] constexpr utf_error validate_next(octet_iterator& it, octet_iterator end,
                                      uint32_t* code_point)
{
	using enum utf_error;
	uint32_t cp = narrow_cast<unsigned char>(*it);
	// Check the lead octet
	int length = sequence_length(*it);

	// "Shortcut" for ASCII characters
	if (length == 1) {
		if (end - it <= 0) {
			return NOT_ENOUGH_ROOM;
		}
		if (code_point) {
			*code_point = cp;
		}
		++it;
		return OK;
	}

	// Do we have enough memory?
	if (std::distance(it, end) < length) {
		return NOT_ENOUGH_ROOM;
	}

	// Check trail octets and calculate the code point
	switch (length) {
	case 0:
		return INVALID_LEAD;
	case 2:
		if (is_trail(*(++it))) {
			cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
		} else {
			--it;
			return INCOMPLETE_SEQUENCE;
		}
		break;
	case 3:
		if (is_trail(*(++it))) {
			cp = ((cp << 12) & 0xffff) + ((*it << 6) & 0xfff);
			if (is_trail(*(++it))) {
				cp += (*it) & 0x3f;
			} else {
				std::advance(it, -2);
				return INCOMPLETE_SEQUENCE;
			}
		} else {
			--it;
			return INCOMPLETE_SEQUENCE;
		}
		break;
	case 4:
		if (is_trail(*(++it))) {
			cp = ((cp << 18) & 0x1fffff) + ((*it << 12) & 0x3ffff);
			if (is_trail(*(++it))) {
				cp += (*it << 6) & 0xfff;
				if (is_trail(*(++it))) {
					cp += (*it) & 0x3f;
				} else {
					std::advance(it, -3);
					return INCOMPLETE_SEQUENCE;
				}
			} else {
				std::advance(it, -2);
				return INCOMPLETE_SEQUENCE;
			}
		} else {
			--it;
			return INCOMPLETE_SEQUENCE;
		}
		break;
	}
	// Is the code point valid?
	if (!is_code_point_valid(cp)) {
		repeat(length - 1, [&] { --it; });
		return INVALID_CODE_POINT;
	}

	if (code_point) {
		*code_point = cp;
	}
	if (cp < 0x80) {
		if (length != 1) {
			std::advance(it, -(length-1));
			return OVERLONG_SEQUENCE;
		}
	} else if (cp < 0x800) {
		if (length != 2) {
			std::advance(it, -(length-1));
			return OVERLONG_SEQUENCE;
		}
	} else if (cp < 0x10000) {
		if (length != 3) {
			std::advance(it, -(length-1));
			return OVERLONG_SEQUENCE;
		}
	}

	++it;
	return OK;
}

template<typename octet_iterator>
[[nodiscard]] constexpr utf_error validate_next(octet_iterator& it, octet_iterator end) {
	return validate_next(it, end, nullptr);
}

} // namespace internal

/// The library API - functions intended to be called by the users

template<typename octet_iterator>
[[nodiscard]] constexpr octet_iterator find_invalid(octet_iterator start, octet_iterator end)
{
	auto result = start;
	while (result != end) {
		internal::utf_error err_code = internal::validate_next(result, end);
		if (err_code != internal::utf_error::OK) {
			return result;
		}
	}
	return result;
}

template<typename octet_iterator>
[[nodiscard]] constexpr bool is_valid(octet_iterator start, octet_iterator end)
{
	return find_invalid(start, end) == end;
}

template<typename octet_iterator>
[[nodiscard]] constexpr bool is_bom(octet_iterator it)
{
	// Byte order mark
	constexpr std::array<uint8_t, 3> bom = {0xef, 0xbb, 0xbf};

	return ((*it++ == bom[0]) &&
	        (*it++ == bom[1]) &&
	        (*it   == bom[2]));
}

template<typename octet_iterator>
[[nodiscard]] constexpr octet_iterator sync_forward(octet_iterator it)
{
	while (internal::is_trail(*it)) ++it;
	return it;
}

template<typename octet_iterator>
[[nodiscard]] constexpr octet_iterator sync_backward(octet_iterator it)
{
	while (internal::is_trail(*it)) --it;
	return it;
}

// Is this a code point in the 'Private Use Area' (PUA).
//   https://en.wikipedia.org/wiki/Private_Use_Areas
[[nodiscard]] constexpr bool is_pua(uint32_t cp)
{
	return ((0x00E000 <= cp) && (cp <= 0x00F8FF)) ||
	       ((0x0F0000 <= cp) && (cp <= 0x0FFFFD)) ||
	       ((0x100000 <= cp) && (cp <= 0x10FFFD));
}

} // namespace utf8

#endif

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	// UTF8-CPP http://utfcpp.sourceforge.net/
2	// Slightly simplified (and reformatted) to fit openMSX coding style.
3
4	// Copyright 2006 Nemanja Trifunovic
5
6	/*
7	Permission is hereby granted, free of charge, to any person or organization
8	obtaining a copy of the software and accompanying documentation covered by
9	this license (the "Software") to use, reproduce, display, distribute,
10	execute, and transmit the Software, and to prepare derivative works of the
11	Software, and to permit third-parties to whom the Software is furnished to
12	do so, all subject to the following:
13
14	The copyright notices in the Software and this entire statement, including
15	the above license grant, this restriction and the following disclaimer,
16	must be included in all copies of the Software, in whole or in part, and
17	all derivative works of the Software, unless such copies or derivative
18	works are solely in the form of machine-executable object code generated by
19	a source language processor.
20
21	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23	FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
24	SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
25	FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
26	ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27	DEALINGS IN THE SOFTWARE.
28	*/
29
30
31	#ifndef UTF8_CORE_HH
32	#define UTF8_CORE_HH
33
34	#include "narrow.hh"
35	#include "one_of.hh"
36
37	#include <array>
38	#include <cstdint>
39	#include <iterator>
40
41	namespace utf8 {
42
43	// Helper code - not intended to be directly called by the library users.
44	// May be changed at any time
45	namespace internal {
46
47	// Unicode constants
48	// Leading (high) surrogates: 0xd800 - 0xdbff
49	// Trailing (low) surrogates: 0xdc00 - 0xdfff
50	inline constexpr uint16_t LEAD_SURROGATE_MIN = 0xd800u;
51	inline constexpr uint16_t LEAD_SURROGATE_MAX = 0xdbffu;
52	inline constexpr uint16_t TRAIL_SURROGATE_MIN = 0xdc00u;
53	inline constexpr uint16_t TRAIL_SURROGATE_MAX = 0xdfffu;
54	inline constexpr uint16_t LEAD_OFFSET = LEAD_SURROGATE_MIN - (0x10000 >> 10);
55	inline constexpr uint32_t SURROGATE_OFFSET = 0x10000u - (LEAD_SURROGATE_MIN << 10) - TRAIL_SURROGATE_MIN;
56
57	// Maximum valid value for a Unicode code point
58	inline constexpr uint32_t CODE_POINT_MAX = 0x0010ffffu;
59
60	[[nodiscard]] constexpr bool is_trail(uint8_t oc)
61	{
62	return (oc >> 6) == 0x2;
63	}
64
65	[[nodiscard]] constexpr bool is_surrogate(uint32_t cp)
66	{
67	return (cp >= LEAD_SURROGATE_MIN) && (cp <= TRAIL_SURROGATE_MAX);
68	}
69
70	[[nodiscard]] constexpr bool is_code_point_valid(uint32_t cp)
71	{
72	return (cp <= CODE_POINT_MAX) && !is_surrogate(cp) &&
73	(cp != one_of(0xfffeu, 0xffffu));
74	}
75
76	[[nodiscard]] constexpr unsigned sequence_length(uint8_t lead)
77	{
78	if (lead < 0x80) {
79	return 1;
80	} else if ((lead >> 5) == 0x06) {
81	return 2;
82	} else if ((lead >> 4) == 0x0e) {
83	return 3;
84	} else if ((lead >> 3) == 0x1e) {
85	return 4;
86	} else {
87	return 0;
88	}
89	}
90
91	enum class utf_error : uint8_t {
92	OK,
93	NOT_ENOUGH_ROOM,
94	INVALID_LEAD,
95	INCOMPLETE_SEQUENCE,
96	OVERLONG_SEQUENCE,
97	INVALID_CODE_POINT
98	};
99
100	template<typename octet_iterator>
101	[[nodiscard]] constexpr utf_error validate_next(octet_iterator& it, octet_iterator end,
102	uint32_t* code_point)
103	{
104	using enum utf_error;
105	uint32_t cp = narrow_cast<unsigned char>(*it);
106	// Check the lead octet
107	int length = sequence_length(*it);
108
109	// "Shortcut" for ASCII characters
110	if (length == 1) {
111	if (end - it <= 0) {
112	return NOT_ENOUGH_ROOM;
113	}
114	if (code_point) {
115	*code_point = cp;
116	}
117	++it;
118	return OK;
119	}
120
121	// Do we have enough memory?
122	if (std::distance(it, end) < length) {
123	return NOT_ENOUGH_ROOM;
124	}
125
126	// Check trail octets and calculate the code point
127	switch (length) {
128	case 0:
129	return INVALID_LEAD;
130	case 2:
131	if (is_trail(*(++it))) {
132	cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
133	} else {
134	--it;
135	return INCOMPLETE_SEQUENCE;
136	}
137	break;
138	case 3:
139	if (is_trail(*(++it))) {
140	cp = ((cp << 12) & 0xffff) + ((*it << 6) & 0xfff);
141	if (is_trail(*(++it))) {
142	cp += (*it) & 0x3f;
143	} else {
144	std::advance(it, -2);
145	return INCOMPLETE_SEQUENCE;
146	}
147	} else {
148	--it;
149	return INCOMPLETE_SEQUENCE;
150	}
151	break;
152	case 4:
153	if (is_trail(*(++it))) {
154	cp = ((cp << 18) & 0x1fffff) + ((*it << 12) & 0x3ffff);
155	if (is_trail(*(++it))) {
156	cp += (*it << 6) & 0xfff;
157	if (is_trail(*(++it))) {
158	cp += (*it) & 0x3f;
159	} else {
160	std::advance(it, -3);
161	return INCOMPLETE_SEQUENCE;
162	}
163	} else {
164	std::advance(it, -2);
165	return INCOMPLETE_SEQUENCE;
166	}
167	} else {
168	--it;
169	return INCOMPLETE_SEQUENCE;
170	}
171	break;
172	}
173	// Is the code point valid?
174	if (!is_code_point_valid(cp)) {
175	repeat(length - 1, [&] { --it; });
176	return INVALID_CODE_POINT;
177	}
178
179	if (code_point) {
180	*code_point = cp;
181	}
182	if (cp < 0x80) {
183	if (length != 1) {
184	std::advance(it, -(length-1));
185	return OVERLONG_SEQUENCE;
186	}
187	} else if (cp < 0x800) {
188	if (length != 2) {
189	std::advance(it, -(length-1));
190	return OVERLONG_SEQUENCE;
191	}
192	} else if (cp < 0x10000) {
193	if (length != 3) {
194	std::advance(it, -(length-1));
195	return OVERLONG_SEQUENCE;
196	}
197	}
198
199	++it;
200	return OK;
201	}
202
203	template<typename octet_iterator>
204	[[nodiscard]] constexpr utf_error validate_next(octet_iterator& it, octet_iterator end) {
205	return validate_next(it, end, nullptr);
206	}
207
208	} // namespace internal
209
210	/// The library API - functions intended to be called by the users
211
212	template<typename octet_iterator>
213	[[nodiscard]] constexpr octet_iterator find_invalid(octet_iterator start, octet_iterator end)
214	{
215	auto result = start;
216	while (result != end) {
217	internal::utf_error err_code = internal::validate_next(result, end);
218	if (err_code != internal::utf_error::OK) {
219	return result;
220	}
221	}
222	return result;
223	}
224
225	template<typename octet_iterator>
226	[[nodiscard]] constexpr bool is_valid(octet_iterator start, octet_iterator end)
227	{
228	return find_invalid(start, end) == end;
229	}
230
231	template<typename octet_iterator>
232	[[nodiscard]] constexpr bool is_bom(octet_iterator it)
233	{
234	// Byte order mark
235	constexpr std::array<uint8_t, 3> bom = {0xef, 0xbb, 0xbf};
236
237	return ((*it++ == bom[0]) &&
238	(*it++ == bom[1]) &&
239	(*it == bom[2]));
240	}
241
242	template<typename octet_iterator>
243	[[nodiscard]] constexpr octet_iterator sync_forward(octet_iterator it)
244	{
245	while (internal::is_trail(*it)) ++it;
246	return it;
247	}
248
249	template<typename octet_iterator>
250	[[nodiscard]] constexpr octet_iterator sync_backward(octet_iterator it)
251	{
252	while (internal::is_trail(*it)) --it;
253	return it;
254	}
255
256	// Is this a code point in the 'Private Use Area' (PUA).
257	// https://en.wikipedia.org/wiki/Private_Use_Areas
258	[[nodiscard]] constexpr bool is_pua(uint32_t cp)
259	{
260	return ((0x00E000 <= cp) && (cp <= 0x00F8FF)) \|\|
261	((0x0F0000 <= cp) && (cp <= 0x0FFFFD)) \|\|
262	((0x100000 <= cp) && (cp <= 0x10FFFD));
263	}
264
265	} // namespace utf8
266
267	#endif

Keyboard