src/utils/utf8_checked.hh at main · doomsdayone/openmsx

◆ main openmsx / src / utils / utf8_checked.hh

9.2 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

// UTF8-CPP   http://utfcpp.sourceforge.net/
//   Slightly simplified (and reformatted) to fit openMSX coding style.

// Copyright 2006 Nemanja Trifunovic

/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:

The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/

#ifndef UTF8_CHECKED_HH
#define UTF8_CHECKED_HH

#include "utf8_core.hh"

#include <stdexcept>
#include <string_view>

namespace utf8 {

// Exceptions that may be thrown from the library functions.
class invalid_code_point : public std::exception
{
	uint32_t cp;
public:
	explicit invalid_code_point(uint32_t cp_) : cp(cp_) {}
	[[nodiscard]] const char* what() const noexcept override { return "Invalid code point"; }
	[[nodiscard]] uint32_t code_point() const { return cp; }
};

class invalid_utf8 : public std::exception
{
	uint8_t u8;
public:
	explicit invalid_utf8(uint8_t u) : u8(u) {}
	[[nodiscard]] const char* what() const noexcept override { return "Invalid UTF-8"; }
	[[nodiscard]] uint8_t utf8_octet() const { return u8; }
};

class invalid_utf16 : public std::exception
{
	uint16_t u16;
public:
	explicit invalid_utf16(uint16_t u) : u16(u) {}
	[[nodiscard]] const char* what() const noexcept override { return "Invalid UTF-16"; }
	[[nodiscard]] uint16_t utf16_word() const { return u16; }
};

class not_enough_room : public std::exception
{
public:
	[[nodiscard]] const char* what() const noexcept override { return "Not enough space"; }
};

// The library API - functions intended to be called by the users

template<typename octet_iterator, typename output_iterator>
output_iterator replace_invalid(octet_iterator start, octet_iterator end,
                                output_iterator out, uint32_t replacement)
{
	while (start != end) {
		auto sequence_start = start;
		internal::utf_error err_code = internal::validate_next(start, end);
		switch (err_code) {
		using enum internal::utf_error;
		case OK:
			for (auto it = sequence_start; it != start; ++it) {
				*out++ = *it;
			}
			break;
		case NOT_ENOUGH_ROOM:
			throw not_enough_room();
		case INVALID_LEAD:
			append(replacement, out);
			++start;
			break;
		case INCOMPLETE_SEQUENCE:
		case OVERLONG_SEQUENCE:
		case INVALID_CODE_POINT:
			append(replacement, out);
			++start;
			// just one replacement mark for the sequence
			while (internal::is_trail(*start) && start != end) {
				++start;
			}
			break;
		}
	}
	return out;
}

template<typename octet_iterator, typename output_iterator>
inline output_iterator replace_invalid(octet_iterator start, octet_iterator end,
                                       output_iterator out)
{
	return replace_invalid(start, end, out, 0xfffd);
}

template<typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result)
{
	if (!internal::is_code_point_valid(cp)) {
		throw invalid_code_point(cp);
	}
	if (cp < 0x80) {
		// one octet
		*result++ = cp;
	} else if (cp < 0x800) {
		// two octets
		*result++ = ((cp >>  6) & 0x1f) | 0xc0; // 0b110.'....  (5)
		*result++ = ((cp >>  0) & 0x3f) | 0x80; // 0b10..'....  (6)
	} else if (cp < 0x10000) {
		// three octets
		*result++ = ((cp >> 12) & 0x0f) | 0xe0; // 0b1110'....  (4)
		*result++ = ((cp >>  6) & 0x3f) | 0x80; // 0b10..'....  (6)
		*result++ = ((cp >>  0) & 0x3f) | 0x80; // 0b10..'....  (6)
	} else if (cp <= internal::CODE_POINT_MAX) {
		// four octets
		*result++ = ((cp >> 18) & 0x07) | 0xf0; // 0b1111'0...  (3)
		*result++ = ((cp >> 12) & 0x3f) | 0x80; // 0b10..'....  (6)
		*result++ = ((cp >>  6) & 0x3f) | 0x80; // 0b10..'....  (6)
		*result++ = ((cp >>  0) & 0x3f) | 0x80; // 0b10..'....  (6)
	} else {
		throw invalid_code_point(cp);
	}
	return result;
}

template<typename octet_iterator>
uint32_t next(octet_iterator& it, octet_iterator end)
{
	uint32_t cp = 0;
	internal::utf_error err_code = internal::validate_next(it, end, &cp);
	switch (err_code) {
	using enum internal::utf_error;
	case OK:
		break;
	case NOT_ENOUGH_ROOM:
		throw not_enough_room();
	case INVALID_LEAD:
	case INCOMPLETE_SEQUENCE:
	case OVERLONG_SEQUENCE:
		throw invalid_utf8(*it);
	case INVALID_CODE_POINT:
		throw invalid_code_point(cp);
	}
	return cp;
}

template<typename octet_iterator>
[[nodiscard]] uint32_t peek_next(octet_iterator it, octet_iterator end)
{
	return next(it, end);
}

template<typename octet_iterator>
uint32_t prior(octet_iterator& it, octet_iterator start)
{
	auto end = it;
	while (internal::is_trail(*(--it))) {
		if (it < start) {
			// error - no lead byte in the sequence
			throw invalid_utf8(*it);
		}
	}
	auto temp = it;
	return next(temp, end);
}

template<typename octet_iterator, typename distance_type>
void advance(octet_iterator& it, distance_type n, octet_iterator end)
{
	repeat(n, [&] {	next(it, end); });
}

template<typename octet_iterator>
[[nodiscard]] auto distance(octet_iterator first, octet_iterator last)
{
	typename std::iterator_traits<octet_iterator>::difference_type dist = 0;
	while (first < last) {
		++dist;
		next(first, last);
	}
	return dist;
}

template<typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end,
                        octet_iterator result)
{
	while (start != end) {
		uint32_t cp = *start++;
		// Take care of surrogate pairs first
		if (internal::is_surrogate(cp)) {
			if (start == end) {
				throw invalid_utf16(*start);
			}
			auto trail_surrogate = *start++;
			if (trail_surrogate < internal::TRAIL_SURROGATE_MIN ||
			    trail_surrogate > internal::TRAIL_SURROGATE_MAX) {
				throw invalid_utf16(trail_surrogate);
			}
			cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
		}
		result = append(cp, result);
	}
	return result;
}

template<typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16(octet_iterator start, octet_iterator end,
                         u16bit_iterator result)
{
	while (start != end) {
		uint32_t cp = next(start, end);
		if (cp > 0xffff) { // make a surrogate pair
			*result++ = (cp >> 10)   + internal::LEAD_OFFSET;
			*result++ = (cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN;
		} else {
			*result++ = cp;
		}
	}
	return result;
}

template<typename octet_iterator, typename u32bit_iterator>
octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end,
                        octet_iterator result)
{
	while (start != end) {
		result = append(*start++, result);
	}
	return result;
}

template<typename octet_iterator, typename u32bit_iterator>
u32bit_iterator utf8to32(octet_iterator start, octet_iterator end,
                         u32bit_iterator result)
{
	while (start < end) {
		*result++ = next(start, end);
	}
	return result;
}

// The iterator class
template<typename octet_iterator>
class iterator
{
	octet_iterator it;
	octet_iterator range_start;
	octet_iterator range_end;

public:
	using iterator_category = std::bidirectional_iterator_tag;
	using difference_type   = ptrdiff_t;
	using value_type        = uint32_t;
	using pointer           = uint32_t*;
	using reference         = uint32_t&;

	iterator() = default;
	iterator(const octet_iterator& octet_it,
	         const octet_iterator& range_start_,
	         const octet_iterator& range_end_)
		: it(octet_it)
		, range_start(range_start_)
		, range_end(range_end_)
	{
		if (it < range_start || it > range_end) {
			throw std::out_of_range("Invalid utf-8 iterator position");
		}
	}
	// the default "big three" are OK
	[[nodiscard]] octet_iterator base() const { return it; }
	[[nodiscard]] uint32_t operator*() const
	{
		auto temp = it;
		return next(temp, range_end);
	}
	[[nodiscard]] bool operator==(const iterator& rhs) const
	{
		if ((range_start != rhs.range_start) ||
		    (range_end   != rhs.range_end)) {
			throw std::logic_error(
				"Comparing utf-8 iterators defined with different ranges");
		}
		return it == rhs.it;
	}
	iterator& operator++()
	{
		next(it, range_end);
		return *this;
	}
	iterator operator++(int)
	{
		auto temp = *this;
		next(it, range_end);
		return temp;
	}
	iterator& operator--()
	{
		prior(it, range_start);
		return *this;
	}
	iterator operator--(int)
	{
		auto temp = *this;
		prior(it, range_start);
		return temp;
	}
};

#ifdef _WIN32
[[nodiscard]] std::wstring utf8to16  (std::string_view utf8);
[[nodiscard]] std::string  utf16to8  (std::wstring_view utf16);
#endif

} // namespace utf8

#endif

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

e8555c89 Andy McCall 1 month ago	1	// UTF8-CPP http://utfcpp.sourceforge.net/
	2	// Slightly simplified (and reformatted) to fit openMSX coding style.
	3
	4	// Copyright 2006 Nemanja Trifunovic
	5
	6	/*
	7	Permission is hereby granted, free of charge, to any person or organization
	8	obtaining a copy of the software and accompanying documentation covered by
	9	this license (the "Software") to use, reproduce, display, distribute,
	10	execute, and transmit the Software, and to prepare derivative works of the
	11	Software, and to permit third-parties to whom the Software is furnished to
	12	do so, all subject to the following:
	13
	14	The copyright notices in the Software and this entire statement, including
	15	the above license grant, this restriction and the following disclaimer,
	16	must be included in all copies of the Software, in whole or in part, and
	17	all derivative works of the Software, unless such copies or derivative
	18	works are solely in the form of machine-executable object code generated by
	19	a source language processor.
	20
	21	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	22	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	23	FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
	24	SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
	25	FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
	26	ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
	27	DEALINGS IN THE SOFTWARE.
	28	*/
	29
	30	#ifndef UTF8_CHECKED_HH
	31	#define UTF8_CHECKED_HH
	32
	33	#include "utf8_core.hh"
	34
	35	#include <stdexcept>
	36	#include <string_view>
	37
	38	namespace utf8 {
	39
	40	// Exceptions that may be thrown from the library functions.
	41	class invalid_code_point : public std::exception
	42	{
	43	uint32_t cp;
	44	public:
	45	explicit invalid_code_point(uint32_t cp_) : cp(cp_) {}
	46	[[nodiscard]] const char* what() const noexcept override { return "Invalid code point"; }
	47	[[nodiscard]] uint32_t code_point() const { return cp; }
	48	};
	49
	50	class invalid_utf8 : public std::exception
	51	{
	52	uint8_t u8;
	53	public:
	54	explicit invalid_utf8(uint8_t u) : u8(u) {}
	55	[[nodiscard]] const char* what() const noexcept override { return "Invalid UTF-8"; }
	56	[[nodiscard]] uint8_t utf8_octet() const { return u8; }
	57	};
	58
	59	class invalid_utf16 : public std::exception
	60	{
	61	uint16_t u16;
	62	public:
	63	explicit invalid_utf16(uint16_t u) : u16(u) {}
	64	[[nodiscard]] const char* what() const noexcept override { return "Invalid UTF-16"; }
65	[[nodiscard]] uint16_t utf16_word() const { return u16; }
66	};
67
68	class not_enough_room : public std::exception
69	{
70	public:
71	[[nodiscard]] const char* what() const noexcept override { return "Not enough space"; }
72	};
73
74	// The library API - functions intended to be called by the users
75
76	template<typename octet_iterator, typename output_iterator>
77	output_iterator replace_invalid(octet_iterator start, octet_iterator end,
78	output_iterator out, uint32_t replacement)
79	{
80	while (start != end) {
81	auto sequence_start = start;
82	internal::utf_error err_code = internal::validate_next(start, end);
83	switch (err_code) {
84	using enum internal::utf_error;
85	case OK:
86	for (auto it = sequence_start; it != start; ++it) {
87	out++ = it;
88	}
89	break;
90	case NOT_ENOUGH_ROOM:
91	throw not_enough_room();
92	case INVALID_LEAD:
93	append(replacement, out);
94	++start;
95	break;
96	case INCOMPLETE_SEQUENCE:
97	case OVERLONG_SEQUENCE:
98	case INVALID_CODE_POINT:
99	append(replacement, out);
100	++start;
101	// just one replacement mark for the sequence
102	while (internal::is_trail(*start) && start != end) {
103	++start;
104	}
105	break;
106	}
107	}
108	return out;
109	}
110
111	template<typename octet_iterator, typename output_iterator>
112	inline output_iterator replace_invalid(octet_iterator start, octet_iterator end,
113	output_iterator out)
114	{
115	return replace_invalid(start, end, out, 0xfffd);
116	}
117
118	template<typename octet_iterator>
119	octet_iterator append(uint32_t cp, octet_iterator result)
120	{
121	if (!internal::is_code_point_valid(cp)) {
122	throw invalid_code_point(cp);
123	}
124	if (cp < 0x80) {
125	// one octet
126	*result++ = cp;
127	} else if (cp < 0x800) {
128	// two octets
129	*result++ = ((cp >> 6) & 0x1f) \| 0xc0; // 0b110.'.... (5)
130	*result++ = ((cp >> 0) & 0x3f) \| 0x80; // 0b10..'.... (6)
131	} else if (cp < 0x10000) {
132	// three octets
133	*result++ = ((cp >> 12) & 0x0f) \| 0xe0; // 0b1110'.... (4)
134	*result++ = ((cp >> 6) & 0x3f) \| 0x80; // 0b10..'.... (6)
135	*result++ = ((cp >> 0) & 0x3f) \| 0x80; // 0b10..'.... (6)
136	} else if (cp <= internal::CODE_POINT_MAX) {
137	// four octets
138	*result++ = ((cp >> 18) & 0x07) \| 0xf0; // 0b1111'0... (3)
139	*result++ = ((cp >> 12) & 0x3f) \| 0x80; // 0b10..'.... (6)
140	*result++ = ((cp >> 6) & 0x3f) \| 0x80; // 0b10..'.... (6)
141	*result++ = ((cp >> 0) & 0x3f) \| 0x80; // 0b10..'.... (6)
142	} else {
143	throw invalid_code_point(cp);
144	}
145	return result;
146	}
147
148	template<typename octet_iterator>
149	uint32_t next(octet_iterator& it, octet_iterator end)
150	{
151	uint32_t cp = 0;
152	internal::utf_error err_code = internal::validate_next(it, end, &cp);
153	switch (err_code) {
154	using enum internal::utf_error;
155	case OK:
156	break;
157	case NOT_ENOUGH_ROOM:
158	throw not_enough_room();
159	case INVALID_LEAD:
160	case INCOMPLETE_SEQUENCE:
161	case OVERLONG_SEQUENCE:
162	throw invalid_utf8(*it);
163	case INVALID_CODE_POINT:
164	throw invalid_code_point(cp);
165	}
166	return cp;
167	}
168
169	template<typename octet_iterator>
170	[[nodiscard]] uint32_t peek_next(octet_iterator it, octet_iterator end)
171	{
172	return next(it, end);
173	}
174
175	template<typename octet_iterator>
176	uint32_t prior(octet_iterator& it, octet_iterator start)
177	{
178	auto end = it;
179	while (internal::is_trail(*(--it))) {
180	if (it < start) {
181	// error - no lead byte in the sequence
182	throw invalid_utf8(*it);
183	}
184	}
185	auto temp = it;
186	return next(temp, end);
187	}
188
189	template<typename octet_iterator, typename distance_type>
190	void advance(octet_iterator& it, distance_type n, octet_iterator end)
191	{
192	repeat(n, [&] { next(it, end); });
193	}
194
195	template<typename octet_iterator>
196	[[nodiscard]] auto distance(octet_iterator first, octet_iterator last)
197	{
198	typename std::iterator_traits<octet_iterator>::difference_type dist = 0;
199	while (first < last) {
200	++dist;
201	next(first, last);
202	}
203	return dist;
204	}
205
206	template<typename u16bit_iterator, typename octet_iterator>
207	octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end,
208	octet_iterator result)
209	{
210	while (start != end) {
211	uint32_t cp = *start++;
212	// Take care of surrogate pairs first
213	if (internal::is_surrogate(cp)) {
214	if (start == end) {
215	throw invalid_utf16(*start);
216	}
217	auto trail_surrogate = *start++;
218	if (trail_surrogate < internal::TRAIL_SURROGATE_MIN \|\|
219	trail_surrogate > internal::TRAIL_SURROGATE_MAX) {
220	throw invalid_utf16(trail_surrogate);
221	}
222	cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
223	}
224	result = append(cp, result);
225	}
226	return result;
227	}
228
229	template<typename u16bit_iterator, typename octet_iterator>
230	u16bit_iterator utf8to16(octet_iterator start, octet_iterator end,
231	u16bit_iterator result)
232	{
233	while (start != end) {
234	uint32_t cp = next(start, end);
235	if (cp > 0xffff) { // make a surrogate pair
236	*result++ = (cp >> 10) + internal::LEAD_OFFSET;
237	*result++ = (cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN;
238	} else {
239	*result++ = cp;
240	}
241	}
242	return result;
243	}
244
245	template<typename octet_iterator, typename u32bit_iterator>
246	octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end,
247	octet_iterator result)
248	{
249	while (start != end) {
250	result = append(*start++, result);
251	}
252	return result;
253	}
254
255	template<typename octet_iterator, typename u32bit_iterator>
256	u32bit_iterator utf8to32(octet_iterator start, octet_iterator end,
257	u32bit_iterator result)
258	{
259	while (start < end) {
260	*result++ = next(start, end);
261	}
262	return result;
263	}
264
265	// The iterator class
266	template<typename octet_iterator>
267	class iterator
268	{
269	octet_iterator it;
270	octet_iterator range_start;
271	octet_iterator range_end;
272
273	public:
274	using iterator_category = std::bidirectional_iterator_tag;
275	using difference_type = ptrdiff_t;
276	using value_type = uint32_t;
277	using pointer = uint32_t*;
278	using reference = uint32_t&;
279
280	iterator() = default;
281	iterator(const octet_iterator& octet_it,
282	const octet_iterator& range_start_,
283	const octet_iterator& range_end_)
284	: it(octet_it)
285	, range_start(range_start_)
286	, range_end(range_end_)
287	{
288	if (it < range_start \|\| it > range_end) {
289	throw std::out_of_range("Invalid utf-8 iterator position");
290	}
291	}
292	// the default "big three" are OK
293	[[nodiscard]] octet_iterator base() const { return it; }
294	[[nodiscard]] uint32_t operator*() const
295	{
296	auto temp = it;
297	return next(temp, range_end);
298	}
299	[[nodiscard]] bool operator==(const iterator& rhs) const
300	{
301	if ((range_start != rhs.range_start) \|\|
302	(range_end != rhs.range_end)) {
303	throw std::logic_error(
304	"Comparing utf-8 iterators defined with different ranges");
305	}
306	return it == rhs.it;
307	}
308	iterator& operator++()
309	{
310	next(it, range_end);
311	return *this;
312	}
313	iterator operator++(int)
314	{
315	auto temp = *this;
316	next(it, range_end);
317	return temp;
318	}
319	iterator& operator--()
320	{
321	prior(it, range_start);
322	return *this;
323	}
324	iterator operator--(int)
325	{
326	auto temp = *this;
327	prior(it, range_start);
328	return temp;
329	}
330	};
331
332	#ifdef _WIN32
333	[[nodiscard]] std::wstring utf8to16 (std::string_view utf8);
334	[[nodiscard]] std::string utf16to8 (std::wstring_view utf16);
335	#endif
336
337	} // namespace utf8
338
339	#endif

Keyboard