src/utils/utf8_unchecked.hh at main · doomsdayone/openmsx

◆ main openmsx / src / utils / utf8_unchecked.hh

6.9 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

// UTF8-CPP   http://utfcpp.sourceforge.net/
//   Slightly simplified (and reformatted) to fit openMSX coding style.

// Copyright 2006 Nemanja Trifunovic

/*
Permission is hereby granted, free of charge, to any person or organization
obtaining a copy of the software and accompanying documentation covered by
this license (the "Software") to use, reproduce, display, distribute,
execute, and transmit the Software, and to prepare derivative works of the
Software, and to permit third-parties to whom the Software is furnished to
do so, all subject to the following:

The copyright notices in the Software and this entire statement, including
the above license grant, this restriction and the following disclaimer,
must be included in all copies of the Software, in whole or in part, and
all derivative works of the Software, unless such copies or derivative
works are solely in the form of machine-executable object code generated by
a source language processor.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*/

#ifndef UTF8_UNCHECKED_HH
#define UTF8_UNCHECKED_HH

#include "utf8_core.hh"

#include "narrow.hh"
#include "xrange.hh"

#include <string_view>

namespace utf8::unchecked {

template<typename octet_iterator>
octet_iterator append(uint32_t cp, octet_iterator result)
{
	if (cp < 0x80) {
		// one octet
		*result++ = narrow_cast<uint8_t>(cp);
	} else if (cp < 0x800) {
		// two octets
		*result++ = narrow_cast<uint8_t>(((cp >>  6) & 0x1f) | 0xc0); // 0b110,'....  (5)
		*result++ = narrow_cast<uint8_t>(((cp >>  0) & 0x3f) | 0x80); // 0b10..'....  (6)
	} else if (cp < 0x10000) {
		// three octets
		*result++ = narrow_cast<uint8_t>(((cp >> 12) & 0x0f) | 0xe0); // 0b1110'....  (4)
		*result++ = narrow_cast<uint8_t>(((cp >>  6) & 0x3f) | 0x80); // 0b10..'....  (6)
		*result++ = narrow_cast<uint8_t>(((cp >>  0) & 0x3f) | 0x80); // 0b10..'....  (6)
	} else {
		// four octets
		*result++ = narrow_cast<uint8_t>(((cp >> 18) & 0x07) | 0xf0); // 0b1111'0...  (3)
		*result++ = narrow_cast<uint8_t>(((cp >> 12) & 0x3f) | 0x80); // 0b10..'....  (6)
		*result++ = narrow_cast<uint8_t>(((cp >>  6) & 0x3f) | 0x80); // 0b10..'....  (6)
		*result++ = narrow_cast<uint8_t>(((cp >>  0) & 0x3f) | 0x80); // 0b10..'....  (6)
	}
	return result;
}

template<typename octet_iterator>
uint32_t next(octet_iterator& it)
{
	uint32_t cp = narrow_cast<uint8_t>(*it);
	switch (utf8::internal::sequence_length(narrow_cast<uint8_t>(cp))) {
	case 1:
		break;
	case 2:
		++it;
		cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
		break;
	case 3:
		++it;
		cp = ((cp << 12) & 0xffff) + ((*it << 6) & 0xfff);
		++it;
		cp += (*it) & 0x3f;
		break;
	case 4:
		++it;
		cp = ((cp << 18) & 0x1fffff) + ((*it << 12) & 0x3ffff);
		++it;
		cp += (*it << 6) & 0xfff;
		++it;
		cp += (*it) & 0x3f;
		break;
	}
	++it;
	return cp;
}

template<typename octet_iterator>
[[nodiscard]] uint32_t peek_next(octet_iterator it)
{
	return next(it);
}

template<typename octet_iterator>
uint32_t prior(octet_iterator& it)
{
	while (internal::is_trail(*(--it))) ;
	auto temp = it;
	return unchecked::next(temp);
}

template<typename octet_iterator, typename distance_type>
void advance(octet_iterator& it, distance_type n)
{
	repeat(n, [&] { unchecked::next(it); });
}

template<typename octet_iterator>
[[nodiscard]] auto distance(octet_iterator first, octet_iterator last)
{
	typename std::iterator_traits<octet_iterator>::difference_type dist = 0;
	while (first < last) {
		++dist;
		unchecked::next(first);
	}
	return dist;
}

template<typename u16bit_iterator, typename octet_iterator>
octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end,
                        octet_iterator result)
{
	while (start != end) {
		uint32_t cp = *start++;
		// Take care of surrogate pairs first
		if (internal::is_surrogate(cp)) {
			uint32_t trail_surrogate = *start++;
			cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
		}
		result = append(cp, result);
	}
	return result;
}

template<typename u16bit_iterator, typename octet_iterator>
u16bit_iterator utf8to16(octet_iterator start, octet_iterator end,
                         u16bit_iterator result)
{
	while (start != end) {
		uint32_t cp = next(start);
		if (cp > 0xffff) {
			// make a surrogate pair
			*result++ = (cp >> 10)   + internal::LEAD_OFFSET;
			*result++ = (cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN;
		} else {
			*result++ = cp;
		}
	}
	return result;
}

template<typename octet_iterator, typename u32bit_iterator>
octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end,
                        octet_iterator result)
{
	while (start != end) {
		result = append(*start++, result);
	}
	return result;
}

template<typename octet_iterator, typename u32bit_iterator>
u32bit_iterator utf8to32(octet_iterator start, octet_iterator end,
                         u32bit_iterator result)
{
	while (start < end) {
		*result++ = next(start);
	}
	return result;
}

// The iterator class
template<typename octet_iterator>
class iterator
{
	octet_iterator it;
public:
	using iterator_category = std::bidirectional_iterator_tag;
	using difference_type   = ptrdiff_t;
	using value_type        = uint32_t;
	using pointer           = uint32_t*;
	using reference         = uint32_t&;

	iterator() = default;
	explicit iterator(const octet_iterator& octet_it)
		: it(octet_it) {}
	// the default "big three" are OK
	[[nodiscard]] octet_iterator base() const { return it; }
	[[nodiscard]] uint32_t operator*() const
	{
		octet_iterator temp = it;
		return next(temp);
	}
	[[nodiscard]] bool operator==(const iterator&) const = default;
	iterator& operator++()
	{
		std::advance(it, internal::sequence_length(*it));
		return *this;
	}
	iterator operator++(int)
	{
		auto temp = *this;
		std::advance(it, internal::sequence_length(*it));
		return temp;
	}
	iterator& operator--()
	{
		prior(it);
		return *this;
	}
	iterator operator--(int)
	{
		auto temp = *this;
		prior(it);
		return temp;
	}
};

// convenience functions
[[nodiscard]] inline size_t size(std::string_view utf8)
{
	return utf8::unchecked::distance(begin(utf8), end(utf8));
}
[[nodiscard]] inline std::string_view substr(std::string_view utf8, std::string_view::size_type first = 0,
                                             std::string_view::size_type len = std::string_view::npos)
{
	auto b = begin(utf8);
	utf8::unchecked::advance(b, first);
	std::string_view::const_iterator e;
	if (len != std::string_view::npos) {
		e = b;
		while (len && (e != end(utf8))) {
			unchecked::next(e); --len;
		}
	} else {
		e = end(utf8);
	}
	return {std::to_address(b), narrow<std::string_view::size_type>(e - b)};
}

} // namespace utf8::unchecked

#endif

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	// UTF8-CPP http://utfcpp.sourceforge.net/
2	// Slightly simplified (and reformatted) to fit openMSX coding style.
3
4	// Copyright 2006 Nemanja Trifunovic
5
6	/*
7	Permission is hereby granted, free of charge, to any person or organization
8	obtaining a copy of the software and accompanying documentation covered by
9	this license (the "Software") to use, reproduce, display, distribute,
10	execute, and transmit the Software, and to prepare derivative works of the
11	Software, and to permit third-parties to whom the Software is furnished to
12	do so, all subject to the following:
13
14	The copyright notices in the Software and this entire statement, including
15	the above license grant, this restriction and the following disclaimer,
16	must be included in all copies of the Software, in whole or in part, and
17	all derivative works of the Software, unless such copies or derivative
18	works are solely in the form of machine-executable object code generated by
19	a source language processor.
20
21	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22	IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23	FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
24	SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
25	FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
26	ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27	DEALINGS IN THE SOFTWARE.
28	*/
29
30	#ifndef UTF8_UNCHECKED_HH
31	#define UTF8_UNCHECKED_HH
32
33	#include "utf8_core.hh"
34
35	#include "narrow.hh"
36	#include "xrange.hh"
37
38	#include <string_view>
39
40	namespace utf8::unchecked {
41
42	template<typename octet_iterator>
43	octet_iterator append(uint32_t cp, octet_iterator result)
44	{
45	if (cp < 0x80) {
46	// one octet
47	*result++ = narrow_cast<uint8_t>(cp);
48	} else if (cp < 0x800) {
49	// two octets
50	*result++ = narrow_cast<uint8_t>(((cp >> 6) & 0x1f) \| 0xc0); // 0b110,'.... (5)
51	*result++ = narrow_cast<uint8_t>(((cp >> 0) & 0x3f) \| 0x80); // 0b10..'.... (6)
52	} else if (cp < 0x10000) {
53	// three octets
54	*result++ = narrow_cast<uint8_t>(((cp >> 12) & 0x0f) \| 0xe0); // 0b1110'.... (4)
55	*result++ = narrow_cast<uint8_t>(((cp >> 6) & 0x3f) \| 0x80); // 0b10..'.... (6)
56	*result++ = narrow_cast<uint8_t>(((cp >> 0) & 0x3f) \| 0x80); // 0b10..'.... (6)
57	} else {
58	// four octets
59	*result++ = narrow_cast<uint8_t>(((cp >> 18) & 0x07) \| 0xf0); // 0b1111'0... (3)
60	*result++ = narrow_cast<uint8_t>(((cp >> 12) & 0x3f) \| 0x80); // 0b10..'.... (6)
61	*result++ = narrow_cast<uint8_t>(((cp >> 6) & 0x3f) \| 0x80); // 0b10..'.... (6)
62	*result++ = narrow_cast<uint8_t>(((cp >> 0) & 0x3f) \| 0x80); // 0b10..'.... (6)
63	}
64	return result;
65	}
66
67	template<typename octet_iterator>
68	uint32_t next(octet_iterator& it)
69	{
70	uint32_t cp = narrow_cast<uint8_t>(*it);
71	switch (utf8::internal::sequence_length(narrow_cast<uint8_t>(cp))) {
72	case 1:
73	break;
74	case 2:
75	++it;
76	cp = ((cp << 6) & 0x7ff) + ((*it) & 0x3f);
77	break;
78	case 3:
79	++it;
80	cp = ((cp << 12) & 0xffff) + ((*it << 6) & 0xfff);
81	++it;
82	cp += (*it) & 0x3f;
83	break;
84	case 4:
85	++it;
86	cp = ((cp << 18) & 0x1fffff) + ((*it << 12) & 0x3ffff);
87	++it;
88	cp += (*it << 6) & 0xfff;
89	++it;
90	cp += (*it) & 0x3f;
91	break;
92	}
93	++it;
94	return cp;
95	}
96
97	template<typename octet_iterator>
98	[[nodiscard]] uint32_t peek_next(octet_iterator it)
99	{
100	return next(it);
101	}
102
103	template<typename octet_iterator>
104	uint32_t prior(octet_iterator& it)
105	{
106	while (internal::is_trail(*(--it))) ;
107	auto temp = it;
108	return unchecked::next(temp);
109	}
110
111	template<typename octet_iterator, typename distance_type>
112	void advance(octet_iterator& it, distance_type n)
113	{
114	repeat(n, [&] { unchecked::next(it); });
115	}
116
117	template<typename octet_iterator>
118	[[nodiscard]] auto distance(octet_iterator first, octet_iterator last)
119	{
120	typename std::iterator_traits<octet_iterator>::difference_type dist = 0;
121	while (first < last) {
122	++dist;
123	unchecked::next(first);
124	}
125	return dist;
126	}
127
128	template<typename u16bit_iterator, typename octet_iterator>
129	octet_iterator utf16to8(u16bit_iterator start, u16bit_iterator end,
130	octet_iterator result)
131	{
132	while (start != end) {
133	uint32_t cp = *start++;
134	// Take care of surrogate pairs first
135	if (internal::is_surrogate(cp)) {
136	uint32_t trail_surrogate = *start++;
137	cp = (cp << 10) + trail_surrogate + internal::SURROGATE_OFFSET;
138	}
139	result = append(cp, result);
140	}
141	return result;
142	}
143
144	template<typename u16bit_iterator, typename octet_iterator>
145	u16bit_iterator utf8to16(octet_iterator start, octet_iterator end,
146	u16bit_iterator result)
147	{
148	while (start != end) {
149	uint32_t cp = next(start);
150	if (cp > 0xffff) {
151	// make a surrogate pair
152	*result++ = (cp >> 10) + internal::LEAD_OFFSET;
153	*result++ = (cp & 0x3ff) + internal::TRAIL_SURROGATE_MIN;
154	} else {
155	*result++ = cp;
156	}
157	}
158	return result;
159	}
160
161	template<typename octet_iterator, typename u32bit_iterator>
162	octet_iterator utf32to8(u32bit_iterator start, u32bit_iterator end,
163	octet_iterator result)
164	{
165	while (start != end) {
166	result = append(*start++, result);
167	}
168	return result;
169	}
170
171	template<typename octet_iterator, typename u32bit_iterator>
172	u32bit_iterator utf8to32(octet_iterator start, octet_iterator end,
173	u32bit_iterator result)
174	{
175	while (start < end) {
176	*result++ = next(start);
177	}
178	return result;
179	}
180
181	// The iterator class
182	template<typename octet_iterator>
183	class iterator
184	{
185	octet_iterator it;
186	public:
187	using iterator_category = std::bidirectional_iterator_tag;
188	using difference_type = ptrdiff_t;
189	using value_type = uint32_t;
190	using pointer = uint32_t*;
191	using reference = uint32_t&;
192
193	iterator() = default;
194	explicit iterator(const octet_iterator& octet_it)
195	: it(octet_it) {}
196	// the default "big three" are OK
197	[[nodiscard]] octet_iterator base() const { return it; }
198	[[nodiscard]] uint32_t operator*() const
199	{
200	octet_iterator temp = it;
201	return next(temp);
202	}
203	[[nodiscard]] bool operator==(const iterator&) const = default;
204	iterator& operator++()
205	{
206	std::advance(it, internal::sequence_length(*it));
207	return *this;
208	}
209	iterator operator++(int)
210	{
211	auto temp = *this;
212	std::advance(it, internal::sequence_length(*it));
213	return temp;
214	}
215	iterator& operator--()
216	{
217	prior(it);
218	return *this;
219	}
220	iterator operator--(int)
221	{
222	auto temp = *this;
223	prior(it);
224	return temp;
225	}
226	};
227
228	// convenience functions
229	[[nodiscard]] inline size_t size(std::string_view utf8)
230	{
231	return utf8::unchecked::distance(begin(utf8), end(utf8));
232	}
233	[[nodiscard]] inline std::string_view substr(std::string_view utf8, std::string_view::size_type first = 0,
234	std::string_view::size_type len = std::string_view::npos)
235	{
236	auto b = begin(utf8);
237	utf8::unchecked::advance(b, first);
238	std::string_view::const_iterator e;
239	if (len != std::string_view::npos) {
240	e = b;
241	while (len && (e != end(utf8))) {
242	unchecked::next(e); --len;
243	}
244	} else {
245	e = end(utf8);
246	}
247	return {std::to_address(b), narrow<std::string_view::size_type>(e - b)};
248	}
249
250	} // namespace utf8::unchecked
251
252	#endif

Keyboard