src/memory/RomDatabase.cc at main · doomsdayone/openmsx

◆ main openmsx / src / memory / RomDatabase.cc

14.9 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

#include "RomDatabase.hh"

#include "CliComm.hh"
#include "File.hh"
#include "FileContext.hh"
#include "MSXException.hh"

#include "String32.hh"
#include "StringOp.hh"
#include "hash_map.hh"
#include "narrow.hh"
#include "ranges.hh"
#include "rapidsax.hh"
#include "stl.hh"
#include "unreachable.hh"
#include "xxhash.hh"

#include <algorithm>
#include <array>
#include <cassert>
#include <ranges>
#include <string_view>

namespace openmsx {

using UnknownTypes = hash_map<std::string, unsigned, XXHasher>;

class DBParser : public rapidsax::NullHandler
{
public:
	DBParser(RomDatabase::RomDB& db_, UnknownTypes& unknownTypes_,
	         CliComm& cliComm_, char* bufStart_)
		: db(db_)
		, unknownTypes(unknownTypes_)
		, cliComm(cliComm_)
		, bufStart(bufStart_)
		, initialSize(db.size())
	{
	}

	// rapidsax handler interface
	void start(zstring_view tag);
	void attribute(zstring_view name, zstring_view value);
	void text(zstring_view txt);
	void stop();
	void doctype(zstring_view txt);

	[[nodiscard]] std::string_view getSystemID() const { return systemID; }

private:
	[[nodiscard]] String32 cIndex(zstring_view str) const;
	void addEntries();
	void addAllEntries();

	enum State : uint8_t {
		BEGIN,
		SOFTWAREDB,
		SOFTWARE,
		SYSTEM,
		TITLE,
		COMPANY,
		YEAR,
		COUNTRY,
		GENMSXID,
		DUMP_REMARK,
		DUMP_TEXT,
		DUMP,
		ORIGINAL,
		ROM,
		TYPE,
		START,
		HASH,
		END
	};

	struct Dump {
		String32 remark;
		Sha1Sum hash;
		String32 origData;
		RomType type;
		bool origValue;
	};

	RomDatabase::RomDB& db;
	UnknownTypes& unknownTypes;
	CliComm& cliComm;
	char* bufStart;

	std::string_view systemID;
	std::string_view type;
	std::string_view startVal;

	std::vector<Dump> dumps;
	std::string_view system;
	String32 title = {};
	String32 company = {};
	String32 year = {};
	String32 country = {};
	unsigned genMSXid = 0;

	State state = BEGIN;
	unsigned unknownLevel = 0;
	size_t initialSize;
};

void DBParser::start(zstring_view tag)
{
	if (unknownLevel) {
		++unknownLevel;
		return;
	}

	assert(!tag.empty()); // rapidsax will reject empty tags
	switch (state) {
	case BEGIN:
		if (tag == "softwaredb") {
			state = SOFTWAREDB;
			return;
		}
		throw MSXException("Expected <softwaredb> as root tag.");
	case SOFTWAREDB:
		if (small_compare<"software">(tag)) {
			system = std::string_view();
			toString32(bufStart, bufStart, title);
			toString32(bufStart, bufStart, company);
			toString32(bufStart, bufStart, year);
			toString32(bufStart, bufStart, country);
			genMSXid = 0;
			dumps.clear();
			state = SOFTWARE;
			return;
		}
		break;
	case SOFTWARE: {
		switch (tag.front()) {
		case 's':
			if (small_compare<"system">(tag)) {
				state = SYSTEM;
				return;
			}
			break;
		case 't':
			tag.remove_prefix(1);
			if (small_compare<"itle">(tag)) {
				state = TITLE;
				return;
			}
			break;
		case 'c':
			if (small_compare<"company">(tag)) {
				state = COMPANY;
				return;
			} else if (small_compare<"country">(tag)) {
				state = COUNTRY;
				return;
			}
			break;
		case 'y':
			if (small_compare<"year">(tag)) {
				state = YEAR;
				return;
			}
			break;
		case 'g':
			if (small_compare<"genmsxid">(tag)) {
				state = GENMSXID;
				return;
			}
			break;
		case 'd':
			if (small_compare<"dump">(tag)) {
				dumps.emplace_back();
				dumps.back().type = RomType::UNKNOWN;
				dumps.back().origValue = false;
				toString32(bufStart, bufStart, dumps.back().remark);
				toString32(bufStart, bufStart, dumps.back().origData);
				state = DUMP;
				return;
			}
			break;
		case 'r':
			tag.remove_prefix(1);
			if (small_compare<"om">(tag)) {
				dumps.emplace_back();
				dumps.back().type = RomType::UNKNOWN;
				dumps.back().origValue = false;
				toString32(bufStart, bufStart, dumps.back().remark);
				toString32(bufStart, bufStart, dumps.back().origData);
				state = DUMP;
				return;
			}
			break;
		}
		break;
	}
	case DUMP: {
		switch (tag.front()) {
		case 'o':
			if (small_compare<"original">(tag)) {
				dumps.back().origValue = false;
				state = ORIGINAL;
				return;
			}
			break;
		case 'm':
			if (small_compare<"megarom">(tag)) {
				type = std::string_view();
				startVal = std::string_view();
				state = ROM;
				return;
			}
			break;
		case 'r':
			tag.remove_prefix(1);
			if (small_compare<"om">(tag)) {
				type = "Mirrored";
				startVal = std::string_view();
				state = ROM;
				return;
			}
			break;
		}
		break;
	}
	case ROM: {
		switch (tag.front()) {
		case 't':
			if (small_compare<"type">(tag)) {
				state = TYPE;
				return;
			}
			break;
		case 's':
			tag.remove_prefix(1);
			if (small_compare<"tart">(tag)) {
				state = START;
				return;
			}
			break;
		case 'r':
			if (small_compare<"remark">(tag)) {
				state = DUMP_REMARK;
				return;
			}
			break;
		case 'h':
			if (small_compare<"hash">(tag)) {
				state = HASH;
				return;
			}
			break;
		}
		break;
	}
	case DUMP_REMARK:
		if (small_compare<"text">(tag)) {
			state = DUMP_TEXT;
			return;
		}
		break;
	case SYSTEM:
	case TITLE:
	case COMPANY:
	case YEAR:
	case COUNTRY:
	case GENMSXID:
	case ORIGINAL:
	case TYPE:
	case START:
	case HASH:
	case DUMP_TEXT:
		break;

	case END:
		throw MSXException("Unexpected opening tag: ", tag);

	default:
		UNREACHABLE;
	}

	++unknownLevel;
}

void DBParser::attribute(zstring_view name, zstring_view value)
{
	if (unknownLevel) return;

	switch (state) {
	case ORIGINAL:
		if (small_compare<"value">(name)) {
			dumps.back().origValue = StringOp::stringToBool(value);
		}
		break;
	case SOFTWARE:
		switch (name.front()) {
		case 't':
			name.remove_prefix(1);
			if (small_compare<"itle">(name)) {
				title = cIndex(value);
			}
			break;
		case 'g':
			if (small_compare<"genmsxid">(name)) {
				if (auto g = StringOp::stringToBase<10, unsigned>(value)) {
					genMSXid = *g;
				} else {
					cliComm.printWarning(
						"Ignoring bad Generation MSX id (genmsxid) "
						"in entry with title '", fromString32(bufStart, title),
						": ", value);
				}
			}
			break;
		case 's':
			if (small_compare<"system">(name)) {
				system = value;
			}
			break;
		case 'c':
			if (small_compare<"company">(name)) {
				company = cIndex(value);
			} else if (small_compare<"country">(name)) {
				country = cIndex(value);
			}
			break;
		case 'y':
			if (small_compare<"year">(name)) {
				year = cIndex(value);
			}
			break;
		}
		break;
	case DUMP:
		switch (name.front()) {
		case 't':
			if (small_compare<"type">(name)) {
				RomType romType = RomInfo::nameToRomType(value);
				if (romType == RomType::UNKNOWN) {
					unknownTypes[std::string(value)]++;
				}
				dumps.back().type = romType;
			}
			break;
		case 's':
			if (small_compare<"sha1">(name)) {
				try {
					dumps.back().hash = Sha1Sum(value);
				} catch (MSXException& e) {
					cliComm.printWarning(
						"Ignoring bad dump for '", fromString32(bufStart, title),
						"': ", e.getMessage());
				}
			} else if (small_compare<"status">(name)) {
				dumps.back().origData = cIndex(value);
			}
			break;
		case 'r':
			if (small_compare<"remark">(name)) {
				dumps.back().remark = cIndex(value);
			}
			break;
		}
		break;
	case HASH:
	case BEGIN:
	case SOFTWAREDB:
	case SYSTEM:
	case TITLE:
	case COMPANY:
	case YEAR:
	case COUNTRY:
	case GENMSXID:
	case DUMP_REMARK:
	case DUMP_TEXT:
	case ROM:
	case TYPE:
	case START:
	case END:
		break;
	default:
		UNREACHABLE;
	}
}

void DBParser::text(zstring_view txt)
{
	if (unknownLevel) return;

	switch (state) {
	case SYSTEM:
		system = txt;
		break;
	case TITLE:
		title = cIndex(txt);
		break;
	case COMPANY:
		company = cIndex(txt);
		break;
	case YEAR:
		year = cIndex(txt);
		break;
	case COUNTRY:
		country = cIndex(txt);
		break;
	case GENMSXID: {
		if (auto g = StringOp::stringToBase<10, unsigned>(txt)) {
			genMSXid = *g;
		} else {
			cliComm.printWarning(
				"Ignoring bad Generation MSX id (genmsxid) "
				"in entry with title '", fromString32(bufStart, title),
				": ", txt);
		}
		break;
	}
	case ORIGINAL:
		dumps.back().origData = cIndex(txt);
		break;
	case TYPE:
		type = txt;
		break;
	case START:
		startVal = txt;
		break;
	case HASH:
		try {
			dumps.back().hash = Sha1Sum(txt);
		} catch (MSXException& e) {
			cliComm.printWarning(
				"Ignoring bad dump for '", fromString32(bufStart, title),
				"': ", e.getMessage());
		}
		break;
	case DUMP_REMARK:
	case DUMP_TEXT:
		dumps.back().remark = cIndex(txt);
		break;
	case BEGIN:
	case SOFTWAREDB:
	case SOFTWARE:
	case DUMP:
	case ROM:
	case END:
		break;
	default:
		UNREACHABLE;
	}
}

String32 DBParser::cIndex(zstring_view str) const
{
	String32 result;
	toString32(bufStart, str.data(), result);
	return result;
}

// called on </software>
void DBParser::addEntries()
{
	append(db, std::views::transform(dumps, [&](auto& d) {
		return RomDatabase::Entry{
			d.hash,
			RomInfo(title, year, company, country, d.origValue,
			        d.origData, d.remark, d.type, genMSXid)};
	}));
}

// called on </softwaredb>
void DBParser::addAllEntries()
{
	// Calculate boundary between old and new entries.
	//  old: [first, mid)   already sorted, no duplicates
	//  new: [mid, last)    not yet sorted, may have duplicates
	//    there may also be duplicates between old and new
	const auto first = begin(db);
	const auto last  = end  (db);
	const auto mid = first + narrow<ptrdiff_t>(initialSize);
	if (mid == last) return; // no new entries

	// Sort new entries, old entries are already sorted.
	std::ranges::sort(mid, last, {}, &RomDatabase::Entry::sha1);

	// Filter duplicates from new entries. This is similar to the
	// unique() algorithm, except that it also warns about duplicates.
	auto it1 = mid;
	auto it2 = mid + 1;
	// skip initial non-duplicates
	while (it2 != last) {
		if (it1->sha1 == it2->sha1) break;
		++it1; ++it2;
	}
	// move non-duplicates up
	while (it2 != last) {
		if (it1->sha1 == it2->sha1) {
			cliComm.printWarning(
				"duplicate softwaredb entry SHA1: ",
				it2->sha1);
		} else {
			++it1;
			*it1 = std::move(*it2);
		}
		++it2;
	}
	// actually erase the duplicates (typically none)
	db.erase(it1 + 1, last);
	// At this point both old and new entries are sorted and unique. But
	// there may still be duplicates between old and new.

	// Merge new and old entries. This is similar to the inplace_merge()
	// algorithm, except that duplicates (between old and new) are removed.
	if (first == mid) return; // no old entries (common case)
	RomDatabase::RomDB result;
	result.reserve(db.size());
	it1 = first;
	it2 = mid;
	// while both new and old still have elements
	while (it1 != mid && it2 != last) {
		if (it1->sha1 < it2->sha1) {
			result.push_back(std::move(*it1));
			++it1;
		} else {
			if (it1->sha1 != it2->sha1) { // *it2 < *it1
				result.push_back(std::move(*it2));
				++it2;
			} else {
				// pick old entry, silently ignore new
				result.push_back(std::move(*it1));
				++it1; ++it2;
			}
		}
	}
	// move remaining old or new entries (one of these is empty)
	move(it1, mid,  back_inserter(result));
	move(it2, last, back_inserter(result));

	// make result the new current database
	swap(result, db);
}

static const char* parseStart(std::string_view s)
{
	// we expect "0x0000", "0x4000", "0x8000", "0xc000" or ""
	return ((s.size() == 6) && s.starts_with("0x")) ? (s.data() + 2) : nullptr;
}

void DBParser::stop()
{
	if (unknownLevel) {
		--unknownLevel;
		return;
	}

	switch (state) {
	case SOFTWAREDB:
		addAllEntries();
		state = END;
		break;
	case SOFTWARE:
		addEntries();
		state = SOFTWAREDB;
		break;
	case SYSTEM:
	case TITLE:
	case COMPANY:
	case YEAR:
	case COUNTRY:
	case GENMSXID:
		state = SOFTWARE;
		break;
	case DUMP:
		if (dumps.back().hash.empty()) {
			// no sha1 sum specified, drop this dump
			dumps.pop_back();
		}
		state = SOFTWARE;
		break;
	case ORIGINAL:
		state = DUMP;
		break;
	case ROM: {
		std::string_view t = type;
		std::array<char, 8 + 4> buf;
		if (small_compare<"Mirrored">(t)) {
			if (const char* s = parseStart(startVal)) {
				copy_to_range(t,                      subspan<8>(buf, 0));
				copy_to_range(std::string_view(s, 4), subspan<4>(buf, 8));
				t = std::string_view(buf.data(), 8 + 4);
			}
		} else if (small_compare<"Normal">(t)) {
			if (const char* s = parseStart(startVal)) {
				copy_to_range(t,                      subspan<6>(buf, 0));
				copy_to_range(std::string_view(s, 4), subspan<4>(buf, 6));
				t = std::string_view(buf.data(), 6 + 4);
			}
		}
		RomType romType = RomInfo::nameToRomType(t);
		if (romType == RomType::UNKNOWN) {
			unknownTypes[std::string(t)]++;
		}
		dumps.back().type = romType;
		state = DUMP;
		break;
	}
	case TYPE:
	case START:
	case HASH:
	case DUMP_REMARK:
		state = ROM;
		break;
	case DUMP_TEXT:
		state = DUMP_REMARK;
		break;
	case BEGIN:
	case END:
		throw MSXException("Unexpected closing tag");

	default:
		UNREACHABLE;
	}
}

void DBParser::doctype(zstring_view txt)
{
	auto pos1 = txt.find(" SYSTEM \"");
	if (pos1 == std::string_view::npos) return;
	auto t = txt.substr(pos1 + 9);
	auto pos2 = t.find('"');
	if (pos2 == std::string_view::npos) return;
	systemID = t.substr(0, pos2);
}

static void parseDB(CliComm& cliComm, char* buf, char* bufStart,
                    RomDatabase::RomDB& db, UnknownTypes& unknownTypes)
{
	DBParser handler(db, unknownTypes, cliComm, bufStart);
	rapidsax::parse<rapidsax::trimWhitespace | rapidsax::zeroTerminateStrings>(handler, buf);

	if (handler.getSystemID() != "softwaredb1.dtd") {
		throw rapidsax::ParseError(
			"Missing or wrong systemID.\n"
			"You're probably using an old incompatible file format.",
			nullptr);
	}
}

RomDatabase::RomDatabase(CliComm& cliComm)
{
	db.reserve(3500);
	UnknownTypes unknownTypes;
	// first user- then system-directory
	std::vector<File> files;
	size_t bufferSize = 0;
	for (const auto& p : systemFileContext().getPaths()) {
		try {
			auto& f = files.emplace_back(p + "/softwaredb.xml");
			bufferSize += f.getSize() + rapidsax::EXTRA_BUFFER_SPACE;
		} catch (MSXException& /*e*/) {
			// Ignore. It's not unusual the DB in the user
			// directory is not found. In case there's an error
			// with both user and system DB, we must give a
			// warning, but that's done below.
		}
	}
	buffer.resize(bufferSize);
	size_t bufferOffset = 0;
	for (auto& file : files) {
		try {
			auto size = file.getSize();
			auto* buf = &buffer[bufferOffset];
			bufferOffset += size + rapidsax::EXTRA_BUFFER_SPACE;
			file.read(std::span{buf, size});
			buf[size] = 0;

			parseDB(cliComm, buf, buffer.data(), db, unknownTypes);
		} catch (rapidsax::ParseError& e) {
			cliComm.printWarning(
				"Rom database parsing failed: ", e.what());
		} catch (MSXException& /*e*/) {
			// Ignore, see above
		}
	}
	if (bufferSize) buffer[0] = 0;
	if (db.empty()) {
		cliComm.printWarning(
			"Couldn't load software database.\n"
			"This may cause incorrect ROM mapper types to be used.");
	}
	if (!unknownTypes.empty()) {
		std::string output = "Unknown mapper types in software database: ";
		for (const auto& [type, count] : unknownTypes) {
			strAppend(output, type, " (", count, "x); ");
		}
		cliComm.printWarning(output);
	}
}

const RomInfo* RomDatabase::fetchRomInfo(const Sha1Sum& sha1sum) const
{
	auto d = binary_find(db, sha1sum, {}, &Entry::sha1);
	return d ? &d->romInfo : nullptr;
}

} // namespace openmsx

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

1	#include "RomDatabase.hh"
2
3	#include "CliComm.hh"
4	#include "File.hh"
5	#include "FileContext.hh"
6	#include "MSXException.hh"
7
8	#include "String32.hh"
9	#include "StringOp.hh"
10	#include "hash_map.hh"
11	#include "narrow.hh"
12	#include "ranges.hh"
13	#include "rapidsax.hh"
14	#include "stl.hh"
15	#include "unreachable.hh"
16	#include "xxhash.hh"
17
18	#include <algorithm>
19	#include <array>
20	#include <cassert>
21	#include <ranges>
22	#include <string_view>
23
24	namespace openmsx {
25
26	using UnknownTypes = hash_map<std::string, unsigned, XXHasher>;
27
28	class DBParser : public rapidsax::NullHandler
29	{
30	public:
31	DBParser(RomDatabase::RomDB& db_, UnknownTypes& unknownTypes_,
32	CliComm& cliComm_, char* bufStart_)
33	: db(db_)
34	, unknownTypes(unknownTypes_)
35	, cliComm(cliComm_)
36	, bufStart(bufStart_)
37	, initialSize(db.size())
38	{
39	}
40
41	// rapidsax handler interface
42	void start(zstring_view tag);
43	void attribute(zstring_view name, zstring_view value);
44	void text(zstring_view txt);
45	void stop();
46	void doctype(zstring_view txt);
47
48	[[nodiscard]] std::string_view getSystemID() const { return systemID; }
49
50	private:
51	[[nodiscard]] String32 cIndex(zstring_view str) const;
52	void addEntries();
53	void addAllEntries();
54
55	enum State : uint8_t {
56	BEGIN,
57	SOFTWAREDB,
58	SOFTWARE,
59	SYSTEM,
60	TITLE,
61	COMPANY,
62	YEAR,
63	COUNTRY,
64	GENMSXID,
65	DUMP_REMARK,
66	DUMP_TEXT,
67	DUMP,
68	ORIGINAL,
69	ROM,
70	TYPE,
71	START,
72	HASH,
73	END
74	};
75
76	struct Dump {
77	String32 remark;
78	Sha1Sum hash;
79	String32 origData;
80	RomType type;
81	bool origValue;
82	};
83
84	RomDatabase::RomDB& db;
85	UnknownTypes& unknownTypes;
86	CliComm& cliComm;
87	char* bufStart;
88
89	std::string_view systemID;
90	std::string_view type;
91	std::string_view startVal;
92
93	std::vector<Dump> dumps;
94	std::string_view system;
95	String32 title = {};
96	String32 company = {};
97	String32 year = {};
98	String32 country = {};
99	unsigned genMSXid = 0;
100
101	State state = BEGIN;
102	unsigned unknownLevel = 0;
103	size_t initialSize;
104	};
105
106	void DBParser::start(zstring_view tag)
107	{
108	if (unknownLevel) {
109	++unknownLevel;
110	return;
111	}
112
113	assert(!tag.empty()); // rapidsax will reject empty tags
114	switch (state) {
115	case BEGIN:
116	if (tag == "softwaredb") {
117	state = SOFTWAREDB;
118	return;
119	}
120	throw MSXException("Expected <softwaredb> as root tag.");
121	case SOFTWAREDB:
122	if (small_compare<"software">(tag)) {
123	system = std::string_view();
124	toString32(bufStart, bufStart, title);
125	toString32(bufStart, bufStart, company);
126	toString32(bufStart, bufStart, year);
127	toString32(bufStart, bufStart, country);
128	genMSXid = 0;
129	dumps.clear();
130	state = SOFTWARE;
131	return;
132	}
133	break;
134	case SOFTWARE: {
135	switch (tag.front()) {
136	case 's':
137	if (small_compare<"system">(tag)) {
138	state = SYSTEM;
139	return;
140	}
141	break;
142	case 't':
143	tag.remove_prefix(1);
144	if (small_compare<"itle">(tag)) {
145	state = TITLE;
146	return;
147	}
148	break;
149	case 'c':
150	if (small_compare<"company">(tag)) {
151	state = COMPANY;
152	return;
153	} else if (small_compare<"country">(tag)) {
154	state = COUNTRY;
155	return;
156	}
157	break;
158	case 'y':
159	if (small_compare<"year">(tag)) {
160	state = YEAR;
161	return;
162	}
163	break;
164	case 'g':
165	if (small_compare<"genmsxid">(tag)) {
166	state = GENMSXID;
167	return;
168	}
169	break;
170	case 'd':
171	if (small_compare<"dump">(tag)) {
172	dumps.emplace_back();
173	dumps.back().type = RomType::UNKNOWN;
174	dumps.back().origValue = false;
175	toString32(bufStart, bufStart, dumps.back().remark);
176	toString32(bufStart, bufStart, dumps.back().origData);
177	state = DUMP;
178	return;
179	}
180	break;
181	case 'r':
182	tag.remove_prefix(1);
183	if (small_compare<"om">(tag)) {
184	dumps.emplace_back();
185	dumps.back().type = RomType::UNKNOWN;
186	dumps.back().origValue = false;
187	toString32(bufStart, bufStart, dumps.back().remark);
188	toString32(bufStart, bufStart, dumps.back().origData);
189	state = DUMP;
190	return;
191	}
192	break;
193	}
194	break;
195	}
196	case DUMP: {
197	switch (tag.front()) {
198	case 'o':
199	if (small_compare<"original">(tag)) {
200	dumps.back().origValue = false;
201	state = ORIGINAL;
202	return;
203	}
204	break;
205	case 'm':
206	if (small_compare<"megarom">(tag)) {
207	type = std::string_view();
208	startVal = std::string_view();
209	state = ROM;
210	return;
211	}
212	break;
213	case 'r':
214	tag.remove_prefix(1);
215	if (small_compare<"om">(tag)) {
216	type = "Mirrored";
217	startVal = std::string_view();
218	state = ROM;
219	return;
220	}
221	break;
222	}
223	break;
224	}
225	case ROM: {
226	switch (tag.front()) {
227	case 't':
228	if (small_compare<"type">(tag)) {
229	state = TYPE;
230	return;
231	}
232	break;
233	case 's':
234	tag.remove_prefix(1);
235	if (small_compare<"tart">(tag)) {
236	state = START;
237	return;
238	}
239	break;
240	case 'r':
241	if (small_compare<"remark">(tag)) {
242	state = DUMP_REMARK;
243	return;
244	}
245	break;
246	case 'h':
247	if (small_compare<"hash">(tag)) {
248	state = HASH;
249	return;
250	}
251	break;
252	}
253	break;
254	}
255	case DUMP_REMARK:
256	if (small_compare<"text">(tag)) {
257	state = DUMP_TEXT;
258	return;
259	}
260	break;
261	case SYSTEM:
262	case TITLE:
263	case COMPANY:
264	case YEAR:
265	case COUNTRY:
266	case GENMSXID:
267	case ORIGINAL:
268	case TYPE:
269	case START:
270	case HASH:
271	case DUMP_TEXT:
272	break;
273
274	case END:
275	throw MSXException("Unexpected opening tag: ", tag);
276
277	default:
278	UNREACHABLE;
279	}
280
281	++unknownLevel;
282	}
283
284	void DBParser::attribute(zstring_view name, zstring_view value)
285	{
286	if (unknownLevel) return;
287
288	switch (state) {
289	case ORIGINAL:
290	if (small_compare<"value">(name)) {
291	dumps.back().origValue = StringOp::stringToBool(value);
292	}
293	break;
294	case SOFTWARE:
295	switch (name.front()) {
296	case 't':
297	name.remove_prefix(1);
298	if (small_compare<"itle">(name)) {
299	title = cIndex(value);
300	}
301	break;
302	case 'g':
303	if (small_compare<"genmsxid">(name)) {
304	if (auto g = StringOp::stringToBase<10, unsigned>(value)) {
305	genMSXid = *g;
306	} else {
307	cliComm.printWarning(
308	"Ignoring bad Generation MSX id (genmsxid) "
309	"in entry with title '", fromString32(bufStart, title),
310	": ", value);
311	}
312	}
313	break;
314	case 's':
315	if (small_compare<"system">(name)) {
316	system = value;
317	}
318	break;
319	case 'c':
320	if (small_compare<"company">(name)) {
321	company = cIndex(value);
322	} else if (small_compare<"country">(name)) {
323	country = cIndex(value);
324	}
325	break;
326	case 'y':
327	if (small_compare<"year">(name)) {
328	year = cIndex(value);
329	}
330	break;
331	}
332	break;
333	case DUMP:
334	switch (name.front()) {
335	case 't':
336	if (small_compare<"type">(name)) {
337	RomType romType = RomInfo::nameToRomType(value);
338	if (romType == RomType::UNKNOWN) {
339	unknownTypes[std::string(value)]++;
340	}
341	dumps.back().type = romType;
342	}
343	break;
344	case 's':
345	if (small_compare<"sha1">(name)) {
346	try {
347	dumps.back().hash = Sha1Sum(value);
348	} catch (MSXException& e) {
349	cliComm.printWarning(
350	"Ignoring bad dump for '", fromString32(bufStart, title),
351	"': ", e.getMessage());
352	}
353	} else if (small_compare<"status">(name)) {
354	dumps.back().origData = cIndex(value);
355	}
356	break;
357	case 'r':
358	if (small_compare<"remark">(name)) {
359	dumps.back().remark = cIndex(value);
360	}
361	break;
362	}
363	break;
364	case HASH:
365	case BEGIN:
366	case SOFTWAREDB:
367	case SYSTEM:
368	case TITLE:
369	case COMPANY:
370	case YEAR:
371	case COUNTRY:
372	case GENMSXID:
373	case DUMP_REMARK:
374	case DUMP_TEXT:
375	case ROM:
376	case TYPE:
377	case START:
378	case END:
379	break;
380	default:
381	UNREACHABLE;
382	}
383	}
384
385	void DBParser::text(zstring_view txt)
386	{
387	if (unknownLevel) return;
388
389	switch (state) {
390	case SYSTEM:
391	system = txt;
392	break;
393	case TITLE:
394	title = cIndex(txt);
395	break;
396	case COMPANY:
397	company = cIndex(txt);
398	break;
399	case YEAR:
400	year = cIndex(txt);
401	break;
402	case COUNTRY:
403	country = cIndex(txt);
404	break;
405	case GENMSXID: {
406	if (auto g = StringOp::stringToBase<10, unsigned>(txt)) {
407	genMSXid = *g;
408	} else {
409	cliComm.printWarning(
410	"Ignoring bad Generation MSX id (genmsxid) "
411	"in entry with title '", fromString32(bufStart, title),
412	": ", txt);
413	}
414	break;
415	}
416	case ORIGINAL:
417	dumps.back().origData = cIndex(txt);
418	break;
419	case TYPE:
420	type = txt;
421	break;
422	case START:
423	startVal = txt;
424	break;
425	case HASH:
426	try {
427	dumps.back().hash = Sha1Sum(txt);
428	} catch (MSXException& e) {
429	cliComm.printWarning(
430	"Ignoring bad dump for '", fromString32(bufStart, title),
431	"': ", e.getMessage());
432	}
433	break;
434	case DUMP_REMARK:
435	case DUMP_TEXT:
436	dumps.back().remark = cIndex(txt);
437	break;
438	case BEGIN:
439	case SOFTWAREDB:
440	case SOFTWARE:
441	case DUMP:
442	case ROM:
443	case END:
444	break;
445	default:
446	UNREACHABLE;
447	}
448	}
449
450	String32 DBParser::cIndex(zstring_view str) const
451	{
452	String32 result;
453	toString32(bufStart, str.data(), result);
454	return result;
455	}
456
457	// called on </software>
458	void DBParser::addEntries()
459	{
460	append(db, std::views::transform(dumps, [&](auto& d) {
461	return RomDatabase::Entry{
462	d.hash,
463	RomInfo(title, year, company, country, d.origValue,
464	d.origData, d.remark, d.type, genMSXid)};
465	}));
466	}
467
468	// called on </softwaredb>
469	void DBParser::addAllEntries()
470	{
471	// Calculate boundary between old and new entries.
472	// old: [first, mid) already sorted, no duplicates
473	// new: [mid, last) not yet sorted, may have duplicates
474	// there may also be duplicates between old and new
475	const auto first = begin(db);
476	const auto last = end (db);
477	const auto mid = first + narrow<ptrdiff_t>(initialSize);
478	if (mid == last) return; // no new entries
479
480	// Sort new entries, old entries are already sorted.
481	std::ranges::sort(mid, last, {}, &RomDatabase::Entry::sha1);
482
483	// Filter duplicates from new entries. This is similar to the
484	// unique() algorithm, except that it also warns about duplicates.
485	auto it1 = mid;
486	auto it2 = mid + 1;
487	// skip initial non-duplicates
488	while (it2 != last) {
489	if (it1->sha1 == it2->sha1) break;
490	++it1; ++it2;
491	}
492	// move non-duplicates up
493	while (it2 != last) {
494	if (it1->sha1 == it2->sha1) {
495	cliComm.printWarning(
496	"duplicate softwaredb entry SHA1: ",
497	it2->sha1);
498	} else {
499	++it1;
500	it1 = std::move(it2);
501	}
502	++it2;
503	}
504	// actually erase the duplicates (typically none)
505	db.erase(it1 + 1, last);
506	// At this point both old and new entries are sorted and unique. But
507	// there may still be duplicates between old and new.
508
509	// Merge new and old entries. This is similar to the inplace_merge()
510	// algorithm, except that duplicates (between old and new) are removed.
511	if (first == mid) return; // no old entries (common case)
512	RomDatabase::RomDB result;
513	result.reserve(db.size());
514	it1 = first;
515	it2 = mid;
516	// while both new and old still have elements
517	while (it1 != mid && it2 != last) {
518	if (it1->sha1 < it2->sha1) {
519	result.push_back(std::move(*it1));
520	++it1;
521	} else {
522	if (it1->sha1 != it2->sha1) { // it2 < it1
523	result.push_back(std::move(*it2));
524	++it2;
525	} else {
526	// pick old entry, silently ignore new
527	result.push_back(std::move(*it1));
528	++it1; ++it2;
529	}
530	}
531	}
532	// move remaining old or new entries (one of these is empty)
533	move(it1, mid, back_inserter(result));
534	move(it2, last, back_inserter(result));
535
536	// make result the new current database
537	swap(result, db);
538	}
539
540	static const char* parseStart(std::string_view s)
541	{
542	// we expect "0x0000", "0x4000", "0x8000", "0xc000" or ""
543	return ((s.size() == 6) && s.starts_with("0x")) ? (s.data() + 2) : nullptr;
544	}
545
546	void DBParser::stop()
547	{
548	if (unknownLevel) {
549	--unknownLevel;
550	return;
551	}
552
553	switch (state) {
554	case SOFTWAREDB:
555	addAllEntries();
556	state = END;
557	break;
558	case SOFTWARE:
559	addEntries();
560	state = SOFTWAREDB;
561	break;
562	case SYSTEM:
563	case TITLE:
564	case COMPANY:
565	case YEAR:
566	case COUNTRY:
567	case GENMSXID:
568	state = SOFTWARE;
569	break;
570	case DUMP:
571	if (dumps.back().hash.empty()) {
572	// no sha1 sum specified, drop this dump
573	dumps.pop_back();
574	}
575	state = SOFTWARE;
576	break;
577	case ORIGINAL:
578	state = DUMP;
579	break;
580	case ROM: {
581	std::string_view t = type;
582	std::array<char, 8 + 4> buf;
583	if (small_compare<"Mirrored">(t)) {
584	if (const char* s = parseStart(startVal)) {
585	copy_to_range(t, subspan<8>(buf, 0));
586	copy_to_range(std::string_view(s, 4), subspan<4>(buf, 8));
587	t = std::string_view(buf.data(), 8 + 4);
588	}
589	} else if (small_compare<"Normal">(t)) {
590	if (const char* s = parseStart(startVal)) {
591	copy_to_range(t, subspan<6>(buf, 0));
592	copy_to_range(std::string_view(s, 4), subspan<4>(buf, 6));
593	t = std::string_view(buf.data(), 6 + 4);
594	}
595	}
596	RomType romType = RomInfo::nameToRomType(t);
597	if (romType == RomType::UNKNOWN) {
598	unknownTypes[std::string(t)]++;
599	}
600	dumps.back().type = romType;
601	state = DUMP;
602	break;
603	}
604	case TYPE:
605	case START:
606	case HASH:
607	case DUMP_REMARK:
608	state = ROM;
609	break;
610	case DUMP_TEXT:
611	state = DUMP_REMARK;
612	break;
613	case BEGIN:
614	case END:
615	throw MSXException("Unexpected closing tag");
616
617	default:
618	UNREACHABLE;
619	}
620	}
621
622	void DBParser::doctype(zstring_view txt)
623	{
624	auto pos1 = txt.find(" SYSTEM \"");
625	if (pos1 == std::string_view::npos) return;
626	auto t = txt.substr(pos1 + 9);
627	auto pos2 = t.find('"');
628	if (pos2 == std::string_view::npos) return;
629	systemID = t.substr(0, pos2);
630	}
631
632	static void parseDB(CliComm& cliComm, char* buf, char* bufStart,
633	RomDatabase::RomDB& db, UnknownTypes& unknownTypes)
634	{
635	DBParser handler(db, unknownTypes, cliComm, bufStart);
636	rapidsax::parse<rapidsax::trimWhitespace \| rapidsax::zeroTerminateStrings>(handler, buf);
637
638	if (handler.getSystemID() != "softwaredb1.dtd") {
639	throw rapidsax::ParseError(
640	"Missing or wrong systemID.\n"
641	"You're probably using an old incompatible file format.",
642	nullptr);
643	}
644	}
645
646	RomDatabase::RomDatabase(CliComm& cliComm)
647	{
648	db.reserve(3500);
649	UnknownTypes unknownTypes;
650	// first user- then system-directory
651	std::vector<File> files;
652	size_t bufferSize = 0;
653	for (const auto& p : systemFileContext().getPaths()) {
654	try {
655	auto& f = files.emplace_back(p + "/softwaredb.xml");
656	bufferSize += f.getSize() + rapidsax::EXTRA_BUFFER_SPACE;
657	} catch (MSXException& /e/) {
658	// Ignore. It's not unusual the DB in the user
659	// directory is not found. In case there's an error
660	// with both user and system DB, we must give a
661	// warning, but that's done below.
662	}
663	}
664	buffer.resize(bufferSize);
665	size_t bufferOffset = 0;
666	for (auto& file : files) {
667	try {
668	auto size = file.getSize();
669	auto* buf = &buffer[bufferOffset];
670	bufferOffset += size + rapidsax::EXTRA_BUFFER_SPACE;
671	file.read(std::span{buf, size});
672	buf[size] = 0;
673
674	parseDB(cliComm, buf, buffer.data(), db, unknownTypes);
675	} catch (rapidsax::ParseError& e) {
676	cliComm.printWarning(
677	"Rom database parsing failed: ", e.what());
678	} catch (MSXException& /e/) {
679	// Ignore, see above
680	}
681	}
682	if (bufferSize) buffer[0] = 0;
683	if (db.empty()) {
684	cliComm.printWarning(
685	"Couldn't load software database.\n"
686	"This may cause incorrect ROM mapper types to be used.");
687	}
688	if (!unknownTypes.empty()) {
689	std::string output = "Unknown mapper types in software database: ";
690	for (const auto& [type, count] : unknownTypes) {
691	strAppend(output, type, " (", count, "x); ");
692	}
693	cliComm.printWarning(output);
694	}
695	}
696
697	const RomInfo* RomDatabase::fetchRomInfo(const Sha1Sum& sha1sum) const
698	{
699	auto d = binary_find(db, sha1sum, {}, &Entry::sha1);
700	return d ? &d->romInfo : nullptr;
701	}
702
703	} // namespace openmsx

Keyboard