src/debugger/SymbolManager.cc at main · doomsdayone/openmsx

◆ main openmsx / src / debugger / SymbolManager.cc

17.9 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw

#include "SymbolManager.hh"

#include "CommandController.hh"
#include "File.hh"
#include "Interpreter.hh"
#include "TclObject.hh"

#include "StringOp.hh"
#include "narrow.hh"
#include "static_vector.hh"
#include "stl.hh"
#include "strCat.hh"
#include "unreachable.hh"

#include <algorithm>
#include <bit>
#include <cassert>
#include <fstream>
#include <ranges>

namespace openmsx {

zstring_view SymbolFile::toString(Type type)
{
	switch (type) {
		using enum Type;
		case AUTO_DETECT:   return "auto-detect";
		case ASMSX:         return "asMSX";
		case GENERIC:       return "generic";
		case HTC:           return "htc";
		case LINKMAP:       return "linkmap";
		case NOICE:         return "NoICE";
		case VASM:          return "vasm";
		case WLALINK_NOGMB: return "wlalink";
		default: UNREACHABLE;
	}
}

std::optional<SymbolFile::Type> SymbolFile::parseType(std::string_view str)
{
	using enum Type;
	if (str == "auto-detect") return AUTO_DETECT;
	if (str == "asMSX")       return ASMSX;
	if (str == "generic")     return GENERIC;
	if (str == "htc")         return HTC;
	if (str == "linkmap")     return LINKMAP;
	if (str == "NoICE")       return NOICE;
	if (str == "vasm")        return VASM;
	if (str == "wlalink")     return WLALINK_NOGMB;
	return {};
}


SymbolManager::SymbolManager(CommandController& commandController_)
	: commandController(commandController_)
{
}

// detection logic taken from old openmsx-debugger, could probably be improved.
[[nodiscard]] SymbolFile::Type SymbolManager::detectType(std::string_view filename, std::string_view buffer)
{
	auto fname = StringOp::toLower(filename);

	using enum SymbolFile::Type;
	if (fname.ends_with(".noi")) {
		// NoICE command file
		return NOICE;
	} else if (fname.ends_with(".map")) {
		auto [line, _] = StringOp::splitOnFirst(buffer, "\n\r");
		if (StringOp::containsCaseInsensitive(line, "hi-tech")) {
			// HiTech link map file
			return LINKMAP;
		}
		// map file output by the Z80ASM from Z88DK
		return GENERIC;
	} else if (fname.ends_with(".sym")) {
		// auto detect which sym file
		auto [line, _] = StringOp::splitOnFirst(buffer, "\n\r");
		if (line.starts_with("; Symbol table")) {
			return ASMSX;
		} else if (StringOp::containsCaseInsensitive(line, " %equ ")) { // TNIASM1
			return GENERIC;
		} else if (StringOp::containsCaseInsensitive(line, " equ ")) {
			return GENERIC;
		} else if (StringOp::containsCaseInsensitive(line, "Sections:")) {
			return VASM;
		} else if (line.starts_with("; this file was created with wlalink")) {
			return WLALINK_NOGMB;
		} else {
			// this is a blunt conclusion but I don't know a way
			// to detect this file type
			return HTC;
		}
	} else if (fname.ends_with(".symbol") || fname.ends_with(".publics") || fname.ends_with(".sys")) {
		/* They are the same type of file. For some reason the Debian
			* manpage uses the extension ".sys"
			* pasmo doc -> pasmo [options] file.asm file.bin [file.symbol [file.publics] ]
			* pasmo manpage in Debian -> pasmo [options]  file.asm file.bin [file.sys]
		*/
		return GENERIC; // pasmo
	}
	return GENERIC;
}

[[nodiscard]] SymbolFile SymbolManager::loadLines(
	std::string_view filename, std::string_view buffer, SymbolFile::Type type,
	function_ref<std::optional<Symbol>(std::span<std::string_view>)> lineParser)
{
	SymbolFile result;
	result.filename = filename;
	result.type = type;

	static constexpr std::string_view whitespace = " \t\r";
	for (std::string_view fullLine : StringOp::split_view(buffer, '\n')) {
		auto [line, _] = StringOp::splitOnFirst(fullLine, ';');

		auto tokens = static_vector<std::string_view, 3 + 1>{from_range,
			std::views::take(StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace), 3 + 1)};
		if (auto symbol = lineParser(tokens)) {
			result.symbols.push_back(std::move(*symbol));
		}
	}

	return result;
}

template<typename T>
[[nodiscard]] std::optional<T> SymbolManager::parseValue(std::string_view str)
{
	if (str.ends_with('h') || str.ends_with('H')) { // hex
		str.remove_suffix(1);
		return StringOp::stringToBase<16, T>(str);
	}
	if (str.starts_with('$') || str.starts_with('#')) { // hex
		str.remove_prefix(1);
		return StringOp::stringToBase<16, T>(str);
	}
	if (str.starts_with('%')) { // bin
		str.remove_prefix(1);
		return StringOp::stringToBase<2, T>(str);
	}
	// this recognizes the prefixes "0x" or "0X" (for hexadecimal)
	//                          and "0b" or "0B" (for binary)
	// no prefix in interpreted as decimal
	// "0" as a prefix for octal is intentionally NOT supported
	return StringOp::stringTo<T>(str);
}

// explicitly instantiate for uint16_t and uint32_t (needed for unittest)
template std::optional<uint16_t> SymbolManager::parseValue<uint16_t>(std::string_view);
template std::optional<uint32_t> SymbolManager::parseValue<uint32_t>(std::string_view);

[[nodiscard]] std::optional<Symbol> SymbolManager::checkLabel(std::string_view label, uint32_t value)
{
	if (label.ends_with(':')) label.remove_suffix(1);
	if (label.empty()) return {};

	auto seg{value > 0xFFFF ? std::optional<uint16_t>(static_cast<uint16_t>(value >> 16)) : std::nullopt};
	return Symbol{.name = std::string(label), .value = static_cast<uint16_t>(value), .slot = {}, .segment = seg};
}

[[nodiscard]] std::optional<Symbol> SymbolManager::checkLabelAndValue(std::string_view label, std::string_view value)
{
	if (auto num = parseValue<uint16_t>(value)) {
		return checkLabel(label, *num);
	}
	return {};
}

[[nodiscard]] std::optional<Symbol> SymbolManager::checkLabelSegmentAndValue(std::string_view label, std::string_view value)
{
	if (auto num = parseValue<uint32_t>(value)) {
		return checkLabel(label, *num);
	}
	return {};
}

[[nodiscard]] SymbolFile SymbolManager::loadGeneric(std::string_view filename, std::string_view buffer)
{
	auto parseLine = [](std::span<std::string_view> tokens) -> std::optional<Symbol> {
		if (tokens.size() != 3) return {};
		auto label = tokens[0];
		auto equ   = tokens[1];
		auto value = tokens[2];
		StringOp::casecmp cmp;
		if (!cmp(equ, "equ") &&      // TNIASM0, PASMO, SJASM, ...
		    !cmp(equ, "%equ") &&     // TNIASM1
		    (equ != "=")) return {}; // Z80ASM map file (Z88DK)
		return checkLabelAndValue(label, value);
	};
	return loadLines(filename, buffer, SymbolFile::Type::GENERIC, parseLine);
}

[[nodiscard]] SymbolFile SymbolManager::loadNoICE(std::string_view filename, std::string_view buffer)
{
	bool anySegment = false;
	auto parseLine = [&](std::span<std::string_view> tokens) -> std::optional<Symbol> {
		if (tokens.size() != 3) return {};
		auto def   = tokens[0];
		auto label = tokens[1];
		auto value = tokens[2];
		if (StringOp::casecmp cmp; !cmp(def, "def")) return {};
		// detecting segment information above 16bits
		auto symbol = checkLabelSegmentAndValue(label, value);
		anySegment |= symbol->segment.has_value();
		return symbol;
	};
	auto file = loadLines(filename, buffer, SymbolFile::Type::NOICE, parseLine);
	 // Heuristic: if all segments in the symbol file are 0,
	 // then assume the file contains no segment information.
	if (anySegment) {
		file.hasSegmentInfo = true;
		for (auto& symbol: file.getSymbols()) {
			if (!symbol.segment) symbol.segment = 0;
		}
	}
	return file;
}

[[nodiscard]] SymbolFile SymbolManager::loadHTC(std::string_view filename, std::string_view buffer)
{
	// TODO check with real HTC file
	auto parseLine = [](std::span<std::string_view> tokens) -> std::optional<Symbol> {
		if (tokens.size() != 3) return {};
		auto label = tokens[0];
		auto value = tokens[1];
		// tokens[2] ???

		auto val = StringOp::stringToBase<16, uint16_t>(value);
		if (!val) return {};
		return checkLabel(label, *val);
	};
	return loadLines(filename, buffer, SymbolFile::Type::HTC, parseLine);
}

[[nodiscard]] SymbolFile SymbolManager::loadVASM(std::string_view filename, std::string_view buffer)
{
	SymbolFile result;
	result.filename = filename;
	result.type = SymbolFile::Type::VASM;

	static constexpr std::string_view whitespace = " \t\r";
	bool skipLines = true;
	for (std::string_view line : StringOp::split_view(buffer, '\n')) {
		if (skipLines) {
			if (line.starts_with("Symbols by value:")) {
				skipLines = false;
			}
			continue;
		}

		auto tokens = static_vector<std::string_view, 2 + 1>{from_range,
			std::views::take(StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace), 2 + 1)};
		if (tokens.size() != 2) continue;
		auto value = tokens[0];
		auto label = tokens[1];

		if (auto val = StringOp::stringToBase<16, uint16_t>(value)) {
			if (auto symbol = checkLabel(label, *val)) {
				result.symbols.push_back(std::move(*symbol));
			}
		}
	}

	return result;
}

[[nodiscard]] SymbolFile SymbolManager::loadNoGmb(std::string_view filename, std::string_view buffer)
{
	auto parseLine = [](std::span<std::string_view> tokens) -> std::optional<Symbol> {
		if (tokens.size() != 2) return {};
		auto value = tokens[0];
		auto label = tokens[1];
		if (!value.starts_with("00:")) return {};
		std::optional<uint16_t> num = StringOp::stringToBase<16, uint16_t>(value.substr(3));
		if (!num.has_value()) return {};
		return checkLabel(label, num.value());
	};
	return loadLines(filename, buffer, SymbolFile::Type::WLALINK_NOGMB, parseLine);
}

[[nodiscard]] SymbolFile SymbolManager::loadASMSX(std::string_view filename, std::string_view buffer)
{
	SymbolFile result;
	result.filename = filename;
	result.type = SymbolFile::Type::ASMSX;

	static constexpr std::string_view whitespace = " \t\r";
	bool symbolPart = false;
	for (std::string_view line : StringOp::split_view(buffer, '\n')) {
		if (line.starts_with(';')) {
			if (line.starts_with("; global and local")) {
				symbolPart = true;
			} else if (line.starts_with("; other")) {
				symbolPart = false;
			}
			continue;
		}
		if (!symbolPart) continue;

		// Possible formats are:  (checked in: https://github.com/Fubukimaru/asMSX/blob/master/src/dura.y#L3987)
		//   <abcd>h <name>         with <abcd> a 4-digit hex value
		//   <xy>h:<abcd>h <name>        <xy>   a 2-digit hex indicating the MegaRom Page (ignored)
		//                               <name> the symbol name
		auto tokens = static_vector<std::string_view, 2 + 1>{from_range,
			std::views::take(StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace), 2 + 1)};
		if (tokens.size() != 2) continue;
		auto value = tokens[0];
		auto label = tokens[1];

		auto [f, l] = StringOp::splitOnFirst(value, ':');
		value = l.empty() ? f : l;

		if (auto symbol = checkLabelAndValue(label, value)) {
			result.symbols.push_back(std::move(*symbol));
		}
	}

	return result;
}

[[nodiscard]] std::optional<unsigned> SymbolManager::isHexDigit(char c)
{
	if ('0' <= c && c <= '9') return c - '0';
	if ('A' <= c && c <= 'F') return c - 'A' + 10;
	if ('a' <= c && c <= 'f') return c - 'a' + 10;
	return {};
}
[[nodiscard]] std::optional<uint16_t> SymbolManager::is4DigitHex(std::string_view s)
{
	if (s.size() != 4) return {};
	unsigned value = 0;
	for (int i = 0; i < 4; ++i) {
		auto digit = isHexDigit(s[i]);
		if (!digit) return {};
		value = (value << 4) | *digit;
	}
	return narrow<uint16_t>(value);
}

[[nodiscard]] SymbolFile SymbolManager::loadLinkMap(std::string_view filename, std::string_view buffer)
{
	// Hi-Tech C link map file. Here's an example of such a file:
	//    https://github.com/artrag/C-experiments-for-msx/blob/master/START.MAP
	SymbolFile result;
	result.filename = filename;
	result.type = SymbolFile::Type::LINKMAP;

	static constexpr std::string_view whitespace = " \t\r";
	bool symbolPart = false;
	for (std::string_view line : StringOp::split_view(buffer, '\n')) {
		if (!symbolPart) {
			if (line.contains("Symbol Table")) {
				symbolPart = true;
			}
			continue;
		}
		// Here's an example of a few lines:
		//   asllmod            text    2CE7  asllsub            text    0AEE
		//   cret               text    2E58  csv                text    2E4C
		//   float_or_long_used (abs)   0001  indir              text    2E5F
		// Note:
		// * Multiple (2 in this case) symbols are defined in a single line.
		// * The width of the columns seems to be the same within a single file, but not across files (?)
		// * Looking at a single symbol:
		//   * There are 3 columns: name, psect, value
		//   * BUT the psect column can be empty!!!
		//     This in combination with an unknown column-width makes parsing difficult.
		//     The heuristic we use is that the last column must match: [0-9A-Fa-f]{4}
		auto tokens = StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace);
		auto it = tokens.begin();
		auto et = tokens.end();
		while (it != et) {
			auto label = *it++;

			if (it == et) break;
			auto value = *it++; // this could either be the psect or the value column
			if (auto val = is4DigitHex(value)) {
				result.symbols.emplace_back(std::string(label), *val, std::nullopt, std::nullopt);
				continue;
			}

			if (it == et) break;
			value = *it++; // try again with 3rd column
			auto val = is4DigitHex(value);
			if (!val) break; // if this also doesn't work there's something wrong, skip this line
			result.symbols.emplace_back(std::string(label), *val, std::nullopt, std::nullopt);
		}
	}

	return result;
}

[[nodiscard]] SymbolFile SymbolManager::loadSymbolFile(
	zstring_view filename, SymbolFile::Type type,
	std::optional<uint8_t> slot, std::optional<uint16_t> segment)
{
	auto buf = File(filename).mmap<const char>();
	std::string_view buffer(buf.data(), buf.size());

	using enum SymbolFile::Type;
	if (type == AUTO_DETECT) {
		type = detectType(filename, buffer);
	}
	assert(type != AUTO_DETECT);

	auto symbolFile = [&]{
		switch (type) {
			case ASMSX:
				return loadASMSX(filename, buffer);
			case GENERIC:
				return loadGeneric(filename, buffer);
			case HTC:
				return loadHTC(filename, buffer);
			case LINKMAP:
				return loadLinkMap(filename, buffer);
			case NOICE:
				return loadNoICE(filename, buffer);
			case VASM:
				return loadVASM(filename, buffer);
			case WLALINK_NOGMB:
				return loadNoGmb(filename, buffer);
			default: UNREACHABLE;
		}
	}();

	// Update slot info for the file and each of its symbol
	symbolFile.slot = slot;
	symbolFile.segment = segment;
	symbolFile.segmentStr = segment ? strCat(*segment) : std::string("-");
	for (auto& symbol: symbolFile.getSymbols()) {
		symbol.slot = slot;
		if (!symbolFile.hasSegmentInfo) {
			symbol.segment = segment;
		}
	}

	return symbolFile;
}

void SymbolManager::refresh()
{
	// Drop caches
	lookupValueCache.clear();

	// Allow to access symbol-values in Tcl expression with syntax: $sym(JIFFY)
	auto& interp = commandController.getInterpreter();
	TclObject arrayName("sym");
	interp.unsetVariable(arrayName.getString().c_str());
	for (const auto& file : files) {
		for (const auto& sym : file.symbols) {
			interp.setVariable(arrayName, TclObject(sym.name), TclObject(sym.value));
		}
	}

	if (observer) observer->notifySymbolsChanged();
}

bool SymbolManager::reloadFile(
	zstring_view filename, LoadEmpty loadEmpty, SymbolFile::Type type,
	std::optional<uint8_t> slot, std::optional<uint16_t> segment)
{
	auto file = loadSymbolFile(filename, type, slot, segment); // might throw
	if (file.symbols.empty() && loadEmpty == LoadEmpty::NOT_ALLOWED) return false;

	if (auto it = std::ranges::find(files, filename, &SymbolFile::filename);
	    it == files.end()) {
		files.push_back(std::move(file));
	} else {
		*it = std::move(file);
	}
	refresh();
	return true;
}

void SymbolManager::removeFile(std::string_view filename)
{
	auto it = std::ranges::find(files, filename, &SymbolFile::filename);
	if (it == files.end()) return; // not found
	files.erase(it);
	refresh();
}

void SymbolManager::removeAllFiles()
{
	files.clear();
	refresh();
}

std::optional<uint16_t> SymbolManager::lookupSymbol(std::string_view str) const
{
	// linear search is fine: only used interactively
	// prefer an exact match
	for (const auto& file : files) {
		if (auto it = std::ranges::find(file.symbols, str, &Symbol::name);
		    it != file.symbols.end()) {
			return it->value;
		}
	}
	// but if not found, a case-insensitive match is fine as well
	for (const auto& file : files) {
		if (auto it = std::ranges::find_if(file.symbols, [&](const auto& sym) {
			return StringOp::casecmp{}(str, sym.name); });
		    it != file.symbols.end()) {
			return it->value;
		}
	}
	return {};
}

std::optional<uint16_t> SymbolManager::parseSymbolOrValue(std::string_view str) const
{
	// first try symbol
	if (auto r = lookupSymbol(str)) return r;
	// if not found, then try to parse as a numerical value
	return parseValue<uint16_t>(str);
}

std::span<Symbol const * const> SymbolManager::lookupValue(uint16_t value)
{
	if (lookupValueCache.empty()) {
		for (const auto& file : files) {
			for (const auto& sym : file.symbols) {
				auto [it, inserted] = lookupValueCache.try_emplace(sym.value, std::vector<const Symbol*>{});
				it->second.push_back(&sym);
			}
		}
	}
	if (auto* sym = lookup(lookupValueCache, value)) {
		return *sym;
	}
	return {};
}

SymbolFile* SymbolManager::findFile(std::string_view filename)
{
	if (auto it = std::ranges::find(files, filename, &SymbolFile::filename); it == files.end()) {
		return nullptr;
	} else {
		return std::to_address(it);
	}
}

std::string SymbolManager::getFileFilters()
{
	return "Auto-detect file type (*){.*},"
	       "asMSX 0.x symbol files (*.sym){.sym},"
	       "HiTech C link map files (*.map){.map},"
	       "HiTech C symbol files (*.sym){.sym},"
	       "NoICE command files (*.noi){.noi},"
	       "pasmo symbol files (*.symbol *.publics *.sys){.symbol,.publics,.sys},"
	       "tniASM 0.x symbol files (*.sym){.sym},"
	       "tniASM 1.x symbol files (*.sym){.sym},"
	       "vasm symbol files (*.sym){.sym},"
	       "wlalink no$gmb symbol files (*.sym){.sym}";
}

SymbolFile::Type SymbolManager::getTypeForFilter(std::string_view filter)
{
	using enum SymbolFile::Type;
	if (filter.starts_with("Auto")) {
		return AUTO_DETECT;
	} else if (filter.starts_with("asMSX")) {
		return ASMSX;
	} else if (filter.starts_with("HiTechC link")) {
		return LINKMAP;
	} else if (filter.starts_with("HiTechC symbol")) {
		return HTC;
	} else if (filter.starts_with("NoICE")) {
		return NOICE;
	} else if (filter.starts_with("vasm")) {
		return VASM;
	} else if (filter.starts_with("wlalink")) {
		return WLALINK_NOGMB;
	} else {
		return GENERIC;
	}
}

} // namespace openmsx

`/`	Search
`g` `c`	Code
`g` `h`	History
`g` `b`	Bookmarks
`g` `t`	Trust
`y`	Permalink to this state (files and trees)
`?`	This sheet

e8555c89 Andy McCall 1 month ago	1	#include "SymbolManager.hh"
	2
	3	#include "CommandController.hh"
	4	#include "File.hh"
	5	#include "Interpreter.hh"
	6	#include "TclObject.hh"
	7
	8	#include "StringOp.hh"
	9	#include "narrow.hh"
	10	#include "static_vector.hh"
	11	#include "stl.hh"
	12	#include "strCat.hh"
	13	#include "unreachable.hh"
	14
	15	#include <algorithm>
	16	#include <bit>
	17	#include <cassert>
	18	#include <fstream>
	19	#include <ranges>
	20
	21	namespace openmsx {
	22
	23	zstring_view SymbolFile::toString(Type type)
	24	{
	25	switch (type) {
	26	using enum Type;
	27	case AUTO_DETECT: return "auto-detect";
	28	case ASMSX: return "asMSX";
	29	case GENERIC: return "generic";
	30	case HTC: return "htc";
	31	case LINKMAP: return "linkmap";
	32	case NOICE: return "NoICE";
	33	case VASM: return "vasm";
	34	case WLALINK_NOGMB: return "wlalink";
	35	default: UNREACHABLE;
	36	}
	37	}
	38
	39	std::optional<SymbolFile::Type> SymbolFile::parseType(std::string_view str)
	40	{
	41	using enum Type;
	42	if (str == "auto-detect") return AUTO_DETECT;
	43	if (str == "asMSX") return ASMSX;
	44	if (str == "generic") return GENERIC;
	45	if (str == "htc") return HTC;
	46	if (str == "linkmap") return LINKMAP;
	47	if (str == "NoICE") return NOICE;
	48	if (str == "vasm") return VASM;
	49	if (str == "wlalink") return WLALINK_NOGMB;
	50	return {};
	51	}
	52
	53
	54	SymbolManager::SymbolManager(CommandController& commandController_)
	55	: commandController(commandController_)
	56	{
	57	}
	58
	59	// detection logic taken from old openmsx-debugger, could probably be improved.
	60	[[nodiscard]] SymbolFile::Type SymbolManager::detectType(std::string_view filename, std::string_view buffer)
	61	{
	62	auto fname = StringOp::toLower(filename);
	63
	64	using enum SymbolFile::Type;
65	if (fname.ends_with(".noi")) {
66	// NoICE command file
67	return NOICE;
68	} else if (fname.ends_with(".map")) {
69	auto [line, _] = StringOp::splitOnFirst(buffer, "\n\r");
70	if (StringOp::containsCaseInsensitive(line, "hi-tech")) {
71	// HiTech link map file
72	return LINKMAP;
73	}
74	// map file output by the Z80ASM from Z88DK
75	return GENERIC;
76	} else if (fname.ends_with(".sym")) {
77	// auto detect which sym file
78	auto [line, _] = StringOp::splitOnFirst(buffer, "\n\r");
79	if (line.starts_with("; Symbol table")) {
80	return ASMSX;
81	} else if (StringOp::containsCaseInsensitive(line, " %equ ")) { // TNIASM1
82	return GENERIC;
83	} else if (StringOp::containsCaseInsensitive(line, " equ ")) {
84	return GENERIC;
85	} else if (StringOp::containsCaseInsensitive(line, "Sections:")) {
86	return VASM;
87	} else if (line.starts_with("; this file was created with wlalink")) {
88	return WLALINK_NOGMB;
89	} else {
90	// this is a blunt conclusion but I don't know a way
91	// to detect this file type
92	return HTC;
93	}
94	} else if (fname.ends_with(".symbol") \|\| fname.ends_with(".publics") \|\| fname.ends_with(".sys")) {
95	/* They are the same type of file. For some reason the Debian
96	* manpage uses the extension ".sys"
97	* pasmo doc -> pasmo [options] file.asm file.bin [file.symbol [file.publics] ]
98	* pasmo manpage in Debian -> pasmo [options] file.asm file.bin [file.sys]
99	*/
100	return GENERIC; // pasmo
101	}
102	return GENERIC;
103	}
104
105	[[nodiscard]] SymbolFile SymbolManager::loadLines(
106	std::string_view filename, std::string_view buffer, SymbolFile::Type type,
107	function_ref<std::optional<Symbol>(std::span<std::string_view>)> lineParser)
108	{
109	SymbolFile result;
110	result.filename = filename;
111	result.type = type;
112
113	static constexpr std::string_view whitespace = " \t\r";
114	for (std::string_view fullLine : StringOp::split_view(buffer, '\n')) {
115	auto [line, _] = StringOp::splitOnFirst(fullLine, ';');
116
117	auto tokens = static_vector<std::string_view, 3 + 1>{from_range,
118	std::views::take(StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace), 3 + 1)};
119	if (auto symbol = lineParser(tokens)) {
120	result.symbols.push_back(std::move(*symbol));
121	}
122	}
123
124	return result;
125	}
126
127	template<typename T>
128	[[nodiscard]] std::optional<T> SymbolManager::parseValue(std::string_view str)
129	{
130	if (str.ends_with('h') \|\| str.ends_with('H')) { // hex
131	str.remove_suffix(1);
132	return StringOp::stringToBase<16, T>(str);
133	}
134	if (str.starts_with('$') \|\| str.starts_with('#')) { // hex
135	str.remove_prefix(1);
136	return StringOp::stringToBase<16, T>(str);
137	}
138	if (str.starts_with('%')) { // bin
139	str.remove_prefix(1);
140	return StringOp::stringToBase<2, T>(str);
141	}
142	// this recognizes the prefixes "0x" or "0X" (for hexadecimal)
143	// and "0b" or "0B" (for binary)
144	// no prefix in interpreted as decimal
145	// "0" as a prefix for octal is intentionally NOT supported
146	return StringOp::stringTo<T>(str);
147	}
148
149	// explicitly instantiate for uint16_t and uint32_t (needed for unittest)
150	template std::optional<uint16_t> SymbolManager::parseValue<uint16_t>(std::string_view);
151	template std::optional<uint32_t> SymbolManager::parseValue<uint32_t>(std::string_view);
152
153	[[nodiscard]] std::optional<Symbol> SymbolManager::checkLabel(std::string_view label, uint32_t value)
154	{
155	if (label.ends_with(':')) label.remove_suffix(1);
156	if (label.empty()) return {};
157
158	auto seg{value > 0xFFFF ? std::optional<uint16_t>(static_cast<uint16_t>(value >> 16)) : std::nullopt};
159	return Symbol{.name = std::string(label), .value = static_cast<uint16_t>(value), .slot = {}, .segment = seg};
160	}
161
162	[[nodiscard]] std::optional<Symbol> SymbolManager::checkLabelAndValue(std::string_view label, std::string_view value)
163	{
164	if (auto num = parseValue<uint16_t>(value)) {
165	return checkLabel(label, *num);
166	}
167	return {};
168	}
169
170	[[nodiscard]] std::optional<Symbol> SymbolManager::checkLabelSegmentAndValue(std::string_view label, std::string_view value)
171	{
172	if (auto num = parseValue<uint32_t>(value)) {
173	return checkLabel(label, *num);
174	}
175	return {};
176	}
177
178	[[nodiscard]] SymbolFile SymbolManager::loadGeneric(std::string_view filename, std::string_view buffer)
179	{
180	auto parseLine = [](std::span<std::string_view> tokens) -> std::optional<Symbol> {
181	if (tokens.size() != 3) return {};
182	auto label = tokens[0];
183	auto equ = tokens[1];
184	auto value = tokens[2];
185	StringOp::casecmp cmp;
186	if (!cmp(equ, "equ") && // TNIASM0, PASMO, SJASM, ...
187	!cmp(equ, "%equ") && // TNIASM1
188	(equ != "=")) return {}; // Z80ASM map file (Z88DK)
189	return checkLabelAndValue(label, value);
190	};
191	return loadLines(filename, buffer, SymbolFile::Type::GENERIC, parseLine);
192	}
193
194	[[nodiscard]] SymbolFile SymbolManager::loadNoICE(std::string_view filename, std::string_view buffer)
195	{
196	bool anySegment = false;
197	auto parseLine = [&](std::span<std::string_view> tokens) -> std::optional<Symbol> {
198	if (tokens.size() != 3) return {};
199	auto def = tokens[0];
200	auto label = tokens[1];
201	auto value = tokens[2];
202	if (StringOp::casecmp cmp; !cmp(def, "def")) return {};
203	// detecting segment information above 16bits
204	auto symbol = checkLabelSegmentAndValue(label, value);
205	anySegment \|= symbol->segment.has_value();
206	return symbol;
207	};
208	auto file = loadLines(filename, buffer, SymbolFile::Type::NOICE, parseLine);
209	// Heuristic: if all segments in the symbol file are 0,
210	// then assume the file contains no segment information.
211	if (anySegment) {
212	file.hasSegmentInfo = true;
213	for (auto& symbol: file.getSymbols()) {
214	if (!symbol.segment) symbol.segment = 0;
215	}
216	}
217	return file;
218	}
219
220	[[nodiscard]] SymbolFile SymbolManager::loadHTC(std::string_view filename, std::string_view buffer)
221	{
222	// TODO check with real HTC file
223	auto parseLine = [](std::span<std::string_view> tokens) -> std::optional<Symbol> {
224	if (tokens.size() != 3) return {};
225	auto label = tokens[0];
226	auto value = tokens[1];
227	// tokens[2] ???
228
229	auto val = StringOp::stringToBase<16, uint16_t>(value);
230	if (!val) return {};
231	return checkLabel(label, *val);
232	};
233	return loadLines(filename, buffer, SymbolFile::Type::HTC, parseLine);
234	}
235
236	[[nodiscard]] SymbolFile SymbolManager::loadVASM(std::string_view filename, std::string_view buffer)
237	{
238	SymbolFile result;
239	result.filename = filename;
240	result.type = SymbolFile::Type::VASM;
241
242	static constexpr std::string_view whitespace = " \t\r";
243	bool skipLines = true;
244	for (std::string_view line : StringOp::split_view(buffer, '\n')) {
245	if (skipLines) {
246	if (line.starts_with("Symbols by value:")) {
247	skipLines = false;
248	}
249	continue;
250	}
251
252	auto tokens = static_vector<std::string_view, 2 + 1>{from_range,
253	std::views::take(StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace), 2 + 1)};
254	if (tokens.size() != 2) continue;
255	auto value = tokens[0];
256	auto label = tokens[1];
257
258	if (auto val = StringOp::stringToBase<16, uint16_t>(value)) {
259	if (auto symbol = checkLabel(label, *val)) {
260	result.symbols.push_back(std::move(*symbol));
261	}
262	}
263	}
264
265	return result;
266	}
267
268	[[nodiscard]] SymbolFile SymbolManager::loadNoGmb(std::string_view filename, std::string_view buffer)
269	{
270	auto parseLine = [](std::span<std::string_view> tokens) -> std::optional<Symbol> {
271	if (tokens.size() != 2) return {};
272	auto value = tokens[0];
273	auto label = tokens[1];
274	if (!value.starts_with("00:")) return {};
275	std::optional<uint16_t> num = StringOp::stringToBase<16, uint16_t>(value.substr(3));
276	if (!num.has_value()) return {};
277	return checkLabel(label, num.value());
278	};
279	return loadLines(filename, buffer, SymbolFile::Type::WLALINK_NOGMB, parseLine);
280	}
281
282	[[nodiscard]] SymbolFile SymbolManager::loadASMSX(std::string_view filename, std::string_view buffer)
283	{
284	SymbolFile result;
285	result.filename = filename;
286	result.type = SymbolFile::Type::ASMSX;
287
288	static constexpr std::string_view whitespace = " \t\r";
289	bool symbolPart = false;
290	for (std::string_view line : StringOp::split_view(buffer, '\n')) {
291	if (line.starts_with(';')) {
292	if (line.starts_with("; global and local")) {
293	symbolPart = true;
294	} else if (line.starts_with("; other")) {
295	symbolPart = false;
296	}
297	continue;
298	}
299	if (!symbolPart) continue;
300
301	// Possible formats are: (checked in: https://github.com/Fubukimaru/asMSX/blob/master/src/dura.y#L3987)
302	// <abcd>h <name> with <abcd> a 4-digit hex value
303	// <xy>h:<abcd>h <name> <xy> a 2-digit hex indicating the MegaRom Page (ignored)
304	// <name> the symbol name
305	auto tokens = static_vector<std::string_view, 2 + 1>{from_range,
306	std::views::take(StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace), 2 + 1)};
307	if (tokens.size() != 2) continue;
308	auto value = tokens[0];
309	auto label = tokens[1];
310
311	auto [f, l] = StringOp::splitOnFirst(value, ':');
312	value = l.empty() ? f : l;
313
314	if (auto symbol = checkLabelAndValue(label, value)) {
315	result.symbols.push_back(std::move(*symbol));
316	}
317	}
318
319	return result;
320	}
321
322	[[nodiscard]] std::optional<unsigned> SymbolManager::isHexDigit(char c)
323	{
324	if ('0' <= c && c <= '9') return c - '0';
325	if ('A' <= c && c <= 'F') return c - 'A' + 10;
326	if ('a' <= c && c <= 'f') return c - 'a' + 10;
327	return {};
328	}
329	[[nodiscard]] std::optional<uint16_t> SymbolManager::is4DigitHex(std::string_view s)
330	{
331	if (s.size() != 4) return {};
332	unsigned value = 0;
333	for (int i = 0; i < 4; ++i) {
334	auto digit = isHexDigit(s[i]);
335	if (!digit) return {};
336	value = (value << 4) \| *digit;
337	}
338	return narrow<uint16_t>(value);
339	}
340
341	[[nodiscard]] SymbolFile SymbolManager::loadLinkMap(std::string_view filename, std::string_view buffer)
342	{
343	// Hi-Tech C link map file. Here's an example of such a file:
344	// https://github.com/artrag/C-experiments-for-msx/blob/master/START.MAP
345	SymbolFile result;
346	result.filename = filename;
347	result.type = SymbolFile::Type::LINKMAP;
348
349	static constexpr std::string_view whitespace = " \t\r";
350	bool symbolPart = false;
351	for (std::string_view line : StringOp::split_view(buffer, '\n')) {
352	if (!symbolPart) {
353	if (line.contains("Symbol Table")) {
354	symbolPart = true;
355	}
356	continue;
357	}
358	// Here's an example of a few lines:
359	// asllmod text 2CE7 asllsub text 0AEE
360	// cret text 2E58 csv text 2E4C
361	// float_or_long_used (abs) 0001 indir text 2E5F
362	// Note:
363	// * Multiple (2 in this case) symbols are defined in a single line.
364	// * The width of the columns seems to be the same within a single file, but not across files (?)
365	// * Looking at a single symbol:
366	// * There are 3 columns: name, psect, value
367	// * BUT the psect column can be empty!!!
368	// This in combination with an unknown column-width makes parsing difficult.
369	// The heuristic we use is that the last column must match: [0-9A-Fa-f]{4}
370	auto tokens = StringOp::split_view<StringOp::EmptyParts::REMOVE>(line, whitespace);
371	auto it = tokens.begin();
372	auto et = tokens.end();
373	while (it != et) {
374	auto label = *it++;
375
376	if (it == et) break;
377	auto value = *it++; // this could either be the psect or the value column
378	if (auto val = is4DigitHex(value)) {
379	result.symbols.emplace_back(std::string(label), *val, std::nullopt, std::nullopt);
380	continue;
381	}
382
383	if (it == et) break;
384	value = *it++; // try again with 3rd column
385	auto val = is4DigitHex(value);
386	if (!val) break; // if this also doesn't work there's something wrong, skip this line
387	result.symbols.emplace_back(std::string(label), *val, std::nullopt, std::nullopt);
388	}
389	}
390
391	return result;
392	}
393
394	[[nodiscard]] SymbolFile SymbolManager::loadSymbolFile(
395	zstring_view filename, SymbolFile::Type type,
396	std::optional<uint8_t> slot, std::optional<uint16_t> segment)
397	{
398	auto buf = File(filename).mmap<const char>();
399	std::string_view buffer(buf.data(), buf.size());
400
401	using enum SymbolFile::Type;
402	if (type == AUTO_DETECT) {
403	type = detectType(filename, buffer);
404	}
405	assert(type != AUTO_DETECT);
406
407	auto symbolFile = [&]{
408	switch (type) {
409	case ASMSX:
410	return loadASMSX(filename, buffer);
411	case GENERIC:
412	return loadGeneric(filename, buffer);
413	case HTC:
414	return loadHTC(filename, buffer);
415	case LINKMAP:
416	return loadLinkMap(filename, buffer);
417	case NOICE:
418	return loadNoICE(filename, buffer);
419	case VASM:
420	return loadVASM(filename, buffer);
421	case WLALINK_NOGMB:
422	return loadNoGmb(filename, buffer);
423	default: UNREACHABLE;
424	}
425	}();
426
427	// Update slot info for the file and each of its symbol
428	symbolFile.slot = slot;
429	symbolFile.segment = segment;
430	symbolFile.segmentStr = segment ? strCat(*segment) : std::string("-");
431	for (auto& symbol: symbolFile.getSymbols()) {
432	symbol.slot = slot;
433	if (!symbolFile.hasSegmentInfo) {
434	symbol.segment = segment;
435	}
436	}
437
438	return symbolFile;
439	}
440
441	void SymbolManager::refresh()
442	{
443	// Drop caches
444	lookupValueCache.clear();
445
446	// Allow to access symbol-values in Tcl expression with syntax: $sym(JIFFY)
447	auto& interp = commandController.getInterpreter();
448	TclObject arrayName("sym");
449	interp.unsetVariable(arrayName.getString().c_str());
450	for (const auto& file : files) {
451	for (const auto& sym : file.symbols) {
452	interp.setVariable(arrayName, TclObject(sym.name), TclObject(sym.value));
453	}
454	}
455
456	if (observer) observer->notifySymbolsChanged();
457	}
458
459	bool SymbolManager::reloadFile(
460	zstring_view filename, LoadEmpty loadEmpty, SymbolFile::Type type,
461	std::optional<uint8_t> slot, std::optional<uint16_t> segment)
462	{
463	auto file = loadSymbolFile(filename, type, slot, segment); // might throw
464	if (file.symbols.empty() && loadEmpty == LoadEmpty::NOT_ALLOWED) return false;
465
466	if (auto it = std::ranges::find(files, filename, &SymbolFile::filename);
467	it == files.end()) {
468	files.push_back(std::move(file));
469	} else {
470	*it = std::move(file);
471	}
472	refresh();
473	return true;
474	}
475
476	void SymbolManager::removeFile(std::string_view filename)
477	{
478	auto it = std::ranges::find(files, filename, &SymbolFile::filename);
479	if (it == files.end()) return; // not found
480	files.erase(it);
481	refresh();
482	}
483
484	void SymbolManager::removeAllFiles()
485	{
486	files.clear();
487	refresh();
488	}
489
490	std::optional<uint16_t> SymbolManager::lookupSymbol(std::string_view str) const
491	{
492	// linear search is fine: only used interactively
493	// prefer an exact match
494	for (const auto& file : files) {
495	if (auto it = std::ranges::find(file.symbols, str, &Symbol::name);
496	it != file.symbols.end()) {
497	return it->value;
498	}
499	}
500	// but if not found, a case-insensitive match is fine as well
501	for (const auto& file : files) {
502	if (auto it = std::ranges::find_if(file.symbols, [&](const auto& sym) {
503	return StringOp::casecmp{}(str, sym.name); });
504	it != file.symbols.end()) {
505	return it->value;
506	}
507	}
508	return {};
509	}
510
511	std::optional<uint16_t> SymbolManager::parseSymbolOrValue(std::string_view str) const
512	{
513	// first try symbol
514	if (auto r = lookupSymbol(str)) return r;
515	// if not found, then try to parse as a numerical value
516	return parseValue<uint16_t>(str);
517	}
518
519	std::span<Symbol const * const> SymbolManager::lookupValue(uint16_t value)
520	{
521	if (lookupValueCache.empty()) {
522	for (const auto& file : files) {
523	for (const auto& sym : file.symbols) {
524	auto [it, inserted] = lookupValueCache.try_emplace(sym.value, std::vector<const Symbol*>{});
525	it->second.push_back(&sym);
526	}
527	}
528	}
529	if (auto* sym = lookup(lookupValueCache, value)) {
530	return *sym;
531	}
532	return {};
533	}
534
535	SymbolFile* SymbolManager::findFile(std::string_view filename)
536	{
537	if (auto it = std::ranges::find(files, filename, &SymbolFile::filename); it == files.end()) {
538	return nullptr;
539	} else {
540	return std::to_address(it);
541	}
542	}
543
544	std::string SymbolManager::getFileFilters()
545	{
546	return "Auto-detect file type (){.},"
547	"asMSX 0.x symbol files (*.sym){.sym},"
548	"HiTech C link map files (*.map){.map},"
549	"HiTech C symbol files (*.sym){.sym},"
550	"NoICE command files (*.noi){.noi},"
551	"pasmo symbol files (.symbol .publics *.sys){.symbol,.publics,.sys},"
552	"tniASM 0.x symbol files (*.sym){.sym},"
553	"tniASM 1.x symbol files (*.sym){.sym},"
554	"vasm symbol files (*.sym){.sym},"
555	"wlalink no$gmb symbol files (*.sym){.sym}";
556	}
557
558	SymbolFile::Type SymbolManager::getTypeForFilter(std::string_view filter)
559	{
560	using enum SymbolFile::Type;
561	if (filter.starts_with("Auto")) {
562	return AUTO_DETECT;
563	} else if (filter.starts_with("asMSX")) {
564	return ASMSX;
565	} else if (filter.starts_with("HiTechC link")) {
566	return LINKMAP;
567	} else if (filter.starts_with("HiTechC symbol")) {
568	return HTC;
569	} else if (filter.starts_with("NoICE")) {
570	return NOICE;
571	} else if (filter.starts_with("vasm")) {
572	return VASM;
573	} else if (filter.starts_with("wlalink")) {
574	return WLALINK_NOGMB;
575	} else {
576	return GENERIC;
577	}
578	}
579
580	} // namespace openmsx

Keyboard