OXYGENHUB
◆ main openmsx / src / commands / TclParser.cc
7.6 KiB · C++ · last changed chore(): initial commit 1 month ago source blame history raw
e8555c89 Andy McCall 1 
#include "TclParser.hh"
2
3 
#include "ScopedAssign.hh"
4 
#include "narrow.hh"
5 
#include "one_of.hh"
6 
#include "strCat.hh"
7
8 
#include <algorithm>
9 
#include <cassert>
10 
#include <iostream>
11
12 
#if DEBUG_TCLPARSER
13 
void TclParser::DEBUG_PRINT(const std::string& s)
14 
{
15 
	std::cout << std::string(2 * level, ' ') << s << '\n';
16 
}
17
18 
static constexpr std::string_view type2string(int type)
19 
{
20 
	switch (type) {
21 
	case TCL_TOKEN_WORD:
22 
		return "word";
23 
	case TCL_TOKEN_SIMPLE_WORD:
24 
		return "simple word";
25 
	case TCL_TOKEN_EXPAND_WORD:
26 
		return "expand word";
27 
	case TCL_TOKEN_TEXT:
28 
		return "text";
29 
	case TCL_TOKEN_BS:
30 
		return "bs";
31 
	case TCL_TOKEN_COMMAND:
32 
		return "command";
33 
	case TCL_TOKEN_VARIABLE:
34 
		return "variable";
35 
	case TCL_TOKEN_SUB_EXPR:
36 
		return "sub expr";
37 
	case TCL_TOKEN_OPERATOR:
38 
		return "operator";
39 
	default:
40 
		assert(false);
41 
		return {};
42 
	}
43 
}
44 
#endif
45
46 
static constexpr bool inRange(char c, char low, char high)
47 
{
48 
	unsigned t = c - low;
49 
	return t <= unsigned(high - low);
50 
}
51
52 
static bool isNumber(std::string_view str)
53 
{
54 
	if (str.starts_with('-') || str.starts_with('+')) {
55 
		str.remove_prefix(1);
56 
	}
57 
	if (str.starts_with("0x") || str.starts_with("0X")) {
58 
		str.remove_prefix(2);
59 
		return std::ranges::all_of(str, [](char c) {
60 
			return inRange(c, '0', '9') ||
61 
			       inRange(c, 'a', 'f') ||
62 
			       inRange(c, 'A', 'F');
63 
		});
64 
	} else {
65 
		return std::ranges::all_of(str,
66 
		                      [](char c) { return inRange(c, '0', '9'); });
67 
	}
68 
}
69
70
71 
TclParser::TclParser(Tcl_Interp* interp_, std::string_view input)
72 
	: interp(interp_)
73 
	, colors(input.size(), '.')
74 
{
75 
	parse(std::string(input), 0, COMMAND);
76 
}
77
78 
void TclParser::parse(std::string parseStr, int offset, ParseType type)
79 
{
80 
	int origSize = narrow<int>(parseStr.size());
81 
	last.push_back(offset);
82
83 
	// The functions Tcl_ParseCommand() and Tcl_ParseExpr() are meant to
84 
	// operate on a complete command. For interactive syntax highlighting
85 
	// we also want to pass incomplete commands (e.g. with an opening, but
86 
	// not yet a closing brace). This loop tries to parse and depending on
87 
	// the parse error retries with a completed command.
88 
	Tcl_Parse parseInfo;
89 
	int retryCount = 0;
90 
	while (true) {
91 
		int parseStatus = (type == EXPRESSION)
92 
			? Tcl_ParseExpr   (interp, parseStr.data(), int(parseStr.size()), &parseInfo)
93 
			: Tcl_ParseCommand(interp, parseStr.data(), int(parseStr.size()), 1, &parseInfo);
94 
		if (parseStatus == TCL_OK) break;
95 
		Tcl_FreeParse(&parseInfo);
96 
		++retryCount;
97
98 
		bool allowComplete = ((offset + parseStr.size()) >= colors.size()) &&
99 
		                     (retryCount < 10);
100 
		Tcl_Obj* resObj = Tcl_GetObjResult(interp);
101 
		Tcl_Size resLen;
102 
		const char* resStr = Tcl_GetStringFromObj(resObj, &resLen);
103 
		std::string_view error(resStr, resLen);
104
105 
		if (allowComplete && error.starts_with("missing close-brace")) {
106 
			parseStr += '}';
107 
		} else if (allowComplete && error.starts_with("missing close-bracket")) {
108 
			parseStr += ']';
109 
		} else if (allowComplete && error.starts_with("missing \"")) {
110 
			parseStr += '"';
111 
		} else if (allowComplete && error.starts_with("unbalanced open paren")) {
112 
			parseStr += ')';
113 
		} else if (allowComplete && error.starts_with("missing operand")) {
114 
			// This also triggers for a (wrong) expression like
115 
			//    'if { / 3'
116 
			// and that can't be solved by adding something at the
117 
			// end. Without the retryCount stuff we would get in an
118 
			// infinite loop here.
119 
			parseStr += '0';
120 
		} else if (allowComplete && error.starts_with("missing )")) {
121 
			parseStr += ')';
122 
		} else {
123 
			DEBUG_PRINT(strCat("ERROR: ", parseStr, ": ", error));
124 
			setColors(parseStr, offset, parseStr.data(), int(parseStr.size()), 'E');
125 
			if ((offset + origSize) < int(colors.size())) last.pop_back();
126 
			return;
127 
		}
128 
	}
129
130 
	if (type == EXPRESSION) {
131 
		DEBUG_PRINT("EXPRESSION: " + parseStr);
132 
	} else {
133 
		if (parseInfo.commentSize) {
134 
			DEBUG_PRINT(strCat("COMMENT: ", std::string_view(parseInfo.commentStart, parseInfo.commentSize)));
135 
			setColors(parseStr, offset, parseInfo.commentStart, parseInfo.commentSize, 'c');
136 
		}
137 
		DEBUG_PRINT(strCat("COMMAND: ", std::string_view(parseInfo.commandStart, parseInfo.commandSize)));
138 
	}
139 
	printTokens(parseStr, offset, {parseInfo.tokenPtr, size_t(parseInfo.numTokens)});
140
141 
	// If the current sub-command stops before the end of the original
142 
	// full command, then it's not the last sub-command. Note that
143 
	// sub-commands can be nested.
144 
	if ((offset + origSize) < int(colors.size())) last.pop_back();
145
146 
	const char* nextStart = parseInfo.commandStart + parseInfo.commandSize;
147 
	Tcl_FreeParse(&parseInfo);
148
149 
	if (type == COMMAND) {
150 
		// next command
151 
		auto nextSize = int((parseStr.data() + parseStr.size()) - nextStart);
152 
		if (nextSize > 0) {
153 
			int newOffset = offset + narrow<int>(nextStart - parseStr.data());
154 
			parse(std::string(nextStart, nextSize), newOffset, type);
155 
		}
156 
	}
157 
}
158
159 
void TclParser::printTokens(std::string_view parseStr, int offset, std::span<const Tcl_Token> tokens)
160 
{
161 
#if DEBUG_TCLPARSER
162 
	ScopedAssign sa(level, level + 1);
163 
#endif
164 
	for (size_t i = 0; i < tokens.size(); /**/) {
165 
		const Tcl_Token& token = tokens[i];
166 
		std::string_view tokenStr(token.start, token.size);
167 
		DEBUG_PRINT(strCat(type2string(token.type), " -> ", tokenStr));
168 
		switch (token.type) {
169 
		case TCL_TOKEN_VARIABLE:
170 
			assert(token.numComponents >= 1);
171 
			setColors(parseStr, offset, tokens[i + 1].start - 1, tokens[i + 1].size + 1, 'v');
172 
			break;
173 
		case TCL_TOKEN_WORD:
174 
		case TCL_TOKEN_SIMPLE_WORD:
175 
			if (*token.start == '"') {
176 
				setColors(parseStr, offset, token.start, token.size, 'l');
177 
			}
178 
			if ((i == 0) && isProc(interp, tokenStr)) {
179 
				setColors(parseStr, offset, token.start, token.size, 'p');
180 
			}
181 
			break;
182 
		case TCL_TOKEN_EXPAND_WORD:
183 
			setColors(parseStr, offset, token.start, 3, 'o');
184 
			break;
185 
		case TCL_TOKEN_OPERATOR:
186 
		case TCL_TOKEN_BS:
187 
			setColors(parseStr, offset, token.start, token.size, 'o');
188 
			break;
189 
		case TCL_TOKEN_TEXT:
190 
			if (isNumber(tokenStr) || (*token.start == '"')) {
191 
				// TODO only works if the same as 'l'
192 
				setColors(parseStr, offset, token.start, token.size, 'l');
193 
			}
194 
			break;
195 
		}
196 
		if (token.type == TCL_TOKEN_COMMAND) {
197 
			int newOffset = offset + narrow<int>(token.start + 1 - parseStr.data());
198 
			parse(std::string(token.start + 1, token.size - 2), newOffset, COMMAND);
199 
		} else if (token.type == TCL_TOKEN_SIMPLE_WORD) {
200 
			ParseType subType = guessSubType(tokens, i);
201 
			if (subType != OTHER) {
202 
				int newOffset = offset + narrow<int>(tokens[i + 1].start - parseStr.data());
203 
				parse(std::string(tokens[i + 1].start, tokens[i + 1].size), newOffset, subType);
204 
			}
205 
		}
206 
		printTokens(parseStr, offset, tokens.subspan(++i, token.numComponents));
207 
		i += token.numComponents;
208 
	}
209 
}
210
211 
TclParser::ParseType TclParser::guessSubType(std::span<const Tcl_Token> tokens, size_t i)
212 
{
213 
	// heuristic: if previous token is 'if' then assume this is an expression
214 
	if ((i >= 1) && (tokens[i - 1].type == TCL_TOKEN_TEXT)) {
215 
		std::string_view prevText(tokens[i - 1].start, tokens[i - 1].size);
216 
		if (prevText == one_of("if", "elseif", "expr")) {
217 
			return EXPRESSION;
218 
		}
219 
	}
220
221 
	// heuristic: parse text that starts with { as a subcommand
222 
	if (*tokens[i].start == '{') {
223 
		return COMMAND;
224 
	}
225
226 
	// a plain text element
227 
	return OTHER;
228 
}
229
230 
bool TclParser::isProc(Tcl_Interp* interp, std::string_view str)
231 
{
232 
	auto command = tmpStrCat("openmsx::is_command_name {", str, '}');
233 
	if (Tcl_Eval(interp, command.c_str()) != TCL_OK) return false;
234 
	int result;
235 
	if (Tcl_GetBooleanFromObj(interp, Tcl_GetObjResult(interp), &result)
236 
		!= TCL_OK) return false;
237 
	return result != 0;
238 
}
239
240 
void TclParser::setColors(std::string_view parseStr, int offset, const char* p, int size, char c)
241 
{
242 
	int start = narrow<int>(p - parseStr.data()) + offset;
243 
	int stop = std::min(start + size, int(colors.size()));
244 
	for (auto i : xrange(start, stop)) {
245 
		colors[i] = c;
246 
	}
247 
}