GCC Code Coverage Report

src/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
363 / 385, 3 excl
94.3%
Branches:
356 / 379
93.9%
Functions:
21 / 21
100.0%
lexer/lexer.spice
Line Branch Exec Source
1 // Copyright (c) 2021-2026 ChilliBits. All rights reserved.
2
3 // Std imports
4 import "std/data/vector";
5 import "std/text/format";
6 import "std/text/analysis";
7 import "std/io/filepath";
8
9 // Own imports
10 import "@/exception/fatal-error";
11 import "@/lexer/token";
12 import "@/reader/reader";
13 import "@/reader/code-loc";
14 import "@/exception/lexer-error";
15
16 3951 public type Lexer struct {
17 Reader reader
18 Vector<Token> tokens // Fully materialized token stream, terminated by a single EOF token
19 unsigned long curIdx // Index of the current token within 'tokens'
20 // Comment-free lookahead index. 'relevantIndices' lists the positions in 'tokens' of all
21 // non-comment (relevant) tokens, and 'relevantOrdinal[i]' is the ordinal of the relevant token
22 // at position i within 'relevantIndices'. Together they give the parser O(1) relevant-token
23 // lookahead (see peekRelevantType) instead of re-scanning the stream on every peek.
24 Vector<unsigned long> relevantIndices
25 Vector<unsigned long> relevantOrdinal
26 }
27
28 3963 public p Lexer.ctor(const FilePath& filePath) {
29 3963 this.reader = Reader(filePath);
30 3963 this.curIdx = 0l;
31
32 // Tokenize the whole input up front. Materializing the token stream once turns getToken(),
33 // advance() and peek() into O(1) index operations. The previous streaming design re-scanned
34 // the reader (and shifted a lookahead buffer) on every peek, which made the parser's repeated
35 // lookahead quadratic and dominated the overall parse time.
36
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2663887 times.
 2663887 while true {
37 2663887 Token token = this.readRawToken();
38 2663887 const bool isEof = token.tokenType == TokenType::EOF;
39 2663887 this.tokens.pushBack(token);
40
2/2
✓ Branch 0 taken 3963 times.
✓ Branch 1 taken 2659924 times.
 2663887 if isEof {
41 3963 break;
42 }
43 2663887 }
44
45 // Build the comment-free lookahead index over the materialized token stream.
46 3963 const unsigned long tokenCount = this.tokens.getSize();
47 3963 unsigned long ordinal = 0l;
48
2/2
✓ Branch 0 taken 2663887 times.
✓ Branch 1 taken 3963 times.
 2667850 for unsigned long i = 0l; i < tokenCount; i++ {
49 2663887 this.relevantOrdinal.pushBack(ordinal);
50
4/4
✓ Branch 0 taken 2617071 times.
✓ Branch 1 taken 46816 times.
✓ Branch 2 taken 95733 times.
✓ Branch 3 taken 2568154 times.
 5280958 if !this.isIgnoredType(this.tokens.get(i).tokenType) {
51 2568154 this.relevantIndices.pushBack(i);
52 2568154 ordinal++;
53 2568154 }
54 2663887 }
55 3963 }
56
57 inline f<bool> Lexer.isIgnoredType(TokenType tokenType) {
58 return tokenType == TokenType::LINE_COMMENT || tokenType == TokenType::BLOCK_COMMENT || tokenType == TokenType::DOC_COMMENT;
59 }
60
61 /**
62 * Return the token type of the k-th relevant (non-comment) token at or after the current one,
63 * where k=0 is the current token. This is O(1) and is the fast path the parser uses for lookahead.
64 *
65 * @param k Zero-based relevant-token lookahead distance
66 * @return The token type at that position (EOF when looking past the end)
67 */
68 4479612 public f<TokenType> Lexer.peekRelevantType(unsigned long k) {
69 4479612 unsigned long ord = this.relevantOrdinal.get(this.curIdx) + k;
70 4479612 const unsigned long lastOrd = this.relevantIndices.getSize() - 1l;
71
1/2
✓ Branch 0 taken 4479612 times.
✗ Branch 1 not taken.
 4479612 if ord > lastOrd {
72 ✗ ord = lastOrd;
73 ✗ }
74 4479612 return this.tokens.get(this.relevantIndices.get(ord)).tokenType;
75 }
76
77 28832386 public f<const Token&> Lexer.getToken() {
78 28832386 return this.tokens.get(this.curIdx);
79 }
80
81 /**
82 * Get the last relevant (non-comment) token before the current one, e.g. the last token of a node, that was just parsed.
83 *
84 * @return Previous relevant token or the current token, if there is none
85 */
86 2230901 public f<const Token&> Lexer.getPreviousRelevantToken() {
87 2230901 const unsigned long ordinal = this.relevantOrdinal.get(this.curIdx);
88
2/2
✓ Branch 0 taken 2230900 times.
✓ Branch 1 taken 1 time.
 2230901 if ordinal == 0l {
89 1 return this.tokens.get(this.curIdx);
90 }
91 2230900 return this.tokens.get(this.relevantIndices.get(ordinal - 1l));
92 }
93
94 2659840 public p Lexer.advance() {
95 // Stop at the terminating EOF token so that getToken() keeps returning EOF once the end is reached.
96
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2659840 times.
 2659840 if this.curIdx + 1l < this.tokens.getSize() {
97 2659840 this.curIdx++;
98 2659840 }
99 2659840 }
100
101 /**
102 * Get the index of the current token within the token stream
103 *
104 * @return Current token index
105 */
106 8478 public f<unsigned long> Lexer.getTokenIndex() {
107 8478 return this.curIdx;
108 }
109
110 /**
111 * Concatenate the texts of all relevant (non-comment) tokens in the index range [startIdx, endIdx), without whitespace.
112 *
113 * @param startIdx Index of the first token
114 * @param endIdx Index after the last token
115 * @return Concatenated token text
116 */
117 159 public f<String> Lexer.getTokenText(unsigned long startIdx, unsigned long endIdx) {
118 159 String text;
119
2/2
✓ Branch 0 taken 270 times.
✓ Branch 1 taken 159 times.
 429 for unsigned long i = startIdx; i < endIdx; i++ {
120 270 const Token& token = this.tokens.get(i);
121
2/4
✓ Branch 0 taken 270 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 270 times.
 540 if !this.isIgnoredType(token.tokenType) {
122 270 text += token.text;
123 270 }
124 270 }
125 159 return text;
126 }
127
128 /**
129 * Get the source code, that is covered by the relevant (non-comment) tokens in the index range [startIdx, endIdx).
130 *
131 * @param startIdx Index of the first token
132 * @param endIdx Index after the last token
133 * @return Source code snippet
134 */
135 4080 public f<String> Lexer.getSourceCode(unsigned long startIdx, unsigned long endIdx) {
136 // Skip trailing comment tokens
137
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4080 times.
✓ Branch 2 taken 4080 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 4080 times.
 8160 while endIdx > startIdx && this.isIgnoredType(this.tokens.get(endIdx - 1l).tokenType) {
138 ✗ endIdx--;
139 ✗ }
140
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4080 times.
 4080 if endIdx <= startIdx {
141 ✗ return String();
142 }
143 4080 const Token& firstToken = this.tokens.get(startIdx);
144 4080 const Token& lastToken = this.tokens.get(endIdx - 1l);
145 4080 return this.reader.getSourceCode(firstToken.startOffset, lastToken.endOffset);
146 }
147
148 /**
149 * Read the next raw token directly from the reader, skipping leading whitespace.
150 * This does not touch the lookahead buffer and is the single source of truth for
151 * turning characters into tokens.
152 */
153 2663888 f<Token> Lexer.readRawToken() {
154 // Skip any whitespaces
155
4/4
✓ Branch 0 taken 2663888 times.
✓ Branch 1 taken 2749080 times.
✓ Branch 2 taken 2749080 times.
✓ Branch 3 taken 2663888 times.
 5412968 while isWhitespace(this.reader.getChar()) && !this.reader.isEOF() {
156 2749080 this.reader.advance();
157 2749080 }
158
159 // Consume the token and remember where it is located within the source code
160 2663888 const unsigned long startOffset = this.reader.getOffset();
161 2663888 Token token = this.consumeToken();
162 2663888 token.startOffset = startOffset;
163 2663888 token.endOffset = this.reader.getOffset();
164 // A code location, created from a token, covers the token
165 2663888 token.codeLoc.sourceIntervalStart = cast<long>(token.startOffset);
166 2663888 token.codeLoc.sourceIntervalStop = cast<long>(token.endOffset) - 1l;
167 2663888 return token;
168 }
169
170 2323751 public f<CodeLoc> Lexer.getCodeLoc() {
171 2323751 return this.getToken().codeLoc;
172 }
173
174 2663888 f<Token> Lexer.consumeToken() {
175 // Get the current char from the reader instance
176 2663888 const char curChar = this.reader.getChar();
177 2663888 const CodeLoc codeLoc = this.reader.getCodeLoc();
178
179 // Check if EOF
180
2/2
✓ Branch 0 taken 3963 times.
✓ Branch 1 taken 2659925 times.
 2663888 if this.reader.isEOF() {
181 3963 return Token(TokenType::EOF, "EOF", codeLoc);
182 }
183 // Check if identifier
184
4/4
✓ Branch 0 taken 1167889 times.
✓ Branch 1 taken 1492036 times.
✓ Branch 2 taken 1488488 times.
✓ Branch 3 taken 1171437 times.
 2659925 if isAlpha(curChar) || curChar == '_' {
185
2/2
✓ Branch 0 taken 136986 times.
✓ Branch 1 taken 1034451 times.
 1171437 if isUpper(curChar) { // Type identifier
186 136986 return this.consumeTypeIdentifier(codeLoc);
187 } else { // Normal identifier or keyword
188 1034451 return this.consumeKeywordOrIdentifier(codeLoc);
189 }
190 ✗ }
191 // Check if number
192
2/2
✓ Branch 0 taken 51680 times.
✓ Branch 1 taken 1436808 times.
 1488488 if isDigit(curChar) {
193 // Consume number literal
194 51680 Token numericLiteral = this.consumeNumberLiteral(codeLoc);
195
2/2
✓ Branch 0 taken 1351 times.
✓ Branch 1 taken 50329 times.
 51680 if numericLiteral.tokenType == TokenType::DOUBLE_LIT {
196 1351 return numericLiteral;
197 }
198
199 // Consume the optional unsigned suffix. It stays part of the token text, so that the parser can detect it
200
2/2
✓ Branch 0 taken 46617 times.
✓ Branch 1 taken 3712 times.
 50329 if this.reader.getChar() == 'u' {
201 3712 numericLiteral.text += 'u';
202 3712 this.reader.advance(); // Consume 'u'
203 3712 }
204
205 // Patch token type if suffix is present and consume the suffix character
206 50329 const char literalSuffix = this.reader.getChar();
207
2/2
✓ Branch 0 taken 2234 times.
✓ Branch 1 taken 48095 times.
 50329 if literalSuffix == 's' {
208 2234 numericLiteral.tokenType = TokenType::SHORT_LIT;
209 2234 this.reader.advance(); // Consume 's'
210
2/2
✓ Branch 0 taken 18718 times.
✓ Branch 1 taken 29377 times.
 50329 } else if literalSuffix == 'l' {
211 29377 numericLiteral.tokenType = TokenType::LONG_LIT;
212 29377 this.reader.advance(); // Consume 'l'
213 48095 }
214
215 50329 return numericLiteral;
216 }
217 // Check if char literal
218
2/2
✓ Branch 0 taken 9889 times.
✓ Branch 1 taken 1426919 times.
 1436808 if curChar == '\'' {
219 9889 return this.consumeCharLiteral(codeLoc);
220 }
221 // Check if string literal
222
2/2
✓ Branch 0 taken 17183 times.
✓ Branch 1 taken 1409736 times.
 1426919 if curChar == '"' {
223 17183 return this.consumeStringLiteral(codeLoc);
224 }
225
226 // Double literal without integer part, e.g. '.5'
227
3/4
✓ Branch 0 taken 1257804 times.
✓ Branch 1 taken 151932 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1409736 times.
 1409736 if curChar == '.' && isDigit(this.reader.peekChar()) {
228 ✗ return this.consumeNumberLiteral(codeLoc);
229 }
230
231 // Consume the first char of the operator and check for longer operators, starting with it
232 1409736 this.reader.advance();
233
25/26
✓ Branch 0 taken 149442 times.
✓ Branch 1 taken 159657 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 98433 times.
✓ Branch 4 taken 98433 times.
✓ Branch 5 taken 149442 times.
✓ Branch 6 taken 18457 times.
✓ Branch 7 taken 18457 times.
✓ Branch 8 taken 34 times.
✓ Branch 9 taken 2600 times.
✓ Branch 10 taken 75919 times.
✓ Branch 11 taken 7958 times.
✓ Branch 12 taken 27956 times.
✓ Branch 13 taken 314 times.
✓ Branch 14 taken 9866 times.
✓ Branch 15 taken 21559 times.
✓ Branch 16 taken 11732 times.
✓ Branch 17 taken 35601 times.
✓ Branch 18 taken 96196 times.
✓ Branch 19 taken 314 times.
✓ Branch 20 taken 91510 times.
✓ Branch 21 taken 89330 times.
✓ Branch 22 taken 85065 times.
✓ Branch 23 taken 6415 times.
✓ Branch 24 taken 151932 times.
✓ Branch 25 taken 3114 times.
 1409736 switch curChar {
234 98433 case '{': { return Token(TokenType::LBRACE, "{", codeLoc); }
235 98433 case '}': { return Token(TokenType::RBRACE, "}", codeLoc); }
236 149442 case '(': { return Token(TokenType::LPAREN, "(", codeLoc); }
237 149442 case ')': { return Token(TokenType::RPAREN, ")", codeLoc); }
238 18457 case '[': { return Token(TokenType::LBRACKET, "[", codeLoc); }
239 18457 case ']': { return Token(TokenType::RBRACKET, "]", codeLoc); }
240 34 case '~': { return Token(TokenType::BITWISE_NOT, "~", codeLoc); }
241 2600 case '?': { return Token(TokenType::QUESTION_MARK, "?", codeLoc); }
242 159657 case ';': { return Token(TokenType::SEMICOLON, ";", codeLoc); }
243 75919 case ',': { return Token(TokenType::COMMA, ",", codeLoc); }
244 case '|': {
245
4/4
✓ Branch 0 taken 2904 times.
✓ Branch 1 taken 5054 times.
✓ Branch 2 taken 5054 times.
✓ Branch 3 taken 2904 times.
 15916 if this.consumeIf('|') { return Token(TokenType::LOGICAL_OR, "||", codeLoc); }
246
4/4
✓ Branch 0 taken 2881 times.
✓ Branch 1 taken 23 times.
✓ Branch 2 taken 23 times.
✓ Branch 3 taken 2881 times.
 5808 if this.consumeIf('=') { return Token(TokenType::OR_EQUAL, "|=", codeLoc); }
247 2881 return Token(TokenType::BITWISE_OR, "|", codeLoc);
248 }
249 case '&': {
250
4/4
✓ Branch 0 taken 25580 times.
✓ Branch 1 taken 2376 times.
✓ Branch 2 taken 2376 times.
✓ Branch 3 taken 25580 times.
 55912 if this.consumeIf('&') { return Token(TokenType::LOGICAL_AND, "&&", codeLoc); }
251
4/4
✓ Branch 0 taken 25559 times.
✓ Branch 1 taken 21 times.
✓ Branch 2 taken 21 times.
✓ Branch 3 taken 25559 times.
 51160 if this.consumeIf('=') { return Token(TokenType::AND_EQUAL, "&=", codeLoc); }
252 25559 return Token(TokenType::BITWISE_AND, "&", codeLoc);
253 }
254 case '^': {
255
4/4
✓ Branch 0 taken 83 times.
✓ Branch 1 taken 231 times.
✓ Branch 2 taken 231 times.
✓ Branch 3 taken 83 times.
 628 if this.consumeIf('=') { return Token(TokenType::XOR_EQUAL, "^=", codeLoc); }
256 83 return Token(TokenType::BITWISE_XOR, "^", codeLoc);
257 }
258 case '!': {
259
4/4
✓ Branch 0 taken 4240 times.
✓ Branch 1 taken 5626 times.
✓ Branch 2 taken 5626 times.
✓ Branch 3 taken 4240 times.
 19732 if this.consumeIf('=') { return Token(TokenType::NOT_EQUAL, "!=", codeLoc); }
260 4240 return Token(TokenType::NOT, "!", codeLoc);
261 }
262 case '+': {
263
4/4
✓ Branch 0 taken 14362 times.
✓ Branch 1 taken 7197 times.
✓ Branch 2 taken 7197 times.
✓ Branch 3 taken 14362 times.
 43118 if this.consumeIf('+') { return Token(TokenType::PLUS_PLUS, "++", codeLoc); }
264
4/4
✓ Branch 0 taken 11118 times.
✓ Branch 1 taken 3244 times.
✓ Branch 2 taken 3244 times.
✓ Branch 3 taken 11118 times.
 28724 if this.consumeIf('=') { return Token(TokenType::PLUS_EQUAL, "+=", codeLoc); }
265 11118 return Token(TokenType::PLUS, "+", codeLoc);
266 }
267 case '-': {
268
4/4
✓ Branch 0 taken 10505 times.
✓ Branch 1 taken 1227 times.
✓ Branch 2 taken 1227 times.
✓ Branch 3 taken 10505 times.
 23464 if this.consumeIf('-') { return Token(TokenType::MINUS_MINUS, "--", codeLoc); }
269
4/4
✓ Branch 0 taken 10202 times.
✓ Branch 1 taken 303 times.
✓ Branch 2 taken 303 times.
✓ Branch 3 taken 10202 times.
 21010 if this.consumeIf('=') { return Token(TokenType::MINUS_EQUAL, "-=", codeLoc); }
270
4/4
✓ Branch 0 taken 10198 times.
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 4 times.
✓ Branch 3 taken 10198 times.
 20404 if this.consumeIf('>') { return Token(TokenType::ARROW, "->", codeLoc); }
271 10198 return Token(TokenType::MINUS, "-", codeLoc);
272 }
273 case '*': {
274
4/4
✓ Branch 0 taken 35191 times.
✓ Branch 1 taken 410 times.
✓ Branch 2 taken 410 times.
✓ Branch 3 taken 35191 times.
 71202 if this.consumeIf('=') { return Token(TokenType::MUL_EQUAL, "*=", codeLoc); }
275 35191 return Token(TokenType::MUL, "*", codeLoc);
276 }
277 case '/': {
278
4/4
✓ Branch 0 taken 96058 times.
✓ Branch 1 taken 138 times.
✓ Branch 2 taken 138 times.
✓ Branch 3 taken 96058 times.
 192392 if this.consumeIf('=') { return Token(TokenType::DIV_EQUAL, "/=", codeLoc); }
279
2/2
✓ Branch 0 taken 46816 times.
✓ Branch 1 taken 49242 times.
 96058 if this.reader.getChar() == '/' { return this.consumeLineComment(codeLoc); }
280
2/2
✓ Branch 0 taken 48917 times.
✓ Branch 1 taken 325 times.
 49242 if this.reader.getChar() == '*' { return this.consumeBlockOrDocComment(codeLoc); }
281 325 return Token(TokenType::DIV, "/", codeLoc);
282 }
283 case '%': {
284
4/4
✓ Branch 0 taken 264 times.
✓ Branch 1 taken 50 times.
✓ Branch 2 taken 50 times.
✓ Branch 3 taken 264 times.
 628 if this.consumeIf('=') { return Token(TokenType::REM_EQUAL, "%=", codeLoc); }
285 264 return Token(TokenType::REM, "%", codeLoc);
286 }
287 case '=': {
288
4/4
✓ Branch 0 taken 67473 times.
✓ Branch 1 taken 24037 times.
✓ Branch 2 taken 24037 times.
✓ Branch 3 taken 67473 times.
 183020 if this.consumeIf('=') { return Token(TokenType::EQUAL, "==", codeLoc); }
289 67473 return Token(TokenType::ASSIGN, "=", codeLoc);
290 }
291 case '<': {
292
4/4
✓ Branch 0 taken 87724 times.
✓ Branch 1 taken 1606 times.
✓ Branch 2 taken 1606 times.
✓ Branch 3 taken 87724 times.
 178660 if this.consumeIf('=') { return Token(TokenType::LESS_EQUAL, "<=", codeLoc); }
293
4/4
✓ Branch 0 taken 87227 times.
✓ Branch 1 taken 497 times.
✓ Branch 2 taken 497 times.
✓ Branch 3 taken 87227 times.
 175448 if this.consumeIf('<') {
294
4/4
✓ Branch 0 taken 493 times.
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 4 times.
✓ Branch 3 taken 493 times.
 994 if this.consumeIf('=') { return Token(TokenType::SHL_EQUAL, "<<=", codeLoc); }
295 493 return Token(TokenType::SHL, "<<", codeLoc);
296 }
297 87227 return Token(TokenType::LESS, "<", codeLoc);
298 }
299 case '>': {
300
4/4
✓ Branch 0 taken 81579 times.
✓ Branch 1 taken 3486 times.
✓ Branch 2 taken 3486 times.
✓ Branch 3 taken 81579 times.
 170130 if this.consumeIf('=') { return Token(TokenType::GREATER_EQUAL, ">=", codeLoc); }
301
4/4
✓ Branch 0 taken 78590 times.
✓ Branch 1 taken 2989 times.
✓ Branch 2 taken 2989 times.
✓ Branch 3 taken 78590 times.
 163158 if this.consumeIf('>') {
302
4/4
✓ Branch 0 taken 2978 times.
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 11 times.
✓ Branch 3 taken 2978 times.
 5978 if this.consumeIf('=') { return Token(TokenType::SHR_EQUAL, ">>=", codeLoc); }
303 2978 return Token(TokenType::SHR, ">>", codeLoc);
304 }
305 78590 return Token(TokenType::GREATER, ">", codeLoc);
306 }
307 case ':': {
308
4/4
✓ Branch 0 taken 3676 times.
✓ Branch 1 taken 2739 times.
✓ Branch 2 taken 2739 times.
✓ Branch 3 taken 3676 times.
 12830 if this.consumeIf(':') { return Token(TokenType::SCOPE_ACCESS, "::", codeLoc); }
309 3676 return Token(TokenType::COLON, ":", codeLoc);
310 }
311 case '.': {
312
4/4
✓ Branch 0 taken 151215 times.
✓ Branch 1 taken 717 times.
✓ Branch 2 taken 717 times.
✓ Branch 3 taken 151215 times.
 303864 if this.consumeIf('.') {
313 717 this.reader.expect('.'); // Consume third '.'
314 717 return Token(TokenType::ELLIPSIS, "...", codeLoc);
315 }
316 151215 return Token(TokenType::DOT, ".", codeLoc);
317 }
318 case '#': {
319
4/4
✓ Branch 0 taken 1574 times.
✓ Branch 1 taken 1540 times.
✓ Branch 2 taken 1540 times.
✓ Branch 3 taken 1574 times.
 6228 if this.consumeIf('!') { return Token(TokenType::MOD_ATTR_PREAMBLE, "#!", codeLoc); }
320 1574 return Token(TokenType::FCT_ATTR_PREAMBLE, "#", codeLoc);
321 }
322 }
323
324 − const String message = String("token recognition error at: '") + curChar + "'"; // LCOV_EXCL_LINE
325 − this.throwLexerError(codeLoc, message.getRaw()); // LCOV_EXCL_LINE
326 − return Token(TokenType::INVALID); // LCOV_EXCL_LINE
327 ✗ }
328
329 /**
330 * Consume the current char, if it is the expected one
331 *
332 * @param expected Expected char
333 * @return Whether the char was consumed
334 */
335 inline f<bool> Lexer.consumeIf(char expected) {
336 if this.reader.getChar() != expected {
337 return false;
338 }
339 this.reader.advance();
340 return true;
341 }
342
343 /**
344 * Abort lexing with a lexer error
345 *
346 * @param codeLoc Code location of the error
347 * @param message Error message suffix
348 */
349 1 p Lexer.throwLexerError(const CodeLoc& codeLoc, string message) {
350 1 const LexerError lexerError = LexerError(codeLoc, LexerErrorType::TOKENIZING_FAILED, message);
351 1 abortWithError(lexerError.getMessage());
352 1 }
353
354 51680 f<Token> Lexer.consumeNumberLiteral(const CodeLoc& codeLoc) {
355 51680 String numberStr;
356
357 // Check for optional minus sign
358
1/2
✓ Branch 0 taken 51680 times.
✗ Branch 1 not taken.
 51680 if this.reader.getChar() == '-' {
359 ✗ numberStr += '-';
360 ✗ this.reader.advance(); // Consume '-'
361 ✗ }
362
363 // Check for different numeric literal formats
364
2/2
✓ Branch 0 taken 27121 times.
✓ Branch 1 taken 24559 times.
 51680 if this.reader.getChar() == '0' { // Potentially a base-prefixed literal
365 24559 this.reader.advance(); // Consume '0'
366 24559 numberStr += '0';
367 // Peek the character after '0', which is the base of the number ('x', 'b', 'o' or 'd') if present
368 24559 const char base = toLower(this.reader.getChar());
369
8/10
✓ Branch 0 taken 23985 times.
✓ Branch 1 taken 574 times.
✓ Branch 2 taken 23985 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 23985 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 734 times.
✓ Branch 7 taken 23251 times.
✓ Branch 8 taken 1308 times.
✓ Branch 9 taken 23251 times.
 24559 if base == 'x' || base == 'h' || base == 'b' || base == 'o' || base == 'd' {
370 1308 numberStr += this.reader.getChar();
371 1308 this.reader.advance(); // Consume the base character
372 // Decide what to do, depending on the base
373
4/4
✓ Branch 0 taken 574 times.
✓ Branch 1 taken 734 times.
✓ Branch 2 taken 574 times.
✓ Branch 3 taken 734 times.
 1308 if base == 'x' || base == 'h' { // Hexadecimal number
374
2/2
✓ Branch 0 taken 5456 times.
✓ Branch 1 taken 574 times.
 6030 while isHexDigit(this.reader.getChar()) {
375 5456 numberStr += this.reader.getChar();
376 5456 this.reader.advance();
377 5456 }
378
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 734 times.
 1308 } else if base == 'b' { // Binary number
379
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ while isBinDigit(this.reader.getChar()) {
380 ✗ numberStr += this.reader.getChar();
381 ✗ this.reader.advance();
382 ✗ }
383
1/2
✓ Branch 0 taken 734 times.
✗ Branch 1 not taken.
 734 } else if base == 'o' { // Octal number
384
2/2
✓ Branch 0 taken 4592 times.
✓ Branch 1 taken 734 times.
 5326 while isOctDigit(this.reader.getChar()) {
385 4592 numberStr += this.reader.getChar();
386 4592 this.reader.advance();
387 4592 }
388 734 } else { // Decimal number with explicit '0d' prefix
389
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ while isDigit(this.reader.getChar()) {
390 ✗ numberStr += this.reader.getChar();
391 ✗ this.reader.advance();
392 ✗ }
393 734 }
394 // The correct token type is set a layer above in the parsing functions for int, short, long, etc.
395 2616 return Token(TokenType::INT_LIT, numberStr, codeLoc);
396 }
397 23251 }
398
399 // Decimal number (or the remainder of a decimal number with leading zero)
400
2/2
✓ Branch 0 taken 40172 times.
✓ Branch 1 taken 50372 times.
 90544 while isDigit(this.reader.getChar()) {
401 40172 numberStr += this.reader.getChar();
402 40172 this.reader.advance();
403 40172 }
404
405 // Check for a double literal: [0-9]*[.][0-9]+([eE][+-]?[0-9]+)?
406
4/4
✓ Branch 0 taken 49021 times.
✓ Branch 1 taken 1351 times.
✓ Branch 2 taken 1351 times.
✓ Branch 3 taken 49021 times.
 50372 if this.reader.getChar() == '.' && isDigit(this.reader.peekChar()) {
407 1351 numberStr += '.';
408 1351 this.reader.advance(); // Consume '.'
409
2/2
✓ Branch 0 taken 2852 times.
✓ Branch 1 taken 1351 times.
 4203 while isDigit(this.reader.getChar()) {
410 2852 numberStr += this.reader.getChar();
411 2852 this.reader.advance();
412 2852 }
413 // Check for optional exponent
414 1351 const char exponentChar = this.reader.getChar();
415 1351 const char afterExponentChar = this.reader.peekChar();
416
2/2
✓ Branch 0 taken 219 times.
✓ Branch 1 taken 1132 times.
 1351 const bool hasExponentSign = afterExponentChar == '+' || afterExponentChar == '-';
417
8/8
✓ Branch 0 taken 10 times.
✓ Branch 1 taken 1341 times.
✓ Branch 2 taken 1341 times.
✓ Branch 3 taken 10 times.
✓ Branch 4 taken 4 times.
✓ Branch 5 taken 6 times.
✓ Branch 6 taken 1341 times.
✓ Branch 7 taken 10 times.
 1351 if (exponentChar == 'e' || exponentChar == 'E') && (isDigit(afterExponentChar) || hasExponentSign) {
418 10 numberStr += exponentChar;
419 10 this.reader.advance(); // Consume 'e' or 'E'
420
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 6 times.
 10 if hasExponentSign {
421 6 numberStr += afterExponentChar;
422 6 this.reader.advance(); // Consume sign
423 6 }
424
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 if !isDigit(this.reader.getChar()) {
425 ✗ const String message = "token recognition error at: '" + numberStr + "'";
426 ✗ this.throwLexerError(codeLoc, message.getRaw());
427 ✗ }
428
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 10 times.
 34 while isDigit(this.reader.getChar()) {
429 24 numberStr += this.reader.getChar();
430 24 this.reader.advance();
431 24 }
432 10 }
433 2702 return Token(TokenType::DOUBLE_LIT, numberStr, codeLoc);
434 }
435
436 // The correct token type is set a layer above in the parsing functions for int, short, long, etc.
437 98042 return Token(TokenType::INT_LIT, numberStr, codeLoc);
438 51680 }
439
440 9889 f<Token> Lexer.consumeCharLiteral(const CodeLoc& codeLoc) {
441 // Parse the following regex: '\'' (~['\\\r\n] | '\\' (. | EOF)) '\''
442 9889 String charStr;
443 9889 this.reader.expect('\''); // Consume '\''
444
2/2
✓ Branch 0 taken 3638 times.
✓ Branch 1 taken 6251 times.
 9889 if this.reader.getChar() == '\\' { // Escape sequence
445 // Keep the escape sequence as is. It gets resolved by the parser
446 3638 charStr += '\\';
447 3638 this.reader.advance(); // Consume '\\'
448 3638 charStr += this.reader.getChar();
449 3638 this.reader.advance(); // Consume escaped character
450 3638 } else { // Normal character
451 6251 charStr += this.reader.getChar();
452 6251 this.reader.advance(); // Consume character
453 6251 }
454 9889 this.reader.expect('\''); // Consume '\''
455 19778 return Token(TokenType::CHAR_LIT, charStr, codeLoc);
456 9889 }
457
458 17183 f<Token> Lexer.consumeStringLiteral(const CodeLoc& codeLoc) {
459 // Parse the following regex: '"' (~["\\\r\n] | '\\' (. | EOF))* '"'
460 17183 String stringStr;
461 17183 this.reader.expect('"'); // Consume '"'
462
2/2
✓ Branch 0 taken 278859 times.
✓ Branch 1 taken 17183 times.
 296042 while this.reader.getChar() != '"' {
463 // A string literal must be terminated in the same line
464
3/4
✗ Branch 0 not taken.
✓ Branch 1 taken 278859 times.
✓ Branch 2 taken 278858 times.
✓ Branch 3 taken 1 time.
 278859 if this.reader.isEOF() || this.reader.getChar() == '\n' {
465 1 const String message = "token recognition error at: '\"" + stringStr + "'";
466 1 this.throwLexerError(codeLoc, message.getRaw());
467 1 }
468
2/2
✓ Branch 0 taken 4233 times.
✓ Branch 1 taken 274626 times.
 278859 if this.reader.getChar() == '\\' { // Escape sequence
469 // Keep the escape sequence as is. It gets resolved by the parser
470 4233 stringStr += '\\';
471 4233 this.reader.advance(); // Consume '\\'
472 4233 stringStr += this.reader.getChar();
473 4233 this.reader.advance(); // Consume escaped character
474 4233 } else { // Normal character
475 274626 stringStr += this.reader.getChar();
476 274626 this.reader.advance(); // Consume character
477 274626 }
478 278859 }
479 17183 this.reader.expect('"'); // Consume '"'
480 34366 return Token(TokenType::STRING_LIT, stringStr, codeLoc);
481 17183 }
482
483 1034451 f<Token> Lexer.consumeKeywordOrIdentifier(const CodeLoc& codeLoc) {
484 1034451 String identifier;
485 1034451 do {
486 5645394 identifier += this.reader.getChar();
487 5645394 this.reader.advance();
488
4/4
✓ Branch 0 taken 4604013 times.
✓ Branch 1 taken 1041381 times.
✓ Branch 2 taken 4610943 times.
✓ Branch 3 taken 1034451 times.
 5645394 } while isAlphaNum(this.reader.getChar()) || this.reader.getChar() == '_';
489
490 // Keywords keep their text as token text, so only the token type needs to be looked up
491 2068902 return Token(this.getKeywordTokenType(identifier), identifier, codeLoc);
492 1034451 }
493
494 /**
495 * Map an identifier to its keyword token type.
496 *
497 * @param identifier Identifier text
498 * @return Token type of the keyword or IDENTIFIER, if the text is not a keyword
499 */
500 1034451 f<TokenType> Lexer.getKeywordTokenType(const String& identifier) {
501 // Dispatch on the length first, so that an identifier is only compared against the keywords of the same length
502
11/11
✓ Branch 0 taken 65896 times.
✓ Branch 1 taken 97654 times.
✓ Branch 2 taken 33550 times.
✓ Branch 3 taken 77197 times.
✓ Branch 4 taken 260082 times.
✓ Branch 5 taken 84704 times.
✓ Branch 6 taken 227811 times.
✓ Branch 7 taken 29730 times.
✓ Branch 8 taken 115544 times.
✓ Branch 9 taken 27734 times.
✓ Branch 10 taken 14549 times.
 1034451 switch identifier.getLength() {
503 case 1l: {
504
2/2
✓ Branch 0 taken 34764 times.
✓ Branch 1 taken 62890 times.
 97654 if identifier == "f" { return TokenType::F; }
505
2/2
✓ Branch 0 taken 18861 times.
✓ Branch 1 taken 44029 times.
 62890 if identifier == "p" { return TokenType::P; }
506 44029 }
507 case 2l: {
508
2/2
✓ Branch 0 taken 25175 times.
✓ Branch 1 taken 8375 times.
 33550 if identifier == "if" { return TokenType::IF; }
509
2/2
✓ Branch 0 taken 50 times.
✓ Branch 1 taken 8325 times.
 8375 if identifier == "do" { return TokenType::DO; }
510
2/2
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 8220 times.
 8325 if identifier == "as" { return TokenType::AS; }
511 8220 }
512 case 3l: {
513
2/2
✓ Branch 0 taken 18417 times.
✓ Branch 1 taken 58780 times.
 77197 if identifier == "int" { return TokenType::TYPE_INT; }
514
2/2
✓ Branch 0 taken 1633 times.
✓ Branch 1 taken 57147 times.
 58780 if identifier == "dyn" { return TokenType::TYPE_DYN; }
515
2/2
✓ Branch 0 taken 4405 times.
✓ Branch 1 taken 52742 times.
 57147 if identifier == "for" { return TokenType::FOR; }
516
2/2
✓ Branch 0 taken 8282 times.
✓ Branch 1 taken 44460 times.
 52742 if identifier == "nil" { return TokenType::NIL; }
517
2/2
✓ Branch 0 taken 5229 times.
✓ Branch 1 taken 39231 times.
 44460 if identifier == "ext" { return TokenType::EXT; }
518 39231 }
519 case 4l: {
520
2/2
✓ Branch 0 taken 34651 times.
✓ Branch 1 taken 225431 times.
 260082 if identifier == "long" { return TokenType::TYPE_LONG; }
521
2/2
✓ Branch 0 taken 12969 times.
✓ Branch 1 taken 212462 times.
 225431 if identifier == "byte" { return TokenType::TYPE_BYTE; }
522
2/2
✓ Branch 0 taken 21951 times.
✓ Branch 1 taken 190511 times.
 212462 if identifier == "char" { return TokenType::TYPE_CHAR; }
523
2/2
✓ Branch 0 taken 13663 times.
✓ Branch 1 taken 176848 times.
 190511 if identifier == "bool" { return TokenType::TYPE_BOOL; }
524
2/2
✓ Branch 0 taken 18577 times.
✓ Branch 1 taken 158271 times.
 176848 if identifier == "heap" { return TokenType::HEAP; }
525
2/2
✓ Branch 0 taken 1397 times.
✓ Branch 1 taken 156874 times.
 158271 if identifier == "else" { return TokenType::ELSE; }
526
2/2
✓ Branch 0 taken 196 times.
✓ Branch 1 taken 156678 times.
 156874 if identifier == "case" { return TokenType::CASE; }
527
2/2
✓ Branch 0 taken 7282 times.
✓ Branch 1 taken 149396 times.
 156678 if identifier == "type" { return TokenType::TYPE; }
528
2/2
✓ Branch 0 taken 466 times.
✓ Branch 1 taken 148930 times.
 149396 if identifier == "enum" { return TokenType::ENUM; }
529
2/2
✓ Branch 0 taken 1037 times.
✓ Branch 1 taken 147893 times.
 148930 if identifier == "main" { return TokenType::MAIN; }
530
2/2
✓ Branch 0 taken 12449 times.
✓ Branch 1 taken 135444 times.
 147893 if identifier == "cast" { return TokenType::CAST; }
531
2/2
✓ Branch 0 taken 3783 times.
✓ Branch 1 taken 131661 times.
 135444 if identifier == "true" { return TokenType::TRUE; }
532 131661 }
533 case 5l: {
534
2/2
✓ Branch 0 taken 3210 times.
✓ Branch 1 taken 81494 times.
 84704 if identifier == "short" { return TokenType::TYPE_SHORT; }
535
2/2
✓ Branch 0 taken 32523 times.
✓ Branch 1 taken 48971 times.
 81494 if identifier == "const" { return TokenType::CONST; }
536
2/2
✓ Branch 0 taken 1915 times.
✓ Branch 1 taken 47056 times.
 48971 if identifier == "while" { return TokenType::WHILE; }
537
2/2
✓ Branch 0 taken 741 times.
✓ Branch 1 taken 46315 times.
 47056 if identifier == "break" { return TokenType::BREAK; }
538
2/2
✓ Branch 0 taken 26 times.
✓ Branch 1 taken 46289 times.
 46315 if identifier == "union" { return TokenType::UNION; }
539
2/2
✓ Branch 0 taken 159 times.
✓ Branch 1 taken 46130 times.
 46289 if identifier == "alias" { return TokenType::ALIAS; }
540
2/2
✓ Branch 0 taken 2961 times.
✓ Branch 1 taken 43169 times.
 46130 if identifier == "false" { return TokenType::FALSE; }
541 43169 }
542 case 6l: {
543
2/2
✓ Branch 0 taken 1423 times.
✓ Branch 1 taken 226388 times.
 227811 if identifier == "double" { return TokenType::TYPE_DOUBLE; }
544
2/2
✓ Branch 0 taken 17968 times.
✓ Branch 1 taken 208420 times.
 226388 if identifier == "string" { return TokenType::TYPE_STRING; }
545
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 208412 times.
 208420 if identifier == "signed" { return TokenType::SIGNED; }
546
2/2
✓ Branch 0 taken 13026 times.
✓ Branch 1 taken 195386 times.
 208412 if identifier == "inline" { return TokenType::INLINE; }
547
2/2
✓ Branch 0 taken 46720 times.
✓ Branch 1 taken 148666 times.
 195386 if identifier == "public" { return TokenType::PUBLIC; }
548
2/2
✓ Branch 0 taken 39 times.
✓ Branch 1 taken 148627 times.
 148666 if identifier == "switch" { return TokenType::SWITCH; }
549
2/2
✓ Branch 0 taken 4080 times.
✓ Branch 1 taken 144547 times.
 148627 if identifier == "assert" { return TokenType::ASSERT; }
550
2/2
✓ Branch 0 taken 2518 times.
✓ Branch 1 taken 142029 times.
 144547 if identifier == "import" { return TokenType::IMPORT; }
551
2/2
✓ Branch 0 taken 40235 times.
✓ Branch 1 taken 101794 times.
 142029 if identifier == "return" { return TokenType::RETURN; }
552
2/2
✓ Branch 0 taken 3233 times.
✓ Branch 1 taken 98561 times.
 101794 if identifier == "struct" { return TokenType::STRUCT; }
553
2/2
✓ Branch 0 taken 12758 times.
✓ Branch 1 taken 85803 times.
 98561 if identifier == "unsafe" { return TokenType::UNSAFE; }
554 85803 }
555 case 7l: {
556
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 29711 times.
 29730 if identifier == "compose" { return TokenType::COMPOSE; }
557
2/2
✓ Branch 0 taken 20 times.
✓ Branch 1 taken 29691 times.
 29711 if identifier == "default" { return TokenType::DEFAULT; }
558
2/2
✓ Branch 0 taken 435 times.
✓ Branch 1 taken 29256 times.
 29691 if identifier == "foreach" { return TokenType::FOREACH; }
559 29256 }
560 case 8l: {
561
2/2
✓ Branch 0 taken 39721 times.
✓ Branch 1 taken 75823 times.
 115544 if identifier == "unsigned" { return TokenType::UNSIGNED; }
562
2/2
✓ Branch 0 taken 762 times.
✓ Branch 1 taken 75061 times.
 75823 if identifier == "continue" { return TokenType::CONTINUE; }
563
2/2
✓ Branch 0 taken 6274 times.
✓ Branch 1 taken 68787 times.
 75061 if identifier == "operator" { return TokenType::OPERATOR; }
564 68787 }
565 case 9l: {
566
2/2
✓ Branch 0 taken 327 times.
✓ Branch 1 taken 27407 times.
 27734 if identifier == "interface" { return TokenType::INTERFACE; }
567 27407 }
568 case 11l: {
569
2/2
✓ Branch 0 taken 16 times.
✓ Branch 1 taken 14533 times.
 14549 if identifier == "fallthrough" { return TokenType::FALLTHROUGH; }
570 14533 }
571 }
572 557992 return TokenType::IDENTIFIER;
573 }
574
575 136986 f<Token> Lexer.consumeTypeIdentifier(const CodeLoc& codeLoc) {
576 136986 String identifier;
577 136986 do {
578 954673 identifier += this.reader.getChar();
579 954673 this.reader.advance();
580
4/4
✓ Branch 0 taken 801912 times.
✓ Branch 1 taken 152761 times.
✓ Branch 2 taken 817687 times.
✓ Branch 3 taken 136986 times.
 954673 } while isAlphaNum(this.reader.getChar()) || this.reader.getChar() == '_';
581 273972 return Token(TokenType::TYPE_IDENTIFIER, identifier, codeLoc);
582 136986 }
583
584 46816 f<Token> Lexer.consumeLineComment(const CodeLoc& codeLoc) {
585 46816 this.reader.expect('/'); // Consume second '/' (first '/' is consumed by the caller)
586
587 46816 String comment = String("//");
588
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 2446993 times.
 2446996 while !this.reader.isEOF() {
589 // Check for comment end
590
2/2
✓ Branch 0 taken 46813 times.
✓ Branch 1 taken 2400180 times.
 2446993 if this.reader.getChar() == '\n' {
591 46813 this.reader.advance(); // Consume '\n'
592 46813 break;
593 }
594
595 2400180 comment += this.reader.getChar();
596 2400180 this.reader.advance();
597 2400180 }
598
599 93632 return Token(TokenType::LINE_COMMENT, comment, codeLoc);
600 46816 }
601
602 48917 f<Token> Lexer.consumeBlockOrDocComment(const CodeLoc& codeLoc) {
603 48917 this.reader.expect('*'); // Consume '*' ('/' is consumed by the caller)
604 48917 String comment = String("/*");
605
606 // Check if doc comment
607 48917 bool isDocComment = false;
608
2/2
✓ Branch 0 taken 4330 times.
✓ Branch 1 taken 44587 times.
 48917 if this.reader.getChar() == '*' {
609 44587 isDocComment = true;
610 44587 this.reader.advance(); // Consume '*'
611 44587 comment += '*';
612 44587 }
613
614
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5895993 times.
 5895993 while !this.reader.isEOF() {
615 // Check for comment end
616
2/2
✓ Branch 0 taken 5698041 times.
✓ Branch 1 taken 197952 times.
 5895993 if this.reader.getChar() == '*' {
617 197952 this.reader.advance(); // Consume '*'
618 197952 comment += '*';
619
2/2
✓ Branch 0 taken 48917 times.
✓ Branch 1 taken 149035 times.
 197952 if this.reader.getChar() == '/' {
620 48917 this.reader.advance(); // Consume '/'
621 48917 comment += '/';
622 48917 break;
623 }
624 149035 }
625
626 5847076 comment += this.reader.getChar();
627 5847076 this.reader.advance();
628 5847076 }
629
630
2/2
✓ Branch 0 taken 44587 times.
✓ Branch 1 taken 4330 times.
 48917 return Token(isDocComment ? TokenType::DOC_COMMENT : TokenType::BLOCK_COMMENT, comment, codeLoc);
631 48917 }
632