import { isGrlKeyword } from "./keywords.js"; import { isGrlUnitLiteral, normalizeUnitLiteral } from "./units.js"; import type { GrlCommentToken, GrlEofToken, GrlIdentifierToken, GrlKeywordToken, GrlNumberToken, GrlOperatorToken, GrlPunctuationToken, GrlSourcePosition, GrlToken } from "./tokens.js"; export interface GrlLexerOptions { preserveComments?: boolean; } interface ScannerState { index: number; line: number; column: number; } const TWO_CHAR_OPERATORS = new Set(["==", "!=", "<=", ">=", "&&", "||", "->", ":="]); const PUNCTUATION = new Set(["(", ")", "{", "}", "[", "]", ",", ":", ";", "."]); const OPERATORS = new Set(["+", "-", "*", "/", "=", "<", ">", "!"]); export function lexGrl(source: string, options: GrlLexerOptions = {}): GrlToken[] { const scanner = new GrlScanner(source, options); return scanner.scanTokens(); } class GrlScanner { private index = 0; private line = 1; private column = 1; private readonly tokens: GrlToken[] = []; private readonly preserveComments: boolean; constructor( private readonly source: string, options: GrlLexerOptions ) { this.preserveComments = options.preserveComments ?? true; } scanTokens(): GrlToken[] { while (!this.isAtEnd()) { const char = this.peek(); if (this.isWhitespace(char)) { this.advance(); continue; } if (char === "/" && this.peek(1) === "/") { this.scanLineComment(); continue; } if (char === "/" && this.peek(1) === "*") { this.scanBlockComment(); continue; } if (char === "\"") { this.scanString(); continue; } if (this.isIdentifierStart(char)) { this.scanIdentifierOrKeyword(); continue; } if (this.isNumberStart(char)) { this.scanNumber(); continue; } this.scanPunctuationOrOperator(); } const start = this.position(); const token: GrlEofToken = { kind: "eof", raw: "", value: "", range: { start, end: start } }; this.tokens.push(token); return this.tokens; } private scanLineComment(): void { const start = this.position(); this.advance(); this.advance(); const contentStart = this.index; while (!this.isAtEnd() && this.peek() !== "\n") { this.advance(); } if (this.preserveComments) { const value = this.source.slice(contentStart, this.index); const token: GrlCommentToken = { kind: "comment", style: "line", raw: this.source.slice(start.offset, this.index), value, range: { start, end: this.position() } }; this.tokens.push(token); } } private scanBlockComment(): void { const start = this.position(); this.advance(); this.advance(); const contentStart = this.index; while (!this.isAtEnd()) { if (this.peek() === "*" && this.peek(1) === "/") { const value = this.source.slice(contentStart, this.index); this.advance(); this.advance(); if (this.preserveComments) { const token: GrlCommentToken = { kind: "comment", style: "block", raw: this.source.slice(start.offset, this.index), value, range: { start, end: this.position() } }; this.tokens.push(token); } return; } this.advance(); } throw this.error(start, "Unterminated block comment"); } private scanString(): void { const start = this.position(); this.advance(); let value = ""; while (!this.isAtEnd()) { const char = this.peek(); if (char === "\"") { this.advance(); const token: GrlToken = { kind: "string", raw: this.source.slice(start.offset, this.index), value, range: { start, end: this.position() } }; this.tokens.push(token); return; } if (char === "\\") { this.advance(); value += this.readEscapedCharacter(start); continue; } value += this.advance(); } throw this.error(start, "Unterminated string literal"); } private readEscapedCharacter(start: GrlSourcePosition): string { if (this.isAtEnd()) { throw this.error(start, "Unterminated string escape"); } const escaped = this.advance(); switch (escaped) { case "n": return "\n"; case "r": return "\r"; case "t": return "\t"; case "\\": case "\"": return escaped; default: return escaped; } } private scanIdentifierOrKeyword(): void { const start = this.position(); while (!this.isAtEnd() && this.isIdentifierPart(this.peek())) { this.advance(); } const raw = this.source.slice(start.offset, this.index); if (isGrlKeyword(raw)) { const token: GrlKeywordToken = { kind: "keyword", raw, value: raw, range: { start, end: this.position() } }; this.tokens.push(token); return; } const token: GrlIdentifierToken = { kind: "identifier", raw, value: raw, range: { start, end: this.position() } }; this.tokens.push(token); } private scanNumber(): void { const start = this.position(); if (this.peek() === ".") { this.advance(); } while (!this.isAtEnd() && this.isDigit(this.peek())) { this.advance(); } if (this.peek() === "." && this.isDigit(this.peek(1))) { this.advance(); while (!this.isAtEnd() && this.isDigit(this.peek())) { this.advance(); } } if ((this.peek() === "e" || this.peek() === "E") && this.isExponentStart()) { this.advance(); if (this.peek() === "+" || this.peek() === "-") { this.advance(); } while (!this.isAtEnd() && this.isDigit(this.peek())) { this.advance(); } } const numericEnd = this.position(); const numericRaw = this.source.slice(start.offset, numericEnd.offset); const value = Number(numericRaw); if (!Number.isFinite(value)) { throw this.error(start, `Invalid number literal: ${numericRaw}`); } const unit = this.tryScanUnitAfterNumber(); const token: GrlNumberToken = { kind: "number", raw: this.source.slice(start.offset, this.index), value, range: { start, end: this.position() }, ...(unit ? { unit: { raw: unit.literal, kind: unit.kind, siUnit: unit.siUnit, normalizedValue: value * unit.factor } } : {}) }; this.tokens.push(token); } private tryScanUnitAfterNumber(): ReturnType | undefined { const state = this.save(); while (!this.isAtEnd() && (this.peek() === " " || this.peek() === "\t")) { this.advance(); } const unitStart = this.index; if (this.peek() === "%") { this.advance(); } else { while (!this.isAtEnd() && /[A-Za-z0-9/^]/.test(this.peek())) { this.advance(); } } const literal = this.source.slice(unitStart, this.index); if (!literal || !isGrlUnitLiteral(literal)) { this.restore(state); return undefined; } return normalizeUnitLiteral(literal); } private scanPunctuationOrOperator(): void { const start = this.position(); const two = `${this.peek()}${this.peek(1)}`; if (TWO_CHAR_OPERATORS.has(two)) { this.advance(); this.advance(); const token: GrlOperatorToken = { kind: "operator", raw: two, value: two, range: { start, end: this.position() } }; this.tokens.push(token); return; } const char = this.advance(); if (PUNCTUATION.has(char)) { const token: GrlPunctuationToken = { kind: "punctuation", raw: char, value: char, range: { start, end: this.position() } }; this.tokens.push(token); return; } if (OPERATORS.has(char)) { const token: GrlOperatorToken = { kind: "operator", raw: char, value: char, range: { start, end: this.position() } }; this.tokens.push(token); return; } throw this.error(start, `Unexpected character: ${char}`); } private isExponentStart(): boolean { const next = this.peek(1); if (this.isDigit(next)) { return true; } return (next === "+" || next === "-") && this.isDigit(this.peek(2)); } private isNumberStart(char: string): boolean { return this.isDigit(char) || (char === "." && this.isDigit(this.peek(1))); } private isIdentifierStart(char: string): boolean { return /[A-Za-z_]/.test(char); } private isIdentifierPart(char: string): boolean { return /[A-Za-z0-9_]/.test(char); } private isDigit(char: string): boolean { return /[0-9]/.test(char); } private isWhitespace(char: string): boolean { return char === " " || char === "\t" || char === "\r" || char === "\n"; } private advance(): string { const char = this.source[this.index] ?? ""; this.index += 1; if (char === "\n") { this.line += 1; this.column = 1; } else { this.column += 1; } return char; } private peek(distance = 0): string { return this.source[this.index + distance] ?? ""; } private isAtEnd(): boolean { return this.index >= this.source.length; } private position(): GrlSourcePosition { return { offset: this.index, line: this.line, column: this.column }; } private save(): ScannerState { return { index: this.index, line: this.line, column: this.column }; } private restore(state: ScannerState): void { this.index = state.index; this.line = state.line; this.column = state.column; } private error(position: GrlSourcePosition, message: string): Error { return new Error(`${message} at ${position.line}:${position.column}`); } }