430 lines
9.9 KiB
TypeScript
430 lines
9.9 KiB
TypeScript
import { isGrlKeyword } from "./keywords.js";
|
|
import { isGrlUnitLiteral, normalizeUnitLiteral } from "./units.js";
|
|
import type {
|
|
GrlCommentToken,
|
|
GrlEofToken,
|
|
GrlIdentifierToken,
|
|
GrlKeywordToken,
|
|
GrlNumberToken,
|
|
GrlOperatorToken,
|
|
GrlPunctuationToken,
|
|
GrlSourcePosition,
|
|
GrlToken
|
|
} from "./tokens.js";
|
|
|
|
export interface GrlLexerOptions {
|
|
preserveComments?: boolean;
|
|
}
|
|
|
|
interface ScannerState {
|
|
index: number;
|
|
line: number;
|
|
column: number;
|
|
}
|
|
|
|
const TWO_CHAR_OPERATORS = new Set(["==", "!=", "<=", ">=", "&&", "||", "->", ":="]);
|
|
const PUNCTUATION = new Set(["(", ")", "{", "}", "[", "]", ",", ":", ";", "."]);
|
|
const OPERATORS = new Set(["+", "-", "*", "/", "=", "<", ">", "!"]);
|
|
|
|
export function lexGrl(source: string, options: GrlLexerOptions = {}): GrlToken[] {
|
|
const scanner = new GrlScanner(source, options);
|
|
return scanner.scanTokens();
|
|
}
|
|
|
|
class GrlScanner {
|
|
private index = 0;
|
|
private line = 1;
|
|
private column = 1;
|
|
private readonly tokens: GrlToken[] = [];
|
|
private readonly preserveComments: boolean;
|
|
|
|
constructor(
|
|
private readonly source: string,
|
|
options: GrlLexerOptions
|
|
) {
|
|
this.preserveComments = options.preserveComments ?? true;
|
|
}
|
|
|
|
scanTokens(): GrlToken[] {
|
|
while (!this.isAtEnd()) {
|
|
const char = this.peek();
|
|
|
|
if (this.isWhitespace(char)) {
|
|
this.advance();
|
|
continue;
|
|
}
|
|
|
|
if (char === "/" && this.peek(1) === "/") {
|
|
this.scanLineComment();
|
|
continue;
|
|
}
|
|
|
|
if (char === "/" && this.peek(1) === "*") {
|
|
this.scanBlockComment();
|
|
continue;
|
|
}
|
|
|
|
if (char === "\"") {
|
|
this.scanString();
|
|
continue;
|
|
}
|
|
|
|
if (this.isIdentifierStart(char)) {
|
|
this.scanIdentifierOrKeyword();
|
|
continue;
|
|
}
|
|
|
|
if (this.isNumberStart(char)) {
|
|
this.scanNumber();
|
|
continue;
|
|
}
|
|
|
|
this.scanPunctuationOrOperator();
|
|
}
|
|
|
|
const start = this.position();
|
|
const token: GrlEofToken = {
|
|
kind: "eof",
|
|
raw: "",
|
|
value: "",
|
|
range: { start, end: start }
|
|
};
|
|
this.tokens.push(token);
|
|
return this.tokens;
|
|
}
|
|
|
|
private scanLineComment(): void {
|
|
const start = this.position();
|
|
this.advance();
|
|
this.advance();
|
|
|
|
const contentStart = this.index;
|
|
while (!this.isAtEnd() && this.peek() !== "\n") {
|
|
this.advance();
|
|
}
|
|
|
|
if (this.preserveComments) {
|
|
const value = this.source.slice(contentStart, this.index);
|
|
const token: GrlCommentToken = {
|
|
kind: "comment",
|
|
style: "line",
|
|
raw: this.source.slice(start.offset, this.index),
|
|
value,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
}
|
|
}
|
|
|
|
private scanBlockComment(): void {
|
|
const start = this.position();
|
|
this.advance();
|
|
this.advance();
|
|
|
|
const contentStart = this.index;
|
|
while (!this.isAtEnd()) {
|
|
if (this.peek() === "*" && this.peek(1) === "/") {
|
|
const value = this.source.slice(contentStart, this.index);
|
|
this.advance();
|
|
this.advance();
|
|
|
|
if (this.preserveComments) {
|
|
const token: GrlCommentToken = {
|
|
kind: "comment",
|
|
style: "block",
|
|
raw: this.source.slice(start.offset, this.index),
|
|
value,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
}
|
|
return;
|
|
}
|
|
|
|
this.advance();
|
|
}
|
|
|
|
throw this.error(start, "Unterminated block comment");
|
|
}
|
|
|
|
private scanString(): void {
|
|
const start = this.position();
|
|
this.advance();
|
|
let value = "";
|
|
|
|
while (!this.isAtEnd()) {
|
|
const char = this.peek();
|
|
if (char === "\"") {
|
|
this.advance();
|
|
const token: GrlToken = {
|
|
kind: "string",
|
|
raw: this.source.slice(start.offset, this.index),
|
|
value,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
return;
|
|
}
|
|
|
|
if (char === "\\") {
|
|
this.advance();
|
|
value += this.readEscapedCharacter(start);
|
|
continue;
|
|
}
|
|
|
|
value += this.advance();
|
|
}
|
|
|
|
throw this.error(start, "Unterminated string literal");
|
|
}
|
|
|
|
private readEscapedCharacter(start: GrlSourcePosition): string {
|
|
if (this.isAtEnd()) {
|
|
throw this.error(start, "Unterminated string escape");
|
|
}
|
|
|
|
const escaped = this.advance();
|
|
switch (escaped) {
|
|
case "n":
|
|
return "\n";
|
|
case "r":
|
|
return "\r";
|
|
case "t":
|
|
return "\t";
|
|
case "\\":
|
|
case "\"":
|
|
return escaped;
|
|
default:
|
|
return escaped;
|
|
}
|
|
}
|
|
|
|
private scanIdentifierOrKeyword(): void {
|
|
const start = this.position();
|
|
|
|
while (!this.isAtEnd() && this.isIdentifierPart(this.peek())) {
|
|
this.advance();
|
|
}
|
|
|
|
const raw = this.source.slice(start.offset, this.index);
|
|
if (isGrlKeyword(raw)) {
|
|
const token: GrlKeywordToken = {
|
|
kind: "keyword",
|
|
raw,
|
|
value: raw,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
return;
|
|
}
|
|
|
|
const token: GrlIdentifierToken = {
|
|
kind: "identifier",
|
|
raw,
|
|
value: raw,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
}
|
|
|
|
private scanNumber(): void {
|
|
const start = this.position();
|
|
|
|
if (this.peek() === ".") {
|
|
this.advance();
|
|
}
|
|
|
|
while (!this.isAtEnd() && this.isDigit(this.peek())) {
|
|
this.advance();
|
|
}
|
|
|
|
if (this.peek() === "." && this.isDigit(this.peek(1))) {
|
|
this.advance();
|
|
while (!this.isAtEnd() && this.isDigit(this.peek())) {
|
|
this.advance();
|
|
}
|
|
}
|
|
|
|
if ((this.peek() === "e" || this.peek() === "E") && this.isExponentStart()) {
|
|
this.advance();
|
|
if (this.peek() === "+" || this.peek() === "-") {
|
|
this.advance();
|
|
}
|
|
while (!this.isAtEnd() && this.isDigit(this.peek())) {
|
|
this.advance();
|
|
}
|
|
}
|
|
|
|
const numericEnd = this.position();
|
|
const numericRaw = this.source.slice(start.offset, numericEnd.offset);
|
|
const value = Number(numericRaw);
|
|
if (!Number.isFinite(value)) {
|
|
throw this.error(start, `Invalid number literal: ${numericRaw}`);
|
|
}
|
|
|
|
const unit = this.tryScanUnitAfterNumber();
|
|
const token: GrlNumberToken = {
|
|
kind: "number",
|
|
raw: this.source.slice(start.offset, this.index),
|
|
value,
|
|
range: { start, end: this.position() },
|
|
...(unit
|
|
? {
|
|
unit: {
|
|
raw: unit.literal,
|
|
kind: unit.kind,
|
|
siUnit: unit.siUnit,
|
|
normalizedValue: value * unit.factor
|
|
}
|
|
}
|
|
: {})
|
|
};
|
|
this.tokens.push(token);
|
|
}
|
|
|
|
private tryScanUnitAfterNumber(): ReturnType<typeof normalizeUnitLiteral> | undefined {
|
|
const state = this.save();
|
|
|
|
while (!this.isAtEnd() && (this.peek() === " " || this.peek() === "\t")) {
|
|
this.advance();
|
|
}
|
|
|
|
const unitStart = this.index;
|
|
|
|
if (this.peek() === "%") {
|
|
this.advance();
|
|
} else {
|
|
while (!this.isAtEnd() && /[A-Za-z0-9/^]/.test(this.peek())) {
|
|
this.advance();
|
|
}
|
|
}
|
|
|
|
const literal = this.source.slice(unitStart, this.index);
|
|
if (!literal || !isGrlUnitLiteral(literal)) {
|
|
this.restore(state);
|
|
return undefined;
|
|
}
|
|
|
|
return normalizeUnitLiteral(literal);
|
|
}
|
|
|
|
private scanPunctuationOrOperator(): void {
|
|
const start = this.position();
|
|
const two = `${this.peek()}${this.peek(1)}`;
|
|
|
|
if (TWO_CHAR_OPERATORS.has(two)) {
|
|
this.advance();
|
|
this.advance();
|
|
const token: GrlOperatorToken = {
|
|
kind: "operator",
|
|
raw: two,
|
|
value: two,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
return;
|
|
}
|
|
|
|
const char = this.advance();
|
|
if (PUNCTUATION.has(char)) {
|
|
const token: GrlPunctuationToken = {
|
|
kind: "punctuation",
|
|
raw: char,
|
|
value: char,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
return;
|
|
}
|
|
|
|
if (OPERATORS.has(char)) {
|
|
const token: GrlOperatorToken = {
|
|
kind: "operator",
|
|
raw: char,
|
|
value: char,
|
|
range: { start, end: this.position() }
|
|
};
|
|
this.tokens.push(token);
|
|
return;
|
|
}
|
|
|
|
throw this.error(start, `Unexpected character: ${char}`);
|
|
}
|
|
|
|
private isExponentStart(): boolean {
|
|
const next = this.peek(1);
|
|
if (this.isDigit(next)) {
|
|
return true;
|
|
}
|
|
return (next === "+" || next === "-") && this.isDigit(this.peek(2));
|
|
}
|
|
|
|
private isNumberStart(char: string): boolean {
|
|
return this.isDigit(char) || (char === "." && this.isDigit(this.peek(1)));
|
|
}
|
|
|
|
private isIdentifierStart(char: string): boolean {
|
|
return /[A-Za-z_]/.test(char);
|
|
}
|
|
|
|
private isIdentifierPart(char: string): boolean {
|
|
return /[A-Za-z0-9_]/.test(char);
|
|
}
|
|
|
|
private isDigit(char: string): boolean {
|
|
return /[0-9]/.test(char);
|
|
}
|
|
|
|
private isWhitespace(char: string): boolean {
|
|
return char === " " || char === "\t" || char === "\r" || char === "\n";
|
|
}
|
|
|
|
private advance(): string {
|
|
const char = this.source[this.index] ?? "";
|
|
this.index += 1;
|
|
|
|
if (char === "\n") {
|
|
this.line += 1;
|
|
this.column = 1;
|
|
} else {
|
|
this.column += 1;
|
|
}
|
|
|
|
return char;
|
|
}
|
|
|
|
private peek(distance = 0): string {
|
|
return this.source[this.index + distance] ?? "";
|
|
}
|
|
|
|
private isAtEnd(): boolean {
|
|
return this.index >= this.source.length;
|
|
}
|
|
|
|
private position(): GrlSourcePosition {
|
|
return {
|
|
offset: this.index,
|
|
line: this.line,
|
|
column: this.column
|
|
};
|
|
}
|
|
|
|
private save(): ScannerState {
|
|
return {
|
|
index: this.index,
|
|
line: this.line,
|
|
column: this.column
|
|
};
|
|
}
|
|
|
|
private restore(state: ScannerState): void {
|
|
this.index = state.index;
|
|
this.line = state.line;
|
|
this.column = state.column;
|
|
}
|
|
|
|
private error(position: GrlSourcePosition, message: string): Error {
|
|
return new Error(`${message} at ${position.line}:${position.column}`);
|
|
}
|
|
}
|