同步KDL工程源码到云仓库
This commit is contained in:
wangdequan
429
kdl-wasm/web/src/grl/lexer/lexer.ts
Normal file
429
kdl-wasm/web/src/grl/lexer/lexer.ts
Normal file
@@ -0,0 +1,429 @@
|
||||
import { isGrlKeyword } from "./keywords.js";
|
||||
import { isGrlUnitLiteral, normalizeUnitLiteral } from "./units.js";
|
||||
import type {
|
||||
GrlCommentToken,
|
||||
GrlEofToken,
|
||||
GrlIdentifierToken,
|
||||
GrlKeywordToken,
|
||||
GrlNumberToken,
|
||||
GrlOperatorToken,
|
||||
GrlPunctuationToken,
|
||||
GrlSourcePosition,
|
||||
GrlToken
|
||||
} from "./tokens.js";
|
||||
|
||||
export interface GrlLexerOptions {
|
||||
preserveComments?: boolean;
|
||||
}
|
||||
|
||||
interface ScannerState {
|
||||
index: number;
|
||||
line: number;
|
||||
column: number;
|
||||
}
|
||||
|
||||
const TWO_CHAR_OPERATORS = new Set(["==", "!=", "<=", ">=", "&&", "||", "->", ":="]);
|
||||
const PUNCTUATION = new Set(["(", ")", "{", "}", "[", "]", ",", ":", ";", "."]);
|
||||
const OPERATORS = new Set(["+", "-", "*", "/", "=", "<", ">", "!"]);
|
||||
|
||||
export function lexGrl(source: string, options: GrlLexerOptions = {}): GrlToken[] {
|
||||
const scanner = new GrlScanner(source, options);
|
||||
return scanner.scanTokens();
|
||||
}
|
||||
|
||||
class GrlScanner {
|
||||
private index = 0;
|
||||
private line = 1;
|
||||
private column = 1;
|
||||
private readonly tokens: GrlToken[] = [];
|
||||
private readonly preserveComments: boolean;
|
||||
|
||||
constructor(
|
||||
private readonly source: string,
|
||||
options: GrlLexerOptions
|
||||
) {
|
||||
this.preserveComments = options.preserveComments ?? true;
|
||||
}
|
||||
|
||||
scanTokens(): GrlToken[] {
|
||||
while (!this.isAtEnd()) {
|
||||
const char = this.peek();
|
||||
|
||||
if (this.isWhitespace(char)) {
|
||||
this.advance();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === "/" && this.peek(1) === "/") {
|
||||
this.scanLineComment();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === "/" && this.peek(1) === "*") {
|
||||
this.scanBlockComment();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (char === "\"") {
|
||||
this.scanString();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (this.isIdentifierStart(char)) {
|
||||
this.scanIdentifierOrKeyword();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (this.isNumberStart(char)) {
|
||||
this.scanNumber();
|
||||
continue;
|
||||
}
|
||||
|
||||
this.scanPunctuationOrOperator();
|
||||
}
|
||||
|
||||
const start = this.position();
|
||||
const token: GrlEofToken = {
|
||||
kind: "eof",
|
||||
raw: "",
|
||||
value: "",
|
||||
range: { start, end: start }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
return this.tokens;
|
||||
}
|
||||
|
||||
private scanLineComment(): void {
|
||||
const start = this.position();
|
||||
this.advance();
|
||||
this.advance();
|
||||
|
||||
const contentStart = this.index;
|
||||
while (!this.isAtEnd() && this.peek() !== "\n") {
|
||||
this.advance();
|
||||
}
|
||||
|
||||
if (this.preserveComments) {
|
||||
const value = this.source.slice(contentStart, this.index);
|
||||
const token: GrlCommentToken = {
|
||||
kind: "comment",
|
||||
style: "line",
|
||||
raw: this.source.slice(start.offset, this.index),
|
||||
value,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
}
|
||||
}
|
||||
|
||||
private scanBlockComment(): void {
|
||||
const start = this.position();
|
||||
this.advance();
|
||||
this.advance();
|
||||
|
||||
const contentStart = this.index;
|
||||
while (!this.isAtEnd()) {
|
||||
if (this.peek() === "*" && this.peek(1) === "/") {
|
||||
const value = this.source.slice(contentStart, this.index);
|
||||
this.advance();
|
||||
this.advance();
|
||||
|
||||
if (this.preserveComments) {
|
||||
const token: GrlCommentToken = {
|
||||
kind: "comment",
|
||||
style: "block",
|
||||
raw: this.source.slice(start.offset, this.index),
|
||||
value,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this.advance();
|
||||
}
|
||||
|
||||
throw this.error(start, "Unterminated block comment");
|
||||
}
|
||||
|
||||
private scanString(): void {
|
||||
const start = this.position();
|
||||
this.advance();
|
||||
let value = "";
|
||||
|
||||
while (!this.isAtEnd()) {
|
||||
const char = this.peek();
|
||||
if (char === "\"") {
|
||||
this.advance();
|
||||
const token: GrlToken = {
|
||||
kind: "string",
|
||||
raw: this.source.slice(start.offset, this.index),
|
||||
value,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
return;
|
||||
}
|
||||
|
||||
if (char === "\\") {
|
||||
this.advance();
|
||||
value += this.readEscapedCharacter(start);
|
||||
continue;
|
||||
}
|
||||
|
||||
value += this.advance();
|
||||
}
|
||||
|
||||
throw this.error(start, "Unterminated string literal");
|
||||
}
|
||||
|
||||
private readEscapedCharacter(start: GrlSourcePosition): string {
|
||||
if (this.isAtEnd()) {
|
||||
throw this.error(start, "Unterminated string escape");
|
||||
}
|
||||
|
||||
const escaped = this.advance();
|
||||
switch (escaped) {
|
||||
case "n":
|
||||
return "\n";
|
||||
case "r":
|
||||
return "\r";
|
||||
case "t":
|
||||
return "\t";
|
||||
case "\\":
|
||||
case "\"":
|
||||
return escaped;
|
||||
default:
|
||||
return escaped;
|
||||
}
|
||||
}
|
||||
|
||||
private scanIdentifierOrKeyword(): void {
|
||||
const start = this.position();
|
||||
|
||||
while (!this.isAtEnd() && this.isIdentifierPart(this.peek())) {
|
||||
this.advance();
|
||||
}
|
||||
|
||||
const raw = this.source.slice(start.offset, this.index);
|
||||
if (isGrlKeyword(raw)) {
|
||||
const token: GrlKeywordToken = {
|
||||
kind: "keyword",
|
||||
raw,
|
||||
value: raw,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
return;
|
||||
}
|
||||
|
||||
const token: GrlIdentifierToken = {
|
||||
kind: "identifier",
|
||||
raw,
|
||||
value: raw,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
}
|
||||
|
||||
private scanNumber(): void {
|
||||
const start = this.position();
|
||||
|
||||
if (this.peek() === ".") {
|
||||
this.advance();
|
||||
}
|
||||
|
||||
while (!this.isAtEnd() && this.isDigit(this.peek())) {
|
||||
this.advance();
|
||||
}
|
||||
|
||||
if (this.peek() === "." && this.isDigit(this.peek(1))) {
|
||||
this.advance();
|
||||
while (!this.isAtEnd() && this.isDigit(this.peek())) {
|
||||
this.advance();
|
||||
}
|
||||
}
|
||||
|
||||
if ((this.peek() === "e" || this.peek() === "E") && this.isExponentStart()) {
|
||||
this.advance();
|
||||
if (this.peek() === "+" || this.peek() === "-") {
|
||||
this.advance();
|
||||
}
|
||||
while (!this.isAtEnd() && this.isDigit(this.peek())) {
|
||||
this.advance();
|
||||
}
|
||||
}
|
||||
|
||||
const numericEnd = this.position();
|
||||
const numericRaw = this.source.slice(start.offset, numericEnd.offset);
|
||||
const value = Number(numericRaw);
|
||||
if (!Number.isFinite(value)) {
|
||||
throw this.error(start, `Invalid number literal: ${numericRaw}`);
|
||||
}
|
||||
|
||||
const unit = this.tryScanUnitAfterNumber();
|
||||
const token: GrlNumberToken = {
|
||||
kind: "number",
|
||||
raw: this.source.slice(start.offset, this.index),
|
||||
value,
|
||||
range: { start, end: this.position() },
|
||||
...(unit
|
||||
? {
|
||||
unit: {
|
||||
raw: unit.literal,
|
||||
kind: unit.kind,
|
||||
siUnit: unit.siUnit,
|
||||
normalizedValue: value * unit.factor
|
||||
}
|
||||
}
|
||||
: {})
|
||||
};
|
||||
this.tokens.push(token);
|
||||
}
|
||||
|
||||
private tryScanUnitAfterNumber(): ReturnType<typeof normalizeUnitLiteral> | undefined {
|
||||
const state = this.save();
|
||||
|
||||
while (!this.isAtEnd() && (this.peek() === " " || this.peek() === "\t")) {
|
||||
this.advance();
|
||||
}
|
||||
|
||||
const unitStart = this.index;
|
||||
|
||||
if (this.peek() === "%") {
|
||||
this.advance();
|
||||
} else {
|
||||
while (!this.isAtEnd() && /[A-Za-z0-9/^]/.test(this.peek())) {
|
||||
this.advance();
|
||||
}
|
||||
}
|
||||
|
||||
const literal = this.source.slice(unitStart, this.index);
|
||||
if (!literal || !isGrlUnitLiteral(literal)) {
|
||||
this.restore(state);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return normalizeUnitLiteral(literal);
|
||||
}
|
||||
|
||||
private scanPunctuationOrOperator(): void {
|
||||
const start = this.position();
|
||||
const two = `${this.peek()}${this.peek(1)}`;
|
||||
|
||||
if (TWO_CHAR_OPERATORS.has(two)) {
|
||||
this.advance();
|
||||
this.advance();
|
||||
const token: GrlOperatorToken = {
|
||||
kind: "operator",
|
||||
raw: two,
|
||||
value: two,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
return;
|
||||
}
|
||||
|
||||
const char = this.advance();
|
||||
if (PUNCTUATION.has(char)) {
|
||||
const token: GrlPunctuationToken = {
|
||||
kind: "punctuation",
|
||||
raw: char,
|
||||
value: char,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
return;
|
||||
}
|
||||
|
||||
if (OPERATORS.has(char)) {
|
||||
const token: GrlOperatorToken = {
|
||||
kind: "operator",
|
||||
raw: char,
|
||||
value: char,
|
||||
range: { start, end: this.position() }
|
||||
};
|
||||
this.tokens.push(token);
|
||||
return;
|
||||
}
|
||||
|
||||
throw this.error(start, `Unexpected character: ${char}`);
|
||||
}
|
||||
|
||||
private isExponentStart(): boolean {
|
||||
const next = this.peek(1);
|
||||
if (this.isDigit(next)) {
|
||||
return true;
|
||||
}
|
||||
return (next === "+" || next === "-") && this.isDigit(this.peek(2));
|
||||
}
|
||||
|
||||
private isNumberStart(char: string): boolean {
|
||||
return this.isDigit(char) || (char === "." && this.isDigit(this.peek(1)));
|
||||
}
|
||||
|
||||
private isIdentifierStart(char: string): boolean {
|
||||
return /[A-Za-z_]/.test(char);
|
||||
}
|
||||
|
||||
private isIdentifierPart(char: string): boolean {
|
||||
return /[A-Za-z0-9_]/.test(char);
|
||||
}
|
||||
|
||||
private isDigit(char: string): boolean {
|
||||
return /[0-9]/.test(char);
|
||||
}
|
||||
|
||||
private isWhitespace(char: string): boolean {
|
||||
return char === " " || char === "\t" || char === "\r" || char === "\n";
|
||||
}
|
||||
|
||||
private advance(): string {
|
||||
const char = this.source[this.index] ?? "";
|
||||
this.index += 1;
|
||||
|
||||
if (char === "\n") {
|
||||
this.line += 1;
|
||||
this.column = 1;
|
||||
} else {
|
||||
this.column += 1;
|
||||
}
|
||||
|
||||
return char;
|
||||
}
|
||||
|
||||
private peek(distance = 0): string {
|
||||
return this.source[this.index + distance] ?? "";
|
||||
}
|
||||
|
||||
private isAtEnd(): boolean {
|
||||
return this.index >= this.source.length;
|
||||
}
|
||||
|
||||
private position(): GrlSourcePosition {
|
||||
return {
|
||||
offset: this.index,
|
||||
line: this.line,
|
||||
column: this.column
|
||||
};
|
||||
}
|
||||
|
||||
private save(): ScannerState {
|
||||
return {
|
||||
index: this.index,
|
||||
line: this.line,
|
||||
column: this.column
|
||||
};
|
||||
}
|
||||
|
||||
private restore(state: ScannerState): void {
|
||||
this.index = state.index;
|
||||
this.line = state.line;
|
||||
this.column = state.column;
|
||||
}
|
||||
|
||||
private error(position: GrlSourcePosition, message: string): Error {
|
||||
return new Error(`${message} at ${position.line}:${position.column}`);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user