bolt/src/scanner.ts

392 lines
8.5 KiB
TypeScript
Raw Normal View History

2020-02-24 18:30:39 +01:00
import XRegExp from "xregexp"
import {
TextFile,
TextPos,
TextSpan,
2020-05-10 15:56:34 +02:00
} from "./text"
import {
SyntaxKind,
BoltToken,
BoltSentence,
createBoltSentence,
createBoltIdentifier,
createBoltRArrow,
createBoltOperator,
createBoltParenthesized,
createBoltBraced,
createBoltBracketed,
createBoltSourceFile,
createBoltSemi,
createBoltComma,
createBoltStringLiteral,
createBoltIntegerLiteral,
createBoltColon,
createBoltEOS,
createBoltDot,
createBoltEqSign,
2020-02-24 18:30:39 +01:00
} from "./ast"
2020-05-10 15:56:34 +02:00
export enum PunctType {
Paren,
Bracket,
Brace,
}
2020-02-24 18:30:39 +01:00
function escapeChar(ch: string) {
switch (ch) {
case '\a': return '\\a';
case '\b': return '\\b';
case '\f': return '\\f';
case '\n': return '\\n';
case '\r': return '\\r';
case '\t': return '\\t';
case '\v': return '\\v';
case '\0': return '\\0';
case '\'': return '\\\'';
default:
const code = ch.charCodeAt(0);
if (code >= 0x20 && code <= 0x7E) {
return ch
} else if (code < 0x7F) {
return `\\x${code.toString(16).padStart(2, '0')}`
} else {
return `\\u${code.toString(16).padStart(4, '0')}`
}
}
}
function getPunctType(ch: string) {
switch (ch) {
case '(':
case ')':
return PunctType.Paren;
case '[':
case ']':
return PunctType.Bracket;
case '{':
case '}':
return PunctType.Brace;
default:
2020-02-25 17:55:17 +01:00
return null;
2020-02-24 18:30:39 +01:00
}
}
function isClosePunct(ch: string) {
switch (ch) {
case '}':
case ']':
case ')':
return true;
default:
return false;
}
}
function isOpenPunct(ch: string) {
switch (ch) {
case '{':
case '(':
case '[':
return true;
default:
return false;
}
}
class ScanError extends Error {
constructor(public file: TextFile, public position: TextPos, public char: string) {
2020-05-10 15:56:34 +02:00
super(`${file.origPath}:${position.line}:${position.column}: unexpected char '${escapeChar(char)}'`)
2020-02-24 18:30:39 +01:00
}
}
function isDigit(ch: string) {
return XRegExp('\\p{Nd}').test(ch)
}
2020-02-24 18:30:39 +01:00
function isWhiteSpace(ch: string) {
return ch == '\n' || XRegExp('\\p{Zs}').test(ch)
}
function isNewLine(ch: string) {
return ch == '\n'
}
function isIdentStart(ch: string) {
return ch == '_' || XRegExp('\\p{L}').test(ch)
}
function isIdentPart(ch: string) {
return ch == '_' || XRegExp('\\p{L}').test(ch)
}
function isOperatorStart(ch: string) {
return /[+\-*\/%$!><]/.test(ch)
}
function isOperatorPart(ch: string) {
return /[=+\-*\/%$!><]/.test(ch)
}
const EOF = ''
export class Scanner {
protected buffer: string[] = [];
2020-05-10 15:56:34 +02:00
protected scanned: BoltToken[] = [];
2020-02-25 17:55:17 +01:00
protected currPos: TextPos;
2020-02-24 18:30:39 +01:00
protected offset = 0;
2020-02-25 17:55:17 +01:00
constructor(public file: TextFile, public input: string, startPos = new TextPos(0,1,1)) {
this.currPos = startPos;
2020-02-24 18:30:39 +01:00
}
protected readChar() {
if (this.offset == this.input.length) {
return EOF
}
return this.input[this.offset++]
}
protected peekChar(count = 1) {
while (this.buffer.length < count) {
this.buffer.push(this.readChar());
}
return this.buffer[count - 1];
}
protected getChar() {
const ch = this.buffer.length > 0
? this.buffer.shift()!
: this.readChar()
if (ch == EOF) {
return EOF
}
if (isNewLine(ch)) {
this.currPos.line += 1;
this.currPos.column = 1;
} else {
this.currPos.column += 1;
}
this.currPos.offset += 1;
return ch
}
protected takeWhile(pred: (ch: string) => boolean) {
let text = this.getChar();
while (true) {
const c0 = this.peekChar();
if (!pred(c0)) {
break;
}
this.getChar()
text += c0;
}
return text;
}
2020-05-10 15:56:34 +02:00
scanToken(): BoltToken {
2020-02-24 18:30:39 +01:00
while (true) {
const c0 = this.peekChar();
if (isWhiteSpace(c0)) {
this.getChar();
continue;
}
const startPos = this.currPos.clone()
2020-02-25 17:55:17 +01:00
if (c0 == EOF) {
2020-05-10 15:56:34 +02:00
return createBoltEOS(new TextSpan(this.file, startPos, startPos));
2020-02-25 17:55:17 +01:00
}
switch (c0) {
case '.':
this.getChar();
2020-05-10 15:56:34 +02:00
return createBoltDot(new TextSpan(this.file, startPos, this.currPos.clone()));
case '=':
this.getChar();
2020-05-10 15:56:34 +02:00
return createBoltEqSign(new TextSpan(this.file, startPos, this.currPos.clone()));
case ';':
this.getChar();
2020-05-10 15:56:34 +02:00
return createBoltSemi(new TextSpan(this.file, startPos, this.currPos.clone()));
case ',':
this.getChar();
2020-05-10 15:56:34 +02:00
return createBoltComma(new TextSpan(this.file, startPos, this.currPos.clone()));
case ':':
this.getChar();
2020-05-10 15:56:34 +02:00
return createBoltColon(new TextSpan(this.file, startPos, this.currPos.clone()));
}
2020-02-25 17:55:17 +01:00
if (c0 === '"') {
2020-02-24 18:30:39 +01:00
this.getChar();
2020-02-25 17:55:17 +01:00
let text = ''
2020-02-24 18:30:39 +01:00
while (true) {
2020-02-25 17:55:17 +01:00
const c1 = this.getChar();
if (c1 === EOF) {
throw new ScanError(this.file, this.currPos.clone(), EOF);
}
if (c1 === '"') {
break;
} else if (c1 === '\\') {
this.scanEscapeSequence()
} else {
text += c1
}
}
2020-02-24 18:30:39 +01:00
2020-02-25 17:55:17 +01:00
const endPos = this.currPos.clone();
2020-02-24 18:30:39 +01:00
2020-05-10 15:56:34 +02:00
return createBoltStringLiteral(text, new TextSpan(this.file, startPos, endPos))
2020-02-25 17:55:17 +01:00
} else if (isDigit(c0)) {
const digits = this.takeWhile(isDigit)
const endPos = this.currPos.clone();
2020-05-10 15:56:34 +02:00
return createBoltIntegerLiteral(BigInt(digits), new TextSpan(this.file, startPos, endPos));
2020-02-25 17:55:17 +01:00
} else if (isOpenPunct(c0)) {
this.getChar();
const punctType = getPunctType(c0);
let punctCount = 1;
let text = ''
while (true) {
const c1 = this.getChar();
2020-02-24 18:35:07 +01:00
2020-02-24 18:30:39 +01:00
if (c1 === EOF) {
throw new ScanError(this.file, this.currPos.clone(), EOF)
}
2020-02-25 17:55:17 +01:00
if (punctType == getPunctType(c1)) {
if (isClosePunct(c1)) {
punctCount--;
if (punctCount === 0)
break;
2020-02-24 18:30:39 +01:00
} else {
2020-02-25 17:55:17 +01:00
punctCount++;
2020-02-24 18:30:39 +01:00
}
}
2020-02-25 17:55:17 +01:00
text += c1
2020-02-24 18:30:39 +01:00
}
const endPos = this.currPos.clone();
switch (punctType) {
case PunctType.Brace:
2020-05-10 15:56:34 +02:00
return createBoltBraced(text, new TextSpan(this.file, startPos, endPos));
case PunctType.Paren:
2020-05-10 15:56:34 +02:00
return createBoltParenthesized(text, new TextSpan(this.file, startPos, endPos));
case PunctType.Bracket:
2020-05-10 15:56:34 +02:00
return createBoltBracketed(text, new TextSpan(this.file, startPos, endPos));
default:
throw new Error("Got an invalid state.")
}
2020-02-24 18:30:39 +01:00
} else if (isIdentStart(c0)) {
const name = this.takeWhile(isIdentPart);
const endPos = this.currPos.clone();
2020-05-10 15:56:34 +02:00
return createBoltIdentifier(name, new TextSpan(this.file, startPos, endPos))
2020-02-24 18:30:39 +01:00
} else if (isOperatorStart(c0)) {
const text = this.takeWhile(isOperatorPart)
const endPos = this.currPos.clone()
const span = new TextSpan(this.file, startPos, endPos);
if (text === '->') {
2020-05-10 15:56:34 +02:00
return createBoltRArrow(span);
} else {
2020-05-10 15:56:34 +02:00
return createBoltOperator(text, span);
}
2020-02-24 18:30:39 +01:00
} else {
throw new ScanError(this.file, this.currPos.clone(), c0);
}
}
}
2020-05-10 15:56:34 +02:00
peek(count = 1): BoltToken {
2020-02-25 17:55:17 +01:00
while (this.scanned.length < count) {
this.scanned.push(this.scanToken());
}
return this.scanned[count - 1];
}
2020-05-10 15:56:34 +02:00
get(): BoltToken {
2020-02-25 17:55:17 +01:00
return this.scanned.length > 0
? this.scanned.shift()!
: this.scanToken();
}
scanTokens() {
2020-02-24 19:16:33 +01:00
2020-05-10 15:56:34 +02:00
const elements: BoltSentence[] = []
2020-02-24 19:16:33 +01:00
outer: while (true) {
2020-02-24 19:16:33 +01:00
2020-05-10 15:56:34 +02:00
const tokens: BoltToken[] = [];
2020-02-24 19:16:33 +01:00
inner: while (true) {
2020-02-24 19:16:33 +01:00
const token = this.scanToken();
2020-05-10 15:56:34 +02:00
if (token.kind === SyntaxKind.BoltEOS) {
if (tokens.length === 0) {
break outer;
} else {
break inner;
}
2020-02-24 19:16:33 +01:00
}
2020-05-10 15:56:34 +02:00
if (token.kind === SyntaxKind.BoltSemi) {
2020-02-24 19:16:33 +01:00
break;
}
tokens.push(token)
2020-05-10 15:56:34 +02:00
if (token.kind === SyntaxKind.BoltBraced) {
2020-02-24 19:16:33 +01:00
break;
}
}
if (tokens.length > 0) {
elements.push(
2020-05-10 15:56:34 +02:00
createBoltSentence(
tokens,
new TextSpan(this.file, tokens[0].span!.start.clone(), tokens[tokens.length-1].span!.end.clone())
)
)
2020-02-24 19:16:33 +01:00
}
}
return elements
}
2020-02-24 18:30:39 +01:00
scan() {
const startPos = this.currPos.clone();
const elements = this.scanTokens();
const endPos = this.currPos.clone();
2020-05-10 15:56:34 +02:00
return createBoltSourceFile(elements, new TextSpan(this.file, startPos, endPos));
2020-02-24 18:30:39 +01:00
}
}