samvv/bolt
1
0
Fork 0
Code Issues 24 Pull requests Projects Releases Packages Wiki Activity Actions

Extend JavaScript scanner and parser

Browse source
This commit is contained in:
Sam Vervaeck 2020-05-22 23:04:40 +02:00
parent 5189804c00
commit 02948613a9
5 changed files with 515 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
spec/ast.txt
View file

@ -290,9 +290,22 @@ node JSIdentifier > JSToken {
text: String,
}
node JSString > JSToken {
value: String,
}
node JSFromKeyword > JSToken;
node JSReturnKeyword > JSToken;
node JSTryKeyword > JSToken;
node JSCatchKeyword > JSToken;
node JSImportKeyword > JSToken;
node JSAsKeyword > JSToken;
node JSConstKeyword > JSToken;
node JSLetKeyword > JSToken;
node JSExportKeyword > JSToken;
node JSFunctionKeyword > JSToken;
node JSWhileKeyword > JSToken;
node JSForKeyword > JSToken;
node JSCloseBrace > JSToken;
node JSCloseBracket > JSToken;
@ -304,6 +317,17 @@ node JSSemi > JSToken;
node JSComma > JSToken;
node JSDot > JSToken;
node JSDotDotDot > JSToken;
node JSMulOp > JSToken;
node JSAddOp > JSToken;
node JSDivOp > JSToken;
node JSSubOp > JSToken;
node JSLtOp > JSToken;
node JSGtOp > JSToken;
node JSBOrOp > JSToken;
node JSBXorOp > JSToken;
node JSBAndOp > JSToken;
node JSBNotOp > JSToken;
node JSNotOp > JSToken;
node JSPattern;
@ -383,6 +407,22 @@ enum JSDeclarationModifiers {
IsExported = 0x1,
}
node JSImportBinding;
node JSImportStarBinding > JSImportBinding {
local: JSIdentifier,
}
node JSImportAsBinding > JSImportBinding {
remote: JSIdentifier,
local: Option<JSIdentifier>,
}
node JSImportDeclaration > JSDeclaration {
bindings: Vec<JSImportBinding>,
filename: JSString,
}
node JSFunctionDeclaration > JSDeclaration {
modifiers: JSDeclarationModifiers,
name: JSIdentifier,

313
src/ast.d.ts vendored
View file

@ -74,36 +74,60 @@ export const enum SyntaxKind {
BoltRecordDeclaration = 85,
JSOperator = 89,
JSIdentifier = 90,
JSReturnKeyword = 91,
JSTryKeyword = 92,
JSCatchKeyword = 93,
JSCloseBrace = 94,
JSCloseBracket = 95,
JSCloseParen = 96,
JSOpenBrace = 97,
JSOpenBracket = 98,
JSOpenParen = 99,
JSSemi = 100,
JSComma = 101,
JSDot = 102,
JSDotDotDot = 103,
JSBindPattern = 105,
JSConstantExpression = 107,
JSMemberExpression = 108,
JSCallExpression = 109,
JSBinaryExpression = 110,
JSUnaryExpression = 111,
JSNewExpression = 112,
JSSequenceExpression = 113,
JSConditionalExpression = 114,
JSReferenceExpression = 115,
JSExpressionStatement = 118,
JSConditionalStatement = 119,
JSParameter = 120,
JSFunctionDeclaration = 123,
JSArrowFunctionDeclaration = 124,
JSLetDeclaration = 125,
JSSourceFile = 126,
JSString = 91,
JSFromKeyword = 92,
JSReturnKeyword = 93,
JSTryKeyword = 94,
JSCatchKeyword = 95,
JSImportKeyword = 96,
JSAsKeyword = 97,
JSConstKeyword = 98,
JSLetKeyword = 99,
JSExportKeyword = 100,
JSFunctionKeyword = 101,
JSWhileKeyword = 102,
JSForKeyword = 103,
JSCloseBrace = 104,
JSCloseBracket = 105,
JSCloseParen = 106,
JSOpenBrace = 107,
JSOpenBracket = 108,
JSOpenParen = 109,
JSSemi = 110,
JSComma = 111,
JSDot = 112,
JSDotDotDot = 113,
JSMulOp = 114,
JSAddOp = 115,
JSDivOp = 116,
JSSubOp = 117,
JSLtOp = 118,
JSGtOp = 119,
JSBOrOp = 120,
JSBXorOp = 121,
JSBAndOp = 122,
JSBNotOp = 123,
JSNotOp = 124,
JSBindPattern = 126,
JSConstantExpression = 128,
JSMemberExpression = 129,
JSCallExpression = 130,
JSBinaryExpression = 131,
JSUnaryExpression = 132,
JSNewExpression = 133,
JSSequenceExpression = 134,
JSConditionalExpression = 135,
JSReferenceExpression = 136,
JSExpressionStatement = 139,
JSConditionalStatement = 140,
JSParameter = 141,
JSImportStarBinding = 145,
JSImportAsBinding = 146,
JSImportDeclaration = 147,
JSFunctionDeclaration = 148,
JSArrowFunctionDeclaration = 149,
JSLetDeclaration = 150,
JSSourceFile = 151,
}
@ -625,9 +649,19 @@ export type JSToken
= EndOfFile
| JSOperator
| JSIdentifier
| JSString
| JSFromKeyword
| JSReturnKeyword
| JSTryKeyword
| JSCatchKeyword
| JSImportKeyword
| JSAsKeyword
| JSConstKeyword
| JSLetKeyword
| JSExportKeyword
| JSFunctionKeyword
| JSWhileKeyword
| JSForKeyword
| JSCloseBrace
| JSCloseBracket
| JSCloseParen
@ -638,6 +672,17 @@ export type JSToken
| JSComma
| JSDot
| JSDotDotDot
| JSMulOp
| JSAddOp
| JSDivOp
| JSSubOp
| JSLtOp
| JSGtOp
| JSBOrOp
| JSBXorOp
| JSBAndOp
| JSBNotOp
| JSNotOp
export interface JSOperator extends SyntaxBase {
@ -650,6 +695,15 @@ export interface JSIdentifier extends SyntaxBase {
text: string;
}
export interface JSString extends SyntaxBase {
kind: SyntaxKind.JSString;
value: string;
}
export interface JSFromKeyword extends SyntaxBase {
kind: SyntaxKind.JSFromKeyword;
}
export interface JSReturnKeyword extends SyntaxBase {
kind: SyntaxKind.JSReturnKeyword;
}
@ -662,6 +716,38 @@ export interface JSCatchKeyword extends SyntaxBase {
kind: SyntaxKind.JSCatchKeyword;
}
export interface JSImportKeyword extends SyntaxBase {
kind: SyntaxKind.JSImportKeyword;
}
export interface JSAsKeyword extends SyntaxBase {
kind: SyntaxKind.JSAsKeyword;
}
export interface JSConstKeyword extends SyntaxBase {
kind: SyntaxKind.JSConstKeyword;
}
export interface JSLetKeyword extends SyntaxBase {
kind: SyntaxKind.JSLetKeyword;
}
export interface JSExportKeyword extends SyntaxBase {
kind: SyntaxKind.JSExportKeyword;
}
export interface JSFunctionKeyword extends SyntaxBase {
kind: SyntaxKind.JSFunctionKeyword;
}
export interface JSWhileKeyword extends SyntaxBase {
kind: SyntaxKind.JSWhileKeyword;
}
export interface JSForKeyword extends SyntaxBase {
kind: SyntaxKind.JSForKeyword;
}
export interface JSCloseBrace extends SyntaxBase {
kind: SyntaxKind.JSCloseBrace;
}
@ -702,6 +788,50 @@ export interface JSDotDotDot extends SyntaxBase {
kind: SyntaxKind.JSDotDotDot;
}
export interface JSMulOp extends SyntaxBase {
kind: SyntaxKind.JSMulOp;
}
export interface JSAddOp extends SyntaxBase {
kind: SyntaxKind.JSAddOp;
}
export interface JSDivOp extends SyntaxBase {
kind: SyntaxKind.JSDivOp;
}
export interface JSSubOp extends SyntaxBase {
kind: SyntaxKind.JSSubOp;
}
export interface JSLtOp extends SyntaxBase {
kind: SyntaxKind.JSLtOp;
}
export interface JSGtOp extends SyntaxBase {
kind: SyntaxKind.JSGtOp;
}
export interface JSBOrOp extends SyntaxBase {
kind: SyntaxKind.JSBOrOp;
}
export interface JSBXorOp extends SyntaxBase {
kind: SyntaxKind.JSBXorOp;
}
export interface JSBAndOp extends SyntaxBase {
kind: SyntaxKind.JSBAndOp;
}
export interface JSBNotOp extends SyntaxBase {
kind: SyntaxKind.JSBNotOp;
}
export interface JSNotOp extends SyntaxBase {
kind: SyntaxKind.JSNotOp;
}
export type JSPattern
= JSBindPattern
@ -779,6 +909,7 @@ export interface JSReferenceExpression extends SyntaxBase {
export type JSSourceElement
= JSExpressionStatement
| JSConditionalStatement
| JSImportDeclaration
| JSFunctionDeclaration
| JSArrowFunctionDeclaration
| JSLetDeclaration
@ -809,7 +940,8 @@ export interface JSParameter extends SyntaxBase {
}
export type JSDeclaration
= JSFunctionDeclaration
= JSImportDeclaration
| JSFunctionDeclaration
| JSArrowFunctionDeclaration
| JSLetDeclaration
@ -817,6 +949,28 @@ export type JSDeclaration
export const enum JSDeclarationModifiers {
IsExported = 1,}
export type JSImportBinding
= JSImportStarBinding
| JSImportAsBinding
export interface JSImportStarBinding extends SyntaxBase {
kind: SyntaxKind.JSImportStarBinding;
local: JSIdentifier;
}
export interface JSImportAsBinding extends SyntaxBase {
kind: SyntaxKind.JSImportAsBinding;
remote: JSIdentifier;
local: JSIdentifier | null;
}
export interface JSImportDeclaration extends SyntaxBase {
kind: SyntaxKind.JSImportDeclaration;
bindings: JSImportBinding[];
filename: JSString;
}
export interface JSFunctionDeclaration extends SyntaxBase {
kind: SyntaxKind.JSFunctionDeclaration;
modifiers: JSDeclarationModifiers;
@ -919,9 +1073,19 @@ export type BoltSyntax
export type JSSyntax
= JSOperator
| JSIdentifier
| JSString
| JSFromKeyword
| JSReturnKeyword
| JSTryKeyword
| JSCatchKeyword
| JSImportKeyword
| JSAsKeyword
| JSConstKeyword
| JSLetKeyword
| JSExportKeyword
| JSFunctionKeyword
| JSWhileKeyword
| JSForKeyword
| JSCloseBrace
| JSCloseBracket
| JSCloseParen
@ -932,6 +1096,17 @@ export type JSSyntax
| JSComma
| JSDot
| JSDotDotDot
| JSMulOp
| JSAddOp
| JSDivOp
| JSSubOp
| JSLtOp
| JSGtOp
| JSBOrOp
| JSBXorOp
| JSBAndOp
| JSBNotOp
| JSNotOp
| JSBindPattern
| JSConstantExpression
| JSMemberExpression
@ -945,6 +1120,9 @@ export type JSSyntax
| JSExpressionStatement
| JSConditionalStatement
| JSParameter
| JSImportStarBinding
| JSImportAsBinding
| JSImportDeclaration
| JSFunctionDeclaration
| JSArrowFunctionDeclaration
| JSLetDeclaration
@ -1026,9 +1204,19 @@ export type Syntax
| BoltRecordDeclaration
| JSOperator
| JSIdentifier
| JSString
| JSFromKeyword
| JSReturnKeyword
| JSTryKeyword
| JSCatchKeyword
| JSImportKeyword
| JSAsKeyword
| JSConstKeyword
| JSLetKeyword
| JSExportKeyword
| JSFunctionKeyword
| JSWhileKeyword
| JSForKeyword
| JSCloseBrace
| JSCloseBracket
| JSCloseParen
@ -1039,6 +1227,17 @@ export type Syntax
| JSComma
| JSDot
| JSDotDotDot
| JSMulOp
| JSAddOp
| JSDivOp
| JSSubOp
| JSLtOp
| JSGtOp
| JSBOrOp
| JSBXorOp
| JSBAndOp
| JSBNotOp
| JSNotOp
| JSBindPattern
| JSConstantExpression
| JSMemberExpression
@ -1052,6 +1251,9 @@ export type Syntax
| JSExpressionStatement
| JSConditionalStatement
| JSParameter
| JSImportStarBinding
| JSImportAsBinding
| JSImportDeclaration
| JSFunctionDeclaration
| JSArrowFunctionDeclaration
| JSLetDeclaration
@ -1134,9 +1336,19 @@ export function createBoltRecordDeclarationField(name: BoltIdentifier, type: Bol
export function createBoltRecordDeclaration(modifiers: BoltDeclarationModifiers, name: BoltQualName, typeParms: BoltTypeParameter[] | null, fields: BoltRecordDeclarationField[], span?: TextSpan | null): BoltRecordDeclaration;
export function createJSOperator(text: string, span?: TextSpan | null): JSOperator;
export function createJSIdentifier(text: string, span?: TextSpan | null): JSIdentifier;
export function createJSString(value: string, span?: TextSpan | null): JSString;
export function createJSFromKeyword(span?: TextSpan | null): JSFromKeyword;
export function createJSReturnKeyword(span?: TextSpan | null): JSReturnKeyword;
export function createJSTryKeyword(span?: TextSpan | null): JSTryKeyword;
export function createJSCatchKeyword(span?: TextSpan | null): JSCatchKeyword;
export function createJSImportKeyword(span?: TextSpan | null): JSImportKeyword;
export function createJSAsKeyword(span?: TextSpan | null): JSAsKeyword;
export function createJSConstKeyword(span?: TextSpan | null): JSConstKeyword;
export function createJSLetKeyword(span?: TextSpan | null): JSLetKeyword;
export function createJSExportKeyword(span?: TextSpan | null): JSExportKeyword;
export function createJSFunctionKeyword(span?: TextSpan | null): JSFunctionKeyword;
export function createJSWhileKeyword(span?: TextSpan | null): JSWhileKeyword;
export function createJSForKeyword(span?: TextSpan | null): JSForKeyword;
export function createJSCloseBrace(span?: TextSpan | null): JSCloseBrace;
export function createJSCloseBracket(span?: TextSpan | null): JSCloseBracket;
export function createJSCloseParen(span?: TextSpan | null): JSCloseParen;
@ -1147,6 +1359,17 @@ export function createJSSemi(span?: TextSpan | null): JSSemi;
export function createJSComma(span?: TextSpan | null): JSComma;
export function createJSDot(span?: TextSpan | null): JSDot;
export function createJSDotDotDot(span?: TextSpan | null): JSDotDotDot;
export function createJSMulOp(span?: TextSpan | null): JSMulOp;
export function createJSAddOp(span?: TextSpan | null): JSAddOp;
export function createJSDivOp(span?: TextSpan | null): JSDivOp;
export function createJSSubOp(span?: TextSpan | null): JSSubOp;
export function createJSLtOp(span?: TextSpan | null): JSLtOp;
export function createJSGtOp(span?: TextSpan | null): JSGtOp;
export function createJSBOrOp(span?: TextSpan | null): JSBOrOp;
export function createJSBXorOp(span?: TextSpan | null): JSBXorOp;
export function createJSBAndOp(span?: TextSpan | null): JSBAndOp;
export function createJSBNotOp(span?: TextSpan | null): JSBNotOp;
export function createJSNotOp(span?: TextSpan | null): JSNotOp;
export function createJSBindPattern(name: JSIdentifier, span?: TextSpan | null): JSBindPattern;
export function createJSConstantExpression(value: BoltValue, span?: TextSpan | null): JSConstantExpression;
export function createJSMemberExpression(value: JSExpression, property: JSIdentifier, span?: TextSpan | null): JSMemberExpression;
@ -1160,6 +1383,9 @@ export function createJSReferenceExpression(name: string, span?: TextSpan | null
export function createJSExpressionStatement(expression: JSExpression, span?: TextSpan | null): JSExpressionStatement;
export function createJSConditionalStatement(test: JSExpression, consequent: JSStatement[], alternate: JSStatement[], span?: TextSpan | null): JSConditionalStatement;
export function createJSParameter(index: number, bindings: JSPattern, defaultValue: JSExpression | null, span?: TextSpan | null): JSParameter;
export function createJSImportStarBinding(local: JSIdentifier, span?: TextSpan | null): JSImportStarBinding;
export function createJSImportAsBinding(remote: JSIdentifier, local: JSIdentifier | null, span?: TextSpan | null): JSImportAsBinding;
export function createJSImportDeclaration(bindings: JSImportBinding[], filename: JSString, span?: TextSpan | null): JSImportDeclaration;
export function createJSFunctionDeclaration(modifiers: JSDeclarationModifiers, name: JSIdentifier, params: JSParameter[], body: JSStatement[], span?: TextSpan | null): JSFunctionDeclaration;
export function createJSArrowFunctionDeclaration(name: JSIdentifier, params: JSParameter[], body: JSExpression, span?: TextSpan | null): JSArrowFunctionDeclaration;
export function createJSLetDeclaration(bindings: JSPattern, value: JSExpression | null, span?: TextSpan | null): JSLetDeclaration;
@ -1251,9 +1477,19 @@ export function isBoltSourceElement(value: any): value is BoltSourceElement;
export function isJSToken(value: any): value is JSToken;
export function isJSOperator(value: any): value is JSOperator;
export function isJSIdentifier(value: any): value is JSIdentifier;
export function isJSString(value: any): value is JSString;
export function isJSFromKeyword(value: any): value is JSFromKeyword;
export function isJSReturnKeyword(value: any): value is JSReturnKeyword;
export function isJSTryKeyword(value: any): value is JSTryKeyword;
export function isJSCatchKeyword(value: any): value is JSCatchKeyword;
export function isJSImportKeyword(value: any): value is JSImportKeyword;
export function isJSAsKeyword(value: any): value is JSAsKeyword;
export function isJSConstKeyword(value: any): value is JSConstKeyword;
export function isJSLetKeyword(value: any): value is JSLetKeyword;
export function isJSExportKeyword(value: any): value is JSExportKeyword;
export function isJSFunctionKeyword(value: any): value is JSFunctionKeyword;
export function isJSWhileKeyword(value: any): value is JSWhileKeyword;
export function isJSForKeyword(value: any): value is JSForKeyword;
export function isJSCloseBrace(value: any): value is JSCloseBrace;
export function isJSCloseBracket(value: any): value is JSCloseBracket;
export function isJSCloseParen(value: any): value is JSCloseParen;
@ -1264,6 +1500,17 @@ export function isJSSemi(value: any): value is JSSemi;
export function isJSComma(value: any): value is JSComma;
export function isJSDot(value: any): value is JSDot;
export function isJSDotDotDot(value: any): value is JSDotDotDot;
export function isJSMulOp(value: any): value is JSMulOp;
export function isJSAddOp(value: any): value is JSAddOp;
export function isJSDivOp(value: any): value is JSDivOp;
export function isJSSubOp(value: any): value is JSSubOp;
export function isJSLtOp(value: any): value is JSLtOp;
export function isJSGtOp(value: any): value is JSGtOp;
export function isJSBOrOp(value: any): value is JSBOrOp;
export function isJSBXorOp(value: any): value is JSBXorOp;
export function isJSBAndOp(value: any): value is JSBAndOp;
export function isJSBNotOp(value: any): value is JSBNotOp;
export function isJSNotOp(value: any): value is JSNotOp;
export function isJSPattern(value: any): value is JSPattern;
export function isJSBindPattern(value: any): value is JSBindPattern;
export function isJSExpression(value: any): value is JSExpression;
@ -1282,6 +1529,10 @@ export function isJSExpressionStatement(value: any): value is JSExpressionStatem
export function isJSConditionalStatement(value: any): value is JSConditionalStatement;
export function isJSParameter(value: any): value is JSParameter;
export function isJSDeclaration(value: any): value is JSDeclaration;
export function isJSImportBinding(value: any): value is JSImportBinding;
export function isJSImportStarBinding(value: any): value is JSImportStarBinding;
export function isJSImportAsBinding(value: any): value is JSImportAsBinding;
export function isJSImportDeclaration(value: any): value is JSImportDeclaration;
export function isJSFunctionDeclaration(value: any): value is JSFunctionDeclaration;
export function isJSArrowFunctionDeclaration(value: any): value is JSArrowFunctionDeclaration;
export function isJSLetDeclaration(value: any): value is JSLetDeclaration;

85
src/foreign/js/parser.ts
View file

@ -14,11 +14,24 @@ import {
JSIdentifier,
JSMemberExpression,
createJSMemberExpression,
createJSCallExpression
createJSCallExpression,
JSDeclaration,
JSString,
createJSImportDeclaration,
createJSImportStarBinding,
createJSImportAsBinding
} from "../../ast"
export type JSTokenStream = Stream<JSToken>;
const T0_DECLARATION = [
SyntaxKind.JSConstKeyword,
SyntaxKind.JSLetKeyword,
SyntaxKind.JSFunctionKeyword,
SyntaxKind.JSImportKeyword,
SyntaxKind.JSExportKeyword,
];
export class JSParser {
public parseJSReferenceExpression(tokens: JSTokenStream): JSReferenceExpression {
@ -96,6 +109,72 @@ export class JSParser {
return this.parseJSExpressionStatement(tokens);
}
public parseImportDeclaration(tokens: JSTokenStream): JSImportDeclaration {
const t0 = tokens.get();
assertToken(t0, SyntaxKind.JSImportKeyword);
const t1 = tokens.peek();
let bindings = [];
let filename;
if (t1.kind === SyntaxKind.JSString) {
tokens.get();
filename = t1 as JSString;
} else {
while (true) {
const t1 = tokens.get();
if (t1.kind === SyntaxKind.JSFromKeyword) {
break;
}
if (t1.kind === SyntaxKind.JSMulOp) {
const t2 = tokens.get();
assertToken(t2, SyntaxKind.JSAsKeyword);
const t3 = tokens.get();
assertToken(t3, SyntaxKind.JSIdentifier);
const binding = createJSImportStarBinding(t3 as JSIdentifier);
setOrigNodeRange(binding, t1, t1);
bindings.push(binding);
} else if (t1.kind === SyntaxKind.JSOpenBrace) {
// TODO
} else if (t1.kind === SyntaxKind.JSIdentifier) {
const binding = createJSImportAsBinding(t1, null)
setOrigNodeRange(binding, t1, t1);
bindings.push(binding);
} else {
throw new ParseError(t1, [SyntaxKind.JSMulOp, SyntaxKind.JSIdentifier, SyntaxKind.JSOpenBrace]);
}
}
const t2 = tokens.get();
assertToken(t2, SyntaxKind.JSString);
filename = t2 as JSString;
}
const result = createJSImportDeclaration(bindings, filename)
setOrigNodeRange(result, t0, filename);
return result;
}
public parseExportDeclaration(tokens: JSTokenStream): JSExportDeclaration {
}
public parseJSDeclaration(tokens: JSTokenStream): JSDeclaration {
const t0 = tokens.peek();
if (t0.kind === SyntaxKind.JSImportKeyword) {
return this.parseImportDeclaration(tokens);
} else if (t0.kind === SyntaxKind.JSExportKeyword) {
return this.parseExportDeclaration(tokens);
} else {
throw new ParseError(t0, T0_DECLARATION);
}
}
public parseJSSourceElement(tokens: JSTokenStream): JSSourceElement {
const t0 = tokens.peek();
if (T0_DECLARATION.indexOf(t0.kind) !== -1) {
return this.parseJSDeclaration(tokens);
} else {
return this.parseJSStatement(tokens);
}
}
public parseJSSourceElementList(tokens: JSTokenStream): JSSourceElement[] {
const elements: JSSourceElement[] = [];
while (true) {
@ -107,8 +186,8 @@ export class JSParser {
tokens.get();
continue;
}
const statement = this.parseJSStatement(tokens)
elements.push(statement);
const element = this.parseJSSourceElement(tokens)
elements.push(element);
}
return elements;
}

86
src/foreign/js/scanner.ts
View file

@ -18,6 +18,29 @@ import {
createJSSemi,
createJSComma,
createEndOfFile,
createJSMulOp,
createJSNotOp,
createJSBOrOp,
createJSBNotOp,
createJSBXorOp,
createJSBAndOp,
createJSGtOp,
createJSLtOp,
createJSDivOp,
createJSSubOp,
createJSAddOp,
createJSLetKeyword,
createJSWhileKeyword,
createJSForKeyword,
createJSFunctionKeyword,
createJSExportKeyword,
createJSImportKeyword,
createJSConstKeyword,
createJSAsKeyword,
createJSReturnKeyword,
createJSCatchKeyword,
createJSFromKeyword,
createJSString,
} from "../../ast"
function isWhiteSpace(ch: string): boolean {
@ -31,6 +54,10 @@ function isLineTerminator(ch: string): boolean {
|| ch === '\u2029';;
}
function isOperator(ch: string): boolean {
return /[-+*/&^|%!<>=]/.test(ch)
}
function isIdentStart(ch: string): boolean {
return /[\p{ID_Start}$_\\]/u.test(ch)
}
@ -190,6 +217,22 @@ export class JSScanner {
throw new Error(`Scanning unicode escape sequences is not yet implemented.`);
}
protected takeWhile(pred: (ch: string) => boolean) {
let text = this.getChar();
while (true) {
const c0 = this.peekChar();
if (c0 === EOF) {
break;
}
if (!pred(c0)) {
break;
}
this.getChar()
text += c0;
}
return text;
}
public scan(): JSToken {
this.skipComments();
@ -202,6 +245,15 @@ export class JSScanner {
const startPos = this.currPos.clone();
if (c0 === '"' || c0 === "'") {
// FIXME
this.getChar();
const value = this.takeWhile(ch => ch !== c0)
this.getChar();
const endPos = this.currPos.clone();
return createJSString(value, new TextSpan(this.file, startPos, endPos))
}
if (/[,;()\[\]{}]/.test(c0)) {
this.getChar();
const span = new TextSpan(this.file, startPos, this.currPos.clone());
@ -235,6 +287,24 @@ export class JSScanner {
}
}
if (isOperator(c0)) {
const text = this.takeWhile(isOperator)
const span = new TextSpan(this.file, startPos, this.currPos.clone());
switch (text) {
case '+': return createJSAddOp(span);
case '-': return createJSSubOp(span);
case '*': return createJSMulOp(span);
case '/': return createJSDivOp(span);
case '<': return createJSLtOp(span);
case '>': return createJSGtOp(span);
case '&': return createJSBAndOp(span);
case '^': return createJSBXorOp(span);
case '~': return createJSBNotOp(span);
case '|': return createJSBOrOp(span);
case '!': return createJSNotOp(span);
}
}
if (isIdentStart(c0)) {
let name = '';
while (true) {
@ -249,7 +319,21 @@ export class JSScanner {
}
}
const endPos = this.currPos.clone();
return createJSIdentifier(name, new TextSpan(this.file, startPos, endPos))
const span = new TextSpan(this.file, startPos, endPos);
switch (name) {
case 'return': return createJSReturnKeyword(span);
case 'catch': return createJSCatchKeyword(span);
case 'from': return createJSFromKeyword(span);
case 'let': return createJSLetKeyword(span);
case 'const': return createJSConstKeyword(span);
case 'import': return createJSImportKeyword(span);
case 'export': return createJSExportKeyword(span);
case 'as': return createJSAsKeyword(span);
case 'function': return createJSFunctionKeyword(span);
case 'for': return createJSForKeyword(span);
case 'while': return createJSWhileKeyword(span);
default: return createJSIdentifier(name, span)
}
} else {
throw new ScanError(this.file, startPos, c0);
}

26
src/util.ts
View file

@ -305,6 +305,32 @@ export function describeKind(kind: SyntaxKind): string {
return "'trait'";
case SyntaxKind.BoltForKeyword:
return "'for'";
case SyntaxKind.JSMulOp:
return "'*'";
case SyntaxKind.JSAddOp:
return "'+'";
case SyntaxKind.JSDivOp:
return "'/'";
case SyntaxKind.JSSubOp:
return "'-'";
case SyntaxKind.JSLtOp:
return "'<'";
case SyntaxKind.JSGtOp:
return "'>'";
case SyntaxKind.JSBOrOp:
return "'|'";
case SyntaxKind.JSBXorOp:
return "'^'";
case SyntaxKind.JSBAndOp:
return "'&'";
case SyntaxKind.JSBNotOp:
return "'~'";
case SyntaxKind.JSNotOp:
return "'~'";
case SyntaxKind.JSString:
return "a JavaScript string"
case SyntaxKind.JSReturnKeyword:
return "'return'";
default:
throw new Error(`failed to describe ${kindToString(kind)}`)
}