Extend scanner/parser to support more syntactic structures
This commit is contained in:
parent
d813e85d00
commit
cda44e4c25
5 changed files with 758 additions and 55 deletions
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import "source-map-support/register"
|
||||
|
||||
import util from "util"
|
||||
import path from "path"
|
||||
import fs from "fs"
|
||||
import yargs from "yargs"
|
||||
|
|
@ -10,6 +11,10 @@ import { Diagnostics } from "../diagnostics"
|
|||
import { Punctuator, Scanner } from "../scanner"
|
||||
import { Parser } from "../parser"
|
||||
|
||||
function debug(value: any) {
|
||||
console.error(util.inspect(value, { colors: true, depth: Infinity }));
|
||||
}
|
||||
|
||||
yargs
|
||||
.string('work-dir')
|
||||
.describe('work-dir', 'Act as if run from this directory')
|
||||
|
|
@ -33,7 +38,7 @@ yargs
|
|||
const parser = new Parser(punctuated);
|
||||
const sourceFile = parser.parseSourceFile();
|
||||
|
||||
console.log(sourceFile);
|
||||
debug(sourceFile.toJSON());
|
||||
|
||||
}
|
||||
)
|
||||
|
|
|
|||
521
src/cst.ts
521
src/cst.ts
|
|
@ -1,3 +1,4 @@
|
|||
import { JSONObject, JSONValue } from "./util";
|
||||
|
||||
export type TextSpan = [number, number];
|
||||
|
||||
|
|
@ -67,6 +68,7 @@ export const enum SyntaxKind {
|
|||
|
||||
// Tokens
|
||||
Identifier,
|
||||
Constructor,
|
||||
CustomOperator,
|
||||
LParen,
|
||||
RParen,
|
||||
|
|
@ -88,6 +90,11 @@ export const enum SyntaxKind {
|
|||
ImportKeyword,
|
||||
StructKeyword,
|
||||
TypeKeyword,
|
||||
ReturnKeyword,
|
||||
MatchKeyword,
|
||||
IfKeyword,
|
||||
ElifKeyword,
|
||||
ElseKeyword,
|
||||
LineFoldEnd,
|
||||
BlockEnd,
|
||||
BlockStart,
|
||||
|
|
@ -99,6 +106,14 @@ export const enum SyntaxKind {
|
|||
// Patterns
|
||||
BindPattern,
|
||||
TuplePattern,
|
||||
StructPattern,
|
||||
NestedPattern,
|
||||
NamedTuplePattern,
|
||||
|
||||
// Struct pattern elements
|
||||
FieldStructPatternElement,
|
||||
PunnedFieldStructPatternElement,
|
||||
VariadicStructPatternElement,
|
||||
|
||||
// Expressions
|
||||
ReferenceExpression,
|
||||
|
|
@ -122,10 +137,16 @@ export const enum SyntaxKind {
|
|||
ImportDeclaration,
|
||||
TypeAliasDeclaration,
|
||||
|
||||
// Other nodes
|
||||
StructDeclarationField,
|
||||
// Let declaration body members
|
||||
ExprBody,
|
||||
BlockBody,
|
||||
|
||||
// Structure declaration members
|
||||
StructDeclarationField,
|
||||
|
||||
// Other nodes
|
||||
Initializer,
|
||||
QualifiedName,
|
||||
TypeAssert,
|
||||
Param,
|
||||
Module,
|
||||
|
|
@ -147,6 +168,39 @@ abstract class SyntaxBase {
|
|||
|
||||
public abstract readonly kind: SyntaxKind;
|
||||
|
||||
public abstract getFirstToken(): Token;
|
||||
|
||||
public abstract getLastToken(): Token;
|
||||
|
||||
public toJSON(): JSONObject {
|
||||
|
||||
const obj: JSONObject = {};
|
||||
|
||||
obj['type'] = this.constructor.name;
|
||||
|
||||
for (const key of Object.getOwnPropertyNames(this)) {
|
||||
if (key === 'kind') {
|
||||
continue;
|
||||
}
|
||||
obj[key] = encode((this as any)[key]);
|
||||
}
|
||||
|
||||
return obj;
|
||||
|
||||
function encode(value: any): JSONValue {
|
||||
if (value === null) {
|
||||
return null;
|
||||
} else if (Array.isArray(value)) {
|
||||
return value.map(encode);
|
||||
} else if (value instanceof SyntaxBase) {
|
||||
return value.toJSON();
|
||||
} else {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
abstract class TokenBase extends SyntaxBase {
|
||||
|
|
@ -159,6 +213,14 @@ abstract class TokenBase extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
throw new Error(`Trying to get the first token of an object that is a token itself.`);
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
throw new Error(`Trying to get the last token of an object that is a token itself.`);
|
||||
}
|
||||
|
||||
public getStartPosition(): TextPosition {
|
||||
return this.startPos;
|
||||
}
|
||||
|
|
@ -221,7 +283,7 @@ export class Integer extends TokenBase {
|
|||
public constructor(
|
||||
public value: bigint,
|
||||
public radix: number,
|
||||
private startPos: TextPosition,
|
||||
startPos: TextPosition,
|
||||
) {
|
||||
super(startPos);
|
||||
}
|
||||
|
|
@ -249,7 +311,7 @@ export class StringLiteral extends TokenBase {
|
|||
|
||||
public constructor(
|
||||
public contents: string,
|
||||
private startPos: TextPosition,
|
||||
startPos: TextPosition,
|
||||
) {
|
||||
super(startPos);
|
||||
}
|
||||
|
|
@ -272,13 +334,26 @@ export class StringLiteral extends TokenBase {
|
|||
|
||||
}
|
||||
|
||||
export class Constructor extends TokenBase {
|
||||
|
||||
public readonly kind = SyntaxKind.Constructor;
|
||||
|
||||
public constructor(
|
||||
public text: string,
|
||||
startPos: TextPosition,
|
||||
) {
|
||||
super(startPos);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class Identifier extends TokenBase {
|
||||
|
||||
public readonly kind = SyntaxKind.Identifier;
|
||||
|
||||
public constructor(
|
||||
public text: string,
|
||||
private startPos: TextPosition,
|
||||
startPos: TextPosition,
|
||||
) {
|
||||
super(startPos);
|
||||
}
|
||||
|
|
@ -291,7 +366,7 @@ export class CustomOperator extends TokenBase {
|
|||
|
||||
public constructor(
|
||||
public text: string,
|
||||
private startPos: TextPosition,
|
||||
startPos: TextPosition,
|
||||
) {
|
||||
super(startPos);
|
||||
}
|
||||
|
|
@ -418,6 +493,26 @@ export class StructKeyword extends TokenBase {
|
|||
|
||||
}
|
||||
|
||||
export class ReturnKeyword extends TokenBase {
|
||||
|
||||
public readonly kind = SyntaxKind.ReturnKeyword;
|
||||
|
||||
public get text(): string {
|
||||
return 'return';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class MatchKeyword extends TokenBase {
|
||||
|
||||
public readonly kind = SyntaxKind.MatchKeyword;
|
||||
|
||||
public get text(): string {
|
||||
return 'match';
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class ModKeyword extends TokenBase {
|
||||
|
||||
public readonly kind = SyntaxKind.ModKeyword;
|
||||
|
|
@ -486,6 +581,7 @@ export type Token
|
|||
| LBracket
|
||||
| RBracket
|
||||
| Identifier
|
||||
| Constructor
|
||||
| CustomOperator
|
||||
| Integer
|
||||
| StringLiteral
|
||||
|
|
@ -501,6 +597,8 @@ export type Token
|
|||
| ImportKeyword
|
||||
| TypeKeyword
|
||||
| StructKeyword
|
||||
| ReturnKeyword
|
||||
| MatchKeyword
|
||||
| EndOfFile
|
||||
| BlockStart
|
||||
| BlockEnd
|
||||
|
|
@ -520,6 +618,17 @@ export class ReferenceTypeExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
if (this.modulePath.length > 0) {
|
||||
return this.modulePath[0][0];
|
||||
}
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type TypeExpression
|
||||
|
|
@ -539,6 +648,14 @@ export class BindPattern extends SyntaxBase {
|
|||
return this.name.text == '_';
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class TuplePattern extends SyntaxBase {
|
||||
|
|
@ -546,15 +663,168 @@ export class TuplePattern extends SyntaxBase {
|
|||
public readonly kind = SyntaxKind.TuplePattern;
|
||||
|
||||
public constructor(
|
||||
public lparen: LParen,
|
||||
public elements: Pattern[],
|
||||
public rparen: RParen,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.lparen;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.rparen;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class NamedTuplePattern extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.NamedTuplePattern;
|
||||
|
||||
public constructor(
|
||||
public name: Constructor,
|
||||
public elements: Pattern[],
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.elements.length > 0) {
|
||||
return this.elements[this.elements.length-1].getLastToken();
|
||||
}
|
||||
return this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class FieldStructPatternElement extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.FieldStructPatternElement;
|
||||
|
||||
public constructor(
|
||||
public name: Identifier,
|
||||
public equals: Equals,
|
||||
public pattern: Pattern,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.pattern.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class VariadicStructPatternElement extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.VariadicStructPatternElement;
|
||||
|
||||
public constructor(
|
||||
public dotdot: DotDot,
|
||||
public pattern: Pattern | null,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.dotdot;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.pattern !== null) {
|
||||
return this.pattern.getLastToken();
|
||||
}
|
||||
return this.dotdot;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class PunnedFieldStructPatternElement extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.PunnedFieldStructPatternElement;
|
||||
|
||||
public constructor(
|
||||
public name: Identifier,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type StructPatternElement
|
||||
= VariadicStructPatternElement
|
||||
| PunnedFieldStructPatternElement
|
||||
| FieldStructPatternElement
|
||||
|
||||
export class StructPattern extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.StructPattern;
|
||||
|
||||
public constructor(
|
||||
public name: Constructor,
|
||||
public lbrace: LBrace,
|
||||
public elements: StructPatternElement[],
|
||||
public rbrace: RBrace,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.rbrace;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class NestedPattern extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.NestedPattern;
|
||||
|
||||
public constructor(
|
||||
public lparen: LParen,
|
||||
public pattern: Pattern,
|
||||
public rparen: RParen,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.lparen;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.rparen;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type Pattern
|
||||
= BindPattern
|
||||
| NestedPattern
|
||||
| StructPattern
|
||||
| NamedTuplePattern
|
||||
| TuplePattern
|
||||
|
||||
export class TupleExpression extends SyntaxBase {
|
||||
|
|
@ -569,6 +839,14 @@ export class TupleExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.lparen;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.rparen;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class NestedExpression extends SyntaxBase {
|
||||
|
|
@ -583,6 +861,14 @@ export class NestedExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.lparen;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.rparen;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class ConstantExpression extends SyntaxBase {
|
||||
|
|
@ -595,6 +881,38 @@ export class ConstantExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.token;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.token;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class QualifiedName extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.QualifiedName;
|
||||
|
||||
public constructor(
|
||||
public modulePath: Array<[Identifier, Dot]>,
|
||||
public name: Identifier,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
if (this.modulePath.length > 0) {
|
||||
return this.modulePath[0][0];
|
||||
}
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class ReferenceExpression extends SyntaxBase {
|
||||
|
|
@ -602,12 +920,19 @@ export class ReferenceExpression extends SyntaxBase {
|
|||
public readonly kind = SyntaxKind.ReferenceExpression;
|
||||
|
||||
public constructor(
|
||||
public modulePath: Array<[Identifier, Dot]>,
|
||||
public name: Identifier
|
||||
public name: QualifiedName,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name.getFirstToken();
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.name.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class PrefixExpression extends SyntaxBase {
|
||||
|
|
@ -621,6 +946,14 @@ export class PrefixExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.operator;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.expression.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class PostfixExpression extends SyntaxBase {
|
||||
|
|
@ -634,6 +967,14 @@ export class PostfixExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.expression.getFirstToken();
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.operator;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class InfixExpression extends SyntaxBase {
|
||||
|
|
@ -648,6 +989,14 @@ export class InfixExpression extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.left.getFirstToken();
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.right.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type Expression
|
||||
|
|
@ -664,11 +1013,23 @@ export class ReturnStatement extends SyntaxBase {
|
|||
public readonly kind = SyntaxKind.ReturnStatement;
|
||||
|
||||
public constructor(
|
||||
public expr: Expression
|
||||
public returnKeyword: ReturnKeyword,
|
||||
public expression: Expression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.returnKeyword;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.expression !== null) {
|
||||
return this.expression.getLastToken();
|
||||
}
|
||||
return this.returnKeyword;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class ExpressionStatement extends SyntaxBase {
|
||||
|
|
@ -676,11 +1037,19 @@ export class ExpressionStatement extends SyntaxBase {
|
|||
public readonly kind = SyntaxKind.ExpressionStatement;
|
||||
|
||||
public constructor(
|
||||
public expresion: Expression,
|
||||
public expression: Expression,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.expression.getFirstToken();
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.expression.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type Statement
|
||||
|
|
@ -697,6 +1066,14 @@ export class Param extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.pattern.getFirstToken();
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.pattern.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class StructDeclarationField extends SyntaxBase {
|
||||
|
|
@ -711,6 +1088,14 @@ export class StructDeclarationField extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.name;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.typeExpr.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class StructDeclaration extends SyntaxBase {
|
||||
|
|
@ -725,6 +1110,17 @@ export class StructDeclaration extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.structKeyword;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.members && this.members.length > 0) {
|
||||
return this.members[this.members.length-1].getLastToken();
|
||||
}
|
||||
return this.name;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class TypeAssert extends SyntaxBase {
|
||||
|
|
@ -738,6 +1134,14 @@ export class TypeAssert extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.colon;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.typeExpression.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type Body
|
||||
|
|
@ -755,6 +1159,14 @@ export class ExprBody extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.equals;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.expression.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type LetBodyElement
|
||||
|
|
@ -772,6 +1184,17 @@ export class BlockBody extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.blockStart;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.elements.length > 0) {
|
||||
return this.elements[this.elements.length-1].getLastToken();
|
||||
}
|
||||
return this.blockStart;
|
||||
}
|
||||
|
||||
}
|
||||
export class LetDeclaration extends SyntaxBase {
|
||||
|
||||
|
|
@ -789,6 +1212,26 @@ export class LetDeclaration extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
if (this.pubKeyword !== null) {
|
||||
return this.pubKeyword;
|
||||
}
|
||||
return this.letKeyword;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.body !== null) {
|
||||
return this.body.getLastToken();
|
||||
}
|
||||
if (this.typeAssert !== null) {
|
||||
return this.typeAssert.getLastToken();
|
||||
}
|
||||
if (this.params.length > 0) {
|
||||
return this.params[this.params.length-1].getLastToken();
|
||||
}
|
||||
return this.pattern.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class ImportDeclaration extends SyntaxBase {
|
||||
|
|
@ -802,6 +1245,14 @@ export class ImportDeclaration extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.importKeyword;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.importSource;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type Declaration
|
||||
|
|
@ -809,11 +1260,33 @@ export type Declaration
|
|||
| ImportDeclaration
|
||||
| StructDeclaration
|
||||
|
||||
export class Initializer extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.Initializer;
|
||||
|
||||
public constructor(
|
||||
public equals: Equals,
|
||||
public expression: Expression
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
return this.equals;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.expression.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export class Module extends SyntaxBase {
|
||||
|
||||
public readonly kind = SyntaxKind.Module;
|
||||
|
||||
public constructor(
|
||||
public pubKeyword: PubKeyword | null,
|
||||
public modKeyword: ModKeyword,
|
||||
public name: Identifier,
|
||||
public body: Body,
|
||||
|
|
@ -821,6 +1294,17 @@ export class Module extends SyntaxBase {
|
|||
super();
|
||||
}
|
||||
|
||||
public getFirstToken(): Token {
|
||||
if (this.pubKeyword !== null) {
|
||||
return this.pubKeyword;
|
||||
}
|
||||
return this.modKeyword;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
return this.body.getLastToken();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
export type SourceFileElement
|
||||
|
|
@ -833,15 +1317,24 @@ export class SourceFile extends SyntaxBase {
|
|||
public readonly kind = SyntaxKind.SourceFile;
|
||||
|
||||
public constructor(
|
||||
public elements: SourceFileElement[]
|
||||
public elements: SourceFileElement[],
|
||||
public eof: EndOfFile,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public *getChildNodes(): Iterable<Syntax> {
|
||||
for (const element in this.elements) {
|
||||
yield element;
|
||||
public getFirstToken(): Token {
|
||||
if (this.elements.length > 0) {
|
||||
return this.elements[0].getFirstToken();
|
||||
}
|
||||
return this.eof;
|
||||
}
|
||||
|
||||
public getLastToken(): Token {
|
||||
if (this.elements.length > 0) {
|
||||
return this.elements[this.elements.length-1].getLastToken();
|
||||
}
|
||||
return this.eof;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
220
src/parser.ts
220
src/parser.ts
|
|
@ -1,5 +1,5 @@
|
|||
|
||||
import { kMaxLength } from "buffer";
|
||||
import { privateDecrypt } from "crypto";
|
||||
import {
|
||||
ReferenceTypeExpression,
|
||||
SourceFile,
|
||||
|
|
@ -19,7 +19,6 @@ import {
|
|||
PrefixExpression,
|
||||
ExpressionStatement,
|
||||
ImportDeclaration,
|
||||
FunctionDeclaration,
|
||||
Param,
|
||||
Pattern,
|
||||
BindPattern,
|
||||
|
|
@ -27,6 +26,15 @@ import {
|
|||
TypeAssert,
|
||||
ExprBody,
|
||||
BlockBody,
|
||||
QualifiedName,
|
||||
NestedExpression,
|
||||
NamedTuplePattern,
|
||||
StructPattern,
|
||||
VariadicStructPatternElement,
|
||||
PunnedFieldStructPatternElement,
|
||||
FieldStructPatternElement,
|
||||
TuplePattern,
|
||||
InfixExpression,
|
||||
} from "./cst"
|
||||
import { Stream, MultiDict } from "./util";
|
||||
|
||||
|
|
@ -91,28 +99,60 @@ function isConstructor(token: Token): boolean {
|
|||
&& token.text[0].toUpperCase() === token.text[0];
|
||||
}
|
||||
|
||||
function isBinaryOperatorLike(token: Token): boolean {
|
||||
return token.kind === SyntaxKind.CustomOperator;
|
||||
}
|
||||
|
||||
function isPrefixOperatorLike(token: Token): boolean {
|
||||
return token.kind === SyntaxKind.CustomOperator;
|
||||
}
|
||||
|
||||
const enum OperatorMode {
|
||||
None = 0,
|
||||
Prefix = 1,
|
||||
InfixL = 2,
|
||||
InfixR = 4,
|
||||
Infix = 6,
|
||||
Suffix = 8,
|
||||
}
|
||||
|
||||
interface OperatorInfo {
|
||||
name: string,
|
||||
mode: OperatorMode,
|
||||
precedence?: number,
|
||||
precedence: number,
|
||||
}
|
||||
|
||||
const EXPR_OPERATOR_TABLE: Array<[string, OperatorMode, number]> = [
|
||||
["**", OperatorMode.InfixR, 11],
|
||||
["*", OperatorMode.InfixL, 8],
|
||||
["/", OperatorMode.InfixL, 8],
|
||||
["+", OperatorMode.InfixL, 7],
|
||||
["-", OperatorMode.InfixL, 7],
|
||||
["<", OperatorMode.InfixL, 6],
|
||||
[">", OperatorMode.InfixL, 6],
|
||||
["<=", OperatorMode.InfixL, 5],
|
||||
[">=", OperatorMode.InfixL, 5],
|
||||
["==", OperatorMode.InfixL, 5],
|
||||
["!=", OperatorMode.InfixL, 5],
|
||||
["<*", OperatorMode.InfixL, 4],
|
||||
[":", OperatorMode.InfixL, 3],
|
||||
["<|>", OperatorMode.InfixL, 2],
|
||||
["<?>", OperatorMode.InfixL, 1],
|
||||
["$", OperatorMode.InfixR, 0]
|
||||
];
|
||||
|
||||
export class Parser {
|
||||
|
||||
private exprOperators = new MultiDict<string, OperatorInfo>();
|
||||
private prefixExprOperators = new Set<string>();
|
||||
private binaryExprOperators = new Map<string, OperatorInfo>();
|
||||
private suffixExprOperators = new Set<string>();
|
||||
|
||||
public constructor(
|
||||
public tokens: Stream<Token>,
|
||||
) {
|
||||
|
||||
for (const [name, mode, precedence] of EXPR_OPERATOR_TABLE) {
|
||||
this.binaryExprOperators.set(name, { name, mode, precedence });
|
||||
}
|
||||
}
|
||||
|
||||
private getToken(): Token {
|
||||
|
|
@ -152,20 +192,6 @@ export class Parser {
|
|||
return t0;
|
||||
}
|
||||
|
||||
private isPrefixOperator(token: Token): boolean {
|
||||
const name = token.text;
|
||||
for (const operator of this.exprOperators.get(name)) {
|
||||
if (operator.mode & OperatorMode.Prefix) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private isBinaryOperator(token: Token): boolean {
|
||||
return token.kind === SyntaxKind.CustomOperator;
|
||||
}
|
||||
|
||||
public parseReferenceTypeExpression(): ReferenceTypeExpression {
|
||||
const name = this.expectToken(SyntaxKind.Identifier);
|
||||
return new ReferenceTypeExpression([], name);
|
||||
|
|
@ -190,7 +216,7 @@ export class Parser {
|
|||
return new ConstantExpression(token);
|
||||
}
|
||||
|
||||
public parseReferenceExpression(): ReferenceExpression {
|
||||
public parseQualifiedName(): QualifiedName {
|
||||
const modulePath: Array<[Identifier, Dot]> = [];
|
||||
let name = this.expectToken(SyntaxKind.Identifier)
|
||||
for (;;) {
|
||||
|
|
@ -201,18 +227,24 @@ export class Parser {
|
|||
modulePath.push([name, t1]);
|
||||
name = this.expectToken(SyntaxKind.Identifier)
|
||||
}
|
||||
return new ReferenceExpression(modulePath, name);
|
||||
return new QualifiedName(modulePath, name);
|
||||
}
|
||||
|
||||
public parseReferenceExpression(): ReferenceExpression {
|
||||
return new ReferenceExpression(this.parseQualifiedName());
|
||||
}
|
||||
|
||||
private parseExpressionWithParens(): Expression {
|
||||
const t0 = this.expectToken(SyntaxKind.LParen)
|
||||
const lparen = this.expectToken(SyntaxKind.LParen)
|
||||
const t1 = this.peekToken();
|
||||
if (t1.kind === SyntaxKind.RParen) {
|
||||
this.getToken();
|
||||
return new TupleExpression(t0, [], t1);
|
||||
}
|
||||
if (isConstructor(t1)) {
|
||||
|
||||
return new TupleExpression(lparen, [], t1);
|
||||
} else if (t1.kind === SyntaxKind.Constructor) {
|
||||
} else {
|
||||
const expression = this.parseExpression();
|
||||
const t2 = this.expectToken(SyntaxKind.RParen);
|
||||
return new NestedExpression(lparen, expression, t2);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -237,36 +269,59 @@ export class Parser {
|
|||
}
|
||||
|
||||
private parseUnaryExpression(): Expression {
|
||||
let out = this.parseExpressionNoOperators()
|
||||
const prefixOperators = [];
|
||||
let result = this.parseExpressionNoOperators()
|
||||
const prefixes = [];
|
||||
for (;;) {
|
||||
const t0 = this.peekToken();
|
||||
if (!this.isPrefixOperator(t0)) {
|
||||
if (!isPrefixOperatorLike(t0)) {
|
||||
break;
|
||||
}
|
||||
prefixOperators.push(t0);
|
||||
if (!this.prefixExprOperators.has(t0.text)) {
|
||||
break;
|
||||
}
|
||||
prefixes.push(t0);
|
||||
this.getToken()
|
||||
}
|
||||
for (let i = prefixOperators.length-1; i >= 0; i--) {
|
||||
const op = prefixOperators[i];
|
||||
out = new PrefixExpression(op, out);
|
||||
for (let i = prefixes.length-1; i >= 0; i--) {
|
||||
const operator = prefixes[i];
|
||||
result = new PrefixExpression(operator, result);
|
||||
}
|
||||
return out;
|
||||
return result;
|
||||
}
|
||||
|
||||
private parseExpressionWithBinaryOperator(lhs: Expression, minPrecedence: number) {
|
||||
private parseBinaryOperatorAfterExpr(lhs: Expression, minPrecedence: number) {
|
||||
for (;;) {
|
||||
const t0 = this.peekToken();
|
||||
if (!this.isBinaryOperator(t0)) {
|
||||
if (!isBinaryOperatorLike(t0)) {
|
||||
break;
|
||||
}
|
||||
const info0 = this.binaryExprOperators.get(t0.text);
|
||||
if (info0 === undefined || info0.precedence < minPrecedence) {
|
||||
break;
|
||||
}
|
||||
this.getToken();
|
||||
let rhs = this.parseUnaryExpression();
|
||||
for (;;) {
|
||||
const t1 = this.peekToken();
|
||||
if (!isBinaryOperatorLike(t1)) {
|
||||
break;
|
||||
}
|
||||
const info1 = this.binaryExprOperators.get(t1.text);
|
||||
if (info1 === undefined
|
||||
|| info1.precedence < info0.precedence
|
||||
|| (info1.precedence === info0.precedence && (info1.mode & OperatorMode.InfixR) === 0)) {
|
||||
break;
|
||||
}
|
||||
rhs = this.parseBinaryOperatorAfterExpr(rhs, info0.precedence);
|
||||
}
|
||||
lhs = new InfixExpression(lhs, t0, rhs);
|
||||
}
|
||||
return lhs;
|
||||
}
|
||||
|
||||
public parseExpression(): Expression {
|
||||
const lhs = this.parseUnaryExpression();
|
||||
return this.parseExpressionWithBinaryOperator(lhs, 0);
|
||||
return this.parseBinaryOperatorAfterExpr(lhs, 0);
|
||||
}
|
||||
|
||||
public parseStructDeclaration(): StructDeclaration {
|
||||
|
|
@ -290,9 +345,96 @@ export class Parser {
|
|||
return new StructDeclaration(structKeyword, name, members);
|
||||
}
|
||||
|
||||
private parsePatternStartingWithConstructor() {
|
||||
const name = this.expectToken(SyntaxKind.Constructor);
|
||||
const t2 = this.peekToken();
|
||||
if (t2.kind === SyntaxKind.LBrace) {
|
||||
this.getToken();
|
||||
const fields = [];
|
||||
let rbrace;
|
||||
for (;;) {
|
||||
const t3 = this.peekToken();
|
||||
if (t3.kind === SyntaxKind.RBrace) {
|
||||
rbrace = t3;
|
||||
break;
|
||||
} else if (t3.kind === SyntaxKind.Identifier) {
|
||||
this.getToken();
|
||||
const t4 = this.peekToken();
|
||||
if (t4.kind === SyntaxKind.Equals) {
|
||||
this.getToken();
|
||||
const pattern = this.parsePattern();
|
||||
fields.push(new FieldStructPatternElement(t3, t4, pattern));
|
||||
} else {
|
||||
fields.push(new PunnedFieldStructPatternElement(t3));
|
||||
}
|
||||
} else if (t3.kind === SyntaxKind.DotDot) {
|
||||
this.getToken();
|
||||
fields.push(new VariadicStructPatternElement(t3, null));
|
||||
} else {
|
||||
this.raiseParseError(t3, [ SyntaxKind.Identifier, SyntaxKind.DotDot ]);
|
||||
}
|
||||
const t5 = this.peekToken();
|
||||
if (t5.kind === SyntaxKind.Comma) {
|
||||
this.getToken();
|
||||
} else if (t5.kind === SyntaxKind.RBrace) {
|
||||
rbrace = t5;
|
||||
break;
|
||||
} else {
|
||||
this.raiseParseError(t5, [ SyntaxKind.Comma, SyntaxKind.RBrace ]);
|
||||
}
|
||||
}
|
||||
return new StructPattern(name, t2, fields, rbrace);
|
||||
} else {
|
||||
const patterns = [];
|
||||
for (;;) {
|
||||
const t3 = this.peekToken();
|
||||
if (t3.kind === SyntaxKind.RParen) {
|
||||
break;
|
||||
}
|
||||
patterns.push(this.parsePattern());
|
||||
}
|
||||
return new NamedTuplePattern(name, patterns);
|
||||
}
|
||||
}
|
||||
|
||||
public parseTuplePattern(): TuplePattern {
|
||||
const lparen = this.expectToken(SyntaxKind.LParen);
|
||||
const elements = [];
|
||||
let rparen;
|
||||
for (;;) {
|
||||
const t1 = this.peekToken();
|
||||
if (t1.kind === SyntaxKind.RParen) {
|
||||
rparen = t1;
|
||||
break;
|
||||
}
|
||||
elements.push(this.parsePattern());
|
||||
const t2 = this.peekToken();
|
||||
if (t2.kind === SyntaxKind.Comma) {
|
||||
this.getToken();
|
||||
} else if (t2.kind === SyntaxKind.RParen) {
|
||||
rparen = t2;
|
||||
break;
|
||||
} else {
|
||||
this.raiseParseError(t2, [ SyntaxKind.Comma, SyntaxKind.RParen ]);
|
||||
}
|
||||
}
|
||||
this.getToken();
|
||||
return new TuplePattern(lparen, elements, rparen);
|
||||
}
|
||||
|
||||
public parsePattern(): Pattern {
|
||||
const t0 = this.peekToken();
|
||||
switch (t0.kind) {
|
||||
case SyntaxKind.LParen:
|
||||
{
|
||||
this.getToken();
|
||||
const t1 = this.peekToken();
|
||||
if (t1.kind === SyntaxKind.Constructor) {
|
||||
return this.parsePatternStartingWithConstructor();
|
||||
} else {
|
||||
return this.parseTuplePattern();
|
||||
}
|
||||
}
|
||||
case SyntaxKind.Identifier:
|
||||
this.getToken();
|
||||
return new BindPattern(t0);
|
||||
|
|
@ -410,15 +552,17 @@ export class Parser {
|
|||
|
||||
public parseSourceFile(): SourceFile {
|
||||
const elements = [];
|
||||
let eof;
|
||||
for (;;) {
|
||||
const t0 = this.peekToken();
|
||||
if (t0.kind === SyntaxKind.EndOfFile) {
|
||||
eof = t0;
|
||||
break;
|
||||
}
|
||||
const element = this.parseSourceFileElement();
|
||||
elements.push(element);
|
||||
}
|
||||
return new SourceFile(elements);
|
||||
return new SourceFile(elements, eof);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,13 +23,20 @@ import {
|
|||
LBracket,
|
||||
RBrace,
|
||||
RBracket,
|
||||
ReturnKeyword,
|
||||
CustomOperator,
|
||||
Constructor,
|
||||
Integer,
|
||||
} from "./cst"
|
||||
import { Diagnostics, UnexpectedCharDiagnostic } from "./diagnostics"
|
||||
import { Stream, BufferedStream } from "./util";
|
||||
import { Stream, BufferedStream, assert } from "./util";
|
||||
|
||||
const EOF = '\uFFFF'
|
||||
|
||||
function isUpper(ch: string): boolean {
|
||||
return ch.toUpperCase() === ch;
|
||||
}
|
||||
|
||||
function isWhiteSpace(ch: string): boolean {
|
||||
return /[\r\n\t ]/.test(ch);
|
||||
}
|
||||
|
|
@ -42,6 +49,16 @@ function isIdentStart(ch: string): boolean {
|
|||
return /[a-zA-Z_]/.test(ch)
|
||||
}
|
||||
|
||||
function isDecimalDigit(ch: string): boolean {
|
||||
return /[0-9]/.test(ch);
|
||||
}
|
||||
|
||||
function toDecimal(ch: string): number {
|
||||
const code = ch.charCodeAt(0);
|
||||
assert(code >= 48 && code <= 57);
|
||||
return code - 48;
|
||||
}
|
||||
|
||||
function isOperatorPart(ch: string): boolean {
|
||||
return /\+-*\/%^&|$<>!?=/.test(ch);
|
||||
}
|
||||
|
|
@ -208,6 +225,37 @@ export class Scanner extends BufferedStream<Token> {
|
|||
}
|
||||
}
|
||||
|
||||
case '0':
|
||||
{
|
||||
const c1 = this.peekChar();
|
||||
switch (c1) {
|
||||
case 'x': // TODO
|
||||
case 'o': // TODO
|
||||
case 'b': // TODO
|
||||
}
|
||||
}
|
||||
case '1':
|
||||
case '2':
|
||||
case '3':
|
||||
case '4':
|
||||
case '5':
|
||||
case '6':
|
||||
case '7':
|
||||
case '8':
|
||||
case '9':
|
||||
{
|
||||
let value = BigInt(toDecimal(c0));
|
||||
for (;;) {
|
||||
const c1 = this.peekChar();
|
||||
if (!isDecimalDigit(c1)) {
|
||||
break;
|
||||
}
|
||||
this.getChar();
|
||||
value = value * BigInt(10) + BigInt(toDecimal(c1));
|
||||
}
|
||||
return new Integer(value, 10, startPos);
|
||||
}
|
||||
|
||||
case 'a':
|
||||
case 'b':
|
||||
case 'c':
|
||||
|
|
@ -272,9 +320,13 @@ export class Scanner extends BufferedStream<Token> {
|
|||
case 'return': return new ReturnKeyword(startPos);
|
||||
case 'type': return new TypeKeyword(startPos);
|
||||
default:
|
||||
if (isUpper(text[0])) {
|
||||
return new Constructor(text, startPos);
|
||||
} else {
|
||||
return new Identifier(text, startPos);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,13 @@
|
|||
|
||||
export function assert(test: boolean): asserts test {
|
||||
if (!test) {
|
||||
throw new Error(`Assertion failed. See the stack trace for more information.`);
|
||||
}
|
||||
}
|
||||
|
||||
export type JSONValue = null | boolean | number | string | JSONArray | JSONObject
|
||||
export type JSONArray = Array<JSONValue>;
|
||||
export type JSONObject = { [key: string]: JSONValue };
|
||||
|
||||
export class MultiDict<K, V> {
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue