From d06c8f577d9518e66caa526bb04bfaafc9bda2f8 Mon Sep 17 00:00:00 2001 From: Sam Vervaeck Date: Mon, 24 Feb 2020 19:16:33 +0100 Subject: [PATCH] Add support for paring sentences --- src/ast.ts | 171 ++++++++++++++++++++++++++++++++++++------------ src/bin/bolt.ts | 24 ++++--- src/scanner.ts | 41 +++++++++++- 3 files changed, 183 insertions(+), 53 deletions(-) diff --git a/src/ast.ts b/src/ast.ts index 167c739b3..de9fb9974 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -13,16 +13,25 @@ export enum SyntaxKind { Identifier, Operator, Punctuated, + Semi, + + // Special nodes SourceFile, QualName, + Sentence, + // Expressions ConstantExpr, ReferenceExpr, + // Statements + + ReturnStatement, + // Type declarations TypeReference, @@ -97,9 +106,14 @@ export class TextSpan { } -export class Literal { +abstract class SyntaxBase { + abstract kind: SyntaxKind; + abstract parentNode: Syntax | null; +} - kind = SyntaxKind.Literal; +export class Literal extends SyntaxBase { + + kind: SyntaxKind.Literal = SyntaxKind.Literal; static META = { value: EdgeType.Primitive, @@ -107,9 +121,10 @@ export class Literal { constructor( public value: string | bigint, - public span: TextSpan + public span: TextSpan, + public parentNode: Syntax | null = null ) { - + super(); } toJSON(): Json { @@ -127,9 +142,9 @@ export enum PunctType { Brace, } -export class Punctuated { +export class Punctuated extends SyntaxBase { - kind = SyntaxKind.Punctuated + kind: SyntaxKind.Punctuated = SyntaxKind.Punctuated static META = { punctuator: EdgeType.Primitive, @@ -139,9 +154,10 @@ export class Punctuated { constructor( public punctuator: PunctType, public elements: Token[], - public span: TextSpan + public span: TextSpan, + public parentNode: Syntax | null = null ) { - + super(); } toJSON(): Json { @@ -154,9 +170,9 @@ export class Punctuated { } -export class Identifier { +export class Identifier extends SyntaxBase { - kind = SyntaxKind.Identifier; + kind: SyntaxKind.Identifier = SyntaxKind.Identifier; static META = { text: EdgeType.Primitive @@ -164,9 +180,10 @@ export class Identifier { constructor( public text: string, - public span: TextSpan + public span: TextSpan, + public parentNode: Syntax | null = null ) { - + super(); } toJSON(): Json { @@ -179,9 +196,9 @@ export class Identifier { } -export class Operator { +export class Operator extends SyntaxBase { - kind = SyntaxKind.Operator; + kind: SyntaxKind.Operator = SyntaxKind.Operator; static META = { text: EdgeType.Primitive @@ -192,7 +209,7 @@ export class Operator { public span: TextSpan, public parentNode: Syntax | null = null ) { - + super(); } toJSON(): Json { @@ -205,15 +222,58 @@ export class Operator { } +export class Semi extends SyntaxBase { + + kind: SyntaxKind.Semi = SyntaxKind.Semi; + + constructor( + public span: TextSpan, + public parentNode: Syntax | null = null + ) { + super(); + } + + toJSON() { + return { + kind: 'Semi', + span: this.span.toJSON(), + } + } + +} + export type Token - = Identifier + = Semi + | Identifier | Operator | Literal | Punctuated +export class Sentence extends SyntaxBase { + + kind = SyntaxKind.Sentence; + + constructor( + public tokens: Token[], + public span: TextSpan, + public parentNode: Syntax | null = null + ) { + super(); + } + + toJSON(): Json { + return { + kind: 'Sentence', + tokens: this.tokens.map(token => token.toJSON()), + span: this.span.toJSON(), + } + } + +} + export class QualName { - kind = SyntaxKind.QualName; + kind: SyntaxKind.QualName = SyntaxKind.QualName; static META = { name: EdgeType.Node, @@ -252,9 +312,25 @@ export class ConstantExpr { export type Expr = ConstantExpr +class ReturnStatement extends SyntaxBase { + + kind: SyntaxKind.ReturnStatement = SyntaxKind.ReturnStatement; + + constructor( + public value: Expr | null, + public span: TextSpan, + public parentNode: Syntax | null = null + ) { + super(); + } +} + +export type Statement + = ReturnStatement + export class TypeReference { - kind = SyntaxKind.TypeReference; + kind: SyntaxKind.TypeReference = SyntaxKind.TypeReference; static META = { name: EdgeType.Node, @@ -275,23 +351,23 @@ export class TypeReference { export type TypeDecl = TypeReference -export class Unexpanded { +// export class Unexpanded { +// +// static META = { +// tokens: EdgeType.Node | EdgeType.List +// } +// +// constructor( +// public tokens: Token[], +// public span: TextSpan, +// public parentNode: Syntax | null = null +// ) { +// +// } +// +// } - static META = { - tokens: EdgeType.Node | EdgeType.List - } - - constructor( - public tokens: Token[], - public span: TextSpan, - public parentNode: Syntax | null = null - ) { - - } - -} - -export class FunctionDecl { +export class FunctionDecl extends SyntaxBase { kind = SyntaxKind.FunctionDecl; @@ -310,13 +386,13 @@ export class FunctionDecl { public span: TextSpan, public parentNode: Syntax | null = null ) { - + super(); } } -export class VariableDecl { +export class VariableDecl extends SyntaxBase { kind = SyntaxKind.VariableDecl; @@ -332,13 +408,13 @@ export class VariableDecl { public value: Expr | null, public span: TextSpan ) { - + super(); } } export type Decl - = Unexpanded + = Sentence | FunctionDecl | VariableDecl @@ -348,11 +424,24 @@ export type Syntax | SourceFile | QualName -export class SourceFile { +export class SourceFile extends SyntaxBase { - constructor(public elements: Decl[], public span: TextSpan) { + kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile; - } + constructor( + public elements: (Decl | Statement)[], + public span: TextSpan, + public parentNode: Syntax | null = null + ) { + super(); + } + + toJSON() { + return { + elements: this.elements.map(element => element.toJSON()), + span: this.span.toJSON(), + } + } } diff --git a/src/bin/bolt.ts b/src/bin/bolt.ts index 8dcb22251..877c552a5 100644 --- a/src/bin/bolt.ts +++ b/src/bin/bolt.ts @@ -7,7 +7,7 @@ import * as fs from "fs" import yargs from "yargs" import { Scanner } from "../scanner" -import { Token, TextFile } from "../ast" +import { Token, TextFile, SourceFile } from "../ast" function toArray(value: T): T extends Array ? T : T[] { if (Array.isArray(value)) { @@ -63,13 +63,13 @@ yargs args => { const hooks: Hook[] = toArray(args.hook as string[] | string).map(parseHook); + const sourceFiles: SourceFile[] = []; for (const filepath of toArray(args.files as string[] | string)) { const file = new TextFile(filepath); const content = fs.readFileSync(filepath, 'utf8') const scanner = new Scanner(file, content) - const tokens: Token[] = []; for (const hook of hooks) { if (hook.name === 'scan' && hook.timing === 'before') { @@ -85,20 +85,21 @@ yargs } } - while (true) { - const token = scanner.scanToken() - if (token === null) { - break; - } - tokens.push(token); - } + const sourceFile = scanner.scan(); + // while (true) { + // const token = scanner.scanToken() + // if (token === null) { + // break; + // } + // tokens.push(token); + // } for (const hook of hooks) { if (hook.name === 'scan' && hook.timing == 'after') { for (const effect of hook.effects) { switch (effect) { case 'dump': - console.log(JSON.stringify(tokens.map(t => t.toJSON()), undefined, 2)); + console.log(JSON.stringify(sourceFile.toJSON(), undefined, 2)); break; case 'abort': process.exit(0); @@ -110,7 +111,10 @@ yargs } } + sourceFiles.push(sourceFile); + } + }) .help() diff --git a/src/scanner.ts b/src/scanner.ts index fcbcf76bd..0d51a89a0 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -2,6 +2,7 @@ import XRegExp from "xregexp" import { + SyntaxKind, TextFile, TextPos, TextSpan, @@ -9,7 +10,10 @@ import { Operator, PunctType, Token, + Decl, Punctuated, + Sentence, + SourceFile } from "./ast" function escapeChar(ch: string) { @@ -168,7 +172,7 @@ export class Scanner { return text; } - scanToken() { + scanToken(): Token | null { while (true) { @@ -248,7 +252,40 @@ export class Scanner { } - scanTokenList() { + scan() { + + const elements: Decl[] = [] + const startPos = this.currPos.clone() + + while (true) { + + const tokens: Token[] = []; + + while (true) { + const token = this.scanToken(); + if (token === null) { + break; + } + if (token.kind === SyntaxKind.Semi) { + break; + } + tokens.push(token) + if (token.kind === SyntaxKind.Punctuated && token.punctuator === PunctType.Brace) { + break; + } + } + + if (tokens.length === 0) { + break; + } + + elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone()))) + + } + + const endPos = this.currPos.clone(); + + return new SourceFile(elements, new TextSpan(this.file, startPos, endPos)) }