samvv/bolt
1
0
Fork 0
Code Issues 24 Pull requests Projects Releases Packages Wiki Activity Actions

Add support for paring sentences

Browse source
This commit is contained in:
Sam Vervaeck 2020-02-24 19:16:33 +01:00
parent c68c6fdf1b
commit d06c8f577d
3 changed files with 183 additions and 53 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

171
src/ast.ts
View file

@ -13,16 +13,25 @@ export enum SyntaxKind {
Identifier,
Operator,
Punctuated,
Semi,
// Special nodes
SourceFile,
QualName,
Sentence,
// Expressions
ConstantExpr,
ReferenceExpr,
// Statements
ReturnStatement,
// Type declarations
TypeReference,
@ -97,9 +106,14 @@ export class TextSpan {
}
export class Literal {
abstract class SyntaxBase {
abstract kind: SyntaxKind;
abstract parentNode: Syntax | null;
}
kind = SyntaxKind.Literal;
export class Literal extends SyntaxBase {
kind: SyntaxKind.Literal = SyntaxKind.Literal;
static META = {
value: EdgeType.Primitive,
@ -107,9 +121,10 @@ export class Literal {
constructor(
public value: string | bigint,
public span: TextSpan
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON(): Json {
@ -127,9 +142,9 @@ export enum PunctType {
Brace,
}
export class Punctuated {
export class Punctuated extends SyntaxBase {
kind = SyntaxKind.Punctuated
kind: SyntaxKind.Punctuated = SyntaxKind.Punctuated
static META = {
punctuator: EdgeType.Primitive,
@ -139,9 +154,10 @@ export class Punctuated {
constructor(
public punctuator: PunctType,
public elements: Token[],
public span: TextSpan
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON(): Json {
@ -154,9 +170,9 @@ export class Punctuated {
}
export class Identifier {
export class Identifier extends SyntaxBase {
kind = SyntaxKind.Identifier;
kind: SyntaxKind.Identifier = SyntaxKind.Identifier;
static META = {
text: EdgeType.Primitive
@ -164,9 +180,10 @@ export class Identifier {
constructor(
public text: string,
public span: TextSpan
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON(): Json {
@ -179,9 +196,9 @@ export class Identifier {
}
export class Operator {
export class Operator extends SyntaxBase {
kind = SyntaxKind.Operator;
kind: SyntaxKind.Operator = SyntaxKind.Operator;
static META = {
text: EdgeType.Primitive
@ -192,7 +209,7 @@ export class Operator {
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON(): Json {
@ -205,15 +222,58 @@ export class Operator {
}
export class Semi extends SyntaxBase {
kind: SyntaxKind.Semi = SyntaxKind.Semi;
constructor(
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON() {
return {
kind: 'Semi',
span: this.span.toJSON(),
}
}
}
export type Token
= Identifier
= Semi
| Identifier
| Operator
| Literal
| Punctuated
export class Sentence extends SyntaxBase {
kind = SyntaxKind.Sentence;
constructor(
public tokens: Token[],
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON(): Json {
return {
kind: 'Sentence',
tokens: this.tokens.map(token => token.toJSON()),
span: this.span.toJSON(),
}
}
}
export class QualName {
kind = SyntaxKind.QualName;
kind: SyntaxKind.QualName = SyntaxKind.QualName;
static META = {
name: EdgeType.Node,
@ -252,9 +312,25 @@ export class ConstantExpr {
export type Expr
= ConstantExpr
class ReturnStatement extends SyntaxBase {
kind: SyntaxKind.ReturnStatement = SyntaxKind.ReturnStatement;
constructor(
public value: Expr | null,
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
}
export type Statement
= ReturnStatement
export class TypeReference {
kind = SyntaxKind.TypeReference;
kind: SyntaxKind.TypeReference = SyntaxKind.TypeReference;
static META = {
name: EdgeType.Node,
@ -275,23 +351,23 @@ export class TypeReference {
export type TypeDecl
= TypeReference
export class Unexpanded {
// export class Unexpanded {
//
// static META = {
// tokens: EdgeType.Node | EdgeType.List
// }
//
// constructor(
// public tokens: Token[],
// public span: TextSpan,
// public parentNode: Syntax | null = null
// ) {
//
// }
//
// }
static META = {
tokens: EdgeType.Node | EdgeType.List
}
constructor(
public tokens: Token[],
public span: TextSpan,
public parentNode: Syntax | null = null
) {
}
}
export class FunctionDecl {
export class FunctionDecl extends SyntaxBase {
kind = SyntaxKind.FunctionDecl;
@ -310,13 +386,13 @@ export class FunctionDecl {
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
}
export class VariableDecl {
export class VariableDecl extends SyntaxBase {
kind = SyntaxKind.VariableDecl;
@ -332,13 +408,13 @@ export class VariableDecl {
public value: Expr | null,
public span: TextSpan
) {
super();
}
}
export type Decl
= Unexpanded
= Sentence
| FunctionDecl
| VariableDecl
@ -348,11 +424,24 @@ export type Syntax
| SourceFile
| QualName
export class SourceFile {
export class SourceFile extends SyntaxBase {
constructor(public elements: Decl[], public span: TextSpan) {
kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile;
}
constructor(
public elements: (Decl | Statement)[],
public span: TextSpan,
public parentNode: Syntax | null = null
) {
super();
}
toJSON() {
return {
elements: this.elements.map(element => element.toJSON()),
span: this.span.toJSON(),
}
}
}

24
src/bin/bolt.ts
View file

@ -7,7 +7,7 @@ import * as fs from "fs"
import yargs from "yargs"
import { Scanner } from "../scanner"
import { Token, TextFile } from "../ast"
import { Token, TextFile, SourceFile } from "../ast"
function toArray<T>(value: T): T extends Array<any> ? T : T[] {
if (Array.isArray(value)) {
@ -63,13 +63,13 @@ yargs
args => {
const hooks: Hook[] = toArray(args.hook as string[] | string).map(parseHook);
const sourceFiles: SourceFile[] = [];
for (const filepath of toArray(args.files as string[] | string)) {
const file = new TextFile(filepath);
const content = fs.readFileSync(filepath, 'utf8')
const scanner = new Scanner(file, content)
const tokens: Token[] = [];
for (const hook of hooks) {
if (hook.name === 'scan' && hook.timing === 'before') {
@ -85,20 +85,21 @@ yargs
}
}
while (true) {
const token = scanner.scanToken()
if (token === null) {
break;
}
tokens.push(token);
}
const sourceFile = scanner.scan();
// while (true) {
// const token = scanner.scanToken()
// if (token === null) {
// break;
// }
// tokens.push(token);
// }
for (const hook of hooks) {
if (hook.name === 'scan' && hook.timing == 'after') {
for (const effect of hook.effects) {
switch (effect) {
case 'dump':
console.log(JSON.stringify(tokens.map(t => t.toJSON()), undefined, 2));
console.log(JSON.stringify(sourceFile.toJSON(), undefined, 2));
break;
case 'abort':
process.exit(0);
@ -110,7 +111,10 @@ yargs
}
}
sourceFiles.push(sourceFile);
}
})
.help()

41
src/scanner.ts
View file

@ -2,6 +2,7 @@
import XRegExp from "xregexp"
import {
SyntaxKind,
TextFile,
TextPos,
TextSpan,
@ -9,7 +10,10 @@ import {
Operator,
PunctType,
Token,
Decl,
Punctuated,
Sentence,
SourceFile
} from "./ast"
function escapeChar(ch: string) {
@ -168,7 +172,7 @@ export class Scanner {
return text;
}
scanToken() {
scanToken(): Token | null {
while (true) {
@ -248,7 +252,40 @@ export class Scanner {
}
scanTokenList() {
scan() {
const elements: Decl[] = []
const startPos = this.currPos.clone()
while (true) {
const tokens: Token[] = [];
while (true) {
const token = this.scanToken();
if (token === null) {
break;
}
if (token.kind === SyntaxKind.Semi) {
break;
}
tokens.push(token)
if (token.kind === SyntaxKind.Punctuated && token.punctuator === PunctType.Brace) {
break;
}
}
if (tokens.length === 0) {
break;
}
elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone())))
}
const endPos = this.currPos.clone();
return new SourceFile(elements, new TextSpan(this.file, startPos, endPos))
}