From c421721766f8706cbe4776d60c57985906a9e98c Mon Sep 17 00:00:00 2001 From: Sam Vervaeck Date: Tue, 25 Feb 2020 17:55:17 +0100 Subject: [PATCH] Add a very naive compiler --- .gitignore | 3 + package-lock.json | 43 ++++++ package.json | 4 + src/ast.ts | 385 ++++++++++++++++++++++++++++++++-------------- src/bin/bolt.ts | 57 ++++++- src/checker.ts | 58 +++++++ src/compiler.ts | 125 +++++++++++++++ src/emitter.ts | 30 ++++ src/expander.ts | 5 +- src/parser.ts | 113 +++++++++++--- src/scanner.ts | 98 ++++++++---- 11 files changed, 746 insertions(+), 175 deletions(-) create mode 100644 src/checker.ts create mode 100644 src/compiler.ts create mode 100644 src/emitter.ts diff --git a/.gitignore b/.gitignore index 11e8db48c..2e2aef008 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ node_modules/ # local development files Makefile +# bolt +.bolt-work/ + diff --git a/package-lock.json b/package-lock.json index 22e5edca5..70828f5bf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.4.tgz", "integrity": "sha512-oVeL12C6gQS/GAExndigSaLxTrKpQPxewx9bOcwfvJiJge4rr7wNaph4J+ns5hrmIV2as5qxqN8YKthn9qh0jw==" }, + "@types/xregexp": { + "version": "3.0.30", + "resolved": "https://registry.npmjs.org/@types/xregexp/-/xregexp-3.0.30.tgz", + "integrity": "sha512-u1dpabg81Rd660bYebOqMXO0+E63H1hxunPAWGebNb7TpxqZYe9YaVLgkkj6ZnzLs3yLumtVB956o8u8OZdhXw==" + }, "@types/yargs": { "version": "15.0.3", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.3.tgz", @@ -48,6 +53,11 @@ "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-15.0.0.tgz", "integrity": "sha512-FA/BWv8t8ZWJ+gEOnLLd8ygxH/2UFbAvgEonyfN6yWGLKc7zVjbpl2Y4CTjid9h2RfgPP6SEt6uHwEOply00yw==" }, + "acorn": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz", + "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ==" + }, "ansi-colors": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", @@ -94,6 +104,11 @@ "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "dev": true }, + "astring": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/astring/-/astring-1.4.3.tgz", + "integrity": "sha512-yJlJU/bmN820vL+cbWShu2YQU87dBP5V7BH2N4wODapRv27A2dZtUD0LgjP9lZENvPe9XRoSyWx+pZR6qKqNBw==" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -374,6 +389,16 @@ "is-buffer": "~2.0.3" } }, + "fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -425,6 +450,11 @@ "is-glob": "^4.0.1" } }, + "graceful-fs": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz", + "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ==" + }, "growl": { "version": "1.10.5", "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", @@ -560,6 +590,14 @@ "esprima": "^4.0.0" } }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, "locate-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", @@ -971,6 +1009,11 @@ "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", "dev": true }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/package.json b/package.json index a398fc965..8511aa746 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,11 @@ "repository": "https://github.com/samvv/Bolt", "dependencies": { "@types/node": "^13.7.4", + "@types/xregexp": "^3.0.30", "@types/yargs": "^15.0.3", + "acorn": "^7.1.0", + "astring": "^1.4.3", + "fs-extra": "^8.1.0", "glob": "^7.1.6", "pegjs": "^0.11.0-master.b7b87ea", "reflect-metadata": "^0.1.13", diff --git a/src/ast.ts b/src/ast.ts index eb2eff467..3e95ff3be 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -1,5 +1,8 @@ +// FIXME SyntaxBase.getSpan() does not work then [n1, n2] is given as origNode + import { Stream, StreamWrapper } from "./util" +import { Scanner } from "./scanner" interface JsonArray extends Array { }; interface JsonObject { [key: string]: Json } @@ -7,23 +10,12 @@ type Json = null | string | boolean | number | JsonArray | JsonObject; export type TokenStream = Stream; -function jsonify(value: any): Json { - if (value === null || typeof value === 'string' || typeof value === 'number') { - return value; - } else if (Array.isArray(value)) { - return value.map(element => jsonify(element)); - } else if (typeof value === 'object' && 'toJSON' in value) { - return value.toJSON(); - } else { - throw new Error(`I don't know how to convert ${value} to a JSON representation`) - } -} - export enum SyntaxKind { // Tokens - Literal, + StringLiteral, + IntegerLiteral, Identifier, Operator, Parenthesized, @@ -36,6 +28,13 @@ export enum SyntaxKind { RArrow, EqSign, + // Keywords + + FunctionKeyword, + ForeignKeyword, + LetKeyword, + ImportKeyword, + // Special nodes SourceFile, @@ -55,6 +54,7 @@ export enum SyntaxKind { // Expressions + CallExpr, ConstExpr, RefExpr, @@ -70,6 +70,7 @@ export enum SyntaxKind { VarDecl, FuncDecl, + ForeignDecl, } @@ -143,7 +144,7 @@ abstract class SyntaxBase { abstract parentNode: Syntax | null; abstract span: TextSpan | null; - getSpan() { + getSpan(): TextSpan { let curr: Syntax | null = this as any as Syntax; @@ -151,25 +152,27 @@ abstract class SyntaxBase { if (curr.span !== null ) { return curr.span; } - curr = curr.origNode + curr = curr.origNode; } while (curr !== null) throw new Error(`No TextSpan object found in this node or any of its originating nodes.`); } -} - -export class Literal extends SyntaxBase { - - kind: SyntaxKind.Literal = SyntaxKind.Literal; - - static META = { - value: EdgeType.Primitive, + getFile(): TextFile { + return this.getSpan().file; } + abstract getChildren(): IterableIterator; + +} + +export class StringLiteral extends SyntaxBase { + + kind: SyntaxKind.StringLiteral = SyntaxKind.StringLiteral; + constructor( - public value: string | bigint, + public value: string, public span: TextSpan | null = null, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -179,20 +182,51 @@ export class Literal extends SyntaxBase { toJSON(): Json { return { - value: typeof this.value === 'bigint' ? { type: 'bigint', value: this.value.toString() } : this.value, + value: this.value, span: this.span !== null ? this.span.toJSON() : null, } } + *getChildren(): IterableIterator { + + } + } +export class IntegerLiteral extends SyntaxBase { + + kind: SyntaxKind.IntegerLiteral = SyntaxKind.IntegerLiteral; + + constructor( + public value: string, + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + toJSON(): Json { + return { + value: ['bigint', this.value.toString()], + span: this.span !== null ? this.span.toJSON() : null, + } + } + + *getChildren(): IterableIterator { + + } + +} + + export enum PunctType { Paren, Bracket, Brace, } -class EOS extends SyntaxBase { +export class EOS extends SyntaxBase { kind: SyntaxKind.EOS = SyntaxKind.EOS; @@ -210,18 +244,20 @@ class EOS extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export class Parenthesized extends SyntaxBase { kind: SyntaxKind.Parenthesized = SyntaxKind.Parenthesized; - static META = { - elements: EdgeType.Node | EdgeType.List - } + protected buffered = null; constructor( - public elements: Token[], + public text: string, public span: TextSpan, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -229,21 +265,24 @@ export class Parenthesized extends SyntaxBase { super(); } - toStream() { - return new StreamWrapper( - this.elements, - () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone())) - ); + toTokenStream() { + const span = this.getSpan(); + const startPos = span.start; + return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1)); } toJSON(): Json { return { kind: 'Parenthesized', - elements: this.elements.map(element => element.toJSON()), + text: this.text, span: this.span !== null ? this.span.toJSON() : null, } } + *getChildren(): IterableIterator { + + } + } @@ -251,12 +290,8 @@ export class Braced extends SyntaxBase { kind: SyntaxKind.Braced = SyntaxKind.Braced; - static META = { - elements: EdgeType.Node | EdgeType.List - } - constructor( - public elements: Token[], + public text: string, public span: TextSpan, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -264,33 +299,32 @@ export class Braced extends SyntaxBase { super(); } - toStream() { - return new StreamWrapper( - this.elements, - () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone())) - ); + toTokenStream() { + const span = this.getSpan(); + const startPos = span.start; + return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1)); } toJSON(): Json { return { kind: 'Braced', - elements: this.elements.map(element => element.toJSON()), + text: this.text, span: this.span !== null ? this.span.toJSON() : null, } } + *getChildren(): IterableIterator { + + } + } export class Bracketed extends SyntaxBase { kind: SyntaxKind.Bracketed = SyntaxKind.Bracketed; - static META = { - elements: EdgeType.Node | EdgeType.List - } - constructor( - public elements: Token[], + public text: string, public span: TextSpan, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -298,31 +332,30 @@ export class Bracketed extends SyntaxBase { super(); } - toStream() { - return new StreamWrapper( - this.elements, - () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone())) - ); + toTokenStream() { + const span = this.getSpan(); + const startPos = span.start; + return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1)); } toJSON(): Json { return { kind: 'Bracketed', - elements: this.elements.map(element => element.toJSON()), + text: this.text, span: this.span !== null ? this.span.toJSON() : null, } } + *getChildren(): IterableIterator { + + } + } export class Identifier extends SyntaxBase { kind: SyntaxKind.Identifier = SyntaxKind.Identifier; - static META = { - text: EdgeType.Primitive - } - constructor( public text: string, public span: TextSpan | null = null, @@ -340,16 +373,16 @@ export class Identifier extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export class Operator extends SyntaxBase { kind: SyntaxKind.Operator = SyntaxKind.Operator; - static META = { - text: EdgeType.Primitive - } - constructor( public text: string, public span: TextSpan | null = null, @@ -367,6 +400,10 @@ export class Operator extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export class Semi extends SyntaxBase { @@ -388,6 +425,10 @@ export class Semi extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export class Colon extends SyntaxBase { @@ -409,6 +450,10 @@ export class Colon extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export class Comma extends SyntaxBase { @@ -430,6 +475,10 @@ export class Comma extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } @@ -452,6 +501,10 @@ export class RArrow extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } @@ -475,6 +528,10 @@ export class EqSign extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export class Dot extends SyntaxBase { @@ -496,6 +553,10 @@ export class Dot extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + } export type Token @@ -508,7 +569,8 @@ export type Token | EOS | Identifier | Operator - | Literal + | StringLiteral + | IntegerLiteral | Parenthesized | Braced | Bracketed @@ -526,7 +588,7 @@ export class Sentence extends SyntaxBase { super(); } - toStream() { + toTokenStream() { return new StreamWrapper( this.tokens, () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone())) @@ -541,17 +603,18 @@ export class Sentence extends SyntaxBase { } } + *getChildren(): IterableIterator { + for (const token of this.tokens) { + yield token; + } + } + } export class QualName { kind: SyntaxKind.QualName = SyntaxKind.QualName; - static META = { - name: EdgeType.Node, - path: EdgeType.Node | EdgeType.List, - } - constructor( public name: Identifier | Operator, public path: Identifier[], @@ -571,6 +634,13 @@ export class QualName { } } + *getChildren(): IterableIterator { + for (const chunk of this.path) { + yield chunk + } + yield this.name + } + } export class Param extends SyntaxBase { @@ -598,6 +668,16 @@ export class Param extends SyntaxBase { } } + *getChildren() { + yield this.bindings + if (this.typeDecl !== null) { + yield this.typeDecl + } + if (this.defaultValue !== null) { + yield this.defaultValue + } + } + } export class BindPatt extends SyntaxBase { @@ -621,6 +701,10 @@ export class BindPatt extends SyntaxBase { } } + *getChildren(): IterableIterator { + yield this.name + } + } export type Patt @@ -647,16 +731,48 @@ export class RefExpr extends SyntaxBase { } } + *getChildren(): IterableIterator { + yield this.name + } + +} + +export class CallExpr extends SyntaxBase { + + kind: SyntaxKind.CallExpr = SyntaxKind.CallExpr; + + constructor( + public operator: Expr, + public args: Expr[], + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + toJSON(): Json { + return { + kind: 'CallExpr', + operator: this.operator.toJSON(), + args: this.args.map(a => a.toJSON()), + span: this.span !== null ? this.span.toJSON() : null, + } + } + + *getChildren(): IterableIterator { + yield this.operator + for (const arg of this.args) { + yield arg + } + } + } export class ConstExpr extends SyntaxBase { kind: SyntaxKind.ConstExpr = SyntaxKind.ConstExpr; - static META = { - value: EdgeType.Primitive, - } - constructor( public value: string | bigint, public span: TextSpan | null = null, @@ -674,11 +790,17 @@ export class ConstExpr extends SyntaxBase { } } + *getChildren(): IterableIterator { + + } + + } export type Expr = ConstExpr | RefExpr + | CallExpr export class RetStmt extends SyntaxBase { @@ -701,6 +823,12 @@ export class RetStmt extends SyntaxBase { } } + *getChildren(): IterableIterator { + if (this.value !== null) { + yield this.value + } + } + } export type Stmt @@ -710,14 +838,9 @@ export class TypeRef extends SyntaxBase { kind: SyntaxKind.TypeRef = SyntaxKind.TypeRef; - static META = { - name: EdgeType.Node, - args: EdgeType.Node | EdgeType.List, - } - constructor( public name: QualName, - public args: TypeDecl[], + public typeArgs: TypeDecl[], public span: TextSpan | null = null, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -729,44 +852,29 @@ export class TypeRef extends SyntaxBase { return { kind: 'TypeRef', name: this.name.toJSON(), - args: this.args.map(a => a.toJSON()), + args: this.typeArgs.map(a => a.toJSON()), span: this.span !== null ? this.span.toJSON() : null, } } + *getChildren(): IterableIterator { + yield this.name + for (const arg of this.typeArgs) { + yield arg + } + } + } export type TypeDecl = TypeRef -// export class Unexpanded { -// -// static META = { -// tokens: EdgeType.Node | EdgeType.List -// } -// -// constructor( -// public tokens: Token[], -// public span: TextSpan, -// public parentNode: Syntax | null = null -// ) { -// -// } -// -// } - export class FuncDecl extends SyntaxBase { kind: SyntaxKind.FuncDecl = SyntaxKind.FuncDecl; - static META = { - name: EdgeType.Node, - params: EdgeType.Node | EdgeType.List, - returnType: EdgeType.Node | EdgeType.Nullable, - body: EdgeType.Node | EdgeType.List, - } - constructor( + public target: string, public name: QualName, public params: Param[], public returnType: TypeDecl | null, @@ -781,6 +889,7 @@ export class FuncDecl extends SyntaxBase { toJSON(): Json { return { kind: 'FuncDecl', + target: this.target, name: this.name.toJSON(), params: this.params.map(p => p.toJSON()), returnType: this.returnType !== null ? this.returnType.toJSON() : null, @@ -789,18 +898,27 @@ export class FuncDecl extends SyntaxBase { } } + *getChildren(): IterableIterator { + yield this.name + for (const param of this.params) { + yield param + } + if (this.returnType !== null) { + yield this.returnType; + } + if (this.body !== null) { + for (const stmt of this.body) { + yield stmt + } + } + } + } export class VarDecl extends SyntaxBase { kind: SyntaxKind.VarDecl = SyntaxKind.VarDecl; - static META = { - bindings: EdgeType.Node, - typeDecl: EdgeType.Node | EdgeType.Nullable, - value: EdgeType.Node | EdgeType.Nullable, - } - constructor( public bindings: Patt, public typeDecl: TypeDecl | null, @@ -814,7 +932,7 @@ export class VarDecl extends SyntaxBase { toJSON(): Json { return { - type: 'VarDecl', + kind: 'VarDecl', bindings: this.bindings.toJSON(), typeDecl: this.typeDecl !== null ? this.typeDecl.toJSON() : null, value: this.value !== null ? this.value.toJSON() : null, @@ -822,6 +940,16 @@ export class VarDecl extends SyntaxBase { } } + *getChildren(): IterableIterator { + yield this.bindings + if (this.typeDecl !== null) { + yield this.typeDecl + } + if (this.value !== null) { + yield this.value; + } + } + } export type Decl @@ -833,6 +961,9 @@ export type Syntax = Decl | Expr | Token + | Stmt + | Patt + | TypeDecl | SourceFile | QualName | Param @@ -843,7 +974,7 @@ export class SourceFile extends SyntaxBase { kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile; constructor( - public elements: (Decl | Stmt)[], + public elements: (Decl | Stmt | Expr)[], public span: TextSpan | null = null, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -859,5 +990,29 @@ export class SourceFile extends SyntaxBase { } } + *getChildren(): IterableIterator { + for (const element of this.elements) { + yield element + } + } + +} + +export function isExpr(node: Syntax): node is Expr { + return node.kind === SyntaxKind.ConstExpr || node.kind === SyntaxKind.CallExpr; +} + +export function isJSNode(node: Syntax) { + return typeof node.type === 'string' +} + +export function setParents(node: Syntax) { + if (isJSNode(node)) { + return; + } + for (const child of node.getChildren()) { + child.parentNode = node + setParents(child) + } } diff --git a/src/bin/bolt.ts b/src/bin/bolt.ts index bbff75e16..79c5d4970 100644 --- a/src/bin/bolt.ts +++ b/src/bin/bolt.ts @@ -3,16 +3,21 @@ import "reflect-metadata" import "source-map-support/register" -import * as fs from "fs" +import * as path from "path" +import * as fs from "fs-extra" +import { spawnSync } from "child_process" import yargs from "yargs" import { Scanner } from "../scanner" import { Parser } from "../parser" import { Expander } from "../expander" -import { Token, TextFile, SourceFile } from "../ast" +import { TypeChecker } from "../checker" +import { Compiler } from "../compiler" +import { Emitter } from "../emitter" +import { TextFile, SourceFile, setParents } from "../ast" -function toArray(value: T): T extends Array ? T : T[] { +function toArray(value: T | T[]): T[] { if (Array.isArray(value)) { return value as T[] } @@ -25,6 +30,11 @@ function pushAll(array: T[], elements: T[]) { } } +function stripExtension(filepath: string) { + const i = filepath.lastIndexOf('.'); + return i !== -1 ? filepath.substring(0, i) : filepath +} + function flatMap(array: T[], proc: (element: T) => T[]) { let out: T[] = [] for (const element of array) { @@ -54,6 +64,7 @@ function parseHook(str: string): Hook { } yargs + .command( 'compile [files..]', @@ -144,6 +155,46 @@ yargs }) + .command( + + 'exec [files..]', + 'Run the specified Bolt scripts', + + yargs => + yargs, + + args => { + + const parser = new Parser() + + const sourceFiles = toArray(args.files as string[]).map(filepath => { + const file = new TextFile(filepath) + const contents = fs.readFileSync(filepath, 'utf8') + const scanner = new Scanner(file, contents) + const sourceFile = scanner.scan(); + const expander = new Expander(parser) + const expanded = expander.getFullyExpanded(sourceFile) + // console.log(require('util').inspect(expanded.toJSON(), { colors: true, depth: Infinity })) + setParents(expanded) + return expanded; + }) + + const checker = new TypeChecker() + const compiler = new Compiler(checker, { target: "JS" }) + const bundle = compiler.compile(sourceFiles) + const emitter = new Emitter() + for (const file of bundle) { + const text = emitter.emit(file); + fs.mkdirpSync('.bolt-work') + const filepath = path.join('.bolt-work', path.relative(process.cwd(), stripExtension(path.resolve(file.loc.source)) + '.js')) + fs.writeFileSync(filepath, text, 'utf8') + spawnSync('node', [filepath], { stdio: 'inherit' }) + } + + } + + ) + .help() .version() .argv diff --git a/src/checker.ts b/src/checker.ts new file mode 100644 index 000000000..89782a2dc --- /dev/null +++ b/src/checker.ts @@ -0,0 +1,58 @@ + +import { + Syntax, + SyntaxKind +} from "./ast" + +class Type { + +} + +interface FastStringMap { + [key: string]: T +} + +export class Scope { + + constructor(public origin: Syntax) { + + } + +} + +export class TypeChecker { + + protected stringType = new Type(); + protected intType = new Type(); + + protected scopes = new Map(); + + createType(node: Syntax) { + switch (node.kind) { + case SyntaxKind.ConstExpr: + if (typeof node.value === 'bigint') { + return this.intType; + } else if (typeof node.value === 'string') { + return this.stringType; + } + } + } + + getScope(node: Syntax): Scope { + while (node.kind !== SyntaxKind.FuncDecl && node.kind !== SyntaxKind.SourceFile) { + node = node.parentNode!; + } + if (this.scopes.has(node)) { + return this.scopes.get(node)! + } + const scope = new Scope(node) + this.scopes.set(node, scope) + return scope + } + + getMapperForNode(target: string, node: Syntax): Mapper { + return this.getScope(node).getMapper(target) + } + +} + diff --git a/src/compiler.ts b/src/compiler.ts new file mode 100644 index 000000000..e89516952 --- /dev/null +++ b/src/compiler.ts @@ -0,0 +1,125 @@ + +import acorn from "acorn" + +import { + TypeChecker, + Scope +} from "./checker" + +import { + Syntax, + SyntaxKind, + SourceFile, + Stmt, + Expr, + Decl, + isExpr, +} from "./ast" + +export interface CompilerOptions { + target: string; +} + +function pushAll(arr: T[], els: T[]) { + for (const el of els) { + arr.push(el) + } +} + +export class Compiler { + + readonly target: string; + + constructor(public checker: TypeChecker, options: CompilerOptions) { + this.target = options.target + } + + compile(files: SourceFile[]) { + return files.map(s => { + const body: (Decl | Stmt | Expr)[] = []; + for (const element of s.elements) { + this.compileDecl(element, body); + } + return { + type: 'Program', + body, + loc: { + source: s.getFile().path + } + } + }); + } + + protected compileExpr(node: Syntax, preamble: Syntax[]): Expr { + + switch (node.kind) { + + case SyntaxKind.CallExpr: + const compiledOperator = this.compileExpr(node.operator, preamble); + const compiledArgs = node.args.map(a => this.compileExpr(a, preamble)) + return { + type: 'CallExpression', + callee: compiledOperator, + arguments: compiledArgs, + }; + + case SyntaxKind.RefExpr: + return { + type: 'Identifier', + name: node.name.name.text, + } + + case SyntaxKind.ConstExpr: + return { + type: 'Literal', + value: node.value, + } + + default: + throw new Error(`Could not compile expression node ${SyntaxKind[node.kind]}`) + } + + } + + protected compileDecl(node: Syntax, preamble: Syntax[]): Expr | undefined { + + console.log(`compiling ${SyntaxKind[node.kind]}`) + + if (isExpr(node)) { + const compiled = this.compileExpr(node, preamble); + preamble.push({ + type: 'ExpressionStatement', + expression: compiled + }) + return; + } + + switch (node.kind) { + + case SyntaxKind.FuncDecl: + const params = []; + if (node.target === this.target) { + console.log(node.body) + preamble.push({ + type: 'FunctionDeclaration', + id: { type: 'Identifier', name: node.name.name.text }, + params: node.params.map(p => ({ type: 'Identifier', name: p.bindings.name.text })), + body: { + type: 'BlockStatement', + body: node.body + } + }) + } else { + throw new Error(`Cannot yet compile Bolt functions`) + } + break; + + default: + throw new Error(`Could not compile node ${SyntaxKind[node.kind]}`); + + } + + } + +} + diff --git a/src/emitter.ts b/src/emitter.ts new file mode 100644 index 000000000..f980f2878 --- /dev/null +++ b/src/emitter.ts @@ -0,0 +1,30 @@ + +import * as astring from "astring" + +import { Syntax, SyntaxKind, isJSNode } from "./ast" + +export class Emitter { + + emit(node: Syntax) { + + if (isJSNode(node)) { + return astring.generate(node) + } + + switch (node.kind) { + + case SyntaxKind.SourceFile: + let out = '' + for (const element of node.elements) { + out += this.emit(element); + } + return out; + + default: + throw new Error(`Could not emit source code for ${SyntaxKind[node.kind]}`) + + } + + } +} + diff --git a/src/expander.ts b/src/expander.ts index bf5671615..57a6e8ffd 100644 --- a/src/expander.ts +++ b/src/expander.ts @@ -18,6 +18,7 @@ export class Expander { constructor(public parser: Parser) { this.transformers.set('fn', parser.parseFuncDecl.bind(parser)) + this.transformers.set('foreign', parser.parseFuncDecl.bind(parser)) this.transformers.set('let', parser.parseVarDecl.bind(parser)) this.transformers.set('return', parser.parseRetStmt.bind(parser)) } @@ -41,7 +42,7 @@ export class Expander { console.log('expanding sententce') - const tokens: TokenStream = node.toStream() + const tokens: TokenStream = node.toTokenStream() const t0 = tokens.peek(); if (t0.kind !== SyntaxKind.Identifier) { @@ -49,7 +50,7 @@ export class Expander { } if (!this.transformers.has(t0.text)) { - throw new Error(`the macro '${t0.text}' does not seem to exist`) + return this.parser.parseCallExpr(tokens) } node = this.transformers.get(t0.text)!(tokens) diff --git a/src/parser.ts b/src/parser.ts index e3136497a..840a48cf7 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1,4 +1,6 @@ +import * as acorn from "acorn" + import { Syntax, Token, @@ -17,7 +19,9 @@ import { TypeRef, TypeDecl, ConstExpr, - QualName + QualName, + ForeignDecl, + CallExpr, } from "./ast" function describeKind(kind: SyntaxKind): string { @@ -26,8 +30,16 @@ function describeKind(kind: SyntaxKind): string { return "an identifier" case SyntaxKind.Operator: return "an operator" - case SyntaxKind.Literal: - return "a constant literal" + case SyntaxKind.StringLiteral: + return "a string" + case SyntaxKind.IntegerLiteral: + return "an integer" + case SyntaxKind.FunctionKeyword: + return "'fn'" + case SyntaxKind.ForeignKeyword: + return "'foreign'" + case SyntaxKind.LetKeyword: + return "'let'" case SyntaxKind.Semi: return "';'" case SyntaxKind.Colon: @@ -42,8 +54,10 @@ function describeKind(kind: SyntaxKind): string { return "'[' .. ']'" case SyntaxKind.Parenthesized: return "'(' .. ')'" + case SyntaxKind.EOS: + return "'}', ')', ']' or end-of-file" default: - return "a token" + throw new Error(`failed to describe ${SyntaxKind[kind]}`) } } @@ -51,7 +65,7 @@ function enumerate(elements: string[]) { if (elements.length === 1) { return elements[0] } else { - return elements.slice(0, elements.length-2).join(',') + ' or ' + elements[elements.length-1] + return elements.slice(0, elements.length-1).join(',') + ' or ' + elements[elements.length-1] } } @@ -112,19 +126,23 @@ export class Parser { } } - parseExpr(tokens: TokenStream): Expr { + parsePrimExpr(tokens: TokenStream): Expr { const t0 = tokens.peek(); - if (t0.kind === SyntaxKind.Literal) { + if (t0.kind === SyntaxKind.StringLiteral) { + tokens.get(); return new ConstExpr(t0.value, null, t0); - } else if (t0.kind === SyntaxKind.Identifier) { const name = this.parseQualName(tokens); return new RefExpr(name, null, name.origNode); } else { - throw new ParseError(t0, [SyntaxKind.Literal, SyntaxKind.Identifier]); + throw new ParseError(t0, [SyntaxKind.StringLiteral, SyntaxKind.Identifier]); } } + parseExpr(tokens: TokenStream) { + return this.parsePrimExpr(tokens) + } + parseParam(tokens: TokenStream) { let defaultValue = null; @@ -152,6 +170,9 @@ export class Parser { parseVarDecl(tokens: TokenStream): VarDecl { + // Assuming first token is 'let' + tokens.get(); + } parseRetStmt(tokens: TokenStream): RetStmt { @@ -183,14 +204,31 @@ export class Parser { parseFuncDecl(tokens: TokenStream, origNode: Syntax | null) { - // Assuming the first identifier is the 'fn' keyword - tokens.get(); + let target = "Bolt"; + + const k0 = tokens.get(); + if (k0.kind !== SyntaxKind.Identifier) { + throw new ParseError(k0, [SyntaxKind.ForeignKeyword, SyntaxKind.FunctionKeyword]) + } + if (k0.text === 'foreign') { + const l1 = tokens.get(); + if (l1.kind !== SyntaxKind.StringLiteral) { + throw new ParseError(l1, [SyntaxKind.StringLiteral]) + } + target = l1.value; + } + const k1 = tokens.get(); + if (k1.text !== 'fn') { + throw new ParseError(k1, [SyntaxKind.FunctionKeyword]) + } let name: QualName; let returnType = null; let body = null; let params: Param[] = []; + // Parse parameters + const t0 = tokens.peek(1); const t1 = tokens.peek(2); @@ -204,7 +242,7 @@ export class Parser { return new Param(new BindPatt(t0, null, t0), null, null, null, t0) } else if (t0.kind === SyntaxKind.Parenthesized) { tokens.get(); - const innerTokens = t0.toStream(); + const innerTokens = t0.toTokenStream(); const param = this.parseParam(innerTokens) this.assertEmpty(innerTokens); return param @@ -213,8 +251,6 @@ export class Parser { } } - // Parse parameters - if (t0.kind === SyntaxKind.Operator) { name = new QualName(t0, [], null, t0); @@ -242,7 +278,7 @@ export class Parser { name = this.parseQualName(tokens) const t2 = tokens.get(); if (t2.kind === SyntaxKind.Parenthesized) { - const innerTokens = t2.toStream(); + const innerTokens = t2.toTokenStream(); while (true) { const t3 = innerTokens.peek(); if (t3.kind === SyntaxKind.EOS) { @@ -271,25 +307,58 @@ export class Parser { const t2 = tokens.peek(); if (t2.kind === SyntaxKind.RArrow) { tokens.get(); - returnType = this.parseTypeDecl(tokens, t2); + returnType = this.parseTypeDecl(tokens); } // Parse function body const t3 = tokens.peek(); if (t3.kind === SyntaxKind.Braced) { - body = this.parseStmts(tokens, t3); + tokens.get(); + switch (target) { + case "Bolt": + body = this.parseStmts(tokens, t3); + break; + case "JS": + body = acorn.parse(t3.text).body; + break; + default: + throw new Error(`Unrecognised language: ${target}`); + } } - return new FuncDecl(name, params, returnType, body, null, origNode) + return new FuncDecl(target, name, params, returnType, body, null, origNode) } - parseDecl(tokens: TokenStream, origNode: Syntax | null) { - const t0 = tokens.peek(1); - if (t0.kind === SyntaxKind.Identifier && t0.text === 'fn') { - this.parseFuncDecl(tokens, origNode) + parseCallExpr(tokens: TokenStream) { + + const operator = this.parsePrimExpr(tokens) + const args: Expr[] = [] + + const t2 = tokens.get(); + if (t2.kind !== SyntaxKind.Parenthesized) { + throw new ParseError(t2, [SyntaxKind.Parenthesized]) } + + const innerTokens = t2.toTokenStream(); + + while (true) { + const t3 = innerTokens.peek(); + if (t3.kind === SyntaxKind.EOS) { + break; + } + args.push(this.parseExpr(innerTokens)) + const t4 = innerTokens.get(); + if (t4.kind === SyntaxKind.EOS) { + break + } else if (t4.kind !== SyntaxKind.Comma){ + throw new ParseError(t4, [SyntaxKind.Comma]) + } + } + + return new CallExpr(operator, args, null) + } } diff --git a/src/scanner.ts b/src/scanner.ts index 58dffce21..5c9aa4a8b 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -19,7 +19,10 @@ import { SourceFile, Semi, Comma, - Colon + StringLiteral, + IntegerLiteral, + Colon, + EOS, } from "./ast" function escapeChar(ch: string) { @@ -57,7 +60,7 @@ function getPunctType(ch: string) { case '}': return PunctType.Brace; default: - throw new Error(`given character is not a valid punctuator`) + return null; } } @@ -123,11 +126,12 @@ const EOF = '' export class Scanner { protected buffer: string[] = []; - protected currPos = new TextPos(0,1,1); + protected scanned: Token[] = []; + protected currPos: TextPos; protected offset = 0; - constructor(public file: TextFile, public input: string) { - + constructor(public file: TextFile, public input: string, startPos = new TextPos(0,1,1)) { + this.currPos = startPos; } protected readChar() { @@ -178,7 +182,7 @@ export class Scanner { return text; } - scanToken(): Token | null { + scanToken(): Token { while (true) { @@ -189,12 +193,11 @@ export class Scanner { continue; } - if (c0 == EOF) { - return null; - } - const startPos = this.currPos.clone() + if (c0 == EOF) { + return new EOS(new TextSpan(this.file, startPos, startPos)); + } switch (c0) { case ';': @@ -208,40 +211,57 @@ export class Scanner { return new Colon(new TextSpan(this.file, startPos, this.currPos.clone())); } - if (isOpenPunct(c0)) { + if (c0 === '"') { + + this.getChar(); + + let text = '' + + while (true) { + const c1 = this.getChar(); + if (c1 === EOF) { + throw new ScanError(this.file, this.currPos.clone(), EOF); + } + if (c1 === '"') { + break; + } else if (c1 === '\\') { + this.scanEscapeSequence() + } else { + text += c1 + } + } + + const endPos = this.currPos.clone(); + + return new StringLiteral(text, new TextSpan(this.file, startPos, endPos)) + + } else if (isOpenPunct(c0)) { this.getChar(); const punctType = getPunctType(c0); - const elements: Token[] = []; + let punctCount = 1; + let text = '' while (true) { - const c1 = this.peekChar(); - - if (isWhiteSpace(c1)) { - this.getChar() - continue; - } + const c1 = this.getChar(); if (c1 === EOF) { throw new ScanError(this.file, this.currPos.clone(), EOF) } - if (isClosePunct(c1)) { - if (punctType == getPunctType(c1)) { - this.getChar(); - break; + if (punctType == getPunctType(c1)) { + if (isClosePunct(c1)) { + punctCount--; + if (punctCount === 0) + break; } else { - throw new ScanError(this.file, this.currPos, c1); + punctCount++; } } - const token = this.scanToken(); - if (token === null) { - throw new ScanError(this.file, this.currPos.clone(), EOF) - } - elements.push(token!); + text += c1 } @@ -249,11 +269,11 @@ export class Scanner { switch (punctType) { case PunctType.Brace: - return new Braced(elements, new TextSpan(this.file, startPos, endPos)); + return new Braced(text, new TextSpan(this.file, startPos, endPos)); case PunctType.Paren: - return new Parenthesized(elements, new TextSpan(this.file, startPos, endPos)); + return new Parenthesized(text, new TextSpan(this.file, startPos, endPos)); case PunctType.Bracket: - return new Bracketed(elements, new TextSpan(this.file, startPos, endPos)); + return new Bracketed(text, new TextSpan(this.file, startPos, endPos)); default: throw new Error("Got an invalid state.") } @@ -286,6 +306,19 @@ export class Scanner { } + peek(count = 1): Token { + while (this.scanned.length < count) { + this.scanned.push(this.scanToken()); + } + return this.scanned[count - 1]; + } + + get(): Token { + return this.scanned.length > 0 + ? this.scanned.shift()! + : this.scanToken(); + } + scan() { const elements: Decl[] = [] @@ -297,7 +330,7 @@ export class Scanner { inner: while (true) { const token = this.scanToken(); - if (token === null) { + if (token.kind === SyntaxKind.EOS) { if (tokens.length === 0) { break outer; } else { @@ -326,4 +359,3 @@ export class Scanner { } } -