Add a very naive compiler

.gitignore

@@ -8,3 +8,6 @@ node_modules/
 # local development files
 Makefile
 
+# bolt
+.bolt-work/
+

package-lock.json

@@ -35,6 +35,11 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.4.tgz",
       "integrity": "sha512-oVeL12C6gQS/GAExndigSaLxTrKpQPxewx9bOcwfvJiJge4rr7wNaph4J+ns5hrmIV2as5qxqN8YKthn9qh0jw=="
     },
+    "@types/xregexp": {
+      "version": "3.0.30",
+      "resolved": "https://registry.npmjs.org/@types/xregexp/-/xregexp-3.0.30.tgz",
+      "integrity": "sha512-u1dpabg81Rd660bYebOqMXO0+E63H1hxunPAWGebNb7TpxqZYe9YaVLgkkj6ZnzLs3yLumtVB956o8u8OZdhXw=="
+    },
     "@types/yargs": {
       "version": "15.0.3",
       "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-15.0.3.tgz",
@@ -48,6 +53,11 @@
       "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-15.0.0.tgz",
       "integrity": "sha512-FA/BWv8t8ZWJ+gEOnLLd8ygxH/2UFbAvgEonyfN6yWGLKc7zVjbpl2Y4CTjid9h2RfgPP6SEt6uHwEOply00yw=="
     },
+    "acorn": {
+      "version": "7.1.0",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.1.0.tgz",
+      "integrity": "sha512-kL5CuoXA/dgxlBbVrflsflzQ3PAas7RYZB52NOm/6839iVYJgKMJ3cQJD+t2i5+qFa8h3MDpEOJiS64E8JLnSQ=="
+    },
     "ansi-colors": {
       "version": "3.2.3",
       "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz",
@@ -94,6 +104,11 @@
       "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
       "dev": true
     },
+    "astring": {
+      "version": "1.4.3",
+      "resolved": "https://registry.npmjs.org/astring/-/astring-1.4.3.tgz",
+      "integrity": "sha512-yJlJU/bmN820vL+cbWShu2YQU87dBP5V7BH2N4wODapRv27A2dZtUD0LgjP9lZENvPe9XRoSyWx+pZR6qKqNBw=="
+    },
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
@@ -374,6 +389,16 @@
         "is-buffer": "~2.0.3"
       }
     },
+    "fs-extra": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
+      "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==",
+      "requires": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^4.0.0",
+        "universalify": "^0.1.0"
+      }
+    },
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -425,6 +450,11 @@
         "is-glob": "^4.0.1"
       }
     },
+    "graceful-fs": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
+      "integrity": "sha512-a30VEBm4PEdx1dRB7MFK7BejejvCvBronbLjht+sHuGYj8PHs7M/5Z+rt5lw551vZ7yfTCj4Vuyy3mSJytDWRQ=="
+    },
     "growl": {
       "version": "1.10.5",
       "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz",
@@ -560,6 +590,14 @@
         "esprima": "^4.0.0"
       }
     },
+    "jsonfile": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
+      "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=",
+      "requires": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
     "locate-path": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz",
@@ -971,6 +1009,11 @@
       "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==",
       "dev": true
     },
+    "universalify": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
+      "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg=="
+    },
     "which": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",

package.json

@@ -14,7 +14,11 @@
   "repository": "https://github.com/samvv/Bolt",
   "dependencies": {
     "@types/node": "^13.7.4",
+    "@types/xregexp": "^3.0.30",
     "@types/yargs": "^15.0.3",
+    "acorn": "^7.1.0",
+    "astring": "^1.4.3",
+    "fs-extra": "^8.1.0",
     "glob": "^7.1.6",
     "pegjs": "^0.11.0-master.b7b87ea",
     "reflect-metadata": "^0.1.13",

src/ast.ts

@@ -1,5 +1,8 @@
 
+// FIXME SyntaxBase.getSpan() does not work then [n1, n2] is given as origNode
+
 import { Stream, StreamWrapper } from "./util"
+import { Scanner } from "./scanner"
 
 interface JsonArray extends Array<Json> {  };
 interface JsonObject { [key: string]: Json }
@@ -7,23 +10,12 @@ type Json = null | string | boolean | number | JsonArray | JsonObject;
 
 export type TokenStream = Stream<Token>;
 
-function jsonify(value: any): Json {
-  if (value === null || typeof value === 'string' || typeof value === 'number') {
-    return value;
-  } else if (Array.isArray(value)) {
-    return value.map(element => jsonify(element));
-  } else if (typeof value === 'object' && 'toJSON' in value) {
-    return value.toJSON();
-  } else {
-    throw new Error(`I don't know how to convert ${value} to a JSON representation`)
-  }
-}
-
 export enum SyntaxKind {
 
   // Tokens
 
-  Literal,
+  StringLiteral,
+  IntegerLiteral,
   Identifier,
   Operator,
   Parenthesized,
@@ -36,6 +28,13 @@ export enum SyntaxKind {
   RArrow,
   EqSign,
 
+  // Keywords
+
+  FunctionKeyword,
+  ForeignKeyword,
+  LetKeyword,
+  ImportKeyword,
+
   // Special nodes
 
   SourceFile,
@@ -55,6 +54,7 @@ export enum SyntaxKind {
 
   // Expressions
 
+  CallExpr,
   ConstExpr,
   RefExpr,
 
@@ -70,6 +70,7 @@ export enum SyntaxKind {
 
   VarDecl,
   FuncDecl,
+  ForeignDecl,
 
 }
 
@@ -143,7 +144,7 @@ abstract class SyntaxBase {
   abstract parentNode: Syntax | null;
   abstract span: TextSpan | null;
 
-  getSpan() {
+  getSpan(): TextSpan {
 
     let curr: Syntax | null = this as any as Syntax;
 
@@ -151,25 +152,27 @@ abstract class SyntaxBase {
       if (curr.span !== null ) {
         return curr.span;
       }
-      curr = curr.origNode
+      curr = curr.origNode;
     } while (curr !== null)
 
     throw new Error(`No TextSpan object found in this node or any of its originating nodes.`);
 
   }
 
-}
+  getFile(): TextFile {
-
+    return this.getSpan().file;
-export class Literal extends SyntaxBase {
-
-  kind: SyntaxKind.Literal = SyntaxKind.Literal;
-
-  static META = {
-    value: EdgeType.Primitive,
   }
 
+  abstract getChildren(): IterableIterator<Syntax>;
+
+}
+
+export class StringLiteral extends SyntaxBase {
+
+  kind: SyntaxKind.StringLiteral = SyntaxKind.StringLiteral;
+
   constructor(
-    public value: string | bigint,
+    public value: string,
     public span: TextSpan | null = null,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -179,20 +182,51 @@ export class Literal extends SyntaxBase {
 
   toJSON(): Json {
     return {
-      value: typeof this.value === 'bigint' ? { type: 'bigint', value: this.value.toString() } : this.value,
+      value: this.value,
       span: this.span !== null ? this.span.toJSON() : null,
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
+export class IntegerLiteral extends SyntaxBase {
+
+  kind: SyntaxKind.IntegerLiteral = SyntaxKind.IntegerLiteral;
+
+  constructor(
+    public value: string,
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON(): Json {
+    return {
+      value: ['bigint', this.value.toString()],
+      span: this.span !== null ? this.span.toJSON() : null,
+    }
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
+}
+
+
 export enum PunctType {
   Paren,
   Bracket,
   Brace,
 }
 
-class EOS extends SyntaxBase {
+export class EOS extends SyntaxBase {
 
   kind: SyntaxKind.EOS = SyntaxKind.EOS;
 
@@ -210,18 +244,20 @@ class EOS extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Parenthesized extends SyntaxBase {
 
   kind: SyntaxKind.Parenthesized = SyntaxKind.Parenthesized;
 
-  static META = {
+  protected buffered = null;
-    elements: EdgeType.Node | EdgeType.List
-  }
 
   constructor(
-    public elements: Token[],
+    public text: string,
     public span: TextSpan,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -229,21 +265,24 @@ export class Parenthesized extends SyntaxBase {
     super();
   }
 
-  toStream() {
+  toTokenStream() {
-    return new StreamWrapper(
+    const span = this.getSpan();
-      this.elements,
+    const startPos = span.start;
-      () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone()))
+    return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1));
-    );
   }
 
   toJSON(): Json {
     return {
       kind: 'Parenthesized',
-      elements: this.elements.map(element => element.toJSON()),
+      text: this.text,
       span: this.span !== null ? this.span.toJSON() : null,
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 
@@ -251,12 +290,8 @@ export class Braced extends SyntaxBase {
 
   kind: SyntaxKind.Braced = SyntaxKind.Braced;
 
-  static META = {
-    elements: EdgeType.Node | EdgeType.List
-  }
-
   constructor(
-    public elements: Token[],
+    public text: string,
     public span: TextSpan,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -264,33 +299,32 @@ export class Braced extends SyntaxBase {
     super();
   }
 
-  toStream() {
+  toTokenStream() {
-    return new StreamWrapper(
+    const span = this.getSpan();
-      this.elements,
+    const startPos = span.start;
-      () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone()))
+    return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1));
-    );
   }
 
   toJSON(): Json {
     return {
       kind: 'Braced',
-      elements: this.elements.map(element => element.toJSON()),
+      text: this.text,
       span: this.span !== null ? this.span.toJSON() : null,
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Bracketed extends SyntaxBase {
 
   kind: SyntaxKind.Bracketed = SyntaxKind.Bracketed;
 
-  static META = {
-    elements: EdgeType.Node | EdgeType.List
-  }
-
   constructor(
-    public elements: Token[],
+    public text: string,
     public span: TextSpan,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -298,31 +332,30 @@ export class Bracketed extends SyntaxBase {
     super();
   }
 
-  toStream() {
+  toTokenStream() {
-    return new StreamWrapper(
+    const span = this.getSpan();
-      this.elements,
+    const startPos = span.start;
-      () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone()))
+    return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1));
-    );
   }
 
   toJSON(): Json {
     return {
       kind: 'Bracketed',
-      elements: this.elements.map(element => element.toJSON()),
+      text: this.text,
       span: this.span !== null ? this.span.toJSON() : null,
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Identifier extends SyntaxBase {
 
   kind: SyntaxKind.Identifier = SyntaxKind.Identifier;
 
-  static META = {
-    text: EdgeType.Primitive
-  }
-
   constructor(
     public text: string,
     public span: TextSpan | null = null,
@@ -340,16 +373,16 @@ export class Identifier extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Operator extends SyntaxBase {
 
   kind: SyntaxKind.Operator = SyntaxKind.Operator;
 
-  static META = {
-    text: EdgeType.Primitive
-  }
-
   constructor(
     public text: string,
     public span: TextSpan | null = null,
@@ -367,6 +400,10 @@ export class Operator extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Semi extends SyntaxBase {
@@ -388,6 +425,10 @@ export class Semi extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Colon extends SyntaxBase {
@@ -409,6 +450,10 @@ export class Colon extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Comma extends SyntaxBase {
@@ -430,6 +475,10 @@ export class Comma extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 
@@ -452,6 +501,10 @@ export class RArrow extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 
@@ -475,6 +528,10 @@ export class EqSign extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export class Dot extends SyntaxBase {
@@ -496,6 +553,10 @@ export class Dot extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+
+  }
+
 }
 
 export type Token
@@ -508,7 +569,8 @@ export type Token
   | EOS
   | Identifier
   | Operator
-  | Literal
+  | StringLiteral
+  | IntegerLiteral
   | Parenthesized
   | Braced
   | Bracketed
@@ -526,7 +588,7 @@ export class Sentence extends SyntaxBase {
     super();
   }
 
-  toStream() {
+  toTokenStream() {
     return new StreamWrapper(
       this.tokens,
       () => new EOS(new TextSpan(this.getSpan().file, this.getSpan().end.clone(), this.getSpan().end.clone()))
@@ -541,17 +603,18 @@ export class Sentence extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    for (const token of this.tokens) {
+      yield token;
+    }
+  }
+
 }
 
 export class QualName {
 
   kind: SyntaxKind.QualName = SyntaxKind.QualName;
 
-  static META = {
-    name: EdgeType.Node,
-    path: EdgeType.Node | EdgeType.List,
-  }
-
   constructor(
     public name: Identifier | Operator,
     public path: Identifier[],
@@ -571,6 +634,13 @@ export class QualName {
     }
   }
 
+    *getChildren(): IterableIterator<Syntax> {
+      for (const chunk of this.path) {
+        yield chunk
+      }
+      yield this.name
+    }
+
 }
 
 export class Param extends SyntaxBase {
@@ -598,6 +668,16 @@ export class Param extends SyntaxBase {
     }
   }
 
+  *getChildren() { 
+    yield this.bindings
+    if (this.typeDecl !== null) {
+      yield this.typeDecl
+    }
+    if (this.defaultValue !== null) {
+      yield this.defaultValue
+    }
+  }
+
 }
 
 export class BindPatt extends SyntaxBase {
@@ -621,6 +701,10 @@ export class BindPatt extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.name
+  }
+
 }
 
 export type Patt
@@ -647,16 +731,48 @@ export class RefExpr extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.name
+  }
+
+}
+
+export class CallExpr extends SyntaxBase {
+
+  kind: SyntaxKind.CallExpr = SyntaxKind.CallExpr;
+
+  constructor(
+    public operator: Expr,
+    public args: Expr[],
+    public span: TextSpan | null = null, 
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON(): Json {
+    return {
+      kind: 'CallExpr',
+      operator: this.operator.toJSON(),
+      args: this.args.map(a => a.toJSON()),
+      span: this.span !== null ? this.span.toJSON() : null,
+    }
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.operator
+    for (const arg of this.args) {
+      yield arg
+    }
+  }
+
 }
 
 export class ConstExpr extends SyntaxBase {
 
   kind: SyntaxKind.ConstExpr = SyntaxKind.ConstExpr;
 
-  static META = {
-    value: EdgeType.Primitive,
-  }
-
   constructor(
     public value: string | bigint, 
     public span: TextSpan | null = null, 
@@ -674,11 +790,17 @@ export class ConstExpr extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    
+  }
+
+
 }
 
 export type Expr
   = ConstExpr
   | RefExpr
+  | CallExpr
 
 export class RetStmt extends SyntaxBase {
 
@@ -701,6 +823,12 @@ export class RetStmt extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    if (this.value !== null) {
+      yield this.value
+    }
+  }
+
 }
 
 export type Stmt
@@ -710,14 +838,9 @@ export class TypeRef extends SyntaxBase {
 
   kind: SyntaxKind.TypeRef = SyntaxKind.TypeRef;
 
-  static META = {
-    name: EdgeType.Node,
-    args: EdgeType.Node | EdgeType.List,
-  }
-
   constructor(
     public name: QualName,
-    public args: TypeDecl[],
+    public typeArgs: TypeDecl[],
     public span: TextSpan | null = null,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -729,44 +852,29 @@ export class TypeRef extends SyntaxBase {
     return {
       kind: 'TypeRef',
       name: this.name.toJSON(),
-      args: this.args.map(a => a.toJSON()),
+      args: this.typeArgs.map(a => a.toJSON()),
       span: this.span !== null ? this.span.toJSON() : null,
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.name
+    for (const arg of this.typeArgs) {
+      yield arg
+    }
+  }
+
 }
 
 export type TypeDecl
   = TypeRef
 
-// export class Unexpanded {
-// 
-//   static META = {
-//     tokens: EdgeType.Node | EdgeType.List
-//   }
-// 
-//   constructor(
-//     public tokens: Token[],
-//     public span: TextSpan,
-//     public parentNode: Syntax | null = null
-//   ) {
-// 
-//   }
-// 
-// }
-
 export class FuncDecl extends SyntaxBase {
 
   kind: SyntaxKind.FuncDecl = SyntaxKind.FuncDecl;
 
-  static META = {
-    name: EdgeType.Node,
-    params: EdgeType.Node | EdgeType.List,
-    returnType: EdgeType.Node | EdgeType.Nullable,
-    body: EdgeType.Node | EdgeType.List,
-  }
-
   constructor(
+    public target: string,
     public name: QualName, 
     public params: Param[],
     public returnType: TypeDecl | null,
@@ -781,6 +889,7 @@ export class FuncDecl extends SyntaxBase {
   toJSON(): Json {
     return {
       kind: 'FuncDecl',
+      target: this.target,
       name: this.name.toJSON(),
       params: this.params.map(p => p.toJSON()),
       returnType: this.returnType !== null ? this.returnType.toJSON() : null,
@@ -789,18 +898,27 @@ export class FuncDecl extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.name
+    for (const param of this.params) {
+      yield param
+    }
+    if (this.returnType !== null) {
+      yield this.returnType;
+    }
+    if (this.body !== null) {
+      for (const stmt of this.body) {
+        yield stmt
+      }
+    }
+  }
+
 }
 
 export class VarDecl extends SyntaxBase {
 
   kind: SyntaxKind.VarDecl = SyntaxKind.VarDecl;
 
-  static META = {
-    bindings: EdgeType.Node,
-    typeDecl: EdgeType.Node | EdgeType.Nullable,
-    value: EdgeType.Node | EdgeType.Nullable,
-  }
-
   constructor(
     public bindings: Patt, 
     public typeDecl: TypeDecl | null,
@@ -814,7 +932,7 @@ export class VarDecl extends SyntaxBase {
 
   toJSON(): Json {
     return {
-      type: 'VarDecl',
+      kind: 'VarDecl',
       bindings: this.bindings.toJSON(),
       typeDecl: this.typeDecl !== null ? this.typeDecl.toJSON() : null,
       value: this.value !== null ? this.value.toJSON() : null,
@@ -822,6 +940,16 @@ export class VarDecl extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.bindings
+    if (this.typeDecl !== null) {
+      yield this.typeDecl
+    }
+    if (this.value !== null) {
+      yield this.value;
+    }
+  }
+
 }
 
 export type Decl
@@ -833,6 +961,9 @@ export type Syntax
   = Decl
   | Expr
   | Token
+  | Stmt
+  | Patt
+  | TypeDecl
   | SourceFile
   | QualName
   | Param
@@ -843,7 +974,7 @@ export class SourceFile extends SyntaxBase {
   kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile;
 
   constructor(
-    public elements: (Decl | Stmt)[],
+    public elements: (Decl | Stmt | Expr)[],
     public span: TextSpan | null = null,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -859,5 +990,29 @@ export class SourceFile extends SyntaxBase {
     }
   }
 
+  *getChildren(): IterableIterator<Syntax> {
+    for (const element of this.elements) {
+      yield element
+    }
+  }
+
+}
+
+export function isExpr(node: Syntax): node is Expr {
+  return node.kind === SyntaxKind.ConstExpr || node.kind === SyntaxKind.CallExpr;
+}
+
+export function isJSNode(node: Syntax) {
+  return typeof node.type === 'string'
+}
+
+export function setParents(node: Syntax) {
+  if (isJSNode(node)) {
+    return;
+  }
+  for (const child of node.getChildren()) {
+    child.parentNode = node
+    setParents(child)
+  }
 }
 

src/bin/bolt.ts

@@ -3,16 +3,21 @@
 import "reflect-metadata"
 import "source-map-support/register"
 
-import * as fs from "fs"
+import * as path from "path"
+import * as fs from "fs-extra"
+import { spawnSync } from "child_process"
 
 import yargs from "yargs"
 
 import { Scanner } from "../scanner"
 import { Parser } from "../parser"
 import { Expander } from "../expander"
-import { Token, TextFile, SourceFile } from "../ast"
+import { TypeChecker } from "../checker"
+import { Compiler } from "../compiler"
+import { Emitter } from "../emitter"
+import { TextFile, SourceFile, setParents } from "../ast"
 
-function toArray<T>(value: T): T extends Array<any> ? T : T[] {
+function toArray<T>(value: T | T[]): T[] {
   if (Array.isArray(value)) {
     return value as T[]
   }
@@ -25,6 +30,11 @@ function pushAll<T>(array: T[], elements: T[]) {
   }
 }
 
+function stripExtension(filepath: string) {
+  const i = filepath.lastIndexOf('.');
+  return i !== -1 ? filepath.substring(0, i) : filepath
+}
+
 function flatMap<T>(array: T[], proc: (element: T) => T[]) {
   let out: T[] = []
   for (const element of array) {
@@ -54,6 +64,7 @@ function parseHook(str: string): Hook {
 }
 
 yargs
+
   .command(
 
     'compile [files..]',
@@ -144,6 +155,46 @@ yargs
 
     })
 
+  .command(
+
+    'exec [files..]',
+    'Run the specified Bolt scripts',
+
+    yargs =>
+      yargs,
+
+    args => {
+
+      const parser = new Parser()
+
+      const sourceFiles = toArray(args.files as string[]).map(filepath => {
+        const file = new TextFile(filepath)
+        const contents = fs.readFileSync(filepath, 'utf8')
+        const scanner = new Scanner(file, contents)
+        const sourceFile = scanner.scan();
+        const expander = new Expander(parser)
+        const expanded = expander.getFullyExpanded(sourceFile)
+        // console.log(require('util').inspect(expanded.toJSON(), { colors: true, depth: Infinity }))
+        setParents(expanded)
+        return expanded;
+      })
+
+      const checker = new TypeChecker()
+      const compiler = new Compiler(checker, { target: "JS" })
+      const bundle = compiler.compile(sourceFiles)
+      const emitter = new Emitter()
+      for (const file of bundle) {
+        const text = emitter.emit(file);
+        fs.mkdirpSync('.bolt-work')
+        const filepath = path.join('.bolt-work', path.relative(process.cwd(), stripExtension(path.resolve(file.loc.source)) + '.js'))
+        fs.writeFileSync(filepath, text, 'utf8')
+        spawnSync('node', [filepath], { stdio: 'inherit' })
+      }
+
+    }
+
+  )
+
   .help()
   .version()
   .argv

src/checker.ts

@@ -0,0 +1,58 @@
+
+import {
+  Syntax,
+  SyntaxKind
+} from "./ast"
+
+class Type {
+  
+}
+
+interface FastStringMap<T> {
+  [key: string]: T
+}
+
+export class Scope {
+
+  constructor(public origin: Syntax) {
+
+  }
+
+}
+
+export class TypeChecker {
+
+  protected stringType = new Type();
+  protected intType = new Type();
+
+  protected scopes = new Map<Syntax, Scope>();
+
+  createType(node: Syntax) {
+    switch (node.kind) {
+      case SyntaxKind.ConstExpr:
+        if (typeof node.value === 'bigint') {
+          return this.intType;
+        } else if (typeof node.value === 'string') {
+          return this.stringType;
+        }
+    }
+  }
+
+  getScope(node: Syntax): Scope {
+    while (node.kind !== SyntaxKind.FuncDecl && node.kind !== SyntaxKind.SourceFile) {
+      node = node.parentNode!;
+    }
+    if (this.scopes.has(node)) {
+      return this.scopes.get(node)!
+    }
+    const scope = new Scope(node)
+    this.scopes.set(node, scope)
+    return scope
+  }
+
+  getMapperForNode(target: string, node: Syntax): Mapper {
+    return this.getScope(node).getMapper(target)
+  }
+
+}
+

src/compiler.ts

@@ -0,0 +1,125 @@
+
+import acorn from "acorn"
+
+import { 
+  TypeChecker,
+  Scope
+} from "./checker"
+
+import {
+  Syntax,
+  SyntaxKind,
+  SourceFile,
+  Stmt,
+  Expr,
+  Decl,
+  isExpr,
+} from "./ast"
+
+export interface CompilerOptions {
+  target: string;
+}
+
+function pushAll<T>(arr: T[], els: T[]) {
+  for (const el of els) {
+    arr.push(el)
+  }
+}
+
+export class Compiler {
+
+  readonly target: string;
+
+  constructor(public checker: TypeChecker, options: CompilerOptions) {
+    this.target = options.target
+  }
+
+  compile(files: SourceFile[]) {
+    return files.map(s => {
+      const body: (Decl | Stmt | Expr)[] = [];
+      for (const element of s.elements) {
+        this.compileDecl(element, body);
+      }
+      return {
+        type: 'Program',
+        body,
+        loc: {
+          source: s.getFile().path
+        }
+      }
+    });
+  }
+
+  protected compileExpr(node: Syntax, preamble: Syntax[]): Expr {
+  
+    switch (node.kind) {
+
+      case SyntaxKind.CallExpr:
+        const compiledOperator = this.compileExpr(node.operator, preamble);
+        const compiledArgs = node.args.map(a => this.compileExpr(a, preamble))
+        return {
+          type: 'CallExpression',
+          callee: compiledOperator,
+          arguments: compiledArgs,
+        };
+
+      case SyntaxKind.RefExpr:
+        return {
+          type: 'Identifier',
+          name: node.name.name.text,
+        }
+
+      case SyntaxKind.ConstExpr:
+        return {
+          type: 'Literal',
+          value: node.value,
+        }
+
+      default:
+        throw new Error(`Could not compile expression node ${SyntaxKind[node.kind]}`)
+    }
+
+  }
+
+  protected compileDecl(node: Syntax, preamble: Syntax[]): Expr | undefined {
+
+    console.log(`compiling ${SyntaxKind[node.kind]}`)
+
+    if (isExpr(node)) {
+      const compiled = this.compileExpr(node, preamble);
+      preamble.push({
+        type: 'ExpressionStatement',
+        expression: compiled
+      })
+      return;
+    }
+
+    switch (node.kind) {
+
+      case SyntaxKind.FuncDecl:
+        const params = [];
+        if (node.target === this.target) {
+          console.log(node.body)
+          preamble.push({
+            type: 'FunctionDeclaration',
+            id: { type: 'Identifier',  name: node.name.name.text },
+            params: node.params.map(p => ({ type: 'Identifier', name: p.bindings.name.text })),
+            body: {
+              type: 'BlockStatement',
+              body: node.body
+            }
+          })
+        } else {
+          throw new Error(`Cannot yet compile Bolt functions`)
+        }
+        break;
+
+      default:
+        throw new Error(`Could not compile node ${SyntaxKind[node.kind]}`);
+
+    }
+
+  }
+
+}
+

src/emitter.ts

@@ -0,0 +1,30 @@
+
+import * as astring from "astring"
+
+import { Syntax, SyntaxKind, isJSNode } from "./ast"
+
+export class Emitter {
+
+  emit(node: Syntax) {
+
+    if (isJSNode(node)) {
+      return astring.generate(node)
+    }
+
+    switch (node.kind) {
+
+      case SyntaxKind.SourceFile:
+        let out = ''
+        for (const element of node.elements) {
+          out += this.emit(element);
+        }
+        return out;
+
+      default:
+        throw new Error(`Could not emit source code for ${SyntaxKind[node.kind]}`)
+
+    }
+
+  }
+}
+

src/expander.ts

@@ -18,6 +18,7 @@ export class Expander {
 
   constructor(public parser: Parser) {
     this.transformers.set('fn', parser.parseFuncDecl.bind(parser))
+    this.transformers.set('foreign', parser.parseFuncDecl.bind(parser))
     this.transformers.set('let', parser.parseVarDecl.bind(parser))
     this.transformers.set('return', parser.parseRetStmt.bind(parser))
   }
@@ -41,7 +42,7 @@ export class Expander {
 
         console.log('expanding sententce')
 
-        const tokens: TokenStream = node.toStream()
+        const tokens: TokenStream = node.toTokenStream()
 
         const t0 = tokens.peek();
         if (t0.kind !== SyntaxKind.Identifier) {
@@ -49,7 +50,7 @@ export class Expander {
         }
 
         if (!this.transformers.has(t0.text)) {
-          throw new Error(`the macro '${t0.text}' does not seem to exist`)
+          return this.parser.parseCallExpr(tokens)
         }
 
         node = this.transformers.get(t0.text)!(tokens)

src/parser.ts

@@ -1,4 +1,6 @@
 
+import * as acorn from "acorn"
+
 import {
   Syntax,
   Token, 
@@ -17,7 +19,9 @@ import {
   TypeRef,
   TypeDecl,
   ConstExpr,
-  QualName
+  QualName,
+  ForeignDecl,
+  CallExpr,
 } from "./ast"
 
 function describeKind(kind: SyntaxKind): string {
@@ -26,8 +30,16 @@ function describeKind(kind: SyntaxKind): string {
       return "an identifier"
     case SyntaxKind.Operator:
       return "an operator"
-    case SyntaxKind.Literal:
+    case SyntaxKind.StringLiteral:
-      return "a constant literal"
+      return "a string"
+    case SyntaxKind.IntegerLiteral:
+      return "an integer"
+    case SyntaxKind.FunctionKeyword:
+      return "'fn'"
+    case SyntaxKind.ForeignKeyword:
+      return "'foreign'"
+    case SyntaxKind.LetKeyword:
+      return "'let'"
     case SyntaxKind.Semi:
       return "';'"
     case SyntaxKind.Colon:
@@ -42,8 +54,10 @@ function describeKind(kind: SyntaxKind): string {
       return "'[' .. ']'"
     case SyntaxKind.Parenthesized:
       return "'(' .. ')'"
+    case SyntaxKind.EOS:
+      return "'}', ')', ']' or end-of-file"
     default:
-      return "a token"
+      throw new Error(`failed to describe ${SyntaxKind[kind]}`)
   }
 }
 
@@ -51,7 +65,7 @@ function enumerate(elements: string[]) {
   if (elements.length === 1) {
     return elements[0]
   } else {
-    return elements.slice(0, elements.length-2).join(',') + ' or ' + elements[elements.length-1]
+    return elements.slice(0, elements.length-1).join(',') + ' or ' + elements[elements.length-1]
   }
 }
 
@@ -112,19 +126,23 @@ export class Parser {
     }
   }
 
-  parseExpr(tokens: TokenStream): Expr {
+  parsePrimExpr(tokens: TokenStream): Expr {
     const t0 = tokens.peek();
-    if (t0.kind === SyntaxKind.Literal) {
+    if (t0.kind === SyntaxKind.StringLiteral) {
+      tokens.get();
       return new ConstExpr(t0.value, null, t0);
-    
     } else if (t0.kind === SyntaxKind.Identifier) {
       const name = this.parseQualName(tokens);
       return new RefExpr(name, null, name.origNode);
     } else {
-      throw new ParseError(t0, [SyntaxKind.Literal, SyntaxKind.Identifier]);
+      throw new ParseError(t0, [SyntaxKind.StringLiteral, SyntaxKind.Identifier]);
     }
   }
 
+  parseExpr(tokens: TokenStream) {
+    return this.parsePrimExpr(tokens)
+  }
+
   parseParam(tokens: TokenStream) {
 
     let defaultValue = null;
@@ -152,6 +170,9 @@ export class Parser {
 
   parseVarDecl(tokens: TokenStream): VarDecl {
 
+    // Assuming first token is 'let'
+    tokens.get();
+
   }
 
   parseRetStmt(tokens: TokenStream): RetStmt {
@@ -183,14 +204,31 @@ export class Parser {
 
   parseFuncDecl(tokens: TokenStream, origNode: Syntax | null) {
 
-    // Assuming the first identifier is the 'fn' keyword
+    let target = "Bolt";
-    tokens.get();
+
+    const k0 = tokens.get();
+    if (k0.kind !== SyntaxKind.Identifier) {
+      throw new ParseError(k0, [SyntaxKind.ForeignKeyword, SyntaxKind.FunctionKeyword])
+    }
+    if (k0.text === 'foreign') {
+      const l1 = tokens.get();
+      if (l1.kind !== SyntaxKind.StringLiteral) {
+        throw new ParseError(l1, [SyntaxKind.StringLiteral])
+      }
+      target = l1.value;
+    }
+    const k1 = tokens.get();
+    if (k1.text !== 'fn') {
+      throw new ParseError(k1, [SyntaxKind.FunctionKeyword])
+    }
 
     let name: QualName;
     let returnType = null;
     let body = null;
     let params: Param[] = [];
 
+    // Parse parameters
+
     const t0 = tokens.peek(1);
     const t1 = tokens.peek(2);
 
@@ -204,7 +242,7 @@ export class Parser {
         return new Param(new BindPatt(t0, null, t0), null, null, null, t0)
       } else if (t0.kind === SyntaxKind.Parenthesized) {
         tokens.get();
-        const innerTokens = t0.toStream();
+        const innerTokens = t0.toTokenStream();
         const param = this.parseParam(innerTokens)
         this.assertEmpty(innerTokens);
         return param
@@ -213,8 +251,6 @@ export class Parser {
       }
     }
 
-    // Parse parameters
-
     if (t0.kind === SyntaxKind.Operator) {
 
       name = new QualName(t0, [], null, t0);
@@ -242,7 +278,7 @@ export class Parser {
       name = this.parseQualName(tokens)
       const t2 = tokens.get();
       if (t2.kind === SyntaxKind.Parenthesized) {
-        const innerTokens = t2.toStream();
+        const innerTokens = t2.toTokenStream();
         while (true) {
           const t3 = innerTokens.peek();
           if (t3.kind === SyntaxKind.EOS) {
@@ -271,25 +307,58 @@ export class Parser {
     const t2 = tokens.peek();
     if (t2.kind === SyntaxKind.RArrow) {
       tokens.get();
-      returnType = this.parseTypeDecl(tokens, t2);
+      returnType = this.parseTypeDecl(tokens);
     }
 
     // Parse function body
 
     const t3 = tokens.peek();
     if (t3.kind === SyntaxKind.Braced) {
-      body = this.parseStmts(tokens, t3);
+      tokens.get();
+      switch (target) {
+        case "Bolt":
+          body = this.parseStmts(tokens, t3);
+          break;
+        case "JS":
+          body = acorn.parse(t3.text).body;
+          break;
+        default:
+          throw new Error(`Unrecognised language: ${target}`);
+      }
     }
 
-    return new FuncDecl(name, params, returnType, body, null, origNode)
+    return new FuncDecl(target, name, params, returnType, body, null, origNode)
 
   }
 
-  parseDecl(tokens: TokenStream, origNode: Syntax | null) {
+  parseCallExpr(tokens: TokenStream) {
-    const t0 = tokens.peek(1);   
+
-    if (t0.kind === SyntaxKind.Identifier && t0.text === 'fn') {
+    const operator = this.parsePrimExpr(tokens)
-      this.parseFuncDecl(tokens, origNode)
+    const args: Expr[] = []
+
+    const t2 = tokens.get();
+    if (t2.kind !== SyntaxKind.Parenthesized) {
+      throw new ParseError(t2, [SyntaxKind.Parenthesized])
     }
+
+    const innerTokens = t2.toTokenStream();
+
+    while (true) {
+      const t3 = innerTokens.peek();
+      if (t3.kind === SyntaxKind.EOS) {
+        break; 
+      }
+      args.push(this.parseExpr(innerTokens))
+      const t4 = innerTokens.get();
+      if (t4.kind === SyntaxKind.EOS) {
+        break
+      } else if (t4.kind !== SyntaxKind.Comma){
+        throw new ParseError(t4, [SyntaxKind.Comma])
+      }
+    }
+
+    return new CallExpr(operator, args, null)
+
   }
 
 }

src/scanner.ts

@@ -19,7 +19,10 @@ import {
   SourceFile,
   Semi,
   Comma,
-  Colon
+  StringLiteral,
+  IntegerLiteral,
+  Colon,
+  EOS,
 } from "./ast"
 
 function escapeChar(ch: string) {
@@ -57,7 +60,7 @@ function getPunctType(ch: string) {
     case '}':
       return PunctType.Brace;
     default:
-      throw new Error(`given character is not a valid punctuator`)
+      return null;
   }
 }
 
@@ -123,11 +126,12 @@ const EOF = ''
 export class Scanner {
 
   protected buffer: string[] = [];
-  protected currPos = new TextPos(0,1,1);
+  protected scanned: Token[] = [];
+  protected currPos: TextPos;
   protected offset = 0;
 
-  constructor(public file: TextFile, public input: string) {
+  constructor(public file: TextFile, public input: string, startPos = new TextPos(0,1,1)) {
-    
+    this.currPos = startPos;
   }
 
   protected readChar() {
@@ -178,7 +182,7 @@ export class Scanner {
     return text;
   }
 
-  scanToken(): Token | null {
+  scanToken(): Token {
 
     while (true) {
 
@@ -189,12 +193,11 @@ export class Scanner {
         continue;
       }
 
-      if (c0 == EOF) {
-        return null;
-      }
-
       const startPos = this.currPos.clone()
 
+      if (c0 == EOF) {
+        return new EOS(new TextSpan(this.file, startPos, startPos));
+      }
 
       switch (c0) {
         case ';':
@@ -208,40 +211,57 @@ export class Scanner {
           return new Colon(new TextSpan(this.file, startPos, this.currPos.clone()));
       }
 
-      if (isOpenPunct(c0)) {
+      if (c0 === '"') {
+
+        this.getChar();
+
+        let text = ''
+
+        while (true) {
+          const c1 = this.getChar();
+          if (c1 === EOF) {
+            throw new ScanError(this.file, this.currPos.clone(), EOF);
+          }
+          if (c1 === '"') {
+            break;
+          } else if (c1 === '\\') {
+            this.scanEscapeSequence()
+          } else {
+            text += c1
+          }
+        }
+
+        const endPos = this.currPos.clone();
+
+        return new StringLiteral(text, new TextSpan(this.file, startPos, endPos))
+
+      } else if (isOpenPunct(c0)) {
 
         this.getChar();
 
         const punctType = getPunctType(c0);
-        const elements: Token[] = [];
+        let punctCount = 1;
+        let text = ''
 
         while (true) {
 
-          const c1 = this.peekChar();
+          const c1 = this.getChar();
-
-          if (isWhiteSpace(c1)) {
-            this.getChar()
-            continue;
-          }
 
           if (c1 === EOF) {
             throw new ScanError(this.file, this.currPos.clone(), EOF)
           }
 
-          if (isClosePunct(c1)) {
+          if (punctType == getPunctType(c1)) {
-            if (punctType == getPunctType(c1)) {
+            if (isClosePunct(c1)) {
-              this.getChar();
+              punctCount--;
-              break;
+              if (punctCount === 0)
+                break;
             } else {
-              throw new ScanError(this.file, this.currPos, c1);
+              punctCount++;
             }
           }
 
-          const token = this.scanToken();
+          text += c1
-          if (token === null) {
-            throw new ScanError(this.file, this.currPos.clone(), EOF)
-          }
-          elements.push(token!);
 
         }
 
@@ -249,11 +269,11 @@ export class Scanner {
 
         switch (punctType) {
           case PunctType.Brace:
-            return new Braced(elements, new TextSpan(this.file, startPos, endPos));
+            return new Braced(text, new TextSpan(this.file, startPos, endPos));
           case PunctType.Paren:
-            return new Parenthesized(elements, new TextSpan(this.file, startPos, endPos));
+            return new Parenthesized(text, new TextSpan(this.file, startPos, endPos));
           case PunctType.Bracket:
-            return new Bracketed(elements, new TextSpan(this.file, startPos, endPos));
+            return new Bracketed(text, new TextSpan(this.file, startPos, endPos));
           default:
             throw new Error("Got an invalid state.")
         }
@@ -286,6 +306,19 @@ export class Scanner {
 
   }
 
+  peek(count = 1): Token {
+    while (this.scanned.length < count) {
+      this.scanned.push(this.scanToken());
+    }
+    return this.scanned[count - 1];
+  }
+
+  get(): Token {
+    return this.scanned.length > 0
+      ? this.scanned.shift()!
+      : this.scanToken();
+  }
+
   scan() {
 
     const elements: Decl[] = []
@@ -297,7 +330,7 @@ export class Scanner {
 
       inner: while (true) {
         const token = this.scanToken();
-        if (token === null) {
+        if (token.kind === SyntaxKind.EOS) {
           if (tokens.length === 0) {
             break outer;
           } else {
@@ -326,4 +359,3 @@ export class Scanner {
   }
 
 }
-