Major update to code

- Many small fixes to parser and scanner
 - Extended type checking algorithm for existing nodes
 - Add partial support for module definitions and records
 - Add an evaluator
 - Prepare for new version of macro system

package-lock.json

@@ -24,6 +24,14 @@
       "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
       "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
     },
+    "@types/fs-extra": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.0.tgz",
+      "integrity": "sha512-UoOfVEzAUpeSPmjm7h1uk5MH6KZma2z2O7a75onTGjnNvAvMVrPzPL/vBbT65iIGHWj6rokwfmYcmxmlSf2uwg==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/mocha": {
       "version": "7.0.1",
       "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-7.0.1.tgz",

package.json

@@ -13,6 +13,7 @@
   "license": "GPL-3.0",
   "repository": "https://github.com/samvv/Bolt",
   "dependencies": {
+    "@types/fs-extra": "^8.1.0",
     "@types/node": "^13.7.4",
     "@types/xregexp": "^3.0.30",
     "@types/yargs": "^15.0.3",

src/ast.ts

@@ -3,6 +3,9 @@
 
 import { Stream, StreamWrapper } from "./util"
 import { Scanner } from "./scanner"
+import { RecordType, PrimType, OptionType, VariantType, stringType, intType, boolType } from "./checker"
+import { bind } from "./bindings"
+import { Value } from "./evaluator"
 
 interface JsonArray extends Array<Json> {  };
 interface JsonObject { [key: string]: Json }
@@ -30,15 +33,21 @@ export enum SyntaxKind {
 
   // Keywords
 
-  FunctionKeyword,
+  FnKeyword,
   ForeignKeyword,
   LetKeyword,
   ImportKeyword,
+  PubKeyword,
+  ModKeyword,
+  EnumKeyword,
+  StructKeyword,
 
   // Special nodes
 
   SourceFile,
 
+  Module,
+
   QualName,
 
   Sentence,
@@ -50,17 +59,21 @@ export enum SyntaxKind {
   // Patterns
 
   BindPatt,
-  ExprPatt,
+  TypePatt,
+  RecordPatt,
+  TuplePatt,
 
   // Expressions
 
   CallExpr,
   ConstExpr,
   RefExpr,
+  MatchExpr,
 
-  // Stmts
+  // Statements
 
   RetStmt,
+  CondStmt,
 
   // Type declarations
 
@@ -71,16 +84,11 @@ export enum SyntaxKind {
   VarDecl,
   FuncDecl,
   ImportDecl,
+  RecordDecl,
+  VariantDecl,
 
 }
 
-enum EdgeType {
-  Primitive  = 1,
-  Node       = 2,
-  Nullable   = 4,
-  List       = 8,
-}
-
 export class TextFile {
 
   constructor(public path: string) {
@@ -167,6 +175,9 @@ abstract class SyntaxBase {
 
 }
 
+export const fileType = new PrimType();
+
+@bind('Bolt.AST.StringLiteral')
 export class StringLiteral extends SyntaxBase {
 
   kind: SyntaxKind.StringLiteral = SyntaxKind.StringLiteral;
@@ -198,7 +209,7 @@ export class IntegerLiteral extends SyntaxBase {
   kind: SyntaxKind.IntegerLiteral = SyntaxKind.IntegerLiteral;
 
   constructor(
-    public value: string,
+    public value: bigint,
     public span: TextSpan | null = null,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -250,11 +261,7 @@ export class EOS extends SyntaxBase {
 
 }
 
-export class Parenthesized extends SyntaxBase {
+export abstract class Punctuated extends SyntaxBase {
-
-  kind: SyntaxKind.Parenthesized = SyntaxKind.Parenthesized;
-
-  protected buffered = null;
 
   constructor(
     public text: string,
@@ -265,38 +272,10 @@ export class Parenthesized extends SyntaxBase {
     super();
   }
 
-  toTokenStream() {
+  toSentences() {
     const span = this.getSpan();
-    const startPos = span.start;
+    const scanner = new Scanner(span.file, this.text)
-    return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1));
+    return scanner.scanTokens()
-  }
-
-  toJSON(): Json {
-    return {
-      kind: 'Parenthesized',
-      text: this.text,
-      span: this.span !== null ? this.span.toJSON() : null,
-    }
-  }
-
-  *getChildren(): IterableIterator<Syntax> {
-
-  }
-
-}
-
-
-export class Braced extends SyntaxBase {
-
-  kind: SyntaxKind.Braced = SyntaxKind.Braced;
-
-  constructor(
-    public text: string,
-    public span: TextSpan,
-    public origNode: [Syntax, Syntax] | Syntax | null = null,
-    public parentNode: Syntax | null = null
-  ) {
-    super();
   }
 
   toTokenStream() {
@@ -319,37 +298,16 @@ export class Braced extends SyntaxBase {
 
 }
 
-export class Bracketed extends SyntaxBase {
+export class Parenthesized extends Punctuated {
+  kind: SyntaxKind.Parenthesized = SyntaxKind.Parenthesized;
+}
 
+export class Braced extends Punctuated {
+  kind: SyntaxKind.Braced = SyntaxKind.Braced;
+}
+
+export class Bracketed extends Punctuated {
   kind: SyntaxKind.Bracketed = SyntaxKind.Bracketed;
-
-  constructor(
-    public text: string,
-    public span: TextSpan,
-    public origNode: [Syntax, Syntax] | Syntax | null = null,
-    public parentNode: Syntax | null = null
-  ) {
-    super();
-  }
-
-  toTokenStream() {
-    const span = this.getSpan();
-    const startPos = span.start;
-    return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1));
-  }
-
-  toJSON(): Json {
-    return {
-      kind: 'Bracketed',
-      text: this.text,
-      span: this.span !== null ? this.span.toJSON() : null,
-    }
-  }
-
-  *getChildren(): IterableIterator<Syntax> {
-
-  }
-
 }
 
 export class Identifier extends SyntaxBase {
@@ -625,6 +583,14 @@ export class QualName {
 
   }
 
+  get fullText() {
+    let out = ''
+    for (const chunk of this.path) {
+      out += chunk.text + '.'
+    }
+    return out + this.name
+  }
+
   toJSON(): Json {
     return {
       kind: 'QualName',
@@ -707,8 +673,82 @@ export class BindPatt extends SyntaxBase {
 
 }
 
+export interface RecordPattField {
+  name: Identifier;
+  pattern: Patt,
+}
+
+export class RecordPatt extends SyntaxBase {
+
+  kind: SyntaxKind.RecordPatt = SyntaxKind.RecordPatt;
+
+  constructor(
+    public typeDecl: TypeDecl,
+    public fields: RecordPattField[],
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+    for (const field of this.fields) {
+      yield field.name;
+      yield field.pattern;
+    }
+  }
+
+}
+
+export class TuplePatt extends SyntaxBase {
+
+  kind: SyntaxKind.TuplePatt = SyntaxKind.TuplePatt;
+
+  constructor(
+    public elements: Patt[],
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+    for (const element of this.elements) {
+      yield element;
+    }
+  }
+
+}
+
+export class TypePatt extends SyntaxBase {
+
+  kind: SyntaxKind.TypePatt = SyntaxKind.TypePatt;
+
+  constructor(
+    public typeDecl: TypeDecl,
+    public pattern: Patt,
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  *getChildren() {
+    yield this.typeDecl;
+    yield this.pattern;
+  }
+
+}
+
 export type Patt
   = BindPatt
+  | Expr
+  | TypePatt
+  | TuplePatt 
+  | RecordPatt
 
 export class RefExpr extends SyntaxBase {
 
@@ -769,12 +809,44 @@ export class CallExpr extends SyntaxBase {
 
 }
 
+export type MatchArm = [Patt, Expr | Stmt[]];
+
+export class MatchExpr extends SyntaxBase {
+
+  kind: SyntaxKind.MatchExpr = SyntaxKind.MatchExpr;
+
+  constructor(
+    public value: Expr,
+    public arms: MatchArm[],
+    public span: TextSpan | null = null, 
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+    yield this.value;
+    for (const [pattern, result] of this.arms) {
+      yield pattern
+      if (Array.isArray(result)) {
+        for (const stmt of result) {
+          yield stmt
+        }
+      } else {
+        yield result
+      }
+    }
+  }
+
+}
+
 export class ConstExpr extends SyntaxBase {
 
   kind: SyntaxKind.ConstExpr = SyntaxKind.ConstExpr;
 
   constructor(
-    public value: string | bigint, 
+    public value: Value,
     public span: TextSpan | null = null, 
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -801,6 +873,7 @@ export type Expr
   = ConstExpr
   | RefExpr
   | CallExpr
+  | MatchExpr
 
 export class RetStmt extends SyntaxBase {
 
@@ -831,8 +904,38 @@ export class RetStmt extends SyntaxBase {
 
 }
 
+export interface Case {
+  conditional: Expr,
+  consequent: Stmt[]
+}
+
+export class CondStmt {
+  
+  kind: SyntaxKind.CondStmt = SyntaxKind.CondStmt
+
+  constructor(
+    public cases: Case[],
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+    for (const kase of this.cases) {
+      yield kase.conditional
+      for (const stmt of kase.consequent) {
+        yield stmt
+      }
+    }
+  }
+
+}
+
 export type Stmt
   = RetStmt
+  | CondStmt
 
 export class TypeRef extends SyntaxBase {
 
@@ -874,6 +977,7 @@ export class FuncDecl extends SyntaxBase {
   kind: SyntaxKind.FuncDecl = SyntaxKind.FuncDecl;
 
   constructor(
+    public isPublic: boolean,
     public target: string,
     public name: QualName, 
     public params: Param[],
@@ -889,6 +993,7 @@ export class FuncDecl extends SyntaxBase {
   toJSON(): Json {
     return {
       kind: 'FuncDecl',
+      isPublic: this.isPublic,
       target: this.target,
       name: this.name.toJSON(),
       params: this.params.map(p => p.toJSON()),
@@ -972,12 +1077,40 @@ export class ImportDecl {
 
 }
 
+export class RecordDecl extends SyntaxBase {
+
+  kind: SyntaxKind.RecordDecl = SyntaxKind.RecordDecl;
+
+  fields: Map<Identifier, TypeDecl>;
+
+  constructor(
+    public isPublic: boolean,
+    public name: QualName,
+    fields: Iterable<[Identifier, TypeDecl]>,
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super()
+    this.fields = new Map(fields);
+  }
+
+  *getChildren() {
+    yield this.name;
+    for (const [name, typeDecl] of this.fields) {
+      yield name
+      yield typeDecl
+    }
+  }
+
+}
+
 export type Decl
   = Sentence 
   | FuncDecl
   | ImportDecl
   | VarDecl
-
+  | RecordDecl
 
 export type Syntax
   = Decl
@@ -986,17 +1119,43 @@ export type Syntax
   | Stmt
   | Patt
   | TypeDecl
+  | Module
   | SourceFile
   | QualName
   | Param
   | EOS
 
+export type SourceElement = (Module | Decl | Stmt | Expr);
+
+export class Module extends SyntaxBase {
+
+  kind: SyntaxKind.Module = SyntaxKind.Module;
+
+  constructor(
+    public isPublic: boolean,
+    public name: QualName,
+    public elements: SourceElement[],
+    public span: TextSpan | null = null,
+    public origNode: [Syntax, Syntax] | Syntax | null = null,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  *getChildren(): IterableIterator<Syntax> {
+    for (const element of this.elements) {
+      yield element;
+    }
+  }
+
+}
+
 export class SourceFile extends SyntaxBase {
 
   kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile;
 
   constructor(
-    public elements: (Decl | Stmt | Expr)[],
+    public elements: SourceElement[],
     public span: TextSpan | null = null,
     public origNode: [Syntax, Syntax] | Syntax | null = null,
     public parentNode: Syntax | null = null
@@ -1021,9 +1180,15 @@ export class SourceFile extends SyntaxBase {
 }
 
 export function isExpr(node: Syntax): node is Expr {
+
   return node.kind === SyntaxKind.ConstExpr || node.kind === SyntaxKind.CallExpr;
 }
 
+export function isNode(value: any): value is Syntax {
+  return typeof value.kind === 'number'
+      && Object.prototype.hasOwnProperty.call(SyntaxKind, value.kind)
+}
+
 export function isJSNode(node: Syntax) {
   return typeof node.type === 'string'
 }

src/bin/bolt.ts

@@ -14,6 +14,7 @@ import { Parser } from "../parser"
 import { Expander } from "../expander"
 import { TypeChecker } from "../checker"
 import { Compiler } from "../compiler"
+import { Evaluator } from "../evaluator"
 import { Emitter } from "../emitter"
 import { TextFile, SourceFile, setParents } from "../ast"
 
@@ -129,9 +130,12 @@ yargs
 
       }
 
+      const checker = new TypeChecker()
+
       for (const sourceFile of sourceFiles) {
         const parser = new Parser()
-        const expander = new Expander(parser)
+        const evaluator = new Evaluator(checker)
+        const expander = new Expander(parser, evaluator, checker)
         const expandedSourceFile = expander.getFullyExpanded(sourceFile)
 
         for (const hook of hooks) {
@@ -166,20 +170,21 @@ yargs
     args => {
 
       const parser = new Parser()
+      const checker = new TypeChecker()
 
       const sourceFiles = toArray(args.files as string[]).map(filepath => {
         const file = new TextFile(filepath)
         const contents = fs.readFileSync(filepath, 'utf8')
         const scanner = new Scanner(file, contents)
         const sourceFile = scanner.scan();
-        const expander = new Expander(parser)
+        const evaluator = new Evaluator(checker)
-        const expanded = expander.getFullyExpanded(sourceFile)
+        const expander = new Expander(parser, evaluator, checker)
+        const expanded = expander.getFullyExpanded(sourceFile) as SourceFile;
         // console.log(require('util').inspect(expanded.toJSON(), { colors: true, depth: Infinity }))
         setParents(expanded)
         return expanded;
       })
 
-      const checker = new TypeChecker()
       const compiler = new Compiler(checker, { target: "JS" })
       const bundle = compiler.compile(sourceFiles)
       const emitter = new Emitter()
@@ -196,6 +201,26 @@ yargs
 
   )
 
+  .command(
+
+    'dump [file]',
+    'Dump a representation of a given primitive node to disk',
+
+    yargs => yargs,
+
+    args => {
+
+      const file = new TextFile(args.file as string)
+      const contents = fs.readFileSync(args.file, 'utf8')
+      const scanner = new Scanner(file, contents)
+      const parser = new Parser();
+      const patt = parser.parsePattern(scanner)
+      console.log(JSON.stringify(patt.toJSON(), undefined, 2))
+
+    }
+
+  )
+
   .help()
   .version()
   .argv

src/bindings.ts

@@ -0,0 +1,22 @@
+
+import { Value, RecordValue } from "./evaluator"
+
+interface Binding {
+  name: string;
+  createValue: () => Value,
+}
+
+export const bindings = new Map<string, Binding>();
+
+export function bind(name: string) {
+  return function (target: any) {
+    if (bindings.has(name)) {
+      throw new Error(`A binding with the name '${name}' already exists.`)
+    }
+    bindings.set(name, {
+      name,
+      createValue: (...args) => new RecordValue(target.META_TYPE, new target(...args)),
+    });
+  }
+}
+

src/checker.ts

@@ -3,14 +3,55 @@ import {
   Syntax,
   SyntaxKind,
   ImportDecl,
+  isNode,
 } from "./ast"
 
-class Type {
+import { FastStringMap } from "./util"
+
+export class Type {
   
 }
 
-interface FastStringMap<T> {
+export class PrimType extends Type {
-  [key: string]: T
+
+}
+
+export class VariantType extends Type {
+
+  constructor(public elementTypes: Type[]) {
+    super();
+  }
+
+}
+
+export const stringType = new PrimType()
+export const intType = new PrimType()
+export const boolType = new PrimType()
+export const voidType = new PrimType()
+
+export class RecordType {
+
+  fieldTypes: FastStringMap<Type> = Object.create(null);
+
+  constructor(
+    iterable: IterableIterator<[string, Type]>,
+  ) {
+    for (const [name, typ] of iterable) {
+      this.fieldTypes[name] = typ;
+    }
+  }
+
+  hasField(name: string) {
+    return name in this.fieldTypes;
+  }
+
+  getTypeOfField(name: string) {
+    if (name in this.fieldTypes) {
+      return this.fieldTypes[name]
+    }
+    throw new Error(`Field '${name}' does not exist on this record type.`)
+  }
+
 }
 
 export class Scope {
@@ -21,22 +62,117 @@ export class Scope {
 
 }
 
+function* map<T, R>(iterable: Iterable<T>, proc: (value: T) => R): IterableIterator<R> {
+  const iterator = iterable[Symbol.iterator]();
+  while (true) {
+    let { done, value }= iterator.next();
+    if (done) {
+      break
+    }
+    yield proc(value)
+  }
+}
+
+function getFullName(node: Syntax) {
+  let out = []
+  let curr: Syntax | null = node;
+  while (true) {
+    switch (curr.kind) {
+      case SyntaxKind.Module:
+        out.unshift(curr.name.fullText);
+        break;
+      case SyntaxKind.RecordDecl:
+        out.unshift(curr.name.fullText)
+        break;
+    } 
+    curr = curr.parentNode;
+    if (curr === null) {
+      break;
+    }
+  }
+  return out.join('.');
+}
+
 export class TypeChecker {
 
-  protected stringType = new Type();
+  protected symbols: FastStringMap<Type> = Object.create(null)
-  protected intType = new Type();
+  protected types = new Map<Syntax, Type>();
-
   protected scopes = new Map<Syntax, Scope>();
 
-  protected createType(node: Syntax) {
+  protected createType(node: Syntax): Type {
+
+    console.log(node)
+
     switch (node.kind) {
+
       case SyntaxKind.ConstExpr:
-        if (typeof node.value === 'bigint') {
+        return node.value.type;
-          return this.intType;
+
-        } else if (typeof node.value === 'string') {
+      case SyntaxKind.RecordDecl:
-          return this.stringType;
+
+        const typ = new RecordType(map(node.fields, ([name, typ]) => ([name.text, typ])));
+
+        this.symbols[getFullName(node)] = typ;
+
+        return typ;
+
+        // if (typeof node.value === 'bigint') {
+        //   return intType;
+        // } else if (typeof node.value === 'string') {
+        //   return stringType;
+        // } else if (typeof node.value === 'boolean') {
+        //   return boolType;
+        // } else if (isNode(node.value)) {
+        //   return this.getTypeNamed(`Bolt.AST.${SyntaxKind[node.value.kind]}`)!
+        // } else {
+        //   throw new Error(`Unrecognised kind of value associated with ConstExpr`)
+        // }
+
+      default:
+        throw new Error(`Could not derive type of ${SyntaxKind[node.kind]}`)
+
     }
+
   }
+
+  getTypeNamed(name: string) {
+    return name in this.typeNames
+      ? this.typeNames[name]
+      : null
+  }
+
+  getTypeOfNode(node: Syntax): Type {
+    if (this.types.has(node)) {
+      return this.types.get(node)!
+    }
+    const newType = this.createType(node)
+    this.types.set(node, newType)
+    return newType;
+  }
+
+  check(node: Syntax) {
+
+    switch (node.kind) {
+
+      case SyntaxKind.Sentence:
+        break;
+
+      case SyntaxKind.RecordDecl:
+        this.getTypeOfNode(node);
+        break;
+
+      case SyntaxKind.Module:
+      case SyntaxKind.SourceFile:
+        for (const element of node.elements) {
+          this.check(element)
+        }
+        break;
+
+      default:
+        throw new Error(`Could not type-check node ${SyntaxKind[node.kind]}`)
+
+    }
+
   }
 
   getImportedSymbols(node: ImportDecl) {
@@ -55,9 +191,9 @@ export class TypeChecker {
     return scope
   }
 
-  getMapperForNode(target: string, node: Syntax): Mapper {
+  // getMapperForNode(target: string, node: Syntax): Mapper {
-    return this.getScope(node).getMapper(target)
+  //   return this.getScope(node).getMapper(target)
-  }
+  // }
 
 }
 

src/compiler.ts

@@ -99,7 +99,7 @@ export class Compiler {
       case SyntaxKind.ImportDecl:
         preamble.push({
           type: 'ImportDeclaration',
-          source: { type: 'Literal', value: node.file },
+          source: { type: 'Literal', value: node.file + '.mjs' },
           specifiers: this.checker.getImportedSymbols(node).map(s => ({
             type: 'ImportSpecifier',
             imported: { type: 'Identifier', name: s.name },
@@ -123,9 +123,17 @@ export class Compiler {
 
       case SyntaxKind.FuncDecl:
         const params = [];
+        if (node.body !== null) {
+          let body;
           if (node.target === this.target) {
-          console.log(node.body)
+            body = node.body;
-          preamble.push({
+          } else if (node.target === 'Bolt') {
+            let body: Stmt[] = [];
+            for (const stmt in node.body) {
+              this.compileDecl(stmt, body)
+            }
+          }
+          let result = {
             type: 'FunctionDeclaration',
             id: { type: 'Identifier',  name: node.name.name.text },
             params: node.params.map(p => ({ type: 'Identifier', name: p.bindings.name.text })),
@@ -133,9 +141,14 @@ export class Compiler {
               type: 'BlockStatement',
               body: node.body
             }
-          })
+          }
-        } else {
+          if (node.isPublic) {
-          throw new Error(`Cannot yet compile Bolt functions`)
+            result = {
+              type: 'ExportNamedDeclaration',
+              declaration: result,
+            }
+          }
+          preamble.push(result)
         }
         break;
 

src/evaluator.ts

@@ -0,0 +1,126 @@
+
+import { Syntax, SyntaxKind, Expr, isNode } from "./ast"
+import { TypeChecker, Type, RecordType, PrimType, boolType } from "./checker"
+import { FastStringMap } from "./util"
+
+export interface Value {
+  type: Type;
+}
+
+export class PrimValue implements Value {
+
+  constructor(
+    public type: PrimType,
+    public value: any
+  ) {
+    
+  }
+
+}
+
+export const TRUE = new PrimValue(boolType, true);
+export const FALSE = new PrimValue(boolType, false);
+
+export abstract class RecordValue implements Value {
+
+  abstract type: RecordType;
+
+  abstract getValueOfField(name: string): Value;
+
+}
+
+export class NativeRecord implements Value {
+
+  constructor(
+    public type: RecordType,
+    protected fields: FastStringMap<Value>,
+  ) {
+    
+  }
+
+  getValueOfField(name: string): Value {
+    if (!this.type.hasField(name)) {
+      throw new Error(`Field '${name}' does not exist on this record.`)
+    }
+    return this.fields[name]
+  }
+
+}
+
+export class RecordWrapper extends RecordValue {
+
+  constructor(
+    public type: RecordType,
+    protected data: any,
+  ) {
+    super();
+  }
+
+  getValueOfField(name: string): Value {
+    if (!this.type.hasField(name)) {
+      throw new Error(`Field '${name}' does not exist on this record.`)
+    }
+    return this.data[name]
+  }
+
+}
+
+export class Evaluator {
+
+  constructor(public checker: TypeChecker) {
+
+  }
+
+  match(value: Value, node: Syntax) {
+
+    switch (node.kind) {
+
+      case SyntaxKind.RecordPatt:
+        for (const field of node.fields) {
+          if (!this.match((value as RecordValue).getValueOfField(field.name.text), field.pattern)) {
+            return false;
+          }
+        }
+        return true;
+
+      case SyntaxKind.TypePatt:
+        return value.type === this.checker.getTypeOfNode(node)
+
+      default:
+        throw new Error(`I did not know how to match on pattern ${SyntaxKind[node.kind]}`)
+
+    }
+
+  }
+
+  createValue(data: any) {
+    if (isNode(data)) {
+      return new RecordWrapper(this.checker.getTypeNamed(`Bolt.AST.${SyntaxKind[data.kind]}`)! as RecordType, data)
+    }
+  }
+
+  eval(node: Syntax): Value { 
+
+    switch (node.kind) {
+
+      case SyntaxKind.MatchExpr:
+        const value = this.eval(node.value);
+        for (const [pattern, result] of node.arms) {
+          if (this.match(value, pattern)) {
+            return this.eval(result as Expr)
+          }
+        }
+        return new PrimValue(this.checker.getTypeNamed('Void')!, null);
+
+      case SyntaxKind.ConstExpr:
+        return new PrimValue(this.checker.getTypeOfNode(node), node.value)
+
+      default:
+        throw new Error(`Could not evaluate node ${SyntaxKind[node.kind]}`)
+
+    }
+
+  }
+
+}
+

src/expander.ts

@@ -1,72 +1,192 @@
 
+// FIXME Actually, the syntax expander could make use of algebraic effects to
+// easily specify how the next expansion should happen. Just a thought.
+
 import { 
   TokenStream,
   SyntaxKind,
   Syntax, 
   SourceFile, 
   Decl,
-  Statement
+  RecordPatt,
+  Identifier,
+  TypeRef,
+  Patt,
+  ConstExpr,
+  QualName,
+  TuplePatt,
+  BindPatt,
+  TypePatt,
+  MatchExpr,
+  Stmt,
+  Module,
 } from "./ast"
 
+import { TypeChecker } from "./checker"
 import { Parser, ParseError } from "./parser"
+import { Evaluator, TRUE, FALSE } from "./evaluator"
 
-type Transformer = (tokens: TokenStream) => Syntax;
+interface Transformer {
+  pattern: Patt;
+  transform: (node: TokenStream) => Syntax;
+}
+
+function createTypeRef(text: string) {
+  const ids = text.split('.').map(name => new Identifier(name))
+  return new TypeRef(new QualName(ids[ids.length-1], ids.slice(0, -1)), [])
+}
+
+/// This is actually a hand-parsed version of the following:
+///
+/// Bolt.AST.Braced {
+///   elements = [
+///     Bolt.AST.Identifier { text = "name" },
+///     Bolt.AST.Braced {
+///       elements = [
+///         pattern: Bolt.AST.Pattern,
+///         _: RArrow,
+///         expression: Bolt.AST.Expr
+///       ]
+///     }
+///   ],
+/// }
+const PATTERN_SYNTAX: Patt = 
+  new RecordPatt(
+    createTypeRef('Bolt.AST.Sentence'),
+    [{
+      name: new Identifier('elements'),
+      pattern: new TuplePatt([
+        new RecordPatt(
+          createTypeRef('Bolt.AST.Identifier'),
+          [{
+            name: new Identifier('text'), 
+            pattern: new ConstExpr('syntax')
+          }]
+        ),
+        new RecordPatt(
+          createTypeRef('Bolt.AST.Braced'),
+          [{
+            name: new Identifier('elements'),
+            pattern: new TuplePatt([
+              new TypePatt(createTypeRef('Bolt.AST.Pattern'), new BindPatt(new Identifier('pattern'))),
+              new TypePatt(createTypeRef('Bolt.AST.RArrow'), new BindPatt(new Identifier('_'))),
+              new TypePatt(createTypeRef('Bolt.AST.Expr'), new BindPatt(new Identifier('expression')))
+            ])
+          }]
+        )
+      ])
+    }]
+  )
 
 export class Expander {
 
-  transformers = new Map<string, Transformer>();
+  protected transformers: Transformer[] = []
 
-  constructor(public parser: Parser) {
+  constructor(public parser: Parser, public evaluator: Evaluator, public checker: TypeChecker) {
-    this.transformers.set('fn', parser.parseFuncDecl.bind(parser))
+    // this.transformers.push({
-    this.transformers.set('import', parser.parseImportDecl.bind(parser))
+    //   pattern: PATTERN_SYNTAX,
-    this.transformers.set('foreign', parser.parseFuncDecl.bind(parser))
+    //   transform: this.parser.parseSyntax.bind(this.parser)
-    this.transformers.set('let', parser.parseVarDecl.bind(parser))
+    // })
-    this.transformers.set('return', parser.parseRetStmt.bind(parser))
   }
 
   getFullyExpanded(node: Syntax): Syntax {
 
     if (node.kind === SyntaxKind.SourceFile) {
 
-      const expanded: (Decl | Statement)[] = [];
+      const expanded: (Decl | Stmt)[] = [];
+
+      let didExpand = false;
+
       for (const element of node.elements) {
-        if (element.kind === SyntaxKind.Sentence) {
+        let newElement = this.getFullyExpanded(element);
-          const newElement = this.getFullyExpanded(element)
+        if (newElement !== element) {
-          expanded.push(newElement as Decl | Statement)
+          didExpand = true;
         }
+        expanded.push(newElement as Decl | Stmt)
       }
+
+      if (!didExpand) {
+        return node;
+      }
+
       return new SourceFile(expanded, null, node);
 
+    } else if (node.kind == SyntaxKind.Module) {
+
+      const expanded = [];
+
+      let didExpand = false;
+
+      for (const element of node.elements) {
+        let newElement = this.getFullyExpanded(element);
+        if (newElement !== element) {
+          didExpand = true;
+        }
+        expanded.push(newElement as Decl | Stmt)
+      }
+
+      if (!didExpand) {
+        return node;
+      }
+
+      return new Module(node.isPublic, node.name, node.elements, null, node);
+
     } else if (node.kind === SyntaxKind.Sentence) {
 
+      let newElement;
+
+      const tokens = node.toTokenStream();
+
+      try {
+
+        newElement = this.parser.parseSourceElement(tokens)
+
+      } catch (e) {
+
+        // Regular errors should be propagated.
+
+        if (!(e instanceof ParseError)) {
+          throw e;
+        }
+
+        // The following applies a user-defined transformer to the token tree.
+
         while (true) {
-
+          let didExpand = false;
-        console.log('expanding sententce')
+          const expanded: Syntax[] = [];
-
+          const tokens = node.toTokenStream();
-        const tokens: TokenStream = node.toTokenStream()
+          for (const transformer of this.transformers) {
-
+            if (this.evaluator.eval(new MatchExpr(new ConstExpr(this.evaluator.createValue(node)), [
-        const t0 = tokens.peek();
+              [transformer.pattern, new ConstExpr(TRUE)],
-        if (t0.kind !== SyntaxKind.Identifier) {
+              [new ConstExpr(TRUE), new ConstExpr(FALSE)]
-          throw new ParseError(t0, [SyntaxKind.Identifier]);
+            ]))) {
+              expanded.push(transformer.transform(tokens))
+              didExpand = true;
+              // break; // FIXME
             }
-
-        if (!this.transformers.has(t0.text)) {
-          return this.parser.parseCallExpr(tokens)
           }
-
+          if (!didExpand) {
-        node = this.transformers.get(t0.text)!(tokens)
-
-        if (node.kind !== SyntaxKind.Sentence) {
             break;
           }
+        }
+
+        // If no transformer matched, then throw the original parse error.
+
+        if (!newElement) {
+          throw e;
+        }
 
       }
 
-      return node
+      // Perform a full expansion on the transformed element.
+
+      return this.getFullyExpanded(newElement)
 
     } else {
 
-      throw new Error(`unrecognised node of kind ${node.kind}`)
+      this.checker.check(node);
+
+      return node;
 
     }
 

src/parser.ts

@@ -22,8 +22,15 @@ import {
   QualName,
   CallExpr,
   ImportDecl,
+  SourceElement,
+  Module,
+  RecordDecl,
 } from "./ast"
 
+import { stringType, intType } from "./checker"
+
+import { PrimValue } from "./evaluator"
+
 function describeKind(kind: SyntaxKind): string {
   switch (kind) {
     case SyntaxKind.Identifier:
@@ -34,10 +41,12 @@ function describeKind(kind: SyntaxKind): string {
       return "a string"
     case SyntaxKind.IntegerLiteral:
       return "an integer"
-    case SyntaxKind.FunctionKeyword:
+    case SyntaxKind.FnKeyword:
       return "'fn'"
     case SyntaxKind.ForeignKeyword:
       return "'foreign'"
+    case SyntaxKind.PubKeyword:
+      return "'pub'"
     case SyntaxKind.LetKeyword:
       return "'let'"
     case SyntaxKind.Semi:
@@ -48,6 +57,12 @@ function describeKind(kind: SyntaxKind): string {
       return "'.'"
     case SyntaxKind.Comma:
       return "','"
+    case SyntaxKind.ModKeyword:
+      return "'mod'"
+    case SyntaxKind.StructKeyword:
+      return "'struct'"
+    case SyntaxKind.EnumKeyword:
+      return "'enum'"
     case SyntaxKind.Braced:
       return "'{' .. '}'"
     case SyntaxKind.Bracketed:
@@ -78,12 +93,7 @@ export class ParseError extends Error {
 
 export class Parser {
 
-
+  parseQualName(tokens: TokenStream): QualName {
-  constructor() {
-
-  }
-
-  parseQualName(tokens: TokenStream) {
 
     const path: Identifier[] = [];
 
@@ -116,7 +126,7 @@ export class Parser {
     }
   }
 
-  parseImportDecl(tokens: TokenStream) {
+  parseImportDecl(tokens: TokenStream): ImportDecl {
 
     // Assuming first keyword is 'import'
     tokens.get();
@@ -144,7 +154,10 @@ export class Parser {
     const t0 = tokens.peek();
     if (t0.kind === SyntaxKind.StringLiteral) {
       tokens.get();
-      return new ConstExpr(t0.value, null, t0);
+      return new ConstExpr(new PrimValue(stringType, t0.value), null, t0);
+    } else if (t0.kind === SyntaxKind.IntegerLiteral) {
+      tokens.get();
+      return new ConstExpr(new PrimValue(intType, t0.value), null, t0);
     } else if (t0.kind === SyntaxKind.Identifier) {
       const name = this.parseQualName(tokens);
       return new RefExpr(name, null, name.origNode);
@@ -153,11 +166,20 @@ export class Parser {
     }
   }
 
-  parseExpr(tokens: TokenStream) {
+  parseSyntax(tokens: TokenStream): Syntax {
+
+    // Assuming first token is 'syntax'
+    tokens.get();
+
+    throw new Error('not implemented')
+
+  }
+
+  parseExpr(tokens: TokenStream): Expr {
     return this.parsePrimExpr(tokens)
   }
 
-  parseParam(tokens: TokenStream) {
+  parseParam(tokens: TokenStream): Param {
 
     let defaultValue = null;
     let typeDecl = null;
@@ -173,7 +195,9 @@ export class Parser {
         tokens.get();
         defaultValue = this.parseExpr(tokens);
       }
-    } else if (t0.kind === SyntaxKind.EqSign) {
+    }
+
+    if (t0.kind === SyntaxKind.EqSign) {
       tokens.get();
       defaultValue = this.parseExpr(tokens);
     }
@@ -230,11 +254,74 @@ export class Parser {
     return new RetStmt(expr, null, [t0, expr.getEndNode()]);
   }
 
+  parseStmt(tokens: TokenStream): Stmt {
+
+  }
+
+  parseRecordDecl(tokens: TokenStream): RecordDecl {
+
+    let isPublic = false;
+
+    let kw = tokens.get();
+    if (kw.kind !== SyntaxKind.Identifier) {
+      throw new ParseError(kw, [SyntaxKind.PubKeyword, SyntaxKind.StructKeyword]);
+    }
+    if (kw.text === 'pub') {
+      isPublic = true;
+      kw = tokens.get();
+    }
+
+    if (kw.kind !== SyntaxKind.Identifier || kw.text !== 'struct') {
+      throw new ParseError(kw, [SyntaxKind.StructKeyword])
+    }
+
+    const name = this.parseQualName(tokens);
+
+    const t2 = tokens.get();
+
+    if (t2.kind !== SyntaxKind.Braced) {
+      throw new ParseError(kw, [SyntaxKind.Braced])
+    }
+
+    let fields = [];
+  
+    return new RecordDecl(isPublic, name, fields);
+
+  }
+
   parseStmts(tokens: TokenStream, origNode: Syntax | null): Stmt[] {
     // TODO
     return []
   }
 
+  parseModDecl(tokens: TokenStream): Module {
+
+    let isPublic = false;
+
+    let kw = tokens.get();
+    if (kw.kind !== SyntaxKind.Identifier) {
+      throw new ParseError(kw, [SyntaxKind.PubKeyword, SyntaxKind.ModKeyword]);
+    }
+    if (kw.text === 'pub') {
+      isPublic = true;
+      kw = tokens.get();
+    }
+
+    if (kw.kind !== SyntaxKind.Identifier || kw.text !== 'mod') {
+      throw new ParseError(kw, [SyntaxKind.ModKeyword])
+    }
+
+    const name = this.parseQualName(tokens);
+
+    const t1 = tokens.get();
+    if (t1.kind !== SyntaxKind.Braced) {
+      throw new ParseError(t1, [SyntaxKind.Braced])
+    }
+
+    return new Module(isPublic, name, t1.toSentences());
+
+  }
+
   protected assertEmpty(tokens: TokenStream) {
     const t0 = tokens.peek(1);
     if (t0.kind !== SyntaxKind.EOS) {
@@ -242,24 +329,35 @@ export class Parser {
     }
   }
 
-  parseFuncDecl(tokens: TokenStream, origNode: Syntax | null) {
+  parseFuncDecl(tokens: TokenStream, origNode: Syntax | null): FuncDecl {
 
     let target = "Bolt";
+    let isPublic = false;
 
     const k0 = tokens.peek();
     if (k0.kind !== SyntaxKind.Identifier) {
-      throw new ParseError(k0, [SyntaxKind.ForeignKeyword, SyntaxKind.FunctionKeyword])
+      throw new ParseError(k0, [SyntaxKind.PubKeyword, SyntaxKind.ForeignKeyword, SyntaxKind.FnKeyword])
     }
-    if (k0.text === 'foreign') {
+    if (k0.text === 'pub') {
+      tokens.get();
+      isPublic = true;
+    }
+
+    const k1 = tokens.peek();
+    if (k1.kind !== SyntaxKind.Identifier) {
+      throw new ParseError(k1, [SyntaxKind.ForeignKeyword, SyntaxKind.FnKeyword])
+    }
+    if (k1.text === 'foreign') {
+      tokens.get();
       const l1 = tokens.get();
       if (l1.kind !== SyntaxKind.StringLiteral) {
         throw new ParseError(l1, [SyntaxKind.StringLiteral])
       }
       target = l1.value;
     }
-    const k1 = tokens.get();
+    const k2 = tokens.get();
-    if (k1.kind !== SyntaxKind.Identifier || k1.text !== 'fn') {
+    if (k2.kind !== SyntaxKind.Identifier || k2.text !== 'fn') {
-      throw new ParseError(k1, [SyntaxKind.FunctionKeyword])
+      throw new ParseError(k2, [SyntaxKind.FnKeyword])
     }
 
     let name: QualName;
@@ -367,11 +465,49 @@ export class Parser {
       }
     }
 
-    return new FuncDecl(target, name, params, returnType, body, null, origNode)
+    return new FuncDecl(isPublic, target, name, params, returnType, body, null, origNode)
 
   }
 
-  parseCallExpr(tokens: TokenStream) {
+  parseSourceElement(tokens: TokenStream): SourceElement {
+    const t0 = tokens.peek(1);
+    if (t0.kind === SyntaxKind.Identifier) {
+      let i = 1;
+      let kw: Token = t0;
+      if (t0.text === 'pub') {
+        i++;
+        kw = tokens.peek(i);
+        if (kw.kind !== SyntaxKind.Identifier) {
+          throw new ParseError(kw, [SyntaxKind.ForeignKeyword, SyntaxKind.ModKeyword, SyntaxKind.LetKeyword, SyntaxKind.FnKeyword, SyntaxKind.EnumKeyword, SyntaxKind.StructKeyword])
+        }
+      }
+      if (t0.text === 'foreign') {
+        i += 2;
+        kw = tokens.peek(i);
+        if (kw.kind !== SyntaxKind.Identifier) {
+          throw new ParseError(kw, [SyntaxKind.ModKeyword, SyntaxKind.LetKeyword, SyntaxKind.FnKeyword, SyntaxKind.EnumKeyword, SyntaxKind.StructKeyword])
+        }
+      }
+      switch (kw.text) {
+        case 'mod':
+          return this.parseModDecl(tokens);
+        case 'fn':
+          return this.parseFuncDecl(tokens, null);
+        case 'let':
+          return this.parseVarDecl(tokens);
+        case 'struct':
+          return this.parseRecordDecl(tokens);
+        case 'enum':
+          return this.parseVariantDecl(tokens);
+        default:
+          throw new ParseError(kw, [SyntaxKind.ModKeyword, SyntaxKind.LetKeyword, SyntaxKind.FnKeyword, SyntaxKind.EnumKeyword, SyntaxKind.StructKeyword])
+      }
+    } else {
+      return this.parseStmt(tokens)
+    }
+  }
+
+  parseCallExpr(tokens: TokenStream): CallExpr {
 
     const operator = this.parsePrimExpr(tokens)
     const args: Expr[] = []

src/scanner.ts

@@ -23,6 +23,7 @@ import {
   IntegerLiteral,
   Colon,
   EOS,
+  Dot,
   EqSign,
 } from "./ast"
 
@@ -97,6 +98,10 @@ interface Stream<T> {
   read(): T
 }
 
+function isDigit(ch: string) {
+  return XRegExp('\\p{Nd}').test(ch)
+}
+
 function isWhiteSpace(ch: string) {
   return ch == '\n' || XRegExp('\\p{Zs}').test(ch)
 }
@@ -201,6 +206,9 @@ export class Scanner {
       }
 
       switch (c0) {
+        case '.':
+          this.getChar();
+          return new Dot(new TextSpan(this.file, startPos, this.currPos.clone()));
         case '=':
           this.getChar();
           return new EqSign(new TextSpan(this.file, startPos, this.currPos.clone()));
@@ -239,6 +247,12 @@ export class Scanner {
 
         return new StringLiteral(text, new TextSpan(this.file, startPos, endPos))
 
+      } else if (isDigit(c0)) {
+
+        const digits = this.takeWhile(isDigit)
+        const endPos = this.currPos.clone();
+        return new IntegerLiteral(BigInt(digits), new TextSpan(this.file, startPos, endPos));
+
       } else if (isOpenPunct(c0)) {
 
         this.getChar();
@@ -323,10 +337,9 @@ export class Scanner {
       : this.scanToken();
   }
 
-  scan() {
+  scanTokens() {
 
-    const elements: Decl[] = []
+    const elements: Sentence[] = []
-    const startPos = this.currPos.clone()
 
     outer: while (true) {
 
@@ -351,15 +364,24 @@ export class Scanner {
       }
 
       if (tokens.length > 0) {
-        elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone())))
+        elements.push(
+          new Sentence(
+            tokens,
+            new TextSpan(this.file, tokens[0].span!.start.clone(), tokens[tokens.length-1].span!.end.clone())
+          )
+        )
       }
 
     }
 
+    return elements
+  }
+
+  scan() {
+    const startPos = this.currPos.clone();
+    const elements = this.scanTokens();
     const endPos = this.currPos.clone();
-
+    return new SourceFile(elements, new TextSpan(this.file, startPos, endPos));
-    return new SourceFile(elements, new TextSpan(this.file, startPos, endPos))
-
   }
 
 }

src/util.ts

@@ -1,4 +1,8 @@
 
+export interface FastStringMap<T> {
+  [key: string]: T
+}
+
 export interface Stream<T> {
   get(): T;
   peek(count?: number): T;