Extend scanner/parser to support more syntactic structures

src/bin/bolt.ts

@@ -2,6 +2,7 @@
 
 import "source-map-support/register"
 
+import util from "util"
 import path from "path"
 import fs from "fs"
 import yargs from "yargs"
@@ -10,6 +11,10 @@ import { Diagnostics } from "../diagnostics"
 import { Punctuator, Scanner } from "../scanner"
 import { Parser } from "../parser"
 
+function debug(value: any) {
+  console.error(util.inspect(value, { colors: true, depth: Infinity }));
+}
+
 yargs
   .string('work-dir')
   .describe('work-dir', 'Act as if run from this directory')
@@ -33,7 +38,7 @@ yargs
       const parser = new Parser(punctuated);
       const sourceFile = parser.parseSourceFile();
 
-      console.log(sourceFile);
+      debug(sourceFile.toJSON());
 
     }
   )

src/cst.ts

@@ -1,3 +1,4 @@
+import { JSONObject, JSONValue } from "./util";
 
 export type TextSpan = [number, number];
 
@@ -67,6 +68,7 @@ export const enum SyntaxKind {
 
   // Tokens
   Identifier,
+  Constructor,
   CustomOperator,
   LParen,
   RParen,
@@ -88,6 +90,11 @@ export const enum SyntaxKind {
   ImportKeyword,
   StructKeyword,
   TypeKeyword,
+  ReturnKeyword,
+  MatchKeyword,
+  IfKeyword,
+  ElifKeyword,
+  ElseKeyword,
   LineFoldEnd,
   BlockEnd,
   BlockStart,
@@ -99,6 +106,14 @@ export const enum SyntaxKind {
   // Patterns
   BindPattern,
   TuplePattern,
+  StructPattern,
+  NestedPattern,
+  NamedTuplePattern,
+
+  // Struct pattern elements
+  FieldStructPatternElement,
+  PunnedFieldStructPatternElement,
+  VariadicStructPatternElement,
 
   // Expressions
   ReferenceExpression,
@@ -122,10 +137,16 @@ export const enum SyntaxKind {
   ImportDeclaration,
   TypeAliasDeclaration,
 
-  // Other nodes
+  // Let declaration body members
-  StructDeclarationField,
   ExprBody,
   BlockBody,
+
+  // Structure declaration members
+  StructDeclarationField,
+
+  // Other nodes
+  Initializer,
+  QualifiedName,
   TypeAssert,
   Param,
   Module,
@@ -147,6 +168,39 @@ abstract class SyntaxBase {
 
   public abstract readonly kind: SyntaxKind;
 
+  public abstract getFirstToken(): Token;
+
+  public abstract getLastToken(): Token;
+
+  public toJSON(): JSONObject {
+
+    const obj: JSONObject = {};
+
+    obj['type'] = this.constructor.name;
+
+    for (const key of Object.getOwnPropertyNames(this)) {
+      if (key === 'kind') {
+        continue;
+      }
+      obj[key] = encode((this as any)[key]);
+    }
+
+    return obj;
+
+    function encode(value: any): JSONValue {
+      if (value === null) {
+        return null;
+      } else if (Array.isArray(value)) {
+        return value.map(encode);
+      } else if (value instanceof SyntaxBase) {
+        return value.toJSON();
+      } else {
+        return value;
+      }
+    }
+
+  }
+
 }
 
 abstract class TokenBase extends SyntaxBase {
@@ -159,6 +213,14 @@ abstract class TokenBase extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    throw new Error(`Trying to get the first token of an object that is a token itself.`);
+  }
+
+  public getLastToken(): Token {
+    throw new Error(`Trying to get the last token of an object that is a token itself.`);
+  }
+
   public getStartPosition(): TextPosition {
     return this.startPos;
   }
@@ -221,7 +283,7 @@ export class Integer extends TokenBase {
   public constructor(
     public value: bigint,
     public radix: number,
-    private startPos: TextPosition,
+    startPos: TextPosition,
   ) {
     super(startPos);
   }
@@ -249,7 +311,7 @@ export class StringLiteral extends TokenBase {
 
   public constructor(
     public contents: string,
-    private startPos: TextPosition,
+    startPos: TextPosition,
   ) {
     super(startPos);
   }
@@ -272,13 +334,26 @@ export class StringLiteral extends TokenBase {
 
 }
 
+export class Constructor extends TokenBase {
+
+  public readonly kind = SyntaxKind.Constructor;
+
+  public constructor(
+    public text: string,
+    startPos: TextPosition,
+  ) {
+    super(startPos);
+  }
+
+}
+
 export class Identifier extends TokenBase {
 
   public readonly kind = SyntaxKind.Identifier;
 
   public constructor(
     public text: string,
-    private startPos: TextPosition,
+    startPos: TextPosition,
   ) {
     super(startPos);
   }
@@ -291,7 +366,7 @@ export class CustomOperator extends TokenBase {
 
   public constructor(
     public text: string,
-    private startPos: TextPosition,
+    startPos: TextPosition,
   ) {
     super(startPos);
   }
@@ -418,6 +493,26 @@ export class StructKeyword extends TokenBase {
 
 }
 
+export class ReturnKeyword extends TokenBase {
+
+  public readonly kind = SyntaxKind.ReturnKeyword;
+
+  public get text(): string {
+    return 'return';
+  }
+
+}
+
+export class MatchKeyword extends TokenBase {
+
+  public readonly kind = SyntaxKind.MatchKeyword;
+
+  public get text(): string {
+    return 'match';
+  }
+
+}
+
 export class ModKeyword extends TokenBase {
 
   public readonly kind = SyntaxKind.ModKeyword;
@@ -486,6 +581,7 @@ export type Token
   | LBracket
   | RBracket
   | Identifier
+  | Constructor
   | CustomOperator
   | Integer
   | StringLiteral
@@ -501,6 +597,8 @@ export type Token
   | ImportKeyword
   | TypeKeyword
   | StructKeyword
+  | ReturnKeyword
+  | MatchKeyword
   | EndOfFile
   | BlockStart
   | BlockEnd
@@ -520,6 +618,17 @@ export class ReferenceTypeExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    if (this.modulePath.length > 0) {
+      return this.modulePath[0][0];
+    }
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.name;
+  }
+
 }
 
 export type TypeExpression
@@ -539,6 +648,14 @@ export class BindPattern extends SyntaxBase {
     return this.name.text == '_';
   }
 
+  public getFirstToken(): Token {
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.name;
+  }
+
 }
 
 export class TuplePattern extends SyntaxBase {
@@ -546,15 +663,168 @@ export class TuplePattern extends SyntaxBase {
   public readonly kind = SyntaxKind.TuplePattern;
 
   public constructor(
+    public lparen: LParen,
+    public elements: Pattern[],
+    public rparen: RParen,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.lparen;
+  }
+
+  public getLastToken(): Token {
+    return this.rparen;
+  }
+
+}
+
+export class NamedTuplePattern extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.NamedTuplePattern;
+
+  public constructor(
+    public name: Constructor,
     public elements: Pattern[],
   ) {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    if (this.elements.length > 0) {
+      return this.elements[this.elements.length-1].getLastToken();
+    }
+    return this.name;
+  }
+
+}
+
+export class FieldStructPatternElement extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.FieldStructPatternElement;
+
+  public constructor(
+    public name: Identifier,
+    public equals: Equals,
+    public pattern: Pattern,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.pattern.getLastToken();
+  }
+
+}
+
+export class VariadicStructPatternElement extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.VariadicStructPatternElement;
+
+  public constructor(
+    public dotdot: DotDot,
+    public pattern: Pattern | null,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.dotdot;
+  }
+
+  public getLastToken(): Token {
+    if (this.pattern !== null) {
+      return this.pattern.getLastToken();
+    }
+    return this.dotdot;
+  }
+
+}
+
+export class PunnedFieldStructPatternElement extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.PunnedFieldStructPatternElement;
+
+  public constructor(
+    public name: Identifier,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.name;
+  }
+
+}
+
+export type StructPatternElement
+  = VariadicStructPatternElement
+  | PunnedFieldStructPatternElement
+  | FieldStructPatternElement
+
+export class StructPattern extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.StructPattern;
+
+  public constructor(
+    public name: Constructor,
+    public lbrace: LBrace,
+    public elements: StructPatternElement[],
+    public rbrace: RBrace,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.rbrace;
+  }
+
+}
+
+export class NestedPattern extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.NestedPattern;
+
+  public constructor(
+    public lparen: LParen,
+    public pattern: Pattern,
+    public rparen: RParen,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.lparen;
+  }
+
+  public getLastToken(): Token {
+    return this.rparen;
+  }
+
 }
 
 export type Pattern
   = BindPattern
+  | NestedPattern
+  | StructPattern
+  | NamedTuplePattern
   | TuplePattern
 
 export class TupleExpression extends SyntaxBase {
@@ -569,6 +839,14 @@ export class TupleExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.lparen;
+  }
+
+  public getLastToken(): Token {
+    return this.rparen;
+  }
+
 }
 
 export class NestedExpression extends SyntaxBase {
@@ -583,6 +861,14 @@ export class NestedExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.lparen;
+  }
+
+  public getLastToken(): Token {
+    return this.rparen;
+  }
+
 }
 
 export class ConstantExpression extends SyntaxBase {
@@ -595,6 +881,38 @@ export class ConstantExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.token;
+  }
+
+  public getLastToken(): Token {
+    return this.token;
+  }
+
+}
+
+export class QualifiedName extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.QualifiedName;
+
+  public constructor(
+    public modulePath: Array<[Identifier, Dot]>,
+    public name: Identifier,
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    if (this.modulePath.length > 0) {
+      return this.modulePath[0][0];
+    }
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.name;
+  }
+
 }
 
 export class ReferenceExpression extends SyntaxBase {
@@ -602,12 +920,19 @@ export class ReferenceExpression extends SyntaxBase {
   public readonly kind = SyntaxKind.ReferenceExpression;
 
   public constructor(
-    public modulePath: Array<[Identifier, Dot]>,
+    public name: QualifiedName,
-    public name: Identifier
   ) {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.name.getFirstToken();
+  }
+
+  public getLastToken(): Token {
+     return this.name.getLastToken();
+  }
+
 }
 
 export class PrefixExpression extends SyntaxBase {
@@ -621,6 +946,14 @@ export class PrefixExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.operator;
+  }
+
+  public getLastToken(): Token {
+    return this.expression.getLastToken();
+  }
+
 }
 
 export class PostfixExpression extends SyntaxBase {
@@ -634,6 +967,14 @@ export class PostfixExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.expression.getFirstToken();
+  }
+
+  public getLastToken(): Token {
+    return this.operator;
+  }
+
 }
 
 export class InfixExpression extends SyntaxBase {
@@ -648,6 +989,14 @@ export class InfixExpression extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.left.getFirstToken();
+  }
+
+  public getLastToken(): Token {
+    return this.right.getLastToken();
+  }
+
 }
 
 export type Expression
@@ -664,11 +1013,23 @@ export class ReturnStatement extends SyntaxBase {
   public readonly kind = SyntaxKind.ReturnStatement;
 
   public constructor(
-    public expr: Expression
+    public returnKeyword: ReturnKeyword,
+    public expression: Expression
   ) {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.returnKeyword;
+  }
+
+  public getLastToken(): Token {
+    if (this.expression !== null) {
+      return this.expression.getLastToken();
+    }
+    return this.returnKeyword;
+  }
+
 }
 
 export class ExpressionStatement extends SyntaxBase {
@@ -676,11 +1037,19 @@ export class ExpressionStatement extends SyntaxBase {
   public readonly kind = SyntaxKind.ExpressionStatement;
 
   public constructor(
-    public expresion: Expression,
+    public expression: Expression,
   ) {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.expression.getFirstToken();
+  }
+
+  public getLastToken(): Token {
+    return this.expression.getLastToken();
+  }
+
 }
 
 export type Statement
@@ -697,6 +1066,14 @@ export class Param extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.pattern.getFirstToken();
+  }
+
+  public getLastToken(): Token {
+    return this.pattern.getLastToken();
+  }
+
 }
 
 export class StructDeclarationField extends SyntaxBase {
@@ -711,6 +1088,14 @@ export class StructDeclarationField extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.name;
+  }
+
+  public getLastToken(): Token {
+    return this.typeExpr.getLastToken();
+  }
+
 }
 
 export class StructDeclaration extends SyntaxBase {
@@ -725,6 +1110,17 @@ export class StructDeclaration extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.structKeyword;
+  }
+
+  public getLastToken(): Token {
+    if (this.members && this.members.length > 0) {
+      return this.members[this.members.length-1].getLastToken();
+    }
+    return this.name;
+  }
+
 }
 
 export class TypeAssert extends SyntaxBase {
@@ -738,6 +1134,14 @@ export class TypeAssert extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.colon;
+  }
+
+  public getLastToken(): Token {
+    return this.typeExpression.getLastToken();
+  }
+
 }
 
 export type Body 
@@ -755,6 +1159,14 @@ export class ExprBody extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.equals;
+  }
+
+  public getLastToken(): Token {
+    return this.expression.getLastToken();
+  }
+
 }
 
 export type LetBodyElement 
@@ -772,6 +1184,17 @@ export class BlockBody extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    return this.blockStart;
+  }
+
+  public getLastToken(): Token {
+    if (this.elements.length > 0) {
+      return this.elements[this.elements.length-1].getLastToken();
+    }
+    return this.blockStart;
+  }
+
 }
 export class LetDeclaration extends SyntaxBase {
 
@@ -789,6 +1212,26 @@ export class LetDeclaration extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    if (this.pubKeyword !== null) {
+      return this.pubKeyword;
+    }
+    return this.letKeyword;
+  }
+
+  public getLastToken(): Token {
+    if (this.body !== null) {
+      return this.body.getLastToken();
+    }
+    if (this.typeAssert !== null) {
+      return this.typeAssert.getLastToken();
+    }
+    if (this.params.length > 0) {
+      return this.params[this.params.length-1].getLastToken();
+    }
+    return this.pattern.getLastToken();
+  }
+
 }
 
 export class ImportDeclaration extends SyntaxBase {
@@ -802,6 +1245,14 @@ export class ImportDeclaration extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+     return this.importKeyword;
+  }
+
+  public getLastToken(): Token {
+    return this.importSource;
+  }
+
 }
 
 export type Declaration
@@ -809,11 +1260,33 @@ export type Declaration
   | ImportDeclaration
   | StructDeclaration
 
+export class Initializer extends SyntaxBase {
+
+  public readonly kind = SyntaxKind.Initializer;
+
+  public constructor(
+    public equals: Equals,
+    public expression: Expression
+  ) {
+    super();
+  }
+
+  public getFirstToken(): Token {
+    return this.equals;
+  }
+
+  public getLastToken(): Token {
+    return this.expression.getLastToken();
+  }
+
+}
+
 export class Module extends SyntaxBase {
 
   public readonly kind = SyntaxKind.Module;
 
   public constructor(
+    public pubKeyword: PubKeyword | null,
     public modKeyword: ModKeyword,
     public name: Identifier,
     public body: Body,
@@ -821,6 +1294,17 @@ export class Module extends SyntaxBase {
     super();
   }
 
+  public getFirstToken(): Token {
+    if (this.pubKeyword !== null) {
+      return this.pubKeyword;
+    }
+    return this.modKeyword;
+  }
+
+  public getLastToken(): Token {
+    return this.body.getLastToken();
+  }
+
 }
 
 export type SourceFileElement
@@ -833,15 +1317,24 @@ export class SourceFile extends SyntaxBase {
   public readonly kind = SyntaxKind.SourceFile;
 
   public constructor(
-    public elements: SourceFileElement[]
+    public elements: SourceFileElement[],
+    public eof: EndOfFile,
   ) {
     super();
   }
 
-  public *getChildNodes(): Iterable<Syntax> {
+  public getFirstToken(): Token {
-    for (const element in this.elements) {
+    if (this.elements.length > 0) {
-      yield element;
+      return this.elements[0].getFirstToken();
     }
+    return this.eof;
+  }
+
+  public getLastToken(): Token {
+    if (this.elements.length > 0) {
+      return this.elements[this.elements.length-1].getLastToken();
+    }
+    return this.eof;
   }
 
 }

src/parser.ts

@@ -1,5 +1,5 @@
 
-import { kMaxLength } from "buffer";
+import { privateDecrypt } from "crypto";
 import {
   ReferenceTypeExpression,
   SourceFile,
@@ -19,7 +19,6 @@ import {
   PrefixExpression,
   ExpressionStatement,
   ImportDeclaration,
-  FunctionDeclaration,
   Param,
   Pattern,
   BindPattern,
@@ -27,6 +26,15 @@ import {
   TypeAssert,
   ExprBody,
   BlockBody,
+  QualifiedName,
+  NestedExpression,
+  NamedTuplePattern,
+  StructPattern,
+  VariadicStructPatternElement,
+  PunnedFieldStructPatternElement,
+  FieldStructPatternElement,
+  TuplePattern,
+  InfixExpression,
 } from "./cst"
 import { Stream, MultiDict } from "./util";
 
@@ -91,28 +99,60 @@ function isConstructor(token: Token): boolean {
       && token.text[0].toUpperCase() === token.text[0];
 }
 
+function isBinaryOperatorLike(token: Token): boolean {
+  return token.kind === SyntaxKind.CustomOperator;
+}
+
+function isPrefixOperatorLike(token: Token): boolean {
+  return token.kind === SyntaxKind.CustomOperator;
+}
+
 const enum OperatorMode {
   None   = 0,
   Prefix = 1,
   InfixL = 2,
   InfixR = 4,
+  Infix = 6,
   Suffix = 8,
 }
 
 interface OperatorInfo {
   name: string,
   mode: OperatorMode,
-  precedence?: number,
+  precedence: number,
 }
 
+const EXPR_OPERATOR_TABLE: Array<[string, OperatorMode, number]> = [
+  ["**", OperatorMode.InfixR, 11],
+  ["*", OperatorMode.InfixL, 8],
+  ["/", OperatorMode.InfixL, 8],
+  ["+", OperatorMode.InfixL, 7],
+  ["-", OperatorMode.InfixL, 7],
+  ["<", OperatorMode.InfixL, 6],
+  [">", OperatorMode.InfixL, 6],
+  ["<=", OperatorMode.InfixL, 5],
+  [">=", OperatorMode.InfixL, 5],
+  ["==", OperatorMode.InfixL, 5],
+  ["!=", OperatorMode.InfixL, 5],
+  ["<*", OperatorMode.InfixL, 4],
+  [":", OperatorMode.InfixL, 3],
+  ["<|>", OperatorMode.InfixL, 2],
+  ["<?>", OperatorMode.InfixL, 1],
+  ["$", OperatorMode.InfixR, 0]
+];
+
 export class Parser {
 
-  private exprOperators = new MultiDict<string, OperatorInfo>();
+  private prefixExprOperators = new Set<string>();
+  private binaryExprOperators = new Map<string, OperatorInfo>();
+  private suffixExprOperators = new Set<string>();
 
   public constructor(
     public tokens: Stream<Token>,
   ) {
-
+    for (const [name, mode, precedence] of EXPR_OPERATOR_TABLE) {
+      this.binaryExprOperators.set(name, { name, mode, precedence });
+    }
   }
 
   private getToken(): Token {
@@ -152,20 +192,6 @@ export class Parser {
     return t0;
   }
 
-  private isPrefixOperator(token: Token): boolean {
-    const name = token.text;
-    for (const operator of this.exprOperators.get(name)) {
-      if (operator.mode & OperatorMode.Prefix) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  private isBinaryOperator(token: Token): boolean {
-    return token.kind === SyntaxKind.CustomOperator;
-  }
-
   public parseReferenceTypeExpression(): ReferenceTypeExpression {
     const name = this.expectToken(SyntaxKind.Identifier);
     return new ReferenceTypeExpression([], name);
@@ -190,7 +216,7 @@ export class Parser {
     return new ConstantExpression(token);
   }
 
-  public parseReferenceExpression(): ReferenceExpression {
+  public parseQualifiedName(): QualifiedName {
     const modulePath: Array<[Identifier, Dot]> = [];
     let name = this.expectToken(SyntaxKind.Identifier)
     for (;;) {
@@ -201,18 +227,24 @@ export class Parser {
       modulePath.push([name, t1]);
       name = this.expectToken(SyntaxKind.Identifier)
     }
-    return new ReferenceExpression(modulePath, name);
+    return new QualifiedName(modulePath, name);
+  }
+
+  public parseReferenceExpression(): ReferenceExpression {
+    return new ReferenceExpression(this.parseQualifiedName());
   }
 
   private parseExpressionWithParens(): Expression {
-    const t0 = this.expectToken(SyntaxKind.LParen)
+    const lparen = this.expectToken(SyntaxKind.LParen)
     const t1 = this.peekToken();
     if (t1.kind === SyntaxKind.RParen) {
       this.getToken();
-      return new TupleExpression(t0, [], t1);
+      return new TupleExpression(lparen, [], t1);
-    }
+    } else if (t1.kind === SyntaxKind.Constructor) {
-    if (isConstructor(t1)) {
+    } else {
-
+      const expression = this.parseExpression();
+      const t2 = this.expectToken(SyntaxKind.RParen);
+      return new NestedExpression(lparen, expression, t2);
     }
   }
 
@@ -237,36 +269,59 @@ export class Parser {
   }
 
   private parseUnaryExpression(): Expression {
-    let out = this.parseExpressionNoOperators()
+    let result = this.parseExpressionNoOperators()
-    const prefixOperators = [];
+    const prefixes = [];
     for (;;) {
       const t0 = this.peekToken();
-      if (!this.isPrefixOperator(t0)) {
+      if (!isPrefixOperatorLike(t0)) {
         break;
       }
-      prefixOperators.push(t0);
+      if (!this.prefixExprOperators.has(t0.text)) {
+        break;
+      }
+      prefixes.push(t0);
       this.getToken()
     }
-    for (let i = prefixOperators.length-1; i >= 0; i--) {
+    for (let i = prefixes.length-1; i >= 0; i--) {
-      const op = prefixOperators[i];
+      const operator = prefixes[i];
-      out = new PrefixExpression(op, out);
+      result = new PrefixExpression(operator, result);
     }
-    return out;
+    return result;
   }
 
-  private parseExpressionWithBinaryOperator(lhs: Expression, minPrecedence: number) {
+  private parseBinaryOperatorAfterExpr(lhs: Expression, minPrecedence: number) {
     for (;;) {
       const t0 = this.peekToken();
-      if (!this.isBinaryOperator(t0)) {
+      if (!isBinaryOperatorLike(t0)) {
         break;
       }
+      const info0 = this.binaryExprOperators.get(t0.text);
+      if (info0 === undefined || info0.precedence < minPrecedence) {
+        break;
+      }
+      this.getToken();
+      let rhs = this.parseUnaryExpression();
+      for (;;) {
+        const t1 = this.peekToken();
+        if (!isBinaryOperatorLike(t1)) {
+          break;
+        }
+        const info1 = this.binaryExprOperators.get(t1.text);
+        if (info1 === undefined
+          || info1.precedence < info0.precedence
+          || (info1.precedence === info0.precedence && (info1.mode & OperatorMode.InfixR) === 0)) {
+          break;
+        }
+        rhs = this.parseBinaryOperatorAfterExpr(rhs, info0.precedence);
+      }
+      lhs = new InfixExpression(lhs, t0, rhs);
     }
     return lhs;
   }
 
   public parseExpression(): Expression {
     const lhs = this.parseUnaryExpression();
-    return this.parseExpressionWithBinaryOperator(lhs, 0);
+    return this.parseBinaryOperatorAfterExpr(lhs, 0);
   }
 
   public parseStructDeclaration(): StructDeclaration {
@@ -290,9 +345,96 @@ export class Parser {
     return new StructDeclaration(structKeyword, name, members);
   }
 
+  private parsePatternStartingWithConstructor() {
+    const name = this.expectToken(SyntaxKind.Constructor);
+    const t2 = this.peekToken();
+    if (t2.kind === SyntaxKind.LBrace) {
+      this.getToken();
+      const fields = [];
+      let rbrace;
+      for (;;) {
+        const t3 = this.peekToken();
+        if (t3.kind === SyntaxKind.RBrace) {
+          rbrace = t3;
+          break;
+        } else if (t3.kind === SyntaxKind.Identifier) {
+          this.getToken();
+          const t4 = this.peekToken();
+          if (t4.kind === SyntaxKind.Equals) {
+            this.getToken();
+            const pattern = this.parsePattern();
+            fields.push(new FieldStructPatternElement(t3, t4, pattern));
+          } else {
+            fields.push(new PunnedFieldStructPatternElement(t3));
+          }
+        } else if (t3.kind === SyntaxKind.DotDot) {
+          this.getToken();
+          fields.push(new VariadicStructPatternElement(t3, null));
+        } else {
+          this.raiseParseError(t3, [ SyntaxKind.Identifier, SyntaxKind.DotDot ]);
+        }
+        const t5 = this.peekToken();
+        if (t5.kind === SyntaxKind.Comma) {
+          this.getToken();
+        } else if (t5.kind === SyntaxKind.RBrace) {
+          rbrace = t5;
+          break;
+        } else {
+          this.raiseParseError(t5, [ SyntaxKind.Comma, SyntaxKind.RBrace ]);
+        }
+      }
+      return new StructPattern(name, t2, fields, rbrace);
+    } else {
+      const patterns = [];
+      for (;;) {
+        const t3 = this.peekToken();
+        if (t3.kind === SyntaxKind.RParen) {
+          break;
+        }
+        patterns.push(this.parsePattern());
+      }
+      return new NamedTuplePattern(name, patterns);
+    }
+  }
+
+  public parseTuplePattern(): TuplePattern {
+    const lparen = this.expectToken(SyntaxKind.LParen);
+    const elements = [];
+    let rparen;
+    for (;;) {
+      const t1 = this.peekToken();
+      if (t1.kind === SyntaxKind.RParen) {
+        rparen = t1;
+        break;
+      }
+      elements.push(this.parsePattern());
+      const t2 = this.peekToken();
+      if (t2.kind === SyntaxKind.Comma) {
+        this.getToken();
+      } else if (t2.kind === SyntaxKind.RParen) {
+        rparen = t2;
+        break;
+      } else {
+        this.raiseParseError(t2, [ SyntaxKind.Comma, SyntaxKind.RParen ]);
+      }
+    }
+    this.getToken();
+    return new TuplePattern(lparen, elements, rparen);
+  }
+
   public parsePattern(): Pattern {
     const t0 = this.peekToken();
     switch (t0.kind) {
+      case SyntaxKind.LParen:
+      {
+        this.getToken();
+        const t1 = this.peekToken();
+        if (t1.kind === SyntaxKind.Constructor) {
+          return this.parsePatternStartingWithConstructor();
+        } else {
+          return this.parseTuplePattern();
+        }
+      }
       case SyntaxKind.Identifier:
         this.getToken();
         return new BindPattern(t0);
@@ -410,15 +552,17 @@ export class Parser {
 
   public parseSourceFile(): SourceFile {
     const elements = [];
+    let eof;
     for (;;) {
       const t0 = this.peekToken();
       if (t0.kind === SyntaxKind.EndOfFile) {
+        eof = t0;
         break;
       }
       const element = this.parseSourceFileElement();
       elements.push(element);
     }
-    return new SourceFile(elements);
+    return new SourceFile(elements, eof);
   }
 
 }

src/scanner.ts

@@ -23,13 +23,20 @@ import {
   LBracket,
   RBrace,
   RBracket,
+  ReturnKeyword,
   CustomOperator,
+  Constructor,
+  Integer,
 } from "./cst"
 import { Diagnostics, UnexpectedCharDiagnostic } from "./diagnostics"
-import { Stream, BufferedStream } from "./util";
+import { Stream, BufferedStream, assert } from "./util";
 
 const EOF = '\uFFFF'
 
+function isUpper(ch: string): boolean {
+  return ch.toUpperCase() === ch;
+}
+
 function isWhiteSpace(ch: string): boolean {
   return /[\r\n\t ]/.test(ch);
 }
@@ -42,6 +49,16 @@ function isIdentStart(ch: string): boolean {
   return /[a-zA-Z_]/.test(ch)
 }
 
+function isDecimalDigit(ch: string): boolean {
+  return /[0-9]/.test(ch);
+}
+
+function toDecimal(ch: string): number {
+  const code = ch.charCodeAt(0);
+  assert(code >= 48 && code <= 57);
+  return code - 48;
+}
+
 function isOperatorPart(ch: string): boolean {
   return /\+-*\/%^&|$<>!?=/.test(ch);
 }
@@ -208,6 +225,37 @@ export class Scanner extends BufferedStream<Token> {
           }
         }
 
+        case '0':
+        {
+          const c1 = this.peekChar();
+          switch (c1) {
+            case 'x': // TODO
+            case 'o': // TODO
+            case 'b': // TODO
+          }
+        }
+        case '1':
+        case '2':
+        case '3':
+        case '4':
+        case '5':
+        case '6':
+        case '7':
+        case '8':
+        case '9':
+        {
+          let value = BigInt(toDecimal(c0));
+          for (;;) {
+            const c1 = this.peekChar();
+            if (!isDecimalDigit(c1)) {
+              break;
+            }
+            this.getChar();
+            value = value * BigInt(10) + BigInt(toDecimal(c1));
+          }
+          return new Integer(value, 10, startPos);
+        }
+
         case 'a':
         case 'b':
         case 'c':
@@ -272,7 +320,11 @@ export class Scanner extends BufferedStream<Token> {
             case 'return': return new ReturnKeyword(startPos);
             case 'type': return new TypeKeyword(startPos);
             default:
-              return new Identifier(text, startPos);
+              if (isUpper(text[0])) {
+                return new Constructor(text, startPos);
+              } else {
+                return new Identifier(text, startPos);
+              }
           }
         }
 

src/util.ts

@@ -1,4 +1,13 @@
 
+export function assert(test: boolean): asserts test {
+  if (!test) {
+    throw new Error(`Assertion failed. See the stack trace for more information.`);
+  }
+}
+
+export type JSONValue = null | boolean | number | string | JSONArray | JSONObject
+export type JSONArray = Array<JSONValue>;
+export type JSONObject = { [key: string]: JSONValue };
 
 export class MultiDict<K, V> {