diff --git a/package-lock.json b/package-lock.json index 70828f5bf..362a1df9a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,14 @@ "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==" }, + "@types/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-UoOfVEzAUpeSPmjm7h1uk5MH6KZma2z2O7a75onTGjnNvAvMVrPzPL/vBbT65iIGHWj6rokwfmYcmxmlSf2uwg==", + "requires": { + "@types/node": "*" + } + }, "@types/mocha": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-7.0.1.tgz", diff --git a/package.json b/package.json index 8511aa746..c399dacec 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "license": "GPL-3.0", "repository": "https://github.com/samvv/Bolt", "dependencies": { + "@types/fs-extra": "^8.1.0", "@types/node": "^13.7.4", "@types/xregexp": "^3.0.30", "@types/yargs": "^15.0.3", diff --git a/src/ast.ts b/src/ast.ts index d2138c075..1dac3c142 100644 --- a/src/ast.ts +++ b/src/ast.ts @@ -3,6 +3,9 @@ import { Stream, StreamWrapper } from "./util" import { Scanner } from "./scanner" +import { RecordType, PrimType, OptionType, VariantType, stringType, intType, boolType } from "./checker" +import { bind } from "./bindings" +import { Value } from "./evaluator" interface JsonArray extends Array { }; interface JsonObject { [key: string]: Json } @@ -30,15 +33,21 @@ export enum SyntaxKind { // Keywords - FunctionKeyword, + FnKeyword, ForeignKeyword, LetKeyword, ImportKeyword, + PubKeyword, + ModKeyword, + EnumKeyword, + StructKeyword, // Special nodes SourceFile, + Module, + QualName, Sentence, @@ -50,17 +59,21 @@ export enum SyntaxKind { // Patterns BindPatt, - ExprPatt, + TypePatt, + RecordPatt, + TuplePatt, // Expressions CallExpr, ConstExpr, RefExpr, + MatchExpr, - // Stmts + // Statements RetStmt, + CondStmt, // Type declarations @@ -71,16 +84,11 @@ export enum SyntaxKind { VarDecl, FuncDecl, ImportDecl, + RecordDecl, + VariantDecl, } -enum EdgeType { - Primitive = 1, - Node = 2, - Nullable = 4, - List = 8, -} - export class TextFile { constructor(public path: string) { @@ -167,6 +175,9 @@ abstract class SyntaxBase { } +export const fileType = new PrimType(); + +@bind('Bolt.AST.StringLiteral') export class StringLiteral extends SyntaxBase { kind: SyntaxKind.StringLiteral = SyntaxKind.StringLiteral; @@ -198,7 +209,7 @@ export class IntegerLiteral extends SyntaxBase { kind: SyntaxKind.IntegerLiteral = SyntaxKind.IntegerLiteral; constructor( - public value: string, + public value: bigint, public span: TextSpan | null = null, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -250,11 +261,7 @@ export class EOS extends SyntaxBase { } -export class Parenthesized extends SyntaxBase { - - kind: SyntaxKind.Parenthesized = SyntaxKind.Parenthesized; - - protected buffered = null; +export abstract class Punctuated extends SyntaxBase { constructor( public text: string, @@ -265,38 +272,10 @@ export class Parenthesized extends SyntaxBase { super(); } - toTokenStream() { + toSentences() { const span = this.getSpan(); - const startPos = span.start; - return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1)); - } - - toJSON(): Json { - return { - kind: 'Parenthesized', - text: this.text, - span: this.span !== null ? this.span.toJSON() : null, - } - } - - *getChildren(): IterableIterator { - - } - -} - - -export class Braced extends SyntaxBase { - - kind: SyntaxKind.Braced = SyntaxKind.Braced; - - constructor( - public text: string, - public span: TextSpan, - public origNode: [Syntax, Syntax] | Syntax | null = null, - public parentNode: Syntax | null = null - ) { - super(); + const scanner = new Scanner(span.file, this.text) + return scanner.scanTokens() } toTokenStream() { @@ -319,37 +298,16 @@ export class Braced extends SyntaxBase { } -export class Bracketed extends SyntaxBase { +export class Parenthesized extends Punctuated { + kind: SyntaxKind.Parenthesized = SyntaxKind.Parenthesized; +} +export class Braced extends Punctuated { + kind: SyntaxKind.Braced = SyntaxKind.Braced; +} + +export class Bracketed extends Punctuated { kind: SyntaxKind.Bracketed = SyntaxKind.Bracketed; - - constructor( - public text: string, - public span: TextSpan, - public origNode: [Syntax, Syntax] | Syntax | null = null, - public parentNode: Syntax | null = null - ) { - super(); - } - - toTokenStream() { - const span = this.getSpan(); - const startPos = span.start; - return new Scanner(span.file, this.text, new TextPos(startPos.offset+1, startPos.line, startPos.column+1)); - } - - toJSON(): Json { - return { - kind: 'Bracketed', - text: this.text, - span: this.span !== null ? this.span.toJSON() : null, - } - } - - *getChildren(): IterableIterator { - - } - } export class Identifier extends SyntaxBase { @@ -625,6 +583,14 @@ export class QualName { } + get fullText() { + let out = '' + for (const chunk of this.path) { + out += chunk.text + '.' + } + return out + this.name + } + toJSON(): Json { return { kind: 'QualName', @@ -707,8 +673,82 @@ export class BindPatt extends SyntaxBase { } +export interface RecordPattField { + name: Identifier; + pattern: Patt, +} + +export class RecordPatt extends SyntaxBase { + + kind: SyntaxKind.RecordPatt = SyntaxKind.RecordPatt; + + constructor( + public typeDecl: TypeDecl, + public fields: RecordPattField[], + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + *getChildren(): IterableIterator { + for (const field of this.fields) { + yield field.name; + yield field.pattern; + } + } + +} + +export class TuplePatt extends SyntaxBase { + + kind: SyntaxKind.TuplePatt = SyntaxKind.TuplePatt; + + constructor( + public elements: Patt[], + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + *getChildren(): IterableIterator { + for (const element of this.elements) { + yield element; + } + } + +} + +export class TypePatt extends SyntaxBase { + + kind: SyntaxKind.TypePatt = SyntaxKind.TypePatt; + + constructor( + public typeDecl: TypeDecl, + public pattern: Patt, + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + *getChildren() { + yield this.typeDecl; + yield this.pattern; + } + +} + export type Patt = BindPatt + | Expr + | TypePatt + | TuplePatt + | RecordPatt export class RefExpr extends SyntaxBase { @@ -769,12 +809,44 @@ export class CallExpr extends SyntaxBase { } +export type MatchArm = [Patt, Expr | Stmt[]]; + +export class MatchExpr extends SyntaxBase { + + kind: SyntaxKind.MatchExpr = SyntaxKind.MatchExpr; + + constructor( + public value: Expr, + public arms: MatchArm[], + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + *getChildren(): IterableIterator { + yield this.value; + for (const [pattern, result] of this.arms) { + yield pattern + if (Array.isArray(result)) { + for (const stmt of result) { + yield stmt + } + } else { + yield result + } + } + } + +} + export class ConstExpr extends SyntaxBase { kind: SyntaxKind.ConstExpr = SyntaxKind.ConstExpr; constructor( - public value: string | bigint, + public value: Value, public span: TextSpan | null = null, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -801,6 +873,7 @@ export type Expr = ConstExpr | RefExpr | CallExpr + | MatchExpr export class RetStmt extends SyntaxBase { @@ -831,8 +904,38 @@ export class RetStmt extends SyntaxBase { } +export interface Case { + conditional: Expr, + consequent: Stmt[] +} + +export class CondStmt { + + kind: SyntaxKind.CondStmt = SyntaxKind.CondStmt + + constructor( + public cases: Case[], + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + + } + + *getChildren(): IterableIterator { + for (const kase of this.cases) { + yield kase.conditional + for (const stmt of kase.consequent) { + yield stmt + } + } + } + +} + export type Stmt = RetStmt + | CondStmt export class TypeRef extends SyntaxBase { @@ -874,6 +977,7 @@ export class FuncDecl extends SyntaxBase { kind: SyntaxKind.FuncDecl = SyntaxKind.FuncDecl; constructor( + public isPublic: boolean, public target: string, public name: QualName, public params: Param[], @@ -889,6 +993,7 @@ export class FuncDecl extends SyntaxBase { toJSON(): Json { return { kind: 'FuncDecl', + isPublic: this.isPublic, target: this.target, name: this.name.toJSON(), params: this.params.map(p => p.toJSON()), @@ -972,12 +1077,40 @@ export class ImportDecl { } +export class RecordDecl extends SyntaxBase { + + kind: SyntaxKind.RecordDecl = SyntaxKind.RecordDecl; + + fields: Map; + + constructor( + public isPublic: boolean, + public name: QualName, + fields: Iterable<[Identifier, TypeDecl]>, + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super() + this.fields = new Map(fields); + } + + *getChildren() { + yield this.name; + for (const [name, typeDecl] of this.fields) { + yield name + yield typeDecl + } + } + +} + export type Decl = Sentence | FuncDecl | ImportDecl | VarDecl - + | RecordDecl export type Syntax = Decl @@ -986,17 +1119,43 @@ export type Syntax | Stmt | Patt | TypeDecl + | Module | SourceFile | QualName | Param | EOS +export type SourceElement = (Module | Decl | Stmt | Expr); + +export class Module extends SyntaxBase { + + kind: SyntaxKind.Module = SyntaxKind.Module; + + constructor( + public isPublic: boolean, + public name: QualName, + public elements: SourceElement[], + public span: TextSpan | null = null, + public origNode: [Syntax, Syntax] | Syntax | null = null, + public parentNode: Syntax | null = null + ) { + super(); + } + + *getChildren(): IterableIterator { + for (const element of this.elements) { + yield element; + } + } + +} + export class SourceFile extends SyntaxBase { kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile; constructor( - public elements: (Decl | Stmt | Expr)[], + public elements: SourceElement[], public span: TextSpan | null = null, public origNode: [Syntax, Syntax] | Syntax | null = null, public parentNode: Syntax | null = null @@ -1021,9 +1180,15 @@ export class SourceFile extends SyntaxBase { } export function isExpr(node: Syntax): node is Expr { + return node.kind === SyntaxKind.ConstExpr || node.kind === SyntaxKind.CallExpr; } +export function isNode(value: any): value is Syntax { + return typeof value.kind === 'number' + && Object.prototype.hasOwnProperty.call(SyntaxKind, value.kind) +} + export function isJSNode(node: Syntax) { return typeof node.type === 'string' } diff --git a/src/bin/bolt.ts b/src/bin/bolt.ts index 4c2c8bdeb..af7f127bb 100644 --- a/src/bin/bolt.ts +++ b/src/bin/bolt.ts @@ -14,6 +14,7 @@ import { Parser } from "../parser" import { Expander } from "../expander" import { TypeChecker } from "../checker" import { Compiler } from "../compiler" +import { Evaluator } from "../evaluator" import { Emitter } from "../emitter" import { TextFile, SourceFile, setParents } from "../ast" @@ -129,9 +130,12 @@ yargs } + const checker = new TypeChecker() + for (const sourceFile of sourceFiles) { const parser = new Parser() - const expander = new Expander(parser) + const evaluator = new Evaluator(checker) + const expander = new Expander(parser, evaluator, checker) const expandedSourceFile = expander.getFullyExpanded(sourceFile) for (const hook of hooks) { @@ -166,20 +170,21 @@ yargs args => { const parser = new Parser() + const checker = new TypeChecker() const sourceFiles = toArray(args.files as string[]).map(filepath => { const file = new TextFile(filepath) const contents = fs.readFileSync(filepath, 'utf8') const scanner = new Scanner(file, contents) const sourceFile = scanner.scan(); - const expander = new Expander(parser) - const expanded = expander.getFullyExpanded(sourceFile) + const evaluator = new Evaluator(checker) + const expander = new Expander(parser, evaluator, checker) + const expanded = expander.getFullyExpanded(sourceFile) as SourceFile; // console.log(require('util').inspect(expanded.toJSON(), { colors: true, depth: Infinity })) setParents(expanded) return expanded; }) - const checker = new TypeChecker() const compiler = new Compiler(checker, { target: "JS" }) const bundle = compiler.compile(sourceFiles) const emitter = new Emitter() @@ -196,6 +201,26 @@ yargs ) + .command( + + 'dump [file]', + 'Dump a representation of a given primitive node to disk', + + yargs => yargs, + + args => { + + const file = new TextFile(args.file as string) + const contents = fs.readFileSync(args.file, 'utf8') + const scanner = new Scanner(file, contents) + const parser = new Parser(); + const patt = parser.parsePattern(scanner) + console.log(JSON.stringify(patt.toJSON(), undefined, 2)) + + } + + ) + .help() .version() .argv diff --git a/src/bindings.ts b/src/bindings.ts new file mode 100644 index 000000000..a842ba20a --- /dev/null +++ b/src/bindings.ts @@ -0,0 +1,22 @@ + +import { Value, RecordValue } from "./evaluator" + +interface Binding { + name: string; + createValue: () => Value, +} + +export const bindings = new Map(); + +export function bind(name: string) { + return function (target: any) { + if (bindings.has(name)) { + throw new Error(`A binding with the name '${name}' already exists.`) + } + bindings.set(name, { + name, + createValue: (...args) => new RecordValue(target.META_TYPE, new target(...args)), + }); + } +} + diff --git a/src/checker.ts b/src/checker.ts index 920b587a8..9be3700ed 100644 --- a/src/checker.ts +++ b/src/checker.ts @@ -3,14 +3,55 @@ import { Syntax, SyntaxKind, ImportDecl, + isNode, } from "./ast" -class Type { +import { FastStringMap } from "./util" + +export class Type { } -interface FastStringMap { - [key: string]: T +export class PrimType extends Type { + +} + +export class VariantType extends Type { + + constructor(public elementTypes: Type[]) { + super(); + } + +} + +export const stringType = new PrimType() +export const intType = new PrimType() +export const boolType = new PrimType() +export const voidType = new PrimType() + +export class RecordType { + + fieldTypes: FastStringMap = Object.create(null); + + constructor( + iterable: IterableIterator<[string, Type]>, + ) { + for (const [name, typ] of iterable) { + this.fieldTypes[name] = typ; + } + } + + hasField(name: string) { + return name in this.fieldTypes; + } + + getTypeOfField(name: string) { + if (name in this.fieldTypes) { + return this.fieldTypes[name] + } + throw new Error(`Field '${name}' does not exist on this record type.`) + } + } export class Scope { @@ -21,22 +62,117 @@ export class Scope { } +function* map(iterable: Iterable, proc: (value: T) => R): IterableIterator { + const iterator = iterable[Symbol.iterator](); + while (true) { + let { done, value }= iterator.next(); + if (done) { + break + } + yield proc(value) + } +} + +function getFullName(node: Syntax) { + let out = [] + let curr: Syntax | null = node; + while (true) { + switch (curr.kind) { + case SyntaxKind.Module: + out.unshift(curr.name.fullText); + break; + case SyntaxKind.RecordDecl: + out.unshift(curr.name.fullText) + break; + } + curr = curr.parentNode; + if (curr === null) { + break; + } + } + return out.join('.'); +} + export class TypeChecker { - protected stringType = new Type(); - protected intType = new Type(); - + protected symbols: FastStringMap = Object.create(null) + protected types = new Map(); protected scopes = new Map(); - protected createType(node: Syntax) { + protected createType(node: Syntax): Type { + + console.log(node) + switch (node.kind) { + case SyntaxKind.ConstExpr: - if (typeof node.value === 'bigint') { - return this.intType; - } else if (typeof node.value === 'string') { - return this.stringType; - } + return node.value.type; + + case SyntaxKind.RecordDecl: + + const typ = new RecordType(map(node.fields, ([name, typ]) => ([name.text, typ]))); + + this.symbols[getFullName(node)] = typ; + + return typ; + + // if (typeof node.value === 'bigint') { + // return intType; + // } else if (typeof node.value === 'string') { + // return stringType; + // } else if (typeof node.value === 'boolean') { + // return boolType; + // } else if (isNode(node.value)) { + // return this.getTypeNamed(`Bolt.AST.${SyntaxKind[node.value.kind]}`)! + // } else { + // throw new Error(`Unrecognised kind of value associated with ConstExpr`) + // } + + default: + throw new Error(`Could not derive type of ${SyntaxKind[node.kind]}`) + } + + } + + getTypeNamed(name: string) { + return name in this.typeNames + ? this.typeNames[name] + : null + } + + getTypeOfNode(node: Syntax): Type { + if (this.types.has(node)) { + return this.types.get(node)! + } + const newType = this.createType(node) + this.types.set(node, newType) + return newType; + } + + check(node: Syntax) { + + switch (node.kind) { + + case SyntaxKind.Sentence: + break; + + case SyntaxKind.RecordDecl: + this.getTypeOfNode(node); + break; + + case SyntaxKind.Module: + case SyntaxKind.SourceFile: + for (const element of node.elements) { + this.check(element) + } + break; + + default: + throw new Error(`Could not type-check node ${SyntaxKind[node.kind]}`) + + } + } getImportedSymbols(node: ImportDecl) { @@ -55,9 +191,9 @@ export class TypeChecker { return scope } - getMapperForNode(target: string, node: Syntax): Mapper { - return this.getScope(node).getMapper(target) - } + // getMapperForNode(target: string, node: Syntax): Mapper { + // return this.getScope(node).getMapper(target) + // } } diff --git a/src/compiler.ts b/src/compiler.ts index 060ab21c8..80ae27f6d 100644 --- a/src/compiler.ts +++ b/src/compiler.ts @@ -99,7 +99,7 @@ export class Compiler { case SyntaxKind.ImportDecl: preamble.push({ type: 'ImportDeclaration', - source: { type: 'Literal', value: node.file }, + source: { type: 'Literal', value: node.file + '.mjs' }, specifiers: this.checker.getImportedSymbols(node).map(s => ({ type: 'ImportSpecifier', imported: { type: 'Identifier', name: s.name }, @@ -123,9 +123,17 @@ export class Compiler { case SyntaxKind.FuncDecl: const params = []; - if (node.target === this.target) { - console.log(node.body) - preamble.push({ + if (node.body !== null) { + let body; + if (node.target === this.target) { + body = node.body; + } else if (node.target === 'Bolt') { + let body: Stmt[] = []; + for (const stmt in node.body) { + this.compileDecl(stmt, body) + } + } + let result = { type: 'FunctionDeclaration', id: { type: 'Identifier', name: node.name.name.text }, params: node.params.map(p => ({ type: 'Identifier', name: p.bindings.name.text })), @@ -133,9 +141,14 @@ export class Compiler { type: 'BlockStatement', body: node.body } - }) - } else { - throw new Error(`Cannot yet compile Bolt functions`) + } + if (node.isPublic) { + result = { + type: 'ExportNamedDeclaration', + declaration: result, + } + } + preamble.push(result) } break; diff --git a/src/evaluator.ts b/src/evaluator.ts new file mode 100644 index 000000000..7d9c3c47e --- /dev/null +++ b/src/evaluator.ts @@ -0,0 +1,126 @@ + +import { Syntax, SyntaxKind, Expr, isNode } from "./ast" +import { TypeChecker, Type, RecordType, PrimType, boolType } from "./checker" +import { FastStringMap } from "./util" + +export interface Value { + type: Type; +} + +export class PrimValue implements Value { + + constructor( + public type: PrimType, + public value: any + ) { + + } + +} + +export const TRUE = new PrimValue(boolType, true); +export const FALSE = new PrimValue(boolType, false); + +export abstract class RecordValue implements Value { + + abstract type: RecordType; + + abstract getValueOfField(name: string): Value; + +} + +export class NativeRecord implements Value { + + constructor( + public type: RecordType, + protected fields: FastStringMap, + ) { + + } + + getValueOfField(name: string): Value { + if (!this.type.hasField(name)) { + throw new Error(`Field '${name}' does not exist on this record.`) + } + return this.fields[name] + } + +} + +export class RecordWrapper extends RecordValue { + + constructor( + public type: RecordType, + protected data: any, + ) { + super(); + } + + getValueOfField(name: string): Value { + if (!this.type.hasField(name)) { + throw new Error(`Field '${name}' does not exist on this record.`) + } + return this.data[name] + } + +} + +export class Evaluator { + + constructor(public checker: TypeChecker) { + + } + + match(value: Value, node: Syntax) { + + switch (node.kind) { + + case SyntaxKind.RecordPatt: + for (const field of node.fields) { + if (!this.match((value as RecordValue).getValueOfField(field.name.text), field.pattern)) { + return false; + } + } + return true; + + case SyntaxKind.TypePatt: + return value.type === this.checker.getTypeOfNode(node) + + default: + throw new Error(`I did not know how to match on pattern ${SyntaxKind[node.kind]}`) + + } + + } + + createValue(data: any) { + if (isNode(data)) { + return new RecordWrapper(this.checker.getTypeNamed(`Bolt.AST.${SyntaxKind[data.kind]}`)! as RecordType, data) + } + } + + eval(node: Syntax): Value { + + switch (node.kind) { + + case SyntaxKind.MatchExpr: + const value = this.eval(node.value); + for (const [pattern, result] of node.arms) { + if (this.match(value, pattern)) { + return this.eval(result as Expr) + } + } + return new PrimValue(this.checker.getTypeNamed('Void')!, null); + + case SyntaxKind.ConstExpr: + return new PrimValue(this.checker.getTypeOfNode(node), node.value) + + default: + throw new Error(`Could not evaluate node ${SyntaxKind[node.kind]}`) + + } + + } + +} + diff --git a/src/expander.ts b/src/expander.ts index 740ac4f94..e5f9076c9 100644 --- a/src/expander.ts +++ b/src/expander.ts @@ -1,72 +1,192 @@ +// FIXME Actually, the syntax expander could make use of algebraic effects to +// easily specify how the next expansion should happen. Just a thought. + import { TokenStream, SyntaxKind, Syntax, SourceFile, Decl, - Statement + RecordPatt, + Identifier, + TypeRef, + Patt, + ConstExpr, + QualName, + TuplePatt, + BindPatt, + TypePatt, + MatchExpr, + Stmt, + Module, } from "./ast" +import { TypeChecker } from "./checker" import { Parser, ParseError } from "./parser" +import { Evaluator, TRUE, FALSE } from "./evaluator" -type Transformer = (tokens: TokenStream) => Syntax; +interface Transformer { + pattern: Patt; + transform: (node: TokenStream) => Syntax; +} + +function createTypeRef(text: string) { + const ids = text.split('.').map(name => new Identifier(name)) + return new TypeRef(new QualName(ids[ids.length-1], ids.slice(0, -1)), []) +} + +/// This is actually a hand-parsed version of the following: +/// +/// Bolt.AST.Braced { +/// elements = [ +/// Bolt.AST.Identifier { text = "name" }, +/// Bolt.AST.Braced { +/// elements = [ +/// pattern: Bolt.AST.Pattern, +/// _: RArrow, +/// expression: Bolt.AST.Expr +/// ] +/// } +/// ], +/// } +const PATTERN_SYNTAX: Patt = + new RecordPatt( + createTypeRef('Bolt.AST.Sentence'), + [{ + name: new Identifier('elements'), + pattern: new TuplePatt([ + new RecordPatt( + createTypeRef('Bolt.AST.Identifier'), + [{ + name: new Identifier('text'), + pattern: new ConstExpr('syntax') + }] + ), + new RecordPatt( + createTypeRef('Bolt.AST.Braced'), + [{ + name: new Identifier('elements'), + pattern: new TuplePatt([ + new TypePatt(createTypeRef('Bolt.AST.Pattern'), new BindPatt(new Identifier('pattern'))), + new TypePatt(createTypeRef('Bolt.AST.RArrow'), new BindPatt(new Identifier('_'))), + new TypePatt(createTypeRef('Bolt.AST.Expr'), new BindPatt(new Identifier('expression'))) + ]) + }] + ) + ]) + }] + ) export class Expander { - transformers = new Map(); + protected transformers: Transformer[] = [] - constructor(public parser: Parser) { - this.transformers.set('fn', parser.parseFuncDecl.bind(parser)) - this.transformers.set('import', parser.parseImportDecl.bind(parser)) - this.transformers.set('foreign', parser.parseFuncDecl.bind(parser)) - this.transformers.set('let', parser.parseVarDecl.bind(parser)) - this.transformers.set('return', parser.parseRetStmt.bind(parser)) + constructor(public parser: Parser, public evaluator: Evaluator, public checker: TypeChecker) { + // this.transformers.push({ + // pattern: PATTERN_SYNTAX, + // transform: this.parser.parseSyntax.bind(this.parser) + // }) } getFullyExpanded(node: Syntax): Syntax { if (node.kind === SyntaxKind.SourceFile) { - const expanded: (Decl | Statement)[] = []; + const expanded: (Decl | Stmt)[] = []; + + let didExpand = false; + for (const element of node.elements) { - if (element.kind === SyntaxKind.Sentence) { - const newElement = this.getFullyExpanded(element) - expanded.push(newElement as Decl | Statement) + let newElement = this.getFullyExpanded(element); + if (newElement !== element) { + didExpand = true; } + expanded.push(newElement as Decl | Stmt) } + + if (!didExpand) { + return node; + } + return new SourceFile(expanded, null, node); + } else if (node.kind == SyntaxKind.Module) { + + const expanded = []; + + let didExpand = false; + + for (const element of node.elements) { + let newElement = this.getFullyExpanded(element); + if (newElement !== element) { + didExpand = true; + } + expanded.push(newElement as Decl | Stmt) + } + + if (!didExpand) { + return node; + } + + return new Module(node.isPublic, node.name, node.elements, null, node); + } else if (node.kind === SyntaxKind.Sentence) { - while (true) { + let newElement; - console.log('expanding sententce') + const tokens = node.toTokenStream(); - const tokens: TokenStream = node.toTokenStream() + try { - const t0 = tokens.peek(); - if (t0.kind !== SyntaxKind.Identifier) { - throw new ParseError(t0, [SyntaxKind.Identifier]); + newElement = this.parser.parseSourceElement(tokens) + + } catch (e) { + + // Regular errors should be propagated. + + if (!(e instanceof ParseError)) { + throw e; } - if (!this.transformers.has(t0.text)) { - return this.parser.parseCallExpr(tokens) + // The following applies a user-defined transformer to the token tree. + + while (true) { + let didExpand = false; + const expanded: Syntax[] = []; + const tokens = node.toTokenStream(); + for (const transformer of this.transformers) { + if (this.evaluator.eval(new MatchExpr(new ConstExpr(this.evaluator.createValue(node)), [ + [transformer.pattern, new ConstExpr(TRUE)], + [new ConstExpr(TRUE), new ConstExpr(FALSE)] + ]))) { + expanded.push(transformer.transform(tokens)) + didExpand = true; + // break; // FIXME + } + } + if (!didExpand) { + break; + } } - node = this.transformers.get(t0.text)!(tokens) + // If no transformer matched, then throw the original parse error. - if (node.kind !== SyntaxKind.Sentence) { - break; + if (!newElement) { + throw e; } } - return node + // Perform a full expansion on the transformed element. + + return this.getFullyExpanded(newElement) } else { - throw new Error(`unrecognised node of kind ${node.kind}`) + this.checker.check(node); + + return node; } diff --git a/src/parser.ts b/src/parser.ts index b1f2283d9..780066e54 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -22,8 +22,15 @@ import { QualName, CallExpr, ImportDecl, + SourceElement, + Module, + RecordDecl, } from "./ast" +import { stringType, intType } from "./checker" + +import { PrimValue } from "./evaluator" + function describeKind(kind: SyntaxKind): string { switch (kind) { case SyntaxKind.Identifier: @@ -34,10 +41,12 @@ function describeKind(kind: SyntaxKind): string { return "a string" case SyntaxKind.IntegerLiteral: return "an integer" - case SyntaxKind.FunctionKeyword: + case SyntaxKind.FnKeyword: return "'fn'" case SyntaxKind.ForeignKeyword: return "'foreign'" + case SyntaxKind.PubKeyword: + return "'pub'" case SyntaxKind.LetKeyword: return "'let'" case SyntaxKind.Semi: @@ -48,6 +57,12 @@ function describeKind(kind: SyntaxKind): string { return "'.'" case SyntaxKind.Comma: return "','" + case SyntaxKind.ModKeyword: + return "'mod'" + case SyntaxKind.StructKeyword: + return "'struct'" + case SyntaxKind.EnumKeyword: + return "'enum'" case SyntaxKind.Braced: return "'{' .. '}'" case SyntaxKind.Bracketed: @@ -78,12 +93,7 @@ export class ParseError extends Error { export class Parser { - - constructor() { - - } - - parseQualName(tokens: TokenStream) { + parseQualName(tokens: TokenStream): QualName { const path: Identifier[] = []; @@ -116,7 +126,7 @@ export class Parser { } } - parseImportDecl(tokens: TokenStream) { + parseImportDecl(tokens: TokenStream): ImportDecl { // Assuming first keyword is 'import' tokens.get(); @@ -144,7 +154,10 @@ export class Parser { const t0 = tokens.peek(); if (t0.kind === SyntaxKind.StringLiteral) { tokens.get(); - return new ConstExpr(t0.value, null, t0); + return new ConstExpr(new PrimValue(stringType, t0.value), null, t0); + } else if (t0.kind === SyntaxKind.IntegerLiteral) { + tokens.get(); + return new ConstExpr(new PrimValue(intType, t0.value), null, t0); } else if (t0.kind === SyntaxKind.Identifier) { const name = this.parseQualName(tokens); return new RefExpr(name, null, name.origNode); @@ -153,11 +166,20 @@ export class Parser { } } - parseExpr(tokens: TokenStream) { + parseSyntax(tokens: TokenStream): Syntax { + + // Assuming first token is 'syntax' + tokens.get(); + + throw new Error('not implemented') + + } + + parseExpr(tokens: TokenStream): Expr { return this.parsePrimExpr(tokens) } - parseParam(tokens: TokenStream) { + parseParam(tokens: TokenStream): Param { let defaultValue = null; let typeDecl = null; @@ -173,7 +195,9 @@ export class Parser { tokens.get(); defaultValue = this.parseExpr(tokens); } - } else if (t0.kind === SyntaxKind.EqSign) { + } + + if (t0.kind === SyntaxKind.EqSign) { tokens.get(); defaultValue = this.parseExpr(tokens); } @@ -230,11 +254,74 @@ export class Parser { return new RetStmt(expr, null, [t0, expr.getEndNode()]); } + parseStmt(tokens: TokenStream): Stmt { + + } + + parseRecordDecl(tokens: TokenStream): RecordDecl { + + let isPublic = false; + + let kw = tokens.get(); + if (kw.kind !== SyntaxKind.Identifier) { + throw new ParseError(kw, [SyntaxKind.PubKeyword, SyntaxKind.StructKeyword]); + } + if (kw.text === 'pub') { + isPublic = true; + kw = tokens.get(); + } + + if (kw.kind !== SyntaxKind.Identifier || kw.text !== 'struct') { + throw new ParseError(kw, [SyntaxKind.StructKeyword]) + } + + const name = this.parseQualName(tokens); + + const t2 = tokens.get(); + + if (t2.kind !== SyntaxKind.Braced) { + throw new ParseError(kw, [SyntaxKind.Braced]) + } + + let fields = []; + + return new RecordDecl(isPublic, name, fields); + + } + parseStmts(tokens: TokenStream, origNode: Syntax | null): Stmt[] { // TODO return [] } + parseModDecl(tokens: TokenStream): Module { + + let isPublic = false; + + let kw = tokens.get(); + if (kw.kind !== SyntaxKind.Identifier) { + throw new ParseError(kw, [SyntaxKind.PubKeyword, SyntaxKind.ModKeyword]); + } + if (kw.text === 'pub') { + isPublic = true; + kw = tokens.get(); + } + + if (kw.kind !== SyntaxKind.Identifier || kw.text !== 'mod') { + throw new ParseError(kw, [SyntaxKind.ModKeyword]) + } + + const name = this.parseQualName(tokens); + + const t1 = tokens.get(); + if (t1.kind !== SyntaxKind.Braced) { + throw new ParseError(t1, [SyntaxKind.Braced]) + } + + return new Module(isPublic, name, t1.toSentences()); + + } + protected assertEmpty(tokens: TokenStream) { const t0 = tokens.peek(1); if (t0.kind !== SyntaxKind.EOS) { @@ -242,24 +329,35 @@ export class Parser { } } - parseFuncDecl(tokens: TokenStream, origNode: Syntax | null) { + parseFuncDecl(tokens: TokenStream, origNode: Syntax | null): FuncDecl { let target = "Bolt"; + let isPublic = false; const k0 = tokens.peek(); if (k0.kind !== SyntaxKind.Identifier) { - throw new ParseError(k0, [SyntaxKind.ForeignKeyword, SyntaxKind.FunctionKeyword]) + throw new ParseError(k0, [SyntaxKind.PubKeyword, SyntaxKind.ForeignKeyword, SyntaxKind.FnKeyword]) } - if (k0.text === 'foreign') { + if (k0.text === 'pub') { + tokens.get(); + isPublic = true; + } + + const k1 = tokens.peek(); + if (k1.kind !== SyntaxKind.Identifier) { + throw new ParseError(k1, [SyntaxKind.ForeignKeyword, SyntaxKind.FnKeyword]) + } + if (k1.text === 'foreign') { + tokens.get(); const l1 = tokens.get(); if (l1.kind !== SyntaxKind.StringLiteral) { throw new ParseError(l1, [SyntaxKind.StringLiteral]) } target = l1.value; } - const k1 = tokens.get(); - if (k1.kind !== SyntaxKind.Identifier || k1.text !== 'fn') { - throw new ParseError(k1, [SyntaxKind.FunctionKeyword]) + const k2 = tokens.get(); + if (k2.kind !== SyntaxKind.Identifier || k2.text !== 'fn') { + throw new ParseError(k2, [SyntaxKind.FnKeyword]) } let name: QualName; @@ -367,11 +465,49 @@ export class Parser { } } - return new FuncDecl(target, name, params, returnType, body, null, origNode) + return new FuncDecl(isPublic, target, name, params, returnType, body, null, origNode) } - parseCallExpr(tokens: TokenStream) { + parseSourceElement(tokens: TokenStream): SourceElement { + const t0 = tokens.peek(1); + if (t0.kind === SyntaxKind.Identifier) { + let i = 1; + let kw: Token = t0; + if (t0.text === 'pub') { + i++; + kw = tokens.peek(i); + if (kw.kind !== SyntaxKind.Identifier) { + throw new ParseError(kw, [SyntaxKind.ForeignKeyword, SyntaxKind.ModKeyword, SyntaxKind.LetKeyword, SyntaxKind.FnKeyword, SyntaxKind.EnumKeyword, SyntaxKind.StructKeyword]) + } + } + if (t0.text === 'foreign') { + i += 2; + kw = tokens.peek(i); + if (kw.kind !== SyntaxKind.Identifier) { + throw new ParseError(kw, [SyntaxKind.ModKeyword, SyntaxKind.LetKeyword, SyntaxKind.FnKeyword, SyntaxKind.EnumKeyword, SyntaxKind.StructKeyword]) + } + } + switch (kw.text) { + case 'mod': + return this.parseModDecl(tokens); + case 'fn': + return this.parseFuncDecl(tokens, null); + case 'let': + return this.parseVarDecl(tokens); + case 'struct': + return this.parseRecordDecl(tokens); + case 'enum': + return this.parseVariantDecl(tokens); + default: + throw new ParseError(kw, [SyntaxKind.ModKeyword, SyntaxKind.LetKeyword, SyntaxKind.FnKeyword, SyntaxKind.EnumKeyword, SyntaxKind.StructKeyword]) + } + } else { + return this.parseStmt(tokens) + } + } + + parseCallExpr(tokens: TokenStream): CallExpr { const operator = this.parsePrimExpr(tokens) const args: Expr[] = [] diff --git a/src/scanner.ts b/src/scanner.ts index 273cb22d4..04f41308e 100644 --- a/src/scanner.ts +++ b/src/scanner.ts @@ -23,6 +23,7 @@ import { IntegerLiteral, Colon, EOS, + Dot, EqSign, } from "./ast" @@ -97,6 +98,10 @@ interface Stream { read(): T } +function isDigit(ch: string) { + return XRegExp('\\p{Nd}').test(ch) +} + function isWhiteSpace(ch: string) { return ch == '\n' || XRegExp('\\p{Zs}').test(ch) } @@ -201,6 +206,9 @@ export class Scanner { } switch (c0) { + case '.': + this.getChar(); + return new Dot(new TextSpan(this.file, startPos, this.currPos.clone())); case '=': this.getChar(); return new EqSign(new TextSpan(this.file, startPos, this.currPos.clone())); @@ -239,6 +247,12 @@ export class Scanner { return new StringLiteral(text, new TextSpan(this.file, startPos, endPos)) + } else if (isDigit(c0)) { + + const digits = this.takeWhile(isDigit) + const endPos = this.currPos.clone(); + return new IntegerLiteral(BigInt(digits), new TextSpan(this.file, startPos, endPos)); + } else if (isOpenPunct(c0)) { this.getChar(); @@ -323,10 +337,9 @@ export class Scanner { : this.scanToken(); } - scan() { + scanTokens() { - const elements: Decl[] = [] - const startPos = this.currPos.clone() + const elements: Sentence[] = [] outer: while (true) { @@ -351,15 +364,24 @@ export class Scanner { } if (tokens.length > 0) { - elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone()))) + elements.push( + new Sentence( + tokens, + new TextSpan(this.file, tokens[0].span!.start.clone(), tokens[tokens.length-1].span!.end.clone()) + ) + ) } } + return elements + } + + scan() { + const startPos = this.currPos.clone(); + const elements = this.scanTokens(); const endPos = this.currPos.clone(); - - return new SourceFile(elements, new TextSpan(this.file, startPos, endPos)) - + return new SourceFile(elements, new TextSpan(this.file, startPos, endPos)); } } diff --git a/src/util.ts b/src/util.ts index 0ee0a1a20..44df16c32 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,4 +1,8 @@ +export interface FastStringMap { + [key: string]: T +} + export interface Stream { get(): T; peek(count?: number): T;