Add support for paring sentences

src/ast.ts

@@ -13,16 +13,25 @@ export enum SyntaxKind {
   Identifier,
   Operator,
   Punctuated,
+  Semi,
+
+  // Special nodes
 
   SourceFile,
 
   QualName,
 
+  Sentence,
+
   // Expressions
 
   ConstantExpr,
   ReferenceExpr,
 
+  // Statements
+
+  ReturnStatement,
+
   // Type declarations
 
   TypeReference,
@@ -97,9 +106,14 @@ export class TextSpan {
 
 }
 
-export class Literal {
+abstract class SyntaxBase {
+  abstract kind: SyntaxKind;
+  abstract parentNode: Syntax | null;
+}
 
-  kind = SyntaxKind.Literal;
+export class Literal extends SyntaxBase {
+
+  kind: SyntaxKind.Literal = SyntaxKind.Literal;
 
   static META = {
     value: EdgeType.Primitive,
@@ -107,9 +121,10 @@ export class Literal {
 
   constructor(
     public value: string | bigint,
-    public span: TextSpan
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
   ) {
-
+    super();
   }
 
   toJSON(): Json {
@@ -127,9 +142,9 @@ export enum PunctType {
   Brace,
 }
 
-export class Punctuated {
+export class Punctuated extends SyntaxBase {
 
-  kind = SyntaxKind.Punctuated
+  kind: SyntaxKind.Punctuated = SyntaxKind.Punctuated
 
   static META = {
     punctuator: EdgeType.Primitive,
@@ -139,9 +154,10 @@ export class Punctuated {
   constructor(
     public punctuator: PunctType,
     public elements: Token[],
-    public span: TextSpan
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
   ) {
-
+    super();
   }
 
   toJSON(): Json {
@@ -154,9 +170,9 @@ export class Punctuated {
 
 }
 
-export class Identifier {
+export class Identifier extends SyntaxBase {
 
-  kind = SyntaxKind.Identifier;
+  kind: SyntaxKind.Identifier = SyntaxKind.Identifier;
 
   static META = {
     text: EdgeType.Primitive
@@ -164,9 +180,10 @@ export class Identifier {
 
   constructor(
     public text: string,
-    public span: TextSpan
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
   ) {
-
+    super();
   }
 
   toJSON(): Json {
@@ -179,9 +196,9 @@ export class Identifier {
 
 }
 
-export class Operator {
+export class Operator extends SyntaxBase {
 
-  kind = SyntaxKind.Operator;
+  kind: SyntaxKind.Operator = SyntaxKind.Operator;
 
   static META = {
     text: EdgeType.Primitive
@@ -192,7 +209,7 @@ export class Operator {
     public span: TextSpan,
     public parentNode: Syntax | null = null
   ) {
-
+    super();
   }
 
   toJSON(): Json {
@@ -205,15 +222,58 @@ export class Operator {
 
 }
 
+export class Semi extends SyntaxBase {
+  
+  kind: SyntaxKind.Semi = SyntaxKind.Semi;
+
+  constructor(
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON() {
+    return {
+      kind: 'Semi',
+      span: this.span.toJSON(),
+    }
+  }
+
+}
+
 export type Token
-  = Identifier
+  = Semi
+  | Identifier
   | Operator
   | Literal
   | Punctuated
 
+export class Sentence extends SyntaxBase {
+
+  kind = SyntaxKind.Sentence;
+
+  constructor(
+    public tokens: Token[],
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON(): Json {
+    return {
+      kind: 'Sentence',
+      tokens: this.tokens.map(token => token.toJSON()),
+      span: this.span.toJSON(),
+    }
+  }
+
+}
+
 export class QualName {
 
-  kind = SyntaxKind.QualName;
+  kind: SyntaxKind.QualName = SyntaxKind.QualName;
 
   static META = {
     name: EdgeType.Node,
@@ -252,9 +312,25 @@ export class ConstantExpr {
 export type Expr
   = ConstantExpr
 
+class ReturnStatement extends SyntaxBase {
+
+  kind: SyntaxKind.ReturnStatement = SyntaxKind.ReturnStatement;
+
+  constructor(
+    public value: Expr | null,
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+}
+
+export type Statement
+  = ReturnStatement
+
 export class TypeReference {
 
-  kind = SyntaxKind.TypeReference;
+  kind: SyntaxKind.TypeReference = SyntaxKind.TypeReference;
 
   static META = {
     name: EdgeType.Node,
@@ -275,23 +351,23 @@ export class TypeReference {
 export type TypeDecl
   = TypeReference
 
-export class Unexpanded {
+// export class Unexpanded {
+// 
+//   static META = {
+//     tokens: EdgeType.Node | EdgeType.List
+//   }
+// 
+//   constructor(
+//     public tokens: Token[],
+//     public span: TextSpan,
+//     public parentNode: Syntax | null = null
+//   ) {
+// 
+//   }
+// 
+// }
 
-  static META = {
+export class FunctionDecl extends SyntaxBase {
-    tokens: EdgeType.Node | EdgeType.List
-  }
-
-  constructor(
-    public tokens: Token[],
-    public span: TextSpan,
-    public parentNode: Syntax | null = null
-  ) {
-
-  }
-
-}
-
-export class FunctionDecl {
 
   kind = SyntaxKind.FunctionDecl;
 
@@ -310,13 +386,13 @@ export class FunctionDecl {
     public span: TextSpan, 
     public parentNode: Syntax | null = null
   ) {
-
+    super();
   }
 
 
 }
 
-export class VariableDecl {
+export class VariableDecl extends SyntaxBase {
 
   kind = SyntaxKind.VariableDecl;
 
@@ -332,13 +408,13 @@ export class VariableDecl {
     public value: Expr | null,
     public span: TextSpan
   ) {
-
+    super();
   }
 
 }
 
 export type Decl
-  = Unexpanded
+  = Sentence 
   | FunctionDecl
   | VariableDecl
 
@@ -348,10 +424,23 @@ export type Syntax
   | SourceFile
   | QualName
 
-export class SourceFile {
+export class SourceFile extends SyntaxBase {
 
-  constructor(public elements: Decl[], public span: TextSpan) {
+  kind: SyntaxKind.SourceFile = SyntaxKind.SourceFile;
 
+  constructor(
+    public elements: (Decl | Statement)[],
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON() {
+    return {
+      elements: this.elements.map(element => element.toJSON()),
+      span: this.span.toJSON(),
+    }
   }
 
 }

src/bin/bolt.ts

@@ -7,7 +7,7 @@ import * as fs from "fs"
 import yargs from "yargs"
 
 import { Scanner } from "../scanner"
-import { Token, TextFile } from "../ast"
+import { Token, TextFile, SourceFile } from "../ast"
 
 function toArray<T>(value: T): T extends Array<any> ? T : T[] {
   if (Array.isArray(value)) {
@@ -63,13 +63,13 @@ yargs
     args => {
 
       const hooks: Hook[] = toArray(args.hook as string[] | string).map(parseHook);
+      const sourceFiles: SourceFile[] = [];
 
       for (const filepath of toArray(args.files as string[] | string)) {  
 
         const file = new TextFile(filepath);
         const content = fs.readFileSync(filepath, 'utf8')
         const scanner = new Scanner(file, content)
-        const tokens: Token[] = [];
 
         for (const hook of hooks) {
           if (hook.name === 'scan' && hook.timing === 'before') {
@@ -85,20 +85,21 @@ yargs
           }
         }
 
-        while (true) {
+        const sourceFile = scanner.scan();
-          const token = scanner.scanToken()
+        // while (true) {
-          if (token === null) {
+        //   const token = scanner.scanToken()
-            break;
+        //   if (token === null) {
-          }
+        //     break;
-          tokens.push(token);
+        //   }
-        }
+        //   tokens.push(token);
+        // }
 
         for (const hook of hooks) {
           if (hook.name === 'scan' && hook.timing == 'after') {
             for (const effect of hook.effects) {
               switch (effect) {
                 case 'dump':
-                  console.log(JSON.stringify(tokens.map(t => t.toJSON()), undefined, 2));
+                  console.log(JSON.stringify(sourceFile.toJSON(), undefined, 2));
                   break;
                 case 'abort':
                   process.exit(0);
@@ -110,7 +111,10 @@ yargs
           }
         }
 
+        sourceFiles.push(sourceFile);
+
       }
+
     })
 
   .help()

src/scanner.ts

@@ -2,6 +2,7 @@
 import XRegExp from "xregexp"
 
 import {
+  SyntaxKind,
   TextFile,
   TextPos,
   TextSpan,
@@ -9,7 +10,10 @@ import {
   Operator,
   PunctType,
   Token,
+  Decl,
   Punctuated,
+  Sentence,
+  SourceFile
 } from "./ast"
 
 function escapeChar(ch: string) {
@@ -168,7 +172,7 @@ export class Scanner {
     return text;
   }
 
-  scanToken() {
+  scanToken(): Token | null {
 
     while (true) {
 
@@ -248,7 +252,40 @@ export class Scanner {
 
   }
 
-  scanTokenList() {
+  scan() {
+
+    const elements: Decl[] = []
+    const startPos = this.currPos.clone()
+
+    while (true) {
+
+      const tokens: Token[] = [];
+
+      while (true) {
+        const token = this.scanToken();
+        if (token === null) {
+          break;
+        }
+        if (token.kind === SyntaxKind.Semi) {
+          break;
+        }
+        tokens.push(token)
+        if (token.kind === SyntaxKind.Punctuated && token.punctuator === PunctType.Brace) {
+          break;
+        }
+      }
+
+      if (tokens.length === 0) {
+        break;
+      }
+
+      elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone())))
+
+    }
+
+    const endPos = this.currPos.clone();
+
+    return new SourceFile(elements, new TextSpan(this.file, startPos, endPos))
 
   }