From 96aa6b6991446a6f33cadf41e91021dff388ee00 Mon Sep 17 00:00:00 2001
From: Sam Vervaeck <vervaeck.sam@skynet.be>
Date: Mon, 24 Feb 2020 19:23:34 +0100
Subject: [PATCH] Fix support for single-character tokens

---
 src/ast.ts                  | 42 +++++++++++++++++++++++++++++++++++++
 src/scanner.ts              | 34 +++++++++++++++++++++++-------
 test/001-punct-depth-1.bolt |  6 +++---
 3 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/src/ast.ts b/src/ast.ts
index de9fb9974..997244cfa 100644
--- a/src/ast.ts
+++ b/src/ast.ts
@@ -14,6 +14,8 @@ export enum SyntaxKind {
   Operator,
   Punctuated,
   Semi,
+  Comma,
+  Colon,
 
   // Special nodes
 
@@ -242,6 +244,46 @@ export class Semi extends SyntaxBase {
 
 }
 
+export class Colon extends SyntaxBase {
+  
+  kind: SyntaxKind.Colon = SyntaxKind.Colon;
+
+  constructor(
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON() {
+    return {
+      kind: 'Colon',
+      span: this.span.toJSON(),
+    }
+  }
+
+}
+
+export class Comma extends SyntaxBase {
+
+  kind: SyntaxKind.Comma = SyntaxKind.Comma;
+
+  constructor(
+    public span: TextSpan,
+    public parentNode: Syntax | null = null
+  ) {
+    super();
+  }
+
+  toJSON() {
+    return {
+      kind: 'Comma',
+      span: this.span.toJSON(),
+    }
+  }
+
+}
+
 export type Token
   = Semi
   | Identifier
diff --git a/src/scanner.ts b/src/scanner.ts
index 0d51a89a0..1a0793699 100644
--- a/src/scanner.ts
+++ b/src/scanner.ts
@@ -13,7 +13,10 @@ import {
   Decl,
   Punctuated,
   Sentence,
-  SourceFile
+  SourceFile,
+  Semi,
+  Comma,
+  Colon
 } from "./ast"
 
 function escapeChar(ch: string) {
@@ -189,6 +192,19 @@ export class Scanner {
 
       const startPos = this.currPos.clone()
 
+      switch (c0) {
+        case ';':
+          this.getChar();
+          return new Semi(new TextSpan(this.file, startPos, this.currPos.clone()));
+        case ',':
+          this.getChar();
+          return new Comma(new TextSpan(this.file, startPos, this.currPos.clone()));
+        case ':':
+          this.getChar();
+          return new Colon(new TextSpan(this.file, startPos, this.currPos.clone()));
+
+      }
+
       if (isOpenPunct(c0)) {
 
         this.getChar();
@@ -257,14 +273,18 @@ export class Scanner {
     const elements: Decl[] = []
     const startPos = this.currPos.clone()
 
-    while (true) {
+    outer: while (true) {
 
       const tokens: Token[] = [];
 
-      while (true) {
+      inner: while (true) {
         const token = this.scanToken();
         if (token === null) {
-          break;
+          if (tokens.length === 0) {
+            break outer;
+          } else {
+            break inner;
+          }  
         }
         if (token.kind === SyntaxKind.Semi) {
           break;
@@ -275,12 +295,10 @@ export class Scanner {
         }
       }
 
-      if (tokens.length === 0) {
-        break;
+      if (tokens.length > 0) {
+        elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone())))
       }
 
-      elements.push(new Sentence(tokens, new TextSpan(this.file, tokens[0].span.start.clone(), tokens[tokens.length-1].span.end.clone())))
-
     }
 
     const endPos = this.currPos.clone();
diff --git a/test/001-punct-depth-1.bolt b/test/001-punct-depth-1.bolt
index ef5665f11..b5bc91879 100644
--- a/test/001-punct-depth-1.bolt
+++ b/test/001-punct-depth-1.bolt
@@ -1,3 +1,3 @@
-{ foo baz bar }
-( foo baz bar )
-[ foo baz bar ]
+{ foo baz bar };
+( foo baz bar );
+[ foo baz bar ];