Make test infrasture more usable by adding a diff reporter

package.json

@@ -10,14 +10,16 @@
   "scripts": {
     "watch": "webpack --watch --config webpack.dev.js",
     "prepare": "webpack --config webpack.dev.js",
-    "test": "node build/bin/bolt-test.js compare",
+    "test": "node lib/bin/bolt-test.js compare",
     "generate-ast": "tsastgen src/ast-spec.ts:src/ast.ts",
-    "update-lkg": "node build/bin/bolt-test.js create-snapshot lkg"
+    "update-lkg": "node lib/bin/bolt-test.js create-snapshot lkg"
   },
   "author": "Sam Vervaeck <vervaeck.sam@skynet.be>",
   "license": "GPL-3.0",
   "repository": "https://github.com/samvv/Bolt",
   "dependencies": {
+    "@types/commonmark": "^0.27.4",
+    "@types/diff": "^4.0.2",
     "@types/fs-extra": "^9.0.1",
     "@types/glob": "^7.1.2",
     "@types/js-yaml": "^3.12.4",
@@ -29,6 +31,8 @@
     "@types/uuid": "^8.0.0",
     "@types/yargs": "^15.0.5",
     "chalk": "^4.1.0",
+    "commonmark": "^0.29.1",
+    "diff": "^4.0.2",
     "fs-extra": "^9.0.1",
     "glob": "^7.1.6",
     "js-yaml": "^3.14.0",
@@ -47,11 +51,11 @@
     "@babel/core": "^7.10.2",
     "@babel/plugin-proposal-class-properties": "^7.10.1",
     "@types/chai": "^4.2.11",
-    "@types/mocha": "^7.0.2",
+    "@types/tape": "^4.13.0",
     "babel-loader": "^8.1.0",
-    "chai": "^4.2.0",
     "concurrently": "^5.2.0",
-    "mocha": "^8.0.1",
+    "tap-min": "^2.0.0",
+    "tape": "^5.0.1",
     "ts-loader": "^7.0.5",
     "typescript": "^3.9.5",
     "webpack": "^4.43.0",

src/ast-spec.ts

@@ -4,7 +4,8 @@ import { TextSpan } from "./text"
 import { Value } from "./evaluator"
 import { Package } from "./package"
 import { Diagnostic } from "./diagnostics";
-import { serializeTag, serialize } from "./util";
+import { MapLike, serializeTag, inspectTag, indent } from "./util";
+import { InspectOptions, InspectOptionsStylized, inspect } from "util";
 
 let nextNodeId = 1;
 
@@ -30,10 +31,30 @@ export abstract class Syntax {
     this.id = nextNodeId++;
   }
 
+  protected [inspectTag](depth: number | null, options: InspectOptionsStylized) {
+    const proto = Object.getPrototypeOf(this);
+    if (depth !== null && depth < 0) {
+      return options.stylize(`[${proto.constructor.name}]`, 'special')
+    }
+    const newOptions = {
+      ...options,
+      depth: options.depth === null ? null : options.depth!-1,
+    }
+    let out = `${proto.constructor.name} {\n`;
+    for (const key of Object.keys(this)) {
+      if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
+        continue;
+      }
+      out += `${key}: ${inspect((this as any)[key], newOptions)},\n`;
+    }
+    out += '}\n';
+    return out;
+  }
+
   [serializeTag]() {
     const result: any[] = [];
     for (const key of Object.keys(this)) {
-      if (key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
+      if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
         continue;
       }
       result.push((this as any)[key]);

src/ast.ts

@@ -8,7 +8,9 @@ import { Package } from "./package";
 
 import { Diagnostic } from "./diagnostics";
 
-import { serializeTag, serialize } from "./util";
+import { MapLike, serializeTag, inspectTag, indent } from "./util";
+
+import { InspectOptions, InspectOptionsStylized, inspect } from "util";
 
 let nextNodeId = 1;
 
@@ -26,10 +28,29 @@ export abstract class SyntaxBase {
     constructor(public span: TextSpan | null = null) {
         this.id = nextNodeId++;
     }
+    protected [inspectTag](depth: number | null, options: InspectOptionsStylized) {
+        const proto = Object.getPrototypeOf(this);
+        if (depth !== null && depth < 0) {
+            return options.stylize(`[${proto.constructor.name}]`, 'special');
+        }
+        const newOptions = {
+            ...options,
+            depth: options.depth === null ? null : options.depth! - 1,
+        };
+        let out = `${proto.constructor.name} {\n`;
+        for (const key of Object.keys(this)) {
+            if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
+                continue;
+            }
+            out += `${key}: ${inspect((this as any)[key], newOptions)},\n`;
+        }
+        out += '}\n';
+        return out;
+    }
     [serializeTag]() {
         const result: any[] = [];
         for (const key of Object.keys(this)) {
-            if (key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
+            if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
                 continue;
             }
             result.push((this as any)[key]);

src/bin/bolt-test.ts

@@ -1,4 +1,7 @@
 
+// NOTE The code in this file is not as clean as we want it to be, but we'll be upgrading our
+//      test infrastructure anyways with version 1.0.0 so it does not matter much.
+
 import "source-map-support/register"
 import "reflect-metadata"
 
@@ -6,24 +9,27 @@ import * as fs from "fs-extra"
 import * as path from "path"
 import * as crypto from "crypto"
 
+import chalk from "chalk"
 import { v4 as uuidv4 } from "uuid"
 import yargs from "yargs"
 import yaml, { FAILSAFE_SCHEMA } from "js-yaml"
 import { sync as globSync } from "glob"
-import ora from "ora"
+import ora, { Ora } from "ora"
-
+import { Parser as CommonmarkParser } from "commonmark"
 import { Parser } from "../parser"
 import { Scanner } from "../scanner"
 import { SyntaxKind, Syntax } from "../ast"
-import { Json, serialize, JsonObject, MapLike, upsearchSync, deepEqual, serializeTag, deserializable, deserialize } from "../util"
+import { Json, serialize, JsonObject, MapLike, upsearchSync, deepEqual, serializeTag, deserializable, deserialize, JsonArray, verbose, diffpatcher } from "../util"
-import { DiagnosticIndex, DiagnosticPrinter, E_TESTS_DO_NOT_COMPARE, E_INVALID_TEST_COMPARE } from "../diagnostics"
+import { DiagnosticIndex, DiagnosticPrinter, E_TESTS_DO_NOT_COMPARE, E_INVALID_TEST_COMPARE, E_NO_BOLTFILE_FOUND_IN_PATH_OR_PARENT_DIRS, Diagnostic } from "../diagnostics"
 import { TextFile, TextPos, TextSpan } from "../text"
+import { diffLines } from "diff"
+import { inspect } from "util"
 
 const PACKAGE_ROOT = path.dirname(upsearchSync('package.json')!);
 const STORAGE_DIR = path.join(PACKAGE_ROOT, '.test-storage');
 
 const diagnostics = new DiagnosticPrinter();
-const spinner = ora(`Initializing test session ...`).start();
+let spinner: Ora;
 
 // TODO move some logic from TestSession to TestSuite
 // TODO hash the entire code base and have it serve as a unique key for TestSession
@@ -40,7 +46,7 @@ class Test {
 
   public key: string;
 
-  public result?: Json;
+  public result?: any;
   public error: Error | null = null;
 
   constructor(
@@ -148,20 +154,40 @@ class TestSession {
 
 }
 
+function toString(value: any): string {
+  return inspect(value, {
+    colors: false,
+    depth: Infinity,
+  })
+}
+
 function compare(actualKey: string, expectedKey: string) {
 
   for (const testKey of fs.readdirSync(path.join(STORAGE_DIR, 'snapshots', actualKey))) { 
 
     const test = deserialize(readJson(path.join(STORAGE_DIR, 'tests', testKey)))
 
-    const actualTestData = deserialize(readJson(path.join(STORAGE_DIR, 'snapshots', actualKey, testKey))!);
+    const actualData = readJson(path.join(STORAGE_DIR, 'snapshots', actualKey, testKey))!;
-    const expectedTestData = deserialize(readJson(path.join(STORAGE_DIR, 'snapshots', expectedKey, testKey)));
+    const expectedData = readJson(path.join(STORAGE_DIR, 'snapshots', expectedKey, testKey))!;
-    if (!deepEqual(actualTestData.result, expectedTestData.result)) {
+    const actual = deserialize(actualData);
+    const expected = deserialize(expectedData);
+    const diffs = diffLines(toString(actual), toString(expected));
+    if (diffs.some(diff => diff.added || diff.removed)) {
       diagnostics.add({
         message: E_TESTS_DO_NOT_COMPARE,
         severity: 'error',
         node: test,
-      })
+      });
+      for (const diff of diffs) {
+        let out = diff.value;
+        if (diff.removed) {
+          out = chalk.red(out);
+        } else if (diff.added) {
+          out = chalk.green(out);
+        }
+        process.stderr.write(out);
+      }
+      //lconsole.error(jsondiffpatch.formatters.console.format(delta, expected) + '\n');
     }
 
   }
@@ -177,115 +203,60 @@ interface TestFileMetadata {
   expect: string;
 }
 
-function* loadTests(filepath: string): IterableIterator<Test> {
+interface SimpleToken {
-
+  text: string;
-  const file = new TextFile(filepath);
+  startPos: TextPos;
-  const contents = file.getText('utf8');
+  endPos: TextPos;
-
-  let i = 0;
-  let column = 1
-  let line = 1;
-  let atNewLine = true;
-
-  assertText('---');
-  let yamlStr = '';
-  i += 3;
-  while (!lookaheadEquals('---')) {
-    yamlStr += contents[i++];
-  }
-  i += 3;
-  const metadata = yaml.safeLoad(yamlStr);
-
-  while (i < contents.length) {
-    skipWhiteSpace();
-    if (atNewLine && column >= 5) {
-      const startPos = new TextPos(i, line, column);
-      const text = scanCodeBlock();
-      const endPos = new TextPos(i, line, column);
-      if (metadata['split-lines']) {
-        for (const line of text.split('\n')) {
-          if (line.trim() !== '') {
-            yield new Test(new TextSpan(file, startPos.clone(), endPos), metadata.type, line, metadata);
-            startPos.advance(line);
-          }
-        }
-      } else {
-        yield new Test(new TextSpan(file, startPos, endPos), metadata.type, text, metadata);
-      }
-    } else {
-      getChar();
-    }
 }
 
-  function getChar() {
+const PREAMBLE_START = '---\n';
-    const ch = contents[i++];
+const PREAMBLE_END = '---\n';
-    if (ch === '\n') {
-      column = 1;
-      line++;
-      atNewLine = true;
-    } else {
-      if (!isEmpty(ch)) {
-        atNewLine = false;
-      }
-      column++;
-    }
-    return ch;
-  }
 
-  function assertText(str: string) {
+function getPreamble(text: string): string {
-    for (let k = 0; k < str.length; k++) {
+  if (!text.startsWith(PREAMBLE_START)) {
-      if (contents[i+k] !== str[k]) {
+    return '';
-        throw new Error(`Expected '${str}' but got ${contents.substr(i, i+str.length)}`)
   }
-    }
+  let out = '';
-  }
+  for (let i = PREAMBLE_START.length; i < text.length; i++) {
-
+    if (text.startsWith(PREAMBLE_END, i)) {
-  function isEmpty(ch: string): boolean {
-    return /[\t ]/.test(ch);
-  }
-
-  function lookaheadEquals(str: string): boolean {
-    for (let k = 0; k < str.length; k++) {
-      if (contents[i+k] !== str[k]) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  function scanCodeBlock() {
-    let out = ''
-    while (i < contents.length) {
-      const ch = getChar();
-      if (ch === '\n') {
-        out += ch;
-        skipWhiteSpace();
-        continue;
-      }
-      if (column < 5) {
       break;
     }
-      out += ch;
+    out += text[i];
   }
   return out;
 }
 
-  function skipWhiteSpace() {
+function* loadTests(filepath: string): IterableIterator<Test> {
-    takeWhile(isWhiteSpace)
+  const file = new TextFile(filepath);
+  const contents = file.getText('utf8');
+  const preamble = getPreamble(contents);
+  const metadata = yaml.safeLoad(preamble);
+  const parser = new CommonmarkParser();
+  const rootNode = parser.parse(contents);
+  if (rootNode.firstChild === null) {
+    return;
   }
-
+  for (let node = rootNode.firstChild; node.next !== null; node = node.next) {
-  function takeWhile(pred: (ch: string) => boolean) {
+    if (node.type === 'code_block') {
-    let out = '';
+      if (metadata['split-lines']) {
-    while (true) {
+        let startPos = new TextPos(0, node.sourcepos[0][0], node.sourcepos[0][1]);
-      const c0 = contents[i];
+        startPos.advance('```')
-      if (!pred(c0)) {
+        startPos.advance(node.info! + '\n')
-        break
+        let endPos = startPos.clone();
+        for (const line of node.literal!.split('\n')) {
+          if (line.length > 0) {
+            yield new Test(new TextSpan(file, startPos.clone(), endPos.clone()), metadata.type, line, metadata);
+            startPos = endPos;
+          }
+          endPos.advance(line + '\n');
+        }
+      } else {
+        const startPos = new TextPos(0, node.sourcepos[0][0], node.sourcepos[0][1]);
+        const endPos = new TextPos(0, node.sourcepos[1][0], node.sourcepos[1][1]);
+        yield new Test(new TextSpan(file, startPos, endPos), metadata.type, node.literal!, metadata);
       }
-      out += getChar();
     }
-    return out;
   }
-
 }
 
 function findSnapshot(ref: string): string | null {
@@ -344,9 +315,9 @@ type TestRunner = (test: Test) => Json;
 
 const TEST_RUNNERS: MapLike<TestRunner> = {
 
-  scan(test: Test): Json {
+  scan(test: Test): any {
       const diagnostics = new DiagnosticIndex;
-      const scanner = new Scanner(test.span.file, test.text, test.span.start);
+      const scanner = new Scanner(test.span.file, test.text, test.span.start.clone());
       const tokens = []
       while (true) {
           const token = scanner.scan();
@@ -355,13 +326,13 @@ const TEST_RUNNERS: MapLike<TestRunner> = {
           }
           tokens.push(token);
       }
-      return serialize({
+      return {
           diagnostics: [...diagnostics.getAllDiagnostics()],
           tokens,
-      });
+      };
   },
 
-  parse(test: Test): Json {
+  parse(test: Test): any {
       const kind = test.data.expect ?? 'SourceFile';
       const diagnostics = new DiagnosticIndex;
       const parser = new Parser();
@@ -377,15 +348,27 @@ const TEST_RUNNERS: MapLike<TestRunner> = {
           default:
               throw new Error(`I did not know how to parse ${kind}`)
       }
-      return serialize({
+      return {
           diagnostics: [...diagnostics.getAllDiagnostics()],
           results,
-      })
+      };
   },
 
 }
 
+const TEST_REPORTERS = {
+
+  scan(test: Test) {
+    const printer = new DiagnosticPrinter();
+    for (const diagnostic of test.result!.diagnostics) {
+      printer.add(diagnostic as Diagnostic);
+    }
+  }
+
+}
+
 yargs
+
   .command(['$0 [pattern..]', 'run [pattern..]'], 'Run all tests on the current version of the compiler',
     yargs => yargs
       .array('pattern')
@@ -401,6 +384,8 @@ yargs
       .default('alias', [])
     , args => {
 
+      spinner = ora(`Initializing test session ...`).start();
+
       const session = new TestSession();
       session.scanForTests(args as LoadTestsOptions);
       session.run();
@@ -424,6 +409,7 @@ yargs
 
     }
   )
+
   .command(['create-snapshot [alias..]'], 'Create a new snapshot from the output of the current compiler',
     yargs => yargs
       .array('alias')
@@ -437,6 +423,8 @@ yargs
       .default('exclude', [])
     , args => {
 
+      spinner = ora(`Initializing test session ...`).start();
+
       // Load and run all tests, saving the results to disk
       const session = new TestSession();
       session.scanForTests(args as LoadTestsOptions);
@@ -450,28 +438,56 @@ yargs
       }
     }
   )
-  .command('compare [snapshot-a] [snapshot-b]', 'Compare the output of two given tests',
+
+  .command('compare [expected] [actual]', 'Compare the output of two given tests',
     yargs => yargs
     , args => {
-      const keyA = findSnapshot(args['snapshot-a'] as string);
+
+      spinner = ora(`Initializing test session ...`).start();
+
+      let expectedSessionKey;
+      let actualSessionKey;
+
+      if (args.expected !== undefined) {
+        expectedSessionKey = args.expected;
+      } else {
+        expectedSessionKey = 'lkg';
+      }
+
+      if (args.actual !== undefined) {
+        actualSessionKey = args.actual;
+      } else {
+        // Load and run all tests, saving the results to disk
+        const session = new TestSession();
+        session.scanForTests(args as LoadTestsOptions);
+        session.run();
+        session.save();
+        actualSessionKey = session.key;
+      }
+
+      spinner.info(`Comparing ${actualSessionKey} to ${expectedSessionKey}`)
+
+      const keyA = findSnapshot(expectedSessionKey as string);
       if (keyA === null) {
-        spinner.fail(`A test snapshot named '${keyA}' was not found.`)
+        spinner.fail(`A test snapshot named '${expectedSessionKey}' was not found.`)
         return 1;
       }
-      const keyB = findSnapshot(args['snapshot-b'] as string);
+      const keyB = findSnapshot(actualSessionKey as string);
       if (keyB === null) {
-        spinner.fail(`A test snapshot named '${keyB}' was not found.`)
+        spinner.fail(`A test snapshot named '${actualSessionKey}' was not found.`)
         return 1;
       }
       compare(keyA, keyB);
     }
   )
+
   .command( 'clean', 'Clean up test snapshots that are unused', 
     yargs => yargs
       .array('keep')
       .default('keep', ['lkg'])
       .describe('keep', 'Keep the given aliases and anything they refer to')
     , args => {
+      spinner = ora(`Initializing test session ...`).start();
       const snapshotsToKeep = new Set();
       for (const alias of fs.readdirSync(path.join(STORAGE_DIR, 'aliases'))) {
         if (args.keep.indexOf(alias) !== -1) {
@@ -492,6 +508,7 @@ yargs
       spinner.succeed('Cleanup complete.')
     }
   )
+
   .version()
   .help()
   .argv;

src/test/types.ts

@@ -1,18 +1,4 @@
 
-import { assert } from "chai"
+import { AnyType, UnionType } from "../types"
-import { AnyType, simplifyType, UnionType } from "../types"
 import { createBoltIdentifier } from "../ast";
-import { type } from "os";
 
-describe('a function that merges two equivalent types', () => {
-
-    it('can merge two any types', () =>{
-        const type1 = new AnyType;
-        type1.node = createBoltIdentifier('a');
-        const type2 = new AnyType;
-        type2.node = createBoltIdentifier('b');
-        const types = new UnionType([type1, type2]);
-        mergeTypes(types);
-    })
-
-})

src/text.ts

@@ -1,7 +1,8 @@
 
 import * as path from "path"
 import * as fs from "fs"
-import { serializeTag, serialize, deserializable } from "./util";
+import { serializeTag, serialize, deserializable, inspectTag } from "./util";
+import { InspectOptionsStylized } from "util";
 
 @deserializable()
 export class TextFile {
@@ -16,7 +17,11 @@ export class TextFile {
     return path.resolve(this.origPath)
   }
 
-  [serializeTag]() {
+  private [inspectTag](depth: numbber | null, options: InspectOptionsStylized) {
+    return `TextFile { ${this.origPath} }`
+  }
+
+  private [serializeTag]() {
     return [ this.origPath ];
   }
 

src/util.ts

@@ -6,7 +6,6 @@ import * as os from "os"
 import moment from "moment"
 import chalk from "chalk"
 import { LOG_DATETIME_FORMAT } from "./constants"
-import { NODE_TYPES } from "./ast"
 
 export function isPowerOf(x: number, n: number):boolean {
   const a = Math.log(x) / Math.log(n);
@@ -518,6 +517,24 @@ export class StreamWrapper<T> {
 
 }
 
+export const inspectTag = require('util').inspect.custom;
+
+export function indent(text: string, indentation = '  ', afterNewLine = true) {
+  let out = ''
+  for (const ch of text) {
+    if (ch === '\n') {
+      afterNewLine = true;
+      out += ch;
+    } else if (afterNewLine) {
+      out += indentation + ch;
+      afterNewLine = false;
+    } else {
+      out += ch;
+    }
+  }
+  return out;
+}
+
 export function expandPath(filepath: string) {
   let out = ''
   for (const ch of filepath) {
@@ -684,7 +701,6 @@ export function format(message: string, data: MapLike<FormatArg>) {
 
 }
 
-
 export function deepEqual(a: any, b: any): boolean {
   if (isPrimitive(a) && isPrimitive(b)) {
     return a === b;
@@ -710,4 +726,3 @@ export function deepEqual(a: any, b: any): boolean {
   }
   return false;
 }
-