samvv/bolt
1
0
Fork 0
Code Issues 24 Pull requests Projects Releases Packages Wiki Activity Actions

Make test infrasture more usable by adding a diff reporter

Browse source
This commit is contained in:
Sam Vervaeck 2020-06-12 16:38:35 +02:00
parent 7368643180
commit 8c92f689cd
8 changed files with 612 additions and 661 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

906
package-lock.json generated
View file

File diff suppressed because it is too large Load diff

14
package.json
View file

@ -10,14 +10,16 @@
"scripts": {
"watch": "webpack --watch --config webpack.dev.js",
"prepare": "webpack --config webpack.dev.js",
"test": "node build/bin/bolt-test.js compare",
"test": "node lib/bin/bolt-test.js compare",
"generate-ast": "tsastgen src/ast-spec.ts:src/ast.ts",
"update-lkg": "node build/bin/bolt-test.js create-snapshot lkg"
"update-lkg": "node lib/bin/bolt-test.js create-snapshot lkg"
},
"author": "Sam Vervaeck <vervaeck.sam@skynet.be>",
"license": "GPL-3.0",
"repository": "https://github.com/samvv/Bolt",
"dependencies": {
"@types/commonmark": "^0.27.4",
"@types/diff": "^4.0.2",
"@types/fs-extra": "^9.0.1",
"@types/glob": "^7.1.2",
"@types/js-yaml": "^3.12.4",
@ -29,6 +31,8 @@
"@types/uuid": "^8.0.0",
"@types/yargs": "^15.0.5",
"chalk": "^4.1.0",
"commonmark": "^0.29.1",
"diff": "^4.0.2",
"fs-extra": "^9.0.1",
"glob": "^7.1.6",
"js-yaml": "^3.14.0",
@ -47,11 +51,11 @@
"@babel/core": "^7.10.2",
"@babel/plugin-proposal-class-properties": "^7.10.1",
"@types/chai": "^4.2.11",
"@types/mocha": "^7.0.2",
"@types/tape": "^4.13.0",
"babel-loader": "^8.1.0",
"chai": "^4.2.0",
"concurrently": "^5.2.0",
"mocha": "^8.0.1",
"tap-min": "^2.0.0",
"tape": "^5.0.1",
"ts-loader": "^7.0.5",
"typescript": "^3.9.5",
"webpack": "^4.43.0",

25
src/ast-spec.ts
View file

@ -4,7 +4,8 @@ import { TextSpan } from "./text"
import { Value } from "./evaluator"
import { Package } from "./package"
import { Diagnostic } from "./diagnostics";
import { serializeTag, serialize } from "./util";
import { MapLike, serializeTag, inspectTag, indent } from "./util";
import { InspectOptions, InspectOptionsStylized, inspect } from "util";
let nextNodeId = 1;
@ -30,10 +31,30 @@ export abstract class Syntax {
this.id = nextNodeId++;
}
protected [inspectTag](depth: number | null, options: InspectOptionsStylized) {
const proto = Object.getPrototypeOf(this);
if (depth !== null && depth < 0) {
return options.stylize(`[${proto.constructor.name}]`, 'special')
}
const newOptions = {
...options,
depth: options.depth === null ? null : options.depth!-1,
}
let out = `${proto.constructor.name} {\n`;
for (const key of Object.keys(this)) {
if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
continue;
}
out += `${key}: ${inspect((this as any)[key], newOptions)},\n`;
}
out += '}\n';
return out;
}
[serializeTag]() {
const result: any[] = [];
for (const key of Object.keys(this)) {
if (key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
continue;
}
result.push((this as any)[key]);

25
src/ast.ts
View file

@ -8,7 +8,9 @@ import { Package } from "./package";
import { Diagnostic } from "./diagnostics";
import { serializeTag, serialize } from "./util";
import { MapLike, serializeTag, inspectTag, indent } from "./util";
import { InspectOptions, InspectOptionsStylized, inspect } from "util";
let nextNodeId = 1;
@ -26,10 +28,29 @@ export abstract class SyntaxBase {
constructor(public span: TextSpan | null = null) {
this.id = nextNodeId++;
}
protected [inspectTag](depth: number | null, options: InspectOptionsStylized) {
const proto = Object.getPrototypeOf(this);
if (depth !== null && depth < 0) {
return options.stylize(`[${proto.constructor.name}]`, 'special');
}
const newOptions = {
...options,
depth: options.depth === null ? null : options.depth! - 1,
};
let out = `${proto.constructor.name} {\n`;
for (const key of Object.keys(this)) {
if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
continue;
}
out += `${key}: ${inspect((this as any)[key], newOptions)},\n`;
}
out += '}\n';
return out;
}
[serializeTag]() {
const result: any[] = [];
for (const key of Object.keys(this)) {
if (key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
if (key === 'kind' || key === 'parentNode' || key === 'errors' || key === 'type' || key === 'id') {
continue;
}
result.push((this as any)[key]);

255
src/bin/bolt-test.ts
View file

@ -1,4 +1,7 @@
// NOTE The code in this file is not as clean as we want it to be, but we'll be upgrading our
// test infrastructure anyways with version 1.0.0 so it does not matter much.
import "source-map-support/register"
import "reflect-metadata"
@ -6,24 +9,27 @@ import * as fs from "fs-extra"
import * as path from "path"
import * as crypto from "crypto"
import chalk from "chalk"
import { v4 as uuidv4 } from "uuid"
import yargs from "yargs"
import yaml, { FAILSAFE_SCHEMA } from "js-yaml"
import { sync as globSync } from "glob"
import ora from "ora"
import ora, { Ora } from "ora"
import { Parser as CommonmarkParser } from "commonmark"
import { Parser } from "../parser"
import { Scanner } from "../scanner"
import { SyntaxKind, Syntax } from "../ast"
import { Json, serialize, JsonObject, MapLike, upsearchSync, deepEqual, serializeTag, deserializable, deserialize } from "../util"
import { DiagnosticIndex, DiagnosticPrinter, E_TESTS_DO_NOT_COMPARE, E_INVALID_TEST_COMPARE } from "../diagnostics"
import { Json, serialize, JsonObject, MapLike, upsearchSync, deepEqual, serializeTag, deserializable, deserialize, JsonArray, verbose, diffpatcher } from "../util"
import { DiagnosticIndex, DiagnosticPrinter, E_TESTS_DO_NOT_COMPARE, E_INVALID_TEST_COMPARE, E_NO_BOLTFILE_FOUND_IN_PATH_OR_PARENT_DIRS, Diagnostic } from "../diagnostics"
import { TextFile, TextPos, TextSpan } from "../text"
import { diffLines } from "diff"
import { inspect } from "util"
const PACKAGE_ROOT = path.dirname(upsearchSync('package.json')!);
const STORAGE_DIR = path.join(PACKAGE_ROOT, '.test-storage');
const diagnostics = new DiagnosticPrinter();
const spinner = ora(`Initializing test session ...`).start();
let spinner: Ora;
// TODO move some logic from TestSession to TestSuite
// TODO hash the entire code base and have it serve as a unique key for TestSession
@ -40,7 +46,7 @@ class Test {
public key: string;
public result?: Json;
public result?: any;
public error: Error | null = null;
constructor(
@ -148,20 +154,40 @@ class TestSession {
}
function toString(value: any): string {
return inspect(value, {
colors: false,
depth: Infinity,
})
}
function compare(actualKey: string, expectedKey: string) {
for (const testKey of fs.readdirSync(path.join(STORAGE_DIR, 'snapshots', actualKey))) {
const test = deserialize(readJson(path.join(STORAGE_DIR, 'tests', testKey)))
const actualTestData = deserialize(readJson(path.join(STORAGE_DIR, 'snapshots', actualKey, testKey))!);
const expectedTestData = deserialize(readJson(path.join(STORAGE_DIR, 'snapshots', expectedKey, testKey)));
if (!deepEqual(actualTestData.result, expectedTestData.result)) {
const actualData = readJson(path.join(STORAGE_DIR, 'snapshots', actualKey, testKey))!;
const expectedData = readJson(path.join(STORAGE_DIR, 'snapshots', expectedKey, testKey))!;
const actual = deserialize(actualData);
const expected = deserialize(expectedData);
const diffs = diffLines(toString(actual), toString(expected));
if (diffs.some(diff => diff.added || diff.removed)) {
diagnostics.add({
message: E_TESTS_DO_NOT_COMPARE,
severity: 'error',
node: test,
})
});
for (const diff of diffs) {
let out = diff.value;
if (diff.removed) {
out = chalk.red(out);
} else if (diff.added) {
out = chalk.green(out);
}
process.stderr.write(out);
}
//lconsole.error(jsondiffpatch.formatters.console.format(delta, expected) + '\n');
}
}
@ -177,115 +203,60 @@ interface TestFileMetadata {
expect: string;
}
function* loadTests(filepath: string): IterableIterator<Test> {
interface SimpleToken {
text: string;
startPos: TextPos;
endPos: TextPos;
}
const file = new TextFile(filepath);
const contents = file.getText('utf8');
const PREAMBLE_START = '---\n';
const PREAMBLE_END = '---\n';
let i = 0;
let column = 1
let line = 1;
let atNewLine = true;
assertText('---');
let yamlStr = '';
i += 3;
while (!lookaheadEquals('---')) {
yamlStr += contents[i++];
function getPreamble(text: string): string {
if (!text.startsWith(PREAMBLE_START)) {
return '';
}
i += 3;
const metadata = yaml.safeLoad(yamlStr);
while (i < contents.length) {
skipWhiteSpace();
if (atNewLine && column >= 5) {
const startPos = new TextPos(i, line, column);
const text = scanCodeBlock();
const endPos = new TextPos(i, line, column);
if (metadata['split-lines']) {
for (const line of text.split('\n')) {
if (line.trim() !== '') {
yield new Test(new TextSpan(file, startPos.clone(), endPos), metadata.type, line, metadata);
startPos.advance(line);
}
}
} else {
yield new Test(new TextSpan(file, startPos, endPos), metadata.type, text, metadata);
}
} else {
getChar();
}
}
function getChar() {
const ch = contents[i++];
if (ch === '\n') {
column = 1;
line++;
atNewLine = true;
} else {
if (!isEmpty(ch)) {
atNewLine = false;
}
column++;
}
return ch;
}
function assertText(str: string) {
for (let k = 0; k < str.length; k++) {
if (contents[i+k] !== str[k]) {
throw new Error(`Expected '${str}' but got ${contents.substr(i, i+str.length)}`)
}
}
}
function isEmpty(ch: string): boolean {
return /[\t ]/.test(ch);
}
function lookaheadEquals(str: string): boolean {
for (let k = 0; k < str.length; k++) {
if (contents[i+k] !== str[k]) {
return false;
}
}
return true;
}
function scanCodeBlock() {
let out = ''
while (i < contents.length) {
const ch = getChar();
if (ch === '\n') {
out += ch;
skipWhiteSpace();
continue;
}
if (column < 5) {
let out = '';
for (let i = PREAMBLE_START.length; i < text.length; i++) {
if (text.startsWith(PREAMBLE_END, i)) {
break;
}
out += ch;
out += text[i];
}
return out;
}
}
function skipWhiteSpace() {
takeWhile(isWhiteSpace)
function* loadTests(filepath: string): IterableIterator<Test> {
const file = new TextFile(filepath);
const contents = file.getText('utf8');
const preamble = getPreamble(contents);
const metadata = yaml.safeLoad(preamble);
const parser = new CommonmarkParser();
const rootNode = parser.parse(contents);
if (rootNode.firstChild === null) {
return;
}
function takeWhile(pred: (ch: string) => boolean) {
let out = '';
while (true) {
const c0 = contents[i];
if (!pred(c0)) {
break
for (let node = rootNode.firstChild; node.next !== null; node = node.next) {
if (node.type === 'code_block') {
if (metadata['split-lines']) {
let startPos = new TextPos(0, node.sourcepos[0][0], node.sourcepos[0][1]);
startPos.advance('```')
startPos.advance(node.info! + '\n')
let endPos = startPos.clone();
for (const line of node.literal!.split('\n')) {
if (line.length > 0) {
yield new Test(new TextSpan(file, startPos.clone(), endPos.clone()), metadata.type, line, metadata);
startPos = endPos;
}
endPos.advance(line + '\n');
}
} else {
const startPos = new TextPos(0, node.sourcepos[0][0], node.sourcepos[0][1]);
const endPos = new TextPos(0, node.sourcepos[1][0], node.sourcepos[1][1]);
yield new Test(new TextSpan(file, startPos, endPos), metadata.type, node.literal!, metadata);
}
out += getChar();
}
return out;
}
}
function findSnapshot(ref: string): string | null {
@ -344,9 +315,9 @@ type TestRunner = (test: Test) => Json;
const TEST_RUNNERS: MapLike<TestRunner> = {
scan(test: Test): Json {
scan(test: Test): any {
const diagnostics = new DiagnosticIndex;
const scanner = new Scanner(test.span.file, test.text, test.span.start);
const scanner = new Scanner(test.span.file, test.text, test.span.start.clone());
const tokens = []
while (true) {
const token = scanner.scan();
@ -355,13 +326,13 @@ const TEST_RUNNERS: MapLike<TestRunner> = {
}
tokens.push(token);
}
return serialize({
return {
diagnostics: [...diagnostics.getAllDiagnostics()],
tokens,
});
};
},
parse(test: Test): Json {
parse(test: Test): any {
const kind = test.data.expect ?? 'SourceFile';
const diagnostics = new DiagnosticIndex;
const parser = new Parser();
@ -377,15 +348,27 @@ const TEST_RUNNERS: MapLike<TestRunner> = {
default:
throw new Error(`I did not know how to parse ${kind}`)
}
return serialize({
return {
diagnostics: [...diagnostics.getAllDiagnostics()],
results,
})
};
},
}
const TEST_REPORTERS = {
scan(test: Test) {
const printer = new DiagnosticPrinter();
for (const diagnostic of test.result!.diagnostics) {
printer.add(diagnostic as Diagnostic);
}
}
}
yargs
.command(['$0 [pattern..]', 'run [pattern..]'], 'Run all tests on the current version of the compiler',
yargs => yargs
.array('pattern')
@ -401,6 +384,8 @@ yargs
.default('alias', [])
, args => {
spinner = ora(`Initializing test session ...`).start();
const session = new TestSession();
session.scanForTests(args as LoadTestsOptions);
session.run();
@ -424,6 +409,7 @@ yargs
}
)
.command(['create-snapshot [alias..]'], 'Create a new snapshot from the output of the current compiler',
yargs => yargs
.array('alias')
@ -437,6 +423,8 @@ yargs
.default('exclude', [])
, args => {
spinner = ora(`Initializing test session ...`).start();
// Load and run all tests, saving the results to disk
const session = new TestSession();
session.scanForTests(args as LoadTestsOptions);
@ -450,28 +438,56 @@ yargs
}
}
)
.command('compare [snapshot-a] [snapshot-b]', 'Compare the output of two given tests',
.command('compare [expected] [actual]', 'Compare the output of two given tests',
yargs => yargs
, args => {
const keyA = findSnapshot(args['snapshot-a'] as string);
spinner = ora(`Initializing test session ...`).start();
let expectedSessionKey;
let actualSessionKey;
if (args.expected !== undefined) {
expectedSessionKey = args.expected;
} else {
expectedSessionKey = 'lkg';
}
if (args.actual !== undefined) {
actualSessionKey = args.actual;
} else {
// Load and run all tests, saving the results to disk
const session = new TestSession();
session.scanForTests(args as LoadTestsOptions);
session.run();
session.save();
actualSessionKey = session.key;
}
spinner.info(`Comparing ${actualSessionKey} to ${expectedSessionKey}`)
const keyA = findSnapshot(expectedSessionKey as string);
if (keyA === null) {
spinner.fail(`A test snapshot named '${keyA}' was not found.`)
spinner.fail(`A test snapshot named '${expectedSessionKey}' was not found.`)
return 1;
}
const keyB = findSnapshot(args['snapshot-b'] as string);
const keyB = findSnapshot(actualSessionKey as string);
if (keyB === null) {
spinner.fail(`A test snapshot named '${keyB}' was not found.`)
spinner.fail(`A test snapshot named '${actualSessionKey}' was not found.`)
return 1;
}
compare(keyA, keyB);
}
)
.command( 'clean', 'Clean up test snapshots that are unused',
yargs => yargs
.array('keep')
.default('keep', ['lkg'])
.describe('keep', 'Keep the given aliases and anything they refer to')
, args => {
spinner = ora(`Initializing test session ...`).start();
const snapshotsToKeep = new Set();
for (const alias of fs.readdirSync(path.join(STORAGE_DIR, 'aliases'))) {
if (args.keep.indexOf(alias) !== -1) {
@ -492,6 +508,7 @@ yargs
spinner.succeed('Cleanup complete.')
}
)
.version()
.help()
.argv;

16
src/test/types.ts
View file

@ -1,18 +1,4 @@
import { assert } from "chai"
import { AnyType, simplifyType, UnionType } from "../types"
import { AnyType, UnionType } from "../types"
import { createBoltIdentifier } from "../ast";
import { type } from "os";
describe('a function that merges two equivalent types', () => {
it('can merge two any types', () =>{
const type1 = new AnyType;
type1.node = createBoltIdentifier('a');
const type2 = new AnyType;
type2.node = createBoltIdentifier('b');
const types = new UnionType([type1, type2]);
mergeTypes(types);
})
})

9
src/text.ts
View file

@ -1,7 +1,8 @@
import * as path from "path"
import * as fs from "fs"
import { serializeTag, serialize, deserializable } from "./util";
import { serializeTag, serialize, deserializable, inspectTag } from "./util";
import { InspectOptionsStylized } from "util";
@deserializable()
export class TextFile {
@ -16,7 +17,11 @@ export class TextFile {
return path.resolve(this.origPath)
}
[serializeTag]() {
private [inspectTag](depth: numbber | null, options: InspectOptionsStylized) {
return `TextFile { ${this.origPath} }`
}
private [serializeTag]() {
return [ this.origPath ];
}

21
src/util.ts
View file

@ -6,7 +6,6 @@ import * as os from "os"
import moment from "moment"
import chalk from "chalk"
import { LOG_DATETIME_FORMAT } from "./constants"
import { NODE_TYPES } from "./ast"
export function isPowerOf(x: number, n: number):boolean {
const a = Math.log(x) / Math.log(n);
@ -518,6 +517,24 @@ export class StreamWrapper<T> {
}
export const inspectTag = require('util').inspect.custom;
export function indent(text: string, indentation = ' ', afterNewLine = true) {
let out = ''
for (const ch of text) {
if (ch === '\n') {
afterNewLine = true;
out += ch;
} else if (afterNewLine) {
out += indentation + ch;
afterNewLine = false;
} else {
out += ch;
}
}
return out;
}
export function expandPath(filepath: string) {
let out = ''
for (const ch of filepath) {
@ -684,7 +701,6 @@ export function format(message: string, data: MapLike<FormatArg>) {
}
export function deepEqual(a: any, b: any): boolean {
if (isPrimitive(a) && isPrimitive(b)) {
return a === b;
@ -710,4 +726,3 @@ export function deepEqual(a: any, b: any): boolean {
}
return false;
}