import "source-map-support/register" import * as fs from "fs-extra" import * as path from "path" import * as crypto from "crypto" import yargs from "yargs" import yaml from "js-yaml" import { sync as globSync } from "glob" import ora from "ora" import { Parser } from "../parser" import { Scanner } from "../scanner" import { SyntaxKind, Syntax } from "../ast" import { Json, serialize, JsonObject, MapLike, upsearchSync, deepEqual } from "../util" import { DiagnosticIndex } from "../diagnostics" import { TextFile, TextPos, TextSpan } from "../text" const PACKAGE_ROOT = path.dirname(upsearchSync('package.json')!); const STORAGE_DIR = path.join(PACKAGE_ROOT, '.test-storage'); const spinner = ora(`Initializing test session ...`).start(); function toArray(value: T | T[]): T[] { if (Array.isArray(value)) { return value; } return value === undefined || value === null ? [] : [ value ] } class Test { public key: string; public result?: Json; public error: Error | null = null; constructor( public readonly span: TextSpan, public readonly type: string, public readonly text: string, public readonly data: JsonObject, ) { this.key = hash([text, data]); } } interface LoadTestsOptions { include: string[]; exclude: string[]; } class TestSession { private failCount = 0; public key: string; constructor(private tests: Test[] = []) { this.key = `${Date.now().toString()}-${Math.random()}`; } public getAllTests() { return this.tests[Symbol.iterator]();; } public scanForTests(options?: LoadTestsOptions) { const includes = options?.include ?? ['test/**/*.md']; const excludes = options?.exclude ?? []; spinner.text = 'Scanning for tests [0 found]'; for (const include of includes) { for (const filepath of globSync(include, { ignore: excludes })) { spinner.info(`Found file ${filepath}`) for (const test of loadTests(filepath)) { this.tests.push(test); spinner.text = `Scanning for tests [${this.tests.length} found]`; } } } } public run() { let i = 1; //let failed = []; for (const test of this.tests) { spinner.text = `Running tests [${i}/${this.tests.length}]` const runner = TEST_RUNNERS[test.type] if (runner === undefined) { spinner.warn(`Test runner '${test.type}' not found.`) continue; } let result; try { test.result = runner(test); } catch (e) { test.error = e; this.failCount++; //failed.push(test); spinner.warn(`The following test from ${path.relative(process.cwd(), test.span.file.fullPath)} failed with "${e.message}":\n\n${test.text}\n`) } i++; } if (this.failCount > 0) { spinner.fail(`${this.failCount} tests failed.`) } } public save() { fs.mkdirpSync(path.join(STORAGE_DIR, 'snapshots', this.key)) for (const test of this.tests) { fs.writeFileSync(path.join(STORAGE_DIR, 'snapshots', this.key, test.key), JSON.stringify(test.result), 'utf8'); } } public hasFailedTests() { return this.failCount > 0; } } function compare(actual: string, expected: string) { for (const testKey of fs.readdirSync(path.join(STORAGE_DIR, 'snapshots', actual))) { const actualTestResult = readJson(path.join(STORAGE_DIR, 'snapshots', actual, testKey))!; const expectedTestResult = readJson(path.join(STORAGE_DIR, 'snapshots', expected, testKey)); if (!deepEqual(actualTestResult, expectedTestResult)) { spinner.warn(`Test ${testKey} does not compare to its expected value.`) } } } function isWhiteSpace(ch: string): boolean { return /[\n\r\t ]/.test(ch); } interface TestFileMetadata { type: string; expect: string; } function* loadTests(filepath: string): IterableIterator { const file = new TextFile(filepath); const contents = file.getText('utf8'); let i = 0; let column = 1 let line = 1; let atNewLine = true; assertText('---'); let yamlStr = ''; i += 3; while (!lookaheadEquals('---')) { yamlStr += contents[i++]; } i += 3; const metadata = yaml.safeLoad(yamlStr); while (i < contents.length) { skipWhiteSpace(); if (atNewLine && column >= 5) { const startPos = new TextPos(i, line, column); const text = scanCodeBlock(); const endPos = new TextPos(i, line, column); if (metadata['split-lines']) { for (const line of text.split('\n')) { if (line.trim() !== '') { yield new Test(new TextSpan(file, startPos.clone(), endPos), metadata.type, line, metadata); startPos.advance(line); } } } else { yield new Test(new TextSpan(file, startPos, endPos), metadata.type, text, metadata); } } else { getChar(); } } function getChar() { const ch = contents[i++]; if (ch === '\n') { column = 1; line++; atNewLine = true; } else { if (!isEmpty(ch)) { atNewLine = false; } column++; } return ch; } function assertText(str: string) { for (let k = 0; k < str.length; k++) { if (contents[i+k] !== str[k]) { throw new Error(`Expected '${str}' but got ${contents.substr(i, i+str.length)}`) } } } function isEmpty(ch: string): boolean { return /[\t ]/.test(ch); } function lookaheadEquals(str: string): boolean { for (let k = 0; k < str.length; k++) { if (contents[i+k] !== str[k]) { return false; } } return true; } function scanCodeBlock() { let out = '' while (i < contents.length) { const ch = getChar(); if (ch === '\n') { out += ch; skipWhiteSpace(); continue; } if (column < 5) { break; } out += ch; } return out; } function skipWhiteSpace() { takeWhile(isWhiteSpace) } function takeWhile(pred: (ch: string) => boolean) { let out = ''; while (true) { const c0 = contents[i]; if (!pred(c0)) { break } out += getChar(); } return out; } } function readJson(filename: string): Json | null { try { return JSON.parse(fs.readFileSync(filename, 'utf8')); } catch (e) { if (e.code === 'ENOENT') { return null } throw e; } } function hash(value: Json) { const hasher = crypto.createHash('sha256'); hasher.update(JSON.stringify(value)); return hasher.digest('hex'); } type TestRunner = (test: Test) => Json; const TEST_RUNNERS: MapLike = { scan(test: Test): Json { const diagnostics = new DiagnosticIndex; const scanner = new Scanner(test.span.file, test.text, test.span.start); const tokens = [] while (true) { const token = scanner.scan(); if (token.kind === SyntaxKind.EndOfFile) { break; } tokens.push(token); } return serialize({ diagnostics: [...diagnostics.getAllDiagnostics()], tokens, }); }, parse(test: Test): Json { const kind = test.data.expect ?? 'SourceFile'; const diagnostics = new DiagnosticIndex; const parser = new Parser(); const tokens = new Scanner(test.span.file, test.text); let results: Syntax[]; switch (kind) { case 'SourceFile': results = [ parser.parseSourceFile(tokens) ]; break; case 'SourceElements': results = parser.parseSourceElements(tokens); break; default: throw new Error(`I did not know how to parse ${kind}`) } return serialize({ diagnostics: [...diagnostics.getAllDiagnostics()], results, }) }, } yargs .command(['$0 [pattern..]', 'run [pattern..]'], 'Run all tests on the current version of the compiler', yargs => yargs .array('pattern') .describe('pattern', 'Only run the tests matching the given pattern') .array('include') .describe('include', 'Files to scan for tests') .default('include', ['test/**/*.md']) .array('exclude') .describe('exclude', 'Files to never scan for tests') .default('exclude', []) , args => { const session = new TestSession(); session.scanForTests(args as LoadTestsOptions); session.run(); session.save(); if (session.hasFailedTests()) { return; } const expectedKey = fs.readFileSync(path.join(STORAGE_DIR, 'aliases', 'lkg'), 'utf8') compare(session.key, expectedKey) } ) .command(['snapshot [name]'], 'Create a new snapshot from the output of the current compiler', yargs => yargs .array('include') .describe('include', 'Files to scan for tests') .default('include', ['test/**/*.md']) .array('exclude') .describe('exclude', 'Files to never scan for tests') .default('exclude', []) , args => { // Load and run all tests, saving the results to disk const session = new TestSession(); session.scanForTests(args as LoadTestsOptions); session.run(); session.save(); // Set the tests that we have run as being the new 'lkg' fs.mkdirpSync(path.join(STORAGE_DIR, 'aliases')); fs.writeFileSync(path.join(STORAGE_DIR, 'aliases', 'lkg'), session.key, 'utf8') } ) .command('inspect ', 'Inspect the ouput of a given test', yargs => yargs , args => { } ) .version() .help() .argv;