From f9ea7ecb4d00851963ec9124c8e8c679417ed4c9 Mon Sep 17 00:00:00 2001 From: Sam Vervaeck Date: Sat, 9 May 2020 23:41:42 +0200 Subject: [PATCH] treegen: Create a small emitter for AST metadata --- treegen/src/ast-generated.js | 179 +++++++++++++++++++++++++++++++++++ treegen/src/ast.ts | 1 + treegen/src/bin/treegen.ts | 108 +++++++++++++++++++-- treegen/src/index.ts | 0 treegen/src/parser.pegjs | 11 ++- 5 files changed, 289 insertions(+), 10 deletions(-) create mode 100644 treegen/src/ast-generated.js create mode 100644 treegen/src/index.ts diff --git a/treegen/src/ast-generated.js b/treegen/src/ast-generated.js new file mode 100644 index 000000000..2f64f712d --- /dev/null +++ b/treegen/src/ast-generated.js @@ -0,0 +1,179 @@ + +const NODE_TYPES = [ + 'StringLiteral': new Map([ + [value, string], + ]), + 'IntegerLiteral': new Map([ + [value, bigint], + ]), + 'Identifier': new Map([ + [text, string], + ]), + 'Operator': new Map([ + [text, string], + ]), + 'EOS': new Map([ + ]), + 'Comma': new Map([ + ]), + 'Semi': new Map([ + ]), + 'Dot': new Map([ + ]), + 'DotDot': new Map([ + ]), + 'RArrow': new Map([ + ]), + 'LArrow': new Map([ + ]), + 'EqSign': new Map([ + ]), + 'FnKeyword': new Map([ + ]), + 'ForeignKeyword': new Map([ + ]), + 'LetKeyword': new Map([ + ]), + 'ImportKeyword': new Map([ + ]), + 'PubKeyword': new Map([ + ]), + 'ModKeyword': new Map([ + ]), + 'EnumKeyword': new Map([ + ]), + 'StructKeyword': new Map([ + ]), + 'NewTypeKeyword': new Map([ + ]), + 'Parenthesized': new Map([ + [text, string], + ]), + 'Braced': new Map([ + [text, string], + ]), + 'Bracketed': new Map([ + [text, string], + ]), + 'SourceElement': new Map([ + ]), + 'SourceFile': new Map([ + [elements, SourceElement[]], + ]), + 'QualName': new Map([ + [modulePath, Identifier[]], + [name, Symbol], + ]), + 'TypeReference': new Map([ + [name, QualName], + [arguments, TypeNode[] | null], + ]), + 'BindPattern': new Map([ + [name, string], + ]), + 'TypePattern': new Map([ + [typeNode, TypeNode], + [nestedPattern, Pattern], + ]), + 'ExpressionPattern': new Map([ + [expression, Expression], + ]), + 'TuplePatternElement': new Map([ + [index, bigint], + [pattern, Pattern], + ]), + 'TuplePattern': new Map([ + [elements, TuplePatternElement[]], + ]), + 'RecordPatternField': new Map([ + [name, Identifier], + [pattern, Pattern], + ]), + 'RecordPattern': new Map([ + [fields, RecordPatternField[]], + ]), + 'CallExpression': new Map([ + [operator, Expression], + [operands, Expression[]], + ]), + 'YieldExpression': new Map([ + [value, Expression], + ]), + 'MatchArm': new Map([ + [pattern, Pattern], + [body, Expression], + ]), + 'MatchExpression': new Map([ + [value, Expression], + [arms, MatchArm[]], + ]), + 'Case': new Map([ + [test, Expression], + [result, Expression], + ]), + 'CaseExpression': new Map([ + [cases, Case[]], + ]), + 'BlockExpression': new Map([ + [statements, Statement[]], + ]), + 'ConstantExpression': new Map([ + [value, Value], + ]), + 'ReturnStatement': new Map([ + [value, Expression | null], + ]), + 'ResumeStatement': new Map([ + [value, Expression], + ]), + 'ExpressionStatement': new Map([ + [expression, Expression], + ]), + 'Module': new Map([ + [modifiers, Modifiers], + [name, QualName], + [elements, SourceElement], + ]), + 'Parameter': new Map([ + [index, bigint], + [bindings, Pattern], + [typeNode, TypeNode | null], + [defaultValue, Expression | null], + ]), + 'FunctionDeclaration': new Map([ + [modifiers, Modifiers], + [name, Symbol], + [params, Parameter[]], + [type, TypeNode | null], + [body, Expression], + ]), + 'ForeignFunctionDeclaration': new Map([ + [modifiers, Modifiers], + [name, Symbol], + [params, Parameter[]], + [type, TypeNode | null], + [body, Statement[]], + ]), + 'VariableDeclaration': new Map([ + [modifiers, Modifiers], + [name, Symbol], + [type, TypeNode | null], + [value, Expression | null], + ]), + 'PlainImportSymbol': new Map([ + [name, QualName], + ]), + 'ImportDeclaration': new Map([ + [file, string], + [symbols, ImportSymbol[]], + ]), + 'RecordDeclarationField': new Map([ + [name, Identifier], + [type, TypeNode], + ]), + 'RecordDeclaration': new Map([ + [name, QualName], + [fields, RecordDeclarationField[]], + ]), +]; + diff --git a/treegen/src/ast.ts b/treegen/src/ast.ts index 5ca183ac8..910377539 100644 --- a/treegen/src/ast.ts +++ b/treegen/src/ast.ts @@ -22,6 +22,7 @@ export interface NodeField { export interface NodeDeclaration { type: 'NodeDeclaration'; name: string; + parents: string[]; fields: NodeField[]; } diff --git a/treegen/src/bin/treegen.ts b/treegen/src/bin/treegen.ts index d9711bcf2..9e6a3aecc 100755 --- a/treegen/src/bin/treegen.ts +++ b/treegen/src/bin/treegen.ts @@ -4,7 +4,8 @@ import * as path from "path" import * as fs from "fs" import { parse, SyntaxError } from "../parser" -import { Declaration } from "../ast" +import { Declaration, NodeDeclaration, TypeDeclaration, EnumDeclaration, TypeNode, NodeField } from "../ast" +import { FileWriter } from "../util" for (const filename of process.argv.slice(2)) { const contents = fs.readFileSync(filename, 'utf8'); @@ -19,22 +20,117 @@ for (const filename of process.argv.slice(2)) { throw e; } } - console.error(jsonify(decls)); - const generated = generateAST(decls); - fs.writeFileSync(getFileStem(filename).toLowerCase(), generated, 'utf8'); + const generated = generateAST(decls, getFileStem(filename)); + fs.writeFileSync('src/ast-generated.js', generated, 'utf8'); } interface FastStringMap { [key: string]: T } -function generateAST(decls: Declaration[]) { +function generateAST(decls: Declaration[], langName: string) { + + let jsFile = new FileWriter(); + + const nodeDecls: NodeDeclaration[] = decls.filter(decl => decl.type === 'NodeDeclaration') as NodeDeclaration[]; + const typeDecls: TypeDeclaration[] = decls.filter(decl => decl.type === 'TypeDeclaration') as TypeDeclaration[]; + const enumDecls: EnumDeclaration[] = decls.filter(decl => decl.type === 'EnumDeclaration') as EnumDeclaration[]; const declByName: FastStringMap = Object.create(null); for (const decl of decls) { declByName[decl.name] = decl; } - return '' + // Generate a mapping from parent node to child node + // This makes it easy to generate union types for the intermediate nodes. + const childrenOf: FastStringMap = Object.create(null); + for (const nodeDecl of nodeDecls) { + for (const parentName of nodeDecl.parents) { + if (childrenOf[parentName] === undefined) { + childrenOf[parentName] = []; + } + childrenOf[parentName].push(nodeDecl.name); + } + } + + // Write a JavaScript file that contains all AST definitions. + + jsFile.write(`\nconst NODE_TYPES = [\n`); + jsFile.indent(); + for (const decl of decls) { + if (decl.type === 'NodeDeclaration' && isLeafNode(decl.name)) { + jsFile.write(`'${decl.name}': new Map([\n`); + jsFile.indent(); + for (const field of getAllFields(decl)) { + jsFile.write(`[${field.name}, ${emitTypeNode(field.typeNode)}],\n`); + } + jsFile.dedent(); + jsFile.write(']),\n'); + } + } + jsFile.dedent(); + jsFile.write('];\n\n'); + + function emitTypeNode(typeNode: TypeNode): string { + console.error(typeNode); + if (typeNode.type === 'ReferenceTypeNode') { + if (hasDeclarationNamed(typeNode.name)) { + return typeNode.name; + } else if (typeNode.name === 'Bool') { + return 'boolean'; + } else if (typeNode.name === 'String') { + return 'string'; + } else if (typeNode.name === 'usize') { + return 'bigint'; + } else if (typeNode.name === 'Vec') { + return `${emitTypeNode(typeNode.typeArgs[0])}[]`; + } else if (typeNode.name === 'Option') { + return `${emitTypeNode(typeNode.typeArgs[0])} | null`; + } + } + throw new Error(`Could not emit TypeScript type for type node ${typeNode.type}.`); + } + + function hasDeclarationNamed(name: string): boolean { + return name in declByName; + } + + function getAllFields(nodeDecl: NodeDeclaration) { + let out: NodeField[] = []; + pushAll(out, nodeDecl.fields); + for (const parentName of nodeDecl.parents) { + const parentDecl = getDeclarationNamed(parentName); + if (parentDecl.type !== 'NodeDeclaration') { + throw new Error(`Parent declaration '${parentName}' of '${nodeDecl.name}' must be a node declaration.`); + } + pushAll(out, getAllFields(parentDecl)); + } + return out; + } + + function getDeclarationNamed(name: string): Declaration { + const decl = declByName[name]; + if (decl === undefined) { + throw new Error(`Declaration '${name}' was not found in any of the definition files.`); + } + return decl; + } + + function isLeafNode(name: string): boolean { + const decl = getDeclarationNamed(name); + if (decl.type !== 'NodeDeclaration') { + throw new Error(`Declaration '${name}' is not a node declaration.`) + } + return childrenOf[name] === undefined || childrenOf[name].length === 0; + } + + return jsFile.currentText; + +} + +function pushAll(arr: T[], els: T[]): void { + for (const el of els) { + arr.push(el); + } } function jsonify(value: any) { diff --git a/treegen/src/index.ts b/treegen/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/treegen/src/parser.pegjs b/treegen/src/parser.pegjs index cb5d19afd..ec481a2da 100644 --- a/treegen/src/parser.pegjs +++ b/treegen/src/parser.pegjs @@ -28,7 +28,7 @@ Declaration / TypeDeclaration NodeDeclaration - = NodeToken __ name:Identifier parents:(__ '>' __ ExtendsList)? fields:(__ '{' __ (@NodeField __)* '}')? EOS { + = NodeToken __ name:Identifier parents:(__ '>' __ @ExtendsList)? fields:(__ '{' __ @(@NodeField __)* '}')? EOS { return createNode('NodeDeclaration', { name, parents: liftArray(parents), fields: liftArray(fields) }); } @@ -47,12 +47,15 @@ TypeNode UnionTypeNode = head:ReferenceTypeNode tail:(__ '|' __ @ReferenceTypeNode)* { - return [head, ...tail]; + if (tail.length === 0) { + return head; + } + return createNode('UnionTypeNode', { elements: [head, ...tail] }); } ReferenceTypeNode - = name:Identifier genericArgs:(__ '<' __ @TypeNodeList __ '>')? { - return createNode('ReferenceTypeNode', { name, genericArgs }); + = name:Identifier typeArgs:(__ '<' __ @TypeNodeList __ '>')? { + return createNode('ReferenceTypeNode', { name, typeArgs }); } TypeNodeList