treegen: Create a small emitter for AST metadata

This commit is contained in:
Sam Vervaeck 2020-05-09 23:41:42 +02:00
parent ff28963403
commit f9ea7ecb4d
5 changed files with 289 additions and 10 deletions

View file

@ -0,0 +1,179 @@
const NODE_TYPES = [
'StringLiteral': new Map([
[value, string],
]),
'IntegerLiteral': new Map([
[value, bigint],
]),
'Identifier': new Map([
[text, string],
]),
'Operator': new Map([
[text, string],
]),
'EOS': new Map([
]),
'Comma': new Map([
]),
'Semi': new Map([
]),
'Dot': new Map([
]),
'DotDot': new Map([
]),
'RArrow': new Map([
]),
'LArrow': new Map([
]),
'EqSign': new Map([
]),
'FnKeyword': new Map([
]),
'ForeignKeyword': new Map([
]),
'LetKeyword': new Map([
]),
'ImportKeyword': new Map([
]),
'PubKeyword': new Map([
]),
'ModKeyword': new Map([
]),
'EnumKeyword': new Map([
]),
'StructKeyword': new Map([
]),
'NewTypeKeyword': new Map([
]),
'Parenthesized': new Map([
[text, string],
]),
'Braced': new Map([
[text, string],
]),
'Bracketed': new Map([
[text, string],
]),
'SourceElement': new Map([
]),
'SourceFile': new Map([
[elements, SourceElement[]],
]),
'QualName': new Map([
[modulePath, Identifier[]],
[name, Symbol],
]),
'TypeReference': new Map([
[name, QualName],
[arguments, TypeNode[] | null],
]),
'BindPattern': new Map([
[name, string],
]),
'TypePattern': new Map([
[typeNode, TypeNode],
[nestedPattern, Pattern],
]),
'ExpressionPattern': new Map([
[expression, Expression],
]),
'TuplePatternElement': new Map([
[index, bigint],
[pattern, Pattern],
]),
'TuplePattern': new Map([
[elements, TuplePatternElement[]],
]),
'RecordPatternField': new Map([
[name, Identifier],
[pattern, Pattern],
]),
'RecordPattern': new Map([
[fields, RecordPatternField[]],
]),
'CallExpression': new Map([
[operator, Expression],
[operands, Expression[]],
]),
'YieldExpression': new Map([
[value, Expression],
]),
'MatchArm': new Map([
[pattern, Pattern],
[body, Expression],
]),
'MatchExpression': new Map([
[value, Expression],
[arms, MatchArm[]],
]),
'Case': new Map([
[test, Expression],
[result, Expression],
]),
'CaseExpression': new Map([
[cases, Case[]],
]),
'BlockExpression': new Map([
[statements, Statement[]],
]),
'ConstantExpression': new Map([
[value, Value],
]),
'ReturnStatement': new Map([
[value, Expression | null],
]),
'ResumeStatement': new Map([
[value, Expression],
]),
'ExpressionStatement': new Map([
[expression, Expression],
]),
'Module': new Map([
[modifiers, Modifiers],
[name, QualName],
[elements, SourceElement],
]),
'Parameter': new Map([
[index, bigint],
[bindings, Pattern],
[typeNode, TypeNode | null],
[defaultValue, Expression | null],
]),
'FunctionDeclaration': new Map([
[modifiers, Modifiers],
[name, Symbol],
[params, Parameter[]],
[type, TypeNode | null],
[body, Expression],
]),
'ForeignFunctionDeclaration': new Map([
[modifiers, Modifiers],
[name, Symbol],
[params, Parameter[]],
[type, TypeNode | null],
[body, Statement[]],
]),
'VariableDeclaration': new Map([
[modifiers, Modifiers],
[name, Symbol],
[type, TypeNode | null],
[value, Expression | null],
]),
'PlainImportSymbol': new Map([
[name, QualName],
]),
'ImportDeclaration': new Map([
[file, string],
[symbols, ImportSymbol[]],
]),
'RecordDeclarationField': new Map([
[name, Identifier],
[type, TypeNode],
]),
'RecordDeclaration': new Map([
[name, QualName],
[fields, RecordDeclarationField[]],
]),
];

View file

@ -22,6 +22,7 @@ export interface NodeField {
export interface NodeDeclaration { export interface NodeDeclaration {
type: 'NodeDeclaration'; type: 'NodeDeclaration';
name: string; name: string;
parents: string[];
fields: NodeField[]; fields: NodeField[];
} }

View file

@ -4,7 +4,8 @@ import * as path from "path"
import * as fs from "fs" import * as fs from "fs"
import { parse, SyntaxError } from "../parser" import { parse, SyntaxError } from "../parser"
import { Declaration } from "../ast" import { Declaration, NodeDeclaration, TypeDeclaration, EnumDeclaration, TypeNode, NodeField } from "../ast"
import { FileWriter } from "../util"
for (const filename of process.argv.slice(2)) { for (const filename of process.argv.slice(2)) {
const contents = fs.readFileSync(filename, 'utf8'); const contents = fs.readFileSync(filename, 'utf8');
@ -19,22 +20,117 @@ for (const filename of process.argv.slice(2)) {
throw e; throw e;
} }
} }
console.error(jsonify(decls)); const generated = generateAST(decls, getFileStem(filename));
const generated = generateAST(decls); fs.writeFileSync('src/ast-generated.js', generated, 'utf8');
fs.writeFileSync(getFileStem(filename).toLowerCase(), generated, 'utf8');
} }
interface FastStringMap<T> { [key: string]: T } interface FastStringMap<T> { [key: string]: T }
function generateAST(decls: Declaration[]) { function generateAST(decls: Declaration[], langName: string) {
let jsFile = new FileWriter();
const nodeDecls: NodeDeclaration[] = decls.filter(decl => decl.type === 'NodeDeclaration') as NodeDeclaration[];
const typeDecls: TypeDeclaration[] = decls.filter(decl => decl.type === 'TypeDeclaration') as TypeDeclaration[];
const enumDecls: EnumDeclaration[] = decls.filter(decl => decl.type === 'EnumDeclaration') as EnumDeclaration[];
const declByName: FastStringMap<Declaration> = Object.create(null); const declByName: FastStringMap<Declaration> = Object.create(null);
for (const decl of decls) { for (const decl of decls) {
declByName[decl.name] = decl; declByName[decl.name] = decl;
} }
return '' // Generate a mapping from parent node to child node
// This makes it easy to generate union types for the intermediate nodes.
const childrenOf: FastStringMap<string[]> = Object.create(null);
for (const nodeDecl of nodeDecls) {
for (const parentName of nodeDecl.parents) {
if (childrenOf[parentName] === undefined) {
childrenOf[parentName] = [];
}
childrenOf[parentName].push(nodeDecl.name);
}
}
// Write a JavaScript file that contains all AST definitions.
jsFile.write(`\nconst NODE_TYPES = [\n`);
jsFile.indent();
for (const decl of decls) {
if (decl.type === 'NodeDeclaration' && isLeafNode(decl.name)) {
jsFile.write(`'${decl.name}': new Map([\n`);
jsFile.indent();
for (const field of getAllFields(decl)) {
jsFile.write(`[${field.name}, ${emitTypeNode(field.typeNode)}],\n`);
}
jsFile.dedent();
jsFile.write(']),\n');
}
}
jsFile.dedent();
jsFile.write('];\n\n');
function emitTypeNode(typeNode: TypeNode): string {
console.error(typeNode);
if (typeNode.type === 'ReferenceTypeNode') {
if (hasDeclarationNamed(typeNode.name)) {
return typeNode.name;
} else if (typeNode.name === 'Bool') {
return 'boolean';
} else if (typeNode.name === 'String') {
return 'string';
} else if (typeNode.name === 'usize') {
return 'bigint';
} else if (typeNode.name === 'Vec') {
return `${emitTypeNode(typeNode.typeArgs[0])}[]`;
} else if (typeNode.name === 'Option') {
return `${emitTypeNode(typeNode.typeArgs[0])} | null`;
}
}
throw new Error(`Could not emit TypeScript type for type node ${typeNode.type}.`);
}
function hasDeclarationNamed(name: string): boolean {
return name in declByName;
}
function getAllFields(nodeDecl: NodeDeclaration) {
let out: NodeField[] = [];
pushAll(out, nodeDecl.fields);
for (const parentName of nodeDecl.parents) {
const parentDecl = getDeclarationNamed(parentName);
if (parentDecl.type !== 'NodeDeclaration') {
throw new Error(`Parent declaration '${parentName}' of '${nodeDecl.name}' must be a node declaration.`);
}
pushAll(out, getAllFields(parentDecl));
}
return out;
}
function getDeclarationNamed(name: string): Declaration {
const decl = declByName[name];
if (decl === undefined) {
throw new Error(`Declaration '${name}' was not found in any of the definition files.`);
}
return decl;
}
function isLeafNode(name: string): boolean {
const decl = getDeclarationNamed(name);
if (decl.type !== 'NodeDeclaration') {
throw new Error(`Declaration '${name}' is not a node declaration.`)
}
return childrenOf[name] === undefined || childrenOf[name].length === 0;
}
return jsFile.currentText;
}
function pushAll<T>(arr: T[], els: T[]): void {
for (const el of els) {
arr.push(el);
}
} }
function jsonify(value: any) { function jsonify(value: any) {

0
treegen/src/index.ts Normal file
View file

View file

@ -28,7 +28,7 @@ Declaration
/ TypeDeclaration / TypeDeclaration
NodeDeclaration NodeDeclaration
= NodeToken __ name:Identifier parents:(__ '>' __ ExtendsList)? fields:(__ '{' __ (@NodeField __)* '}')? EOS { = NodeToken __ name:Identifier parents:(__ '>' __ @ExtendsList)? fields:(__ '{' __ @(@NodeField __)* '}')? EOS {
return createNode('NodeDeclaration', { name, parents: liftArray(parents), fields: liftArray(fields) }); return createNode('NodeDeclaration', { name, parents: liftArray(parents), fields: liftArray(fields) });
} }
@ -47,12 +47,15 @@ TypeNode
UnionTypeNode UnionTypeNode
= head:ReferenceTypeNode tail:(__ '|' __ @ReferenceTypeNode)* { = head:ReferenceTypeNode tail:(__ '|' __ @ReferenceTypeNode)* {
return [head, ...tail]; if (tail.length === 0) {
return head;
}
return createNode('UnionTypeNode', { elements: [head, ...tail] });
} }
ReferenceTypeNode ReferenceTypeNode
= name:Identifier genericArgs:(__ '<' __ @TypeNodeList __ '>')? { = name:Identifier typeArgs:(__ '<' __ @TypeNodeList __ '>')? {
return createNode('ReferenceTypeNode', { name, genericArgs }); return createNode('ReferenceTypeNode', { name, typeArgs });
} }
TypeNodeList TypeNodeList