commit d813e85d000c8fa216780e5e7aa8b86ad61d680e Author: Sam Vervaeck Date: Sun Aug 28 21:12:25 2022 +0200 Enable parsing of simple let-expressions diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..9d850e6e8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +/lib/ +Makefile diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..7059a4a8c --- /dev/null +++ b/package-lock.json @@ -0,0 +1,391 @@ +{ + "name": "@samvv/bolt", + "version": "0.0.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "@samvv/bolt", + "version": "0.0.1", + "license": "MIT", + "dependencies": { + "source-map-support": "^0.5.21", + "tslib": "^2.4.0", + "yargs": "^17.5.1" + }, + "bin": { + "bolt": "lib/bin/bolt.js" + }, + "devDependencies": { + "@types/node": "^18.7.13", + "@types/yargs": "^17.0.11" + } + }, + "node_modules/@types/node": { + "version": "18.7.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.13.tgz", + "integrity": "sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw==", + "dev": true + }, + "node_modules/@types/yargs": { + "version": "17.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.11.tgz", + "integrity": "sha512-aB4y9UDUXTSMxmM4MH+YnuR0g5Cph3FLQBoWoMB21DSvFVAxRVEHEMx3TLh+zUZYMCQtKiqazz0Q4Rre31f/OA==", + "dev": true, + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "20.2.1", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-20.2.1.tgz", + "integrity": "sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==", + "dev": true + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.5.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz", + "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-z9kApYUOCwoeZ78rfRYYWdiU/iNL6mwwYlkkZfJoyMR1xps+NEBX5X7XmRpxkZHhXJ6+Ey00IwKxBBSW9FIjyA==", + "engines": { + "node": ">=12" + } + } + }, + "dependencies": { + "@types/node": { + "version": "18.7.13", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.13.tgz", + "integrity": "sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw==", + "dev": true + }, + "@types/yargs": { + "version": "17.0.11", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.11.tgz", + "integrity": "sha512-aB4y9UDUXTSMxmM4MH+YnuR0g5Cph3FLQBoWoMB21DSvFVAxRVEHEMx3TLh+zUZYMCQtKiqazz0Q4Rre31f/OA==", + "dev": true, + "requires": { + "@types/yargs-parser": "*" + } + }, + "@types/yargs-parser": { + "version": "20.2.1", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-20.2.1.tgz", + "integrity": "sha512-7tFImggNeNBVMsn0vLrpn1H1uPrUBdnARPTpZoitY37ZrdJREzf7I16tMrlK3hen349gr1NYh8CmZQa7CTG6Aw==", + "dev": true + }, + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==" + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=" + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, + "source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "tslib": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.0.tgz", + "integrity": "sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ==" + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yargs": { + "version": "17.5.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.5.1.tgz", + "integrity": "sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==", + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.0.0" + } + }, + "yargs-parser": { + "version": "21.0.0", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.0.0.tgz", + "integrity": "sha512-z9kApYUOCwoeZ78rfRYYWdiU/iNL6mwwYlkkZfJoyMR1xps+NEBX5X7XmRpxkZHhXJ6+Ey00IwKxBBSW9FIjyA==" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 000000000..ed058d4a5 --- /dev/null +++ b/package.json @@ -0,0 +1,33 @@ +{ + "name": "@samvv/bolt", + "version": "0.0.1", + "description": "A new programming language for the web", + "main": "lib/index.js", + "bin": { + "bolt": "lib/bin/bolt.js" + }, + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "https://github.com/BoltJS" + }, + "keywords": [ + "programming-language", + "productivity", + "development", + "performance" + ], + "author": "Sam Vervaeck", + "license": "MIT", + "dependencies": { + "source-map-support": "^0.5.21", + "tslib": "^2.4.0", + "yargs": "^17.5.1" + }, + "devDependencies": { + "@types/node": "^18.7.13", + "@types/yargs": "^17.0.11" + } +} diff --git a/src/bin/bolt.ts b/src/bin/bolt.ts new file mode 100644 index 000000000..4aa4bd01b --- /dev/null +++ b/src/bin/bolt.ts @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +import "source-map-support/register" + +import path from "path" +import fs from "fs" +import yargs from "yargs" + +import { Diagnostics } from "../diagnostics" +import { Punctuator, Scanner } from "../scanner" +import { Parser } from "../parser" + +yargs + .string('work-dir') + .describe('work-dir', 'Act as if run from this directory') + .default('work-dir', '.') + .alias('work-dir', 'C') + .command( + ['$0 ', 'exec '], + 'Execute a Bolt script', + yargs => yargs + .string('file') + .demandOption('file') + .describe('file', 'Path to the script to execute') + , args => { + + const cwd = args.C; + const filename = path.resolve(cwd, args.file); + const diagnostics = new Diagnostics(); + const text = fs.readFileSync(filename, 'utf8') + const scanner = new Scanner(text, 0, diagnostics); + const punctuated = new Punctuator(scanner); + const parser = new Parser(punctuated); + const sourceFile = parser.parseSourceFile(); + + console.log(sourceFile); + + } + ) + .help() + .version() + .argv diff --git a/src/checker.ts b/src/checker.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/cst.ts b/src/cst.ts new file mode 100644 index 000000000..e47a4fd52 --- /dev/null +++ b/src/cst.ts @@ -0,0 +1,847 @@ + +export type TextSpan = [number, number]; + +export class TextPosition { + + public constructor( + public offset: number, + public line: number, + public column: number, + ) { + + } + + public clone(): TextPosition { + return new TextPosition( + this.offset, + this.line, + this.column, + ); + } + + public advance(text: string): void { + for (const ch of text) { + if (ch === '\n') { + this.line++; + this.column = 1; + } else { + this.column++; + } + this.offset += text.length; + } + + } + +} + +export class TextRange { + + constructor( + public start: TextPosition, + public end: TextPosition, + ) { + + } + + public clone(): TextRange { + return new TextRange( + this.start.clone(), + this.end.clone(), + ); + } + +} + +export class TextFile { + + public constructor( + public origPath: string, + public text: string, + ) { + + } + +} + +export const enum SyntaxKind { + + // Tokens + Identifier, + CustomOperator, + LParen, + RParen, + LBrace, + RBrace, + LBracket, + RBracket, + Dot, + DotDot, + Comma, + Colon, + Equals, + Integer, + StringLiteral, + LetKeyword, + PubKeyword, + MutKeyword, + ModKeyword, + ImportKeyword, + StructKeyword, + TypeKeyword, + LineFoldEnd, + BlockEnd, + BlockStart, + EndOfFile, + + // Type expressions + ReferenceTypeExpression, + + // Patterns + BindPattern, + TuplePattern, + + // Expressions + ReferenceExpression, + TupleExpression, + NestedExpression, + ConstantExpression, + PrefixExpression, + PostfixExpression, + InfixExpression, + + // Statements + ReturnStatement, + ExpressionStatement, + + // Declarations + VariableDeclaration, + PrefixFuncDecl, + SuffixFuncDecl, + LetDeclaration, + StructDeclaration, + ImportDeclaration, + TypeAliasDeclaration, + + // Other nodes + StructDeclarationField, + ExprBody, + BlockBody, + TypeAssert, + Param, + Module, + SourceFile, + +} + +export type Syntax + = SourceFile + | Param + | StructDeclarationField + | Declaration + | Statement + | Expression + | TypeExpression + | Pattern + +abstract class SyntaxBase { + + public abstract readonly kind: SyntaxKind; + +} + +abstract class TokenBase extends SyntaxBase { + + private endPos: TextPosition | null = null; + + constructor( + private startPos: TextPosition, + ) { + super(); + } + + public getStartPosition(): TextPosition { + return this.startPos; + } + + public getStartLine(): number { + return this.getStartPosition().line; + } + + public getStartColumn(): number { + return this.getStartPosition().column; + } + + public getEndPosition(): TextPosition { + if (this.endPos === null) { + const endPos = this.getStartPosition().clone(); + endPos.advance(this.text); + return this.endPos = endPos; + } + return this.endPos; + } + + public getEndLine(): number { + return this.getEndPosition().line; + } + + public getEndColumn(): number { + return this.getEndPosition().column; + } + + public abstract readonly text: string; + +} + +abstract class VirtualTokenBase extends TokenBase { + public get text(): string { + return ''; + } +} + +export class EndOfFile extends VirtualTokenBase { + public readonly kind = SyntaxKind.EndOfFile; +} + +export class BlockEnd extends VirtualTokenBase { + public readonly kind = SyntaxKind.BlockEnd; +} + +export class BlockStart extends VirtualTokenBase { + public readonly kind = SyntaxKind.BlockStart; +} + +export class LineFoldEnd extends VirtualTokenBase { + public readonly kind = SyntaxKind.LineFoldEnd; +} + +export class Integer extends TokenBase { + + public readonly kind = SyntaxKind.Integer; + + public constructor( + public value: bigint, + public radix: number, + private startPos: TextPosition, + ) { + super(startPos); + } + + public get text(): string { + switch (this.radix) { + case 16: + return '0x' + this.value.toString(16); + case 10: + return this.value.toString(10) + case 8: + return '0o' + this.value.toString(8) + case 2: + return '0b' + this.value.toString(2); + default: + throw new Error(`Radix ${this.radix} of Integer not recognised.`) + } + } + +} + +export class StringLiteral extends TokenBase { + + public readonly kind = SyntaxKind.StringLiteral; + + public constructor( + public contents: string, + private startPos: TextPosition, + ) { + super(startPos); + } + + public get text(): string { + let out = '"'; + for (const ch of this.contents) { + const code = ch.charCodeAt(0); + if (code >= 32 && code <= 127) { + out += ch; + } else if (code <= 127) { + out += '\\x' + code.toString(16).padStart(2, '0'); + } else { + out += '\\u' + code.toString(17).padStart(4, '0'); + } + } + out += '"'; + return out; + } + +} + +export class Identifier extends TokenBase { + + public readonly kind = SyntaxKind.Identifier; + + public constructor( + public text: string, + private startPos: TextPosition, + ) { + super(startPos); + } + +} + +export class CustomOperator extends TokenBase { + + public readonly kind = SyntaxKind.CustomOperator; + + public constructor( + public text: string, + private startPos: TextPosition, + ) { + super(startPos); + } + +} + +export class LParen extends TokenBase { + + public readonly kind = SyntaxKind.LParen; + + public get text(): string { + return '('; + } + +} + +export class RParen extends TokenBase { + + public readonly kind = SyntaxKind.RParen; + + public get text(): string { + return ')'; + } + +} + +export class LBrace extends TokenBase { + + public readonly kind = SyntaxKind.LBrace; + + public get text(): string { + return '{'; + } + +} + +export class RBrace extends TokenBase { + + public readonly kind = SyntaxKind.RBrace; + + public get text(): string { + return '}'; + } + +} + +export class LBracket extends TokenBase { + + public readonly kind = SyntaxKind.LBracket; + + public get text(): string { + return '['; + } + +} + +export class RBracket extends TokenBase { + + public readonly kind = SyntaxKind.RBracket; + + public get text(): string { + return ']'; + } + +} + +export class Dot extends TokenBase { + + public readonly kind = SyntaxKind.Dot; + + public get text(): string { + return '.'; + } + +} + +export class Comma extends TokenBase { + + public readonly kind = SyntaxKind.Comma; + + public get text(): string { + return ','; + } + +} + +export class DotDot extends TokenBase { + + public readonly kind = SyntaxKind.DotDot; + + public get text(): string { + return '..'; + } + +} + +export class Colon extends TokenBase { + + public readonly kind = SyntaxKind.Colon; + + public get text(): string { + return ':'; + } + +} + +export class Equals extends TokenBase { + + public readonly kind = SyntaxKind.Equals; + + public get text(): string { + return '='; + } + +} + +export class StructKeyword extends TokenBase { + + public readonly kind = SyntaxKind.StructKeyword; + + public get text(): string { + return 'struct'; + } + +} + +export class ModKeyword extends TokenBase { + + public readonly kind = SyntaxKind.ModKeyword; + + public get text(): string { + return 'mod'; + } + +} + +export class MutKeyword extends TokenBase { + + public readonly kind = SyntaxKind.MutKeyword; + + public get text(): string { + return 'mut'; + } + +} + +export class ImportKeyword extends TokenBase { + + public readonly kind = SyntaxKind.ImportKeyword; + + public get text(): string { + return 'import' + } + +} + +export class TypeKeyword extends TokenBase { + + public readonly kind = SyntaxKind.TypeKeyword; + + public get text(): string { + return 'type'; + } + +} + +export class PubKeyword extends TokenBase { + + public readonly kind = SyntaxKind.PubKeyword; + + public get text(): string { + return 'pub'; + } + +} + +export class LetKeyword extends TokenBase { + + public readonly kind = SyntaxKind.LetKeyword; + + public get text(): string { + return 'let'; + } + +} + +export type Token + = LParen + | RParen + | LBrace + | RBrace + | LBracket + | RBracket + | Identifier + | CustomOperator + | Integer + | StringLiteral + | Comma + | Dot + | DotDot + | Colon + | Equals + | LetKeyword + | PubKeyword + | MutKeyword + | ModKeyword + | ImportKeyword + | TypeKeyword + | StructKeyword + | EndOfFile + | BlockStart + | BlockEnd + | LineFoldEnd + +export type TokenKind + = Token['kind'] + +export class ReferenceTypeExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.ReferenceTypeExpression; + + public constructor( + public modulePath: Array<[Identifier, Dot]>, + public name: Identifier, + ) { + super(); + } + +} + +export type TypeExpression + = ReferenceTypeExpression + +export class BindPattern extends SyntaxBase { + + public readonly kind = SyntaxKind.BindPattern; + + public constructor( + public name: Identifier, + ) { + super(); + } + + public get isHole(): boolean { + return this.name.text == '_'; + } + +} + +export class TuplePattern extends SyntaxBase { + + public readonly kind = SyntaxKind.TuplePattern; + + public constructor( + public elements: Pattern[], + ) { + super(); + } + +} + +export type Pattern + = BindPattern + | TuplePattern + +export class TupleExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.TupleExpression; + + public constructor( + public lparen: LParen, + public elements: Expression[], + public rparen: RParen, + ) { + super(); + } + +} + +export class NestedExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.NestedExpression; + + public constructor( + public lparen: LParen, + public expression: Expression, + public rparen: RParen, + ) { + super(); + } + +} + +export class ConstantExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.ConstantExpression; + + public constructor( + public token: Integer | StringLiteral, + ) { + super(); + } + +} + +export class ReferenceExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.ReferenceExpression; + + public constructor( + public modulePath: Array<[Identifier, Dot]>, + public name: Identifier + ) { + super(); + } + +} + +export class PrefixExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.PrefixExpression; + + public constructor( + public operator: Token, + public expression: Expression, + ) { + super(); + } + +} + +export class PostfixExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.PostfixExpression; + + public constructor( + public expression: Expression, + public operator: Token, + ) { + super(); + } + +} + +export class InfixExpression extends SyntaxBase { + + public readonly kind = SyntaxKind.InfixExpression; + + public constructor( + public left: Expression, + public operator: Token, + public right: Expression, + ) { + super(); + } + +} + +export type Expression + = ReferenceExpression + | ConstantExpression + | TupleExpression + | NestedExpression + | PrefixExpression + | InfixExpression + | PostfixExpression + +export class ReturnStatement extends SyntaxBase { + + public readonly kind = SyntaxKind.ReturnStatement; + + public constructor( + public expr: Expression + ) { + super(); + } + +} + +export class ExpressionStatement extends SyntaxBase { + + public readonly kind = SyntaxKind.ExpressionStatement; + + public constructor( + public expresion: Expression, + ) { + super(); + } + +} + +export type Statement + = ReturnStatement + | ExpressionStatement + +export class Param extends SyntaxBase { + + public readonly kind = SyntaxKind.Param; + + public constructor( + public pattern: Pattern, + ) { + super(); + } + +} + +export class StructDeclarationField extends SyntaxBase { + + public readonly kind = SyntaxKind.StructDeclarationField; + + public constructor( + public name: Identifier, + public colon: Colon, + public typeExpr: TypeExpression, + ) { + super(); + } + +} + +export class StructDeclaration extends SyntaxBase { + + public readonly kind = SyntaxKind.StructDeclaration; + + public constructor( + public structKeyword: StructKeyword, + public name: Identifier, + public members: StructDeclarationField[] | null, + ) { + super(); + } + +} + +export class TypeAssert extends SyntaxBase { + + public readonly kind = SyntaxKind.TypeAssert; + + public constructor( + public colon: Colon, + public typeExpression: TypeExpression, + ) { + super(); + } + +} + +export type Body + = ExprBody + | BlockBody + +export class ExprBody extends SyntaxBase { + + public readonly kind = SyntaxKind.ExprBody; + + public constructor( + public equals: Equals, + public expression: Expression, + ) { + super(); + } + +} + +export type LetBodyElement + = LetDeclaration + | Statement + +export class BlockBody extends SyntaxBase { + + public readonly kind = SyntaxKind.BlockBody; + + public constructor( + public blockStart: BlockStart, + public elements: LetBodyElement[], + ) { + super(); + } + +} +export class LetDeclaration extends SyntaxBase { + + public readonly kind = SyntaxKind.LetDeclaration; + + public constructor( + public pubKeyword: PubKeyword | null, + public letKeyword: LetKeyword, + public mutKeyword: MutKeyword | null, + public pattern: Pattern, + public params: Param[], + public typeAssert: TypeAssert | null, + public body: Body | null, + ) { + super(); + } + +} + +export class ImportDeclaration extends SyntaxBase { + + public readonly kind = SyntaxKind.ImportDeclaration; + + public constructor( + public importKeyword: ImportKeyword, + public importSource: StringLiteral, + ) { + super(); + } + +} + +export type Declaration + = LetDeclaration + | ImportDeclaration + | StructDeclaration + +export class Module extends SyntaxBase { + + public readonly kind = SyntaxKind.Module; + + public constructor( + public modKeyword: ModKeyword, + public name: Identifier, + public body: Body, + ) { + super(); + } + +} + +export type SourceFileElement + = Statement + | Declaration + | Module + +export class SourceFile extends SyntaxBase { + + public readonly kind = SyntaxKind.SourceFile; + + public constructor( + public elements: SourceFileElement[] + ) { + super(); + } + + public *getChildNodes(): Iterable { + for (const element in this.elements) { + yield element; + } + } + +} diff --git a/src/diagnostics.ts b/src/diagnostics.ts new file mode 100644 index 000000000..03e71ced3 --- /dev/null +++ b/src/diagnostics.ts @@ -0,0 +1,32 @@ + +export class UnexpectedCharDiagnostic { + + public constructor( + public text: string, + public offset: number, + public actual: string, + ) { + + } + + public format(): string { + let out = `error: unexpeced character '${this.actual}'.`; + return out; + } + +} + +export type Diagnostic + = UnexpectedCharDiagnostic; + +export class Diagnostics { + + private savedDiagnostics: Diagnostic[] = []; + + public add(diagnostic: Diagnostic): void { + this.savedDiagnostics.push(diagnostic); + process.stderr.write(diagnostic.format()); + } + +} + diff --git a/src/parser.ts b/src/parser.ts new file mode 100644 index 000000000..d123098ea --- /dev/null +++ b/src/parser.ts @@ -0,0 +1,425 @@ + +import { kMaxLength } from "buffer"; +import { + ReferenceTypeExpression, + SourceFile, + SourceFileElement, + StructDeclaration, + StructDeclarationField, + SyntaxKind, + Token, + TokenKind, + Expression, + TypeExpression, + ConstantExpression, + ReferenceExpression, + Dot, + Identifier, + TupleExpression, + PrefixExpression, + ExpressionStatement, + ImportDeclaration, + FunctionDeclaration, + Param, + Pattern, + BindPattern, + LetDeclaration, + TypeAssert, + ExprBody, + BlockBody, +} from "./cst" +import { Stream, MultiDict } from "./util"; + +const DESCRIPTIONS: Record = { + [SyntaxKind.StringLiteral]: 'a string literal', + [SyntaxKind.Identifier]: "an identifier", + [SyntaxKind.Comma]: "','", + [SyntaxKind.Colon]: "':'", + [SyntaxKind.Integer]: "an integer", + [SyntaxKind.LParen]: "'('", + [SyntaxKind.RParen]: "')'", + [SyntaxKind.LBrace]: "'{'", + [SyntaxKind.RBrace]: "'}'", + [SyntaxKind.LBracket]: "'['", + [SyntaxKind.RBracket]: "']'", + [SyntaxKind.ConstantExpression]: 'a constant expression', + [SyntaxKind.ReferenceExpression]: 'a reference expression', + [SyntaxKind.LineFoldEnd]: 'the end of the current line-fold', + [SyntaxKind.TupleExpression]: 'a tuple expression such as (1, 2)', + [SyntaxKind.ReferenceExpression]: 'a reference to some variable', + [SyntaxKind.NestedExpression]: 'an expression nested with parentheses', + [SyntaxKind.ConstantExpression]: 'a constant expression such as 1 or "foo"', +} + +function describeSyntaxKind(kind: SyntaxKind): string { + const desc = DESCRIPTIONS[kind]; + if (desc === undefined) { + throw new Error(`Could not describe SyntaxKind '${kind}'`); + } + return desc +} + +function describeExpected(expected: SyntaxKind[]) { + if (expected.length === 0) { + return 'nothing'; + } + let out = describeSyntaxKind(expected[0]); + if (expected.length === 1) { + return out; + } + for (let i = 1; i < expected.length-1; i++) { + const kind = expected[i]; + out += ', ' + describeSyntaxKind(kind); + } + out += ' or ' + describeSyntaxKind(expected[expected.length-1]) + return out; +} + +class ParseError extends Error { + + public constructor( + public actual: Token, + public expected: SyntaxKind[], + ) { + super(`got '${actual.text}' but expected ${describeExpected(expected)}`); + } + +} + +function isConstructor(token: Token): boolean { + return token.kind === SyntaxKind.Identifier + && token.text[0].toUpperCase() === token.text[0]; +} + +const enum OperatorMode { + None = 0, + Prefix = 1, + InfixL = 2, + InfixR = 4, + Suffix = 8, +} + +interface OperatorInfo { + name: string, + mode: OperatorMode, + precedence?: number, +} + +export class Parser { + + private exprOperators = new MultiDict(); + + public constructor( + public tokens: Stream, + ) { + + } + + private getToken(): Token { + return this.tokens.get(); + } + + private peekToken(offset = 1): Token { + return this.tokens.peek(offset); + } + + private assertToken(token: Token, expectedKind: K): void { + if (token.kind !== expectedKind) { + this.raiseParseError(token, [ expectedKind ]); + } + } + + private expectToken(expectedKind: K): Token & { kind: K } { + const token = this.getToken(); + if (token.kind !== expectedKind) { + this.raiseParseError(token, [ expectedKind ]) + } + return token as Token & { kind: K }; + } + + private raiseParseError(actual: Token, expected: SyntaxKind[]): never { + throw new ParseError(actual, expected); + } + + private peekTokenAfterModifiers(): Token { + let t0; + for (let i = 1;;i++) { + t0 = this.peekToken(i); + if (t0.kind !== SyntaxKind.PubKeyword) { + break; + } + } + return t0; + } + + private isPrefixOperator(token: Token): boolean { + const name = token.text; + for (const operator of this.exprOperators.get(name)) { + if (operator.mode & OperatorMode.Prefix) { + return true; + } + } + return false; + } + + private isBinaryOperator(token: Token): boolean { + return token.kind === SyntaxKind.CustomOperator; + } + + public parseReferenceTypeExpression(): ReferenceTypeExpression { + const name = this.expectToken(SyntaxKind.Identifier); + return new ReferenceTypeExpression([], name); + } + + public parseTypeExpression(): TypeExpression { + const t0 = this.peekToken(); + switch (t0.kind) { + case SyntaxKind.Identifier: + return this.parseReferenceTypeExpression(); + default: + throw new ParseError(t0, [ SyntaxKind.Identifier ]); + } + } + + public parseConstantExpression(): ConstantExpression { + const token = this.getToken() + if (token.kind !== SyntaxKind.StringLiteral + && token.kind !== SyntaxKind.Integer) { + this.raiseParseError(token, [ SyntaxKind.StringLiteral, SyntaxKind.Integer ]) + } + return new ConstantExpression(token); + } + + public parseReferenceExpression(): ReferenceExpression { + const modulePath: Array<[Identifier, Dot]> = []; + let name = this.expectToken(SyntaxKind.Identifier) + for (;;) { + const t1 = this.peekToken() + if (t1.kind !== SyntaxKind.Dot) { + break; + } + modulePath.push([name, t1]); + name = this.expectToken(SyntaxKind.Identifier) + } + return new ReferenceExpression(modulePath, name); + } + + private parseExpressionWithParens(): Expression { + const t0 = this.expectToken(SyntaxKind.LParen) + const t1 = this.peekToken(); + if (t1.kind === SyntaxKind.RParen) { + this.getToken(); + return new TupleExpression(t0, [], t1); + } + if (isConstructor(t1)) { + + } + } + + private parseExpressionNoOperators(): Expression { + const t0 = this.peekToken(); + switch (t0.kind) { + case SyntaxKind.LParen: + return this.parseExpressionWithParens(); + case SyntaxKind.Identifier: + return this.parseReferenceExpression(); + case SyntaxKind.Integer: + case SyntaxKind.StringLiteral: + return this.parseConstantExpression(); + default: + this.raiseParseError(t0, [ + SyntaxKind.TupleExpression, + SyntaxKind.NestedExpression, + SyntaxKind.ConstantExpression, + SyntaxKind.ReferenceExpression + ]); + } + } + + private parseUnaryExpression(): Expression { + let out = this.parseExpressionNoOperators() + const prefixOperators = []; + for (;;) { + const t0 = this.peekToken(); + if (!this.isPrefixOperator(t0)) { + break; + } + prefixOperators.push(t0); + this.getToken() + } + for (let i = prefixOperators.length-1; i >= 0; i--) { + const op = prefixOperators[i]; + out = new PrefixExpression(op, out); + } + return out; + } + + private parseExpressionWithBinaryOperator(lhs: Expression, minPrecedence: number) { + for (;;) { + const t0 = this.peekToken(); + if (!this.isBinaryOperator(t0)) { + break; + } + } + return lhs; + } + + public parseExpression(): Expression { + const lhs = this.parseUnaryExpression(); + return this.parseExpressionWithBinaryOperator(lhs, 0); + } + + public parseStructDeclaration(): StructDeclaration { + const structKeyword = this.expectToken(SyntaxKind.StructKeyword); + const name = this.expectToken(SyntaxKind.Identifier); + const t2 = this.peekToken() + let members = null; + if (t2.kind === SyntaxKind.BlockStart) { + this.getToken(); + members = []; + for (;;) { + const name = this.expectToken(SyntaxKind.Identifier); + const colon = this.expectToken(SyntaxKind.Colon); + const typeExpr = this.parseTypeExpression(); + const member = new StructDeclarationField(name, colon, typeExpr); + members.push(member); + } + } else { + this.assertToken(t2, SyntaxKind.LineFoldEnd); + } + return new StructDeclaration(structKeyword, name, members); + } + + public parsePattern(): Pattern { + const t0 = this.peekToken(); + switch (t0.kind) { + case SyntaxKind.Identifier: + this.getToken(); + return new BindPattern(t0); + default: + this.raiseParseError(t0, [ SyntaxKind.Identifier ]); + } + } + + public parseParam(): Param { + const pattern = this.parsePattern(); + return new Param(pattern); + } + + public parseDeclartionWithLetKeyword(): LetDeclaration { + let t0 = this.getToken(); + let pubKeyword = null; + let mutKeyword = null; + if (t0.kind === SyntaxKind.PubKeyword) { + pubKeyword = t0; + t0 = this.getToken(); + } + if (t0.kind !== SyntaxKind.LetKeyword) { + this.raiseParseError(t0, [ SyntaxKind.LetKeyword ]); + } + const t1 = this.peekToken(); + if (t1.kind === SyntaxKind.MutKeyword) { + this.getToken(); + mutKeyword = t1; + } + const pattern = this.parsePattern(); + const params = []; + for (;;) { + const t2 = this.peekToken(); + if (t2.kind === SyntaxKind.Colon + || t2.kind === SyntaxKind.BlockStart + || t2.kind === SyntaxKind.Equals + || t2.kind === SyntaxKind.LineFoldEnd) { + break; + } + params.push(this.parseParam()); + } + let typeAssert = null; + let t3 = this.getToken(); + if (t3.kind === SyntaxKind.Colon) { + const typeExpression = this.parseTypeExpression(); + typeAssert = new TypeAssert(t3, typeExpression); + t3 = this.getToken(); + } + let body = null; + switch (t3.kind) { + case SyntaxKind.BlockStart: + { + const elements = []; + for (;;) { + const t4 = this.peekToken(); + if (t4.kind === SyntaxKind.BlockEnd) { + break; + } + elements.push(this.parseLetBodyElement()); + } + body = new BlockBody(t3, elements); + t3 = this.getToken(); + break; + } + case SyntaxKind.Equals: + { + const expression = this.parseExpression(); + body = new ExprBody(t3, expression); + t3 = this.getToken(); + break; + } + case SyntaxKind.LineFoldEnd: + break; + } + if (t3.kind !== SyntaxKind.LineFoldEnd) { + this.raiseParseError(t3, [ SyntaxKind.LineFoldEnd ]); + } + return new LetDeclaration( + pubKeyword, + t0, + mutKeyword, + pattern, + params, + typeAssert, + body + ); + } + + public parseExpressionStatement(): ExpressionStatement { + const expression = this.parseExpression(); + this.expectToken(SyntaxKind.LineFoldEnd) + return new ExpressionStatement(expression); + } + + public parseImportDeclaration(): ImportDeclaration { + const importKeyword = this.expectToken(SyntaxKind.ImportKeyword); + const importSource = this.expectToken(SyntaxKind.StringLiteral); + return new ImportDeclaration(importKeyword, importSource); + } + + public parseSourceFileElement(): SourceFileElement { + + const t0 = this.peekTokenAfterModifiers(); + switch (t0.kind) { + case SyntaxKind.LetKeyword: + return this.parseDeclartionWithLetKeyword(); + case SyntaxKind.ImportKeyword: + return this.parseImportDeclaration(); + case SyntaxKind.StructKeyword: + return this.parseStructDeclaration(); + default: + return this.parseExpressionStatement(); + } + } + + public parseSourceFile(): SourceFile { + const elements = []; + for (;;) { + const t0 = this.peekToken(); + if (t0.kind === SyntaxKind.EndOfFile) { + break; + } + const element = this.parseSourceFileElement(); + elements.push(element); + } + return new SourceFile(elements); + } + +} + diff --git a/src/scanner.ts b/src/scanner.ts new file mode 100644 index 000000000..0e9713804 --- /dev/null +++ b/src/scanner.ts @@ -0,0 +1,389 @@ + +import { + SyntaxKind, + Token, + Identifier, + StringLiteral, + EndOfFile, + BlockStart, + BlockEnd, + LineFoldEnd, + PubKeyword, + MutKeyword, + LetKeyword, + ImportKeyword, + TypeKeyword, + TextPosition, + Colon, + Comma, + Equals, + LParen, + RParen, + LBrace, + LBracket, + RBrace, + RBracket, + CustomOperator, +} from "./cst" +import { Diagnostics, UnexpectedCharDiagnostic } from "./diagnostics" +import { Stream, BufferedStream } from "./util"; + +const EOF = '\uFFFF' + +function isWhiteSpace(ch: string): boolean { + return /[\r\n\t ]/.test(ch); +} + +function isIdentPart(ch: string): boolean { + return /[a-zA-Z0-9_]/.test(ch); +} + +function isIdentStart(ch: string): boolean { + return /[a-zA-Z_]/.test(ch) +} + +function isOperatorPart(ch: string): boolean { + return /\+-*\/%^&|$<>!?=/.test(ch); +} + +class ScanError extends Error {} + +export class Scanner extends BufferedStream { + + private currLine = 1; + private currColumn = 1; + + public constructor( + public text: string, + public textOffset: number = 0, + public diagnostics: Diagnostics, + ) { + super(); + } + + private peekChar(offset = 1): string { + const i = this.textOffset + offset - 1; + return i < this.text.length ? this.text[i] : EOF; + } + + private getChar(): string { + const ch = this.textOffset < this.text.length + ? this.text[this.textOffset++] + : EOF + if (ch === '\n') { + this.currLine++; + this.currColumn = 1; + } else { + this.currColumn++; + } + return ch; + } + + private takeWhile(pred: (ch: string) => boolean): string { + let out = '' + for (;;) { + const c0 = this.peekChar() + if (!pred(c0)) { + break; + } + this.getChar() + out += c0; + } + return out; + } + + private getCurrentPosition(): TextPosition { + return new TextPosition( + this.textOffset, + this.currLine, + this.currColumn + ); + } + + public read(): Token { + + let c0: string; + + // Skip whitespace and comments + for (;;) { + + for (;;) { + c0 = this.peekChar(); + if (isWhiteSpace(c0)) { + this.getChar(); + continue; + } + if (c0 === '#') { + this.getChar(); + for (;;) { + const c1 = this.getChar(); + if (c1 === '\n' || c1 === EOF) { + break; + } + } + continue; + } + + // We failed to match a newline or line comment, so there's nothing to skip + break; + + } + + const startPos = this.getCurrentPosition(); + this.getChar(); + + switch (c0) { + + case '"': + { + const startPos = this.getCurrentPosition(); + let contents = ''; + let escaping = false; + for (;;) { + const c1 = this.getChar(); + if (escaping) { + switch (c1) { + case 'a': contents += '\a'; break; + case 'b': contents += '\b'; break; + case 'f': contents += '\f'; break; + case 'n': contents += '\n'; break; + case 'r': contents += '\r'; break; + case 't': contents += '\t'; break; + case 'v': contents += '\v'; break; + case '0': contents += '\0'; break; + case '\'': contents += '\''; break; + case '\"': contents += '\"'; break; + default: + this.diagnostics.add(new UnexpectedCharDiagnostic(this.text, this.textOffset, c1)); + throw new ScanError(); + } + escaping = false; + } else { + if (c1 === '"') { + break; + } else { + contents += c1; + } + } + } + return new StringLiteral(contents, startPos); + } + + case EOF: + { + return new EndOfFile(startPos); + } + + case '(': return new LParen(startPos); + case ')': return new RParen(startPos); + case '[': return new LBracket(startPos); + case ']': return new RBracket(startPos); + case '{': return new LBrace(startPos); + case '}': return new RBrace(startPos); + case ',': return new Comma(startPos); + case ':': return new Colon(startPos); + + case '+': + case '-': + case '*': + case '/': + case '%': + case '&': + case '^': + case '|': + case '$': + case '<': + case '>': + case '=': + case '!': + case '?': + { + const text = c0 + this.takeWhile(isOperatorPart); + if (text === '=') { + return new Equals(startPos); + } else if (text.endsWith('=') && text[text.length-2] !== '=') { + return new Assignment(startPos); + } else { + return new CustomOperator(text, startPos); + } + } + + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + { + const text = c0 + this.takeWhile(isIdentPart); + switch (text) { + case 'import': return new ImportKeyword(startPos); + case 'pub': return new PubKeyword(startPos); + case 'mut': return new MutKeyword(startPos); + case 'let': return new LetKeyword(startPos); + case 'import': return new ImportKeyword(startPos); + case 'return': return new ReturnKeyword(startPos); + case 'type': return new TypeKeyword(startPos); + default: + return new Identifier(text, startPos); + } + } + + default: + + // Nothing matched, so the current character is unrecognisable + this.diagnostics.add(new UnexpectedCharDiagnostic(this.text, this.textOffset, c0)); + throw new ScanError(); + } + + } + + } + +} + +const enum FrameType { + Block, + LineFold, +} + +const INIT_POS = new TextPosition(0, 0, 0); + +export class Punctuator extends BufferedStream { + + private referencePositions: TextPosition[] = [ INIT_POS ]; + + private frameTypes: FrameType[] = [ FrameType.Block ]; + + public constructor( + private tokens: Stream, + ) { + super(); + } + + public read(): Token { + + const refPos = this.referencePositions[this.referencePositions.length-1]; + const frameType = this.frameTypes[this.frameTypes.length-1]; + const t0 = this.tokens.peek(1); + + if (t0.kind === SyntaxKind.EndOfFile) { + if (this.frameTypes.length === 1) { + return t0; + } + this.frameTypes.pop(); + switch (frameType) { + case FrameType.LineFold: + return new LineFoldEnd(t0.getStartPosition()); + case FrameType.Block: + return new BlockEnd(t0.getStartPosition()); + } + } + + switch (frameType) { + + case FrameType.LineFold: + { + + // This important check verifies we're still inside the line-fold. If + // we aren't, we need to clean up the stack a bit and eventually return + // a token that indicates the line-fold ended. + if (t0.getStartLine() > refPos.line + && t0.getStartColumn() <= refPos.column) { + this.frameTypes.pop(); + this.referencePositions.pop(); + return new LineFoldEnd(t0.getStartPosition()); + } + + const t1 = this.tokens.peek(2); + if (t0.kind === SyntaxKind.Dot && t0.getEndLine() < t1.getStartLine()) { + this.tokens.get(); + this.frameTypes.push(FrameType.Block); + return new BlockStart(t0.getStartPosition()); + } + + // If we got here, this is an ordinary token that is part of the + // line-fold. Make sure to consume it and return it to the caller. + this.tokens.get(); + return t0; + } + + case FrameType.Block: + { + + if (t0.getStartColumn() <= refPos.column) { + + // We only get here if the current token is less indented than the + // current reference token. Pop the block indicator and leave the + // reference position be for the edge case where the parent line-fold + // continues after the block. + this.frameTypes.pop(); + return new BlockEnd(t0.getStartPosition()); + + } + + this.frameTypes.push(FrameType.LineFold); + this.referencePositions.push(t0.getStartPosition()); + + // In theory, we could explictly issue a LineFoldStart and let all + // tokens be passed through in the FrameType.LineFold case. It does add + // more logic to the parser for no real benefit, which is why it was + // omitted. + this.tokens.get(); + return t0; + } + + } + + } + +} + + diff --git a/src/util.ts b/src/util.ts new file mode 100644 index 000000000..371f0b33d --- /dev/null +++ b/src/util.ts @@ -0,0 +1,64 @@ + + +export class MultiDict { + + private mapping = new Map(); + + public constructor(iterable?: Iterable<[K, V]>) { + if (iterable) { + for (const [key, value] of iterable) { + this.add(key, value); + } + } + } + + public get(key: K): Iterable { + return this.mapping.get(key) ?? []; + } + + public add(key: K, value: V): void { + const values = this.mapping.get(key); + if (values) { + values.push(value); + } else { + this.mapping.set(key, [ value ]) + } + } + + public *[Symbol.iterator](): Iterator<[K, V]> { + for (const [key, values] of this.mapping) { + for (const value of values) { + yield [key, value]; + } + } + } + +} + +export interface Stream { + get(): T; + peek(offset?: number): T; +} + +export abstract class BufferedStream { + + private buffer: Array = []; + + public abstract read(): T; + + public get(): T { + if (this.buffer.length > 0) { + return this.buffer.shift()!; + } + return this.read(); + } + + public peek(offset = 1): T { + while (this.buffer.length < offset) { + this.buffer.push(this.read()); + } + return this.buffer[offset-1]; + } + +} + diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 000000000..a4c57943a --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2016", + "module": "commonjs", + "moduleResolution": "node", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./lib", + "removeComments": true, + "importHelpers": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true + } +}