samvv/bolt
1
0
Fork 0
Code Issues 24 Pull requests Projects Releases Packages Wiki Activity Actions

Refactor parser to pass around a 'forkable' TokenStream

Browse source
This commit is contained in:
Sam Vervaeck 2024-07-11 21:08:52 +02:00
parent aa23420c5f
commit 4e1771cf2a
Signed by: samvv
SSH key fingerprint: SHA256:dIg0ywU1OP+ZYifrYxy8c5esO72cIKB+4/9wkZj1VaY
3 changed files with 432 additions and 326 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

142
include/bolt/Parser.hpp
View file

@ -58,90 +58,134 @@ public:
}; };
class TokenStream {
std::vector<Token*>& Buffer;
std::size_t Offset;
public:
TokenStream(
std::vector<Token*>& Buffer,
std::size_t Offset = 0
): Buffer(Buffer), Offset(Offset) {}
std::size_t getAbsoluteOffset() const {
return Offset;
}
Token* peek(std::size_t I = 0) {
auto RealOffset = Offset + I;
if (RealOffset >= Buffer.size()) {
return Buffer.back();
}
return Buffer[RealOffset];
}
TokenStream fork() {
return TokenStream { Buffer, Offset };
}
void skip(std::size_t Count) {
Offset = std::min(Buffer.size()-1, Offset + Count);
}
Token* get() {
auto Tok = Buffer[Offset];
if (Offset+1 < Buffer.size()) {
++Offset;
}
return Tok;
}
};
class Parser { class Parser {
TextFile& File; TextFile& File;
DiagnosticEngine& DE; DiagnosticEngine& DE;
Stream<Token*>& Tokens;
OperatorTable ExprOperators; OperatorTable ExprOperators;
Token* peekFirstTokenAfterAnnotationsAndModifiers(); std::optional<std::pair<std::size_t, std::vector<Annotation*>>> CachedAnnotations;
Token* expectToken(NodeKind Ty); void cacheAnnotations(TokenStream& Tokens);
std::vector<RecordDeclarationField*> parseRecordDeclarationFields(); Token* peekTokenAfterAnnotations(TokenStream& Tokens);
std::vector<std::tuple<RecordPatternField*, Comma*>> parseRecordPatternFields(); Token* peekTokenAfterAnnotationsAndModifiers(TokenStream& Tokens);
std::vector<RecordDeclarationField*> parseRecordDeclarationFields(TokenStream& Tokens);
std::vector<std::tuple<RecordPatternField*, Comma*>> parseRecordPatternFields(TokenStream& Tokens);
template<typename T> template<typename T>
T* expectToken(); T* expectToken(TokenStream& Tokens);
Expression* parseInfixOperatorAfterExpression(Expression* LHS, int MinPrecedence); Expression* parseInfixOperatorAfterExpression(TokenStream& Tokens, Expression* LHS, int MinPrecedence);
MatchExpression* parseMatchExpression(); MatchExpression* parseMatchExpression(TokenStream& Tokens);
Expression* parseMemberExpression(); Expression* parseMemberExpression(TokenStream& Tokens);
RecordExpression* parseRecordExpression(); RecordExpression* parseRecordExpression(TokenStream& Tokens);
Expression* parsePrimitiveExpression(); Expression* parsePrimitiveExpression(TokenStream& Tokens);
ConstraintExpression* parseConstraintExpression(); ConstraintExpression* parseConstraintExpression(TokenStream& Tokens);
TypeExpression* parseAppTypeExpression(); TypeExpression* parseAppTypeExpression(TokenStream& Tokens);
TypeExpression* parsePrimitiveTypeExpression(); TypeExpression* parsePrimitiveTypeExpression(TokenStream& Tokens);
TypeExpression* parseQualifiedTypeExpression(); TypeExpression* parseQualifiedTypeExpression(TokenStream& Tokens);
TypeExpression* parseArrowTypeExpression(); TypeExpression* parseArrowTypeExpression(TokenStream& Tokens);
VarTypeExpression* parseVarTypeExpression(); VarTypeExpression* parseVarTypeExpression(TokenStream& Tokens);
ReferenceTypeExpression* parseReferenceTypeExpression(); ReferenceTypeExpression* parseReferenceTypeExpression(TokenStream& Tokens);
std::vector<Annotation*> parseAnnotations(); std::vector<Annotation*> parseAnnotations(TokenStream& Tokens);
void checkLineFoldEnd(); void checkLineFoldEnd(TokenStream& Tokens);
void skipPastLineFoldEnd(); void skipPastLineFoldEnd(TokenStream& Tokens);
void skipToRBrace(); void skipToRBrace(TokenStream& Tokens);
public: public:
Parser(TextFile& File, Stream<Token*>& S, DiagnosticEngine& DE); Parser(TextFile& File, DiagnosticEngine& DE);
TypeExpression* parseTypeExpression(); TypeExpression* parseTypeExpression(TokenStream& Tokens);
ListPattern* parseListPattern(); ListPattern* parseListPattern(TokenStream& Tokens);
Pattern* parsePrimitivePattern(bool IsNarrow); Pattern* parsePrimitivePattern(TokenStream& Tokkens, bool IsNarrow);
Pattern* parseWidePattern(); Pattern* parseWidePattern(TokenStream& Tokens);
Pattern* parseNarrowPattern(); Pattern* parseNarrowPattern(TokenStream& Tokens);
Parameter* parseParam(); Parameter* parseParam(TokenStream& Tokens);
FunctionExpression* parseFunctionExpression(); LiteralExpression* parseLiteralExpression(TokenStream& Tokens);
ReferenceExpression* parseReferenceExpression(); FunctionExpression* parseFunctionExpression(TokenStream& Tokens);
Expression* parseUnaryExpression(); ReferenceExpression* parseReferenceExpression(TokenStream& Tokens);
Expression* parseExpression(); Expression* parseUnaryExpression(TokenStream& Tokens);
BlockExpression* parseBlockExpression(std::vector<Annotation*> Annotations = {}); Expression* parseExpression(TokenStream& Tokens);
Expression* parseCallExpression(); BlockExpression* parseBlockExpression(TokenStream& Tokens);
IfExpression* parseIfExpression(); Expression* parseCallExpression(TokenStream& Tokens);
IfExpression* parseIfExpression(TokenStream& Tokens);
ReturnExpression* parseReturnExpression(); ReturnExpression* parseReturnExpression(TokenStream& Tokens);
Expression* parseExpressionStatement(); Expression* parseExpressionStatement(TokenStream& Tokens);
Node* parseLetBodyElement(); Node* parseLetBodyElement(TokenStream& Tokens);
FunctionDeclaration* parseFunctionDeclaration(); FunctionDeclaration* parseFunctionDeclaration(TokenStream& Tokens);
VariableDeclaration* parseVariableDeclaration(); VariableDeclaration* parseVariableDeclaration(TokenStream& Tokens);
Node* parseClassElement(); Node* parseClassElement(TokenStream& Tokens);
ClassDeclaration* parseClassDeclaration(); ClassDeclaration* parseClassDeclaration(TokenStream& Tokens);
InstanceDeclaration* parseInstanceDeclaration(); InstanceDeclaration* parseInstanceDeclaration(TokenStream& Tokens);
RecordDeclaration* parseRecordDeclaration(); RecordDeclaration* parseRecordDeclaration(TokenStream& Tokens);
VariantDeclaration* parseVariantDeclaration(); VariantDeclaration* parseVariantDeclaration(TokenStream& Tokens);
Node* parseSourceElement(); Node* parseSourceElement(TokenStream& Tokens);
SourceFile* parseSourceFile(); SourceFile* parseSourceFile(TokenStream& Tokens);
}; };

593
src/Parser.cc
View file

File diff suppressed because it is too large Load diff

23
src/main.cc
View file

@ -44,6 +44,18 @@ ByteString readFile(std::string Path) {
namespace po = zen::po; namespace po = zen::po;
auto getAllTokens(Stream<Token*>& S) {
std::vector<Token*> Tokens;
for (;;) {
auto Tok = S.get();
Tokens.push_back(Tok);
if (Tok->getKind() == NodeKind::EndOfFile) {
break;
}
}
return Tokens;
}
int main(int Argc, const char* Argv[]) { int main(int Argc, const char* Argv[]) {
auto Match = po::program("bolt", "The offical compiler for the Bolt programming language") auto Match = po::program("bolt", "The offical compiler for the Bolt programming language")
@ -84,9 +96,11 @@ int main(int Argc, const char* Argv[]) {
VectorStream<ByteString, Char> Chars(Text, EOF); VectorStream<ByteString, Char> Chars(Text, EOF);
Scanner S(DE, File, Chars); Scanner S(DE, File, Chars);
Punctuator PT(S); Punctuator PT(S);
Parser P(File, PT, DE); auto Buffer = getAllTokens(PT);
Parser P(File, DE);
TokenStream Tokens { Buffer };
auto SF = P.parseSourceFile(); auto SF = P.parseSourceFile(Tokens);
if (SF == nullptr) { if (SF == nullptr) {
continue; continue;
} }
@ -112,11 +126,12 @@ int main(int Argc, const char* Argv[]) {
void visitExpression(Expression* N) { void visitExpression(Expression* N) {
for (auto A: N->Annotations) { for (auto A: N->Annotations) {
if (A->getKind() == NodeKind::TypeAssertAnnotation) { if (A->getKind() == NodeKind::TypeAssertAnnotation) {
auto TA = static_cast<TypeAssertAnnotation*>(A);
auto Left = C.getTypeOfNode(N); auto Left = C.getTypeOfNode(N);
auto Right = static_cast<TypeAssertAnnotation*>(A)->getTypeExpression()->getType(); auto Right = TA->getTypeExpression()->getType();
std::cerr << "verify " << Left->toString() << " == " << Right->toString() << std::endl; std::cerr << "verify " << Left->toString() << " == " << Right->toString() << std::endl;
if (*Left != *Right) { if (*Left != *Right) {
DE.add<TypeMismatchError>(Left, Right, A); DE.add<TypeMismatchError>(Left, Right, TA->getTypeExpression());
} }
} }
} }