From 556fc28eb70d4f538bc2214c5cdc91d1cd5ba254 Mon Sep 17 00:00:00 2001 From: Sam Vervaeck Date: Wed, 10 Jul 2024 23:09:45 +0200 Subject: [PATCH] Add parsing support for function expressions and format Parser.cc --- include/bolt/CST.hpp | 67 +++- include/bolt/CSTVisitor.hpp | 355 ++++++++-------- include/bolt/Parser.hpp | 3 +- src/CST.cc | 12 + src/Parser.cc | 777 ++++++++++++++++++++++++++++++++---- src/Scanner.cc | 3 + 6 files changed, 964 insertions(+), 253 deletions(-) diff --git a/include/bolt/CST.hpp b/include/bolt/CST.hpp index 267c3e7e5..f36209a90 100644 --- a/include/bolt/CST.hpp +++ b/include/bolt/CST.hpp @@ -10,6 +10,7 @@ #include #include "zen/config.hpp" +#include "zen/range.hpp" #include "bolt/Common.hpp" #include "bolt/Integer.hpp" @@ -92,6 +93,7 @@ enum class NodeKind { // Plain tokens Assignment, At, + Backslash, Colon, Comma, CustomOperator, @@ -150,14 +152,14 @@ enum class NodeKind { TypeclassConstraintExpression, EqualityConstraintExpression, - RecordTypeExpression, + RecordTypeExpressionField, // Type expressions AppTypeExpression, ArrowTypeExpression, NestedTypeExpression, QualifiedTypeExpression, - RecordTypeExpressionField, + RecordTypeExpression, ReferenceTypeExpression, TupleTypeExpression, VarTypeExpression, @@ -181,6 +183,7 @@ enum class NodeKind { // Expressions BlockExpression, CallExpression, + FunctionExpression, IfExpression, InfixExpression, LiteralExpression, @@ -471,6 +474,18 @@ public: }; +class Backslash : public Token { +public: + + inline Backslash(TextLoc StartLoc): + Token(NodeKind::Backslash, StartLoc) {} + + std::string getText() const override; + + static constexpr const NodeKind Kind = NodeKind::Backslash; + +}; + class DoKeyword : public Token { public: @@ -1747,6 +1762,7 @@ public: return N->getKind() == NodeKind::ReferenceExpression || N->getKind() == NodeKind::NestedExpression || N->getKind() == NodeKind::CallExpression + || N->getKind() == NodeKind::FunctionExpression || N->getKind() == NodeKind::TupleExpression || N->getKind() == NodeKind::InfixExpression || N->getKind() == NodeKind::RecordExpression @@ -2043,6 +2059,53 @@ public: }; +class FunctionExpression : public Expression { + +public: + + Backslash* Backslash; + std::vector Params; + RArrow* RArrow; + Expression* E; + + inline FunctionExpression( + std::vector Annotations, + class Backslash* Backslash, + std::vector Params, + class RArrow* RArrow, + class Expression* E + ): Expression(NodeKind::FunctionExpression, Annotations), + Backslash(Backslash), + Params(Params), + RArrow(RArrow), + E(E) {} + + inline FunctionExpression( + class Backslash* Backslash, + std::vector Params, + class RArrow* RArrow, + class Expression* Expression + ): FunctionExpression({}, Backslash, Params, RArrow, Expression) {} + + std::size_t countParams() { + return Params.size(); + } + + auto getParameters() { + return zen::make_iterator_range(Params); + } + + Expression* getExpression() const { + return E; + } + + Token* getFirstToken() const override; + Token* getLastToken() const override; + + static constexpr const NodeKind Kind = NodeKind::FunctionExpression; + +}; + class InfixExpression : public Expression { public: diff --git a/include/bolt/CSTVisitor.hpp b/include/bolt/CSTVisitor.hpp index 462943f23..73c0da596 100644 --- a/include/bolt/CSTVisitor.hpp +++ b/include/bolt/CSTVisitor.hpp @@ -20,106 +20,108 @@ public: #define BOLT_VISIT_OPERATOR(node) static_cast(this)->dispatchOperator(node) switch (N->getKind()) { - BOLT_GEN_CASE(VBar) - BOLT_GEN_CASE(Equals) + BOLT_GEN_CASE(AppTypeExpression) + BOLT_GEN_CASE(ArrowTypeExpression) + BOLT_GEN_CASE(Assignment) + BOLT_GEN_CASE(At) + BOLT_GEN_CASE(Backslash) + BOLT_GEN_CASE(BindPattern) + BOLT_GEN_CASE(BlockEnd) + BOLT_GEN_CASE(BlockExpression) + BOLT_GEN_CASE(BlockStart) + BOLT_GEN_CASE(CallExpression) + BOLT_GEN_CASE(ClassDeclaration) + BOLT_GEN_CASE(ClassKeyword) BOLT_GEN_CASE(Colon) BOLT_GEN_CASE(Comma) + BOLT_GEN_CASE(CustomOperator) + BOLT_GEN_CASE(DoKeyword) BOLT_GEN_CASE(Dot) BOLT_GEN_CASE(DotDot) - BOLT_GEN_CASE(Tilde) - BOLT_GEN_CASE(At) - BOLT_GEN_CASE(DoKeyword) - BOLT_GEN_CASE(LParen) - BOLT_GEN_CASE(RParen) - BOLT_GEN_CASE(LBracket) - BOLT_GEN_CASE(RBracket) - BOLT_GEN_CASE(LBrace) - BOLT_GEN_CASE(RBrace) - BOLT_GEN_CASE(RArrow) - BOLT_GEN_CASE(RArrowAlt) - BOLT_GEN_CASE(LetKeyword) - BOLT_GEN_CASE(ForeignKeyword) - BOLT_GEN_CASE(MutKeyword) - BOLT_GEN_CASE(PubKeyword) - BOLT_GEN_CASE(TypeKeyword) - BOLT_GEN_CASE(ReturnKeyword) - BOLT_GEN_CASE(ModKeyword) - BOLT_GEN_CASE(StructKeyword) - BOLT_GEN_CASE(EnumKeyword) - BOLT_GEN_CASE(FnKeyword) - BOLT_GEN_CASE(ClassKeyword) - BOLT_GEN_CASE(InstanceKeyword) BOLT_GEN_CASE(ElifKeyword) - BOLT_GEN_CASE(IfKeyword) BOLT_GEN_CASE(ElseKeyword) - BOLT_GEN_CASE(MatchKeyword) - BOLT_GEN_CASE(Invalid) BOLT_GEN_CASE(EndOfFile) - BOLT_GEN_CASE(BlockStart) - BOLT_GEN_CASE(BlockEnd) - BOLT_GEN_CASE(LineFoldEnd) - BOLT_GEN_CASE(CustomOperator) - BOLT_GEN_CASE(Assignment) + BOLT_GEN_CASE(EnumKeyword) + BOLT_GEN_CASE(EqualityConstraintExpression) + BOLT_GEN_CASE(Equals) + BOLT_GEN_CASE(ExpressionAnnotation) + BOLT_GEN_CASE(FnKeyword) + BOLT_GEN_CASE(ForeignKeyword) + BOLT_GEN_CASE(FunctionExpression) BOLT_GEN_CASE(Identifier) BOLT_GEN_CASE(IdentifierAlt) - BOLT_GEN_CASE(WrappedOperator) - BOLT_GEN_CASE(StringLiteral) - BOLT_GEN_CASE(IntegerLiteral) - BOLT_GEN_CASE(ExpressionAnnotation) - BOLT_GEN_CASE(TypeAssertAnnotation) - BOLT_GEN_CASE(TypeclassConstraintExpression) - BOLT_GEN_CASE(EqualityConstraintExpression) - BOLT_GEN_CASE(RecordTypeExpressionField) - BOLT_GEN_CASE(RecordTypeExpression) - BOLT_GEN_CASE(QualifiedTypeExpression) - BOLT_GEN_CASE(ReferenceTypeExpression) - BOLT_GEN_CASE(ArrowTypeExpression) - BOLT_GEN_CASE(AppTypeExpression) - BOLT_GEN_CASE(VarTypeExpression) - BOLT_GEN_CASE(NestedTypeExpression) - BOLT_GEN_CASE(TupleTypeExpression) - BOLT_GEN_CASE(BindPattern) - BOLT_GEN_CASE(LiteralPattern) - BOLT_GEN_CASE(RecordPatternField) - BOLT_GEN_CASE(RecordPattern) - BOLT_GEN_CASE(NamedRecordPattern) - BOLT_GEN_CASE(NamedTuplePattern) - BOLT_GEN_CASE(TuplePattern) - BOLT_GEN_CASE(NestedPattern) - BOLT_GEN_CASE(ListPattern) - BOLT_GEN_CASE(ReferenceExpression) - BOLT_GEN_CASE(MatchCase) - BOLT_GEN_CASE(MatchExpression) - BOLT_GEN_CASE(BlockExpression) - BOLT_GEN_CASE(MemberExpression) - BOLT_GEN_CASE(TupleExpression) - BOLT_GEN_CASE(NestedExpression) - BOLT_GEN_CASE(LiteralExpression) - BOLT_GEN_CASE(CallExpression) - BOLT_GEN_CASE(InfixExpression) - BOLT_GEN_CASE(PrefixExpression) - BOLT_GEN_CASE(RecordExpressionField) - BOLT_GEN_CASE(RecordExpression) - BOLT_GEN_CASE(ReturnExpression) BOLT_GEN_CASE(IfExpression) BOLT_GEN_CASE(IfExpressionPart) - BOLT_GEN_CASE(TypeAssert) - BOLT_GEN_CASE(Parameter) + BOLT_GEN_CASE(IfKeyword) + BOLT_GEN_CASE(InfixExpression) + BOLT_GEN_CASE(InfixFunctionDeclaration) + BOLT_GEN_CASE(InstanceDeclaration) + BOLT_GEN_CASE(InstanceKeyword) + BOLT_GEN_CASE(IntegerLiteral) + BOLT_GEN_CASE(Invalid) + BOLT_GEN_CASE(LBrace) + BOLT_GEN_CASE(LBracket) + BOLT_GEN_CASE(LParen) BOLT_GEN_CASE(LetBlockBody) BOLT_GEN_CASE(LetExprBody) - BOLT_GEN_CASE(PrefixFunctionDeclaration) - BOLT_GEN_CASE(InfixFunctionDeclaration) - BOLT_GEN_CASE(SuffixFunctionDeclaration) + BOLT_GEN_CASE(LetKeyword) + BOLT_GEN_CASE(LineFoldEnd) + BOLT_GEN_CASE(ListPattern) + BOLT_GEN_CASE(LiteralExpression) + BOLT_GEN_CASE(LiteralPattern) + BOLT_GEN_CASE(MatchCase) + BOLT_GEN_CASE(MatchExpression) + BOLT_GEN_CASE(MatchKeyword) + BOLT_GEN_CASE(MemberExpression) + BOLT_GEN_CASE(ModKeyword) + BOLT_GEN_CASE(MutKeyword) BOLT_GEN_CASE(NamedFunctionDeclaration) - BOLT_GEN_CASE(VariableDeclaration) + BOLT_GEN_CASE(NamedRecordPattern) + BOLT_GEN_CASE(NamedTuplePattern) + BOLT_GEN_CASE(NestedExpression) + BOLT_GEN_CASE(NestedPattern) + BOLT_GEN_CASE(NestedTypeExpression) + BOLT_GEN_CASE(Parameter) + BOLT_GEN_CASE(PrefixExpression) + BOLT_GEN_CASE(PrefixFunctionDeclaration) + BOLT_GEN_CASE(PubKeyword) + BOLT_GEN_CASE(QualifiedTypeExpression) + BOLT_GEN_CASE(RArrow) + BOLT_GEN_CASE(RArrowAlt) + BOLT_GEN_CASE(RBrace) + BOLT_GEN_CASE(RBracket) + BOLT_GEN_CASE(RParen) BOLT_GEN_CASE(RecordDeclaration) BOLT_GEN_CASE(RecordDeclarationField) - BOLT_GEN_CASE(VariantDeclaration) - BOLT_GEN_CASE(TupleVariantDeclarationMember) + BOLT_GEN_CASE(RecordExpression) + BOLT_GEN_CASE(RecordExpressionField) + BOLT_GEN_CASE(RecordPattern) + BOLT_GEN_CASE(RecordPatternField) + BOLT_GEN_CASE(RecordTypeExpression) + BOLT_GEN_CASE(RecordTypeExpressionField) BOLT_GEN_CASE(RecordVariantDeclarationMember) - BOLT_GEN_CASE(ClassDeclaration) - BOLT_GEN_CASE(InstanceDeclaration) + BOLT_GEN_CASE(ReferenceExpression) + BOLT_GEN_CASE(ReferenceTypeExpression) + BOLT_GEN_CASE(ReturnExpression) + BOLT_GEN_CASE(ReturnKeyword) BOLT_GEN_CASE(SourceFile) + BOLT_GEN_CASE(StringLiteral) + BOLT_GEN_CASE(StructKeyword) + BOLT_GEN_CASE(SuffixFunctionDeclaration) + BOLT_GEN_CASE(Tilde) + BOLT_GEN_CASE(TupleExpression) + BOLT_GEN_CASE(TuplePattern) + BOLT_GEN_CASE(TupleTypeExpression) + BOLT_GEN_CASE(TupleVariantDeclarationMember) + BOLT_GEN_CASE(TypeAssert) + BOLT_GEN_CASE(TypeAssertAnnotation) + BOLT_GEN_CASE(TypeKeyword) + BOLT_GEN_CASE(TypeclassConstraintExpression) + BOLT_GEN_CASE(VBar) + BOLT_GEN_CASE(VarTypeExpression) + BOLT_GEN_CASE(VariableDeclaration) + BOLT_GEN_CASE(VariantDeclaration) + BOLT_GEN_CASE(WrappedOperator) } } @@ -194,6 +196,10 @@ protected: static_cast(this)->visitToken(N); } + void visitBackslash(Backslash* N) { + static_cast(this)->visitToken(N); + } + void visitDoKeyword(DoKeyword* N) { static_cast(this)->visitToken(N); } @@ -486,6 +492,10 @@ protected: static_cast(this)->visitExpression(N); } + void visitFunctionExpression(FunctionExpression* N) { + static_cast(this)->visitExpression(N); + } + void visitInfixExpression(InfixExpression* N) { static_cast(this)->visitExpression(N); } @@ -604,106 +614,108 @@ public: break; switch (N->getKind()) { - BOLT_GEN_CHILD_CASE(VBar) - BOLT_GEN_CHILD_CASE(Equals) + BOLT_GEN_CHILD_CASE(AppTypeExpression) + BOLT_GEN_CHILD_CASE(ArrowTypeExpression) + BOLT_GEN_CHILD_CASE(Assignment) + BOLT_GEN_CHILD_CASE(At) + BOLT_GEN_CHILD_CASE(Backslash) + BOLT_GEN_CHILD_CASE(BindPattern) + BOLT_GEN_CHILD_CASE(BlockEnd) + BOLT_GEN_CHILD_CASE(BlockExpression) + BOLT_GEN_CHILD_CASE(BlockStart) + BOLT_GEN_CHILD_CASE(CallExpression) + BOLT_GEN_CHILD_CASE(ClassDeclaration) + BOLT_GEN_CHILD_CASE(ClassKeyword) BOLT_GEN_CHILD_CASE(Colon) BOLT_GEN_CHILD_CASE(Comma) + BOLT_GEN_CHILD_CASE(CustomOperator) + BOLT_GEN_CHILD_CASE(DoKeyword) BOLT_GEN_CHILD_CASE(Dot) BOLT_GEN_CHILD_CASE(DotDot) - BOLT_GEN_CHILD_CASE(Tilde) - BOLT_GEN_CHILD_CASE(At) - BOLT_GEN_CHILD_CASE(DoKeyword) - BOLT_GEN_CHILD_CASE(LParen) - BOLT_GEN_CHILD_CASE(RParen) - BOLT_GEN_CHILD_CASE(LBracket) - BOLT_GEN_CHILD_CASE(RBracket) - BOLT_GEN_CHILD_CASE(LBrace) - BOLT_GEN_CHILD_CASE(RBrace) - BOLT_GEN_CHILD_CASE(RArrow) - BOLT_GEN_CHILD_CASE(RArrowAlt) - BOLT_GEN_CHILD_CASE(LetKeyword) - BOLT_GEN_CHILD_CASE(ForeignKeyword) - BOLT_GEN_CHILD_CASE(MutKeyword) - BOLT_GEN_CHILD_CASE(PubKeyword) - BOLT_GEN_CHILD_CASE(TypeKeyword) - BOLT_GEN_CHILD_CASE(ReturnKeyword) - BOLT_GEN_CHILD_CASE(ModKeyword) - BOLT_GEN_CHILD_CASE(StructKeyword) - BOLT_GEN_CHILD_CASE(EnumKeyword) - BOLT_GEN_CHILD_CASE(FnKeyword) - BOLT_GEN_CHILD_CASE(ClassKeyword) - BOLT_GEN_CHILD_CASE(InstanceKeyword) BOLT_GEN_CHILD_CASE(ElifKeyword) - BOLT_GEN_CHILD_CASE(IfKeyword) BOLT_GEN_CHILD_CASE(ElseKeyword) - BOLT_GEN_CHILD_CASE(MatchKeyword) - BOLT_GEN_CHILD_CASE(Invalid) BOLT_GEN_CHILD_CASE(EndOfFile) - BOLT_GEN_CHILD_CASE(BlockStart) - BOLT_GEN_CHILD_CASE(BlockEnd) - BOLT_GEN_CHILD_CASE(LineFoldEnd) - BOLT_GEN_CHILD_CASE(CustomOperator) - BOLT_GEN_CHILD_CASE(Assignment) + BOLT_GEN_CHILD_CASE(EnumKeyword) + BOLT_GEN_CHILD_CASE(EqualityConstraintExpression) + BOLT_GEN_CHILD_CASE(Equals) + BOLT_GEN_CHILD_CASE(ExpressionAnnotation) + BOLT_GEN_CHILD_CASE(FnKeyword) + BOLT_GEN_CHILD_CASE(ForeignKeyword) + BOLT_GEN_CHILD_CASE(FunctionExpression) BOLT_GEN_CHILD_CASE(Identifier) BOLT_GEN_CHILD_CASE(IdentifierAlt) - BOLT_GEN_CHILD_CASE(WrappedOperator) - BOLT_GEN_CHILD_CASE(StringLiteral) - BOLT_GEN_CHILD_CASE(IntegerLiteral) - BOLT_GEN_CHILD_CASE(ExpressionAnnotation) - BOLT_GEN_CHILD_CASE(TypeAssertAnnotation) - BOLT_GEN_CHILD_CASE(TypeclassConstraintExpression) - BOLT_GEN_CHILD_CASE(EqualityConstraintExpression) - BOLT_GEN_CHILD_CASE(RecordTypeExpressionField) - BOLT_GEN_CHILD_CASE(RecordTypeExpression) - BOLT_GEN_CHILD_CASE(QualifiedTypeExpression) - BOLT_GEN_CHILD_CASE(ReferenceTypeExpression) - BOLT_GEN_CHILD_CASE(ArrowTypeExpression) - BOLT_GEN_CHILD_CASE(AppTypeExpression) - BOLT_GEN_CHILD_CASE(VarTypeExpression) - BOLT_GEN_CHILD_CASE(NestedTypeExpression) - BOLT_GEN_CHILD_CASE(TupleTypeExpression) - BOLT_GEN_CHILD_CASE(BindPattern) - BOLT_GEN_CHILD_CASE(LiteralPattern) - BOLT_GEN_CHILD_CASE(RecordPatternField) - BOLT_GEN_CHILD_CASE(RecordPattern) - BOLT_GEN_CHILD_CASE(NamedRecordPattern) - BOLT_GEN_CHILD_CASE(NamedTuplePattern) - BOLT_GEN_CHILD_CASE(TuplePattern) - BOLT_GEN_CHILD_CASE(NestedPattern) - BOLT_GEN_CHILD_CASE(ListPattern) - BOLT_GEN_CHILD_CASE(ReferenceExpression) - BOLT_GEN_CHILD_CASE(MatchCase) - BOLT_GEN_CHILD_CASE(MatchExpression) - BOLT_GEN_CHILD_CASE(BlockExpression) - BOLT_GEN_CHILD_CASE(MemberExpression) - BOLT_GEN_CHILD_CASE(TupleExpression) - BOLT_GEN_CHILD_CASE(NestedExpression) - BOLT_GEN_CHILD_CASE(LiteralExpression) - BOLT_GEN_CHILD_CASE(CallExpression) - BOLT_GEN_CHILD_CASE(InfixExpression) - BOLT_GEN_CHILD_CASE(PrefixExpression) - BOLT_GEN_CHILD_CASE(RecordExpressionField) - BOLT_GEN_CHILD_CASE(RecordExpression) - BOLT_GEN_CHILD_CASE(ReturnExpression) BOLT_GEN_CHILD_CASE(IfExpression) BOLT_GEN_CHILD_CASE(IfExpressionPart) - BOLT_GEN_CHILD_CASE(TypeAssert) - BOLT_GEN_CHILD_CASE(Parameter) + BOLT_GEN_CHILD_CASE(IfKeyword) + BOLT_GEN_CHILD_CASE(InfixExpression) + BOLT_GEN_CHILD_CASE(InfixFunctionDeclaration) + BOLT_GEN_CHILD_CASE(InstanceDeclaration) + BOLT_GEN_CHILD_CASE(InstanceKeyword) + BOLT_GEN_CHILD_CASE(IntegerLiteral) + BOLT_GEN_CHILD_CASE(Invalid) + BOLT_GEN_CHILD_CASE(LBrace) + BOLT_GEN_CHILD_CASE(LBracket) + BOLT_GEN_CHILD_CASE(LParen) BOLT_GEN_CHILD_CASE(LetBlockBody) BOLT_GEN_CHILD_CASE(LetExprBody) - BOLT_GEN_CHILD_CASE(PrefixFunctionDeclaration) - BOLT_GEN_CHILD_CASE(InfixFunctionDeclaration) - BOLT_GEN_CHILD_CASE(SuffixFunctionDeclaration) + BOLT_GEN_CHILD_CASE(LetKeyword) + BOLT_GEN_CHILD_CASE(LineFoldEnd) + BOLT_GEN_CHILD_CASE(ListPattern) + BOLT_GEN_CHILD_CASE(LiteralExpression) + BOLT_GEN_CHILD_CASE(LiteralPattern) + BOLT_GEN_CHILD_CASE(MatchCase) + BOLT_GEN_CHILD_CASE(MatchExpression) + BOLT_GEN_CHILD_CASE(MatchKeyword) + BOLT_GEN_CHILD_CASE(MemberExpression) + BOLT_GEN_CHILD_CASE(ModKeyword) + BOLT_GEN_CHILD_CASE(MutKeyword) BOLT_GEN_CHILD_CASE(NamedFunctionDeclaration) - BOLT_GEN_CHILD_CASE(VariableDeclaration) + BOLT_GEN_CHILD_CASE(NamedRecordPattern) + BOLT_GEN_CHILD_CASE(NamedTuplePattern) + BOLT_GEN_CHILD_CASE(NestedExpression) + BOLT_GEN_CHILD_CASE(NestedPattern) + BOLT_GEN_CHILD_CASE(NestedTypeExpression) + BOLT_GEN_CHILD_CASE(Parameter) + BOLT_GEN_CHILD_CASE(PrefixExpression) + BOLT_GEN_CHILD_CASE(PrefixFunctionDeclaration) + BOLT_GEN_CHILD_CASE(PubKeyword) + BOLT_GEN_CHILD_CASE(QualifiedTypeExpression) + BOLT_GEN_CHILD_CASE(RArrow) + BOLT_GEN_CHILD_CASE(RArrowAlt) + BOLT_GEN_CHILD_CASE(RBrace) + BOLT_GEN_CHILD_CASE(RBracket) + BOLT_GEN_CHILD_CASE(RParen) BOLT_GEN_CHILD_CASE(RecordDeclaration) BOLT_GEN_CHILD_CASE(RecordDeclarationField) - BOLT_GEN_CHILD_CASE(VariantDeclaration) - BOLT_GEN_CHILD_CASE(TupleVariantDeclarationMember) + BOLT_GEN_CHILD_CASE(RecordExpression) + BOLT_GEN_CHILD_CASE(RecordExpressionField) + BOLT_GEN_CHILD_CASE(RecordPattern) + BOLT_GEN_CHILD_CASE(RecordPatternField) + BOLT_GEN_CHILD_CASE(RecordTypeExpression) + BOLT_GEN_CHILD_CASE(RecordTypeExpressionField) BOLT_GEN_CHILD_CASE(RecordVariantDeclarationMember) - BOLT_GEN_CHILD_CASE(ClassDeclaration) - BOLT_GEN_CHILD_CASE(InstanceDeclaration) + BOLT_GEN_CHILD_CASE(ReferenceExpression) + BOLT_GEN_CHILD_CASE(ReferenceTypeExpression) + BOLT_GEN_CHILD_CASE(ReturnExpression) + BOLT_GEN_CHILD_CASE(ReturnKeyword) BOLT_GEN_CHILD_CASE(SourceFile) + BOLT_GEN_CHILD_CASE(StringLiteral) + BOLT_GEN_CHILD_CASE(StructKeyword) + BOLT_GEN_CHILD_CASE(SuffixFunctionDeclaration) + BOLT_GEN_CHILD_CASE(Tilde) + BOLT_GEN_CHILD_CASE(TupleExpression) + BOLT_GEN_CHILD_CASE(TuplePattern) + BOLT_GEN_CHILD_CASE(TupleTypeExpression) + BOLT_GEN_CHILD_CASE(TupleVariantDeclarationMember) + BOLT_GEN_CHILD_CASE(TypeAssert) + BOLT_GEN_CHILD_CASE(TypeAssertAnnotation) + BOLT_GEN_CHILD_CASE(TypeKeyword) + BOLT_GEN_CHILD_CASE(TypeclassConstraintExpression) + BOLT_GEN_CHILD_CASE(VBar) + BOLT_GEN_CHILD_CASE(VarTypeExpression) + BOLT_GEN_CHILD_CASE(VariableDeclaration) + BOLT_GEN_CHILD_CASE(VariantDeclaration) + BOLT_GEN_CHILD_CASE(WrappedOperator) } } @@ -731,6 +743,9 @@ public: void visitEachChildImpl(At* N) { } + void visitEachChildImpl(Backslash* N) { + } + void visitEachChildImpl(DoKeyword* N) { } @@ -1125,6 +1140,18 @@ public: } } + void visitEachChildImpl(FunctionExpression* N) { + for (auto A: N->Annotations) { + BOLT_VISIT(A); + } + BOLT_VISIT(N->Backslash); + for (auto P: N->getParameters()) { + BOLT_VISIT(P); + } + BOLT_VISIT(N->RArrow); + BOLT_VISIT(N->getExpression()); + } + void visitEachChildImpl(InfixExpression* N) { for (auto A: N->Annotations) { BOLT_VISIT(A); diff --git a/include/bolt/Parser.hpp b/include/bolt/Parser.hpp index 29884e868..47a08a514 100644 --- a/include/bolt/Parser.hpp +++ b/include/bolt/Parser.hpp @@ -72,7 +72,7 @@ class Parser { Token* expectToken(NodeKind Ty); std::vector parseRecordDeclarationFields(); - std::optional>> parseRecordPatternFields(); + std::vector> parseRecordPatternFields(); template T* expectToken(); @@ -112,6 +112,7 @@ public: Parameter* parseParam(); + FunctionExpression* parseFunctionExpression(); ReferenceExpression* parseReferenceExpression(); Expression* parseUnaryExpression(); Expression* parseExpression(); diff --git a/src/CST.cc b/src/CST.cc index 919fdf5f1..f75d42ae0 100644 --- a/src/CST.cc +++ b/src/CST.cc @@ -492,6 +492,14 @@ Token* CallExpression::getLastToken() const { return Function->getLastToken(); } +Token* FunctionExpression::getFirstToken() const { + return Backslash; +} + +Token* FunctionExpression::getLastToken() const { + return E->getLastToken(); +} + Token* InfixExpression::getFirstToken() const { return Left->getFirstToken(); } @@ -775,6 +783,10 @@ Token* SourceFile::getLastToken() const { return nullptr; } +std::string Backslash::getText() const { + return "\\"; +} + std::string VBar::getText() const { return "|"; } diff --git a/src/Parser.cc b/src/Parser.cc index af0007160..1762d1ec4 100644 --- a/src/Parser.cc +++ b/src/Parser.cc @@ -35,6 +35,13 @@ namespace bolt { +enum class Expected { + PrimitiveExpression, + Expression, + IdentifierAlt, + Identifier, +}; + std::optional OperatorTable::getInfix(Token* T) { auto Match = Mapping.find(T->getText()); if (Match == Mapping.end() || !Match->second.isInfix()) { @@ -62,11 +69,6 @@ void OperatorTable::add(std::string Name, unsigned Flags, int Precedence) { Mapping.emplace(Name, OperatorInfo { Precedence, Flags }); } -#define BOLT_EACH_UNREF(nodes) \ - for (auto N: nodes) { \ - N->unref(); \ - } - Parser::Parser(TextFile& File, Stream& S, DiagnosticEngine& DE): File(File), Tokens(S), DE(DE) { ExprOperators.add("**", OperatorFlags_InfixR, 10); @@ -88,15 +90,21 @@ Parser::Parser(TextFile& File, Stream& S, DiagnosticEngine& DE): template T* Parser::expectToken() { - auto Tok = Tokens.peek(); + auto Tok = Tokens.get(); if (Tok->getKind() != T::Kind) { DE.add(File, Tok, std::vector { T::Kind }); return nullptr; } - Tokens.get(); return static_cast(Tok); } +template +void unrefAll(std::vector& Container) { + for (auto Element: Container) { + Element->unref(); + } +} + Token* Parser::peekFirstTokenAfterAnnotationsAndModifiers() { std::size_t I = 0; for (;;) { @@ -120,143 +128,255 @@ Token* Parser::peekFirstTokenAfterAnnotationsAndModifiers() { } ListPattern* Parser::parseListPattern() { + auto LBracket = expectToken(); if (!LBracket) { return nullptr; } + std::vector> Elements; + RBracket* RBracket; + auto T0 = Tokens.peek(); if (T0->getKind() == NodeKind::RBracket) { Tokens.get(); RBracket = static_cast(T0); goto finish; } + for (;;) { + auto P = parseWidePattern(); if (!P) { LBracket->unref(); - for (auto [Element, Separator]: Elements) { - Element->unref(); - Separator->unref(); + for (auto [Pattern, Comma]: Elements) { + Pattern->unref(); + if (Comma) { + Comma->unref(); + } } return nullptr; } + auto T1 = Tokens.peek(); + switch (T1->getKind()) { + case NodeKind::Comma: Tokens.get(); Elements.push_back(std::make_tuple(P, static_cast(T1))); break; + case NodeKind::RBracket: Tokens.get(); Elements.push_back(std::make_tuple(P, nullptr)); RBracket = static_cast(T1); goto finish; + default: - DE.add(File, T1, std::vector { NodeKind::Comma, NodeKind::RBracket }); + Tokens.get(); // Error recovery + DE.add(File, T1, std::vector { + NodeKind::Comma, + NodeKind::RBracket + }); } + } + finish: return new ListPattern { LBracket, Elements, RBracket }; } -std::optional>> Parser::parseRecordPatternFields() { +std::vector> Parser::parseRecordPatternFields() { + std::vector> Fields; + for (;;) { + auto T0 = Tokens.peek(); + + // If the next token on the stream is '}', then we are finished since this + // loop always cleanly starts at the beginning of a list element. + // The caller expects RBrace to be on the stream so we leave it there. if (T0->getKind() == NodeKind::RBrace) { break; } + + // We encountered the last element in a pattern: '..'. It may have an + // expression associated with it, which we must parse. if (T0->getKind() == NodeKind::DotDot) { + + // Parse the '..' Tokens.get(); auto DotDot = static_cast(T0); + auto T1 = Tokens.peek(); + + // If the '..' is immediately followed by '}' we can exit early. if (T1->getKind() == NodeKind::RBrace) { Fields.push_back(std::make_tuple(new RecordPatternField(DotDot), nullptr)); break; } + + // We only get here if there's still something interesting on the stream. + // Go parse it now. auto P = parseWidePattern(); - auto T2 = Tokens.peek(); - if (T2->getKind() != NodeKind::RBrace) { - DE.add(File, T2, std::vector { NodeKind::RBrace, NodeKind::Comma }); - return {}; + + if (P) { + + auto T2 = Tokens.peek(); + + // We parsed the rest pattern but there's still something left on the + // stream. That's an error. + if (T2->getKind() != NodeKind::RBrace) { + DE.add(File, T2, std::vector { NodeKind::RBrace }); + // We assume that this really was the last element on the stream. + break; + } + + // Everything OK; create the new field and stop looking for more elements. + // The caller expects RBrace to be on the stream so we leave it there. + Fields.push_back(std::make_tuple(new RecordPatternField(DotDot, P), nullptr)); + break; + + } else { + + // We failed to parse a pattern, so we'll act as if this was due to a small typo by the user. + // parseWidePattern() should have already advanced the stream so we don't have to. + continue; + } - Fields.push_back(std::make_tuple(new RecordPatternField(DotDot, P), nullptr)); - break; + } + + // If we get here we are NOT at a '..' pattern and also not at the end of + // the contents of the braces. + auto Name = expectToken(); + + if (!Name) { + // Something else than an identifier on the stream. Since we can't guess + // names, we ignore it and try again. + continue; + } + Equals* Equals = nullptr; Pattern* Pattern = nullptr; + auto T1 = Tokens.peek(); + + // The pattern on the stream certainly is not a punned field pattern, so + // assume that what follows will be '=' and an ordinary pattern. if (T1->getKind() == NodeKind::Equals) { + Tokens.get(); + Equals = static_cast(T1); Pattern = parseWidePattern(); + + // We failed to parse a pattern, so we'll act as if this was due to a small typo by the user. + // parseWidePattern() should have already advanced the stream so we don't have to. + if (!Pattern) { + continue; + } + } + auto Field = new RecordPatternField(Name, Equals, Pattern); + auto T2 = Tokens.peek(); + + // We break a second time here when encountering the same '}' because we + // want to push the right delimiter in the list of fields. Encountering a + // '}' means no delimiter was present. + // The caller expects RBrace to be on the stream so we leave it there. if (T2->getKind() == NodeKind::RBrace) { Fields.push_back(std::make_tuple(Field, nullptr)); break; } + + // If there's anything other than a '}' or ',' it's an error. if (T2->getKind() != NodeKind::Comma) { DE.add(File, T2, std::vector { NodeKind::RBrace, NodeKind::Comma }); - return {}; + // We assume the user forgot to place their ',' and continue parsing fields as if the ',' was present. + continue; } + + // The only case left is that our token is a ','. We consume it and add it + // in the list of fields. Tokens.get(); auto Comma = static_cast(T2); Fields.push_back(std::make_tuple(Field, Comma)); } + return Fields; } Pattern* Parser::parsePrimitivePattern(bool IsNarrow) { + auto T0 = Tokens.peek(); + switch (T0->getKind()) { + + // Parse literals such as 1 or "foo" case NodeKind::StringLiteral: case NodeKind::IntegerLiteral: Tokens.get(); return new LiteralPattern(static_cast(T0)); + + // Parse a very basic pattern case NodeKind::Identifier: Tokens.get(); return new BindPattern(static_cast(T0)); + + // Parse a record pattern with no name, such as { foo = 12, bar = True } case NodeKind::LBrace: { Tokens.get(); auto LBrace = static_cast(T0); auto Fields = parseRecordPatternFields(); - if (!Fields) { - LBrace->unref(); - skipToRBrace(); - return nullptr; - } auto RBrace = static_cast(Tokens.get()); - return new RecordPattern(LBrace, *Fields, RBrace); + return new RecordPattern(LBrace, Fields, RBrace); } + + // If we enounter an identifier of which the first letter is uppercase, then + // we must be dealing with some named pattern. After all, module paths such + // as Foo.Bar.bax don't make sense in BindPattern so those are excluded. case NodeKind::IdentifierAlt: { Tokens.get(); auto Name = static_cast(T0); + + // If 'spaces' aren't allowed (such as 'Foo a b'), then we must + // immediately return with whatever we were allowed to parse. if (IsNarrow) { return new NamedTuplePattern(Name, {}); } + auto T1 = Tokens.peek(); + + // If we encounter '{', we almost certainly are dealing with a named + // record pattern. These are parsed in much the same way as above, except + // that we return a named record pattern instead of an anonymous one. if (T1->getKind() == NodeKind::LBrace) { - auto LBrace = static_cast(T1); Tokens.get(); + auto LBrace = static_cast(T1); auto Fields = parseRecordPatternFields(); - if (!Fields) { - LBrace->unref(); - skipToRBrace(); - return nullptr; - } auto RBrace = static_cast(Tokens.get()); - return new NamedRecordPattern({}, Name, LBrace, *Fields, RBrace); + return new NamedRecordPattern({}, Name, LBrace, Fields, RBrace); } + + // From here on we assume we are dealing with a named tuple pattern. + std::vector Patterns; + for (;;) { + auto T2 = Tokens.peek(); + + // This check enables less 'primitive' patterns, such as a disjunctive + // pattern, to be parsed. if (T2->getKind() == NodeKind::RParen || T2->getKind() == NodeKind::RBracket || T2->getKind() == NodeKind::RBrace @@ -267,38 +387,51 @@ Pattern* Parser::parsePrimitivePattern(bool IsNarrow) { || T2->getKind() == NodeKind::RArrowAlt) { break; } + + // Parse one element of the named tuple pattern, e.g. the 'a' in 'Foo a b' auto P = parseNarrowPattern(); - if (!P) { - Name->unref(); - for (auto P: Patterns) { - P->unref(); - } - return nullptr; + + // A simple check ensures that no nullptrs are pushed in the tuple element list. + // parseNarrowPattern() returns a nullptr if it failed to parse a + // pattern, but it should still advance the stream (even on error). + if (P) { + Patterns.push_back(P); } - Patterns.push_back(P); + } return new NamedTuplePattern { Name, Patterns }; } + + // Parse a list pattern such as [ first, second, third ] case NodeKind::LBracket: return parseListPattern(); + + // Parse something between parentheses; either a nested pattern or a tuple pattern. case NodeKind::LParen: { Tokens.get(); auto LParen = static_cast(T0); + + // The basic tactic is as follows: assume a tuple pattern, parse as many + // tuple elements as possible, and in the case that there is only one then + // return a NestedPattern + std::vector> Elements; + RParen* RParen; for (;;) { + auto P = parseWidePattern(); + if (!P) { - LParen->unref(); - for (auto [P, Comma]: Elements) { - P->unref(); - Comma->unref(); - } - // TODO maybe skip to next comma? - return nullptr; + // If the tuple element failed to parse, simply ignore it and try again. + // parseNarrowPattern() returns a nullptr if it failed to parse a + // pattern, but it should still advance the stream (even on error). + continue;; } + auto T1 = Tokens.peek(); + if (T1->getKind() == NodeKind::Comma) { Tokens.get(); Elements.push_back(std::make_tuple(P, static_cast(T1))); @@ -309,23 +442,23 @@ Pattern* Parser::parsePrimitivePattern(bool IsNarrow) { break; } else { DE.add(File, T1, std::vector { NodeKind::Comma, NodeKind::RParen }); - LParen->unref(); - for (auto [P, Comma]: Elements) { - P->unref(); - Comma->unref(); - } - // TODO maybe skip to next comma? - return nullptr; - } + } + + // As promised, we reduce the tuple pattern to a simple nested pattern if + // it only contains one element. if (Elements.size() == 1) { return new NestedPattern { LParen, std::get<0>(Elements.front()), RParen }; } + return new TuplePattern(LParen, Elements, RParen); } + default: - // Tokens.get(); + // We must consume at least one token. Since we haven't consumed a single + // token yet, we do so now. + Tokens.get(); DE.add(File, T0, std::vector { NodeKind::Identifier, NodeKind::IdentifierAlt, @@ -352,6 +485,9 @@ TypeExpression* Parser::parseTypeExpression() { TypeExpression* Parser::parseQualifiedTypeExpression() { bool HasConstraints = false; + + // First we check if there is a '=>', indicating that there are (class) + // constraints that need to be parsed. auto T0 = Tokens.peek(); if (isa(T0)) { std::size_t I = 1; @@ -372,56 +508,75 @@ TypeExpression* Parser::parseQualifiedTypeExpression() { } } after_lookahead: + + // Skip the logic below if we can see that there is are no constraint expressions. if (!HasConstraints) { return parseArrowTypeExpression(); } + Tokens.get(); LParen* LParen = static_cast(T0); + std::vector> Constraints; + RParen* RParen; RArrowAlt* RArrowAlt; + auto T1 = Tokens.peek(); + + // Special case for when there are no actual constraints if (T1->getKind() == NodeKind::RParen) { Tokens.get(); RParen = static_cast(T1); goto after_constraints; } + for (;;) { + auto C = parseConstraintExpression(); + + if (!C) { + // If we couldn't parse the constraint expression then we assume that + // parseConstraintExpression() put the stream in a good position to + // continue parsing other constraints. + continue; + } + Comma* Comma = nullptr; + auto T2 = Tokens.get(); + switch (T2->getKind()) { case NodeKind::Comma: { auto Comma = static_cast(T2); - if (C) { - Constraints.push_back(std::make_tuple(C, Comma)); - } else { - Comma->unref(); - } + Constraints.push_back(std::make_tuple(C, Comma)); continue; } case NodeKind::RParen: RParen = static_cast(T2); - if (C) { - Constraints.push_back(std::make_tuple(C, nullptr)); - } + Constraints.push_back(std::make_tuple(C, nullptr)); goto after_constraints; default: DE.add(File, T2, std::vector { NodeKind::Comma, NodeKind::RArrowAlt }); - return nullptr; } } + after_constraints: + RArrowAlt = expectToken(); if (!RArrowAlt) { LParen->unref(); for (auto [CE, Comma]: Constraints) { CE->unref(); + if (Comma) { + Comma->unref(); + } } RParen->unref(); return nullptr; } + auto TE = parseArrowTypeExpression(); if (!TE) { LParen->unref(); @@ -435,59 +590,81 @@ after_constraints: RArrowAlt->unref(); return nullptr; } - return new QualifiedTypeExpression(Constraints, RArrowAlt, TE); + + return new QualifiedTypeExpression { Constraints, RArrowAlt, TE }; } TypeExpression* Parser::parsePrimitiveTypeExpression() { + auto T0 = Tokens.peek(); + switch (T0->getKind()) { + case NodeKind::Identifier: return parseVarTypeExpression(); + case NodeKind::LBrace: { Tokens.get(); auto LBrace = static_cast(T0); + std::vector> Fields; + VBar* VBar = nullptr; TypeExpression* Rest = nullptr; + for (;;) { + auto T1 = Tokens.peek(); + if (T1->getKind() == NodeKind::RBrace) { break; } + auto Name = expectToken(); if (Name == nullptr) { for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } return nullptr; } + auto Colon = expectToken(); if (Colon == nullptr) { for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } Name->unref(); return nullptr; } + auto TE = parseTypeExpression(); if (TE == nullptr) { for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } Name->unref(); Colon->unref(); return nullptr; } + auto Field = new RecordTypeExpressionField(Name, Colon, TE); + auto T3 = Tokens.peek(); if (T3->getKind() == NodeKind::RBrace) { Fields.push_back(std::make_tuple(Field, nullptr)); break; } + if (T3->getKind() == NodeKind::VBar) { Tokens.get(); Fields.push_back(std::make_tuple(Field, nullptr)); @@ -496,7 +673,9 @@ TypeExpression* Parser::parsePrimitiveTypeExpression() { if (!Rest) { for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } Field->unref(); return nullptr; @@ -505,7 +684,9 @@ TypeExpression* Parser::parsePrimitiveTypeExpression() { if (T4->getKind() != NodeKind::RBrace) { for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } Field->unref(); Rest->unref(); @@ -514,16 +695,20 @@ TypeExpression* Parser::parsePrimitiveTypeExpression() { } break; } + if (T3->getKind() == NodeKind::Comma) { Tokens.get(); auto Comma = static_cast(T3); Fields.push_back(std::make_tuple(Field, Comma)); continue; } + DE.add(File, T3, std::vector { NodeKind::RBrace, NodeKind::Comma, NodeKind::VBar }); for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } Field->unref(); return nullptr; @@ -531,19 +716,26 @@ TypeExpression* Parser::parsePrimitiveTypeExpression() { auto RBrace = static_cast(Tokens.get()); return new RecordTypeExpression(LBrace, Fields, VBar, Rest, RBrace); } + case NodeKind::LParen: { Tokens.get(); auto LParen = static_cast(T0); + std::vector> Elements; + RParen* RParen; + for (;;) { + auto T1 = Tokens.peek(); + if (isa(T1)) { Tokens.get(); RParen = static_cast(T1); break; } + auto TE = parseTypeExpression(); if (!TE) { LParen->unref(); @@ -553,6 +745,7 @@ TypeExpression* Parser::parsePrimitiveTypeExpression() { } return nullptr; } + auto T2 = Tokens.get(); switch (T2->getKind()) { case NodeKind::RParen: @@ -567,39 +760,56 @@ TypeExpression* Parser::parsePrimitiveTypeExpression() { LParen->unref(); for (auto [TE, Comma]: Elements) { TE->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } return nullptr; } + } + after_tuple_element: + if (Elements.size() == 1) { return new NestedTypeExpression { LParen, std::get<0>(Elements.front()), RParen }; } + return new TupleTypeExpression { LParen, Elements, RParen }; } + case NodeKind::IdentifierAlt: return parseReferenceTypeExpression(); + default: - // Tokens.get(); + Tokens.get(); DE.add(File, T0, std::vector { NodeKind::Identifier, NodeKind::IdentifierAlt, NodeKind::LParen }); return nullptr; } + } ReferenceTypeExpression* Parser::parseReferenceTypeExpression() { + std::vector> ModulePath; + auto Name = expectToken(); if (!Name) { return nullptr; } + for (;;) { + auto T1 = Tokens.peek(); + if (T1->getKind() != NodeKind::Dot) { break; } + Tokens.get(); + ModulePath.push_back(std::make_tuple(static_cast(Name), static_cast(T1))); + Name = expectToken(); if (!Name) { for (auto [Name, Dot]: ModulePath) { @@ -608,16 +818,22 @@ ReferenceTypeExpression* Parser::parseReferenceTypeExpression() { } return nullptr; } + } + return new ReferenceTypeExpression(ModulePath, static_cast(Name)); } TypeExpression* Parser::parseAppTypeExpression() { + auto OpTy = parsePrimitiveTypeExpression(); + if (!OpTy) { return nullptr; } + std::vector ArgTys; + for (;;) { auto T1 = Tokens.peek(); auto Kind = T1->getKind(); @@ -643,18 +859,24 @@ TypeExpression* Parser::parseAppTypeExpression() { } ArgTys.push_back(TE); } + if (ArgTys.empty()) { return OpTy; } + return new AppTypeExpression { OpTy, ArgTys }; } TypeExpression* Parser::parseArrowTypeExpression() { + auto RetType = parseAppTypeExpression(); + if (RetType == nullptr) { return nullptr; } + std::vector ParamTypes; + for (;;) { auto T1 = Tokens.peek(); if (T1->getKind() != NodeKind::RArrow) { @@ -670,20 +892,27 @@ TypeExpression* Parser::parseArrowTypeExpression() { return nullptr; } } + if (!ParamTypes.empty()) { return new ArrowTypeExpression(ParamTypes, RetType); } + return RetType; } MatchExpression* Parser::parseMatchExpression() { + auto T0 = expectToken(); + if (!T0) { return nullptr; } + auto T1 = Tokens.peek(); + Expression* Value; BlockStart* BlockStart; + if (isa(T1)) { Value = nullptr; BlockStart = static_cast(T1); @@ -701,7 +930,9 @@ MatchExpression* Parser::parseMatchExpression() { return nullptr; } } + std::vector Cases; + for (;;) { auto T2 = Tokens.peek(); if (isa(T2)) { @@ -729,22 +960,32 @@ MatchExpression* Parser::parseMatchExpression() { checkLineFoldEnd(); Cases.push_back(new MatchCase { Pattern, RArrowAlt, Expression }); } + return new MatchExpression(static_cast(T0), Value, BlockStart, Cases); } RecordExpression* Parser::parseRecordExpression() { + auto LBrace = expectToken(); if (!LBrace) { return nullptr; } + RBrace* RBrace; + auto T1 = Tokens.peek(); + std::vector> Fields; + if (T1->getKind() == NodeKind::RBrace) { + Tokens.get(); RBrace = static_cast(T1); + } else { + for (;;) { + auto Name = expectToken(); if (!Name) { LBrace->unref(); @@ -754,6 +995,7 @@ RecordExpression* Parser::parseRecordExpression() { } return nullptr; } + auto Equals = expectToken(); if (!Equals) { LBrace->unref(); @@ -764,6 +1006,7 @@ RecordExpression* Parser::parseRecordExpression() { Name->unref(); return nullptr; } + auto E = parseExpression(); if (!E) { LBrace->unref(); @@ -775,6 +1018,7 @@ RecordExpression* Parser::parseRecordExpression() { Equals->unref(); return nullptr; } + auto T2 = Tokens.peek(); if (T2->getKind() == NodeKind::Comma) { Tokens.get(); @@ -789,37 +1033,109 @@ RecordExpression* Parser::parseRecordExpression() { LBrace->unref(); for (auto [Field, Comma]: Fields) { Field->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } Name->unref(); Equals->unref(); E->unref(); return nullptr; } + } + } + return new RecordExpression { LBrace, Fields, RBrace }; } +FunctionExpression* Parser::parseFunctionExpression() { + + auto Backslash = expectToken(); + if (!Backslash) { + return nullptr; + } + + std::vector Params; + + class RArrow* RArrow; + + for (;;) { + + auto T1 = Tokens.peek(); + + if (isa(T1)) { + DE.add(File, T1, std::vector { + NodeKind::RArrow, + }); + Backslash->unref(); + unrefAll(Params); + return nullptr; + } + + if (isa(T1)) { + Tokens.get(); + RArrow = static_cast(T1); + break; + } + + auto P = parseNarrowPattern(); + if (!P) { + Backslash->unref(); + unrefAll(Params); + return nullptr; + } + + Params.push_back(P); + } + + auto E = parseExpression(); + if (!E) { + Backslash->unref(); + unrefAll(Params); + RArrow->unref(); + return nullptr; + } + + return new FunctionExpression { + Backslash, + Params, + RArrow, + E + }; +} + Expression* Parser::parsePrimitiveExpression() { + auto Annotations = parseAnnotations(); + auto T0 = Tokens.peek(); + switch (T0->getKind()) { + case NodeKind::Identifier: case NodeKind::IdentifierAlt: { std::vector> ModulePath; + for (;;) { + auto T1 = Tokens.peek(0); auto T2 = Tokens.peek(1); + if (!isa(T1) || !isa(T2)) { break; } + Tokens.get(); Tokens.get(); + ModulePath.push_back(std::make_tuple(static_cast(T1), static_cast(T2))); } + auto T3 = Tokens.get(); + if (!isa(T3) && !isa(T3)) { for (auto [Name, Dot]: ModulePath) { Name->unref(); @@ -828,40 +1144,57 @@ Expression* Parser::parsePrimitiveExpression() { DE.add(File, T3, std::vector { NodeKind::Identifier, NodeKind::IdentifierAlt }); return nullptr; } + return new ReferenceExpression(Annotations, ModulePath, Symbol::from_raw_node(T3)); } + case NodeKind::LParen: { + Tokens.get(); - std::vector> Elements; auto LParen = static_cast(T0); + + std::vector> Elements; + RParen* RParen; + auto T1 = Tokens.peek(); + if (isa(T1)) { Tokens.get(); RParen = static_cast(T1); goto after_tuple_elements; } + for (;;) { + auto T1 = Tokens.peek(); + auto E = parseExpression(); if (!E) { LParen->unref(); for (auto [E, Comma]: Elements) { E->unref(); - Comma->unref(); + if (Comma) { + Comma->unref(); + } } return nullptr; } + auto T2 = Tokens.get(); + switch (T2->getKind()) { + case NodeKind::RParen: RParen = static_cast(T2); Elements.push_back({ E, nullptr }); goto after_tuple_elements; + case NodeKind::Comma: Elements.push_back({ E, static_cast(T2) }); break; + default: DE.add(File, T2, std::vector { NodeKind::RParen, NodeKind::Comma }); LParen->unref(); @@ -870,6 +1203,7 @@ Expression* Parser::parsePrimitiveExpression() { Comma->unref(); } return nullptr; + case NodeKind::LineFoldEnd: case NodeKind::BlockStart: case NodeKind::EndOfFile: @@ -878,33 +1212,49 @@ Expression* Parser::parsePrimitiveExpression() { DE.add(File, T2, std::vector { NodeKind::RParen, NodeKind::Comma }); goto after_tuple_elements; } + } + after_tuple_elements: + if (Elements.size() == 1 && !std::get<1>(Elements.front())) { return new NestedExpression(Annotations, LParen, std::get<0>(Elements.front()), RParen); } + return new TupleExpression { Annotations, LParen, Elements, RParen }; } + + case NodeKind::Backslash: + return parseFunctionExpression(); + case NodeKind::MatchKeyword: return parseMatchExpression(); + case NodeKind::DoKeyword: return parseBlockExpression(); + case NodeKind::IfKeyword: return parseIfExpression(); + case NodeKind::ReturnKeyword: return parseReturnExpression(); + case NodeKind::IntegerLiteral: case NodeKind::StringLiteral: Tokens.get(); return new LiteralExpression(Annotations, static_cast(T0)); + case NodeKind::LBrace: return parseRecordExpression(); + default: - // Tokens.get(); + Tokens.get(); DE.add(File, T0, std::vector { NodeKind::MatchKeyword, + NodeKind::DoKeyword, NodeKind::Identifier, NodeKind::IdentifierAlt, + NodeKind::Backslash, NodeKind::LParen, NodeKind::LBrace, NodeKind::IntegerLiteral, @@ -912,37 +1262,47 @@ after_tuple_elements: }); return nullptr; } + } BlockExpression* Parser::parseBlockExpression(std::vector Annotations) { + auto DoKeyword = expectToken(); if (!DoKeyword) { - BOLT_EACH_UNREF(Annotations); + unrefAll(Annotations); return nullptr; } + auto BlockStart = expectToken(); if (!BlockStart) { - BOLT_EACH_UNREF(Annotations); + unrefAll(Annotations); DoKeyword->unref(); return nullptr; } + std::vector Elements; + for (;;) { + auto T2 = Tokens.peek(); + if (T2->getKind() == NodeKind::BlockEnd) { Tokens.get()->unref(); break; } + auto Element = parseLetBodyElement(); if (Element == nullptr) { - BOLT_EACH_UNREF(Annotations); + unrefAll(Annotations); DoKeyword->unref(); BlockStart->unref(); - BOLT_EACH_UNREF(Elements); + unrefAll(Elements); return nullptr; } + Elements.push_back(Element); } + return new BlockExpression { DoKeyword, BlockStart, @@ -1016,15 +1376,22 @@ Expression* Parser::parseCallExpression() { } Expression* Parser::parseUnaryExpression() { + std::vector Prefix; + for (;;) { + auto T0 = Tokens.peek(); + if (!ExprOperators.isPrefix(T0)) { break; } + Tokens.get(); + Prefix.push_back(T0); } + auto E = parseCallExpression(); if (!E) { for (auto Tok: Prefix) { @@ -1032,32 +1399,43 @@ Expression* Parser::parseUnaryExpression() { } return nullptr; } + for (auto Iter = Prefix.rbegin(); Iter != Prefix.rend(); Iter++) { E = new PrefixExpression(*Iter, E); } + return E; } Expression* Parser::parseInfixOperatorAfterExpression(Expression* Left, int MinPrecedence) { + for (;;) { + auto T0 = Tokens.peek(); + auto Info0 = ExprOperators.getInfix(T0); if (!Info0 || Info0->Precedence < MinPrecedence) { break; } + Tokens.get(); + auto Right = parseUnaryExpression(); if (!Right) { Left->unref(); T0->unref(); return nullptr; } + for (;;) { + auto T1 = Tokens.peek(); auto Info1 = ExprOperators.getInfix(T1); + if (!Info1 || Info1->Precedence < Info0->Precedence && (Info1->Precedence > Info0->Precedence || Info1->isRightAssoc())) { break; } + auto NewRight = parseInfixOperatorAfterExpression(Right, Info1->Precedence); if (!NewRight) { Left->unref(); @@ -1065,10 +1443,13 @@ Expression* Parser::parseInfixOperatorAfterExpression(Expression* Left, int MinP Right->unref(); return nullptr; } + Right = NewRight; } + Left = new InfixExpression(Left, Operator::from_raw_node(T0), Right); } + return Left; } @@ -1091,41 +1472,52 @@ Expression* Parser::parseExpressionStatement() { } ReturnExpression* Parser::parseReturnExpression() { + auto Annotations = parseAnnotations(); + auto ReturnKeyword = expectToken(); if (!ReturnKeyword) { - BOLT_EACH_UNREF(Annotations); + unrefAll(Annotations); return nullptr; } Expression* Expression; + auto T1 = Tokens.peek(); + if (T1->getKind() == NodeKind::LineFoldEnd) { Tokens.get()->unref(); Expression = nullptr; } else { Expression = parseExpression(); if (!Expression) { + unrefAll(Annotations); ReturnKeyword->unref(); return nullptr; } } + return new ReturnExpression(Annotations, ReturnKeyword, Expression); } IfExpression* Parser::parseIfExpression() { - std::vector Parts; + auto Annotations = parseAnnotations(); + + std::vector Parts; + auto IfKeyword = expectToken(); if (!IfKeyword) { - BOLT_EACH_UNREF(Annotations); + unrefAll(Annotations); return nullptr; } + auto Test = parseExpression(); if (!Test) { IfKeyword->unref(); skipPastLineFoldEnd(); return nullptr; } + auto T1 = expectToken(); if (!T1) { IfKeyword->unref(); @@ -1133,31 +1525,47 @@ IfExpression* Parser::parseIfExpression() { skipPastLineFoldEnd(); return nullptr; } + std::vector Then; + for (;;) { + auto T2 = Tokens.peek(); + if (T2->getKind() == NodeKind::BlockEnd) { Tokens.get()->unref(); break; } + auto Element = parseLetBodyElement(); if (Element) { Then.push_back(Element); } + } + Tokens.get()->unref(); // Always a LineFoldEnd + Parts.push_back(new IfExpressionPart(Annotations, IfKeyword, Test, T1, Then)); + for (;;) { + auto T3 = peekFirstTokenAfterAnnotationsAndModifiers(); + if (T3->getKind() != NodeKind::ElseKeyword && T3->getKind() != NodeKind::ElifKeyword) { break; } + auto Annotations = parseAnnotations(); + Tokens.get(); + Expression* Test = nullptr; + if (T3->getKind() == NodeKind::ElifKeyword) { Test = parseExpression(); } + auto T4 = expectToken(); if (!T4) { for (auto Part: Parts) { @@ -1165,24 +1573,35 @@ IfExpression* Parser::parseIfExpression() { } return nullptr; } + std::vector Alt; + for (;;) { + auto T5 = Tokens.peek(); + if (T5->getKind() == NodeKind::BlockEnd) { Tokens.get()->unref(); break; } + auto Element = parseLetBodyElement(); if (Element) { Alt.push_back(Element); } + } + Tokens.get()->unref(); // Always a LineFoldEnd + Parts.push_back(new IfExpressionPart(Annotations, T3, Test, T4, Alt)); + if (T3->getKind() == NodeKind::ElseKeyword) { break; } + } + return new IfExpression(Parts); } @@ -1495,37 +1914,52 @@ VariableDeclaration* Parser::parseVariableDeclaration() { } switch (T5->getKind()) { + case NodeKind::BlockStart: { Tokens.get(); + std::vector Elements; + for (;;) { + auto T6 = Tokens.peek(); if (T6->getKind() == NodeKind::BlockEnd) { break; } + auto Element = parseLetBodyElement(); if (Element) { Elements.push_back(Element); } + } + Tokens.get()->unref(); // Always a BlockEnd + Body = new LetBlockBody(static_cast(T5), Elements); + break; } + case NodeKind::Equals: { Tokens.get(); + auto E = parseExpression(); if (!E) { skipPastLineFoldEnd(); goto finish; } + Body = new LetExprBody(static_cast(T5), E); + break; } + case NodeKind::LineFoldEnd: break; + default: std::vector Expected { NodeKind::BlockStart, NodeKind::LineFoldEnd, NodeKind::Equals }; if (TA == nullptr) { @@ -1565,9 +1999,13 @@ Node* Parser::parseLetBodyElement() { } ConstraintExpression* Parser::parseConstraintExpression() { + bool HasTilde = false; + for (std::size_t I = 0; ; I++) { + auto Tok = Tokens.peek(I); + switch (Tok->getKind()) { case NodeKind::Tilde: HasTilde = true; @@ -1580,57 +2018,77 @@ ConstraintExpression* Parser::parseConstraintExpression() { default: continue; } + } + after_lookahead: + if (HasTilde) { + auto Left = parseArrowTypeExpression(); if (!Left) { return nullptr; } + auto Tilde = expectToken(); if (!Tilde) { Left->unref(); return nullptr; } + auto Right = parseArrowTypeExpression(); if (!Right) { Left->unref(); Tilde->unref(); return nullptr; } + return new EqualityConstraintExpression { Left, Tilde, Right }; } + auto Name = expectToken(); if (!Name) { return nullptr; } + std::vector TEs; + for (;;) { + auto T1 = Tokens.peek(); + switch (T1->getKind()) { + case NodeKind::RParen: case NodeKind::RArrowAlt: case NodeKind::Comma: goto after_vars; + case NodeKind::Identifier: Tokens.get(); TEs.push_back(new VarTypeExpression { static_cast(T1) }); break; + default: DE.add(File, T1, std::vector { NodeKind::RParen, NodeKind::RArrowAlt, NodeKind::Comma, NodeKind::Identifier }); Name->unref(); return nullptr; } + } + after_vars: + return new TypeclassConstraintExpression { Name, TEs }; } VarTypeExpression* Parser::parseVarTypeExpression() { + auto Name = expectToken(); if (!Name) { return nullptr; } + for (auto Ch: Name->Text) { if (!std::islower(Ch)) { // TODO @@ -1639,27 +2097,34 @@ VarTypeExpression* Parser::parseVarTypeExpression() { return nullptr; } } + return new VarTypeExpression { Name }; } InstanceDeclaration* Parser::parseInstanceDeclaration() { + auto InstanceKeyword = expectToken(); if (!InstanceKeyword) { skipPastLineFoldEnd(); return nullptr; } + auto Name = expectToken(); if (!Name) { InstanceKeyword->unref(); skipPastLineFoldEnd(); return nullptr; } + std::vector TypeExps; + for (;;) { + auto T1 = Tokens.peek(); if (isa(T1)) { break; } + auto TE = parseTypeExpression(); if (!TE) { InstanceKeyword->unref(); @@ -1670,8 +2135,10 @@ InstanceDeclaration* Parser::parseInstanceDeclaration() { skipPastLineFoldEnd(); return nullptr; } + TypeExps.push_back(TE); } + auto BlockStart = expectToken(); if (!BlockStart) { InstanceKeyword->unref(); @@ -1682,19 +2149,27 @@ InstanceDeclaration* Parser::parseInstanceDeclaration() { skipPastLineFoldEnd(); return nullptr; } + std::vector Elements; + for (;;) { + auto T2 = Tokens.peek(); + if (isa(T2)) { Tokens.get()->unref(); break; } + auto Element = parseClassElement(); if (Element) { Elements.push_back(Element); } + } + checkLineFoldEnd(); + return new InstanceDeclaration( InstanceKeyword, Name, @@ -1705,12 +2180,15 @@ InstanceDeclaration* Parser::parseInstanceDeclaration() { } ClassDeclaration* Parser::parseClassDeclaration() { + PubKeyword* PubKeyword = nullptr; + auto T0 = Tokens.peek(); if (T0->getKind() == NodeKind::PubKeyword) { Tokens.get(); PubKeyword = static_cast(T0); } + auto ClassKeyword = expectToken(); if (!ClassKeyword) { if (PubKeyword) { @@ -1719,6 +2197,7 @@ ClassDeclaration* Parser::parseClassDeclaration() { skipPastLineFoldEnd(); return nullptr; } + auto Name = expectToken(); if (!Name) { if (PubKeyword) { @@ -1728,12 +2207,16 @@ ClassDeclaration* Parser::parseClassDeclaration() { skipPastLineFoldEnd(); return nullptr; } + std::vector TypeVars; + for (;;) { + auto T2 = Tokens.peek(); if (T2->getKind() == NodeKind::BlockStart) { break; } + auto TE = parseVarTypeExpression(); if (!TE) { if (PubKeyword) { @@ -1746,8 +2229,10 @@ ClassDeclaration* Parser::parseClassDeclaration() { skipPastLineFoldEnd(); return nullptr; } + TypeVars.push_back(TE); } + auto BlockStart = expectToken(); if (!BlockStart) { if (PubKeyword) { @@ -1760,19 +2245,29 @@ ClassDeclaration* Parser::parseClassDeclaration() { skipPastLineFoldEnd(); return nullptr; } + std::vector Elements; + for (;;) { + auto T2 = Tokens.peek(); + if (isa(T2)) { Tokens.get()->unref(); break; } + auto Element = parseClassElement(); if (Element) { Elements.push_back(Element); + } else { + skipPastLineFoldEnd(); } + } + Tokens.get()->unref(); // Always a LineFoldEnd + return new ClassDeclaration( PubKeyword, ClassKeyword, @@ -1784,24 +2279,30 @@ ClassDeclaration* Parser::parseClassDeclaration() { } std::vector Parser::parseRecordDeclarationFields() { + std::vector Fields; + for (;;) { + auto T1 = Tokens.peek(); if (T1->getKind() == NodeKind::BlockEnd) { Tokens.get()->unref(); break; } + auto Name = expectToken(); if (!Name) { skipPastLineFoldEnd(); continue; } + auto Colon = expectToken(); if (!Colon) { Name->unref(); skipPastLineFoldEnd(); continue; } + auto TE = parseTypeExpression(); if (!TE) { Name->unref(); @@ -1809,19 +2310,25 @@ std::vector Parser::parseRecordDeclarationFields() { skipPastLineFoldEnd(); continue; } + checkLineFoldEnd(); + Fields.push_back(new RecordDeclarationField { Name, Colon, TE }); } + return Fields; } RecordDeclaration* Parser::parseRecordDeclaration() { - auto T0 = Tokens.peek(); + PubKeyword* Pub = nullptr; + + auto T0 = Tokens.peek(); if (T0->getKind() == NodeKind::MutKeyword) { Tokens.get(); Pub = static_cast(T0); } + auto Struct = expectToken(); if (!Struct) { if (Pub) { @@ -1830,6 +2337,7 @@ RecordDeclaration* Parser::parseRecordDeclaration() { skipPastLineFoldEnd(); return nullptr; } + auto Name = expectToken(); if (!Name) { if (Pub) { @@ -1839,6 +2347,7 @@ RecordDeclaration* Parser::parseRecordDeclaration() { skipPastLineFoldEnd(); return nullptr; } + std::vector Vars; for (;;) { auto T1 = Tokens.peek(); @@ -1848,8 +2357,11 @@ RecordDeclaration* Parser::parseRecordDeclaration() { auto Var = parseVarTypeExpression(); if (Var) { Vars.push_back(Var); + } else { + Tokens.get(); } } + auto BS = expectToken(); if (!BS) { if (Pub) { @@ -1857,21 +2369,30 @@ RecordDeclaration* Parser::parseRecordDeclaration() { } Struct->unref(); Name->unref(); + for (auto V: Vars) { + V->unref(); + } skipPastLineFoldEnd(); return nullptr; } + auto Fields = parseRecordDeclarationFields(); + Tokens.get()->unref(); // Always a LineFoldEnd + return new RecordDeclaration { Pub, Struct, Name, Vars, BS, Fields }; } VariantDeclaration* Parser::parseVariantDeclaration() { - auto T0 = Tokens.peek(); + PubKeyword* Pub = nullptr; + + auto T0 = Tokens.peek(); if (T0->getKind() == NodeKind::MutKeyword) { Tokens.get(); Pub = static_cast(T0); } + auto Enum = expectToken(); if (!Enum) { if (Pub) { @@ -1880,6 +2401,7 @@ VariantDeclaration* Parser::parseVariantDeclaration() { skipPastLineFoldEnd(); return nullptr; } + auto Name = expectToken(); if (!Name) { if (Pub) { @@ -1889,17 +2411,23 @@ VariantDeclaration* Parser::parseVariantDeclaration() { skipPastLineFoldEnd(); return nullptr; } + std::vector TVs; + for (;;) { + auto T0 = Tokens.peek(); if (T0->getKind() == NodeKind::BlockStart) { break; } + auto Var = parseVarTypeExpression(); if (Var) { TVs.push_back(Var); } + } + auto BS = expectToken(); if (!BS) { if (Pub) { @@ -1910,19 +2438,24 @@ VariantDeclaration* Parser::parseVariantDeclaration() { skipPastLineFoldEnd(); return nullptr; } + std::vector Members; + for (;;) { next_member: + auto T0 = Tokens.peek(); if (T0->getKind() == NodeKind::BlockEnd) { Tokens.get()->unref(); break; } + auto Name = expectToken(); if (!Name) { skipPastLineFoldEnd(); continue; } + auto T1 = Tokens.peek(); if (T1->getKind() == NodeKind::BlockStart) { Tokens.get(); @@ -1931,13 +2464,17 @@ next_member: // TODO continue; on error in Fields Members.push_back(new RecordVariantDeclarationMember { Name, BS, Fields }); } else { + std::vector Elements; + for (;;) { + auto T2 = Tokens.peek(); if (T2->getKind() == NodeKind::LineFoldEnd) { Tokens.get()->unref(); break; } + auto TE = parsePrimitiveTypeExpression(); if (!TE) { Name->unref(); @@ -1946,22 +2483,32 @@ next_member: } goto next_member; } + Elements.push_back(TE); } + Members.push_back(new TupleVariantDeclarationMember { Name, Elements }); } + } + checkLineFoldEnd(); + return new VariantDeclaration { Pub, Enum, Name, TVs, BS, Members }; } Node* Parser::parseClassElement() { + auto T0 = Tokens.peek(); + switch (T0->getKind()) { + case NodeKind::LetKeyword: return parseVariableDeclaration(); + case NodeKind::TypeKeyword: // TODO + default: DE.add(File, T0, std::vector { NodeKind::LetKeyword, NodeKind::TypeKeyword }); skipPastLineFoldEnd(); @@ -1970,49 +2517,68 @@ Node* Parser::parseClassElement() { } Node* Parser::parseSourceElement() { + auto T0 = peekFirstTokenAfterAnnotationsAndModifiers(); + switch (T0->getKind()) { case NodeKind::LetKeyword: return parseVariableDeclaration(); + case NodeKind::FnKeyword: return parseFunctionDeclaration(); + case NodeKind::ClassKeyword: return parseClassDeclaration(); + case NodeKind::InstanceKeyword: return parseInstanceDeclaration(); + case NodeKind::StructKeyword: return parseRecordDeclaration(); + case NodeKind::EnumKeyword: return parseVariantDeclaration(); + default: return parseExpressionStatement(); } } SourceFile* Parser::parseSourceFile() { + std::vector Elements; + for (;;) { + auto T0 = Tokens.peek(); if (isa(T0)) { break; } + auto Element = parseSourceElement(); if (Element) { Elements.push_back(Element); } + } + return new SourceFile(File, Elements); } std::vector Parser::parseAnnotations() { + std::vector Annotations; + for (;;) { + auto T0 = Tokens.peek(); if (T0->getKind() != NodeKind::At) { break; } + auto At = static_cast(T0); Tokens.get(); + auto T1 = Tokens.peek(); switch (T1->getKind()) { case NodeKind::Colon: @@ -2047,55 +2613,71 @@ std::vector Parser::parseAnnotations() { // skipToLineFoldEnd(); // break; } + next_annotation:; } + return Annotations; } void Parser::skipToRBrace() { + unsigned ParenLevel = 0; unsigned BracketLevel = 0; unsigned BraceLevel = 0; unsigned BlockLevel = 0; + for (;;) { + auto T0 = Tokens.peek(); + switch (T0->getKind()) { + case NodeKind::EndOfFile: return; + case NodeKind::LineFoldEnd: Tokens.get()->unref(); if (BlockLevel == 0 && ParenLevel == 0 && BracketLevel == 0 && BlockLevel == 0) { return; } break; + case NodeKind::BlockStart: Tokens.get()->unref(); BlockLevel++; break; + case NodeKind::BlockEnd: Tokens.get()->unref(); BlockLevel--; break; + case NodeKind::LParen: Tokens.get()->unref(); ParenLevel++; break; + case NodeKind::LBracket: Tokens.get()->unref(); BracketLevel++; break; + case NodeKind::LBrace: Tokens.get()->unref(); BraceLevel++; break; + case NodeKind::RParen: Tokens.get()->unref(); ParenLevel--; break; + case NodeKind::RBracket: Tokens.get()->unref(); BracketLevel--; break; + case NodeKind::RBrace: if (BlockLevel == 0 && ParenLevel == 0 && BracketLevel == 0 && BlockLevel == 0) { return; @@ -2103,76 +2685,99 @@ void Parser::skipToRBrace() { Tokens.get()->unref(); BraceLevel--; break; + default: Tokens.get()->unref(); break; } + } + } void Parser::skipPastLineFoldEnd() { + unsigned ParenLevel = 0; unsigned BracketLevel = 0; unsigned BraceLevel = 0; unsigned BlockLevel = 0; + for (;;) { + auto T0 = Tokens.get(); + switch (T0->getKind()) { + case NodeKind::EndOfFile: return; + case NodeKind::LineFoldEnd: T0->unref(); if (BlockLevel == 0 && ParenLevel == 0 && BracketLevel == 0 && BlockLevel == 0) { return; } break; + case NodeKind::BlockStart: T0->unref(); BlockLevel++; break; + case NodeKind::BlockEnd: T0->unref(); BlockLevel--; break; + case NodeKind::LParen: T0->unref(); ParenLevel++; break; + case NodeKind::LBracket: T0->unref(); BracketLevel++; break; + case NodeKind::LBrace: T0->unref(); BraceLevel++; break; + case NodeKind::RParen: T0->unref(); ParenLevel--; break; + case NodeKind::RBracket: T0->unref(); BracketLevel--; break; + case NodeKind::RBrace: T0->unref(); BraceLevel--; break; + default: T0->unref(); break; } + } + } void Parser::checkLineFoldEnd() { + auto T0 = Tokens.peek(); + if (T0->getKind() == NodeKind::LineFoldEnd) { Tokens.get()->unref(); } else { DE.add(File, T0, std::vector { NodeKind::LineFoldEnd }); skipPastLineFoldEnd(); } + } } diff --git a/src/Scanner.cc b/src/Scanner.cc index 1c6d58003..4baf43718 100644 --- a/src/Scanner.cc +++ b/src/Scanner.cc @@ -344,6 +344,9 @@ after_string_contents: return new StringLiteral(Text, StartLoc); } + case '\\': + return new Backslash { getCurrentLoc() }; + case '.': { auto C1 = peekChar();