From 655cd79df81cc4ffacb4d9e94be6036cb24fe6f7 Mon Sep 17 00:00:00 2001 From: Rintaro Ishizaki Date: Thu, 8 Feb 2024 11:04:14 -0800 Subject: [PATCH] [Parser] Introduce TokenSpec for fixed text Matches a specific tokenText. Use it in decl/type attribute TokenSpecSet. --- .../TokenSpecStaticMembersFile.swift | 2 - Sources/SwiftParser/Attributes.swift | 117 +++++++++------- Sources/SwiftParser/TokenSpec.swift | 131 ++++++++++++------ Sources/SwiftParser/TokenSpecSet.swift | 53 ++++--- .../generated/TokenSpecStaticMembers.swift | 4 - 5 files changed, 190 insertions(+), 117 deletions(-) diff --git a/CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/TokenSpecStaticMembersFile.swift b/CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/TokenSpecStaticMembersFile.swift index 9d3272d4ee8..bb26183e2fd 100644 --- a/CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/TokenSpecStaticMembersFile.swift +++ b/CodeGeneration/Sources/generate-swift-syntax/templates/swiftparser/TokenSpecStaticMembersFile.swift @@ -22,7 +22,5 @@ let tokenSpecStaticMembersFile = SourceFileSyntax(leadingTrivia: copyrightHeader for tokenSpec in Token.allCases.map(\.spec) where tokenSpec.kind != .keyword { DeclSyntax("static var \(tokenSpec.varOrCaseName): TokenSpec { return TokenSpec(.\(tokenSpec.varOrCaseName)) }") } - - DeclSyntax("static func keyword(_ keyword: Keyword) -> TokenSpec { return TokenSpec(keyword) }") } } diff --git a/Sources/SwiftParser/Attributes.swift b/Sources/SwiftParser/Attributes.swift index 5b39dccbcd5..bd7aaca4b7a 100644 --- a/Sources/SwiftParser/Attributes.swift +++ b/Sources/SwiftParser/Attributes.swift @@ -67,52 +67,41 @@ extension Parser { case transpose init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) { - switch lexeme.rawTokenKind { - case .keyword: - switch Keyword(lexeme.tokenText) { - case .rethrows: self = .rethrows - default: - return nil - } - case .identifier: - switch lexeme.tokenText { - case "_alignment": self = ._alignment - case "_backDeploy": self = ._backDeploy - case "_cdecl": self = ._cdecl - case "_documentation": self = ._documentation - case "_dynamicReplacement": self = ._dynamicReplacement - case "_effects": self = ._effects - case "_expose": self = ._expose - case "_implements": self = ._implements - case "_nonSendable": self = ._nonSendable - case "_objcImplementation": self = ._objcImplementation - case "_objcRuntimeName": self = ._objcRuntimeName - case "_optimize": self = ._optimize - case "_originallyDefinedIn": self = ._originallyDefinedIn - case "_private": self = ._private - case "_projectedValueProperty": self = ._projectedValueProperty - case "_semantics": self = ._semantics - case "_specialize": self = ._specialize - case "_spi": self = ._spi - case "_spi_available": self = ._spi_available - case "_swift_native_objc_runtime_base": self = ._swift_native_objc_runtime_base - case "_typeEraser": self = ._typeEraser - case "_unavailableFromAsync": self = ._unavailableFromAsync - case "rethrows": self = .rethrows - case "attached": self = .attached - case "backDeployed": self = .backDeployed - case "derivative": self = .derivative - case "differentiable": self = .differentiable - case "exclusivity": self = .exclusivity - case "freestanding": self = .freestanding - case "inline": self = .inline - case "objc": self = .objc - case "available": self = .available - case "Sendable": self = .Sendable - case "transpose": self = .transpose - default: - return nil - } + switch lexeme { + case TokenSpec("_alignment"): self = ._alignment + case TokenSpec("_backDeploy"): self = ._backDeploy + case TokenSpec("_cdecl"): self = ._cdecl + case TokenSpec("_documentation"): self = ._documentation + case TokenSpec("_dynamicReplacement"): self = ._dynamicReplacement + case TokenSpec("_effects"): self = ._effects + case TokenSpec("_expose"): self = ._expose + case TokenSpec("_implements"): self = ._implements + case TokenSpec("_nonSendable"): self = ._nonSendable + case TokenSpec("_objcImplementation"): self = ._objcImplementation + case TokenSpec("_objcRuntimeName"): self = ._objcRuntimeName + case TokenSpec("_optimize"): self = ._optimize + case TokenSpec("_originallyDefinedIn"): self = ._originallyDefinedIn + case TokenSpec("_private"): self = ._private + case TokenSpec("_projectedValueProperty"): self = ._projectedValueProperty + case TokenSpec("_semantics"): self = ._semantics + case TokenSpec("_specialize"): self = ._specialize + case TokenSpec("_spi"): self = ._spi + case TokenSpec("_spi_available"): self = ._spi_available + case TokenSpec("_swift_native_objc_runtime_base"): self = ._swift_native_objc_runtime_base + case TokenSpec("_typeEraser"): self = ._typeEraser + case TokenSpec("_unavailableFromAsync"): self = ._unavailableFromAsync + case TokenSpec("rethrows"): self = .rethrows + case TokenSpec("attached"): self = .attached + case TokenSpec("available"): self = .available + case TokenSpec("backDeployed"): self = .backDeployed + case TokenSpec("derivative"): self = .derivative + case TokenSpec("differentiable"): self = .differentiable + case TokenSpec("exclusivity"): self = .exclusivity + case TokenSpec("freestanding"): self = .freestanding + case TokenSpec("inline"): self = .inline + case TokenSpec("objc"): self = .objc + case TokenSpec("Sendable"): self = .Sendable + case TokenSpec("transpose"): self = .transpose default: return nil } @@ -120,8 +109,40 @@ extension Parser { var spec: TokenSpec { switch self { - case .rethrows: return .keyword(.rethrows) - default: return .identifier + case ._alignment: return TokenSpec("_alignment") + case ._backDeploy: return TokenSpec("_backDeploy") + case ._cdecl: return TokenSpec("_cdecl") + case ._documentation: return TokenSpec("_documentation") + case ._dynamicReplacement: return TokenSpec("_dynamicReplacement") + case ._effects: return TokenSpec("_effects") + case ._expose: return TokenSpec("_expose") + case ._implements: return TokenSpec("_implements") + case ._nonSendable: return TokenSpec("_nonSendable") + case ._objcImplementation: return TokenSpec("_objcImplementation") + case ._objcRuntimeName: return TokenSpec("_objcRuntimeName") + case ._optimize: return TokenSpec("_optimize") + case ._originallyDefinedIn: return TokenSpec("_originallyDefinedIn") + case ._private: return TokenSpec("_private") + case ._projectedValueProperty: return TokenSpec("_projectedValueProperty") + case ._semantics: return TokenSpec("_semantics") + case ._specialize: return TokenSpec("_specialize") + case ._spi: return TokenSpec("_spi") + case ._spi_available: return TokenSpec("_spi_available") + case ._swift_native_objc_runtime_base: return TokenSpec("_swift_native_objc_runtime_base") + case ._typeEraser: return TokenSpec("_typeEraser") + case ._unavailableFromAsync: return TokenSpec("_unavailableFromAsync") + case .rethrows: return TokenSpec("rethrows") + case .available: return TokenSpec("available") + case .attached: return TokenSpec("attached") + case .backDeployed: return TokenSpec("backDeployed") + case .derivative: return TokenSpec("derivative") + case .differentiable: return TokenSpec("differentiable") + case .exclusivity: return TokenSpec("exclusivity") + case .freestanding: return TokenSpec("freestanding") + case .inline: return TokenSpec("inline") + case .objc: return TokenSpec("objc") + case .Sendable: return TokenSpec("Sendable") + case .transpose: return TokenSpec("transpose") } } } diff --git a/Sources/SwiftParser/TokenSpec.swift b/Sources/SwiftParser/TokenSpec.swift index da0b3fc74fd..093b5730ce5 100644 --- a/Sources/SwiftParser/TokenSpec.swift +++ b/Sources/SwiftParser/TokenSpec.swift @@ -19,6 +19,8 @@ struct PrepareForKeywordMatch { /// The kind of the lexeme. fileprivate let rawTokenKind: RawTokenKind + fileprivate let rawTokenText: SyntaxText + /// If the lexeme has the same text as a keyword, that keyword, otherwise `nil`. fileprivate let keyword: Keyword? @@ -28,6 +30,7 @@ struct PrepareForKeywordMatch { @inline(__always) init(_ lexeme: Lexer.Lexeme) { self.rawTokenKind = lexeme.rawTokenKind + self.rawTokenText = lexeme.tokenText switch lexeme.rawTokenKind { case .keyword, .identifier: keyword = Keyword(lexeme.tokenText) @@ -46,18 +49,24 @@ struct PrepareForKeywordMatch { /// `matches(rawTokenKind:text:)` based on the matched kind. @_spi(AlternateTokenIntrospection) public struct TokenSpec { - /// The kind we expect the token that we want to consume to have. - /// This can be a keyword, in which case the ``TokenSpec`` will also match an - /// identifier with the same text as the keyword and remap it to that keyword - /// when consumed. - /// - /// `fileprivate` because only functions in this file should access it since - /// they know how to handle the identifier -> keyword remapping. - fileprivate let rawTokenKind: RawTokenKind + enum Matcher { + /// A token with a specific text. + case fixedText(SyntaxText) - /// If `rawTokenKind` is `keyword`, the keyword we are expecting. For all other - /// values of `rawTokenKind`, this is `nil`. - fileprivate let keyword: Keyword? + /// The keyword we are expecting. + case keyword(Keyword) + + /// The kind we expect the token that we want to consume to have. + /// This can be a keyword, in which case the ``TokenSpec`` will also match an + /// identifier with the same text as the keyword and remap it to that keyword + /// when consumed. + /// + /// `fileprivate` because only functions in this file should access it since + /// they know how to handle the identifier -> keyword remapping. + case tokenKind(RawTokenKind) + } + + fileprivate let matcher: Matcher /// If not nil, the token will be remapped to the provided kind when consumed. /// @@ -83,8 +92,7 @@ public struct TokenSpec { allowAtStartOfLine: Bool = true ) { precondition(rawTokenKind != .keyword, "To create a TokenSpec for a keyword use the initializer that takes a keyword") - self.rawTokenKind = rawTokenKind - self.keyword = nil + self.matcher = .tokenKind(rawTokenKind) self.remapping = remapping self.recoveryPrecedence = recoveryPrecedence ?? TokenPrecedence(nonKeyword: rawTokenKind) self.allowAtStartOfLine = allowAtStartOfLine @@ -97,32 +105,42 @@ public struct TokenSpec { recoveryPrecedence: TokenPrecedence? = nil, allowAtStartOfLine: Bool = true ) { - self.rawTokenKind = .keyword - self.keyword = keyword + self.matcher = .keyword(keyword) self.remapping = remapping self.recoveryPrecedence = recoveryPrecedence ?? TokenPrecedence(keyword) self.allowAtStartOfLine = allowAtStartOfLine } + @inline(__always) + init( + _ text: SyntaxText, + remapping: RawTokenKind? = nil, + recoveryPrecedence: TokenPrecedence? = nil, + allowAtStartOfLine: Bool = true + ) { + self.matcher = .fixedText(text) + self.remapping = remapping + self.recoveryPrecedence = recoveryPrecedence ?? .identifierLike + self.allowAtStartOfLine = allowAtStartOfLine + } + @inline(__always) func matches( rawTokenKind: RawTokenKind, + rawTokenText: SyntaxText, keyword: @autoclosure () -> Keyword?, atStartOfLine: @autoclosure () -> Bool ) -> Bool { if !allowAtStartOfLine && atStartOfLine() { return false } - if self.rawTokenKind == .keyword { - precondition(self.keyword != nil) - switch rawTokenKind { - case .keyword, .identifier: - return keyword() == self.keyword - default: - return false - } - } else { - return rawTokenKind == self.rawTokenKind + switch matcher { + case .fixedText(let expectedText): + return rawTokenText == expectedText + case .keyword(let expectedKeyword): + return keyword() == expectedKeyword + case .tokenKind(let expectedRawTokenKind): + return rawTokenKind == expectedRawTokenKind } } @@ -130,6 +148,7 @@ public struct TokenSpec { static func ~= (kind: TokenSpec, lexeme: Lexer.Lexeme) -> Bool { return kind.matches( rawTokenKind: lexeme.rawTokenKind, + rawTokenText: lexeme.tokenText, keyword: Keyword(lexeme.tokenText), atStartOfLine: lexeme.isAtStartOfLine ) @@ -138,7 +157,8 @@ public struct TokenSpec { @inline(__always) static func ~= (kind: TokenSpec, token: TokenSyntax) -> Bool { return kind.matches( - rawTokenKind: token.tokenView.rawKind, + rawTokenKind: token.rawTokenKind, + rawTokenText: token.rawText, keyword: Keyword(token.tokenView.rawText), atStartOfLine: token.leadingTrivia.contains(where: { $0.isNewline }) ) @@ -148,6 +168,7 @@ public struct TokenSpec { static func ~= (kind: TokenSpec, token: RawTokenSyntax) -> Bool { return kind.matches( rawTokenKind: token.tokenKind, + rawTokenText: token.tokenText, keyword: Keyword(token.tokenView.rawText), atStartOfLine: token.leadingTriviaPieces.contains(where: \.isNewline) ) @@ -157,6 +178,7 @@ public struct TokenSpec { static func ~= (kind: TokenSpec, lexeme: PrepareForKeywordMatch) -> Bool { return kind.matches( rawTokenKind: lexeme.rawTokenKind, + rawTokenText: lexeme.rawTokenText, keyword: lexeme.keyword, atStartOfLine: lexeme.isAtStartOfLine ) @@ -169,29 +191,50 @@ public struct TokenSpec { /// modification of test cases. This should never be used in the parser itself. @_spi(AlternateTokenIntrospection) public var synthesizedTokenKind: TokenKind { - switch rawTokenKind { - case .binaryOperator: return .binaryOperator("+") - case .dollarIdentifier: return .dollarIdentifier("$0") - case .floatLiteral: return .floatLiteral("1.0") - case .identifier: return .identifier("myIdent") - case .integerLiteral: return .integerLiteral("1") - case .keyword: return .keyword(keyword!) - case .postfixOperator: return .postfixOperator("++") - case .prefixOperator: return .prefixOperator("!") - case .rawStringPoundDelimiter: return .rawStringPoundDelimiter("#") - case .regexLiteralPattern: return .regexLiteralPattern(".*") - case .regexPoundDelimiter: return .regexPoundDelimiter("#") - case .stringSegment: return .stringSegment("abc") - default: return TokenKind.fromRaw(kind: rawTokenKind, text: "") + switch matcher { + case .fixedText(let text): + return .identifier(String(syntaxText: text)) + case .keyword(let keyword): + return .keyword(keyword) + case .tokenKind(let kind): + switch kind { + case .binaryOperator: return .binaryOperator("+") + case .dollarIdentifier: return .dollarIdentifier("$0") + case .floatLiteral: return .floatLiteral("1.0") + case .identifier: return .identifier("myIdent") + case .integerLiteral: return .integerLiteral("1") + case .postfixOperator: return .postfixOperator("++") + case .prefixOperator: return .prefixOperator("!") + case .rawStringPoundDelimiter: return .rawStringPoundDelimiter("#") + case .regexLiteralPattern: return .regexLiteralPattern(".*") + case .regexPoundDelimiter: return .regexPoundDelimiter("#") + case .stringSegment: return .stringSegment("abc") + default: return TokenKind.fromRaw(kind: kind, text: "") + } } } + + static func keyword(_ keyword: Keyword) -> TokenSpec { + return TokenSpec(keyword) + } + + static func fixedText(_ text: SyntaxText) -> TokenSpec { + return TokenSpec(text) + } } extension TokenConsumer { /// Generates a missing token that has the expected kind of `spec`. @inline(__always) mutating func missingToken(_ spec: TokenSpec) -> Token { - return missingToken(spec.remapping ?? spec.rawTokenKind, text: spec.keyword?.defaultText ?? spec.rawTokenKind.defaultText) + switch spec.matcher { + case .fixedText(let text): + return missingToken(spec.remapping ?? .identifier, text: text) + case .keyword(let keyword): + return missingToken(spec.remapping ?? .keyword, text: keyword.defaultText) + case .tokenKind(let kind): + return missingToken(spec.remapping ?? kind, text: kind.defaultText) + } } /// Asserts that the current token matches `spec` and consumes it, performing @@ -204,9 +247,11 @@ extension TokenConsumer { precondition(spec ~= self.currentToken) if let remapping = spec.remapping { return self.consumeAnyToken(remapping: remapping) - } else if spec.rawTokenKind == .keyword { + } + switch spec.matcher { + case .keyword(_): return self.consumeAnyToken(remapping: .keyword) - } else { + case .fixedText(_), .tokenKind(_): return self.consumeAnyToken() } } diff --git a/Sources/SwiftParser/TokenSpecSet.swift b/Sources/SwiftParser/TokenSpecSet.swift index 5eb18978ab0..512f4e2bc47 100644 --- a/Sources/SwiftParser/TokenSpecSet.swift +++ b/Sources/SwiftParser/TokenSpecSet.swift @@ -646,31 +646,44 @@ enum TypeAttribute: TokenSpecSet { case isolated init?(lexeme: Lexer.Lexeme, experimentalFeatures: Parser.ExperimentalFeatures) { - guard lexeme.rawTokenKind == .identifier else { - return nil - } - switch lexeme.tokenText { - case "_local": self = ._local - case "_noMetadata": self = ._noMetadata - case "_opaqueReturnTypeOf": self = ._opaqueReturnTypeOf - case "async": self = .async - case "autoclosure": self = .autoclosure - case "convention": self = .convention - case "differentiable": self = .differentiable - case "escaping": self = .escaping - case "noDerivative": self = .noDerivative - case "noescape": self = .noescape - case "preconcurrency": self = .preconcurrency - case "Sendable": self = .Sendable - case "retroactive": self = .retroactive - case "unchecked": self = .unchecked - case "isolated": self = .isolated + switch lexeme { + case TokenSpec("_local"): self = ._local + case TokenSpec("_noMetadata"): self = ._noMetadata + case TokenSpec("_opaqueReturnTypeOf"): self = ._opaqueReturnTypeOf + case TokenSpec("async"): self = .async + case TokenSpec("autoclosure"): self = .autoclosure + case TokenSpec("convention"): self = .convention + case TokenSpec("differentiable"): self = .differentiable + case TokenSpec("escaping"): self = .escaping + case TokenSpec("noDerivative"): self = .noDerivative + case TokenSpec("noescape"): self = .noescape + case TokenSpec("preconcurrency"): self = .preconcurrency + case TokenSpec("Sendable"): self = .Sendable + case TokenSpec("retroactive"): self = .retroactive + case TokenSpec("unchecked"): self = .unchecked + case TokenSpec("isolated"): self = .isolated default: return nil } } var spec: TokenSpec { - return .identifier + switch self { + case ._local: return TokenSpec("_local") + case ._noMetadata: return TokenSpec("_noMetadata") + case ._opaqueReturnTypeOf: return TokenSpec("_opaqueReturnTypeOf") + case .async: return TokenSpec("async") + case .autoclosure: return TokenSpec("autoclosure") + case .convention: return TokenSpec("convention") + case .differentiable: return TokenSpec("differentiable") + case .escaping: return TokenSpec("escaping") + case .noDerivative: return TokenSpec("noDerivative") + case .noescape: return TokenSpec("noescape") + case .preconcurrency: return TokenSpec("preconcurrency") + case .Sendable: return TokenSpec("Sendable") + case .retroactive: return TokenSpec("retroactive") + case .unchecked: return TokenSpec("unchecked") + case .isolated: return TokenSpec("isolated") + } } } diff --git a/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift b/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift index 7f0f81a9ba7..622af0b4ca0 100644 --- a/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift +++ b/Sources/SwiftParser/generated/TokenSpecStaticMembers.swift @@ -210,8 +210,4 @@ extension TokenSpec { static var wildcard: TokenSpec { return TokenSpec(.wildcard) } - - static func keyword(_ keyword: Keyword) -> TokenSpec { - return TokenSpec(keyword) - } }