From d1d40201e1af612059e46699418ab5d702b7187f Mon Sep 17 00:00:00 2001 From: Adam Carter <14096630+adamcarter93@users.noreply.github.com> Date: Fri, 29 Mar 2024 00:21:13 +0000 Subject: [PATCH 1/3] Added commentValues on Trivia This holds an array of comments made from the trivia's pieces, with one line per element of the array --- Sources/SwiftSyntax/Trivia.swift | 31 ++++++++++ Tests/SwiftSyntaxTest/TriviaTests.swift | 82 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/Sources/SwiftSyntax/Trivia.swift b/Sources/SwiftSyntax/Trivia.swift index 07e3925ee9e..f17612de2dd 100644 --- a/Sources/SwiftSyntax/Trivia.swift +++ b/Sources/SwiftSyntax/Trivia.swift @@ -42,6 +42,21 @@ public struct Trivia: Sendable { pieces.isEmpty } + /// The string contents of all the comment pieces with any comments tokens trimmed. + /// + /// Each element in the array is the trimmed contents of a line comment, or, in the case of a multi-line comment a trimmed, concatenated single string. + public var commentValues: [String] { + pieces.compactMap { + switch $0 { + case .lineComment(let text), .docLineComment(let text): + sanitizingLineComment(text) + + default: + nil + } + } + } + /// The length of all the pieces in this ``Trivia``. public var sourceLength: SourceLength { return pieces.map({ $0.sourceLength }).reduce(.zero, +) @@ -215,3 +230,19 @@ extension RawTriviaPiece: CustomDebugStringConvertible { TriviaPiece(raw: self).debugDescription } } + +private func sanitizingLineComment(_ text: String) -> String { + // TODO: adammcarter - can we import Foundation instead and use trimmingCharacters(in:) + + var charactersToDrop = 0 + + for character in text { + if (character == "/" || character == " ") { + charactersToDrop += 1 + } else { + break + } + } + + return String(text.dropFirst(charactersToDrop)) +} diff --git a/Tests/SwiftSyntaxTest/TriviaTests.swift b/Tests/SwiftSyntaxTest/TriviaTests.swift index 6360d2f3ec5..2f8c1878a0b 100644 --- a/Tests/SwiftSyntaxTest/TriviaTests.swift +++ b/Tests/SwiftSyntaxTest/TriviaTests.swift @@ -69,4 +69,86 @@ class TriviaTests: XCTestCase { XCTAssertNotEqual(TriviaPiece.unexpectedText("e"), .unexpectedText("f")) XCTAssertNotEqual(TriviaPiece.unexpectedText("e"), .lineComment("e")) } + + func testTriviaCommentValues() { + XCTAssertTrue(Trivia(pieces: []).commentValues.isEmpty) + + // MARK: line comment + + XCTAssertEqual( + Trivia(pieces: [.lineComment("")]).commentValues, + [""] + ) + + XCTAssertEqual( + Trivia(pieces: [.lineComment("Some line comment")]).commentValues, + ["Some line comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [.lineComment("// Some line comment")]).commentValues, + ["Some line comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .lineComment("// Some line comment"), + .lineComment("// Another"), + ]).commentValues, + [ + "Some line comment", + "Another", + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .lineComment("// Some line comment"), + .lineComment("Other"), + ]).commentValues, + [ + "Some line comment", + "Other", + ] + ) + + // MARK: doc line comment + + XCTAssertEqual( + Trivia(pieces: [.docLineComment("")]).commentValues, + [""] + ) + + XCTAssertEqual( + Trivia(pieces: [.docLineComment("Some doc line comment")]).commentValues, + ["Some doc line comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [.docLineComment("/// Some doc line comment")]).commentValues, + ["Some doc line comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docLineComment("/// Some doc line comment"), + .docLineComment("/// Another"), + ]).commentValues, + [ + "Some doc line comment", + "Another", + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docLineComment("/// Some doc line comment"), + .docLineComment("Other"), + ]).commentValues, + [ + "Some doc line comment", + "Other", + ] + ) + } } From e8cd9f998292f7d135a599782317e29b5db46920 Mon Sep 17 00:00:00 2001 From: Adam Carter <14096630+adamcarter93@users.noreply.github.com> Date: Fri, 29 Mar 2024 02:10:47 +0000 Subject: [PATCH 2/3] Parse comment blocks out to commentValues Blocks of comments are now concatenated with their related lines within the same block comment and added to the commentValues property as one stripped, concatenated string If there are n code blocks, there will be n stripped, concatenated elements in the commentValues (plus x line comments) --- Sources/SwiftSyntax/Trivia.swift | 68 +++++++++---- Sources/SwiftSyntax/Utils.swift | 42 ++++++++ Tests/SwiftSyntaxTest/TriviaTests.swift | 129 ++++++++++++++++++++++++ 3 files changed, 220 insertions(+), 19 deletions(-) diff --git a/Sources/SwiftSyntax/Trivia.swift b/Sources/SwiftSyntax/Trivia.swift index f17612de2dd..cdf4da29118 100644 --- a/Sources/SwiftSyntax/Trivia.swift +++ b/Sources/SwiftSyntax/Trivia.swift @@ -46,15 +46,10 @@ public struct Trivia: Sendable { /// /// Each element in the array is the trimmed contents of a line comment, or, in the case of a multi-line comment a trimmed, concatenated single string. public var commentValues: [String] { - pieces.compactMap { - switch $0 { - case .lineComment(let text), .docLineComment(let text): - sanitizingLineComment(text) - - default: - nil - } - } + [ + sanitizedLineCommentValues, + sanitizedBlockCommentValues, + ].flatMap { $0 } } /// The length of all the pieces in this ``Trivia``. @@ -231,18 +226,53 @@ extension RawTriviaPiece: CustomDebugStringConvertible { } } -private func sanitizingLineComment(_ text: String) -> String { - // TODO: adammcarter - can we import Foundation instead and use trimmingCharacters(in:) - - var charactersToDrop = 0 +private extension Trivia { + var sanitizedLineCommentValues: [String] { + compactMap { + switch $0 { + case .lineComment(let text), .docLineComment(let text): + String(sanitizingLineComment(text)) - for character in text { - if (character == "/" || character == " ") { - charactersToDrop += 1 - } else { - break + default: + nil + } } } - return String(text.dropFirst(charactersToDrop)) + func sanitizingLineComment(_ text: String) -> Substring { + text.trimmingPrefix("/ ") + } + + var sanitizedBlockCommentValues: [String] { + var lines = [String]() + var substrings = [Substring]() + var foundTerminator = false + + for piece in self { + switch piece { + case .blockComment(let text), .docBlockComment(let text): + let sanitized = text.trimmingCharacters(in: "/* ") + + if substrings.isEmpty || sanitized.isEmpty == false { + substrings.append(sanitized) + } + + foundTerminator = text.hasSuffix("*/") + + default: + break + } + + if foundTerminator, substrings.isEmpty == false { + lines.append(substrings.joined(separator: " ")) + substrings.removeAll() + } + } + + if substrings.isEmpty == false { + lines.append(substrings.joined(separator: " ")) + } + + return lines + } } diff --git a/Sources/SwiftSyntax/Utils.swift b/Sources/SwiftSyntax/Utils.swift index b9ac1e85a7c..669172bc7cd 100644 --- a/Sources/SwiftSyntax/Utils.swift +++ b/Sources/SwiftSyntax/Utils.swift @@ -120,3 +120,45 @@ extension RawUnexpectedNodesSyntax { self.init(raw: raw) } } + +extension String { + func trimmingCharacters(in charactersToTrim: any BidirectionalCollection) -> Substring { + // TODO: adammcarter - this feels a bit dirty + self[startIndex...].trimmingPrefix(charactersToTrim).trimmingSuffix(charactersToTrim) + } + + func trimmingPrefix(_ charactersToTrim: any BidirectionalCollection) -> Substring { + self[startIndex...].trimmingPrefix(charactersToTrim) + } + + func trimmingSuffix(_ charactersToTrim: any BidirectionalCollection) -> Substring { + self[startIndex...].trimmingSuffix(charactersToTrim) + } +} + +extension Substring { + func trimmingPrefix(_ charactersToTrim: any BidirectionalCollection) -> Self { + dropFirst(countOfSequentialCharacters(charactersToTrim, in: self)) + } + + func trimmingSuffix(_ charactersToTrim: any BidirectionalCollection) -> Self { + dropLast(countOfSequentialCharacters(charactersToTrim, in: reversed())) + } +} + +private func countOfSequentialCharacters( + _ charactersToCount: any BidirectionalCollection, + in characters: any BidirectionalCollection +) -> Int { + var count = 0 + + for character in characters { + if charactersToCount.contains(character) { + count += 1 + } else { + break + } + } + + return count +} diff --git a/Tests/SwiftSyntaxTest/TriviaTests.swift b/Tests/SwiftSyntaxTest/TriviaTests.swift index 2f8c1878a0b..89c7e738faf 100644 --- a/Tests/SwiftSyntaxTest/TriviaTests.swift +++ b/Tests/SwiftSyntaxTest/TriviaTests.swift @@ -150,5 +150,134 @@ class TriviaTests: XCTestCase { "Other", ] ) + + // MARK: block comment + + XCTAssertEqual( + Trivia(pieces: [.blockComment("")]).commentValues, + [""] + ) + + XCTAssertEqual( + Trivia(pieces: [.blockComment("Some block comment")]).commentValues, + ["Some block comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [.blockComment("/* Some block comment */")]).commentValues, + ["Some block comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .blockComment("/* Some block comment"), + .blockComment("* spread on many lines */"), + ]).commentValues, + [ + "Some block comment spread on many lines" + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .blockComment("/* Some block comment"), + .blockComment("* spread on many lines"), + .blockComment("*/"), + ]).commentValues, + [ + "Some block comment spread on many lines" + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .blockComment("/* Some block comment"), + .blockComment("* spread on many lines */"), + .blockComment("/* Another block comment */"), + ]).commentValues, + [ + "Some block comment spread on many lines", + "Another block comment", + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .blockComment("/* Some block comment"), + .blockComment("* spread on many lines */"), + .newlines(2), + .blockComment("/* Another block comment */"), + ]).commentValues, + [ + "Some block comment spread on many lines", + "Another block comment", + ] + ) + + // MARK: doc block comment + + XCTAssertEqual( + Trivia(pieces: [.docBlockComment("")]).commentValues, + [""] + ) + + XCTAssertEqual( + Trivia(pieces: [.docBlockComment("Some doc block comment")]).commentValues, + ["Some doc block comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [.docBlockComment("/** Some doc block comment */")]).commentValues, + ["Some doc block comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment("/** Some doc block comment"), + .docBlockComment("* spread on many lines */"), + ]).commentValues, + [ + "Some doc block comment spread on many lines" + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment("/** Some doc block comment"), + .docBlockComment("* spread on many lines"), + .docBlockComment("*/"), + ]).commentValues, + [ + "Some doc block comment spread on many lines" + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment("/** Some doc block comment"), + .docBlockComment("* spread on many lines */"), + .docBlockComment("/** Another doc block comment */"), + ]).commentValues, + [ + "Some doc block comment spread on many lines", + "Another doc block comment", + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment("/** Some doc block comment"), + .docBlockComment("* spread on many lines */"), + .newlines(2), + .docBlockComment("/** Another doc block comment */"), + ]).commentValues, + [ + "Some doc block comment spread on many lines", + "Another doc block comment", + ] + ) + + // TODO: adammcarter - mixing trivia of lines and blocks + // TODO: adammcarter - newline chars in the prefix/suffix of code block too } } From 36df0fc380a8a62a797fdbf4258264a0f5f594f7 Mon Sep 17 00:00:00 2001 From: Adam Carter <14096630+adamcarter93@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:47:53 +0000 Subject: [PATCH 3/3] Breakdown comment blocks with new lines This also keeps comment lines within comment blocks with their leading comment markers in tact --- Sources/SwiftSyntax/Trivia.swift | 111 +++++++++++------------ Sources/SwiftSyntax/Utils.swift | 14 +-- Tests/SwiftSyntaxTest/TriviaTests.swift | 114 +++++++++++++++++++++++- 3 files changed, 177 insertions(+), 62 deletions(-) diff --git a/Sources/SwiftSyntax/Trivia.swift b/Sources/SwiftSyntax/Trivia.swift index cdf4da29118..b7f9778e6ac 100644 --- a/Sources/SwiftSyntax/Trivia.swift +++ b/Sources/SwiftSyntax/Trivia.swift @@ -46,10 +46,62 @@ public struct Trivia: Sendable { /// /// Each element in the array is the trimmed contents of a line comment, or, in the case of a multi-line comment a trimmed, concatenated single string. public var commentValues: [String] { - [ - sanitizedLineCommentValues, - sanitizedBlockCommentValues, - ].flatMap { $0 } + var comments = [String]() + var partialComments = [String]() + + var foundStartOfCodeBlock = false + var foundEndOfCodeBlock = false + var isInCodeBlock: Bool { foundStartOfCodeBlock && !foundEndOfCodeBlock } + + for piece in pieces { + switch piece { + case .blockComment(let text), .docBlockComment(let text): + let text = text.trimmingCharacters(in: "\n") + + foundStartOfCodeBlock = text.hasPrefix("/*") + foundEndOfCodeBlock = text.hasSuffix("*/") + + let sanitized = + text + .split(separator: "\n") + .map { $0.trimmingAnyCharacters(in: "/*").trimmingAnyCharacters(in: " ") } + .filter { !$0.isEmpty } + .joined(separator: " ") + + appendPartialCommentIfPossible(sanitized) + + case .lineComment(let text), .docLineComment(let text): + if isInCodeBlock { + appendPartialCommentIfPossible(text) + } else { + comments.append(String(text.trimmingPrefix("/ "))) + } + + default: + break + } + + if foundEndOfCodeBlock, !partialComments.isEmpty { + appendSubstringsToLines() + partialComments.removeAll() + } + } + + if !partialComments.isEmpty { + appendSubstringsToLines() + } + + func appendPartialCommentIfPossible(_ text: String) { + guard partialComments.isEmpty || !text.isEmpty else { return } + + partialComments.append(text) + } + + func appendSubstringsToLines() { + comments.append(partialComments.joined(separator: " ")) + } + + return comments } /// The length of all the pieces in this ``Trivia``. @@ -225,54 +277,3 @@ extension RawTriviaPiece: CustomDebugStringConvertible { TriviaPiece(raw: self).debugDescription } } - -private extension Trivia { - var sanitizedLineCommentValues: [String] { - compactMap { - switch $0 { - case .lineComment(let text), .docLineComment(let text): - String(sanitizingLineComment(text)) - - default: - nil - } - } - } - - func sanitizingLineComment(_ text: String) -> Substring { - text.trimmingPrefix("/ ") - } - - var sanitizedBlockCommentValues: [String] { - var lines = [String]() - var substrings = [Substring]() - var foundTerminator = false - - for piece in self { - switch piece { - case .blockComment(let text), .docBlockComment(let text): - let sanitized = text.trimmingCharacters(in: "/* ") - - if substrings.isEmpty || sanitized.isEmpty == false { - substrings.append(sanitized) - } - - foundTerminator = text.hasSuffix("*/") - - default: - break - } - - if foundTerminator, substrings.isEmpty == false { - lines.append(substrings.joined(separator: " ")) - substrings.removeAll() - } - } - - if substrings.isEmpty == false { - lines.append(substrings.joined(separator: " ")) - } - - return lines - } -} diff --git a/Sources/SwiftSyntax/Utils.swift b/Sources/SwiftSyntax/Utils.swift index 669172bc7cd..68777aa3bcc 100644 --- a/Sources/SwiftSyntax/Utils.swift +++ b/Sources/SwiftSyntax/Utils.swift @@ -124,24 +124,28 @@ extension RawUnexpectedNodesSyntax { extension String { func trimmingCharacters(in charactersToTrim: any BidirectionalCollection) -> Substring { // TODO: adammcarter - this feels a bit dirty - self[startIndex...].trimmingPrefix(charactersToTrim).trimmingSuffix(charactersToTrim) + self[startIndex...].trimmingAnyCharacters(in: charactersToTrim) } func trimmingPrefix(_ charactersToTrim: any BidirectionalCollection) -> Substring { - self[startIndex...].trimmingPrefix(charactersToTrim) + self[startIndex...].trimmingAnyCharactersFromPrefix(in: charactersToTrim) } func trimmingSuffix(_ charactersToTrim: any BidirectionalCollection) -> Substring { - self[startIndex...].trimmingSuffix(charactersToTrim) + self[startIndex...].trimmingAnyCharactersFromSuffix(in: charactersToTrim) } } extension Substring { - func trimmingPrefix(_ charactersToTrim: any BidirectionalCollection) -> Self { + func trimmingAnyCharacters(in charactersToTrim: any BidirectionalCollection) -> Substring { + trimmingAnyCharactersFromPrefix(in: charactersToTrim).trimmingAnyCharactersFromSuffix(in: charactersToTrim) + } + + func trimmingAnyCharactersFromPrefix(in charactersToTrim: any BidirectionalCollection) -> Self { dropFirst(countOfSequentialCharacters(charactersToTrim, in: self)) } - func trimmingSuffix(_ charactersToTrim: any BidirectionalCollection) -> Self { + func trimmingAnyCharactersFromSuffix(in charactersToTrim: any BidirectionalCollection) -> Self { dropLast(countOfSequentialCharacters(charactersToTrim, in: reversed())) } } diff --git a/Tests/SwiftSyntaxTest/TriviaTests.swift b/Tests/SwiftSyntaxTest/TriviaTests.swift index 89c7e738faf..329deee7692 100644 --- a/Tests/SwiftSyntaxTest/TriviaTests.swift +++ b/Tests/SwiftSyntaxTest/TriviaTests.swift @@ -214,6 +214,34 @@ class TriviaTests: XCTestCase { ] ) + XCTAssertEqual( + Trivia(pieces: [ + .blockComment( + """ + /* + Some block comment + spread on many lines + */ + """ + ) + ]).commentValues, + ["Some block comment spread on many lines"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .blockComment( + """ + /* + * Some block comment + * spread on many lines + */ + """ + ) + ]).commentValues, + ["Some block comment spread on many lines"] + ) + // MARK: doc block comment XCTAssertEqual( @@ -277,7 +305,89 @@ class TriviaTests: XCTestCase { ] ) - // TODO: adammcarter - mixing trivia of lines and blocks - // TODO: adammcarter - newline chars in the prefix/suffix of code block too + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment( + """ + /** + Some doc block comment + spread on many lines + */ + """ + ) + ]).commentValues, + ["Some doc block comment spread on many lines"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment( + """ + /** + * Some doc block comment + * spread on many lines + */ + """ + ) + ]).commentValues, + ["Some doc block comment spread on many lines"] + ) + + // MARK: Mixing comment styles + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment( + """ + /** + * Some doc block comment + * // spread on many lines + * with a line comment + */ + """ + ) + ]).commentValues, + ["Some doc block comment // spread on many lines with a line comment"] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment("/** Some doc block comment"), + .docBlockComment("* spread on many lines */"), + .newlines(2), + .docLineComment("/// Some doc line comment"), + .docLineComment("// Some line comment"), + .newlines(2), + .spaces(4), + .blockComment("/* Some block comment"), + .blockComment("* spread on many lines */"), + .newlines(2), + .docBlockComment("/** Another doc block comment */"), + ]).commentValues, + [ + "Some doc block comment spread on many lines", + "Some doc line comment", + "Some line comment", + "Some block comment spread on many lines", + "Another doc block comment", + ] + ) + + XCTAssertEqual( + Trivia(pieces: [ + .docBlockComment("/* Some block comment"), + .docLineComment("// A line comment in a block"), + .docBlockComment("* spread on many lines */"), + .newlines(2), + .blockComment("/** Some doc block comment"), + .docLineComment("/// A doc line comment in a block"), + .blockComment("* spread on"), + .blockComment("* many lines */"), + ]).commentValues, + [ + "Some block comment // A line comment in a block spread on many lines", + "Some doc block comment /// A doc line comment in a block spread on many lines", + ] + ) } }