Fix categorization of newlines as whitespace

bash · Jun 17, 2024 · 9e5a008 · 9e5a008
1 parent 39fa4b4
commit 9e5a008
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 2 deletions.
diff --git a/Broccolini.Test/TestData.cs b/Broccolini.Test/TestData.cs
@@ -2,6 +2,9 @@ namespace Broccolini.Test;
 
 internal static class TestData
 {
+    // This tests against a bug in the tokenizer that classifies whitespace + newline as whitespace.
+    private const string NewLineWithPrecedingWhiteSpace = "\t\n";
+
     // TODO: test parsing with newline at the end
     public static IEnumerable<string> GarbageNodes
         => Sequence.Return(
@@ -41,6 +44,7 @@ public static IEnumerable<SectionWithName> SectionsWithNames
             .SelectMany(VaryClosingBracket)
             .SelectMany(VaryTrailingWhitespace)
             .SelectMany(VaryNewLines)
+            .SelectMany(VaryLeadingNewLines)
             .Distinct();
 
     public static IEnumerable<KeyValuePairWithKeyAndValue> KeyValuePairsWithKeyAndValue
@@ -62,7 +66,8 @@ public static IEnumerable<KeyValuePairWithKeyAndValue> KeyValuePairsWithKeyAndVa
             new KeyValuePairWithKeyAndValue("# key = not a comment", "# key", "not a comment"),
             new KeyValuePairWithKeyAndValue("key = \"quoted value'", "key", "\"quoted value'"),
             new KeyValuePairWithKeyAndValue("key = 'quoted value\"", "key", "'quoted value\""))
-            .Concat(KeyValuePairsWithQuotes);
+            .Concat(KeyValuePairsWithQuotes)
+            .SelectMany(VaryLeadingNewLines);
             // TODO: vary leading and trailing whitespace and line break
 
     public static IEnumerable<string> NewLines => Sequence.Return("\r\n", "\r", "\n");
@@ -120,6 +125,16 @@ private static IEnumerable<SectionWithName> VaryLeadingWhitespace(SectionWithNam
             sectionWithName,
             sectionWithName with { Input = " \t\v\f " + sectionWithName.Input });
 
+    private static IEnumerable<SectionWithName> VaryLeadingNewLines(SectionWithName sectionWithName)
+        => [sectionWithName,
+            sectionWithName with { Input = "\n" + sectionWithName.Input },
+            sectionWithName with { Input = "; comment" + NewLineWithPrecedingWhiteSpace + sectionWithName.Input }];
+
+    private static IEnumerable<KeyValuePairWithKeyAndValue> VaryLeadingNewLines(KeyValuePairWithKeyAndValue keyValuePair)
+        => [keyValuePair,
+            keyValuePair with { Input = "\n" + keyValuePair.Input },
+            keyValuePair with { Input = "; comment" + NewLineWithPrecedingWhiteSpace + keyValuePair.Input }];
+
     private static IEnumerable<SectionWithName> VaryTrailingWhitespace(SectionWithName sectionWithName)
         => Sequence.Return(
             sectionWithName,

diff --git a/Broccolini.Test/TokenizerTest.cs b/Broccolini.Test/TokenizerTest.cs
@@ -30,6 +30,12 @@ public void TokenizesWhiteSpace(char whitespace)
         Assert.Single(Tokenize(whitespace.ToString()), new IniToken.WhiteSpace(whitespace.ToString()));
     }
 
+    [Fact]
+    public void CorrectlyTokenizesNewLineFollowingWhitesSpace()
+    {
+        Assert.Equal([new IniToken.WhiteSpace("\t"), new IniToken.NewLine("\n")], Tokenize("\t\n"));
+    }
+
     private static TheoryData<char> GetWhiteSpaceData() => WhiteSpace.ToTheoryData();
 
     private static TheoryData<string, string> GetConsecutiveNewLinesData()

diff --git a/Broccolini/Tokenization/Rules/CharPredicates.cs b/Broccolini/Tokenization/Rules/CharPredicates.cs
@@ -6,5 +6,5 @@ internal static class CharPredicates
 
     public static bool IsNewLine(char c) => c is '\r' or '\n';
 
-    public static bool IsWhitespace(char c) => c <= ' ';
+    public static bool IsWhitespace(char c) => c <= ' ' && !IsNewLine(c);
 }