Temporary workaround for IfPartPattern in reviewer and other fixes

.github/workflows/dotnet-core.yml

@@ -87,13 +87,22 @@ jobs:
           mv target/nerd-0.0.1-SNAPSHOT.jar ../../release/smi-nerd-$(fgrep AssemblyVersion ../../SharedAssemblyInfo.cs | cut -d'"' -f2).jar
       - name: Package ii binary
         run: |
+          tag="$(fgrep AssemblyVersion SharedAssemblyInfo.cs | cut -d'"' -f2)"
           cd ii
           dotnet publish --runtime win-x64 -c Release --self-contained true -o ../ii-win-x64
-          dotnet publish --runtime linux-x64 -c Release --self-contained true -o ../ii-$(fgrep AssemblyVersion ../SharedAssemblyInfo.cs|cut -d'"' -f2)-cli-linux-x64
+          dotnet publish --runtime linux-x64 -c Release --self-contained true -o ../ii-"$tag"-cli-linux-x64
           cd ../ii-win-x64
-          zip -q9r ../release/ii-$(fgrep AssemblyVersion ../SharedAssemblyInfo.cs|cut -d'"' -f2)-cli-win-x64.zip .
+          zip -q9r ../release/ii-"$tag"-cli-win-x64.zip .
           cd ..
-          tar -zcvf ./release/ii-$(fgrep AssemblyVersion SharedAssemblyInfo.cs|cut -d'"' -f2)-cli-linux-x64.tar.gz ii-$(fgrep AssemblyVersion SharedAssemblyInfo.cs|cut -d'"' -f2)-cli-linux-x64
+          tar -zcvf ./release/ii-"$tag"-cli-linux-x64.tar.gz ii-"$tag"-cli-linux-x64
+          git_tag="$(git rev-parse --short HEAD)"
+          cp ./release/ii-"$tag"-cli-linux-x64.tar.gz ./ii-"$git_tag"-cli-linux-x64.tar.gz
+      - name: Store created ii binary
+        uses: actions/upload-artifact@v3
+        with:
+          name: ii
+          path: "*.tar.gz"
+          retention-days: 1
       - name: Test ii binary
         run: |
           set -e
@@ -122,6 +131,7 @@ jobs:
       - name: Store created nupkg files
         uses: actions/upload-artifact@v3
         with:
+          name: IsIdentifiablePlugin
           path: release/IsIdentifiablePlugin.*.nupkg
           retention-days: 1
       - name: Upload binaries to release

IsIdentifiable/Failures/FailurePart.cs

@@ -1,4 +1,4 @@
-using Equ;
+using Equ;
 
 namespace IsIdentifiable.Failures;
 
@@ -71,4 +71,7 @@ public bool Includes(int start, int length)
 
         return false;
     }
+
+    /// <inheritdoc/>
+    public override string ToString() => $"{nameof(FailurePart)}({Word},{Offset},{Classification})";
 }

IsIdentifiable/Redacting/OutBase.cs

@@ -23,6 +23,11 @@
     /// </summary>
     public List<RegexRule> Rules { get; }
 
+    /// <summary>
+    /// Temp -- do not use.
+    /// </summary>
+    public readonly List<PartPatternFilterRule> PartRules_Temp = new();
+
     /// <summary>
     /// Persistence of <see cref="RulesFile"/>
     /// </summary>
@@ -68,8 +73,11 @@
             else
             {
                 //populated rules file already existed
-                var deserializer = new Deserializer();
-                Rules = deserializer.Deserialize<List<RegexRule>>(existingRules) ?? new List<RegexRule>();
+                 var builder = new DeserializerBuilder();
+                builder.WithTagMapping("!IgnorePartRegexRule", typeof(PartPatternFilterRule));
+                var allRules = builder.Build().Deserialize<List<RegexRule>>(existingRules) ?? new List<RegexRule>();
+                Rules = allRules.OfType<RegexRule>().ToList();
+                PartRules_Temp = allRules.OfType<PartPatternFilterRule>().ToList();
             }
         }
     }
@@ -81,7 +89,7 @@
     /// <param name="action"></param>
     /// <param name="overrideRuleFactory">Overrides the current <see cref="RulesFactory"/> and uses this instead</param>
     /// <returns>The new / existing rule that covers failure</returns>
     protected RegexRule Add(Failure f, RuleAction action, IRulePatternFactory overrideRuleFactory = null)
     {
         var factory = overrideRuleFactory ?? RulesFactory;
 
@@ -175,7 +183,7 @@
     /// Serializes the current <see cref="Rules"/> to the provided file
     /// </summary>
     /// <param name="toFile"></param>
     public void Save(IFileInfo toFile = null)
     {
         toFile ??= RulesFile;
 

IsIdentifiable/Redacting/ReportReader.cs

@@ -1,6 +1,8 @@
 using IsIdentifiable.Failures;
 using IsIdentifiable.Reporting.Reports;
+using IsIdentifiable.Rules;
 using System;
+using System.Collections.Generic;
 using System.IO.Abstractions;
 using System.Linq;
 using System.Threading;
@@ -52,10 +54,10 @@ public ReportReader(IFileInfo csvFile)
     /// <param name="loadedRows"></param>
     /// <param name="token"></param>
     /// <param name="fileSystem"></param>
-    public ReportReader(IFileInfo csvFile, Action<int> loadedRows, IFileSystem fileSystem, CancellationToken token)
+    public ReportReader(IFileInfo csvFile, Action<int> loadedRows, IFileSystem fileSystem, CancellationToken token, List<PartPatternFilterRule>? partRules = null)
     {
         var report = new FailureStoreReport("", 0, fileSystem);
-        Failures = FailureStoreReport.Deserialize(csvFile, loadedRows, token).ToArray();
+        Failures = FailureStoreReport.Deserialize(csvFile, loadedRows, token, partRules).ToArray();
     }
 
     /// <summary>
@@ -77,18 +79,17 @@ public bool Next()
     /// by the total number of <see cref="Failures"/>
     /// </summary>
     /// <param name="index"></param>
-    public void GoTo(int index)
+    public bool GoTo(int index)
     {
+        var original = _current;
         _current = Math.Min(Math.Max(0, index), Failures.Length);
+        return _current != original && (_current != Failures.Length);
     }
 
     /// <summary>
     /// Provides a human readable count of how far through the <see cref="Failures"/>
     /// the <see cref="CurrentIndex"/> is.
     /// </summary>
     /// <returns></returns>
-    public string DescribeProgress()
-    {
-        return $"{_current}/{Failures.Length}";
-    }
+    public string DescribeProgress() => $"{_current + 1}/{Failures.Length}";
 }

IsIdentifiable/Reporting/Reports/FailureStoreReport.cs

@@ -2,12 +2,17 @@
 using IsIdentifiable.Failures;
 using IsIdentifiable.Options;
 using IsIdentifiable.Reporting.Destinations;
+using IsIdentifiable.Rules;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Data;
 using System.IO.Abstractions;
 using System.Linq;
+using System.Net;
+using System.Text.RegularExpressions;
 using System.Threading;
+using System.Threading.Tasks;
 
 namespace IsIdentifiable.Reporting.Reports;
 
@@ -130,44 +135,171 @@
     /// <param name="token">Cancellation token for aborting the file deserialication (and closing the file again)</param>
     /// <returns></returns>
     /// <exception cref="Exception"></exception>
-    public static IEnumerable<Failure> Deserialize(IFileInfo oldFile, Action<int> loadedRows, CancellationToken token)
+    public static IEnumerable<Failure> Deserialize(IFileInfo oldFile, Action<int> loadedRows, CancellationToken token, IEnumerable<PartPatternFilterRule>? partRules = null)
     {
-        var lineNumber = 0;
+        partRules ??= new List<PartPatternFilterRule>();
 
         using var stream = oldFile.OpenRead();
         using var sr = new System.IO.StreamReader(stream);
-        using var r = new CsvReader(sr, System.Globalization.CultureInfo.CurrentCulture);
-        if (r.Read())
-            r.ReadHeader();
+        using var reader = new CsvReader(sr, System.Globalization.CultureInfo.CurrentCulture);
+        if (reader.Read())
+            reader.ReadHeader();
         else
-            yield break;
-        lineNumber++;
-        // "Resource", "ResourcePrimaryKey", "ProblemField", "ProblemValue", "PartWords", "PartClassifications", "PartOffsets" 
+            return Enumerable.Empty<Failure>();
 
-        while (r.Read())
-        {
-            token.ThrowIfCancellationRequested();
-            lineNumber++;
-            var words = r["PartWords"].Split(Separator);
-            var classes = r["PartClassifications"].Split(Separator);
-            var offsets = r["PartOffsets"].Split(Separator);
-
-            var parts = words.Select((t, i) => new FailurePart(
-                t,
-                Enum.TryParse<FailureClassification>(classes[i], true, out var classification) ? classification : throw new Exception($"Invalid failure classification '{classes[i]}' on line {lineNumber}"),
-                int.TryParse(offsets[i], out var offset) ? offset : throw new Exception($"Invalid offset '{offsets[i]}' on line {lineNumber}"))).ToList();
-            yield return new Failure(parts)
+        int totalProcessed = 0;
+        var localTokenSource = new CancellationTokenSource();
+        using var timerTask = Task.Run(
+            async () =>
             {
-                Resource = r["Resource"],
-                ResourcePrimaryKey = r["ResourcePrimaryKey"],
-                ProblemField = r["ProblemField"],
-                ProblemValue = r["ProblemValue"],
-            };
-
-            if (lineNumber % 1000 == 0)
-                loadedRows(lineNumber);
+                while (!token.IsCancellationRequested && !localTokenSource.Token.IsCancellationRequested)
+                {
+                    loadedRows(totalProcessed);
+                    await Task.Delay(TimeSpan.FromSeconds(0.1), token);
+                }
+            },
+            token
+        );
+
+        var failures = new ConcurrentBag<Failure>();
+
+        try
+        {
+            Parallel.ForEach(
+                reader.GetRecords<FailureStoreReportRecord>(),
+                new ParallelOptions
+                {
+                    CancellationToken = token,
+                },
+                (FailureStoreReportRecord row) =>
+                {
+                    if (row.ProblemValue == null)
+                        throw new Exception("ProblemValue was null");
+
+                    var words = row.PartWords.Split(Separator);
+                    var classes = row.PartClassifications.Split(Separator);
+                    var offsets = row.PartOffsets.Split(Separator);
+
+                    var parts = words.Select(
+                        (word, index) => new FailurePart(
+                            word,
+                            Enum.TryParse<FailureClassification>(classes[index], true, out var classification) ? classification : throw new Exception($"Invalid failure classification '{classes[index]}'"),
+                            int.TryParse(offsets[index], out var offset) ? offset : throw new Exception($"Invalid offset '{row.PartOffsets}'")
+                        )
+                    ).ToList();
+
+                    if (row.ProblemField != "PixelData")
+                    {
+                        // Fixes any offsets that have been mangled by file endings etc.
+                        foreach (var part in parts)
+                        {
+                            if (row.ProblemValue.Substring(part.Offset, part.Word.Length) == part.Word)
+                                continue;
+
+                            // Test if the ProblemValue has been HTML escaped
+                            var encodedPartWord = WebUtility.HtmlEncode(part.Word);
+                            try
+                            {
+                                if (row.ProblemValue.Substring(part.Offset, encodedPartWord.Length) == encodedPartWord)
+                                {
+                                    part.Word = encodedPartWord;
+                                    continue;
+                                }
+                            }
+                            catch (ArgumentOutOfRangeException)
+                            { }
+
+                            // Test if the ProblemValue has hidden unicode symbols
+                            var withoutInvisible = Regex.Replace(row.ProblemValue, @"\p{C}+", string.Empty);
+                            if (withoutInvisible.Substring(part.Offset, part.Word.Length) == part.Word)
+                            {
+                                part.Word = row.ProblemValue.Substring(part.Offset, part.Word.Length + 1);
+
+                                if (row.ProblemValue.Substring(part.Offset, part.Word.Length) != part.Word)
+                                    throw new Exception($"Could not fix hidden unicode characters in Failure:\n===\n{row}\n===");
+
+                                continue;
+                            }
+
+                            // Finally, try shifting the offset around to find the word
+                            try
+                            {
+                                FixupOffsets(row, part);
+                            }
+                            catch (ArgumentOutOfRangeException e)
+                            {
+                                throw new Exception($"Could not fixup Offset value in Failure:\n{row}", e);
+                            }
+                        }
+                    }
+
+                    /* TEMP - Filter out any FailureParts covered by an PartPatternFilterRule */
+                    var toRemove = new List<FailurePart>();
+                    foreach (var partRule in partRules)
+                    {
+                        if (!string.IsNullOrWhiteSpace(partRule.IfColumn) && !string.Equals(partRule.IfColumn, row.ProblemField, StringComparison.InvariantCultureIgnoreCase))
+                            continue;
+
+                        foreach (var part in parts.Where(x => partRule.Covers(x, row.ProblemValue)))
+                        {
+                            toRemove.Add(part);
+                            partRule.IncrementUsed();
+                        }
+                    }
+                    parts = parts.Except(toRemove).ToList();
+                    /* TEMP */
+
+                    if (parts.Any())
+                        failures.Add(new Failure(parts)
+                        {
+                            Resource = row.Resource,
+                            ResourcePrimaryKey = row.ResourcePrimaryKey,
+                            ProblemField = row.ProblemField,
+                            ProblemValue = row.ProblemValue,
+                        });
+
+                    Interlocked.Increment(ref totalProcessed);
+                }
+            );
+        }
+        finally
+        {
+            localTokenSource.Cancel();
+            timerTask.Wait();
         }
 
-        loadedRows(lineNumber);
+        loadedRows(totalProcessed);
+
+        return failures;
+    }
+
+    private static void FixupOffsets(FailureStoreReportRecord row, FailurePart part)
+    {
+        // Try looking ahead first, then back
+        var origOffset = part.Offset;
+        try
+        {
+            while (row.ProblemValue.Substring(part.Offset, part.Word.Length) != part.Word)
+                part.Offset++;
+        }
+        catch (ArgumentOutOfRangeException)
+        {
+            part.Offset = origOffset;
+            while (row.ProblemValue.Substring(part.Offset, part.Word.Length) != part.Word)
+                part.Offset--;
+        }
+    }
+
+    internal class FailureStoreReportRecord
+    {
+        public string Resource { get; init; }
+        public string ResourcePrimaryKey { get; init; }
+        public string ProblemField { get; init; }
+        public string ProblemValue { get; init; }
+        public string PartWords { get; init; }
+        public string PartClassifications { get; init; }
+        public string PartOffsets { get; init; }
+
+        public override string ToString() => $"Failure({Resource}|{ResourcePrimaryKey}|{ProblemField}|{ProblemValue}|{PartWords}|{PartClassifications}|{PartOffsets})";
     }
 }