Log indexing

src/Nethermind/Directory.Packages.props

@@ -78,11 +78,12 @@
     <PackageVersion Include="Snappier" Version="1.1.6" />
     <PackageVersion Include="System.CommandLine" Version="2.0.0-beta4.24528.1" />
     <PackageVersion Include="System.Configuration.ConfigurationManager" Version="9.0.1" />
+    <PackageVersion Include="System.IO.Hashing" Version="9.0.1" />
     <PackageVersion Include="System.IO.Pipelines" Version="9.0.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="9.0.1" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.TestingHelpers" Version="21.2.1" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.Wrappers" Version="21.2.1" />
     <PackageVersion Include="Websocket.Client" Version="5.1.2" />
   </ItemGroup>
-</Project>
+</Project>

src/Nethermind/Nethermind.Logs.Test/BinaryEncodingTests.cs

@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using FluentAssertions;
+using NUnit.Framework;
+
+namespace Nethermind.Logs.Test;
+
+public class BinaryEncodingTests
+{
+    [TestCase(0u)]
+    [TestCase(1u)]
+    [TestCase(0xFFu)]
+    [TestCase(0xFFFFu)]
+    [TestCase(0xFFFFFu)]
+    [TestCase(0xFFFFFFu)]
+    [TestCase(0xFFFFFFFFu)]
+    public void Test(uint value)
+    {
+        Span<byte> buffer = stackalloc byte[8];
+
+        var written = BinaryEncoding.WriteVarInt(value, buffer);
+        BinaryEncoding.TryReadVarInt(buffer, 0, out var read).Should().Be(written);
+        read.Should().Be(value);
+    }
+}

src/Nethermind/Nethermind.Logs.Test/LogBuilderTests.cs

@@ -0,0 +1,173 @@
+using System.Buffers;
+using FluentAssertions;
+using Nethermind.Core;
+using Nethermind.Core.Crypto;
+using NUnit.Framework;
+
+namespace Nethermind.Logs.Test;
+
+public class LogBuilderTests
+{
+    private const int HashCount = 256;
+    private readonly Hash256[] _hashes;
+
+    public LogBuilderTests()
+    {
+        var random = new Random(13);
+        Span<byte> span = stackalloc byte[Hash256.Size];
+
+        _hashes = new Hash256[HashCount];
+
+        for (int i = 0; i < HashCount; i++)
+        {
+            random.NextBytes(span);
+            _hashes[i] = new Hash256(span);
+        }
+    }
+
+    [Test]
+    public void Simple()
+    {
+        var builder = new LogsBuilder();
+        var hash0 = _hashes[0];
+
+        var entry0 = new LogEntry(Address.SystemUser, [], [hash0]);
+        var entry1 = new LogEntry(Address.MaxValue, [], [hash0]);
+
+        const ushort block = 1;
+        const ushort tx1 = 1;
+        const ushort tx2 = 2;
+
+        builder.Append(entry0, block, tx1);
+        builder.Append(entry1, block, tx2);
+
+        var writer = new ArrayBufferWriter<byte>();
+
+        builder.Build(writer);
+
+        var reader = new LogsBuilder.MemoryReader(writer.WrittenMemory);
+
+        LogsBuilder.Entry e1 = new(block, tx1);
+        LogsBuilder.Entry e2 = new(block, tx2);
+
+        reader.Find(Address.SystemUser).ToArray().Should().BeEquivalentTo([e1]);
+        reader.Find(Address.MaxValue).ToArray().Should().BeEquivalentTo([e2]);
+
+        reader.Find(hash0).ToArray().Should().BeEquivalentTo([e1, e2]);
+
+        reader.Find(hash0, 1).Should().BeEmpty();
+
+        writer.WrittenCount.Should().Be(48);
+    }
+
+    [Test]
+    public void Entries_are_always_ordered_per_block()
+    {
+        var builder = new LogsBuilder();
+        var hash0 = _hashes[0];
+
+        var entry = new LogEntry(Address.SystemUser, [], [hash0]);
+
+        const ushort block1 = 1;
+        const ushort block2 = 2;
+        const ushort tx1 = 1;
+        const ushort tx2 = 2;
+
+        // Report it with different ordering
+        builder.Append(entry, block2, tx2);
+        builder.Append(entry, block1, tx2);
+        builder.Append(entry, block1, tx1);
+
+        var writer = new ArrayBufferWriter<byte>();
+
+        builder.Build(writer);
+
+        var reader = new LogsBuilder.MemoryReader(writer.WrittenMemory);
+
+        LogsBuilder.Entry e1 = new(block1, tx1);
+        LogsBuilder.Entry e2 = new(block1, tx2);
+        LogsBuilder.Entry e3 = new(block2, tx2);
+
+        // order by number of block then by tx
+        LogsBuilder.Entry[] expected = [e1, e2, e3];
+
+        reader.Find(Address.SystemUser).Should().BeEquivalentTo(expected);
+        reader.Find(hash0).ToArray().Should().BeEquivalentTo(expected);
+
+        reader.Find(hash0, 1).Should().BeEmpty();
+
+        writer.WrittenCount.Should().Be(50);
+    }
+
+    [Test]
+    public void Frequent_topics_are_compressed_well()
+    {
+        var builder = new LogsBuilder();
+        var hash0 = _hashes[0];
+
+        var entry = new LogEntry(Address.SystemUser, [], [hash0]);
+
+        const int blocks = 100;
+        const int txs = 100;
+        const int logEntries = blocks * txs;
+
+        foreach ((uint block, ushort tx) in Builder())
+        {
+            builder.Append(entry, block, tx);
+        }
+
+        var writer = new ArrayBufferWriter<byte>();
+
+        builder.Build(writer);
+
+        var reader = new LogsBuilder.MemoryReader(writer.WrittenMemory);
+
+        reader.Find(Address.SystemUser)
+            .Should()
+            .BeEquivalentTo(Builder().Select(t => new LogsBuilder.Entry(t.block, t.tx)));
+
+        Console.WriteLine($"{(double)writer.WrittenCount / logEntries:F1} bytes per {nameof(LogEntry)}");
+        return;
+
+        static IEnumerable<(uint block, ushort tx)> Builder()
+        {
+            for (uint i = 1; i < blocks; i++)
+            {
+                for (ushort j = 1; j < txs; j++)
+                {
+                    yield return (i, j);
+                }
+            }
+        }
+    }
+
+    [Test]
+    public void Repeated_entries_should_be_deduplicated()
+    {
+        var builder = new LogsBuilder();
+
+        var entry = new LogEntry(Address.SystemUser, [], [_hashes[0]]);
+
+        const int logsReported = 1000;
+        const uint block = 1;
+        const ushort tx = 1;
+
+        // Report a lot of times with the same position, to replicate a complex exchange where a lot of
+        // Transfer (index_topic_1 address src, index_topic_2 address dst, uint256 wad) is done.
+        for (ushort i = 0; i < logsReported; i++)
+        {
+            builder.Append(entry, block, tx);
+        }
+
+        var writer = new ArrayBufferWriter<byte>();
+
+        builder.Build(writer);
+
+        var reader = new LogsBuilder.MemoryReader(writer.WrittenMemory);
+
+        LogsBuilder.Entry e = new(block, tx);
+
+        reader.Find(Address.SystemUser).ToArray().Should().BeEquivalentTo([e]);
+        writer.WrittenCount.Should().Be(42);
+    }
+}

src/Nethermind/Nethermind.Logs.Test/Nethermind.Logs.Test.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="FluentAssertions" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="NSubstitute" />
+    <PackageReference Include="NUnit" />
+    <PackageReference Include="NUnit.Analyzers">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="NUnit3TestAdapter" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Nethermind.Logs\Nethermind.Logs.csproj" />
+  </ItemGroup>
+
+</Project>

src/Nethermind/Nethermind.Logs/BinaryEncoding.cs

@@ -0,0 +1,56 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+namespace Nethermind.Logs;
+
+public static class BinaryEncoding
+{
+    public static int TryReadVarInt(ReadOnlySpan<byte> span, int offset, out uint value)
+    {
+        if ((uint)offset >= (uint)span.Length)
+        {
+            value = 0;
+            return 0;
+        }
+
+        const int bits = 7;
+
+        value = span[offset++];
+        if ((value & 0x80) == 0) return 1;
+        value &= 0x7F;
+
+        if ((uint)offset >= (uint)span.Length) return -1;
+        uint chunk = span[offset++];
+        value |= (chunk & 0x7F) << (1 * bits);
+        if ((chunk & 0x80) == 0) return 2;
+
+        if ((uint)offset >= (uint)span.Length) return -1;
+        chunk = span[offset++];
+        value |= (chunk & 0x7F) << (2 * bits);
+        if ((chunk & 0x80) == 0) return 3;
+
+        if ((uint)offset >= (uint)span.Length) return -1;
+        chunk = span[offset++];
+        value |= (chunk & 0x7F) << (3 * bits);
+        if ((chunk & 0x80) == 0) return 4;
+
+        // Use 32 - 28 bits as the last one
+        value |= (chunk & 0b00001111) << (4 * bits);
+        return MaxVarIntByteCount;
+    }
+
+    public const int MaxVarIntByteCount = 5;
+
+    public static int WriteVarInt(uint value, Span<byte> span, int offset = 0)
+    {
+        int count = 0;
+        do
+        {
+            span[offset++] = (byte)((value & 0x7F) | 0x80);
+            count++;
+        } while ((value >>= 7) != 0);
+
+        span[offset - 1] &= 0x7F;
+        return count;
+    }
+}

src/Nethermind/Nethermind.Logs/CountingBufferWriter.cs

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Buffers;
+
+namespace Nethermind.Logs;
+
+public sealed class CountingBufferWriter<T>(IBufferWriter<T> writer) : IBufferWriter<T>
+{
+    public int WrittenCount { get; private set; }
+
+    public void Advance(int count)
+    {
+        writer.Advance(count);
+        WrittenCount += count;
+    }
+
+    public Memory<T> GetMemory(int sizeHint = 0) => writer.GetMemory(sizeHint);
+
+    public Span<T> GetSpan(int sizeHint = 0) => writer.GetSpan(sizeHint);
+
+    public override string ToString() => $"WrittenCount: {WrittenCount}";
+}