support set listpack redis7.2 (#69)

* support set listpack redis7.2 Signed-off-by: catcherwong <[email protected]> * fix set listpack numEle issue Signed-off-by: catcherwong <[email protected]> * add more test for set listpack Signed-off-by: catcherwong <[email protected]> --------- Signed-off-by: catcherwong <[email protected]>
catcherwong · Sep 30, 2024 · 11b3b5b · 11b3b5b
1 parent ee9a142
commit 11b3b5b
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 2 deletions.
diff --git a/src/RDBParser/BinaryReaderRDBParser.Base.cs b/src/RDBParser/BinaryReaderRDBParser.Base.cs
@@ -108,7 +108,8 @@ public void ReadObject(BinaryReader br, byte[] key, int encType, long expiry, In
                 ReadModule(br);
             }
             else if (encType == Constant.DataType.STREAM_LISTPACKS
-                || encType == Constant.DataType.STREAM_LISTPACKS_2)
+                || encType == Constant.DataType.STREAM_LISTPACKS_2
+                || encType == Constant.DataType.STREAM_LISTPACKS_3)
             {
                 ReadStream(br, encType);
             }
@@ -120,6 +121,10 @@ public void ReadObject(BinaryReader br, byte[] key, int encType, long expiry, In
             {
                 ReadZSetFromListPack(br);
             }
+            else if (encType == Constant.DataType.SET_LISTPACK)
+            {
+                ReadSetFromListPack(br);
+            }
             else
             {
                 throw new RDBParserException($"Invalid object type {encType} for {key} ");
@@ -207,6 +212,10 @@ private void SkipObject(BinaryReader br, int encType)
             {
                 skip = 1;
             }
+            else if (encType == Constant.DataType.SET_LISTPACK)
+            {
+                skip = 1;
+            }
             else
             {
                 throw new RDBParserException($"Invalid object type {encType} for {_key} ");

diff --git a/src/RDBParser/BinaryReaderRDBParser.ListPack.cs b/src/RDBParser/BinaryReaderRDBParser.ListPack.cs
@@ -235,6 +235,40 @@ private void ReadZSetFromListPack(BinaryReader br)
             _callback.EndSortedSet(_key);
         }
 
+        private void ReadSetFromListPack(BinaryReader br)
+        {
+            // <total_bytes><size><entry><entry>..<entry><end>
+            var rawString = br.ReadStr();
+            using MemoryStream stream = new MemoryStream(rawString);
+            using var rd = new BinaryReader(stream);
+
+            // <total_bytes>
+            var bytes = lpGetTotalBytes(rd);
+            // <size>
+            var numEle = lpGetNumElements(rd);
+
+            var numEntries = (ushort)numEle;
+
+            Info info = new Info();
+            info.Idle = _idle;
+            info.Freq = _freq;
+            info.Encoding = Constant.ObjEncoding.LISTPACK;
+            info.SizeOfValue = rawString.Length;
+            _callback.StartSet(_key, numEntries, _expiry, info);
+
+            for (int i = 0; i < numEntries; i++)
+            {
+                var entry = ReadListPackEntry(rd);
+
+                _callback.SAdd(_key, entry.data);
+            }
+
+            var lpEnd = rd.ReadByte();
+            if (lpEnd != 0xFF) throw new RDBParserException($"Invalid list pack end - {lpEnd} for key {_key}");
+
+            _callback.EndSet(_key);
+        }
+
         private uint lpGetTotalBytes(BinaryReader br)
         {
             return (uint)br.ReadByte() << 0 |

diff --git a/src/RDBParser/Constant.cs b/src/RDBParser/Constant.cs
@@ -91,6 +91,12 @@ public static class DataType
             public const int ZSET_LISTPACK = 17;
             public const int LIST_QUICKLIST_2 = 18;
             public const int STREAM_LISTPACKS_2 = 19;
+            public const int SET_LISTPACK = 20;
+            public const int STREAM_LISTPACKS_3 = 21;
+            public const int HASH_METADATA_PRE_GA = 22;
+            public const int HASH_LISTPACK_EX_PRE_GA = 23;
+            public const int HASH_METADATA = 24;
+            public const int HASH_LISTPACK_EX = 25;
 
             public static readonly System.Collections.Generic.Dictionary<int, string> MAPPING = new System.Collections.Generic.Dictionary<int, string>
             {
@@ -113,6 +119,12 @@ public static class DataType
                 { 17, "sortedset" },
                 { 18, "list" },
                 { 19, "stream" },
+                { 20, "set" },
+                { 21, "stream" },
+                { 22, "hash" },
+                { 23, "hash" },
+                { 24, "hash" },
+                { 25, "hash" },
             };
         }
 

diff --git a/tests/RDBParserTests/SetTests.cs b/tests/RDBParserTests/SetTests.cs
@@ -104,6 +104,26 @@ public void TestRegularSet()
             {
                 Assert.Contains(Encoding.UTF8.GetBytes(item), sets[0][Encoding.UTF8.GetBytes("regular_set")]);
             }
-        }        
+        }
+
+        [Fact]
+        public void TestListPackSet_72()
+        {
+            var path = TestHelper.GetRDBPath("redis_72_set_listpack.rdb");
+
+            var callback = new TestReaderCallback(_output);
+            var parser = new BinaryReaderRDBParser(callback);
+            parser.Parse(path);
+
+            var sets = callback.GetSets();
+            var lengths = callback.GetLengths();
+
+            Assert.Equal(7, lengths[0][Encoding.UTF8.GetBytes("testrdb")]);
+
+            foreach (var item in new List<string> { "v1", "v2", "100", "-1000", 0x7ffe.ToString(), 0x7ffefffe.ToString(), 0x7ffefffefffefffe .ToString()})
+            {
+                Assert.Contains(Encoding.UTF8.GetBytes(item), sets[0][Encoding.UTF8.GetBytes("testrdb")]);
+            }
+        }
     }
 }
diff --git a/tests/dumps/redis_72_set_listpack.rdb b/tests/dumps/redis_72_set_listpack.rdb