From c3b911df2144c62fd95ecc779875cb7e9241de9f Mon Sep 17 00:00:00 2001 From: Catcher Wong Date: Sun, 30 Jun 2024 14:16:59 +0800 Subject: [PATCH] cmd csv support keyprefix ana (#59) * cmd csv support keyprefix ana #58 Signed-off-by: catcherwong * fix value and update version Signed-off-by: catcherwong --------- Signed-off-by: catcherwong --- build/version.props | 4 +- src/RDBCli/Commands/CsvCommand.cs | 118 ++++++++++++++++++++---- src/RDBCli/Helpers/CommonHelper.Func.cs | 59 ++++++++++++ src/RDBCli/Stats/RdbDataCounter.cs | 59 +----------- 4 files changed, 162 insertions(+), 78 deletions(-) diff --git a/build/version.props b/build/version.props index 7cfc833..39aafe3 100644 --- a/build/version.props +++ b/build/version.props @@ -1,8 +1,8 @@ - 0.9.1 - 0.9.1 + 0.9.2 + 0.9.2 \ No newline at end of file diff --git a/src/RDBCli/Commands/CsvCommand.cs b/src/RDBCli/Commands/CsvCommand.cs index 87560cd..68cb202 100644 --- a/src/RDBCli/Commands/CsvCommand.cs +++ b/src/RDBCli/Commands/CsvCommand.cs @@ -5,6 +5,7 @@ using System.CommandLine.Invocation; using System.Diagnostics; using System.IO; +using System.Linq; using System.Text; using System.Threading.Tasks; using CliCB = RDBCli.Callbacks; @@ -19,8 +20,11 @@ internal class CsvCommand : Command private static Option> _keyPrefixesOption = CommonCLIOptions.KeyPrefixesOption(); private static Option _minIdleOption = CommonCLIOptions.MinIdleOption(); private static Option _minFreqOption = CommonCLIOptions.MinFreqOption(); + private static Option _separatorsOption = CommonCLIOptions.SeparatorsOption(); + private static Option _sepPrefixCountOption = CommonCLIOptions.SepPrefixCountOption(); private static Option _permanentOption = CommonCLIOptions.IsPermanentOption(); private static Option _expiredOption = CommonCLIOptions.IsExpiredOption(); + private static Option _keySuffixEnableOption = CommonCLIOptions.KeySuffixEnableOption(); private static Argument _fileArg = CommonCLIArguments.FileArgument(); public CsvCommand() @@ -32,6 +36,9 @@ public CsvCommand() this.AddOption(_keyPrefixesOption); this.AddOption(_minIdleOption); this.AddOption(_minFreqOption); + this.AddOption(_separatorsOption); + this.AddOption(_sepPrefixCountOption); + this.AddOption(_keySuffixEnableOption); this.AddOption(_permanentOption); this.AddOption(_expiredOption); this.AddArgument(_fileArg); @@ -50,7 +57,7 @@ private void Do(InvocationContext context, CommandOptions options) var cb = new CliCB.MemoryCallback(); var rdbDataInfo = cb.GetRdbDataInfo(); - var counter = new RdbCsvData(rdbDataInfo.Records); + var counter = new RdbCsvData(rdbDataInfo.Records, options.Separators, options.SepPrefixCount, options.keySuffixEnable); var task = counter.Output(options.Output); console.WriteLine($""); @@ -75,6 +82,9 @@ private class CommandOptions { public string Files { get; set; } public string Output { get; set; } + public string Separators { get; set; } + public int SepPrefixCount { get; set; } + public bool keySuffixEnable { get; set; } public RDBParser.ParserFilter ParserFilter { get; set; } public static CommandOptions FromContext(InvocationContext context) @@ -86,6 +96,9 @@ public static CommandOptions FromContext(InvocationContext context) var keyPrefixes = context.ParseResult.GetValueForOption>(_keyPrefixesOption); var minIdle = context.ParseResult.GetValueForOption(_minIdleOption); var minFreq = context.ParseResult.GetValueForOption(_minFreqOption); + var sep = context.ParseResult.GetValueForOption(_separatorsOption); + var sepPrefixCount = context.ParseResult.GetValueForOption(_sepPrefixCountOption); + var keySuffixEnable = context.ParseResult.GetValueForOption(_keySuffixEnableOption); var permanent = context.ParseResult.GetValueForOption(_permanentOption); var expired = context.ParseResult.GetValueForOption(_expiredOption); @@ -104,7 +117,10 @@ public static CommandOptions FromContext(InvocationContext context) { Files = files, Output = output, - ParserFilter = parseFilter + ParserFilter = parseFilter, + Separators = sep, + SepPrefixCount = sepPrefixCount, + keySuffixEnable = keySuffixEnable ?? false }; } } @@ -113,10 +129,20 @@ public static CommandOptions FromContext(InvocationContext context) internal class RdbCsvData { private BlockingCollection _records; - - public RdbCsvData(BlockingCollection records) + private readonly char[] _separators; + private readonly int _sepCount; + private readonly bool _keySuffixEnable; + private Dictionary _prefixDict; + public RdbCsvData(BlockingCollection records, string separators = "", int sepCount = -1, bool keySuffixEnable = false) { this._records = records; + if (!string.IsNullOrWhiteSpace(separators)) + { + this._separators = separators.ToCharArray(); + } + this._sepCount = sepCount > 0 ? sepCount : 1; + this._keySuffixEnable = keySuffixEnable; + this._prefixDict = new Dictionary(); } public Task Output(string output) @@ -140,30 +166,86 @@ public Task Output(string output) System.Threading.CancellationTokenSource cts = new System.Threading.CancellationTokenSource(); var task = Task.Factory.StartNew(() => { - using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None)) + if (_separators != null && _separators.Any()) { - // overwrite - fs.SetLength(0); - var header = Encoding.UTF8.GetBytes("database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry,idle,freq\n"); - fs.Write(header); - while (!_records.IsCompleted) { - try + if (_records.TryTake(out var item)) { - if (_records.TryTake(out var item)) + var prefixs = CommonHelper.GetPrefixes(item.Record.Key, _separators, _sepCount, _keySuffixEnable); + + foreach (var p in prefixs) { - var line = Encoding.UTF8.GetBytes($"{item.Record.Database},{item.Record.Type},{item.Record.Key},{item.Record.Bytes},{item.Record.Encoding},{item.Record.NumOfElem},{item.Record.LenOfLargestElem},{CommonHelper.GetExpireString(item.Record.Expiry)},{item.Record.Idle},{item.Record.Freq}\n"); - fs.Write(line); + var key = $"{item.Record.Database}!{item.Record.Type}!{p}"; + if (_prefixDict.ContainsKey(key)) + { + var value = _prefixDict[key]; + var i1 = value.Item1 + (long)item.Record.Bytes; + var i2 = value.Item2 + 1; + + _prefixDict[key] = (i1, i2); + } + else + { + _prefixDict.Add($"{item.Record.Database}!{item.Record.Type}!{p}", ((long)item.Record.Bytes, 1)); + } } - else + } + else + { + continue; + } + } + + try + { + using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None)) + { + // overwrite + fs.SetLength(0); + var header = Encoding.UTF8.GetBytes("database,type,key_prefix,size_in_bytes,count\n"); + fs.Write(header); + + foreach (var item in _prefixDict) { - continue; + var keys = item.Key.Split('!'); + var line = Encoding.UTF8.GetBytes($"{keys[0]},{keys[1]},{keys[2]},{item.Value.Item1},{item.Value.Item2}\n"); + fs.Write(line); } } - catch (Exception ex) + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + } + else + { + using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None)) + { + // overwrite + fs.SetLength(0); + var header = Encoding.UTF8.GetBytes("database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry,idle,freq\n"); + fs.Write(header); + + while (!_records.IsCompleted) { - Console.WriteLine(ex.Message); + try + { + if (_records.TryTake(out var item)) + { + var line = Encoding.UTF8.GetBytes($"{item.Record.Database},{item.Record.Type},{item.Record.Key},{item.Record.Bytes},{item.Record.Encoding},{item.Record.NumOfElem},{item.Record.LenOfLargestElem},{CommonHelper.GetExpireString(item.Record.Expiry)},{item.Record.Idle},{item.Record.Freq}\n"); + fs.Write(line); + } + else + { + continue; + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } } } } diff --git a/src/RDBCli/Helpers/CommonHelper.Func.cs b/src/RDBCli/Helpers/CommonHelper.Func.cs index 00feee3..64d276a 100644 --- a/src/RDBCli/Helpers/CommonHelper.Func.cs +++ b/src/RDBCli/Helpers/CommonHelper.Func.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; namespace RDBCli { @@ -191,5 +193,62 @@ internal static string GetShortKey(string key) return key; } + + public static List GetPrefixes(string s, char[] separators, int sepCount, bool keySuffixEnable) + { + var res = new List(); + + var span = s.AsSpan(); + + if (!keySuffixEnable) + { + var sepIdx = span.IndexOfAny(separators); + + if (sepIdx < 0) res.Add(s); + + while (sepIdx > -1) + { + var str = new string(span[..(sepIdx + 1)]); + + if (res.Any()) + { + str = string.Concat(res[^1], str); + } + + res.Add(str); + + span = span[(sepIdx + 1)..]; + sepIdx = span.IndexOfAny(separators); + } + } + else + { + var sepIdx = span.LastIndexOfAny(separators); + + if (sepIdx < 0) res.Add(s); + + while (sepIdx > -1) + { + var str = new string(span[(sepIdx + 1)..]) + span[sepIdx]; + + if (res.Any()) + { + str = string.Concat(res[^1], str); + } + + res.Add(str); + + span = span[..sepIdx]; + sepIdx = span.LastIndexOfAny(separators); + } + } + + for (int i = 0; i < res.Count; i++) + { + res[i] = res[i].TrimEnd(separators); + } + + return res.Distinct().Take(sepCount).ToList(); + } } } diff --git a/src/RDBCli/Stats/RdbDataCounter.cs b/src/RDBCli/Stats/RdbDataCounter.cs index 88cb832..3047a31 100644 --- a/src/RDBCli/Stats/RdbDataCounter.cs +++ b/src/RDBCli/Stats/RdbDataCounter.cs @@ -174,7 +174,7 @@ private void CountStreams(StreamsRecord streamsRecord, int num) private void CountByKeyPrefix(Record record) { - var prefixes = GetPrefixes(record.Key); + var prefixes = CommonHelper.GetPrefixes(record.Key, _separators, _sepCount, _keySuffixEnable); var tKey = new TypeKey { Type = record.Type }; @@ -204,63 +204,6 @@ private void CountByKeyPrefix(Record record) } } - private List GetPrefixes(string s) - { - var res = new List(); - - var span = s.AsSpan(); - - if (!_keySuffixEnable) - { - var sepIdx = span.IndexOfAny(_separators); - - if (sepIdx < 0) res.Add(s); - - while (sepIdx > -1) - { - var str = new string(span[..(sepIdx + 1)]); - - if (res.Any()) - { - str = string.Concat(res[^1], str); - } - - res.Add(str); - - span = span[(sepIdx + 1)..]; - sepIdx = span.IndexOfAny(_separators); - } - } - else - { - var sepIdx = span.LastIndexOfAny(_separators); - - if (sepIdx < 0) res.Add(s); - - while (sepIdx > -1) - { - var str = new string(span[(sepIdx + 1)..]) + span[sepIdx]; - - if (res.Any()) - { - str = string.Concat(res[^1], str); - } - - res.Add(str); - - span = span[..sepIdx]; - sepIdx = span.LastIndexOfAny(_separators); - } - } - - for (int i = 0; i < res.Count; i++) - { - res[i] = res[i].TrimEnd(_separators); - } - - return res.Distinct().Take(_sepCount).ToList(); - } - private void CountLargestEntries(Record record, int num) { record.Key = CommonHelper.GetShortKey(record.Key);