Skip to content

Commit

Permalink
cmd csv support keyprefix ana (#59)
Browse files Browse the repository at this point in the history 
* cmd csv support keyprefix ana #58

Signed-off-by: catcherwong <[email protected]>

* fix value and update version

Signed-off-by: catcherwong <[email protected]>

---------

Signed-off-by: catcherwong <[email protected]>
  • Loading branch information
@catcherwong
catcherwong authored Jun 30, 2024
1 parent 1742461 commit c3b911d
Show file tree
Hide file tree
Showing 4 changed files with 162 additions and 78 deletions.
4 changes: 2 additions & 2 deletions build/version.props
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<Project>
<PropertyGroup>

<RDBParserVersion>0.9.1</RDBParserVersion>
<RDBCliVersion>0.9.1</RDBCliVersion>
<RDBParserVersion>0.9.2</RDBParserVersion>
<RDBCliVersion>0.9.2</RDBCliVersion>

</PropertyGroup>
</Project>
118 changes: 100 additions & 18 deletions src/RDBCli/Commands/CsvCommand.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.CommandLine.Invocation;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using CliCB = RDBCli.Callbacks;
Expand All @@ -19,8 +20,11 @@ internal class CsvCommand : Command
private static Option<List<string>> _keyPrefixesOption = CommonCLIOptions.KeyPrefixesOption();
private static Option<ulong?> _minIdleOption = CommonCLIOptions.MinIdleOption();
private static Option<int?> _minFreqOption = CommonCLIOptions.MinFreqOption();
private static Option<string> _separatorsOption = CommonCLIOptions.SeparatorsOption();
private static Option<int> _sepPrefixCountOption = CommonCLIOptions.SepPrefixCountOption();
private static Option<bool?> _permanentOption = CommonCLIOptions.IsPermanentOption();
private static Option<bool?> _expiredOption = CommonCLIOptions.IsExpiredOption();
private static Option<bool?> _keySuffixEnableOption = CommonCLIOptions.KeySuffixEnableOption();
private static Argument<string> _fileArg = CommonCLIArguments.FileArgument();

public CsvCommand()
Expand All @@ -32,6 +36,9 @@ public CsvCommand()
this.AddOption(_keyPrefixesOption);
this.AddOption(_minIdleOption);
this.AddOption(_minFreqOption);
this.AddOption(_separatorsOption);
this.AddOption(_sepPrefixCountOption);
this.AddOption(_keySuffixEnableOption);
this.AddOption(_permanentOption);
this.AddOption(_expiredOption);
this.AddArgument(_fileArg);
Expand All @@ -50,7 +57,7 @@ private void Do(InvocationContext context, CommandOptions options)
var cb = new CliCB.MemoryCallback();
var rdbDataInfo = cb.GetRdbDataInfo();

var counter = new RdbCsvData(rdbDataInfo.Records);
var counter = new RdbCsvData(rdbDataInfo.Records, options.Separators, options.SepPrefixCount, options.keySuffixEnable);
var task = counter.Output(options.Output);

console.WriteLine($"");
Expand All @@ -75,6 +82,9 @@ private class CommandOptions
{
public string Files { get; set; }
public string Output { get; set; }
public string Separators { get; set; }
public int SepPrefixCount { get; set; }
public bool keySuffixEnable { get; set; }
public RDBParser.ParserFilter ParserFilter { get; set; }

public static CommandOptions FromContext(InvocationContext context)
Expand All @@ -86,6 +96,9 @@ public static CommandOptions FromContext(InvocationContext context)
var keyPrefixes = context.ParseResult.GetValueForOption<List<string>>(_keyPrefixesOption);
var minIdle = context.ParseResult.GetValueForOption<ulong?>(_minIdleOption);
var minFreq = context.ParseResult.GetValueForOption<int?>(_minFreqOption);
var sep = context.ParseResult.GetValueForOption<string>(_separatorsOption);
var sepPrefixCount = context.ParseResult.GetValueForOption<int>(_sepPrefixCountOption);
var keySuffixEnable = context.ParseResult.GetValueForOption<bool?>(_keySuffixEnableOption);
var permanent = context.ParseResult.GetValueForOption<bool?>(_permanentOption);
var expired = context.ParseResult.GetValueForOption<bool?>(_expiredOption);

Expand All @@ -104,7 +117,10 @@ public static CommandOptions FromContext(InvocationContext context)
{
Files = files,
Output = output,
ParserFilter = parseFilter
ParserFilter = parseFilter,
Separators = sep,
SepPrefixCount = sepPrefixCount,
keySuffixEnable = keySuffixEnable ?? false
};
}
}
Expand All @@ -113,10 +129,20 @@ public static CommandOptions FromContext(InvocationContext context)
internal class RdbCsvData
{
private BlockingCollection<AnalysisRecord> _records;

public RdbCsvData(BlockingCollection<AnalysisRecord> records)
private readonly char[] _separators;
private readonly int _sepCount;
private readonly bool _keySuffixEnable;
private Dictionary<string, (long, long)> _prefixDict;
public RdbCsvData(BlockingCollection<AnalysisRecord> records, string separators = "", int sepCount = -1, bool keySuffixEnable = false)
{
this._records = records;
if (!string.IsNullOrWhiteSpace(separators))
{
this._separators = separators.ToCharArray();
}
this._sepCount = sepCount > 0 ? sepCount : 1;
this._keySuffixEnable = keySuffixEnable;
this._prefixDict = new Dictionary<string, (long, long)>();
}

public Task<string> Output(string output)
Expand All @@ -140,30 +166,86 @@ public Task<string> Output(string output)
System.Threading.CancellationTokenSource cts = new System.Threading.CancellationTokenSource();
var task = Task.Factory.StartNew(() =>
{
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None))
if (_separators != null && _separators.Any())
{
// overwrite
fs.SetLength(0);
var header = Encoding.UTF8.GetBytes("database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry,idle,freq\n");
fs.Write(header);
while (!_records.IsCompleted)
{
try
if (_records.TryTake(out var item))
{
if (_records.TryTake(out var item))
var prefixs = CommonHelper.GetPrefixes(item.Record.Key, _separators, _sepCount, _keySuffixEnable);
foreach (var p in prefixs)
{
var line = Encoding.UTF8.GetBytes($"{item.Record.Database},{item.Record.Type},{item.Record.Key},{item.Record.Bytes},{item.Record.Encoding},{item.Record.NumOfElem},{item.Record.LenOfLargestElem},{CommonHelper.GetExpireString(item.Record.Expiry)},{item.Record.Idle},{item.Record.Freq}\n");
fs.Write(line);
var key = $"{item.Record.Database}!{item.Record.Type}!{p}";
if (_prefixDict.ContainsKey(key))
{
var value = _prefixDict[key];
var i1 = value.Item1 + (long)item.Record.Bytes;
var i2 = value.Item2 + 1;
_prefixDict[key] = (i1, i2);
}
else
{
_prefixDict.Add($"{item.Record.Database}!{item.Record.Type}!{p}", ((long)item.Record.Bytes, 1));
}
}
else
}
else
{
continue;
}
}
try
{
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None))
{
// overwrite
fs.SetLength(0);
var header = Encoding.UTF8.GetBytes("database,type,key_prefix,size_in_bytes,count\n");
fs.Write(header);
foreach (var item in _prefixDict)
{
continue;
var keys = item.Key.Split('!');
var line = Encoding.UTF8.GetBytes($"{keys[0]},{keys[1]},{keys[2]},{item.Value.Item1},{item.Value.Item2}\n");
fs.Write(line);
}
}
catch (Exception ex)
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
else
{
using (FileStream fs = new FileStream(path, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None))
{
// overwrite
fs.SetLength(0);
var header = Encoding.UTF8.GetBytes("database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry,idle,freq\n");
fs.Write(header);
while (!_records.IsCompleted)
{
Console.WriteLine(ex.Message);
try
{
if (_records.TryTake(out var item))
{
var line = Encoding.UTF8.GetBytes($"{item.Record.Database},{item.Record.Type},{item.Record.Key},{item.Record.Bytes},{item.Record.Encoding},{item.Record.NumOfElem},{item.Record.LenOfLargestElem},{CommonHelper.GetExpireString(item.Record.Expiry)},{item.Record.Idle},{item.Record.Freq}\n");
fs.Write(line);
}
else
{
continue;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
}
Expand Down
59 changes: 59 additions & 0 deletions src/RDBCli/Helpers/CommonHelper.Func.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;

namespace RDBCli
{
Expand Down Expand Up @@ -191,5 +193,62 @@ internal static string GetShortKey(string key)

return key;
}

public static List<string> GetPrefixes(string s, char[] separators, int sepCount, bool keySuffixEnable)
{
var res = new List<string>();

var span = s.AsSpan();

if (!keySuffixEnable)
{
var sepIdx = span.IndexOfAny(separators);

if (sepIdx < 0) res.Add(s);

while (sepIdx > -1)
{
var str = new string(span[..(sepIdx + 1)]);

if (res.Any())
{
str = string.Concat(res[^1], str);
}

res.Add(str);

span = span[(sepIdx + 1)..];
sepIdx = span.IndexOfAny(separators);
}
}
else
{
var sepIdx = span.LastIndexOfAny(separators);

if (sepIdx < 0) res.Add(s);

while (sepIdx > -1)
{
var str = new string(span[(sepIdx + 1)..]) + span[sepIdx];

if (res.Any())
{
str = string.Concat(res[^1], str);
}

res.Add(str);

span = span[..sepIdx];
sepIdx = span.LastIndexOfAny(separators);
}
}

for (int i = 0; i < res.Count; i++)
{
res[i] = res[i].TrimEnd(separators);
}

return res.Distinct().Take(sepCount).ToList();
}
}
}
59 changes: 1 addition & 58 deletions src/RDBCli/Stats/RdbDataCounter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ private void CountStreams(StreamsRecord streamsRecord, int num)

private void CountByKeyPrefix(Record record)
{
var prefixes = GetPrefixes(record.Key);
var prefixes = CommonHelper.GetPrefixes(record.Key, _separators, _sepCount, _keySuffixEnable);

var tKey = new TypeKey { Type = record.Type };

Expand Down Expand Up @@ -204,63 +204,6 @@ private void CountByKeyPrefix(Record record)
}
}

private List<string> GetPrefixes(string s)
{
var res = new List<string>();

var span = s.AsSpan();

if (!_keySuffixEnable)
{
var sepIdx = span.IndexOfAny(_separators);

if (sepIdx < 0) res.Add(s);

while (sepIdx > -1)
{
var str = new string(span[..(sepIdx + 1)]);

if (res.Any())
{
str = string.Concat(res[^1], str);
}

res.Add(str);

span = span[(sepIdx + 1)..];
sepIdx = span.IndexOfAny(_separators);
}
}
else
{
var sepIdx = span.LastIndexOfAny(_separators);

if (sepIdx < 0) res.Add(s);

while (sepIdx > -1)
{
var str = new string(span[(sepIdx + 1)..]) + span[sepIdx];

if (res.Any())
{
str = string.Concat(res[^1], str);
}

res.Add(str);

span = span[..sepIdx];
sepIdx = span.LastIndexOfAny(_separators);
}
}

for (int i = 0; i < res.Count; i++)
{
res[i] = res[i].TrimEnd(_separators);
}

return res.Distinct().Take(_sepCount).ToList();
}

private void CountLargestEntries(Record record, int num)
{
record.Key = CommonHelper.GetShortKey(record.Key);
Expand Down

0 comments on commit c3b911d

Please sign in to comment.