Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support idle and freq anslysis #54

Merged
merged 4 commits into from
Apr 7, 2024
Merged

Support idle and freq anslysis #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
53d8784
Support idle and freq anslysis (#52)
catcherwong Apr 6, 2024
5e88406
clean and add ut
catcherwong Apr 6, 2024
4102d0d
clean and jsdelivr => unpkg
catcherwong Apr 6, 2024
98166e3
filter support min idle and freq
catcherwong Apr 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions build/version.props
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
<Project>
<PropertyGroup>

<RDBParserVersion>0.6.0</RDBParserVersion>
<RDBCliVersion>0.8.1</RDBCliVersion>
<RDBParserVersion>0.7.0</RDBParserVersion>
<RDBCliVersion>0.8.2</RDBCliVersion>

</PropertyGroup>
</Project>
13 changes: 9 additions & 4 deletions src/RDBCli/Callbacks/KeysOnlyCallback.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using RDBParser;
using System;
using System.Collections.Generic;
using System.CommandLine;
using System.Linq;
Expand All @@ -22,15 +23,15 @@ private void OutputInfo(byte[] key, long expiry)
{
var keyStr = System.Text.Encoding.UTF8.GetString(key);

if(CheckPreifx(keyStr))
if (CheckPreifx(keyStr))
{
if(_isPermanent.HasValue)
if (_isPermanent.HasValue)
{
if(_isPermanent.Value && expiry == 0)
if (_isPermanent.Value && expiry == 0)
{
_console.WriteLine(keyStr);
}
else if(!_isPermanent.Value && expiry != 0)
else if (!_isPermanent.Value && expiry != 0)
{
_console.WriteLine(keyStr);
}
Expand Down Expand Up @@ -178,5 +179,9 @@ public void StreamListPack(byte[] key, byte[] entry_id, byte[] data)
public void ZAdd(byte[] key, double score, byte[] member)
{
}

public void SetIdleOrFreq(int val)
{
}
}
}
21 changes: 21 additions & 0 deletions src/RDBCli/Callbacks/MemoryCallback.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,12 @@ internal partial class MemoryCallback : IReaderCallback

private Record _currentRecord = new Record();

private int _idleOrFreq = -1;

public RdbDataInfo GetRdbDataInfo() => _rdbDataInfo;

public int GetIdleOrFreq() => _idleOrFreq;

public MemoryCallback(bool isIgnorefole = false)
{
_isIgnorefole = isIgnorefole;
Expand Down Expand Up @@ -266,6 +270,8 @@ public void Set(byte[] key, byte[] value, long expiry, Info info)
Expiry = expiry,
Database = _dbNum,
LenOfLargestElem = length,
Freq = info.Freq,
Idle = info.Idle,
};

_rdbDataInfo.TotalMem += record.Bytes;
Expand Down Expand Up @@ -305,6 +311,8 @@ public void StartHash(byte[] key, long length, long expiry, Info info)
Encoding = info.Encoding,
Expiry = expiry,
Database = _dbNum,
Freq = info.Freq,
Idle = info.Idle,
};
}

Expand All @@ -324,6 +332,8 @@ public void StartList(byte[] key, long expiry, Info info)
Encoding = encoding,
Expiry = expiry,
Database = _dbNum,
Freq = info.Freq,
Idle = info.Idle,
};
}

Expand All @@ -343,6 +353,8 @@ public bool StartModule(byte[] key, string module_name, long expiry, Info info)
Expiry = expiry,
NumOfElem = 1,
Database = _dbNum,
Freq = info.Freq,
Idle = info.Idle,
};

return false;
Expand Down Expand Up @@ -386,6 +398,8 @@ public void StartSortedSet(byte[] key, long length, long expiry, Info info)
Encoding = info.Encoding,
Expiry = expiry,
Database = _dbNum,
Freq = info.Freq,
Idle = info.Idle,
};
}

Expand All @@ -406,6 +420,8 @@ public void StartStream(byte[] key, long listpacks_count, long expiry, Info info
Encoding = info.Encoding,
Expiry = expiry,
Database = _dbNum,
Freq = info.Freq,
Idle = info.Idle,
};
}

Expand Down Expand Up @@ -456,5 +472,10 @@ public void FuntionLoad(byte[] engine, byte[] libName, byte[] code)

_rdbDataInfo.TotalMem += FunctionOverhead(engine, libName, code);
}

public void SetIdleOrFreq(int val)
{
_idleOrFreq = val;
}
}
}
12 changes: 10 additions & 2 deletions src/RDBCli/Commands/CsvCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ internal class CsvCommand : Command
private static Option<List<int>> _databasesOption = CommonCLIOptions.DBsOption();
private static Option<List<string>> _typesOption = CommonCLIOptions.TypesOption();
private static Option<List<string>> _keyPrefixesOption = CommonCLIOptions.KeyPrefixesOption();
private static Option<ulong> _minIdleOption = CommonCLIOptions.MinIdleOption();
private static Option<int> _minFreqOption = CommonCLIOptions.MinFreqOption();
private static Argument<string> _fileArg = CommonCLIArguments.FileArgument();

public CsvCommand()
Expand All @@ -26,6 +28,8 @@ public CsvCommand()
this.AddOption(_databasesOption);
this.AddOption(_typesOption);
this.AddOption(_keyPrefixesOption);
this.AddOption(_minIdleOption);
this.AddOption(_minFreqOption);
this.AddArgument(_fileArg);

this.SetHandler((InvocationContext context) =>
Expand Down Expand Up @@ -76,12 +80,16 @@ public static CommandOptions FromContext(InvocationContext context)
var databases = context.ParseResult.GetValueForOption<List<int>>(_databasesOption);
var types = context.ParseResult.GetValueForOption<List<string>>(_typesOption);
var keyPrefixes = context.ParseResult.GetValueForOption<List<string>>(_keyPrefixesOption);
var minIdle = context.ParseResult.GetValueForOption<ulong>(_minIdleOption);
var minFreq = context.ParseResult.GetValueForOption<int>(_minFreqOption);

var parseFilter = new RDBParser.ParserFilter()
{
Databases = databases,
Types = types,
KeyPrefixes = keyPrefixes,
MinFreq = minFreq,
MinIdle = minIdle
};

return new CommandOptions
Expand Down Expand Up @@ -128,7 +136,7 @@ public Task<string> Output(string output)
{
// overwrite
fs.SetLength(0);
var header = Encoding.UTF8.GetBytes("database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry\n");
var header = Encoding.UTF8.GetBytes("database,type,key,size_in_bytes,encoding,num_elements,len_largest_element,expiry,idle,freq\n");
fs.Write(header);

while (!_records.IsCompleted)
Expand All @@ -137,7 +145,7 @@ public Task<string> Output(string output)
{
if (_records.TryTake(out var item))
{
var line = Encoding.UTF8.GetBytes($"{item.Record.Database},{item.Record.Type},{item.Record.Key},{item.Record.Bytes},{item.Record.Encoding},{item.Record.NumOfElem},{item.Record.LenOfLargestElem},{CommonHelper.GetExpireString(item.Record.Expiry)}\n");
var line = Encoding.UTF8.GetBytes($"{item.Record.Database},{item.Record.Type},{item.Record.Key},{item.Record.Bytes},{item.Record.Encoding},{item.Record.NumOfElem},{item.Record.LenOfLargestElem},{CommonHelper.GetExpireString(item.Record.Expiry)},{item.Record.Idle},{item.Record.Freq}\n");
fs.Write(line);
}
else
Expand Down
31 changes: 27 additions & 4 deletions src/RDBCli/Commands/MemoryCommand.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ private void Do(InvocationContext context, CommandOptions options)
{
var console = context.Console;
var cb = new CliCB.MemoryCallback(options.IsIgnoreFole ?? false);
var rdbDataInfo = cb.GetRdbDataInfo();
//var rdbDataInfo = cb.GetRdbDataInfo();

var counter = new RdbDataCounter(rdbDataInfo.Records, options.Separators, options.SepPrefixCount, options.keySuffixEnable ?? false);
var counter = new RdbDataCounter(cb, options.Separators, options.SepPrefixCount, options.keySuffixEnable ?? false);
var task = counter.Count();

console.WriteLine($"");
Expand All @@ -80,14 +80,16 @@ private void Do(InvocationContext context, CommandOptions options)
var largestKeyPrefix = counter.GetLargestKeyPrefixes(options.TopPrefixCount);
var typeRecords = counter.GetTypeRecords();
var expiryInfo = counter.GetExpiryInfo();
var streamRecords = counter.GetStreamRecords();
var streamRecords = counter.GetStreamRecords();
var idleOrFreqInfo = counter.GetIdleOrFreqInfo();

var dict = MemoryAnslysisResult.BuildBasicFromRdbDataInfo(rdbDataInfo);
var dict = MemoryAnslysisResult.BuildBasicFromRdbDataInfo(cb.GetRdbDataInfo());
dict.typeRecords = typeRecords;
dict.largestKeyPrefix = largestKeyPrefix;
dict.largestRecords = largestRecords;
dict.expiryInfo = expiryInfo;
dict.largestStreams = streamRecords;
dict.idleOrFreqInfo = idleOrFreqInfo;

var exp = expiryInfo.FirstOrDefault(x => x.Expiry.Equals(CommonHelper.AlreadyExpired));
var perm = expiryInfo.FirstOrDefault(x => x.Expiry.Equals(CommonHelper.Permanent));
Expand Down Expand Up @@ -229,6 +231,7 @@ public class MemoryAnslysisResult
public List<Record> largestRecords { get; set; }
public List<PrefixRecord> largestKeyPrefix { get; set; }
public List<ExpiryRecord> expiryInfo { get; set; }
public List<IdleOrFreqRecord> idleOrFreqInfo { get; set; }
public List<FunctionsRecord> functions { get; set; }
public List<StreamsRecord> largestStreams { get; set; }

Expand Down Expand Up @@ -404,5 +407,25 @@ public static Option<int> SepPrefixCountOption()

return option;
}

public static Option<ulong> MinIdleOption()
{
Option<ulong> option =
new Option<ulong>(
aliases: new string[] { "--min-idle" },
description: "The minimum idle time of the key(must lru policy)");

return option;
}

public static Option<int> MinFreqOption()
{
Option<int> option =
new Option<int>(
aliases: new string[] { "--min-freq" },
description: "The minimum frequency of the key(must lfu policy)");

return option;
}
}
}
Loading
Loading