-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
require name for attachment, add recursive summarizer
- Loading branch information
1 parent
101d1cb
commit 9d00a9b
Showing
10 changed files
with
89 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,8 @@ | ||
// See https://aka.ms/new-console-template for more information | ||
Console.WriteLine("Hello, World!"); | ||
| ||
using ContextFlow.Infrastructure.Providers.OpenAI; | ||
using Demo; | ||
|
||
Console.WriteLine(WriteAnArticle.Write("the history of India")); | ||
//Console.WriteLine(await WriteAnArticleAsync.Write("the history of India")); | ||
//Console.WriteLine(RecursiveSummarizer.Summarize("<book or long article>", new OpenAITokenizer("gpt-3.5-turbo"), 1024)); | ||
//Console.WriteLine(CompleteAssignment.CompleteAssignmentFromSource("<source>")); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,68 @@ | ||
using ContextFlow.Application.Templates; | ||
using ContextFlow.Application.TextUtil; | ||
using ContextFlow.Domain; | ||
using ContextFlow.Infrastructure.Providers; | ||
using ContextFlow.Infrastructure.Providers.OpenAI; | ||
using OpenAI_API.Moderation; | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq; | ||
using System.Text; | ||
using System.Threading.Tasks; | ||
|
||
namespace Demo; | ||
|
||
public static class RecursiveSummarizer | ||
{ | ||
public static (string Summary, int Depth) Summarize(string input, LLMTokenizer tokenizer, int summaryTokenLen, int nSubSummariesPerNextSummary) | ||
{ | ||
// split the inputted text | ||
var inputsSplit = new HierarchichalTextSplitter(tokenizer, summaryTokenLen, HierarchichalTextSplitter.MarkdownBasedHierarchy, HierarchichalTextSplitter.MarkdownBasedAddToBeginnings) | ||
.Split(input); | ||
|
||
var summaries = inputsSplit; | ||
int depth = 0; | ||
|
||
// while there is more than one summary or a summary is longer than the maximum number of tokens | ||
while (summaries.Count > 1 || tokenizer.CountTokens(summaries[0]) > summaryTokenLen) | ||
{ | ||
depth++; | ||
// group by nSubSummariesPerNextSummary and merge into one string per group | ||
var summaryBlocks = GroupByCount(summaries, nSubSummariesPerNextSummary).Select(ls => String.Join("\n", ls)); | ||
|
||
// summarize strings of grouped summaries into the next summary | ||
summaries = SummarizeInner(summaryBlocks, summaryTokenLen); | ||
} | ||
|
||
return (String.Join("\n\n", summaries), depth); | ||
} | ||
|
||
private static List<List<string>> GroupByCount(List<string> inputs, int groupsize) | ||
{ | ||
var result = new List<List<string>>(); | ||
for (int i = 0; i < inputs.Count; i += groupsize) | ||
{ | ||
result.Add(inputs.Skip(i).Take(groupsize).ToList()); | ||
} | ||
return result; | ||
} | ||
|
||
private static List<string> SummarizeInner(IEnumerable<string> inputs, int summaryTokenLen) | ||
{ | ||
var con = new OpenAIChatConnection(); | ||
var result = new List<string>(); | ||
|
||
foreach (var input in inputs) | ||
{ | ||
int availableTokenSpace = summaryTokenLen; | ||
double tokenToWordRatio = 3.5; | ||
double marginOfSafetyMul = 0.8; | ||
int availableWords = (int)Math.Floor(availableTokenSpace / tokenToWordRatio * marginOfSafetyMul); | ||
|
||
var targetLength = $"Below {availableWords} words"; | ||
result.Add(new SummarizeTemplate(input, targetLength).GetLLMRequest(con, "gpt-3.5-turbo").Complete().RawOutput); | ||
} | ||
return result; | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.