(GFM Style)",type:"boolean"},requireSpaceBeforeHeadingText:{defaultValue:!1,describe:"Makes adding a space between `#` and the header text mandatory (GFM Style)",type:"boolean"},ghMentions:{defaultValue:!1,describe:"Enables github @mentions",type:"boolean"},ghMentionsLink:{defaultValue:"https://github.com/{u}",describe:"Changes the link generated by @mentions. Only applies if ghMentions option is enabled.",type:"string"},encodeEmails:{defaultValue:!0,describe:"Encode e-mail addresses through the use of Character Entities, transforming ASCII e-mail addresses into its equivalent decimal entities",type:"boolean"},openLinksInNewWindow:{defaultValue:!1,describe:"Open all links in new windows",type:"boolean"},backslashEscapesHTMLTags:{defaultValue:!1,describe:"Support for HTML Tag escaping. ex:
foo
",type:"boolean"},emoji:{defaultValue:!1,describe:"Enable emoji support. Ex: `this is a :smile: emoji`",type:"boolean"},underline:{defaultValue:!1,describe:"Enable support for underline. Syntax is double or triple underscores: `__underline word__`. With this option enabled, underscores no longer parses into `` and ``",type:"boolean"},ellipsis:{defaultValue:!0,describe:"Replaces three dots with the ellipsis unicode character",type:"boolean"},completeHTMLDocument:{defaultValue:!1,describe:"Outputs a complete html document, including ``, `` and `` tags",type:"boolean"},metadata:{defaultValue:!1,describe:"Enable support for document metadata (defined at the top of the document between `ยซยซยซ` and `ยปยปยป` or between `---` and `---`).",type:"boolean"},splitAdjacentBlockquotes:{defaultValue:!1,describe:"Split adjacent blockquote blocks",type:"boolean"}};if(!1===e)return JSON.parse(JSON.stringify(r));var t,a={};for(t in r)r.hasOwnProperty(t)&&(a[t]=r[t].defaultValue);return a}var x={},t={},d={},p=a(!0),h="vanilla",_={github:{omitExtraWLInCodeBlocks:!0,simplifiedAutoLink:!0,excludeTrailingPunctuationFromURLs:!0,literalMidWordUnderscores:!0,strikethrough:!0,tables:!0,tablesHeaderId:!0,ghCodeBlocks:!0,tasklists:!0,disableForced4SpacesIndentedSublists:!0,simpleLineBreaks:!0,requireSpaceBeforeHeadingText:!0,ghCompatibleHeaderId:!0,ghMentions:!0,backslashEscapesHTMLTags:!0,emoji:!0,splitAdjacentBlockquotes:!0},original:{noHeaderId:!0,ghCodeBlocks:!1},ghost:{omitExtraWLInCodeBlocks:!0,parseImgDimensions:!0,simplifiedAutoLink:!0,excludeTrailingPunctuationFromURLs:!0,literalMidWordUnderscores:!0,strikethrough:!0,tables:!0,tablesHeaderId:!0,ghCodeBlocks:!0,tasklists:!0,smoothLivePreview:!0,simpleLineBreaks:!0,requireSpaceBeforeHeadingText:!0,ghMentions:!1,encodeEmails:!0},vanilla:a(!0),allOn:function(){"use strict";var e,r=a(!0),t={};for(e in r)r.hasOwnProperty(e)&&(t[e]=!0);return t}()};function g(e,r){"use strict";var t=r?"Error in "+r+" extension->":"Error in unnamed extension",a={valid:!0,error:""};x.helper.isArray(e)||(e=[e]);for(var n=0;n
',showdown:"S"},x.Converter=function(e){"use strict";var r,t,n={},i=[],l=[],o={},a=h,s={parsed:{},raw:"",format:""};for(r in e=e||{},p)p.hasOwnProperty(r)&&(n[r]=p[r]);if("object"!=typeof e)throw Error("Converter expects the passed parameter to be an object, but "+typeof e+" was passed instead.");for(t in e)e.hasOwnProperty(t)&&(n[t]=e[t]);function c(e,r){if(r=r||null,x.helper.isString(e)){if(r=e=x.helper.stdExtName(e),x.extensions[e]){console.warn("DEPRECATION WARNING: "+e+" is an old extension that uses a deprecated loading method.Please inform the developer that the extension should be updated!");var t=x.extensions[e],a=e;if("function"==typeof t&&(t=t(new x.Converter)),x.helper.isArray(t)||(t=[t]),!(a=g(t,a)).valid)throw Error(a.error);for(var n=0;n[^\r]+?<\/pre>)/gm,function(e,r){return r.replace(/^ /gm,"ยจ0").replace(/ยจ0/g,"")}),x.subParser("hashBlock")("\n"+e+"\n
",r,t)}),e=t.converter._dispatch("blockQuotes.after",e,r,t)}),x.subParser("codeBlocks",function(e,n,s){"use strict";e=s.converter._dispatch("codeBlocks.before",e,n,s);return e=(e=(e+="ยจ0").replace(/(?:\n\n|^)((?:(?:[ ]{4}|\t).*\n+)+)(\n*[ ]{0,3}[^ \t\n]|(?=ยจ0))/g,function(e,r,t){var a="\n",r=x.subParser("outdent")(r,n,s);return r=x.subParser("encodeCode")(r,n,s),r=""+(r=(r=(r=x.subParser("detab")(r,n,s)).replace(/^\n+/g,"")).replace(/\n+$/g,""))+(a=n.omitExtraWLInCodeBlocks?"":a)+"
",x.subParser("hashBlock")(r,n,s)+t})).replace(/ยจ0/,""),e=s.converter._dispatch("codeBlocks.after",e,n,s)}),x.subParser("codeSpans",function(e,n,s){"use strict";return e=(e=void 0===(e=s.converter._dispatch("codeSpans.before",e,n,s))?"":e).replace(/(^|[^\\])(`+)([^\r]*?[^`])\2(?!`)/gm,function(e,r,t,a){return a=(a=a.replace(/^([ \t]*)/g,"")).replace(/[ \t]*$/g,""),a=r+""+(a=x.subParser("encodeCode")(a,n,s))+"",a=x.subParser("hashHTMLSpans")(a,n,s)}),e=s.converter._dispatch("codeSpans.after",e,n,s)}),x.subParser("completeHTMLDocument",function(e,r,t){"use strict";if(!r.completeHTMLDocument)return e;e=t.converter._dispatch("completeHTMLDocument.before",e,r,t);var a,n="html",s="\n",o="",i='\n',l="",c="";for(a in void 0!==t.metadata.parsed.doctype&&(s="\n","html"!==(n=t.metadata.parsed.doctype.toString().toLowerCase())&&"html5"!==n||(i='')),t.metadata.parsed)if(t.metadata.parsed.hasOwnProperty(a))switch(a.toLowerCase()){case"doctype":break;case"title":o=""+t.metadata.parsed.title+" \n";break;case"charset":i="html"===n||"html5"===n?'\n':'\n';break;case"language":case"lang":l=' lang="'+t.metadata.parsed[a]+'"',c+='\n';break;default:c+='\n'}return e=s+"\n\n"+o+i+c+"\n\n"+e.trim()+"\n\n",e=t.converter._dispatch("completeHTMLDocument.after",e,r,t)}),x.subParser("detab",function(e,r,t){"use strict";return e=(e=(e=(e=(e=(e=t.converter._dispatch("detab.before",e,r,t)).replace(/\t(?=\t)/g," ")).replace(/\t/g,"ยจAยจB")).replace(/ยจB(.+?)ยจA/g,function(e,r){for(var t=r,a=4-t.length%4,n=0;n/g,">"),e=t.converter._dispatch("encodeAmpsAndAngles.after",e,r,t)}),x.subParser("encodeBackslashEscapes",function(e,r,t){"use strict";return e=(e=(e=t.converter._dispatch("encodeBackslashEscapes.before",e,r,t)).replace(/\\(\\)/g,x.helper.escapeCharactersCallback)).replace(/\\([`*_{}\[\]()>#+.!~=|:-])/g,x.helper.escapeCharactersCallback),e=t.converter._dispatch("encodeBackslashEscapes.after",e,r,t)}),x.subParser("encodeCode",function(e,r,t){"use strict";return e=(e=t.converter._dispatch("encodeCode.before",e,r,t)).replace(/&/g,"&").replace(//g,">").replace(/([*_{}\[\]\\=~-])/g,x.helper.escapeCharactersCallback),e=t.converter._dispatch("encodeCode.after",e,r,t)}),x.subParser("escapeSpecialCharsWithinTagAttributes",function(e,r,t){"use strict";return e=(e=(e=t.converter._dispatch("escapeSpecialCharsWithinTagAttributes.before",e,r,t)).replace(/<\/?[a-z\d_:-]+(?:[\s]+[\s\S]+?)?>/gi,function(e){return e.replace(/(.)<\/?code>(?=.)/g,"$1`").replace(/([\\`*_~=|])/g,x.helper.escapeCharactersCallback)})).replace(/-]|-[^>])(?:[^-]|-[^-])*)--)>/gi,function(e){return e.replace(/([\\`*_~=|])/g,x.helper.escapeCharactersCallback)}),e=t.converter._dispatch("escapeSpecialCharsWithinTagAttributes.after",e,r,t)}),x.subParser("githubCodeBlocks",function(e,s,o){"use strict";return s.ghCodeBlocks?(e=o.converter._dispatch("githubCodeBlocks.before",e,s,o),e=(e=(e+="ยจ0").replace(/(?:^|\n)(?: {0,3})(```+|~~~+)(?: *)([^\s`~]*)\n([\s\S]*?)\n(?: {0,3})\1/g,function(e,r,t,a){var n=s.omitExtraWLInCodeBlocks?"":"\n";return a=x.subParser("encodeCode")(a,s,o),a=""+(a=(a=(a=x.subParser("detab")(a,s,o)).replace(/^\n+/g,"")).replace(/\n+$/g,""))+n+"
",a=x.subParser("hashBlock")(a,s,o),"\n\nยจG"+(o.ghCodeBlocks.push({text:e,codeblock:a})-1)+"G\n\n"})).replace(/ยจ0/,""),o.converter._dispatch("githubCodeBlocks.after",e,s,o)):e}),x.subParser("hashBlock",function(e,r,t){"use strict";return e=(e=t.converter._dispatch("hashBlock.before",e,r,t)).replace(/(^\n+|\n+$)/g,""),e="\n\nยจK"+(t.gHtmlBlocks.push(e)-1)+"K\n\n",e=t.converter._dispatch("hashBlock.after",e,r,t)}),x.subParser("hashCodeTags",function(e,n,s){"use strict";e=s.converter._dispatch("hashCodeTags.before",e,n,s);return e=x.helper.replaceRecursiveRegExp(e,function(e,r,t,a){t=t+x.subParser("encodeCode")(r,n,s)+a;return"ยจC"+(s.gHtmlSpans.push(t)-1)+"C"},"]*>","","gim"),e=s.converter._dispatch("hashCodeTags.after",e,n,s)}),x.subParser("hashElement",function(e,r,t){"use strict";return function(e,r){return r=(r=(r=r.replace(/\n\n/g,"\n")).replace(/^\n/,"")).replace(/\n+$/g,""),r="\n\nยจK"+(t.gHtmlBlocks.push(r)-1)+"K\n\n"}}),x.subParser("hashHTMLBlocks",function(e,r,n){"use strict";e=n.converter._dispatch("hashHTMLBlocks.before",e,r,n);function t(e,r,t,a){return-1!==t.search(/\bmarkdown\b/)&&(e=t+n.converter.makeHtml(r)+a),"\n\nยจK"+(n.gHtmlBlocks.push(e)-1)+"K\n\n"}var a=["pre","div","h1","h2","h3","h4","h5","h6","blockquote","table","dl","ol","ul","script","noscript","form","fieldset","iframe","math","style","section","header","footer","nav","article","aside","address","audio","canvas","figure","hgroup","output","video","p"];r.backslashEscapesHTMLTags&&(e=e.replace(/\\<(\/?[^>]+?)>/g,function(e,r){return"<"+r+">"}));for(var s=0;s]*>)","im"),i="<"+a[s]+"\\b[^>]*>",l="";-1!==(c=x.helper.regexIndexOf(e,o));){var c=x.helper.splitAtIndex(e,c),u=x.helper.replaceRecursiveRegExp(c[1],t,i,l,"im");if(u===c[1])break;e=c[0].concat(u)}return e=e.replace(/(\n {0,3}(<(hr)\b([^<>])*?\/?>)[ \t]*(?=\n{2,}))/g,x.subParser("hashElement")(e,r,n)),e=(e=x.helper.replaceRecursiveRegExp(e,function(e){return"\n\nยจK"+(n.gHtmlBlocks.push(e)-1)+"K\n\n"},"^ {0,3}\x3c!--","--\x3e","gm")).replace(/(?:\n\n)( {0,3}(?:<([?%])[^\r]*?\2>)[ \t]*(?=\n{2,}))/g,x.subParser("hashElement")(e,r,n)),e=n.converter._dispatch("hashHTMLBlocks.after",e,r,n)}),x.subParser("hashHTMLSpans",function(e,r,t){"use strict";function a(e){return"ยจC"+(t.gHtmlSpans.push(e)-1)+"C"}return e=(e=(e=(e=(e=t.converter._dispatch("hashHTMLSpans.before",e,r,t)).replace(/<[^>]+?\/>/gi,a)).replace(/<([^>]+?)>[\s\S]*?<\/\1>/g,a)).replace(/<([^>]+?)\s[^>]+?>[\s\S]*?<\/\1>/g,a)).replace(/<[^>]+?>/gi,a),e=t.converter._dispatch("hashHTMLSpans.after",e,r,t)}),x.subParser("unhashHTMLSpans",function(e,r,t){"use strict";e=t.converter._dispatch("unhashHTMLSpans.before",e,r,t);for(var a=0;a]*>\\s*]*>","^ {0,3}\\s* ","gim"),e=s.converter._dispatch("hashPreCodeTags.after",e,n,s)}),x.subParser("headers",function(e,n,s){"use strict";e=s.converter._dispatch("headers.before",e,n,s);var o=isNaN(parseInt(n.headerLevelStart))?1:parseInt(n.headerLevelStart),r=n.smoothLivePreview?/^(.+)[ \t]*\n={2,}[ \t]*\n+/gm:/^(.+)[ \t]*\n=+[ \t]*\n+/gm,t=n.smoothLivePreview?/^(.+)[ \t]*\n-{2,}[ \t]*\n+/gm:/^(.+)[ \t]*\n-+[ \t]*\n+/gm,r=(e=(e=e.replace(r,function(e,r){var t=x.subParser("spanGamut")(r,n,s),r=n.noHeaderId?"":' id="'+i(r)+'"',r="",r,t);return e=(e=(e=e.replace(/^ {0,2}( ?-){3,}[ \t]*$/gm,a)).replace(/^ {0,2}( ?\*){3,}[ \t]*$/gm,a)).replace(/^ {0,2}( ?_){3,}[ \t]*$/gm,a),e=t.converter._dispatch("horizontalRule.after",e,r,t)}),x.subParser("images",function(e,r,d){"use strict";function l(e,r,t,a,n,s,o,i){var l=d.gUrls,c=d.gTitles,u=d.gDimensions;if(t=t.toLowerCase(),i=i||"",-1
\n")):e=e.replace(/ +\n/g,"
\n"),e=t.converter._dispatch("spanGamut.after",e,r,t)}),x.subParser("strikethrough",function(e,t,a){"use strict";return t.strikethrough&&(e=(e=a.converter._dispatch("strikethrough.before",e,t,a)).replace(/(?:~){2}([\s\S]+?)(?:~){2}/g,function(e,r){return r=r,"
+
+
+
+
diff --git a/app/templates/inference.js b/app/templates/inference.js
new file mode 100644
index 0000000..262f65c
--- /dev/null
+++ b/app/templates/inference.js
@@ -0,0 +1,268 @@
+const converter = new showdown.Converter();
+
+let ws;
+let response = "";
+let conversationStarted = false;
+let dontScroll = false;
+
+function setButton(value, disabled) {
+ var button = document.getElementById('send');
+ button.innerHTML = value;
+ button.disabled = disabled
+ if(disabled) {
+ button.classList.add("blink_me");
+ } else {
+ button.classList.remove("blink_me");
+ }
+}
+
+function pickModel(model){
+ messageText.value = "!model " + model;
+ messageText.focus();
+ sendQuery();
+ messageText.value = "";
+ messageText.focus();
+ return false;
+}
+
+function setPromptTemplate(template, cursorLocation){
+ messageText.value = template;
+ messageText.focus();
+ messageText.selectionEnd = cursorLocation;
+}
+
+function localCommandExecuted(e){
+ const messageText = document.getElementById("messageText");
+ switch(messageText.value.toLowerCase()){
+ {% for template in conf.PROMPT_TEMPLATES %}
+ case '#{{ template[0] }}':
+ setPromptTemplate("{{ template[1]|safe }}", {{ template[2] }});
+ if (e) e.preventDefault();
+ return true;
+ {% endfor %}
+ default:
+ return false;
+ }
+}
+
+function sendQuery() {
+ if (localCommandExecuted()) return;
+ if (!ws) return;
+ conversationStarted = true;
+ var message = document.getElementById('messageText').value;
+ var temp = document.getElementById('tempValue').innerHTML;
+ if (message === "") {
+ return;
+ }
+ payload = {
+ query: message,
+ temperature: parseFloat(temp)
+ }
+ ws.send(JSON.stringify(payload));
+ setButton("{{ res.BUTTON_PROCESSING }}", true)
+}
+
+function submitForm(event) {
+ event.preventDefault();
+ sendQuery();
+}
+
+function appendButtons() {
+ var messages = document.getElementById('messages');
+ var div = messages;
+
+ var span = document.createElement('span');
+ span.innerHTML="
+
+
+
+
+
+
+
+
+ Tokens: 0
+ / {{conf.MAX_RESPONSE_TOKENS }}
+ 
"
+ div.appendChild(span);
+}
+
+function copyClipboard(e) {
+ var messages = document.getElementById('messages');
+ var div = messages.firstChild.firstChild;
+ content = div.innerText;
+ navigator.clipboard.writeText(content);
+ label = document.getElementById("clip-button-label")
+ label.innerHTML = "Copied!"
+ setTimeout(()=>{
+ label.innerText = "Copy to clipboard";
+ },
+ 2000)
+}
+
+function updateResponseTokens() {
+ responseTokenCountValue = document.getElementById("responseTokenCountValue");
+ encoded = llamaTokenizer.encode(response)
+ responseTokenCountValue.innerText=encoded.length;
+ responseTokenCount.classList.remove("d-none");
+}
+
+function hideResponseTokens() {
+ responseTokenCount = document.getElementById("responseTokenCount");
+ responseTokenCount.classList.add("d-none");
+}
+
+function connect() {
+ let wsBaseUrl = "{{ wsurl }}";
+ if (wsBaseUrl === "") {
+ let wsProtocol = "https:" === document.location.protocol ? 'wss://' : 'ws://'
+ wsBaseUrl = wsProtocol + window.location.host;
+ }
+ ws = new WebSocket(wsBaseUrl + "/inference");
+ ws.onmessage = function (event) {
+ var messages = document.getElementById('messages');
+ var data = JSON.parse(event.data);
+ handleResBotResponse(data ,messages)
+
+ // Scroll to the bottom of the chat (don't auto scroll if user has scrolled manually)
+ if (!dontScroll){
+ messages.scrollTop = Math.floor(messages.scrollHeight - messages.offsetHeight)
+ }
+ };
+ ws.onopen = function() {
+ setButton("{{ res.BUTTON_SEND }}", false);
+ };
+
+ ws.onclose = function(e) {
+ setButton("{{ res.BUTTON_WAIT }}", true);
+ console.log('Socket is closed. Reconnect will be attempted in 1 second.', e.reason);
+ setTimeout(function() {
+ connect();
+ }, 1000);
+ };
+}
+
+function handleResBotResponse(data, messages) {
+ switch(data.type) {
+ case "start":
+ dontScroll = false;
+ messages.innerHTML = '';
+ response = "";
+ var div = document.createElement('div');
+ div.className = 'server-message';
+ var p = document.createElement('p');
+ response += data.message;
+ p.innerHTML = converter.makeHtml(response);
+ div.appendChild(p);
+ messages.appendChild(div);
+ updateResponseTokens();
+ break;
+
+ case "stream":
+ setButton("{{ res.BUTTON_TYPING }}", true);
+ var p = messages.lastChild.lastChild;
+ response += data.message;
+ p.innerHTML = converter.makeHtml(response);
+ updateResponseTokens();
+ break;
+
+ case "end":
+ var p = messages.lastChild.lastChild;
+ p.innerHTML = converter.makeHtml(response);
+ setButton("{{ res.BUTTON_SEND }}", false);
+ updateResponseTokens();
+ appendButtons();
+ break;
+
+ case "done":
+ hideResponseTokens();
+ setButton("{{ res.BUTTON_SEND }}", false);
+ break;
+
+ case "info":
+ messages.innerHTML = '';
+ var div = document.createElement('div');
+ div.className = 'server-message';
+ var p = document.createElement('p');
+ // p.innerHTML = converter.makeHtml(data.message);
+ p.innerHTML = data.message;
+ div.appendChild(p);
+ messages.appendChild(div);
+ hideResponseTokens();
+ break;
+
+ case "system":
+ messages.innerHTML = '';
+ var div = document.createElement('div');
+ div.className = 'server-message';
+ var p = document.createElement('p');
+ p.innerHTML = data.message;
+ div.appendChild(p);
+ messages.appendChild(div);
+ hideResponseTokens();
+ setButton("{{ res.BUTTON_SEND }}", false);
+ break;
+
+ case "error":
+ messages.innerHTML = '';
+ var div = document.createElement('div');
+ div.className = 'server-message';
+ var p = document.createElement('p');
+ // p.innerHTML = converter.makeHtml(data.message);
+ p.innerHTML = data.message;
+ div.appendChild(p);
+ messages.appendChild(div);
+ hideResponseTokens();
+ setButton("{{ res.BUTTON_SEND }}", false);
+ break;
+ }
+}
+
+document.addEventListener("DOMContentLoaded", function(event) {
+ let messages=document.getElementById("messages");
+ let topbox=document.getElementById("top-box");
+
+ ['touchmove','mousedown','select','wheel'].forEach((evt) => {
+ messages.addEventListener(evt, (e) => {
+ if(Math.floor(messages.scrollTop) === Math.floor(messages.scrollHeight - messages.offsetHeight)) {
+ dontScroll = false;
+ }
+ else{
+ dontScroll = true;
+ }
+ });
+ });
+
+
+ const slider = document.getElementById("tempSlider");
+ const label = document.getElementById("tempValue");
+ slider.addEventListener("input", () => {
+ label.innerText=slider.value;
+ });
+
+ const button = document.getElementById('send');
+ const tokenCount = document.getElementById("tokenCount");
+ const tokenCountValue = document.getElementById("tokenCountValue");
+ const messageText = document.getElementById("messageText");
+ messageText.addEventListener("keydown", (e) => {
+ if (e.ctrlKey && e.key === 'Enter') {
+ if (!button.disabled) sendQuery();
+ e.preventDefault();
+ }
+ if (e.key === "Tab") {
+ localCommandExecuted(e);
+ e.preventDefault();
+ }
+ })
+ messageText.focus();
+
+ messageText.addEventListener("input", () => {
+ encoded = llamaTokenizer.encode(messageText.value)
+ if (encoded.length > 0) {
+ tokenCount.classList.remove("d-none");
+ tokenCountValue.innerText=encoded.length
+ } else {
+ tokenCount.classList.add("d-none");
+ }
+ if (!button.disabled) {
+ if (encoded.length > {{ conf.CONTEXT_TOKENS }}) {
+ button.disabled = true;
+ tokenCount.classList.add("tokenLimitExceeded");
+ } else {
+ button.disabled = false;
+ tokenCount.classList.remove("tokenLimitExceeded");
+ }
+ }
+ });
+
+ connect();
+});
diff --git a/environment.yml b/environment.yml
index 3e5f658..ed530d1 100644
--- a/environment.yml
+++ b/environment.yml
@@ -7,8 +7,11 @@ dependencies:
- python
- beautifulsoup4
- faiss-cpu
+ - fastapi
+ - jinja2
- langchain
- numpydoc
+ - pydantic
- pydata-sphinx-theme
- pytest
- pytest-cov
@@ -16,8 +19,9 @@ dependencies:
- scikit-learn
- sentence-transformers
- sphinx
- - tiktoken
- transformers
+ - uvicorn
+ - websockets
- pip:
# We use return_generator option that will be shipped in 1.4
- git+https://github.com/joblib/joblib.git@master
diff --git a/pyproject.toml b/pyproject.toml
index 23bc517..655d308 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -29,3 +29,4 @@ exclude=[
# folder.
"scripts/*"=["E402"]
"doc/conf.py"=["E402"]
+"app/main.py"=["E402"]
diff --git a/rag_based_llm/prompt/_agent.py b/rag_based_llm/prompt/_agent.py
index afb2048..b825a61 100644
--- a/rag_based_llm/prompt/_agent.py
+++ b/rag_based_llm/prompt/_agent.py
@@ -1,11 +1,3 @@
-import tiktoken
-
-
-def trim(text, max_tokens):
- enc = tiktoken.get_encoding("cl100k_base")
- return enc.decode(enc.encode(text)[:max_tokens])
-
-
class QueryAgent:
def __init__(
self,
@@ -19,13 +11,17 @@ def __init__(
def __call__(self, query, **prompt_kwargs):
# FIXME: We need to come with a design such that it is generic enough to accept
# different prompt strategies.
- max_tokens = prompt_kwargs.get("max_tokens", 1024)
+ prompt_kwargs.get("max_tokens", 1024)
prompt = (
"[INST] Summarize the query provided by extracting keywords from it. "
"Only list the keywords only separated by a comma. \n"
f"query: {query}[/INST]"
)
- response = self.llm(trim(prompt, max_tokens=max_tokens), **prompt_kwargs)
+
+ # do not create a stream generator
+ local_prompt_kwargs = prompt_kwargs.copy()
+ local_prompt_kwargs["stream"] = False
+ response = self.llm(prompt, **local_prompt_kwargs)
keywords = response["choices"][0]["text"].strip()
context = self.retriever.query(
@@ -39,13 +35,9 @@ def __call__(self, query, **prompt_kwargs):
"[INST] Answer to the query related to scikit-learn using the following "
"pair of content and source. The context is provided from the most "
"relevant to the least relevant. Use this priority to answer to the query. "
- "Be succinct. \n"
+ "You have to finish your answer with the source that is an https link of "
+ "the content that you used to answer the query. Be succinct. \n"
f"query: {query}\n"
f"context: {context_query} [/INST]."
)
- response = self.llm(trim(prompt, max_tokens=max_tokens), **prompt_kwargs)
- return (
- response["choices"][0]["text"].strip()
- + "\n\nSource(s):\n"
- + "\n".join(sources)
- )
+ return self.llm(prompt, **prompt_kwargs), sources
diff --git a/requirements.txt b/requirements.txt
index 6229b0a..b29236d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,10 @@
beautifulsoup4
faiss-cpu # this is an unofficial package
+fastapi
+jinja2
langchain
numpydoc
+pydantic
pydata-sphinx-theme
pytest
pytest-cov
@@ -9,8 +12,9 @@ scikit-learn
sentence-transformers
setuptools
sphinx
-tiktoken
torch --index-url https://download.pytorch.org/whl/cpu/
transformers
+uvicorn
+websockets
# We use return_generator option that will be shipped in 1.4
-joblib @ git+https://github.com/joblib/joblib.git@master
\ No newline at end of file
+joblib @ git+https://github.com/joblib/joblib.git@master