+
+

Table of Contents¶

-

Chapter 1: Introduction¶

+

Chapter 1: Introduction¶

  • The Hidden Challenges of LLMs

  • Why This Book Matters

    • @@ -133,7 +135,7 @@

    Chapter 1: Introduction

-

Chapter 2: Non-determinism & Evals¶

+

Chapter 2: Non-determinism & Evals¶

diff --git a/tamingllms/_build/html/objects.inv b/tamingllms/_build/html/objects.inv index d695173e560b1c82310d10b1a66b0616a20d79b1..e98b3dff01df67414820149f06eb26688de04d50 100644 GIT binary patch delta 249 zcmVsac;At@0h!q1Cq%Um&UEOYzK7wW-i_)VJk4X}P1svG@`hN0oc&h%*}dFcnKas| zR}%h3H34g2(lfR*okNfhcobG-jg=R>^5EfAUl$pCNyieIUvgzp>_ql)!;%EW;E5^v z^7dbz9QP_*7h;hplO85*BYk1($dj&z-q>>P+gXMt$?!bOW-cfe`B6h^Z3<}HoUxvN zW6SyP0i!zN0gj|2G(WD(<*F#|pGLPQ9fg<0Ml_)xmtfX1)8zfu3|M^wgV5jXQ=@!h delta 245 zcmVOd^a-4*cdDP$jTP_IVLDb`hqK`eZILAKt#7!S+RK|ela6NW z1etzNNx&MI@{B!MJqQG~+y64KSs8P<*7u8yza+rHJTS4qiEjBEx5Oz(T91U$mlOQe z$+}mjx(Ev+k1~wXM*M`#WF1y8$pN8F2S5-rpfn5-9z;Sq8j2AvAuq6 diff --git a/tamingllms/_build/html/searchindex.js b/tamingllms/_build/html/searchindex.js index 3a687aa..9267fed 100644 --- a/tamingllms/_build/html/searchindex.js +++ b/tamingllms/_build/html/searchindex.js @@ -1 +1 @@ -Search.setIndex({"docnames": ["markdown/intro", "markdown/toc", "notebooks/nondeterminism", "notebooks/output_size_limit", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/toc.md", "notebooks/nondeterminism.ipynb", "notebooks/output_size_limit.ipynb", "notebooks/structured_output.ipynb"], "titles": ["Introduction", "Chapter 1: Introduction", "Non-determinism", "Output Size Limitations", "Wrestling with Structured Output"], "terms": {"In": [0, 3, 4], "recent": 0, "year": [0, 3, 4], "larg": [0, 2, 3, 4], "languag": [0, 2, 3, 4], "model": [0, 2, 3, 4], "llm": [0, 1, 3, 4], "have": [0, 3, 4], "emerg": 0, "transform": 0, "forc": 0, "technologi": [0, 3, 4], "promis": 0, "revolution": 0, "build": [0, 1, 2, 3], "product": [0, 2, 4], "interact": [0, 3, 4], "comput": [0, 3], "from": [0, 2, 3, 4], "chatgpt": [0, 2], "github": 0, "copilot": 0, "system": [0, 1, 2, 3, 4], "captur": 0, "public": 0, "imagin": 0, "spark": 0, "gold": 0, "rush": 0, "ai": 0, "power": [0, 3, 4], "applic": [0, 3, 4], "howev": [0, 2, 3], "beneath": 0, "surfac": 0, "technolog": 0, "revolut": 0, "li": 0, "complex": [0, 2, 3, 4], "landscap": 0, "practition": 0, "must": [0, 3, 4], "navig": 0, "As": [0, 3], "explor": 0, "signific": [0, 2, 3, 4], "engin": [0, 2, 4], "effort": 0, "requir": [0, 3, 4], "manag": [0, 1, 3], "handl": [0, 1, 3, 4], "non": [0, 4], "determinist": [0, 1, 2], "output": [0, 2], "prevent": [0, 4], "hallucin": [0, 4], "rais": [0, 3], "import": [0, 2, 3, 4], "question": [0, 2], "about": [0, 3, 4], "true": [0, 3, 4], "gain": 0, "while": [0, 3], "potenti": [0, 2, 3, 4], "remain": [0, 3, 4], "compel": 0, "hidden": [0, 1], "cost": [0, 3], "reliabl": [0, 1, 2, 4], "should": [0, 3, 4], "neglect": 0, "instead": [0, 3, 4], "mai": [0, 2, 3, 4], "u": 0, "reconsid": 0, "our": [0, 3], "overli": 0, "optimist": 0, "assumpt": 0, "impact": [0, 3], "capabl": [0, 3], "ar": [0, 4], "inde": 0, "remark": 0, "prevail": 0, "narr": 0, "often": [0, 3, 4], "gloss": 0, "over": [0, 3, 4], "fundament": [0, 2], "problem": [0, 1], "organ": [0, 3], "face": 0, "when": [0, 2, 3, 4], "real": [0, 3, 4], "world": [0, 4], "aim": [0, 3], "bridg": 0, "gap": 0, "offer": [0, 4], "clear": [0, 4], "ei": 0, "examin": [0, 3], "pitfal": 0, "work": [0, 2, 4], "throughout": [0, 3], "tackl": 0, "follow": [0, 3, 4], "exhaust": 0, "list": [0, 2, 3, 4], "critic": [0, 3], "behavior": [0, 1, 2], "unlik": [0, 2], "tradit": [0, 2], "softwar": [0, 2, 4], "can": [0, 2, 3, 4], "produc": [0, 2, 4], "differ": [0, 2, 3, 4], "ident": [0, 2], "input": [0, 1, 2, 3, 4], "make": [0, 2, 3, 4], "test": [0, 1, 2, 4], "assur": 0, "particularli": [0, 3, 4], "structur": [0, 3], "struggl": [0, 4], "maintain": [0, 3], "consist": [0, 1, 3, 4], "format": [0, 1, 3, 4], "complic": 0, "integr": [0, 4], "larger": [0, 3], "error": [0, 1, 4], "more": [0, 2, 3, 4], "These": [0, 3, 4], "gener": [0, 1, 2, 4], "plausibl": 0, "sound": 0, "entir": [0, 3], "fabric": 0, "inform": [0, 3, 4], "creat": [0, 2, 3, 4], "risk": [0, 3], "optim": [0, 1, 3], "The": [0, 3], "financi": [0, 2, 3, 4], "oper": [0, 3], "base": [0, 2, 3, 4], "quickli": [0, 3], "becom": [0, 2], "prohibit": 0, "without": [0, 3, 4], "care": 0, "methodologi": 0, "break": [0, 3], "down": [0, 3], "deal": 0, "new": [0, 3, 4], "incorpor": 0, "exist": 0, "architectur": [0, 3], "present": [0, 3, 4], "uniqu": 0, "take": [0, 3, 4], "hand": [0, 3], "provid": [0, 3], "concret": 0, "exampl": [0, 4], "you": [0, 2, 3, 4], "run": [0, 4], "modifi": 0, "scenario": 0, "solut": [0, 1, 3], "strategi": [0, 1, 2, 3], "best": [0, 1], "techniqu": [0, 1, 2, 3], "pattern": [0, 1, 4], "anti": 0, "design": [0, 3, 4], "overse": 0, "initi": [0, 3], "technic": [0, 3], "leader": 0, "decis": [0, 4], "anyon": 0, "seek": 0, "understand": [0, 1, 3, 4], "To": [0, 3, 4], "most": [0, 2, 3, 4], "basic": [0, 3], "program": 0, "experi": [0, 3], "access": [0, 4], "openai": [0, 2, 4], "anthrop": 0, "similar": 0, "desir": [0, 4], "grade": 0, "each": [0, 2, 3, 4], "chapter": 0, "focus": [0, 3, 4], "specif": [0, 3], "explan": [0, 4], "deep": 0, "dive": 0, "implement": [0, 1, 3, 4], "perform": [0, 1, 3], "consider": [0, 1], "conclus": [0, 2], "look": 0, "limit": [0, 4], "goal": [0, 3], "discourag": 0, "enabl": [0, 3], "robust": [0, 3, 4], "By": [0, 3, 4], "upfront": 0, "better": [0, 3], "equip": 0, "leverag": [0, 3, 4], "effect": [0, 1, 3, 4], "avoid": 0, "current": [0, 3], "discours": 0, "around": 0, "tend": 0, "toward": 0, "extrem": 0, "either": [0, 3], "uncrit": 0, "enthusiasm": 0, "wholesal": 0, "dismiss": 0, "focu": [0, 3], "rather": 0, "than": 0, "theoret": 0, "first": [0, 3], "learn": [0, 4], "everi": 0, "concept": 0, "illustr": 0, "execut": 0, "immedi": 0, "analysi": [0, 1, 2, 3], "balanc": [0, 2, 3], "both": [0, 2], "help": [0, 3, 4], "reader": 0, "befor": 0, "need": [0, 3, 4], "develop": [0, 3, 4], "here": [0, 3, 4], "": [0, 2, 3, 4], "get": [0, 2, 3, 4], "start": 0, "activ": 0, "virtual": 0, "m": 0, "venv": 0, "env": [0, 2, 3, 4], "sourc": 0, "bin": 0, "On": 0, "window": [0, 1], "script": [0, 1], "instal": 0, "packag": 0, "pip": 0, "r": [0, 3, 4], "txt": [0, 3, 4], "export": 0, "openai_api_kei": 0, "clone": 0, "companion": 0, "git": 0, "http": 0, "com": 0, "yourusernam": 0, "tame": 0, "cd": 0, "If": [0, 4], "encount": 0, "rate": 0, "consid": [0, 2, 3, 4], "smaller": [0, 3], "retri": [0, 4], "logic": [0, 3], "conflict": 0, "try": [0, 4], "fresh": 0, "like": [0, 2, 3, 4], "poetri": 0, "check": 0, "page": 0, "known": 0, "now": [0, 3, 4], "let": [0, 3], "begin": 0, "challeng": [1, 3], "why": 1, "thi": [1, 2, 3, 4], "book": 1, "matter": 1, "overview": 1, "kei": [1, 3, 4], "temperatur": [1, 3], "random": [1, 2], "evalu": [1, 3], "measur": 1, "observ": 1, "log": 1, "monitor": 1, "debug": 1, "respons": [1, 2, 3, 4], "practic": [1, 3], "workflow": 1, "common": [1, 3, 4], "failur": 1, "mode": [1, 4], "text": [1, 2, 3, 4], "inconsist": [1, 2, 4], "valid": [1, 4], "recoveri": 1, "enforc": [1, 4], "type": [1, 2, 3, 4], "detect": [1, 4], "ground": 1, "retriev": 1, "augment": 1, "rag": 1, "context": [1, 3, 4], "select": 1, "index": [1, 3], "vector": 1, "store": [1, 3], "chunk": 1, "method": [1, 3, 4], "pipelin": 1, "token": [1, 2, 4], "cach": 1, "invalid": [1, 4], "predict": [1, 4], "issu": [1, 3, 4], "guard": 1, "content": [1, 2, 4], "filter": 1, "sanit": 1, "alert": 1, "constraint": [1, 3], "long": 1, "form": [1, 4], "vendor": 1, "lock": 1, "self": 1, "host": 1, "llama": 1, "llamafil": 1, "setup": 1, "usag": 1, "ollama": 1, "deploy": 1, "migrat": 1, "complet": [1, 2, 3, 4], "util": [1, 3], "function": [1, 3, 4], "configur": 1, "templat": [1, 3], "recommend": [1, 3], "librari": [1, 3, 4], "commun": 1, "One": 2, "natur": [2, 3, 4], "where": [2, 3, 4], "same": [2, 3], "time": [2, 3, 4], "thei": [2, 3, 4], "re": [2, 3], "queri": 2, "even": [2, 3, 4], "prompt": [2, 3], "characterist": 2, "strength": 2, "ask": [2, 4], "ani": [2, 3, 4], "other": [2, 3, 4], "multipl": [2, 3, 4], "ll": 2, "isn": 2, "t": [2, 3, 4], "bug": 2, "featur": [2, 4], "how": [2, 3, 4], "paramet": [2, 3, 4], "which": [2, 3, 4], "control": [2, 4], "allow": [2, 3, 4], "creativ": 2, "divers": [2, 3, 4], "incredibli": 2, "difficult": 2, "testabl": 2, "servic": [2, 3, 4], "compani": [2, 3, 4], "us": [2, 3, 4], "invest": [2, 4], "advic": 2, "summari": 2, "mean": [2, 3], "market": [2, 3, 4], "data": [2, 3, 4], "could": [2, 3], "yield": 2, "exponenti": 2, "regulatori": 2, "complianc": [2, 4], "guarante": [2, 4], "user": [2, 3, 4], "trust": 2, "affect": 2, "primari": 2, "come": [2, 3, 4], "dure": 2, "calcul": 2, "probabl": 2, "distribut": 2, "next": 2, "set": [2, 3, 4], "nucleu": 2, "coher": [2, 3], "0": [2, 3, 4], "repetit": [2, 3], "1": [2, 4], "increas": [2, 3, 4], "incoher": 2, "dotenv": [2, 3, 4], "load_dotenv": [2, 3, 4], "o": [2, 3, 4], "load": [2, 3, 4], "environ": [2, 3, 4], "variabl": [2, 3, 4], "file": [2, 3, 4], "panda": 2, "pd": 2, "def": [2, 3, 4], "generate_respons": 2, "model_nam": [2, 3], "str": [2, 3, 4], "float": [2, 3], "attempt": [2, 3], "int": [2, 3], "3": [2, 3, 4], "datafram": 2, "demonstr": [2, 3, 4], "client": [2, 4], "result": [2, 3, 4], "temp": 2, "rang": [2, 3], "chat": [2, 3, 4], "messag": [2, 4], "role": [2, 3, 4], "max_token": 2, "50": 2, "append": [2, 3], "choic": [2, 4], "displai": 2, "group": [2, 3], "df_result": 2, "print": [2, 3, 4], "f": [2, 3, 4], "ntemperatur": 2, "40": 2, "temp_respons": 2, "_": 2, "row": 2, "iterrow": 2, "return": [2, 3, 4], "gpt": [2, 3, 4], "5": [2, 3, 4], "turbo": [2, 3, 4], "suggest": 2, "name": [2, 3, 4], "coffe": 2, "shop": 2, "2": [2, 3, 4], "caf\u00e9": 2, "perk": 2, "bean": 2, "bliss": 2, "cafe": 2, "caffein": 2, "haven": 2, "chronicl": 2, "earthbean": 2, "birch": 2, "grab": 2, "brew": 2, "unit": 3, "process": [3, 4], "A": [3, 4], "short": 3, "singl": [3, 4], "charact": 3, "word": [3, 4], "english": 3, "rule": 3, "thumb": 3, "i": [3, 4], "4": 3, "\u00be": 3, "max_output_token": 3, "avail": 3, "modern": [3, 4], "determin": [3, 4], "maximum": 3, "length": 3, "an": [3, 4], "contrari": 3, "one": [3, 4], "might": [3, 4], "expect": [3, 4], "doe": [3, 4], "summar": 3, "answer": 3, "surpass": 3, "stop": 3, "onc": 3, "reach": [3, 4], "mid": 3, "sentenc": [3, 4], "e": [3, 4], "truncat": 3, "tabl": 3, "comparison": 3, "across": 3, "max_input_token": 3, "input_cost_per_token": 3, "output_cost_per_token": 3, "meta": 3, "llama3": 3, "11b": 3, "instruct": [3, 4], "v1": 3, "4096": 3, "128000": 3, "5e": 3, "7": 3, "claud": 3, "sonnet": 3, "20241022": 3, "8192": 3, "200000": 3, "3e": 3, "6": [3, 4], "0613": 3, "6e": 3, "2024": [3, 4], "04": 3, "09": 3, "1e": 3, "4o": [3, 4], "mini": [3, 4], "16384": 3, "gemini": [3, 4], "flash": 3, "002": 3, "1048576": 3, "8": 3, "pro": 3, "2097152": 3, "05e": 3, "pose": [3, 4], "who": 3, "incomplet": 3, "extens": [3, 4], "detail": [3, 4], "report": [3, 4], "comprehens": 3, "articl": 3, "find": 3, "abruptli": 3, "cut": 3, "off": 3, "due": 3, "disrupt": 3, "flow": 3, "shallow": 3, "thorough": 3, "receiv": 3, "onli": [3, 4], "partial": 3, "lead": [3, 4], "dissatisfact": 3, "frustrat": 3, "especi": 3, "educ": 3, "tool": 3, "creation": 3, "platform": 3, "address": [3, 4], "ensur": [3, 4], "feasibl": 3, "effici": 3, "approach": [3, 4], "section": 3, "split": 3, "individu": 3, "previou": 3, "For": [3, 4], "includ": [3, 4], "its": [3, 4], "combin": 3, "all": [3, 4], "final": 3, "analyz": 3, "10k": 3, "sec": [3, 4], "There": 3, "them": [3, 4], "appropri": [3, 4], "situat": 3, "we": [3, 4], "broadli": [3, 4], "two": [3, 4], "fix": 3, "straightforward": 3, "simpli": 3, "decid": 3, "number": [3, 4], "option": [3, 4], "whether": 3, "overlap": 3, "between": 3, "want": [3, 4], "keep": 3, "some": [3, 4], "sure": 3, "semant": 3, "doesn": 3, "lost": 3, "reason": 3, "path": 3, "mani": 3, "case": [3, 4], "compar": 3, "computation": 3, "cheap": 3, "simpl": [3, 4], "sinc": 3, "speciali": 3, "awar": 3, "advantag": 3, "appli": 3, "sophist": 3, "embed": 3, "level": 3, "would": 3, "sever": [3, 4], "do": [3, 4], "naiv": 3, "g": [3, 4], "period": 3, "nltk": 3, "spaci": 3, "recurs": 3, "divid": 3, "hierarch": 3, "iter": [3, 4], "manner": [3, 4], "separ": 3, "class": [3, 4], "extract": [3, 4], "your": [3, 4], "made": 3, "up": [3, 4], "talk": 3, "theme": 3, "topic": 3, "langchain": 3, "charactertextsplitt": 3, "tiktoken": 3, "count": 3, "per": 3, "get_chunk": 3, "chunk_siz": 3, "chunk_overlap": 3, "specifi": 3, "arg": 3, "langchain_text_splitt": 3, "text_splitt": 3, "from_tiktoken_encod": 3, "split_text": 3, "write": 3, "serv": 3, "foundat": 3, "defin": [3, 4], "persona": 3, "assum": 3, "background": 3, "task": [3, 4], "action": 3, "input_text": 3, "contain": 3, "actual": 3, "langchain_cor": 3, "prompttempl": 3, "get_base_prompt_templ": 3, "base_prompt": 3, "from_templ": 3, "llmchain": 3, "construct": 3, "chain": 3, "togeth": 3, "parser": 3, "output_pars": 3, "stroutputpars": 3, "langchain_commun": 3, "chat_model": 3, "chatlitellm": 3, "get_llm_chain": 3, "prompt_templ": 3, "instanc": 3, "llm_chain": 3, "api_key_label": 3, "upper": 3, "_api_kei": 3, "api_kei": 3, "get_dynamic_prompt_templ": 3, "dynam": 3, "dict": 3, "get_dynamic_prompt_param": 3, "prompt_param": 3, "part_idx": 3, "total_part": 3, "chat_context": 3, "origin": 3, "convers": [3, 4], "part": 3, "total": [3, 4], "param": 3, "dynamic_prompt_param": 3, "copi": 3, "save": 3, "add": 3, "introduct": 3, "don": [3, 4], "cover": 3, "yet": 3, "just": 3, "scope": 3, "elif": 3, "last": [3, 4], "below": 3, "second": 3, "main": 3, "point": 3, "given": [3, 4], "els": 3, "wai": [3, 4], "easi": 3, "read": [3, 4], "merg": 3, "previous": 3, "call": [3, 4], "updat": 3, "concaten": 3, "end": 3, "generate_report": 3, "input_cont": 3, "llm_model_nam": 3, "report_part": 3, "num_part": 3, "len": 3, "dinam": 3, "priovid": 3, "enumer": 3, "invok": 3, "cummul": 3, "n": 3, "join": 3, "sampl": 3, "open": [3, 4], "appl": [3, 4], "max_chunk_s": 3, "10000": [3, 4], "max_chunk_overlap": 3, "latest": 3, "analyst": 3, "readabl": 3, "factor": 3, "move": 3, "insight": [3, 4], "local": 3, "apple_report": 3, "w": 3, "300": 3, "inc": [3, 4], "posit": 3, "fiscal": 3, "septemb": 3, "28": 3, "disclos": 3, "10": [3, 4], "k": [3, 4], "state": 3, "secur": 3, "exchang": 3, "commiss": 3, "identifi": 3, "luation": 3, "term": 3, "oblig": 3, "cash": 3, "highlight": [3, 4], "continu": [3, 4], "disciplin": 3, "deeper": 3, "granular": 3, "note": 3, "assess": 3, "few": 3, "interest": [3, 4], "aspect": [3, 4], "high": 3, "smooth": 3, "upon": 3, "head": 3, "subhead": 3, "clariti": 3, "document": 3, "adher": [3, 4], "request": [3, 4], "variou": 3, "revenu": [3, 4], "segment": 3, "profit": [3, 4], "liquid": 3, "capit": [3, 4], "resourc": 3, "inclus": 3, "valu": [3, 4], "align": 3, "despit": [3, 4], "qualiti": 3, "depth": 3, "wide": 3, "certain": 3, "human": [3, 4], "expert": [3, 4], "nuanc": 3, "overlook": 3, "mitig": 3, "fit": 3, "within": [3, 4], "altern": 3, "meaning": [3, 4], "preserv": 3, "easier": [3, 4], "addit": 3, "preprocess": 3, "significantli": 3, "enhanc": 3, "own": 3, "overcom": 3, "introduc": 3, "layer": 3, "It": [3, 4], "necessit": 3, "meticul": 3, "retain": 3, "necessari": 3, "advanc": 3, "transit": 3, "seamlessli": 3, "circumv": 3, "therebi": 3, "associ": 3, "contribut": 3, "overal": 3, "escal": 3, "frequenc": 3, "volum": 3, "bottleneck": 3, "latenc": 3, "reduc": 3, "prepar": 3, "friendli": 3, "evolv": 3, "improv": [3, 4], "mustafa": 3, "suleyman": 3, "infinit": 3, "memori": 3, "abl": 3, "convei": 3, "amount": [3, 4], "fewer": 3, "compress": 3, "research": 3, "progress": 3, "essenti": 3, "condens": 3, "adapt": 3, "adjust": 3, "flexibl": [3, 4], "innov": 3, "deploi": 3, "constrain": [3, 4], "collect": 3, "versatil": 3, "also": [3, 4], "fulli": 3, "har": 3, "drive": 3, "grace": 3, "fallback": 3, "further": 3, "empow": 3, "scalabl": 3, "crucial": [3, 4], "stai": 3, "full": 3, "inher": [3, 4], "splitter": 3, "excel": 4, "easili": 4, "databas": 4, "sometim": 4, "well": 4, "craft": 4, "unstructur": 4, "json": 4, "max_length": 4, "sec_fil": 4, "person": 4, "discuss": 4, "response_cont": 4, "wow": 4, "seem": 4, "lot": 4, "see": 4, "breakdown": 4, "stream": 4, "absolut": 4, "portfolio": 4, "iphon": 4, "mac": 4, "ipad": 4, "wearabl": 4, "impress": 4, "trend": 4, "notic": 4, "trillion": 4, "march": 4, "29": 4, "That": 4, "huge": 4, "show": 4, "confid": 4, "investor": 4, "definit": 4, "figur": 4, "good": 4, "compli": 4, "regul": 4, "ye": 4, "accur": 4, "date": 4, "transpar": 4, "industri": 4, "those": 4, "serious": 4, "is_json": 4, "myjson": 4, "except": 4, "valueerror": 4, "fals": 4, "clearli": 4, "object": 4, "unpredict": 4, "obtain": 4, "deviat": 4, "lack": 4, "safeti": 4, "guid": 4, "correct": 4, "experiment": 4, "emploi": 4, "schema": 4, "guidanc": 4, "blueprint": 4, "achiev": 4, "pydant": 4, "pars": 4, "nativ": 4, "support": 4, "regular": 4, "express": 4, "dedic": 4, "python": 4, "json_format": 4, "person1": 4, "alic": 4, "q1": 4, "ha": 4, "20": 4, "person2": 4, "bob": 4, "great": 4, "what": 4, "net": 4, "margin": 4, "materi": 4, "though": 4, "suffici": 4, "nest": 4, "restrict": 4, "programmat": 4, "alwai": 4, "via": 4, "explicit": 4, "unend": 4, "whitespac": 4, "until": 4, "forget": 4, "throw": 4, "string": 4, "appear": 4, "somewher": 4, "response_format": 4, "json_object": 4, "approxim": 4, "628": 4, "553": 4, "000": 4, "held": 4, "affili": 4, "scale": 4, "sheer": 4, "size": 4, "mention": 4, "15": 4, "115": 4, "823": 4, "share": 4, "stock": 4, "outstand": 4, "octob": 4, "18": 4, "circul": 4, "plai": 4, "match": 4, "purpos": 4, "target": 4, "suppli": 4, "so": 4, "worri": 4, "omit": 4, "enum": 4, "benefit": 4, "No": 4, "incorrectli": 4, "refus": 4, "simpler": 4, "strongli": 4, "07": 4, "08": 4, "06": 4, "later": 4, "version": 4, "basemodel": 4, "secextract": 4, "mentioned_ent": 4, "mentioned_plac": 4, "extract_from_sec_fil": 4, "sec_filing_text": 4, "beta": 4, "entiti": 4, "place": 4, "convert": 4, "code": 4, "dialogueturn": 4, "financialstatementdiscuss": 4, "repres": 4, "hint": 4, "generate_convers": 4, "send": 4, "simul": 4, "dialogu": 4, "conform": 4, "through": 4, "turn": 4, "speaker": 4, "nasdaq": 4, "llc": 4, "washington": 4, "d": 4, "c": 4, "california": 4, "cupertino": 4, "usabl": 4, "framework": 4, "simplifi": 4, "with_structured_output": 4, "directli": 4, "predefin": 4, "simplic": 4, "v": 4, "greater": 4, "steeper": 4, "curv": 4, "reli": 4, "built": 4, "structuredoutputpars": 4, "custom": 4, "grammar": 4, "know": 4, "exactli": 4, "descript": 4, "field": 4, "element": 4, "chanc": 4, "mechan": 4, "perfect": 4, "thoroughli": 4, "refin": 4, "respond": 4, "connect": 4, "etc": 4, "unlock": 4, "autom": 4, "valuabl": 4}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"introduct": [0, 1], "core": 0, "challeng": [0, 2, 4], "we": 0, "ll": 0, "address": 0, "A": [0, 1], "practic": [0, 4], "approach": 0, "who": 0, "thi": 0, "book": 0, "i": [0, 2], "For": 0, "prerequisit": 0, "how": 0, "us": 0, "note": 0, "perspect": 0, "set": 0, "up": 0, "your": 0, "environ": 0, "1": [0, 1, 3], "python": 0, "setup": 0, "2": [0, 1], "api": [0, 4], "kei": 0, "configur": 0, "3": [0, 1], "code": [0, 1], "repositori": 0, "troubleshoot": 0, "common": 0, "issu": 0, "chapter": 1, "non": [1, 2], "determin": [1, 2], "eval": 1, "wrestl": [1, 4], "structur": [1, 4], "output": [1, 3, 4], "4": 1, "hallucin": 1, "The": [1, 2, 4], "realiti": 1, "gap": 1, "5": 1, "cost": 1, "factor": 1, "6": 1, "safeti": 1, "concern": 1, "7": 1, "size": [1, 3], "length": 1, "limit": [1, 3], "8": 1, "break": 1, "free": 1, "from": 1, "cloud": 1, "provid": [1, 4], "appendix": 1, "exampl": [1, 3], "b": 1, "tool": [1, 4], "resourc": 1, "understand": 2, "what": [2, 3], "llm": 2, "real": 2, "world": 2, "impact": 2, "technic": 2, "deep": 2, "dive": 2, "sourc": 2, "temperatur": 2, "sampl": 2, "spectrum": 2, "ar": 3, "token": 3, "problem": [3, 4], "statement": [3, 4], "content": 3, "chunk": 3, "contextu": 3, "link": 3, "gener": 3, "long": 3, "form": 3, "step": 3, "usag": 3, "discuss": 3, "implic": 3, "futur": 3, "consider": 3, "conclus": [3, 4], "refer": 3, "solut": 4, "strategi": 4, "techniqu": 4, "One": 4, "shot": 4, "prompt": 4, "specif": 4, "langchain": 4, "outlin": 4, "compar": 4, "best": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"Introduction": [[0, "introduction"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Practical Approach": [[0, "a-practical-approach"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Prerequisites": [[0, "prerequisites"]], "How to Use This Book": [[0, "how-to-use-this-book"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "1. Python Environment Setup": [[0, "python-environment-setup"]], "2. API Keys Configuration": [[0, "api-keys-configuration"]], "3. Code Repository": [[0, "code-repository"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "Chapter 1: Introduction": [[1, "chapter-1-introduction"]], "Chapter 2: Non-determinism & Evals": [[1, "chapter-2-non-determinism-evals"]], "Chapter 3: Wrestling with Structured Output": [[1, "chapter-3-wrestling-with-structured-output"]], "Chapter 4: Hallucination: The Reality Gap": [[1, "chapter-4-hallucination-the-reality-gap"]], "Chapter 5: The Cost Factor": [[1, "chapter-5-the-cost-factor"]], "Chapter 6: Safety Concerns": [[1, "chapter-6-safety-concerns"]], "Chapter 7: Size and Length Limitations": [[1, "chapter-7-size-and-length-limitations"]], "Chapter 8: Breaking Free from Cloud Providers": [[1, "chapter-8-breaking-free-from-cloud-providers"]], "Appendix A: Code Examples": [[1, "appendix-a-code-examples"]], "Appendix B: Tools and Resources": [[1, "appendix-b-tools-and-resources"]], "Non-determinism": [[2, "non-determinism"]], "Understanding the Challenge": [[2, "understanding-the-challenge"]], "What is Non-determinism in LLMs?": [[2, "what-is-non-determinism-in-llms"]], "Real-world Impact": [[2, "real-world-impact"]], "Technical Deep-dive: Sources of Non-determinism": [[2, "technical-deep-dive-sources-of-non-determinism"]], "Temperature and Sampling": [[2, "temperature-and-sampling"]], "The Temperature Spectrum": [[2, "the-temperature-spectrum"]], "Output Size Limitations": [[3, "output-size-limitations"]], "What are Token Limits?": [[3, "what-are-token-limits"]], "Problem Statement": [[3, "problem-statement"], [4, "problem-statement"]], "Content Chunking with Contextual Linking": [[3, "content-chunking-with-contextual-linking"]], "Generating long-form content": [[3, "generating-long-form-content"]], "Step 1: Chunking the Content": [[3, "step-1-chunking-the-content"]], "Example Usage": [[3, "example-usage"]], "Discussion": [[3, "discussion"]], "Implications": [[3, "implications"]], "Future Considerations": [[3, "future-considerations"]], "Conclusion": [[3, "conclusion"], [4, "conclusion"]], "References": [[3, "references"]], "Wrestling with Structured Output": [[4, "wrestling-with-structured-output"]], "The Structured Output Challenges": [[4, "the-structured-output-challenges"]], "Solutions": [[4, "solutions"]], "Strategies": [[4, "strategies"]], "Techniques and Tools": [[4, "techniques-and-tools"]], "One-Shot Prompts": [[4, "one-shot-prompts"]], "Structured Output with Provider-Specific APIs": [[4, "structured-output-with-provider-specific-apis"]], "LangChain": [[4, "langchain"]], "Outlines": [[4, "outlines"]], "Comparing Solutions": [[4, "comparing-solutions"]], "Best Practices": [[4, "best-practices"]]}, "indexentries": {}}) \ No newline at end of file +Search.setIndex({"docnames": ["markdown/intro", "markdown/toc", "notebooks/nondeterminism", "notebooks/output_size_limit", "notebooks/structured_output"], "filenames": ["markdown/intro.md", "markdown/toc.md", "notebooks/nondeterminism.ipynb", "notebooks/output_size_limit.ipynb", "notebooks/structured_output.ipynb"], "titles": ["Introduction", "Table of Contents", "Non-determinism", "Output Size Limitations", "Wrestling with Structured Output"], "terms": {"In": [0, 3, 4], "recent": 0, "year": [0, 3, 4], "larg": [0, 2, 3, 4], "languag": [0, 2, 3, 4], "model": [0, 2, 3, 4], "llm": [0, 1, 3, 4], "have": [0, 3, 4], "emerg": 0, "transform": 0, "forc": 0, "technologi": [0, 3, 4], "promis": 0, "revolution": 0, "how": [0, 2, 3, 4], "build": [0, 1, 2, 3], "product": [0, 2, 4], "interact": [0, 3, 4], "comput": [0, 3], "from": [0, 2, 3, 4], "chatgpt": [0, 2], "github": 0, "copilot": 0, "claud": [0, 3], "artifact": 0, "cursor": 0, "com": 0, "replit": 0, "other": [0, 2, 3, 4], "system": [0, 1, 2, 3, 4], "captur": 0, "public": 0, "imagin": 0, "spark": 0, "gold": 0, "rush": 0, "ai": 0, "power": [0, 3, 4], "applic": [0, 3, 4], "howev": [0, 2, 3], "beneath": 0, "surfac": 0, "technolog": 0, "revolut": 0, "li": 0, "complex": [0, 2, 3, 4], "landscap": 0, "practition": 0, "must": [0, 3, 4], "navig": 0, "As": [0, 3], "explor": 0, "engin": [0, 2, 4], "effort": 0, "requir": [0, 3, 4], "manag": [0, 1, 3], "handl": [0, 1, 3, 4], "non": [0, 4], "determinist": [0, 1, 2], "output": [0, 2], "prevent": [0, 4], "hallucin": [0, 4], "cannot": 0, "overst": 0, "while": [0, 3], "potenti": [0, 2, 3, 4], "remain": [0, 3, 4], "compel": 0, "understand": [0, 1, 3, 4], "hidden": [0, 1], "cost": [0, 3], "reliabl": [0, 1, 2, 4], "enabl": [0, 3], "u": 0, "fulli": [0, 3], "har": [0, 3], "impact": [0, 3], "capabl": [0, 3], "ar": [0, 4], "inde": 0, "remark": 0, "prevail": 0, "narr": 0, "often": [0, 3, 4], "gloss": 0, "over": [0, 3, 4], "fundament": [0, 2], "problem": [0, 1], "organ": [0, 3], "face": 0, "when": [0, 2, 3, 4], "real": [0, 3, 4], "world": [0, 4], "aim": [0, 3], "bridg": 0, "gap": 0, "offer": [0, 4], "clear": [0, 4], "ei": 0, "examin": [0, 3], "pitfal": 0, "work": [0, 2, 4], "throughout": [0, 3], "tackl": 0, "follow": [0, 3, 4], "exhaust": 0, "list": [0, 2, 3, 4], "critic": [0, 3], "behavior": [0, 1, 2], "unlik": [0, 2], "tradit": [0, 2], "softwar": [0, 2, 4], "can": [0, 2, 3, 4], "produc": [0, 2, 4], "differ": [0, 2, 3, 4], "ident": [0, 2], "input": [0, 1, 2, 3, 4], "make": [0, 2, 3, 4], "test": [0, 1, 2, 4], "assur": 0, "particularli": [0, 3, 4], "structur": [0, 3], "un": 0, "struggl": [0, 4], "maintain": [0, 3], "consist": [0, 1, 3, 4], "format": [0, 1, 3, 4], "complic": 0, "integr": [0, 4], "larger": [0, 3], "error": [0, 1, 4], "more": [0, 2, 3, 4], "These": [0, 3, 4], "gener": [0, 1, 2, 4], "plausibl": 0, "sound": 0, "entir": [0, 3], "fabric": 0, "inform": [0, 3, 4], "creat": [0, 2, 3, 4], "signific": [0, 2, 3, 4], "risk": [0, 3], "optim": [0, 1, 3], "The": [0, 3], "financi": [0, 2, 3, 4], "oper": [0, 3], "base": [0, 2, 3, 4], "quickli": [0, 3], "becom": [0, 2], "prohibit": 0, "without": [0, 3, 4], "care": 0, "methodologi": 0, "break": [0, 3], "down": [0, 3], "deal": 0, "new": [0, 3, 4], "take": [0, 3, 4], "look": 0, "limit": [0, 4], "our": [0, 3], "goal": [0, 3], "discourag": 0, "us": [0, 2, 3, 4], "robust": [0, 3, 4], "implement": [0, 1, 3, 4], "By": [0, 3, 4], "upfront": 0, "you": [0, 2, 3, 4], "better": [0, 3], "equip": 0, "leverag": [0, 3, 4], "effect": [0, 1, 3, 4], "avoid": 0, "current": [0, 3], "discours": 0, "around": 0, "tend": 0, "toward": 0, "extrem": 0, "either": [0, 3], "uncrit": 0, "enthusiasm": 0, "wholesal": 0, "dismiss": 0, "focu": [0, 3], "rather": 0, "than": 0, "theoret": 0, "solut": [0, 1, 3], "first": [0, 3], "learn": [0, 4], "everi": 0, "concept": 0, "illustr": 0, "execut": 0, "exampl": [0, 4], "immedi": 0, "analysi": [0, 1, 2, 3], "provid": [0, 3], "balanc": [0, 2, 3], "both": [0, 2], "help": [0, 3, 4], "reader": 0, "decis": [0, 4], "about": [0, 3, 4], "hand": [0, 3], "concret": 0, "run": [0, 4], "modifi": 0, "scenario": 0, "strategi": [0, 1, 2, 3], "best": [0, 1], "techniqu": [0, 1, 2, 3], "pattern": [0, 1, 4], "anti": 0, "design": [0, 3, 4], "lead": [0, 3, 4], "initi": [0, 3], "technic": [0, 3], "leader": 0, "architectur": [0, 3], "anyon": 0, "seek": 0, "To": [0, 3, 4], "most": [0, 2, 3, 4], "should": [0, 3, 4], "basic": [0, 3], "program": 0, "experi": [0, 3], "access": [0, 4], "knowledg": 0, "openai": [0, 2, 4], "anthrop": 0, "similar": 0, "desir": [0, 4], "grade": 0, "befor": 0, "dive": 0, "need": [0, 3, 4], "develop": [0, 3, 4], "here": [0, 3, 4], "": [0, 2, 3, 4], "get": [0, 2, 3, 4], "start": 0, "activ": 0, "virtual": 0, "m": 0, "venv": 0, "env": [0, 2, 3, 4], "sourc": 0, "bin": 0, "On": 0, "window": [0, 1], "script": [0, 1], "instal": 0, "packag": 0, "pip": 0, "r": [0, 3, 4], "txt": [0, 3, 4], "export": 0, "openai_api_kei": 0, "clone": 0, "companion": 0, "git": 0, "http": 0, "souzatharsi": 0, "tamingllm": 0, "cd": 0, "If": [0, 4], "encount": 0, "rate": 0, "consid": [0, 2, 3, 4], "smaller": [0, 3], "retri": [0, 4], "logic": [0, 3], "conflict": 0, "try": [0, 4], "fresh": 0, "like": [0, 2, 3, 4], "poetri": 0, "check": 0, "page": 0, "known": 0, "now": [0, 3, 4], "let": [0, 3], "begin": 0, "challeng": [1, 3], "why": 1, "thi": [1, 2, 3, 4], "book": 1, "matter": 1, "overview": 1, "kei": [1, 3, 4], "temperatur": [1, 3], "random": [1, 2], "evalu": [1, 3], "measur": 1, "observ": 1, "log": 1, "monitor": 1, "debug": 1, "respons": [1, 2, 3, 4], "practic": [1, 3], "workflow": 1, "common": [1, 3, 4], "failur": 1, "mode": [1, 4], "text": [1, 2, 3, 4], "inconsist": [1, 2, 4], "valid": [1, 4], "recoveri": 1, "enforc": [1, 4], "type": [1, 2, 3, 4], "detect": [1, 4], "ground": 1, "retriev": 1, "augment": 1, "rag": 1, "context": [1, 3, 4], "select": 1, "index": [1, 3], "vector": 1, "store": [1, 3], "chunk": 1, "method": [1, 3, 4], "pipelin": 1, "token": [1, 2, 4], "cach": 1, "invalid": [1, 4], "predict": [1, 4], "issu": [1, 3, 4], "guard": 1, "filter": 1, "sanit": 1, "alert": 1, "constraint": [1, 3], "long": 1, "form": [1, 4], "vendor": 1, "lock": 1, "self": 1, "host": 1, "llama": 1, "llamafil": 1, "setup": 1, "usag": 1, "ollama": 1, "deploy": 1, "perform": [1, 3], "consider": 1, "migrat": 1, "complet": [1, 2, 3, 4], "util": [1, 3], "function": [1, 3, 4], "configur": 1, "templat": [1, 3], "recommend": [1, 3], "librari": [1, 3, 4], "commun": 1, "One": 2, "natur": [2, 3, 4], "where": [2, 3, 4], "same": [2, 3], "each": [2, 3, 4], "time": [2, 3, 4], "thei": [2, 3, 4], "re": [2, 3], "queri": 2, "even": [2, 3, 4], "prompt": [2, 3], "characterist": 2, "strength": 2, "ask": [2, 4], "ani": [2, 3, 4], "question": 2, "multipl": [2, 3, 4], "ll": 2, "isn": 2, "t": [2, 3, 4], "bug": 2, "featur": [2, 4], "paramet": [2, 3, 4], "which": [2, 3, 4], "control": [2, 4], "allow": [2, 3, 4], "creativ": 2, "divers": [2, 3, 4], "incredibli": 2, "difficult": 2, "testabl": 2, "servic": [2, 3, 4], "compani": [2, 3, 4], "invest": [2, 4], "advic": 2, "summari": 2, "mean": [2, 3], "market": [2, 3, 4], "data": [2, 3, 4], "could": [2, 3], "yield": 2, "conclus": 2, "exponenti": 2, "regulatori": 2, "complianc": [2, 4], "guarante": [2, 4], "user": [2, 3, 4], "trust": 2, "mai": [2, 3, 4], "affect": 2, "primari": 2, "come": [2, 3, 4], "dure": 2, "calcul": 2, "probabl": 2, "distribut": 2, "next": 2, "set": [2, 3, 4], "nucleu": 2, "coher": [2, 3], "0": [2, 3, 4], "repetit": [2, 3], "1": [2, 4], "increas": [2, 3, 4], "incoher": 2, "dotenv": [2, 3, 4], "import": [2, 3, 4], "load_dotenv": [2, 3, 4], "o": [2, 3, 4], "load": [2, 3, 4], "environ": [2, 3, 4], "variabl": [2, 3, 4], "file": [2, 3, 4], "panda": 2, "pd": 2, "def": [2, 3, 4], "generate_respons": 2, "model_nam": [2, 3], "str": [2, 3, 4], "float": [2, 3], "attempt": [2, 3], "int": [2, 3], "3": [2, 3, 4], "datafram": 2, "demonstr": [2, 3, 4], "client": [2, 4], "result": [2, 3, 4], "temp": 2, "rang": [2, 3], "chat": [2, 3, 4], "messag": [2, 4], "role": [2, 3, 4], "content": [2, 4], "max_token": 2, "50": 2, "append": [2, 3], "choic": [2, 4], "displai": 2, "group": [2, 3], "df_result": 2, "print": [2, 3, 4], "f": [2, 3, 4], "ntemperatur": 2, "40": 2, "temp_respons": 2, "_": 2, "row": 2, "iterrow": 2, "return": [2, 3, 4], "gpt": [2, 3, 4], "5": [2, 3, 4], "turbo": [2, 3, 4], "suggest": 2, "name": [2, 3, 4], "coffe": 2, "shop": 2, "2": [2, 3, 4], "caf\u00e9": 2, "perk": 2, "bean": 2, "bliss": 2, "cafe": 2, "caffein": 2, "haven": 2, "chronicl": 2, "earthbean": 2, "birch": 2, "grab": 2, "brew": 2, "unit": 3, "process": [3, 4], "A": [3, 4], "short": 3, "singl": [3, 4], "charact": 3, "word": [3, 4], "english": 3, "rule": 3, "thumb": 3, "i": [3, 4], "4": 3, "\u00be": 3, "max_output_token": 3, "avail": 3, "modern": [3, 4], "determin": [3, 4], "maximum": 3, "length": 3, "an": [3, 4], "contrari": 3, "one": [3, 4], "might": [3, 4], "expect": [3, 4], "doe": [3, 4], "summar": 3, "answer": 3, "surpass": 3, "instead": [3, 4], "stop": 3, "onc": 3, "reach": [3, 4], "mid": 3, "sentenc": [3, 4], "e": [3, 4], "truncat": 3, "tabl": 3, "comparison": 3, "across": 3, "max_input_token": 3, "input_cost_per_token": 3, "output_cost_per_token": 3, "meta": 3, "llama3": 3, "11b": 3, "instruct": [3, 4], "v1": 3, "4096": 3, "128000": 3, "5e": 3, "7": 3, "sonnet": 3, "20241022": 3, "8192": 3, "200000": 3, "3e": 3, "6": [3, 4], "0613": 3, "6e": 3, "2024": [3, 4], "04": 3, "09": 3, "1e": 3, "4o": [3, 4], "mini": [3, 4], "16384": 3, "gemini": [3, 4], "flash": 3, "002": 3, "1048576": 3, "8": 3, "pro": 3, "2097152": 3, "05e": 3, "pose": [3, 4], "who": 3, "incomplet": 3, "extens": [3, 4], "detail": [3, 4], "report": [3, 4], "comprehens": 3, "articl": 3, "find": 3, "abruptli": 3, "cut": 3, "off": 3, "due": 3, "disrupt": 3, "flow": 3, "shallow": 3, "thorough": 3, "receiv": 3, "onli": [3, 4], "partial": 3, "dissatisfact": 3, "frustrat": 3, "especi": 3, "true": [3, 4], "educ": 3, "tool": 3, "creation": 3, "platform": 3, "address": [3, 4], "ensur": [3, 4], "feasibl": 3, "effici": 3, "approach": [3, 4], "section": 3, "split": 3, "individu": 3, "focus": [3, 4], "specif": 3, "previou": 3, "For": [3, 4], "includ": [3, 4], "its": [3, 4], "combin": 3, "all": [3, 4], "final": 3, "analyz": 3, "10k": 3, "sec": [3, 4], "There": 3, "them": [3, 4], "appropri": [3, 4], "situat": 3, "we": [3, 4], "broadli": [3, 4], "two": [3, 4], "fix": 3, "straightforward": 3, "simpli": 3, "decid": 3, "number": [3, 4], "option": [3, 4], "whether": 3, "overlap": 3, "between": 3, "want": [3, 4], "keep": 3, "some": [3, 4], "sure": 3, "semant": 3, "doesn": 3, "lost": 3, "reason": 3, "path": 3, "mani": 3, "case": [3, 4], "compar": 3, "computation": 3, "cheap": 3, "simpl": [3, 4], "sinc": 3, "speciali": 3, "awar": 3, "advantag": 3, "appli": 3, "sophist": 3, "embed": 3, "level": 3, "would": 3, "sever": [3, 4], "do": [3, 4], "naiv": 3, "g": [3, 4], "period": 3, "nltk": 3, "spaci": 3, "recurs": 3, "divid": 3, "hierarch": 3, "iter": [3, 4], "manner": [3, 4], "separ": 3, "class": [3, 4], "extract": [3, 4], "present": [3, 4], "your": [3, 4], "made": 3, "up": [3, 4], "talk": 3, "theme": 3, "topic": 3, "langchain": 3, "charactertextsplitt": 3, "tiktoken": 3, "count": 3, "per": 3, "get_chunk": 3, "chunk_siz": 3, "chunk_overlap": 3, "specifi": 3, "arg": 3, "langchain_text_splitt": 3, "text_splitt": 3, "from_tiktoken_encod": 3, "split_text": 3, "write": 3, "serv": 3, "foundat": 3, "defin": [3, 4], "persona": 3, "assum": 3, "background": 3, "task": [3, 4], "action": 3, "input_text": 3, "contain": 3, "actual": 3, "langchain_cor": 3, "prompttempl": 3, "get_base_prompt_templ": 3, "base_prompt": 3, "from_templ": 3, "llmchain": 3, "construct": 3, "chain": 3, "togeth": 3, "parser": 3, "output_pars": 3, "stroutputpars": 3, "langchain_commun": 3, "chat_model": 3, "chatlitellm": 3, "get_llm_chain": 3, "prompt_templ": 3, "instanc": 3, "llm_chain": 3, "api_key_label": 3, "upper": 3, "_api_kei": 3, "api_kei": 3, "get_dynamic_prompt_templ": 3, "dynam": 3, "dict": 3, "get_dynamic_prompt_param": 3, "prompt_param": 3, "part_idx": 3, "total_part": 3, "chat_context": 3, "origin": 3, "convers": [3, 4], "part": 3, "total": [3, 4], "param": 3, "dynamic_prompt_param": 3, "copi": 3, "save": 3, "add": 3, "introduct": 3, "don": [3, 4], "cover": 3, "yet": 3, "just": 3, "scope": 3, "elif": 3, "last": [3, 4], "below": 3, "second": 3, "main": 3, "point": 3, "given": [3, 4], "els": 3, "wai": [3, 4], "easi": 3, "read": [3, 4], "merg": 3, "previous": 3, "call": [3, 4], "updat": 3, "concaten": 3, "end": 3, "generate_report": 3, "input_cont": 3, "llm_model_nam": 3, "report_part": 3, "num_part": 3, "len": 3, "dinam": 3, "priovid": 3, "enumer": 3, "invok": 3, "cummul": 3, "n": 3, "join": 3, "sampl": 3, "open": [3, 4], "appl": [3, 4], "max_chunk_s": 3, "10000": [3, 4], "max_chunk_overlap": 3, "latest": 3, "analyst": 3, "readabl": 3, "factor": 3, "move": 3, "insight": [3, 4], "local": 3, "apple_report": 3, "w": 3, "300": 3, "inc": [3, 4], "posit": 3, "fiscal": 3, "septemb": 3, "28": 3, "disclos": 3, "10": [3, 4], "k": [3, 4], "state": 3, "secur": 3, "exchang": 3, "commiss": 3, "identifi": 3, "luation": 3, "term": 3, "oblig": 3, "cash": 3, "highlight": [3, 4], "continu": [3, 4], "disciplin": 3, "deeper": 3, "granular": 3, "note": 3, "assess": 3, "few": 3, "interest": [3, 4], "aspect": [3, 4], "high": 3, "smooth": 3, "upon": 3, "head": 3, "subhead": 3, "clariti": 3, "document": 3, "adher": [3, 4], "request": [3, 4], "variou": 3, "revenu": [3, 4], "segment": 3, "profit": [3, 4], "liquid": 3, "capit": [3, 4], "resourc": 3, "inclus": 3, "valu": [3, 4], "align": 3, "despit": [3, 4], "qualiti": 3, "depth": 3, "wide": 3, "certain": 3, "human": [3, 4], "expert": [3, 4], "nuanc": 3, "overlook": 3, "mitig": 3, "fit": 3, "within": [3, 4], "altern": 3, "meaning": [3, 4], "preserv": 3, "easier": [3, 4], "addit": 3, "preprocess": 3, "significantli": 3, "enhanc": 3, "own": 3, "overcom": 3, "introduc": 3, "layer": 3, "It": [3, 4], "necessit": 3, "meticul": 3, "retain": 3, "necessari": 3, "advanc": 3, "transit": 3, "seamlessli": 3, "circumv": 3, "therebi": 3, "rais": 3, "associ": 3, "contribut": 3, "overal": 3, "escal": 3, "frequenc": 3, "volum": 3, "bottleneck": 3, "latenc": 3, "reduc": 3, "prepar": 3, "friendli": 3, "evolv": 3, "improv": [3, 4], "mustafa": 3, "suleyman": 3, "infinit": 3, "memori": 3, "abl": 3, "convei": 3, "amount": [3, 4], "fewer": 3, "compress": 3, "research": 3, "progress": 3, "essenti": 3, "condens": 3, "adapt": 3, "adjust": 3, "flexibl": [3, 4], "innov": 3, "deploi": 3, "constrain": [3, 4], "collect": 3, "versatil": 3, "also": [3, 4], "drive": 3, "grace": 3, "fallback": 3, "further": 3, "empow": 3, "scalabl": 3, "crucial": [3, 4], "stai": 3, "full": 3, "inher": [3, 4], "splitter": 3, "excel": 4, "easili": 4, "databas": 4, "sometim": 4, "well": 4, "craft": 4, "unstructur": 4, "json": 4, "max_length": 4, "sec_fil": 4, "person": 4, "discuss": 4, "response_cont": 4, "wow": 4, "seem": 4, "lot": 4, "see": 4, "breakdown": 4, "stream": 4, "absolut": 4, "portfolio": 4, "iphon": 4, "mac": 4, "ipad": 4, "wearabl": 4, "impress": 4, "trend": 4, "notic": 4, "trillion": 4, "march": 4, "29": 4, "That": 4, "huge": 4, "show": 4, "confid": 4, "investor": 4, "definit": 4, "figur": 4, "good": 4, "compli": 4, "regul": 4, "ye": 4, "accur": 4, "date": 4, "transpar": 4, "industri": 4, "those": 4, "serious": 4, "is_json": 4, "myjson": 4, "except": 4, "valueerror": 4, "fals": 4, "clearli": 4, "object": 4, "unpredict": 4, "obtain": 4, "deviat": 4, "lack": 4, "safeti": 4, "guid": 4, "correct": 4, "experiment": 4, "emploi": 4, "schema": 4, "guidanc": 4, "blueprint": 4, "achiev": 4, "pydant": 4, "pars": 4, "nativ": 4, "support": 4, "regular": 4, "express": 4, "dedic": 4, "python": 4, "json_format": 4, "person1": 4, "alic": 4, "q1": 4, "ha": 4, "20": 4, "person2": 4, "bob": 4, "great": 4, "what": 4, "net": 4, "margin": 4, "materi": 4, "though": 4, "suffici": 4, "nest": 4, "restrict": 4, "programmat": 4, "alwai": 4, "via": 4, "explicit": 4, "unend": 4, "whitespac": 4, "until": 4, "forget": 4, "throw": 4, "string": 4, "appear": 4, "somewher": 4, "response_format": 4, "json_object": 4, "approxim": 4, "628": 4, "553": 4, "000": 4, "held": 4, "affili": 4, "scale": 4, "sheer": 4, "size": 4, "mention": 4, "15": 4, "115": 4, "823": 4, "share": 4, "stock": 4, "outstand": 4, "octob": 4, "18": 4, "circul": 4, "plai": 4, "match": 4, "purpos": 4, "target": 4, "suppli": 4, "so": 4, "worri": 4, "omit": 4, "enum": 4, "benefit": 4, "No": 4, "incorrectli": 4, "refus": 4, "simpler": 4, "strongli": 4, "07": 4, "08": 4, "06": 4, "later": 4, "version": 4, "basemodel": 4, "secextract": 4, "mentioned_ent": 4, "mentioned_plac": 4, "extract_from_sec_fil": 4, "sec_filing_text": 4, "beta": 4, "entiti": 4, "place": 4, "convert": 4, "explan": 4, "code": 4, "dialogueturn": 4, "financialstatementdiscuss": 4, "repres": 4, "hint": 4, "generate_convers": 4, "send": 4, "simul": 4, "dialogu": 4, "conform": 4, "through": 4, "turn": 4, "speaker": 4, "nasdaq": 4, "llc": 4, "washington": 4, "d": 4, "c": 4, "california": 4, "cupertino": 4, "usabl": 4, "framework": 4, "simplifi": 4, "with_structured_output": 4, "directli": 4, "predefin": 4, "simplic": 4, "v": 4, "greater": 4, "steeper": 4, "curv": 4, "reli": 4, "built": 4, "structuredoutputpars": 4, "custom": 4, "grammar": 4, "know": 4, "exactli": 4, "descript": 4, "field": 4, "element": 4, "chanc": 4, "mechan": 4, "perfect": 4, "thoroughli": 4, "refin": 4, "respond": 4, "connect": 4, "etc": 4, "unlock": 4, "autom": 4, "valuabl": 4}, "objects": {}, "objtypes": {}, "objnames": {}, "titleterms": {"introduct": [0, 1], "core": 0, "challeng": [0, 2, 4], "we": 0, "ll": 0, "address": 0, "A": [0, 1], "note": 0, "perspect": 0, "practic": [0, 4], "approach": 0, "who": 0, "thi": 0, "book": 0, "i": [0, 2], "For": 0, "prerequisit": 0, "set": 0, "up": 0, "your": 0, "environ": 0, "1": [0, 1, 3], "python": 0, "setup": 0, "2": [0, 1], "api": [0, 4], "kei": 0, "configur": 0, "3": [0, 1], "code": [0, 1], "repositori": 0, "troubleshoot": 0, "common": 0, "issu": 0, "tabl": 1, "content": [1, 3], "chapter": 1, "non": [1, 2], "determin": [1, 2], "eval": 1, "wrestl": [1, 4], "structur": [1, 4], "output": [1, 3, 4], "4": 1, "hallucin": 1, "The": [1, 2, 4], "realiti": 1, "gap": 1, "5": 1, "cost": 1, "factor": 1, "6": 1, "safeti": 1, "concern": 1, "7": 1, "size": [1, 3], "length": 1, "limit": [1, 3], "8": 1, "break": 1, "free": 1, "from": 1, "cloud": 1, "provid": [1, 4], "appendix": 1, "exampl": [1, 3], "b": 1, "tool": [1, 4], "resourc": 1, "understand": 2, "what": [2, 3], "llm": 2, "real": 2, "world": 2, "impact": 2, "technic": 2, "deep": 2, "dive": 2, "sourc": 2, "temperatur": 2, "sampl": 2, "spectrum": 2, "ar": 3, "token": 3, "problem": [3, 4], "statement": [3, 4], "chunk": 3, "contextu": 3, "link": 3, "gener": 3, "long": 3, "form": 3, "step": 3, "usag": 3, "discuss": 3, "implic": 3, "futur": 3, "consider": 3, "conclus": [3, 4], "refer": 3, "solut": 4, "strategi": 4, "techniqu": 4, "One": 4, "shot": 4, "prompt": 4, "specif": 4, "langchain": 4, "outlin": 4, "compar": 4, "best": 4}, "envversion": {"sphinx.domains.c": 2, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 8, "sphinx.domains.index": 1, "sphinx.domains.javascript": 2, "sphinx.domains.math": 2, "sphinx.domains.python": 3, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.intersphinx": 1, "sphinxcontrib.bibtex": 9, "sphinx": 57}, "alltitles": {"Introduction": [[0, "introduction"]], "Core Challenges We\u2019ll Address": [[0, "core-challenges-we-ll-address"]], "A Note on Perspective": [[0, "a-note-on-perspective"]], "A Practical Approach": [[0, "a-practical-approach"]], "Who This Book Is For": [[0, "who-this-book-is-for"]], "Prerequisites": [[0, "prerequisites"]], "Setting Up Your Environment": [[0, "setting-up-your-environment"]], "1. Python Environment Setup": [[0, "python-environment-setup"]], "2. API Keys Configuration": [[0, "api-keys-configuration"]], "3. Code Repository": [[0, "code-repository"]], "Troubleshooting Common Issues": [[0, "troubleshooting-common-issues"]], "Table of Contents": [[1, "table-of-contents"]], "Chapter 1: Introduction": [[1, "chapter-1-introduction"]], "Chapter 2: Non-determinism & Evals": [[1, "chapter-2-non-determinism-evals"]], "Chapter 3: Wrestling with Structured Output": [[1, "chapter-3-wrestling-with-structured-output"]], "Chapter 4: Hallucination: The Reality Gap": [[1, "chapter-4-hallucination-the-reality-gap"]], "Chapter 5: The Cost Factor": [[1, "chapter-5-the-cost-factor"]], "Chapter 6: Safety Concerns": [[1, "chapter-6-safety-concerns"]], "Chapter 7: Size and Length Limitations": [[1, "chapter-7-size-and-length-limitations"]], "Chapter 8: Breaking Free from Cloud Providers": [[1, "chapter-8-breaking-free-from-cloud-providers"]], "Appendix A: Code Examples": [[1, "appendix-a-code-examples"]], "Appendix B: Tools and Resources": [[1, "appendix-b-tools-and-resources"]], "Non-determinism": [[2, "non-determinism"]], "Understanding the Challenge": [[2, "understanding-the-challenge"]], "What is Non-determinism in LLMs?": [[2, "what-is-non-determinism-in-llms"]], "Real-world Impact": [[2, "real-world-impact"]], "Technical Deep-dive: Sources of Non-determinism": [[2, "technical-deep-dive-sources-of-non-determinism"]], "Temperature and Sampling": [[2, "temperature-and-sampling"]], "The Temperature Spectrum": [[2, "the-temperature-spectrum"]], "Output Size Limitations": [[3, "output-size-limitations"]], "What are Token Limits?": [[3, "what-are-token-limits"]], "Problem Statement": [[3, "problem-statement"], [4, "problem-statement"]], "Content Chunking with Contextual Linking": [[3, "content-chunking-with-contextual-linking"]], "Generating long-form content": [[3, "generating-long-form-content"]], "Step 1: Chunking the Content": [[3, "step-1-chunking-the-content"]], "Example Usage": [[3, "example-usage"]], "Discussion": [[3, "discussion"]], "Implications": [[3, "implications"]], "Future Considerations": [[3, "future-considerations"]], "Conclusion": [[3, "conclusion"], [4, "conclusion"]], "References": [[3, "references"]], "Wrestling with Structured Output": [[4, "wrestling-with-structured-output"]], "The Structured Output Challenges": [[4, "the-structured-output-challenges"]], "Solutions": [[4, "solutions"]], "Strategies": [[4, "strategies"]], "Techniques and Tools": [[4, "techniques-and-tools"]], "One-Shot Prompts": [[4, "one-shot-prompts"]], "Structured Output with Provider-Specific APIs": [[4, "structured-output-with-provider-specific-apis"]], "LangChain": [[4, "langchain"]], "Outlines": [[4, "outlines"]], "Comparing Solutions": [[4, "comparing-solutions"]], "Best Practices": [[4, "best-practices"]]}, "indexentries": {}}) \ No newline at end of file diff --git a/tamingllms/_static/logo_w.png b/tamingllms/_static/logo_w.png new file mode 100644 index 0000000000000000000000000000000000000000..439fac0b4efd3b337d202ef1f62ded630a158c01 GIT binary patch literal 41746 zcmeEtWm}s;*KHC41StfH6?ZEZptywK6ev=PLvSeC;>A6pZ-8Wq&f1+ z?U@WOkHPda4wJuqg(MU!93VZ&XGy?!1(CF&Um#N!saZl=H%~<=thfp^Qx;O9fLS;x z-w{y=SQu?i0$zs@Ar!?5&rZ|m2N51mxSO*oa43Oa^96zUIimwsCq80eEGb}M ztm5F}8jL~R{eiYHo;*cdmZxrgoD~Qzq>-MWL%g03?FfY<$>J23er9t8kE3s97i zdh3z3*Wy`gp=-ExeZRkh>+T`lOvcMgzeS_~RbxX(56~E3xBrpN+EZ;1VL|U!RE*0; zq_e}GJRVXor^Vz_pY*Qo%hcbuhNcS<<2pD*P6(MV;NajWN>h*HrKU?}t;y$qd^`l@J9QAGfAY5X#bMh4$;he^ z=^767MI+A<7a|q?rWl>z``D(D?0y;br|tZHhkC(8&xs%%SUzxDa8KC!b3OkpZF;j&1;9u!<_eIV{;UWBfT01j zfL{=xu6WiSLx?QE3p9a*g<>3!DnHOO#YWD#IhjQ_|s@#rT z?#u9BM7DvzmyZz_H@Z$YxnaJSV;ut~X zW0?M-PDE7%ApXh2LpRAv{{KwJO1QFCtMj}|nVv1ge#OvV09z)QEZ|p^&4e_3?dqnN z^gz;}#j}1PtN&-ESt*V`Ix8pvsLBhZ1jUwWm))C37is6=2PvrVZbM zD@nr<)nHjr4}_H#I~3l$rrk^hz`+l@j@qa6V;ng*5#bm2UxzKVN)I*JU3d3}e_JNw zj=pokrNLU61IGekq6YEkaQ||QGy%@HCM#$`*w63?ir6Zzc6T+Ld+<%u2dCs-&0m`C zcAmQGcF2aF0vlP!TuEejbz>9|J0Oi00J!`$4~2q#?=Gj4WIe7d~Q=>H}U}qHD*iztT-ZCHp{*kXdotbbUQ)m zjNRmS><_PIIBx`%UX{I3P=s~2ME!EP)A5MDD5f( zZ7^s6fCv2RW*q&JXqWO!Io(LB0OxAB z65E)t`$Cbu4YRWdUcc3=1O*Ck=bbnKD17b}=lF>TI6dH) ztLEo-s6i0GLDW-V!r66?EnCY0McTz>8+0U6lyt&@>9klV%6Jl7GJoYI3|KguE3xt$ zRwX>s1;!1mLx8_P)o>=~R_kejzya?@k^6Fmv<>xvj{?mVifCW-77ZlC+@%86_=7>9 zBNI0b?{RAB>u>E@{pkJo%>@@sR3v86V?CM};C(Z3n%^;K#0ktzyPx80f0hIAe|$Jx z$hLdK?EGr9c2V6jFc$YK|F9mQP^gBz*bZ%pubp7Hu`spuNBSUJ6dG8bN1n%nxwZcG zZrF(Q^*@3W#rNM!2-PuSK>dp^`Xz>g)1fRpvNH3&xyIMPEDli7?s|XYSHQM72iiwZ z@1CYo@3mj?jesxmXx2F$J;Kozgd1)_{y z+}=jL(W_mO<=FoY@KqIw zkqH;lvds)YN!kl7LJtO*BH`hZ3q{OUynK{IFHyf%AP^D=sEP<6mg0zlM3Ha)Z;dJ6@5Mi@Fq`AT0pdj;} zSul=ROlqCXLcoJ99;B@&P=NjAQBP;V&bLr&9!ykCR!%P9 z^env;@dbCf(C=@L?VrCruI_()2h<(g#(4o-Jmk2m3!9VgpQ{>s5{fNY&cq*5z(EK? z)cV)7kpKKw$GHw9fQ{+%Bu5cOyg5A<22AtBp++K)gFU$Yde4%VG+-TH%<0jZ|C8-P z&dCjZ46SO)2T(~E84N`h1z~n);cGIqb7Mw>cWDL~S)cvROU8J|4p;KWrMLaXY7!WL zh0S}dY<1BHu|s|-U)_~m^Ss1^j*k-XvQeQgGHd>&A98EG!dv64aZp2wf#b-QqP zos=OtS!4V5+@S8>7cwglx@9*Rn-iE%6KuAb45lqIC@aOL3x4SfA2R16-o^L1iB?+6 z4hCf49M0`hcEFuTipC^=dhX>fFZFC%@&W*nMDSil)go@OAr>7#x#hc|>n-h8D*3^2 zvEoq?iN7vx(Y7B!n^=ppnK;GHed>{XRF=G*gqi z$k`8)?}pMs;?nF#HS+ECtvTJR+0qBK(D|DBzcX-a6SZp9_~jB%qKhe2tTd7oj(v$9 zdrUzsl75psNzYy0zp&(WW-Kz8$0@TqgF_Tz#z008oxdi@VVKgWmy)^IJ&eskM@SO_ zSp|?YNy!WJ6&ZLQ0BT4qRpGDt62rNkitwvPbQ-3sL*^(J07=W+y*4WYZVTBaC)4Wa zT)7&*iYWW#xh#!;Ka%}?U;`J*Z~nHb6|*djMB6(!5F|#cNZ0$;k6sQFKlk?vd%);9 zRE<{C?7lQy@u$m^gHk_wP0p?-j5shm#|vNo3VNKm`OaXh_nRm1sNTU&C09RMUQp~P zJvOF(I@Wu=xA!~#o*M;Uy5=C-0KmzBNCSMEZrd@)7v#@MANZ4eZ%xSJUJg8w-r@X# zPJ|RsR9o?%D^K>)RG$Wc40PcUE|b;>X6?Krk$}eZ9bJd{oh^&`m1Hzg5Zz8YIF<*g zV`z};?B+$Oes5O-v1?h7Bz=tF<5O_Dp|Zz0$g1cC6K-47MH$<fj4#srzpXy5FgDti#=yma(2vsqOvM!`jk8u@y%;~)Wo>knoybcM{ycVyunW=&VHsmohw{fF&Xygp`x^KT?Va*R&nV+R>G zcgvisIMH)j^Tu8fovtTZ9#NJv5){Y+)@r`{su%Apef*LI7p@>9IWT6N1cA|ND3dtR zXdug+$?$h2p+q#OrGO@qQVyt@4p9nW76=(W(T%Y-c$SflV8JIb0F8Ee%!~V5?Xp#tMRVSl^gU1^;w!8O2z>}M zHZu>~sXzal2KaDl>zRW%u(gB|zk`3*xce)1Ma{_Ds8Qq>2=2hYl3F$OKmgk+%|KZ| zjgKK%>Us5HnACpgZQu(1~&)@*h(&@ zdlf_Kt{Z%U2u3fV$-ocbV>T62^cD2H0)S2d)!R41+Q}nJZg-jb5d%34orB>?^HrTGxpEJofU#2&CJQ{2+w?*IiH5Uz(!1EhSDl1MkvLE&C!L6wYTW z*?ZG!_j<)kZ@xN`msm;Ekoh?{3z2P)rR=|FOV$c#`f1M0B5~cH*tFO>fJLuiYis)| z^{+?u*t1@X7y7FL@(5+Fdiqx~j2DN?#$m{mx*IosBR%Z=okp^f#Pg@yqQmmkU`G%p zW@bL6qMO+!tBcHfOPYg&uS0hFH)}pTf}OvADMDrLXCn(z$PM#+t6!1X9s69mKDDQ8 z*_U1s36Ws>C7afg3fk7M;y-HP)P8@O+erOb1`VDSh-0<9O=Dspg`+^c*gW83UNPs# zYFC01|&CAKJt4)r}t?{IrOwi41z0wlhtz09bo70HI zOmT`3&fN}W>kR6F0khg&&UfD$UJVbLc?GvV%5@jyGj*;kI_}XJ!>6L!W0R!&w!MXf_ zpCsSm6)O)ljb2&kKN_?XoQlg_{LY%vvDVqb&}Sr#$vFtWEGmQ@dbY~OkXGsO}NC32|JC6LxwDTcI zaG3^m`0~eHO|k7mEz#Yu9v#yE4s3!Y^wtEuwOlF)yP!{?S?W2kC%_+PSp& zsI&KYmb|XWWFLYB{v9GvhfIGTx4YvRa_9Da&3Q?2GLo%RPuVU7@)ZGyqqsuyhYj#; z-}*_bV5{%?^vuJoA0_UFyQ3#gf)$uyxZ9gCKtZuNCUm_d%JnXV=3!MoR_VdxixTQD zK>?0pfGo_pPw^v%hH~rKv}}&UBF@S(8;bpGkwaYx9I=0Ll%x{G#dq7=A-L#DLEGj% zzp(YMz`IVMM@4JmI&2r{xvQSSG)*HCFg0aC`3xB2kLLcktFk-Kz<4K*5%S>V?y?@_ zrHywtw!Gxy-ll&q$2y#{0df; z-{aO|xzf$>PVM!4fzDEvodYycS~mqkmscYmQ7RKfK!*XLgBejH&1`Y>Y;|Zp+ctIe zrgiD&|mpqHpKZfihy6Zu!eJRs;mEZQo@BH3d9C#jeyzyNzz3{X6+e`fA zD58M1wTL`f{!NaUB7%a<3@RPWu5>mXpg?;Cp#v+ccY5KoC8hD-ySD6CQsAZ-%`f=-fawZw{@{F?5uJ6!)>>o)Yr8FEcezoxKoAeBSfukgbCM_gbbKGAW6 zJoWYMvRCzTeY5xdd*L|07wK;&-0fTxHA;0Wfb>lMP>@y;0qo}2Qp-(#*n!{H^`4jO z`B6<_!Z+|(y}-o9x3W^i;fwi6n@2IPJg#@DFFstf;vYBGWgE>XTocQH*$1k&R-PTM z{c5XvIJ$51YS14JSBSZ!zUfo?MWYisyDdiXLR?j~T^(4&i{q;nxvhdPV(Hv^Y`duI4zOP4gw2PwwM$);Hf(9qXO(_K;S9MA?qT?f%r!{g#Sw8$|1-HkqaZ>^A}5hn^v-lG$UmZ|Ew#h1 zp8sD`Kj&;&aR-Dm5JR<-}4yJ zC1)p|t8Yclm%iJaj>4bVZe1QyJ|u+w(D2H_aT+TuWv{izq3`^>t3LC}8<&=D)eydL z!DI0q^-sd$RN{WFFNXk%{0*|7p;VAsBWyX{z9rFM1(@m|1?ca|fAr^=Itq+wUw6$^ z!0(;$*4Ij1wrY8l@v1HQsAe|r_T?WiSV0yP%=lECNXsl=lEwCIH{vDQpnc`1FKOOg zyb)1QaA^s7hfyq`vz)cZ9kE7C>)Z3Mbyt!~xY=`kcAOjbxX+YXL|iOJiH3I}fK{BRa;%(^v%!w{@p>LOt%ajrTH3R@*S4`RL44*Llu8QzgIVg zR#-mo5P`y?*rdAi(lD@~0l>6sB|%aEO(%BHwsnh&7b+_YBoHPfYZ`F*3%D@`!S{d8pc__^(ft+tx%k{cv}l05Kdp`v>yk z3-)OsbRQ?;;~C$;z^$JWg5a(1)Ll>d$NzB@u#i+^c914WxWLDXk2s+f~Q^ zJr?3^T%P@B*1A$q7QOIgaJzLQ1_EMTu?lLZo}ITt+QX7PpV<6j0muVBV*ec<rB!>&bhBh) zwhMlzJ4Y6xt~ zFH`Gg;*UkEAg3oqv~%T>*#C&Jati>^k#BK)9y@{!2k*}rh z^4}JS?v_Qrsy&ml&fuIyn#eKuqc_K*ve8SiAw>v{i5#whJd( zkAga<(aG0VpYV5g6Jf@W^UGv`%XjXMz5`)gvX8@}{u6o>hUQ=%2*}WpE;*DoytI_Z zp5jXPB8iMa0Xxf{Sn?K(T>)Tw0-M2a<1`SYO!IYN?=VWRJ^+xg*T*tPxSz|DB901rhFzwqAmfBTR#@MPCLv8FdJMV_RiqOp)b=#JjkUGj;YGueuIzvjlvl?6OXp`&0H za~boe^wiw%di>l%YjMZmOE8oyN}iT}c8L=dk?qXWixkNQ#nTl91DNPg{G(VJYV6r> zAXtWEKcRG=d;6m+lrm1ZEP8W|7QOZdxL5eI1t;8oE3tK_E2!X8epiVr$*ssv4yd~b zY;{BtByqLl3T@5y0V=Q|N;7^FKWLdcz)>kFJodNp;#OXlMz?1^Gk>lsD^LBleZ2F> zF&K#(hpTBrs=vUpRbgbfa2e=l+F%_!#)%P`9ZGu#yL=v-^N93SADIRHpY4Jw^9O?X zRINLUPvg|I8Hm>|`?>U8o43vRyriJTIiBG=K{rEr+JMqhc#_7S5a_rN4wh$1krD^9 zJ_3>?i*Bh36XT86ACy6z+2^eH%}nDhIUlWbIKQ)YZqcVlx!`4PSPGMKt!`}g+9@O7 zP62$Z)QQeDdxvHYi7Gg9LQIbT=<#hz&h}g09PSefdop)!?GQ-C z{YFY3m2t_w;R;Ti8W;Q#kYWmWD=w48(hVh$lH-Ayg9ka%G@b88$8sf^_!WN>2Cihk zZoOD_iB-#K01QzEaH*77imeZe8vB}#EZx3QxV~^*%WN`v#)X?v88S{xOKi7=^a|Ui zoUB4@i|Q-@LY^20HOxfmScEl_-R0pgQfJBmm=Wt&MJr zhp~8+vwp{3o+9(CP?*6{oF1GUNXqG571^4i74V#bg)T~Dx1*)mZRMpN_d`ZeLm@t^ z4mtbBAz|Gvfe+kp1`?3}St=M19Q+oYo&^WF!rv9k z2(Qj#J$tEN@^Epqs_!7a+&aB3Zmjoh@9`z`wPT>wx`96oW9c^ObuuIhQo-21Gc`vY zR0=kIA~ThsLZ2X}5YsBu5e)v~Spp~^5#W$5ElgP$+n(Y&pq}PlN!9q*s6c-p@)KwT z4het`w+e60p>4hZD=5vThdd(El`J@L7SZ~E7k*1M z)){Hyw9hk}XJ4PeYU@AzuNFXLw?Hi+9YBb?E8KG`NtO1* z`ugV;f%v2{fYcJOdu(>?M@!dbS7To(%vBH_y;k(aV;cG7VGKfICKN(LAN-cKV9tZE zgL0hg7XhVst@7&{Q1efLDULQC7}N(K4_Ay{VR-qq1GXq1?fN^RxvO`r>$PoI+s2?k z2$^Q~^SJ@|AF_UCYoW*h!8c<@(my z+_jngW7J4~m4f0gB05vvB~LnJ2;eh@X)r4m#DT6iGmN^?oug_eJA_A8B&J244%5F- zH|G}e5f2#kJnl`?`Hye+{*DCSb087P_Us;qZd>VLb=o@%y8(%!`C=mP`MlOt9Cla zOHGJDjxW%pMfy-I7>YuYndSY+vCuW)3F8wmvZD)*tn-$4F?(ese8#8s$|E-JUpf*r z(rim(8_yH$-+K6j&zKVx^^`D3NIq=+>1+uQ)_pfn9lSeY5v|;r%T^SO%h)#mYt=aY z)6O+%S#GRcgv^^QtDL?5H1qvK`!Fadw2Q9%$N9yh{-XOLMW)0Zb^jW(VbW-^Y23<^ zE}ZpWwltF20{0n)O+rBuJfbIRRIiXDQGiL|EIa5Kb$<_nQ9UNzHFc-f*~BoI=kK|` zfcH`RYG&@ehmV1g=o!1_)S(Gsef(Q8F3v({0bTN)GZ4MiJK93eSM?2%eRWAX3lpxx ziiyg6HG6y!)d(vu<>DdtcLRx!?#rz{q#NWXI!?-$yVp+d9F}mxHjNJqVha?|89%4k zXh|tT*i zhqd)LFLl?VOdQ;RFo=NM+d_h3l{S*#t3v4~Z1|GH&Z6mf zAYtr7dnSc2BhYZ-DGCbtpNOlO+DKbLD()X-sUgsftaT^|0@4ykAd&Ouf|$LzldntO zIiM7HPRC6tbFT|VSioC4JI&VYY6h~Ocu&n06Wbc<1>+8|I{FvDAovWB7CckPalY2z zy0Pz=TiV)EU#`@>L!?Av6H@X+fj$;+s74|p9$AJR>+$&e^`DN@%@>Xy=M$=*ZpUpL z^R=)X(d-TZ_JBixEM!UACct+dDXUc>(r{h2aJf1>DKfawOgP>$T(hPW%ZB126kPJ$ z5_BNiWT0sD=znYLqS5htuJqsn`wb%yB4e9h1VVs6_`{WBp2tsoHqUe;imwyaVysuu z1JeL$fuX?`SY^14tdTbAt@+KXjhD+8y-$x#@C$+3$X6YmYLXypqx*4F5nfxSct8 zRKl>v=Af-a5yHcg}UW^_nl?B@Eqm^c_)k_N33JJCJ98 zjqsLaWi}PF%1&3Sk6=@J+~tk1-k$DPP6j%?ee2S;Dt>pdTI#-N^6kQ;B@;GLumV=> zf44aXjwVUH)6;Sv8LCNMmJN_i6AK7T!^qw``C#HvCsb7t<_at7Wi`#MexF9zK06d| z#((c@d-BL}*m4oc^%pkZbQv1-VW17OJ++UlUN?hgk=uDGwjKV8tbiQ%d+Yb`;$6)u zy@h;oTp%6HMo&3|q_}+af@BJ*Foi_5!y0mx^`FYoEd1W2P3_)WM-M6h2lsN-1_J{I zhCBz?)wQedoDTAu7sa~ZSs5xJVY2xZcZ*o4el8IJC}5>i0K}Ml78`2$sF-H_B%&ZE zLM;^AaYODm< z0$FLH*!ED!XoC9fQ4E8l-z|L^(V|a%7S2?ko@>L~^7|G1Kdzg%Q+Yv*vi)OJ{$Y%PmLQ;q>!SxU=EWXlja7#}-Bw zOIsv1!25|SVT!xJu}SteNn-^%#&3fF0`w^|zm%H)Ftucj_ui-3+IBLWahgVh>Q#N3 zIhlX>)_6pnD|rfNh9XWzh(WZIGK{I0%XhmfheoYdBjk;BVbdZcvLR#)!l~sXdbW1< z_W8Ra-Pb1-- z@izxG`Fp;@`0MW*uV7wCz*8!^8K$?`2L>lRaR`kH=>bn=o$dYeRb?q^v)~^#QwnS$ z{nJDmN?Af5Kufoz6^72+q&Y{aS=!P$M>V&B@=MN(+%mz%HfluZ2nZQ22I~MJj1eY5 zwkOW2bk9P4hj00vH2=_xI$G{?QbEJ(86=OtGvDR6720I@1pjP?T|uRi6RLkIB48*w zI%MR^EO8BAHAf6fyk=e+o2lq-zbg#?uwiulgG7`Z=0*MRgC7EAwXg!CWRA~z7fbCY z>*DLDYx}*ivGJ#Ivc`5JKXbB0_^Q)SGJQco!7+Xn>|!^(CvdmqMP{~=#h|!W^3~uT zDDc7cJ9c`}ie^-aYQ_o+j$;3cE5p$1=&1{8`WoM_5&zNoQj|nc0#_uQ!@Opqv?EhZ19%4 zb+qNl_D+md3irSAATC@HtFg1}gwm_%kIk1hq?0=={+GV1?+cRxFPB7jL#KY{8)qo9 z7(dZ9;fa~wId-=V^2iuMplZYx87QZAYn>qJtm~Yja7Ea2T)o%9E2F9`Y9g{N+|i+| z(Gp#4*R|`_9uxd*W?}fS@SW06o9~_+I_@OSYDG_K6XBtl0dCBubxGL<8GBYM*en&!Mq8 z-n+I3jqH)v(U*!yk*&NRy-&&VX;G-1CZ-bN31(ZKa=qtEnndB!q%nZwoy9{|R_tR- zd2}S*h`_35swf_f{7IX#sL)+enZrD=iK{ER;X)qI&18dcP{Yf#A2tZ)BAKZEpDS6S zV*6KHZa1@(VjfPkIQLB)-4!{SBy0kxJ=?N&Vbu8Xca)qcvc!Jo z+7tkN5+Q1=a5p2wcGgLUJBn?(G^o;B8Fn;{^=IR?~!$Zed( zxHKo5yDv4R^~1!i`og7}UcpW=&&%}m&oD21o(rS*zq^oyjQ^LAJLL)=pUd{%i`#Uycc4kfAffUH)oo%0GRyX1c&$pR7=$Dvt1| zqsYIJj8+J7^hcD#K9H=)_oVF3>tauMPw38Jp{}f#EGdYb;Au}%IUp*QBZDb)Qm@2Tq`@2t73x)R9%#^D%amQN zr!}>+&k6=Oi!QDqY%2#qbXQ*>W9Rnfc=5tAObG)KuHsqsv9@#Oo%EtZE+U4X>62v7 zTP&+CBLR8TG~R^?Hw)VDF4{v-WNAO9BLlP{0Ug$0OF6?Cq$U3lScA~{h$GlP4$KD8 zfa1{UK&t4ggw_w{4oZ3b_8T4a@6b$aY_L=&o@gb6Z8RCfy9*M^A3|{u^fGeIkKM2OqRqR2DznK zgIJ|-*6^Uq6vLSUUUDBlcD`-1MXPDOGaq~=O-mXJB?wcS^KxsDD9NazOlS4YE7Gph zBqd-*BaeTcY;n=`zwx<8?@~TS|DlR!5JfS2=JIHSRw_~^4&d%X@d6f@mYiZe&sM%` zVu(+u)7_xJ<|#y3okU0n=2t09zs952sJS?p%>QN1g)70ZJ`Vye>vfZVBcelxSG@mwBfEm4xXy7{5{JG`DYcZ_Fo%eE zRZEDfer@9tNph9#CfIk$0M(ZJ^*XuUN53gvev|aX@*{0**s`~h*j4N7Uu*1GR}|Gb zTG#S3E}VaMZKO7hf;=f}a81u*qcDRc@y7hpk{%SATu;7~4;Gs$8?MCD8fnRKVfG0= zUyH=yG;UvM86?nvBGPozJ*nZm_?yZ!?uMAmNPGKdhBF-*&majkH~o!8a_R0X=tChOJd#^q7}Lz ze4cKVz;)*rCq2kNxP$kkRza97l9y6iU2fu0uqP_nK2xTx+#+M}A84X(d7=Iy=Jf{& z?pLRiInI*XJT(%pe{o~@x39Fr*qPK~gwvgus-v4V#NvSpJuL2s^A*HS6K0h zxEe`K`6rJ!RBb03(kM#ZuhmzVtH6jc{h$9$fRj2=nraEF$_WK` z1bz6`HjHvwvO%-Y!gJ)|vB+lr8{c23!@JqRi?3H&>Qj*}1y+^pIj{$~f2Rlk71HT= zvAR-Xo`0wWn^Pc(jp9O#j>fclNjNR`#`4AC0}la zT}z|R0=||0lek-8`<59U9tt3fR=_5UF@d$%frK4>&&zV_be~D!F|62Os!y+M1MT5T z3=_qg>!!$^IXls@cG~d;>Lss$C~V zcr|stv!N1WTMHWzkT4FJmx-v6rz4VC_C+e3v?1S13hmvt{p2R*BL@i8-Zwei zFv`qn$@DH@#CNSEXM4EpZ;(`fp8^)3eZhBgx!;M>*Yb4rjo$&5?+`_fbrYZ+>scv=2>Q6^2=cM18&xH@0JCOdByYQ7B z4OcBKQ(k*(9x}BJe;dAzyi(0d4?4BvqQj<5ZSZ=kV65bkk>v$o_@$KyL3nb@OmiHg z5l!GN%g%(nal2xO!Y6(ez7Z1GWxY>4(LWq*?J(JeI!aI&S@zQ+Fj>dh4q{dzQ7a*i zh}GSCw0Ez*`Qdioiq{@Fa@ijJ#L+YBHY0diX;7}gIz)PDk)y>nBnRZDyDAX>_&*gG z=iBF)_|%kQTCaadk~`-*+FI#qArziFp^`M>W#GS)=%pb~O_VlfRp1@Fs>idGlK z-uV9Amd;)3_RJv_@&0M#u-*o^dKK8SszA`a#&Imxe@n_$LFMt# z`#tR8%#=ED%z9sd_he+CQSMO8TB5^T?v;Y@yqc&rGXai4g)jcI}5*yl0`)qaOy}9IN%F^Xq*zA4t_9Tnh*l6#9V$rcVB8aNqv$5 z-Qc+&eh|jG=O%}KehV1EW+K6Wz_&SQp9O1F)yW&a$JM{QxIJhn5p|aAc%8`GYJYn~ z7;&74?pyro{p1_H?c^3zOV~c;X9edVA?^6AH82yI>=7A|HT~g|eTCESGEwsW1Hkys zWBL4R?rlTZ=3%bqwSz|{|rB)V@>2$(1r$k8McqDNhzV*Jg`(oH)UJZ0gfM5 z1J^$hURV48Ju!K`KhCJ z$FHgsgBJC6S{M6gTy+VfU?^KKfPr_RlBJM1u%=?g>2l+0e9MvQanCOu>zGB>d2e=0 zmD*?JRkw$t(?vk1^=xa@eGHHLZ3Erom@tw zP&5RT9jK=g_HZEtp000DloWY)s0`Ae_z4aFT$))KOF)z~hG1n&Xlo5f`)hMiDf)ZN z=XP^!M6AZ}@eem_*?k&*?qNqIdNP*SCwcjgc7I5(^pP1ny1><=HfYLRJ00ngtMoN} zitoA-4KyfLUEcOL-1oAOE@bw{tG*CF@v5Vq94nTLb7Kgdj5-m5jSk_|$*KYhuI7OZ zOZ!u!;@b*!{k9wAB`+`AoX)n#hFB&;M||saB`?}X9J4%*ED!fGf!z&NjJ*Xar>i#c zZg0Ol=C%mN*6f%5^YJqiC|mH-c5bXyur>wdAD5!?P}@_xE&qt6pguPMbTlwR0hi7o z(^(5a9>_?32Ci~k>WTd_gXa_Bo%4k?Kt3mwDbSI>9oqf`vE3Xi>EM*^x zqLj{0xIW#JidZ=XCH7M5?l^`7#9*q7!Cl{atHsv;!Y32jZjNr9Ph9f;D;Sw~ti(H& zWlh-?KpPzF1n#e9->xPZ|HKbeOIhJX&4*p6PHM zeR$J}Hexg6?vO*cA)30?Qxs2j2 zM6*!LYKuH&80&8dKyE^nfi$fLf%(e$P zMtbzUTL=iU&b4Yd)-MMyo{p_39p}=9A}tB}U<3qF6<*JwkG?WSFB+|(VTDfmrNhRfEWzt?8By~#Rcxd^$JTiDkaM%q@B56hGd+5 z>FPzZm!GVDj@v!N99IkhGnn6gA6fRjjZiHMT5ieoSq|8}4UwhnhLfR9!xgZ0Da{v^ zH86mItWt#5Ns9s(I3FB8iuo*+IJVsz*L4Ns`(1TibW^!EOJ}Fu$Iy<5h(K19zspz@ zowWW7$*BV9&iZ0d364c?gy5$w?be%iJw->fo6Wr9T)9tJ&K>=LN9!c;a}u>yP=f5? z7ge_kMYfKd2r*HCSHi~$`%9J7qdQ{02B?#^El-heNeFc!T6#uxgDv@rW;GMbBKjZs zXR$-$hnMk;{Z(b#Pr)6>_z1+8&g+z_Z%RRZ@(^)z{Vf zC1mWn2bRXTK~EHA{93PBUeZa&7|_mof*X)Py<|8e{_yJ&#>ZFl9sB4RujJ|MV^Pvi zb+(Y^Up%m};2)8$zjmo2dftmq#mG8;T)qp{OqpIAf6X1OWadbS=+MLefjGVn3><~) z*GI(RB;>9(pAPjr#c+LB3UKf;DZ9|v2Q$m7fisdG8_}%}v3*U{)4L(t7d2;(aqgo$ zkW&o?Xa%s~7-_*atP79hgFbj`XliYB<{)d{r(zGIg#^(9iNU`XPOH>i<`S1bO-0K2 z7}&-u9%C4n{N!{@kTtNFvfLLiXTV4&{H`y|j`y@jD!eW_pjp}YFYvm+J6(Z%+x-5$ z)G9|aA5@R-O;g+l++5wf+HJSZsh1mx zRTKhbD$*fs<& z$F4;hDz|I9gHQ5dc&ViPZusfs^GDG=o$%rCz6Y1asXL(UfaxfW6h|r!IiDqd*T!mp zkc||`an|VPdy@M1=2`eO0%C$*!I6YZBCK?@yOu}|B{gV!ooHOAG&8}crJ&+Vdt2;< z9jEJ8GX@6~(WIHO^UWZJS?O=}zgmDy#~hku72}5eIxkli zA5ZJol7BwX0XS%1#C2_UQMVXjj$*4}`Oos_e;Eg$sj2BSkYw;K zFx!vX(8I*++}-YJOX=w(Md28kz{BxU;bLWDva85qnP^R)1g!KJ(?ujU=hw;Ido$cM zMnwJci-YF!&kW)ycY(})IN`VVdW`qd8=sa(1{YG^*Aj|JN)8ORJt=-xi0L<`=%>i> zFTljX5Elr+c52jkr8qh)i!aTZd6e*5uK3LnP1Hvr%M0BScOSC%JYuzaHd_u*x^D#= zCWK}V4sHWOR44_H!%8~(U@5Szc&Cu={r$@S0Rut&zQH9k&!Csi%{@5sVzFb7Klo8= z^4Eo-W>GihI+a!WE{qwo!>|*?G*Gdso~ND|e9D9=AXz{eT2o8A)ip<2TKgafVDqx- zjllxWx+-NW&?x#{=s=<*!qGJS;s5luw@r`e@YZCW4DoEMb?4P5^O9WG@vCcF{W7l>%;GP z-$zGn*Rl7#=Wkni`741Rc-{h@1f;LJbt+%)#)JV71+}`1gcZs6Zg$KO=Zr6W_NzOT zgNbkS5BT#gdfJPmlwrHuLlh?5?esk}Iq$t=W?#cxU)anJjYz+H`<(jLQQ1>PpsUmw=NwwC7Jy25hKT;W?9bmls!no_ba7r#Bq$GC}2v5(i)US`Heea z+g?F1U`+ALb2E>9XeY+&^=AIxGA%wSKtolVHLSzj>^!FS?|Tl>)*etv>%jNMjn-vS zN_4y3VaL}M8D7jD-kGcNZK}c82wD5Y@3MDyW?8nO4xoDiI(PD;$E*ccX$4fzj6D1? zhy|owJxv7wzI^14ua>>uU)$0Ed5$OuK}rGylzEOMP5`pV`yC(%5KK+$`9!{UB%p&h|0P?pSC7$&m06U^aj=K)VaDLMmv} zsY@gEfE$hV0=8m7uzv@l<9TkL=WNv;+FC1ApalR?Pqait1mZ!-2Ly@xW@aAw(#Q>4 z#L1aQ|59t+*IENqX#ua4YrMqy^zJ#MvS(05ErdWESFP%;wdiy@g)zljM>Xy$wzw=y z?}(xRLJ;C)(X(|GZ@g-wUSf9k1oll&{mtC*$Np@G%mqLGcpr@4UF$vlEJbT_qLTX) z0$QnOZ`JWTQ>aqG_|x6lC*81<;|78JfY!2#e62&0BrwK>lauFNvXwEzBz}FCWytd! zQ&UsO^BhrBUB64}9_omA1^-QN_UM+2SWlPU`O|OLT7O~v#mbny%X$fpX9vEnsq*B9 z87Ziv#iEwM0|?R!NRM?p1ST^0fcGBz_MZon8+d0SYO#W}->sSw#v$NFmfzZ?b>^GCn(p7%Vo<{xE_7yWw_X0JSKBf2b2dzBvi z(};1r+&@ zulfI#LYk%kc@Po${eE!p;3ennka^pVr`6ZhOGFN;YuXFvUc-FPj(UBz$p6t8vjMe` zO476H>Kc6QX2&~p=uk@#eNPYsAX#;bO`;gKG+6n$=K*-uUc7y?V{Qe$@4g}q)EgMy zuM`Ma!Yak(TgC5ukZw=o1h5d9Tiy&xfV_wFi@rZ6T=7M)K=p)?Sn{MPey|To_({;XB^CqlKKNm0a0(1_(3;*MTR% zQGM*gSdIhdT}*@4-MuEUIA?qWMz6q2uDYt#&ody7;Ejdt_7H?2ND5eMKuQ2{1e7_- zyay$LM2N?B#=45Yy1$fc0a$=sUEnXOb6f|KiGg0b1ECZO#>I5_@Wzx&mN8NfIEcc^ z-K{$tF^OT7@6u>cr68~+k;VoHu_o)$^v?5Z9kC$cXwboT+8Vnv5rqst$KW~h85L!KT>!S(?mLOIjUo{(H z_7H+#Y_;-MIslkZ6a@!SxMsw6RLmbe`jt?Mqe9fOi&%IK7%cRN0j(tJ1=yp#_Q`+P zX?wkK|0aZ}G{$|kcoIUOC<;hP06S;Zi z+XmIU?idC*SK;j6NGhd-0|TVNUe?WSY~7H$3eYLi^`gMwv&l1G2Bdq{f0bW%- z%^~WjxdYgO$)kW<*BR%-huuXE9}wQ51gZ;95^~#vP}LkvBx)IRg5?0E4|c4O^QqTd zbNR`TUTheUq^n*Q-Tt-CXP)=q1Ao5W-xY%5VXp-#7oy|5s1yBy z#&~rtZjsXaL@Nn6Q`M&gAxi7OQUM^y!8NWJ2t0CWPzbT35>MLGXPF&@_JA^QWZ>bd z^X&k50WJ)n6)2FfwhW#LOl@YY9ZL#ao)cb)%wc4am6bSl?BSn1&0qQYkr=WLS&=^; z#|CZ%k(GM2=44rNdldXwL-`rO_PI+D+ zZX_u4tWs~vEy744lIl(XFWu~z+W`shR{l`^!t;B#;j`Zn2NvI3_3@I$pe;DK+6Al{ zhz+Y~cEijah2-KUQ&p5w3?s$9O5v!lR8JC)){l$>K%-2LG?Uhiv)%>}x}F1hMPH>X zJepzr$|t|+P5Zw3{`b#r!(pvSJEEVmjR>|ND9^BE4&7|P6t2?ePMT0eDBH);NJ2CV z^ZpNi_`~bY)#R~eKK+29gyN}HvZu|w5NVlbpfE%r1S~YHFX8h7QQWF(YP<)Ogor}; zZix`I>7dAFAZuq@g`HJkr zsn9vM2&6t)lqE?eH@g1Z9%uXvfx%zd=wwrtB^r$etaT$-d|8cNuXj{RJyjZuB&ifM z2C6hwKcZ!@nA~9#cZL3OwQKF98;$1Clk@ErwZ2z^Ja1b6y(sb*1wnvjYhoy=D5{hs zXdMmeZ}032TTkFTVrGGgw#>#pua|MM4-34uJz z0PoP4XrQ0=k>`2U@JC9Z--piyCRG!@GymM3wDJBs?`)sO+XtyW4{v6yb0e&9hT#Oy zI_Ear0gte;75~`+5jL#1Ohi=^C68U(@}HWV>I{@e%bcTs|oDKYXLzWKEK^3>W12S7(L zgp@F42^oe}r_iE=YBZp=1Tn*QdLQ}BEw^ke$C`LoI&h|)?jx==h>`>lKtE5R;|O4` zQYQ-!s2Fk7K)cfiTFoEd4Lf&&u`|90vDW!0iqP-(ciWV`i5DA=s0Kj*DP`y6$wwa< zrAcei>CVp&8Wnl((P%VK6h)n;vLel`k1Edoed;YBr9L@OGua56T_8yczID)Ee$Zry znQ`*O3HZ9zHt~X<|K#K(B&omN%vfX2ZH8%kYZOtRlhjT806^s7hHm;V{NgYE^^_1_ zRo>S*(*nc_3q`b*lwdM31ra7Fqwvy0BksrsN{BE5c|u456f0zT2{+NggJt?(FMsDd zM<(UGn~a_DJ&4k(F7Vq_QSMm}6K~iV&+{COMq}IC=AI7C#zZGc63oxf52YfVb8AR{ zo+F8{yGT*y-E}))E@e}@PoWOKZV>rqW@xQZmL(>qriQyCN(o3M;EaVaCYwKT-al_< ztTE>piyjk;9dTf+Mzb1?P;7Vw9(2+pst%Y83S&WHQHG>P+U>$HV3dTIXpSs8e&JQG zf9AZi-{wfbmnEWB6UYsSWc6GT3S1hPqK`0%5as5P2j6w`hMVIMb|*4k%+3CDBmtyd zfIToV1(Y7%dPu2JUsie?Pqpc>T%53#1j3PG0U*<8y0Ol--UN;b|vck@L^AsTB(Hz z2%8NU=TVdeGNM;L=S^?gHxk1qT5oVk2#^GXLOwf-Xy0^|n&bg%48kaYnrfoxcd*~m zubt)yJDxKR0F$x$Ls?`tn3w>vyjn%71P~h57(~q&#v4F*1kDD}igCi5r+!)ZPhEQ5 zbuZWrn@7di8Q+8CdCrdgw#dzG6h2GK@4ffKqRHJNj6w>607X#@)l{5wYwE|389h7E z)W$h?Ss2EcpKD`kY6@Cw7-N=P1A`*tH0>ddf`1#iQDe>?YTtVps2LO(@;GgLCD@q_ zJDvaft2dnpo&AV_#lqrdAu`V7QTlq0Rw0~}y4y@%v#kcBD%|wV*6T_LxV(VY0YuV3 z-t7+SdaE6`9JV`;sb*s1e*1%`GlpL%KHO36yY<#5nnm%KIxhgBfUIh5AV~wHkQX^b zKoA6!^Ctjffr&|Uw7zh*+5E)QuDj;9zTw(yFWn8>x0|suz8A3on9Q0bT#J$d&sIlz zRZe4#!pxPF+%6(W3>Q{mSVcu?Q6A%%oyU1U@-#5I^wP*#z5u9pX!5)q>iCvr38|{0 z%|Y6dQVL~hK08vQP6Oug1-e)8U|!xCk+{;eUD!GG)z27U?CvvfC)kjZy7#rBO8q-> zV>#mknr!cmnZa=I;96sx05)JL!0Z8~KrB#NAZwu7;> z;TAaWyr-U@dh?7R&`)>PUnT@y>^&U>{6TBoq*5YcrjX!6hENLvRKu@Cp25rvf@ja{ zKp;hbPbBoloVB6$_n0wb_7GFO-US|V*wuD{g;jR9+eH|L$nzXg6e91ZU*2&;$DGlK z6(0XP*uLY8f`vORXZ(R%ZvWWyPhaX)LTluu1t|#@3TzD^b=CLNRm}}g=GkRW zedpW1?JGCk`u}W~&GGq%-f*JGAd?U<9;Pgz;t;lU5K=;t0K*`kn?*d8K()I^0Q~T2 z4MH0$;2cE&>EGY~zI!jZ=B@AV8Ge(DqM^fYy#Zw1xCs`+?z1cauJqrHj$nKt!X){g zvNYdgJ9F)){LIh(uPLVgT^{Rw^u7V|2-kv)HlpfF;2nC=B$Wo|A6DU2D4q-!u zPMU68A?GD;d)qT-BK0{LH{tpvv<%>^1zQK92tF@Bfr4X;Xlep+p5Fh&?RR|abjKm~ z>n>7g;J1tbYIm+X@4^ee=_J{2?4@%+t6A+rSPKCUPrfQjrpg(YwkoK#Ttcc4re`4n z4bhyOe5l*|;RDmHAGU{-z3A{eKTAdW_o1bK5QY24)6@;$QO0R1>u~Vmr(M>Zy5N?B z7d-vTQmQYedHG%)L@zHH7X)zvr2}|ZX=0XzK@>&MDu7Z7&U$$7;T^B~mHoo@PyJ#5 zo=34CG-k}09YZ*AXubC+r2t~2X$mQcj~{>N*iP6s=ImmLlp*=C9@8o?onB-d01i6$ zLC({D5K1IT1MEGTjV4f*2&N}sdQ~H@tS~Ufq6orQUUBWUn{E=e7^~^ZV4?-*8IT%| z0i`v9K*8r3!YG8Z7Lb6;`_R4q=@xDR0CJVDRfK%svvc$8oMR~^;y8x27DR+zuLonyFfVwm7ZXtBr~sb99_)RU95@b*j~R0Y z!-A%k{os`MWc?JjG?<#2f)L{58Q$z?{LI>67CYfgRZ$@T`0>{2%J|doH2KLp?`${8 z|FKK^P)Y#I=$)Jg1rf^mxhiFZE#jmJNR8C9>O*h2Bj=@8UmcYy_{pqv00d+h!V7{e z3uIY_Ac|0A8MKxNghb?E4(0q?n;MLDQ3?-y@z>t_=>FV(%K_*Av{71?-lMvxW#y220pUHI zFmTcUnt)KjnG$d%tg}E-fZ_x~1u&MOgC=AcgM$P;2ed8FvmU)L62~1q<9O7#{#e-j z#J-=u@#Gc%<7a;3o36h0+edbQYcO`kYo?z3Gqc%n9HtimFP+@Q1x6>2%LcO0E7jE0jPuy zC1496(10mHVO0cbvm7i0GUs4eL8=7KD`=5GsR@L>2~j5CJc6SHa}?}(2yvJ%!H1!F9< z465HF1*ib2l$cLKJQ2vRd0eR*AK~Dk3vT@B-@g3%Yo0sm<2^U-*fp3uaPg~yxbZRP zaUnBcy@ybmJ(z@&WUOZ)HG~jU6a~UigV;e!1^9skj{^ijC<*H;JyIsXgHc)o2m%BR zzb7fG{61y`p+u4Qp_N2gI1H zcMQd&)zIG))I#H8P+=viubvh^`w8$2&pdS0H_Nm*`8#2er&?C?xnTe*1cZXOB_KT5 z<$$Ve=*#d&UU=={O%-y^Q~buZBRG|ybOdr1LMy;z09!~|`5L4MWGpTuxv9d<1r{{K z1#xv9`Na=%QRQwEz~CM896$HVzx?>j&F^~6*V_CWCfg@}Kas`=>8j>GG6YzH!WO(G zcmpV*As`_|04_ba^dK2jj=N+)F}wy`DBzWY(HSiD0WJV`06dr^j7rd#5i&HQu57;g z@v!xo12vZCZ?t!rEvXz|5&5hnhBI_E2ZbVo%u3KlCl{$tuY3@Jfmi# zLDoC$Kd>KJKjj1ar+Id6o{e>f;<))ed7i64*Skln2**~kXj|W5A}ntyarQ9Isoyz> zm!9m^Qmb}*{V3t!Kp>VQvxS(8_1>eDLY8F+wSs5Ab1iqroKXZpgw!%zULa~UVSE)C zbjy1CR3RULSfBL^-4e@!Ez6`Gmj?m1Itko++wb%)y!NekM^XG!PKzoC#*9Jl5Wx&T zn5S963PCfr{K?ZD=hCaMj(U^LdDWVREq&e9&jYZKywF;eA;8X{rDcBbnEYJ;cWw6A zYS9P~4|0%0cr#=c4H z`eg^-cGb_Fm^}C{K74pvpV#NCGoG~0e^s0$zgIe2orB@kwW#eafTAeiy+s^W?aZ}Q z5RxE-K$hi*0`&xY_X)QAFNZET@E_g2`CRAtLyy<02p4b4wuy_r)fn?%k|a5$d*m>( zSbrZy5xU)Rs_B?9dktovuO~24?Y4y=_F%X1BC0Xx782FMO@|S(ZntV)?n^-KnN;6% zg6UBGdN0jh=Upl=xgS`xL*WODfjUKqnUOWrPo3@<8QE7^Mg+eQ*>E%r`~x7kT3b*W zhk)t#KJlSjI$JuhMXZ?mL%v?@h}weP;ep%ldU){f%dflPJ92VA-e&(&(<`R65-0?L zS{i9F4+w@yb0Cg^B1fFGV6q&xv>;oh2q?{~ z8{yRSFYGU~|9Z*w*FW{4+iv^Hb{*R}nWl2!z$MRYwpt(R_xo!sMmzwIB#F>!wb1Fd z5ri5ffILr+wL5d~4z>8kxtT{Vm_PCOE9QMu`)8(vx@U5Fx>^ijade&_SU~uyc^@Fq74GRzdE50bIh`@6*?gnYA#A$? z!F3#(eB~sfN=X_BeK$y;>HV*7^|-_!t@9LPR(SIssTTgi?YDgFp__i`wew+c;DVxj z!9^tAJ(>1Dlk|EnDoZp14W$G`SwadRJ8=w>fItbf;s|Ayqe{O~092LUf;!a;Rp(C@ z9x_xYtb=2MlL|c{aZ-pE%tqol1eA?r|gwyS%x48APEoxjU6^{%o#!e0V5?E2M?k%Hw)QV zW9mK)S==$F56^b*w%~6PVYtXA6+?#>sJ>Qt;~rD3Xv2k^Pk-whpFI~v|Bg^1nwW+$ z1`NMYkviNOuO$A0qWtvdZ@u-NtsawDRPeh*CUcOFzlSfU<>?~deD&3n^Dcj(%l(Tp zEnjdVPoJ0U+nd z`yV#FzkTkFH-6{6?|%2+Z}m9NxyX29>Vm5T(G}i%SoXYPzUU$y!7vOVz`os{{na$U z6`hkuKUh2BBSUH8GS8Sdu>6doXsFyWX3PS9r_=hzEJH|t3~QvSGF8tP24Jm+bN=AY zSvTfPLZK?C>-4&SlyKgyrDcjU~1 z@QpSIz)~Ua_ZR)m#lC<5m>?KvNx9{yH~`R~DsuxWJc6rQ$i2!Z@4U19g}d(h>-%qg z&pRG{*DWu8RbZ+P#;&xRHOnhYlTTX%#&zghCiAWNA(+&`S!q z2KwJrcX}%G%*9&2zBBj4Zvc4nsBHuQFq2XWg2W=1YN=z+AP`rPwQ*#8%$U80gFRM7 z##g^TO=%&utd$JJo@&p0#=vKP7&uK+a2P=d4b}>v$`NF@hrOb+l)t+3&is;h{`6hB za98zZ-4(L->zJzI#>%JgaK@ux7SxPhx8C1<%4=U6mrpvs%|VE?mm+RXz!W7G&;7ip zNHuWC_RY@z0e}x|$8ia+a?`0UNL`UL@N&E7$dM%~Ro{5!l|kBSeS6oGKVT%i!U%DQ z$br1CQj7?YOz=X$7CFEQ3=g4$su0+FfEmUbC>020o1efVwHl0$xbN+&s<5LdM{e%pMPHW=+R@lWAT{1%KpiHI7$R>D;ZrlTStt} zxy|i8f$E!Iml9Wz(8IN$0N^kvc&y*(JVX#;#+x^;`$zY+UX^+dO3TpA7P35p95flN zWg0yM1OVX~p|c;l=g5)%_8ifoG?jJUNel>a=C&YQ?3~l#!*aUF{)=+_I!pWmo=JL1 za3tYafQ5z#LYQ6#*=WFLDWElk5+IVW&O*cyj_2vq&IkYeR_xmKJ9&`X#ycRa$L!uD zgg_KU$g&JVC>Y}ap&+G%a}GiXc<&*EfE4s6d3W~xBR6ar)*1*QHk9}0tM;JVe)5)o zE<5r0j0(R8Z6EddWF8lEZ8N9+fz8a!bcIA)N(o~Of1>_c5hJYX_ zEuiuoEiShuMP-1gl~tJ;BxSX!$GmV37hP6oiwG+pKI~@Sb@N}%+& zROH?{2P*)l2&OE7MgnI_KueG}kmTTf2?&9*G{}Mwog{k8bKi3PQ#WhNt~=v9#pIzU zU7F|S8J+~n!hocLV~@$nNu>Qggb)bB5WQXx6BA7oMdpgKf8B1|J!Z_9GZ?}b^E^cq zSI-t^Mi2y*ws;sqN(mtp;wV9B9i&iavNJwrL#rIef7&~)eU1tAH)h7bl@^c!Itr^ryK{(=7EHen zI1lL@!Xp2tuikv~e{9#W7!WHRLhBITIk0mJy4=7yvCRJ5t#^FsftzmnkpnEhdpgj^ zP?S~a6*Ht1)#9Wq0P--t1pBgT(nt~@fs_3#C=N8=wAt~`8E0IQ|L$h9N#1*Ctr5pD zthMNLI!GD~6h#4J3??QfaPs&GghB9E+5G&rX-J+9mQH=tu2dj0X3Utqjez)k9LF%m zR1v4EYWVaA%sAbMBlvIHjmkR)O|N=iOdbfcgNdeBW8NO_^>Wa zm!w2l=E(CLq$NlypeP}wfOifKK%N_jr13+~fAc$6tPAP0db5!lK^Rv~pAsCB~$2h;91Z5t(DzWw&Uo9-7cnk?NT z4FEz3xGV*nK@bH90tLVVNeFhf`n)_rPfXx(@~@`Dhc{Kkx0jspk-D1B`)^}r7-P`S zQuO71V`x;AFan62afgPEbV z9xnJBjRw5;U}iY>n3|eGqtSqr5}kJETXx&xF=qy}=)b_y)VFbp+UU9G$dNKM_U@Pj zd|9QI0svcwu#v#!IS>SZ2b4}wLWy?BrxbEtc;%H1H97I>em_Ie?L&yV<~s?Y;}}v( zWc?gT(gHD{P1%vA_@iw-7$recA~~=h{Z0qAD1ahAXT5^Y-FoYn4hs4%*X@?RNFkI! zm_#VE3}s$GhY8T_0bnFa1IP?a>Cg}5!3(m^*KcObz2uCK6hZPCS(ag9VgjvJ3sOoD zB&3v(bz#gPYHo~yv3ASao!iA?r+W|~CyYboW5(=ty!W3fiUMU>LTg>+tqy8Zq(m5o zXt&$2*1}o~rPSrE)&+-l+wL)Y3kEHb;T3B={>P5^>a4KqycJ|_j>HVK;t1XuD5?Tz zrYHbW=lAOnwpYLkg?=+UB~>N3@Vr-@?53fP8ffjGhIa-q1yE!#X&=sdkW#S5Ls$=O z?1%2Z^Ui!bk0Z4uoV7^19q1^6(7NhAi0WEfXlx|lA;5I@yXjxJ<(4l@hx+HICK~W% z4@KTb(r7|hfIB%t8AEs=ZO>zB{{a95Z#~K&*i<8+bH*8uqUh<60=<5Uq9{amSZLUi{zflXHlp7=j4w4A}Xq@Y32!n>vcn zcMeC3{*#|_{o9rla-JyM8yp0f8QpFda3x4fKuf3~z|{0Kgw!aFfo339__ggm4(A*I zM%Ze>I9mt&2x8F%cP{!P#vHp#s9F~Qi7d@)*6xo*g4Fed|;a-gSG>)iunJS4RQ`X zM#i)9UV!=jym8>7f64Rm1(Q?L=ykeCk_3_jdTAd)*nnjV_KdPL2&IB4vsdi4-2g!6 zK=kWOg@84mYBbJM*)al-yAMw9`>xqEgChYIXb{yZ98f^%Afp(r09eN;iu9_%@1A%4 z_0M(9*1d_2(CK!eV+{iWdkcVI%M$b1ED*$q;uzG={_e}~zV%DneGIM1N%WmZS(bqJ zaD@ScfWV+j<*W7v6BgTOKI8Z>H%fla=fpDY!*0j$jAviL|$9L8`=MxB0AZ5Q{`E}|%^ZW?1M$uS7zc@8NwCMGVP9GO8IgZ(Lw z)3GByX3Uv}7WA$6DC|k46v8lsEpkXL z0P@f}0E~spG5}Rm9TP@BblH_x9t40)>o*nb(ChcnXf$9;Q|r{54d5h6;GBck?!1v1 zv<@ZnH#xV^T8+uquey?1@bKT>+wD$+Tf!i~G7hwCFU!y8cAwwfh>PhcHy0S3GU3xX2@mF^+ne6iamj~?l1FQ|I(hd*pZ zp59HKVZ8$ajhu5JRPes?H2_rkBRK}PfHwe(DmNVx0sxb(e2+R;9H<<0qRl@KiPiCL;ea5n z8YYp4@E*!~C<{mrfCLJXD1}D96qj$?fo=k;#ABJUmAZ`c@Od99l(1PBkOc26;2BaW zKq-{MCz}~(v>}B>4)_v(arjN;1Z%Bf@`AE@^|;;i`z|f>0z`zOC=ewvAOw_BgUJux zdjvs%UatoSMx^6Si??ruR_lU8VG#VB&`NmkPmN41bG*Ya96QQ;pS=`D9J9mBAAjuA zaiAWAcW}N^ZW^emXf0t&17;pZ&AVv^>-;rSQ%~Mhgk?LJIC#+y=rDQ)uaTs@HzdwH zhPuvYGgQr`RW7fGUv%z?v5-qngN4aBO>HNcB7G#$imS#kjW){{?nvM>2$P1SJ$;9GodYp@Od$Umgr+ zE#MuT^WDviGg@bS16xS#b{ou$si`R-jz;uopY4TV=mB7AY6|Uk8^#!vWr?yZ5yvqA zAj>jLPfr5?@+|w_(b%#!GQh7`Yp0kQN@}dPJ(Oh`j4`YJ|BXUphjGl9kzuU;w?Rh7XmLYk&%G#Utk0CBU?kTL(vZrDn6MF^%l(llN5J-hFO*N_Fo-ig+a zjD?1dOMgrN(_oGe!}riRQf;?avZX%vo?m)McKd&!36&!*GzKMkzWz`MwK53+0bC;B ze+kZm8E{PSg5YJfa|Qym_Tjlu|KcD-Kj5n~Mc{XU{78g{?)-Ve5+A*HOl@2)@`{B7- zj1vG7a6(j(XpgEsVE-i2-aML2fAN*~yyu}^J-}_^pWpl5l=9+JaK2hc*Vn2iR2O=k zZ3Xf`ATir(BPIXFD-ItHcf(#|>3O|=A4%MVa}4%?4r9PEkQIn0rr?SKzRY2Y0?i=! z@@B>vwKJZr7jO;35dD516B85AQtuy?AxRQ{snu#hYduU?A(9A!0R3*SQc{s}=%9DI z-HW2u!Pkz;o>T0bI`qmcE#8$DxnO3bX$l=i>;7I*6!6~f%0)V6WSj$2>F%?WGH~sg zN4``P<=-$ffnK@_8(Q09S=X^@jr}5ae3d-j>Bp|uUqlPUJMj@KU-o1NAw*5>6 z>ThCr2=)+MU(Jg%WY{nKP!jS;obR&YZ98q}OAa5_CJJ5}9oUC%(T5WfLWeNjK1c+B z3Xx?6BuNB90c=6i-?y2uMtvoptiE_&NLg9>;7rpLQ50c*ejdiUXOGGdLGly*exJ*- z98QAPmSE?HxtfFg@h}V#1Ob|j=1u$e?;p9U3dz=6!YKKDp@Sd{V?<$$FitiU8TZ~J zilTVbHlE3hQ*g$t!j4`S%{YFaElMu39PB)-GsyECTI)Jpg)lieiL{>~?PpLyh%C!5 z&k_C(5slpI*mvm3FUpMj%`i^r!2W~irF%%-bp?nkCv|Vf@3)7=;$w_@K{Qwx4IH^a zIs^mKoh#1x-hmGvb`9r`DC0m5AQ@CnC7-HxGl-NgfffpItCSWG*a8Xo@dqD$^sh#2 z{^?yWytP6#)tmodXl&((Q|*WjqU4Kf#jS%1L(Jq|9GC9rFTei!=Wb=()8+nN`j&ag zle1Y0N*ZA2z^<~1ovSqDEf6UUawQZP(%|Ep8Ee!z<3~60Hl-9=trj5gpGReix9)Gt zvZN@gqRl}N3=24&t6EfNS%x%CtFCyxE^<>IG6??iDNlLINSxjC&p*EzO`Lc4^!exA zX`I(V7$M6G6vi&eC0&asO3>@2>xa*V0rf z3Z51haxHQQcM$>*dWpcD{T1v2RRr53B|uAn4Hyfv!82BXB?B8YHt2>v5Nx2t>l-Cb3!p8Q1o z;&*S=t-6_ce>Z<8PoDEV1EOEjbsO4wjjZT5tck(I7kp<#1mJKP@r9_Krob=+55)GEfruiq;qkhOA0{&fB+#tX$|LEgs#P*BLB>%{^h^?^o_jV9;>O~ zs^avEN_2HyLqvVu=-9D6krZ5Cf9&IcDoXJ@uK9c zxCFGvv1t#z?scCpoA!g_L4~$y06HIgu~^UsM7u{sqA|WyPeBY@5dkBBAXp*I`I)Ud zDnw;LHk-XpOU&KT3}^u?1BUyYY*7&3%PRT+R0HsMc;d6Z{@62L@v7(VW!s+qs#iR( zSs8!WXITLzKpP;$yuu(J;#;7!fFdvjKnIM<9BMNA*Ejl}du)u$UusQYW?F2^FvrPa zaT8kTd-w5)nGV_8_N+N9?`hi>N)wDxOAXKv!cv30?Hs62xbQyex3)e!pfY)I_3nEf zxY;(8D`oDl>lOkEq)i9{_8fSzdR9tZuyPiW<*wJ_%M3&v_u2YCUp;*MCAXq2 zM~*+^IV*?n`yHjtTY?}XC}dfNrfHC61)K{@Ehi=WQx02HxaQ)KObXr@)Wu$@aNRzH zg^kV22Y}W9`}2GJ%n%5?FamJ%5?i-9v4dBhE2*zvvY3Nn+4sHW5YaDcAL|M%LobLI zx|D$KW&M;!;C^vMy}{SDfbUR^D)^>`+T8iTXW#X%Q+wX%o@@Q?-HNg>pMKjrKV}>E zs}uqt0l|4F#1%`d{HXwA`>=Wo-*^PF=<@2m^-_Q0_$y!e6JK-k6dk_`-i0MN4_*h!&nTa1cqU$$-%H^f1#25vjg zSTw}%c)RJUI2C>%%>37AlN}W@2$0zW0en;WAk38)e zzwJAIc@V#FtioUmV{x8W`D=S?ewl{z1?tk`UpDjk!{cZ5kv%7GRQ?$-m5DibYj9UR_bC}TxV2|0x296Xt zjs(0Lz&9OSZ}yh)#J{SM`hQ(HdSoz})d&<5SbgUJV*t}vBVAl&qTm%{lIgP1`ltUZ z;al(i!5{pGM-Cl&--9Pl-c+TlXZ-l9UV56sYib|fQy#lp@w`LV#6pl=-2m)CqyVEJ z5MVY11|xug)oKL&_?dV7mp8xdnnJTL#hx1D>+4Va>FUvYwym-&!80b4ZR9qC)*4$| zTQPA-pKn6fb+AT5Nd9TH-TwWxBlrHa*7^@Z=RReO+c-xgt4<$*!P6R*(I^)JoT75Z>VpAZ8^ThKMYKlH0s-~y)WB7j zBUk9ZI&W;t?(Mz)Un+Ik=t+`(q1m+_xhi3%bIi_eKlg?ka~rX$>D`;#+pjiN3BOya z4+=3DT7f0z%W>U67Xt)q;hQ-~7@0NDt$XB?Z+OG+9BfZ6jnJ)`#~*#?&pr0V|LiBe z<-a}g*ssyrs*<*b8XbZmg%AQn`tj<+cr?eox}X84fYX!tbB`Q3{wtrG?!4u`fAQ+K zOFjRHt>r&|=FLBIRipTsuY1*MH?=RE*7e`BO215(~)wr^dL5gz@30{a-fAzjHR72w|7rEmPDn zW@5^04%c~%$76I|Cw1M5)|w`h87T#5t%ZnC6ot&^b2XpONoy@~88_n6JKnFfedpnW zzyeP-V?1#ivpXzjsiWINM7N*YexKPq_rwPe-F5Ffy0-gX(gtXH zdxq5t*5+ur!E`#s;lqb9pU=VC-rG6%-DK^{p@@`O#qiJ&hwl0!mp4rfZ3vmh=EkN} z!y(P*HJYx&q19FFOsCP9?gE$@hYlTz6^KKSs;a1MuffqI>Y9qX?Hvm3*>#NWz8iiv z#rkxLqB3yC!YkfRC&mB(AOJ~3K~x2h1`|VpKua&JfPpvVe`X)V3|OF=*y@ zAW+}1W$$h>CwTWtMlP{5yHeN#0YCr@&{~6`u7JzenfiiaK=MmRY<6-QkO=`{5l%yk zN}SQS^lbw>tl&Ej2!oV@v^6Sgko)jX0Ixl$KI|8te*I7Xo7Go*|IgOMKhg~f;^{U_ zK7s^_;aSVYu)-o{0iwYK*wG5kb)Kjk9-hwT-)D3Ey?Lgv@}sXjwW`b`1JG8eo2_9r znAV{yTdRg=yV;6e9siAu+HE3NpsXU(GN|>)Ml0w70G$Du>I?h^WL1u~p2H}DG6j6o z0FA&F3YD>bMVoK`e_#9aTQ@Tfd!G41kav0SzjIM)9sp*u8HR%j^Z6WQHCPtC@ZMuM z8sqHQvna|6op+Sw1x@Euj0QtYr&AO~0i9XOjX^t~LF?;_)?9I`|Z5P=jyTGw$=m9n|Lg`y}(86p8A&#e#tL=7K7iGO(6~gQH?|`H(^nlFlCM=tKRJ)Tdk) zr=LcjEpM`tCxdsm6q1`JxCS@m@@h4TrttOLp!o53o2O~D$HO)oq(+|e;hMa&9zsKVeWLD04tkb7f0L718#hk)gG!X2UrBOLh!nS~fz2hmbCM;p_;m*;V@HT$O0 zwLv9|i?-!C%NMUMs7Fu;pKs zgYpc&6XgDnQ03=iZ~ zskgXVe1$zsgTetHqwd7Fs5%;^R#-xhj$xt^reJhrMC5}u50}+Zlp6Ktp+aOtcALFY z*bm3|`{=93IJKVGb%y~URJ}6|spRoRYfAaXl_i!0jYvPbZ?=u-7{mbXVRm~d| zi<9Pk0I|Dws@j(&`4EtUm^#6rEO!D2-g7=D4;Mwu-5Ld5zmUv00WXg&mH;A=T!|0}s#g**tF< zvfX8a%qE#=)cUkq#eV(9TX+BERrdkn*IAJ~m0&Egn;_ZiA;9b-q8cvGQMeDHehe)} zaFl5+ZMLI?@@^ub+i2r=u_udu&nJWx3uUDEU<-6HQ2`I`njT;ux(pCrV!zcAcYP}T zBhO8Gc$c0@KoOO;Jtxk5STlPDcuTMe8rY^Bd`+KJb+-SYcE7-eA%FWM`lG_-I#>6- z4;s50`cWf|h=vk!oQgrN-kzO=pqzpV8O>D#x!{@SQPACQK~!Mxm>c3bdd*KL7M_5g zEl{Ju-(*pbk>gq7W)i_poWUWyeFNVfk6>l6@ePl#m-Uj4*e3BDmK5&?IbA2ejHdal zR=--%i*Bql>ExYZT@vuXSR^09$Z8IkOrnPI3E8DxK)Uop?pq?7oQ7Q5i+|RR)kO^Z zKD{pQQ+wgo>>0UykW6}GxL@$StM@ttcNmSlnI4}p>lG4+?P`Rn3QlJU%W2-k@$nYu z&{HM#**G^gF0hpyvE041un*OUX$#B(ULkTLnK^Cx&juzRoR1`nGb$D0`eAfKh2w2> zoC1P3sm05@{_>S9@BUl!d`@cij-HlNIuWOS+*?vtS7*9BhY4SL5G|S;DV9gOmmz4F zC`4Zo!d|*6mzQ@J?{LpBvC)_3H}x(Tz^}D6t%jk`9&(+n*CcQ;gu$4CVVWq|Px5O7 z^?gYVh<5_1&3^uEfF=HVplu*Bs&7q^;p5ZfOb2W1lO^|tPsxohR7D{1m(47iSo6t@ zDQw9HC_i~fP)i1t*PmugvNG<4?NQyJ8YT&qQ=LMQl94#dCN9%zatE?_hEd7DTM)?N zn?>)Ux;QoKM`m9<)eI*45>VbU{ENd1k3kZbv|zUVgwzaG9-})`zhh1PYld7}O9r_E z4@$$#!%V1?JQ0s60nUsXynY96$HvM3OoM(PR8pX?h|w?qtQ2ZIxdPu)3S-r|4w7}BV$lKP zcNovy{Z%8R%*Tv9sPybi&XyPhdM=}Bz_yGx!1o?)=AA#?PT zc^ms`Cque^JE0DgdoAiI_?>#j6R#4x{{ntl33`3T?=^|*SehW*FGs6|xO zVe0CX@yqVX7kn$1L{*~XpmYyLEq%i@u?YhP?{ki_b%R};D-k{05w(&#xKD#!qggq{PgVFGe*b#TKF~4NJ>uwayHs>=oybP? zmnaUC5`7V&dFxFO1-4Hf+MbnU5ocr2;-TTc!`3{zRl{X2vC!!otI%IkRu**RS0^&r zDjU{gplJqK1hb`GKKa($u6U-+H}@O+LpEhDz+L>&NXbTH(o)8@5OQ6ApO#B{NH*!**>jw?q4+?6K2G@zP9dG#qkH4;H=PsH9m@oLc7yM{k8 z4g}`9`}OB|V#UH)gd z2rhUC-w-PWS%R(`IKaUmy2mgC>O|ruoeWLwuTMczY(Bm=U&oVQb*nj@ehF~7c~a-( zt5D_^Ti+#kME;d^yb33FrH@LPy3G<6L0N#8{YWQrSJ+*E2HB#hV?C#*H(G8P&xQs$ z?&Q?fZvU(gbAL8FKBlJLn(!6Eh<3S&QkipOl?-Sl4@E}z!L^)Kl*jqH|zLx%ncE!@GdI^(_Aqv5Z7Rrxc&cj-c-7FUmWgRY1IGEHe|cH5|f31AETQ z#AGEAC_)CY>>VX8$rxDVt{R2QU^z;D@-%WpO(lHC>%tjCs0?h?sFcYpnA*yUcW-$p z-!#B54skrY2BJE^t_(Afsc8mRMy8<=J zrG{*-1i{4H51+VXm5lc0_(+B69yi?nN_(ZMzVHr9t4v=M11d`obM+q2Ca>#4k7eo6 zpHV@YB8Chrm7yEf!P7EUB4j(I4VF@9-2^jhgw>2m_^7sC`U^SOa&9~`Hy=(r2d*xWn*mqUS`xGD{+dny53qjvLdNY?e=BkxD zM7u;*exp%^{OR0bRW7`MygVXeEb3UFxyVJNQ9e{};3Zv5#$(DZt<(2*^txB3sSp-S zNwe{)0#FAi3;A$)=Cm;Fh~XH4lY2z{b^ zw}+}?>qpkAoTazcB*dqvvLva0@m{k?zZ;CH6d8msNutu!j0+T1_uhO$4bE`0tzPgq zuNDSga65#bP3FuTSs-h}fVE2rECP{@!f21b5o-fdc2^h9KYjaq!qFXC^{shjg`>?* zOdZU1+?6Y|qYz8+Mg}X>V1317iYrMrTQUOA)_C+Pq-Kgef4 z9ied}H_ywTjw~PFon*Qxza^Bk^f5{hokx0w=%AQqW-4YZ)+3hZlI39ez|{<}wsa)h zb~`MxTE*yM^8j+t5ly+ZOkN#Y+LOM%Vg(VRnd2*0N((UkhS0BHbYNww_t+@mDbuU} zB)CHw`6kKI>TO+uu$^EVF|y@c%s`4Gy1BFm(+h0??wJugdnYFg5yXJU)kwb8VF%zq zj+tUwTR-|*=sM+L^>42~*&l3xjrOQ<{)T@96>q&F`f{;PuKfc6k<%H_Z&^}(Vuku7 zfFo()v}D>P!dRrm9d_0eHtzO6A60lkygp%#1DsGMSi34e6ihcokey!gt>E4Cxpn3i z<1zuEe{IlVSrV1Ebex#A`|?&{vPw|_C~`n@SB7Efgmg{#`mMc&xu!JXKK<3th`={> zYqIBz9%JI=Wbu`IhF||&zCyksmvnkum>w;AolF_)!%zTrZt#;*QAEtwN`nYjkle#v zB6kPZ`2&t(Jngsq7DvZZXs{pya*jaEXM`BNR*r}_jV=9(5m#qf=!gA7lYJhd`A1Mt zBd7_Cg^SFcvC7FU9BUu%e%-2-zRKgbZp=OAb7uUkO6o)XP~VCYdV+SG?Z}Jbd1(Yv zoREyRl}~rb%=#LEjx#%Mxla?$t}^-C+!`|bYLTEe4uZ6tGH07bk`+4Wz7A8KXmVB- zb#fW%MRHZ5j2=(5 z6>hsNY~RD%xNelCA@jY<4>bqOjiPUH>s`xdMcJs|{BXIo$GPuzmXYiZtMm_2X0e^W z|5ZLPnfl;}h*7Kg^Ro<(d^VZ1KAl7={rx_+koOOA2Eu=VsN3&+5`IwS*AplxtXyq+ zLIq-4&dVd%MC0H|{W%>4p~I8_3!SHs_28X+d$uO+3h92gm8x6!<(L!0oVBi9Vt~c8 zP1@T8BD)=Tc0V4pJ*Ui$A5XS_V@O-jthikWDjjfYO~F z3kg!8v5-`CPU^dEU~JsjXDKfz^g?x1(z#A;VuYh8T7)BB=RI%)HehPx_)F20-|1|a zuw&SWk(Z&}nX1!w6FZqC$2#Puedwew3!0&sn~i~z0UImYCD`)ao|eFpn4m)QPVpum zf?(DE4mb;)%Dka!%+}f=lJnP|nwpU(oz^OmJp|+mf4rCwWcYn?~Lfax; zT-lUfsuQUq2>PKiDY4*yWSHl@uBa!~U>j)NHg3ymD;Mr1(rOa!XhW{XATwTB{~hzg zuief5yp^%`z$U<7_@r{PgfdDex;bp~j3>~uD7D8rpxc@ao%(!E9-7CT;z`izh=MY6 zq0^BrnLNk9^0M+n$kmEf0_X@Yp9g zl&z(SzjPwMDT2y<_DRY^ZNwXjRKU5bTs-mR;qTU z#vSugQ){sy4|3{GTZzo7+o{DW1wXY>zJq$yPwi%aEH{pqI5 zkZwTnY@s}W(yY;!J&tU5sa7Zw+E*dcCh{Lp>`aF1ib|t}^44$FDBYW?_<25QP>eam z7x4W6H*C+hQwgyX$KCfCmA4c|Up{7)?OmiV?J-lhqISbfDy6~lN)xyZ;k*UFMqO0G zZU1)r=!beXgH@@uMVO;^Q@^%j$^H~{;{l4mk0wR`_?WObR=JvwKx74J7r#}!#oR;_ zSlC?aA(ps)-)AT^`=5kQFN&BIFAa%}|E}t9C{1L*?xrq9-Tr}Ey!k0tQDvVS@e$7_3e_6~$(xn8GzWaOH5h~2T`-je!pfRK&$O60u8Tt*=FpN76G1ix3k-SIUH z=Hu&*aJ8~0n;m8{kLM>J%BM9VqW*=xKZ0IW7-1+#pE9AMsMxG_N%NUnRXp#BQ#x8K zOjQi18+4K6zBwP%6%W>aX*SMqT^|8^(^$vZ;JNO}**z{>WH6aee=A%}<%>#TW6TNR zI^`;dk%eZ49QL&2k&{nzJq1=+)_Z4pQEG&AXJqJEds`j6BAFK;nU~Ea4&TUQ?(>Dz zcNoK43{jhRZE1CSl0M1@={{}1(?^=U2~xDF+VEHV8L&%OeG24?c~z-&z#?aRxs<N4l~e7%}5y~D+uKE5Co<5u$&pD+K}#?u36ahr%sK1 zJz#okB7qcKMbZ@FZDeMw%A)PXzGB~x$I7UVO&EK zFGZl{%u}}mKAObwHy3%&O;#4&%G;66L8RSwS`g60+;-x$^!9$9G<8db7N=&|aeZ_@ z2qWlQoOi}UZ%8^%tDXI% zRo9-~#2;O~_k4pnsntF?H;CL;UcZLEJSyWnBY95kMnCdcjS+^rfZj>AMO>yl^{g0C z3NTLGynl`P0sZ;Hq2Gg-j((%HhB^91vpDlBs=7Cj_aWM#UTD~$1R++*Z|4_oB96}S zriB8aWo9Zyre$C=h$6JAie)m;&pTE-e<%dvZ)9{sg zFZ3jL)=7#0t!=>n6-@;iji7@+e3rdGO!oZNKAbU5UYCqn%=xsn=8F=1yhV+`&DkGPo&s;TQl@LgWMI)_NJSE`XsCPWtW4ZM3 zHXUn1;qoi#;U^bjaSHrx4iq1>-K{Y{%(LC&yhf~ixt8iAvn2JTd9A?8aKQ)@3`^ZR z39<2WnK!ex9I0^%FiyS@o$asa{?=B%x9;th`kYwm@%P_TzC1WGbrKd9<3FK-e^Ku9 z%QuK{FJ8=6o+`V?>gi>*$1ISUV%g3=wI`!pz(arE)ok+ugjnNmR`aX}9W9-umYQfr z(5(f6RLB1ns~0noKZrbYd?cy>L`k0HfC6{HBMa0}WtL!?HVz;#(Tf_KdigFmdBL+A zu(&fvrwd)*c@y5DA`SeymP-3r5LKW|4D@ z;k^6LQpxQoegQs71-~`c0>=6~pQv~tLmu=NP*rL;pM>@xCW0OCUB;sw=nJS_ zn7w1_fA-pXJM2)Z%)Jmef;3b!=$?xy+f`Mlkz9B^V9il<6pz}Vjj&2l8!0t);Tj=>7cLl ztY{U72|EjXFDH8wur%R>mQt52>FF7eK>Yu6c1FW}72 zFEAHNZe>0kl(P#D8u%t8X7M~)b#R0uHq7yx_*vfA!F+(Y3rKJ@uJbPgi6XDx&m zmFv^GQUv45@o^I(cn@yPj;tyvXI z8FBnezF0*$0X|=V)FhM1mr;G+BcS$9l3L8m1cgj*+m2p9R#Yv#a! z5gKpf6-HL&gy65JzmOq<%|twv6o?8ZZRA&)F1mjH|15WSWngsC@SK*tg<66Ne2FP3 zi+37d0L=rBe^W~}b}i)R`!Spze8{am*kDHBK@fJlVQduIs?aQ? z(P#sLF3wQGfhv?N>7C1LPxQ8PaBr_R^eY$$a}dkQU{R(e+3ogZ5j`5zk}d`y%l{6* zHU_-_h3>Cu59P#D7zsvY;SIjAv(AW#55{!!2GOE$WDcw_6J z43iL#uz5W=eKk{%79_)y9*=|JP-fA%LvCJ(?L!^ftrRug1!~hMWEqc%(piUeL^!ej z?OkBfkWc$-i{Ll(3}>6yZxRL5k=>=)fixhm{>*qXkV~J6aoNYuwTF*cLd*|9-4vmR zj>lh2QK(WRzje+x2i+&E0EcDXEXZ{6c`4K^=3%Ij3JUO2dcVb`P&IVqA1CYsS-;1gP07^6k*$i>Vs4WX)MWlYJtXy*+9=0Wa2=S#lqFPYnX6mm+CEH?lQRI*Hlk z4xBYMl<+f`JK^Cd!1JLo?fc6+Cg+6+ckJvqLp3{?58NlqQ2_fHB!ks7Xdw+%4#u8m zS%zN#$MhS8zmqQ^;?l5Jn-X1CJr$OQ_kQPyrE??T59!?H8tg>v*nvOX=Xa^n-&jRW`N{j}?i`gj?s})xvUn=u|&eEW4`zam>eqiuYcLi47d=~s`Zxe4C2TnGgOOs+UIqGT(Cr+Y8TgVFH$;8v~ zb$O!G0=T9VgWuxC?Y0t40ADFU9}k>qc&>^gdCb4}ERWl#I;?mHVBFikqi1p+tmZ|7_NnRKXW<{9 literal 0 HcmV?d00001 diff --git a/tamingllms/markdown/intro.md b/tamingllms/markdown/intro.md index 6632d69..29886a3 100644 --- a/tamingllms/markdown/intro.md +++ b/tamingllms/markdown/intro.md @@ -1,8 +1,8 @@ # Introduction -In recent years, Large Language Models (LLMs) have emerged as a transformative force in technology, promising to revolutionize how we build products and interact with computers. From ChatGPT to GitHub Copilot, these systems have captured the public imagination and sparked a gold rush of AI-powered applications. However, beneath the surface of this technological revolution lies a complex landscape of challenges that practitioners must navigate. +In recent years, Large Language Models (LLMs) have emerged as a transformative force in technology, promising to revolutionize how we build products and interact with computers. From ChatGPT to GitHub Copilot, Claude Artifacts, cursor.com, replit, and others, these systems have captured the public imagination and sparked a gold rush of AI-powered applications. However, beneath the surface of this technological revolution lies a complex landscape of challenges that practitioners must navigate. -As we'll explore in this book, the significant engineering effort required to manage these challenges - from handling non-deterministic outputs to preventing hallucinations - raises important questions about the true productivity gains promised by LLM technology. While the potential remains compelling, the hidden costs and complexities of building reliable LLM-powered systems should not be neglected and instead may force us to reconsider our overly-optimistic assumptions about their transformative impact. +As we'll explore in this book, the engineering effort required to manage these challenges - from handling non-deterministic outputs to preventing hallucinations - cannot be overstated. While the potential of LLM technology remains compelling, understanding and addressing the hidden costs and complexities of building reliable LLM-powered systems will enable us to fully harness their transformative impact. ## Core Challenges We'll Address While the capabilities of LLMs are indeed remarkable, the prevailing narrative often glosses over fundamental problems that engineers, product managers, and organizations face when building real-world applications. This book aims to bridge that gap, offering a practical, clear-eyed examination of the pitfalls and challenges in working with LLMs. @@ -11,7 +11,7 @@ Throughout this book, we'll tackle the following (non-exhaustive) list of critic 1. **Non-deterministic Behavior**: Unlike traditional software systems, LLMs can produce different outputs for identical inputs, making testing and reliability assurance particularly challenging. -2. **Structural Reliability**: LLMs struggle to maintain consistent output formats, complicating their integration into larger systems and making error handling more complex. +2. **Structural (un)Reliability**: LLMs struggle to maintain consistent output formats, complicating their integration into larger systems and making error handling more complex. 3. **Hallucination Management**: These models can generate plausible-sounding but entirely fabricated information, creating significant risks for production applications. @@ -19,7 +19,16 @@ Throughout this book, we'll tackle the following (non-exhaustive) list of critic 5. **Testing Complexity**: Traditional testing methodologies break down when dealing with non-deterministic systems, requiring new approaches. -6. **Integration Challenges**: Incorporating LLMs into existing software architectures presents unique architectural and operational challenges. +## A Note on Perspective + +While this book takes a critical look at LLM limitations, our goal is not to discourage their use but to enable more robust and reliable implementations. By understanding these challenges upfront, you'll be better equipped to build systems that leverage LLMs effectively while avoiding common pitfalls. + +The current discourse around LLMs tends toward extremes—either uncritical enthusiasm or wholesale dismissal. This book takes a different approach: + +- **Practical Implementation Focus**: Rather than theoretical capabilities, we examine real-world challenges and their solutions. +- **Code-First Learning**: Every concept is illustrated with executable Python examples, enabling immediate practical application. +- **Critical Analysis**: We provide a balanced examination of both capabilities and limitations, helping readers make informed decisions about LLM integration. + ## A Practical Approach @@ -36,7 +45,7 @@ This book takes a hands-on approach to these challenges, providing: This book is designed for: - Software Engineers building LLM-powered applications -- Product Managers overseeing AI initiatives +- Product Managers leading AI initiatives - Technical Leaders making architectural decisions - Anyone seeking to understand the practical challenges of working with LLMs @@ -45,29 +54,8 @@ This book is designed for: To make the most of this book, you should have: - Basic Python programming experience -- Access to LLM APIs (OpenAI, Anthropic, or similar) -- A desire to build reliable, production-grade AI systems - -## How to Use This Book - -Each chapter focuses on a specific challenge, following this structure: - -1. Problem explanation and real-world impact -2. Technical deep-dive with code examples -3. Practical solutions and implementation patterns -4. Testing strategies and best practices -5. Cost and performance considerations -6. Conclusion - -## A Note on Perspective - -While this book takes a critical look at LLM limitations, our goal is not to discourage their use but to enable more robust and reliable implementations. By understanding these challenges upfront, you'll be better equipped to build systems that leverage LLMs effectively while avoiding common pitfalls. - -The current discourse around LLMs tends toward extremes—either uncritical enthusiasm or wholesale dismissal. This book takes a different approach: - -- **Practical Implementation Focus**: Rather than theoretical capabilities, we examine real-world challenges and their solutions. -- **Code-First Learning**: Every concept is illustrated with executable Python examples, enabling immediate practical application. -- **Critical Analysis**: We provide a balanced examination of both capabilities and limitations, helping readers make informed decisions about LLM integration. +- Access to and basic knowledge of LLM APIs (OpenAI, Anthropic, or similar) +- A desire to build reliable, production-grade LLM-powered products ## Setting Up Your Environment @@ -93,8 +81,8 @@ export OPENAI_API_KEY=your-openai-key ### 3. Code Repository Clone the book's companion repository: ```bash -git clone https://github.com/yourusername/taming-llms.git -cd taming-llms +git clone https://github.com/souzatharsis/tamingllms.git +cd tamingllms ``` ### Troubleshooting Common Issues diff --git a/tamingllms/markdown/toc.md b/tamingllms/markdown/toc.md index adf6787..cca120b 100644 --- a/tamingllms/markdown/toc.md +++ b/tamingllms/markdown/toc.md @@ -1,3 +1,5 @@ +# Table of Contents + ## Chapter 1: Introduction - The Hidden Challenges of LLMs - Why This Book Matters