diff --git a/prompt-engineering/README.md b/prompt-engineering/README.md index 7e5ed753b9..11b4820de4 100644 --- a/prompt-engineering/README.md +++ b/prompt-engineering/README.md @@ -20,6 +20,12 @@ Create and activate a [virtual environment](https://realpython.com/python-virtua (venv) $ python -m pip install openai ``` +Or, to make sure that you're using the same versions as shown in the tutorial, you can install from `requirements.txt` instead: + +```bash +(venv) $ python -m pip install -r requirements.txt +``` + ## Usage Read support chat conversations from a file, sanitize the text, classify by sentiment, and format the output as JSON: @@ -399,6 +405,16 @@ instruction_prompt = """ Classify the sentiment of each conversation in >>>>>CONTENT<<<<< with "🔥" for negative and "✅" for positive. +Follow these steps when classifying the conversations: +1. Does the customer use swear words or 😤? +2. Does the customer seem aggravated or angry? + +If you answer "Yes" to one of the above questions, +then classify the conversation as negative with "🔥". +Otherwise classify the conversation as positive with "✅". + +Let's think step by step + #### START EXAMPLES ------ Example Inputs ------ @@ -406,13 +422,17 @@ with "🔥" for negative and "✅" for positive. [Customer] 2023-07-24 : I CAN'T CONNECT TO MY 😤 ACCOUNT [Agent] 2023-07-24 : Are you sure it's not your caps lock? [Customer] 2023-07-24 : 😤! You're right! -The customer uses the 😤 emoji and seems aggravated, so the sentiment is negative. 🔥 + - Does the customer use swear words or 😤? Yes + - Does the customer seem aggravated or angry? Yes + - Sentiment: 🔥 [Agent] 2023-06-15 : Hello! How can I assist you today? [Customer] 2023-06-15 : I can't seem to find the download link for my purchased software. -[Agent] 2023-06-15 : No problem, ********. Let me find that for you. Can you please provide your order number? -[Customer] 2023-06-15 : It's ********. Thanks for helping me out! -The customer does not use any swear words or 😤 emoji and does not seem aggravated or angry, so the sentiment is positive. ✅ +[Agent] 2023-06-15 : No problem, ****. Let me find that for you. Can you please provide your order number? +[Customer] 2023-06-15 : It's ****. Thanks for helping me out! + - Does the customer use swear words or 😤? No + - Does the customer seem aggravated or angry? No + - Sentiment: ✅ ------ Example Outputs ------ 🔥 @@ -424,8 +444,8 @@ The customer does not use any swear words or 😤 emoji and does not seem aggrav ✅ [Agent] 2023-06-15 : Hello! How can I assist you today? [Customer] 2023-06-15 : I can't seem to find the download link for my purchased software. -[Agent] 2023-06-15 : No problem, ********. Let me find that for you. Can you please provide your order number? -[Customer] 2023-06-15 : It's ********. Thanks for helping me out! +[Agent] 2023-06-15 : No problem, ****. Let me find that for you. Can you please provide your order number? +[Customer] 2023-06-15 : It's ****. Thanks for helping me out! #### END EXAMPLES """ @@ -444,21 +464,34 @@ Classify the sentiment of each conversation in >>>>>CONTENT<<<<< as "negative" and "positive". Return the output as valid JSON. -#### START EXAMPLES +Follow these steps when classifying the conversations: +1. Does the customer use swear words or 😤? +2. Does the customer seem aggravated or angry? ------- Example Input ------ +If you answer "Yes" to one of the above questions, +then classify the conversation as "negative". +Otherwise classify the conversation as "positive". +Let's think step by step + +#### START EXAMPLES + +------ Example Inputs ------ [Agent] 2023-07-24 : What can I help you with? [Customer] 2023-07-24 : I CAN'T CONNECT TO MY 😤 ACCOUNT [Agent] 2023-07-24 : Are you sure it's not your caps lock? [Customer] 2023-07-24 : 😤! You're right! -The customer uses the 😤 emoji and seems aggravated, so the sentiment is negative. + - Does the customer use swear words or 😤? Yes + - Does the customer seem aggravated or angry? Yes + - Sentiment: "negative" [Agent] 2023-06-15 : Hello! How can I assist you today? [Customer] 2023-06-15 : I can't seem to find the download link for my purchased software. -[Agent] 2023-06-15 : No problem, ********. Let me find that for you. Can you please provide your order number? -[Customer] 2023-06-15 : It's ********. Thanks for helping me out! -The customer does not use any swear words or 😤 emoji and does not seem aggravated or angry, so the sentiment is positive. +[Agent] 2023-06-15 : No problem, ****. Let me find that for you. Can you please provide your order number? +[Customer] 2023-06-15 : It's ****. Thanks for helping me out! + - Does the customer use swear words or 😤? No + - Does the customer seem aggravated or angry? No + - Sentiment: "positive" ------ Example Output ------ @@ -480,8 +513,8 @@ The customer does not use any swear words or 😤 emoji and does not seem aggrav "conversation": [ "A: Hello! How can I assist you today?", "C: I can't seem to find the download link for my purchased software.", - "A: No problem, ********. Let me find that for you. Can you please provide your order number?", - "C: It's ********. Thanks for helping me out!" + "A: No problem, ****. Let me find that for you. Can you please provide your order number?", + "C: It's ****. Thanks for helping me out!" ] } ] @@ -504,32 +537,33 @@ Classify the sentiment of each conversation in >>>>>CONTENT<<<<< as "negative" and "positive". Return the output as valid JSON. """ -role_prompt = """You are a thoroughly trained machine learning model -that is an expert at sentiment classification. +role_prompt = """You are a thoroughly trained machine learning +model that is an expert at sentiment classification. You diligently complete tasks as instructed. You never make up any information that isn't there.""" positive_example = """ [Agent] 2023-06-15 : Hello! How can I assist you today? [Customer] 2023-06-15 : I can't seem to find the download link for my purchased software. -[Agent] 2023-06-15 : No problem, ********. Let me find that for you. Can you please provide your order number? -[Customer] 2023-06-15 : It's ********. Thanks for helping me out! +[Agent] 2023-06-15 : No problem, ****. Let me find that for you. Can you please provide your order number? +[Customer] 2023-06-15 : It's ****. Thanks for helping me out! +""" +positive_reasoning = """ +- Does the customer use swear words or 😤? No +- Does the customer seem aggravated or angry? No +- Sentiment: "positive" """ -positive_reasoning = """The customer does not use any swear words or 😤 emoji -and does not seem aggravated or angry, so the sentiment is positive.""" positive_output = """ -{ - "positive": [ - { - "date": "2023-06-15", - "conversation": [ - "A: Hello! How can I assist you today?", - "C: I can't seem to find the download link for my purchased software.", - "A: No problem, ********. Let me find that for you. Can you please provide your order number?", - "C: It's ********. Thanks for helping me out!" - ] - } - ] -} +"positive": [ + { + "date": "2023-06-15", + "conversation": [ + "A: Hello! How can I assist you today?", + "C: I can't seem to find the download link for my purchased software.", + "A: No problem, ****. Let me find that for you. Can you please provide your order number?", + "C: It's ****. Thanks for helping me out!" + ] + } +] """ negative_example = """ [Agent] 2023-07-24 : What can I help you with? @@ -537,21 +571,22 @@ negative_example = """ [Agent] 2023-07-24 : Are you sure it's not your caps lock? [Customer] 2023-07-24 : 😤! You're right! """ -negative_reasoning = """The customer uses the 😤 emoji and seems aggravated, -so the sentiment is negative.""" +negative_reasoning = """ +- Does the customer use swear words or 😤? Yes +- Does the customer seem aggravated or angry? Yes +- Sentiment: "negative" +""" negative_output = """ -{ - "negative": [ - { - "date": "2023-07-24", - "conversation": [ - "A: What can I help you with?", - "C: I CAN'T CONNECT TO MY 😤 ACCOUNT", - "A: Are you sure it's not your caps lock?", - "C: 😤! You're right!" - ] - } - ] -} +"negative": [ + { + "date": "2023-07-24", + "conversation": [ + "A: What can I help you with?", + "C: I CAN'T CONNECT TO MY 😤 ACCOUNT", + "A: Are you sure it's not your caps lock?", + "C: 😤! You're right!" + ] + } +] """ ``` diff --git a/prompt-engineering/app.py b/prompt-engineering/app.py index 35cba5a0e4..1405739ac9 100644 --- a/prompt-engineering/app.py +++ b/prompt-engineering/app.py @@ -3,39 +3,17 @@ import tomllib from pathlib import Path -import openai +from openai import OpenAI -# Authenticate -openai.api_key = os.getenv("OPENAI_API_KEY") - - -class Settings(dict): - """Handle loading and accessing application settings from file.""" +__all__ = ["get_chat_completion"] - @classmethod - def load(cls, path) -> "Settings": - """Load TOML settings file and pass it to class constuctor.""" - with path.open("rb") as file: - return cls(tomllib.load(file)) +# Authenticate +client = OpenAI(api_key=os.getenv("OPENAI_API_KEY")) - def __init__(self, *args, **kwargs) -> None: - """Add general settings and prompts as instance attributes.""" - super().__init__(*args, **kwargs) - # Settings - self.chat_models = self["general"]["chat_models"] - self.model = self["general"]["model"] - self.max_tokens = self["general"]["max_tokens"] - self.temperature = self["general"]["temperature"] - self.model_supports_chat_completions = self.model in self.chat_models - # Prompts - self.instruction_prompt = self["prompts"]["instruction_prompt"] - self.role_prompt = self["prompts"]["role_prompt"] - self.positive_example = self["prompts"]["positive_example"] - self.positive_reasoning = self["prompts"]["positive_reasoning"] - self.positive_output = self["prompts"]["positive_output"] - self.negative_example = self["prompts"]["negative_example"] - self.negative_reasoning = self["prompts"]["negative_reasoning"] - self.negative_output = self["prompts"]["negative_output"] +# Load settings file +settings_path = Path("settings.toml") +with settings_path.open("rb") as settings_file: + SETTINGS = tomllib.load(settings_file) def parse_args() -> argparse.Namespace: @@ -47,52 +25,46 @@ def parse_args() -> argparse.Namespace: def main(args: argparse.Namespace) -> None: file_content = args.file_path.read_text("utf-8") - settings = Settings.load(Path("settings.toml")) - if settings.model_supports_chat_completions: - print(get_chat_completion(file_content, settings)) - else: - print(get_completion(file_content, settings)) - + print(get_chat_completion(file_content)) -def get_completion(content: str, settings: Settings) -> str: - """Send a request to the /completions endpoint.""" - response = openai.Completion.create( - model=settings.model, - prompt=assemble_prompt(content, settings), - max_tokens=settings.max_tokens, - temperature=settings.temperature, - ) - return response["choices"][0]["text"] - -def get_chat_completion(content: str, settings: Settings) -> str: +def get_chat_completion(content: str) -> str: """Send a request to the /chat/completions endpoint.""" - response = openai.ChatCompletion.create( - model=settings.model, - messages=assemble_chat_messages(content, settings), - temperature=settings.temperature, + response = client.chat.completions.create( + model=SETTINGS["general"]["model"], + messages=_assemble_chat_messages(content), + temperature=SETTINGS["general"]["temperature"], + seed=12345, # Doesn't do anything for older models ) - return response["choices"][0]["message"]["content"] - - -def assemble_prompt(content: str, settings: Settings) -> str: - """Combine all text input into a single prompt.""" - return f">>>>>\n{content}\n<<<<<\n\n" + settings.instruction_prompt - - -def assemble_chat_messages(content: str, settings: Settings) -> list[dict]: - """Combine all messages into a well-formatted dictionary.""" - return [ - {"role": "system", "content": settings.role_prompt}, - {"role": "user", "content": settings.negative_example}, - {"role": "system", "content": settings.negative_reasoning}, - {"role": "assistant", "content": settings.negative_output}, - {"role": "user", "content": settings.positive_example}, - {"role": "system", "content": settings.positive_reasoning}, - {"role": "assistant", "content": settings.positive_output}, + return response.choices[0].message.content + + +def _assemble_chat_messages(content: str) -> list[dict]: + """Combine all messages into a well-formatted list of dicts.""" + messages = [ + {"role": "system", "content": SETTINGS["prompts"]["role_prompt"]}, + {"role": "user", "content": SETTINGS["prompts"]["negative_example"]}, + { + "role": "system", + "content": SETTINGS["prompts"]["negative_reasoning"], + }, + { + "role": "assistant", + "content": SETTINGS["prompts"]["negative_output"], + }, + {"role": "user", "content": SETTINGS["prompts"]["positive_example"]}, + { + "role": "system", + "content": SETTINGS["prompts"]["positive_reasoning"], + }, + { + "role": "assistant", + "content": SETTINGS["prompts"]["positive_output"], + }, {"role": "user", "content": f">>>>>\n{content}\n<<<<<"}, - {"role": "user", "content": settings.instruction_prompt}, + {"role": "user", "content": SETTINGS["prompts"]["instruction_prompt"]}, ] + return messages if __name__ == "__main__": diff --git a/prompt-engineering/requirements.txt b/prompt-engineering/requirements.txt index 116fc72241..ae127ad5d9 100644 --- a/prompt-engineering/requirements.txt +++ b/prompt-engineering/requirements.txt @@ -1,15 +1,14 @@ -aiohttp==3.8.4 -aiosignal==1.3.1 -async-timeout==4.0.2 -attrs==23.1.0 -certifi==2023.5.7 -charset-normalizer==3.1.0 -frozenlist==1.3.3 -idna==3.4 -multidict==6.0.4 -openai==0.27.8 -python-dotenv==1.0.0 -requests==2.31.0 -tqdm==4.65.0 -urllib3==2.0.3 -yarl==1.9.2 +annotated-types==0.6.0 +anyio==4.3.0 +certifi==2024.2.2 +distro==1.9.0 +h11==0.14.0 +httpcore==1.0.4 +httpx==0.27.0 +idna==3.6 +openai==1.13.3 +pydantic==2.6.3 +pydantic-core==2.16.3 +sniffio==1.3.1 +tqdm==4.66.2 +typing-extensions==4.10.0 \ No newline at end of file diff --git a/prompt-engineering/sanitized-testing-chats.txt b/prompt-engineering/sanitized-testing-chats.txt index 6da3659736..174c3c41c2 100644 --- a/prompt-engineering/sanitized-testing-chats.txt +++ b/prompt-engineering/sanitized-testing-chats.txt @@ -10,7 +10,7 @@ [Agent] 2023-08-13: Good morning! How may I assist you? [Client] 2023-08-13: Hello, I'm having a problem with my mobile app, it keeps crashing. -[Agent] 2023-08-13: I'm sorry to hear that, ********. Could you tell me what device you're using? +[Agent] 2023-08-13: I'm sorry to hear that, ********. Could you tell me what device you're using? [Client] 2023-08-13: I have an iPhone 11. [Agent] 2023-08-30: Good evening! How may I assist you today? diff --git a/prompt-engineering/settings-final.toml b/prompt-engineering/settings-final.toml index 8c8368898d..b2bdf0b611 100644 --- a/prompt-engineering/settings-final.toml +++ b/prompt-engineering/settings-final.toml @@ -1,38 +1,41 @@ [general] -chat_models = ["gpt-3.5", "gpt-3.5-turbo", "gpt-4"] +chat_models = ["gpt-3.5-turbo", "gpt-4"] model = "gpt-4" -max_tokens = 100 temperature = 0 [prompts] instruction_prompt = """ -Classify the sentiment of each conversation in >>>>>CONTENT<<<<< as "negative" and "positive". +Classify the sentiment of each conversation in >>>>>CONTENT<<<<< +as "negative" and "positive". Return the output as valid JSON. """ -role_prompt = """You are a thoroughly trained machine learning model that is an expert at sentiment classification. -You diligently complete tasks as instructed. You never make up any information that isn't there.""" +role_prompt = """You are a thoroughly trained machine learning +model that is an expert at sentiment classification. +You diligently complete tasks as instructed. +You never make up any information that isn't there.""" positive_example = """ [Agent] 2023-06-15 : Hello! How can I assist you today? [Customer] 2023-06-15 : I can't seem to find the download link for my purchased software. -[Agent] 2023-06-15 : No problem, ********. Let me find that for you. Can you please provide your order number? -[Customer] 2023-06-15 : It's ********. Thanks for helping me out! +[Agent] 2023-06-15 : No problem, ****. Let me find that for you. Can you please provide your order number? +[Customer] 2023-06-15 : It's ****. Thanks for helping me out! +""" +positive_reasoning = """ +- Does the customer use swear words or 😤? No +- Does the customer seem aggravated or angry? No +- Sentiment: "positive" """ -positive_reasoning = """The customer does not use any swear words or 😤 emoji -and does not seem aggravated or angry, so the sentiment is positive.""" positive_output = """ -{ - "positive": [ - { - "date": "2023-06-15", - "conversation": [ - "A: Hello! How can I assist you today?", - "C: I can't seem to find the download link for my purchased software.", - "A: No problem, ********. Let me find that for you. Can you please provide your order number?", - "C: It's ********. Thanks for helping me out!" - ] - } - ] -} +"positive": [ + { + "date": "2023-06-15", + "conversation": [ + "A: Hello! How can I assist you today?", + "C: I can't seem to find the download link for my purchased software.", + "A: No problem, ****. Let me find that for you. Can you please provide your order number?", + "C: It's ****. Thanks for helping me out!" + ] + } +] """ negative_example = """ [Agent] 2023-07-24 : What can I help you with? @@ -40,20 +43,21 @@ negative_example = """ [Agent] 2023-07-24 : Are you sure it's not your caps lock? [Customer] 2023-07-24 : 😤! You're right! """ -negative_reasoning = """The customer uses the 😤 emoji and seems aggravated, -so the sentiment is negative.""" -negative_output = """ -{ - "negative": [ - { - "date": "2023-07-24", - "conversation": [ - "A: What can I help you with?", - "C: I CAN'T CONNECT TO MY 😤 ACCOUNT", - "A: Are you sure it's not your caps lock?", - "C: 😤! You're right!" - ] - } - ] -} +negative_reasoning = """ +- Does the customer use swear words or 😤? Yes +- Does the customer seem aggravated or angry? Yes +- Sentiment: "negative" """ +negative_output = """ +"negative": [ + { + "date": "2023-07-24", + "conversation": [ + "A: What can I help you with?", + "C: I CAN'T CONNECT TO MY 😤 ACCOUNT", + "A: Are you sure it's not your caps lock?", + "C: 😤! You're right!" + ] + } +] +""" \ No newline at end of file diff --git a/prompt-engineering/settings.toml b/prompt-engineering/settings.toml index aedae77ffe..554f38a528 100644 --- a/prompt-engineering/settings.toml +++ b/prompt-engineering/settings.toml @@ -1,7 +1,6 @@ [general] -chat_models = ["gpt-3.5-turbo", "gpt-3.5-turbo-16k", "gpt-4"] -model = "text-davinci-003" -max_tokens = 2100 +chat_models = ["gpt-3.5-turbo", "gpt-4"] +model = "gpt-3.5-turbo" temperature = 0 [prompts] diff --git a/prompt-engineering/testing-chats.txt b/prompt-engineering/testing-chats.txt index 04233b0d8e..d54c31c248 100644 --- a/prompt-engineering/testing-chats.txt +++ b/prompt-engineering/testing-chats.txt @@ -10,7 +10,7 @@ [support_luke] 2023-08-13T11:34:02+00:00: Good morning! How may I assist you? [anna_s] 2023-08-13T11:34:30+00:00: Hello, I'm having a problem with my mobile app, it keeps crashing. -[support_luke] 2023-08-13T11:34:58+00:00: I'm sorry to hear that, Anna. Could you tell me what device you're using? +[support_luke] 2023-08-13T11:34:58+00:00: I'm sorry to hear that, Anna. Could you tell me what device you're using? [anna_s] 2023-08-13T11:35:22+00:00: I have an iPhone 11. [support_lisa] 2023-08-30T20:38:00+00:00: Good evening! How may I assist you today?