Skip to content

Commit

Permalink
Handle ignore known errors, disable followup
Browse files Browse the repository at this point in the history
  • Loading branch information
@SketchingDev
SketchingDev committed Nov 9, 2023
1 parent df18856 commit 2ddf250
Show file tree
Hide file tree
Showing 8 changed files with 91 additions and 77 deletions.
20 changes: 10 additions & 10 deletions examples/cli-exploratory-tests/example.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,13 @@ scenarios:
terminatingPhrases:
pass: ["PASS"]
fail: ["FAIL"]
followUp:
prompt: |
Inspect the transcript below for spelling or grammatical errors and list them. Ignore anything
said by the speaker called 'AI'.
Transcript:
%TRANSCRIPT%
terminatingPhrases:
pass: ["PASS"]
fail: ["FAIL"]
# followUp:
# prompt: |
# Inspect the transcript below for spelling or grammatical errors and list them. Ignore anything
# said by the speaker called 'AI'.
#
# Transcript:
# %TRANSCRIPT%
# terminatingPhrases:
# pass: ["PASS"]
# fail: ["FAIL"]
10 changes: 10 additions & 0 deletions packages/genesys-web-messaging-tester-cli/src/commands/CommandExpectedlyFailedError.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
/**
* This error is thrown when you want to fail a command, but not have an error
* message or stack trace printed out by the entrypoint code.
*/
export class CommandExpectedlyFailedError extends Error {
constructor() {
super();
Object.setPrototypeOf(this, CommandExpectedlyFailedError.prototype);
}
}
6 changes: 0 additions & 6 deletions packages/genesys-web-messaging-tester-cli/src/commands/exploratoryTest/AxiosError.ts
View file

This file was deleted.

98 changes: 50 additions & 48 deletions ...sys-web-messaging-tester-cli/src/commands/exploratoryTest/createExploratoryTestCommand.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@ import { shouldEndConversation, ShouldEndConversationResult } from './prompt/sho
import { readableFileValidator } from '../../fileSystem/readableFileValidator';
import { createYamlFileReader } from '../../fileSystem/yamlFileReader';
import { validatePromptScript } from './testScript/validatePromptScript';
import { substituteTemplatePlaceholders } from './prompt/substituteTemplatePlaceholders';
import { containsTerminatingPhrases } from './prompt/containsTerminatingPhrases';
import { CommandExpectedlyFailedError } from '../CommandExpectedlyFailedError';

/**
* This value can be between 0 and 1 and controls the randomness of ChatGPT's completions.
Expand Down Expand Up @@ -89,15 +88,15 @@ export function createExploratoryTestCommand({
outputConfig.writeErr(
ui.validatingOpenAiEnvValidationFailed(sessionValidationResult.error),
);
throw new Error();
throw new CommandExpectedlyFailedError();
}

const openAiEnvValidationResult = validateOpenAiEnvVariables(process.env);
if (!openAiEnvValidationResult.openAikey) {
outputConfig.writeErr(
ui.validatingOpenAiEnvValidationFailed(openAiEnvValidationResult.error),
);
throw new Error();
throw new CommandExpectedlyFailedError();
}

// 1. Read YAML file
Expand All @@ -106,15 +105,15 @@ export function createExploratoryTestCommand({
testScriptFileContents = yamlFileReader(testScriptPath);
} catch (error) {
outputConfig.writeErr(ui.errorReadingTestScriptFile(error as Error));
throw new Error();
throw new CommandExpectedlyFailedError();
}

// 2. Validate Test Script
const testScriptValidationResults = validatePromptScript(testScriptFileContents);
// TODO Update scenario validation object to match
if (testScriptValidationResults.error) {
outputConfig.writeErr(ui.validatingPromptScriptFailed(testScriptValidationResults.error));
throw new Error();
throw new CommandExpectedlyFailedError();
}

// 3. Merge session config from args and Test Script - args take priority
Expand All @@ -131,13 +130,13 @@ export function createExploratoryTestCommand({
outputConfig.writeErr(
ui.validatingSessionConfigFailed(sessionConfigValidationResults.error),
);
throw new Error();
throw new CommandExpectedlyFailedError();
}

const totalScenarios = Object.keys(validPromptScript?.scenarios).length;
if (totalScenarios > 1) {
outputConfig.writeErr(ui.onlyOnePromptSupported(totalScenarios));
throw new Error();
throw new CommandExpectedlyFailedError();
}

const scenario = Object.entries(validPromptScript?.scenarios)[0][1];
Expand Down Expand Up @@ -203,50 +202,53 @@ export function createExploratoryTestCommand({
session.close();

if (endConversation.reason.type === 'fail') {
throw new Error();
throw new CommandExpectedlyFailedError();
}

if (scenario.followUp) {
const content = substituteTemplatePlaceholders(scenario.followUp.prompt, transcript);
const { choices } = await openai.chat.completions.create({
model: chatGptModel,
n: 1, // Number of choices
temperature,
messages: [
{
role: 'system',
content,
},
],
});

if (choices[0].message?.content) {
const result = containsTerminatingPhrases(choices[0].message.content, {
fail: scenario.setup.terminatingPhrases.fail,
pass: scenario.setup.terminatingPhrases.pass,
});

outputConfig.writeOut(ui.followUpDetails(choices[0].message.content));
if (result.phraseFound) {
outputConfig.writeOut(ui.followUpResult(result));
if (result.phraseIndicates === 'fail') {
throw new Error();
}
}
}

// endConversation = shouldEndConversation(
// messages,
// scenario.setup.terminatingPhrases.fail,
// scenario.setup.terminatingPhrases.pass,
// );
// if (choices[0].message?.content) {
// messages.push({ role: 'assistant', content: choices[0].message.content });
// await convo.sendText(choices[0].message.content);
// } else {
// messages.push({ role: 'assistant', content: '' });
// }
outputConfig.writeOut(ui.followUpDetailsUnderDevelopment());
}
// if (scenario.followUp) {
// const content = substituteTemplatePlaceholders(scenario.followUp.prompt, transcript);
// const { choices } = await openai.chat.completions.create({
// model: chatGptModel,
// n: 1, // Number of choices
// temperature,
// messages: [
// {
// role: 'system',
// content,
// },
// ],
// });
//
// if (choices[0].message?.content) {
// const result = containsTerminatingPhrases(choices[0].message.content, {
// fail: scenario.setup.terminatingPhrases.fail,
// pass: scenario.setup.terminatingPhrases.pass,
// });
//
// outputConfig.writeOut(ui.followUpDetails(choices[0].message.content));
// if (result.phraseFound) {
// outputConfig.writeOut(ui.followUpResult(result));
// if (result.phraseIndicates === 'fail') {
// throw new CommandExpectedlyFailedError();
// }
// }
// }
//
// // endConversation = shouldEndConversation(
// // messages,
// // scenario.setup.terminatingPhrases.fail,
// // scenario.setup.terminatingPhrases.pass,
// // );
// // if (choices[0].message?.content) {
// // messages.push({ role: 'assistant', content: choices[0].message.content });
// // await convo.sendText(choices[0].message.content);
// // } else {
// // messages.push({ role: 'assistant', content: '' });
// // }
// }
},
);
}
6 changes: 3 additions & 3 deletions ...sys-web-messaging-tester-cli/src/commands/exploratoryTest/prompt/shouldEndConversation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export function shouldEndConversation(

const lastMessage = messages.slice(-1);
if (lastMessage.length === 1 && lastMessage[0].content === '') {
const who = lastMessage[0].role === 'assistant' ? 'ChatGPT' : 'ChatBot';
const who = lastMessage[0].role === 'assistant' ? 'ChatGPT' : 'Chatbot';
return {
hasEnded: true,
reason: { type: 'fail', description: `${who} didn't have a response` },
Expand All @@ -50,7 +50,7 @@ export function shouldEndConversation(
hasEnded: true,
reason: {
type: phraseResult.phraseIndicates,
description: `Terminating phrase found in response: ${lastChatGptMsg[0].content}`,
description: `Terminating phrase found in response: '${lastChatGptMsg[0].content}'`,
},
};
}
Expand Down Expand Up @@ -79,7 +79,7 @@ export function shouldEndConversation(

reason: {
type: 'fail',
description: 'ChatBot has repeated itself',
description: 'The Chatbot repeated itself',
},
};
}
Expand Down
14 changes: 10 additions & 4 deletions packages/genesys-web-messaging-tester-cli/src/commands/exploratoryTest/ui.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ export class Ui {

public testResult(result: ShouldEndConversationEndedResult): string {
const resultMessage =
result.reason.type === 'fail'
? chalk.bold.red(`FAILED: ${result.reason.description}`)
: chalk.bold.green(`PASSED: ${result.reason.description}`);
result.reason.type === 'pass'
? [chalk.bold.green('PASSED'), chalk.green(result.reason.description)]
: [chalk.bold.red('FAILED'), chalk.red(result.reason.description)];

return Ui.trailingNewline(['\n---------------------', resultMessage].join('\n'));
return Ui.trailingNewline(['\n---------------------', ...resultMessage].join('\n'));
}

public messageTranscribed(msg: TranscribedMessage): string {
Expand All @@ -60,6 +60,12 @@ export class Ui {
return Ui.trailingNewline(`${chalk.bold.grey(`${msg.who}:`)} ${chalk.grey(msg.message)}`);
}

public followUpDetailsUnderDevelopment(): string {
return Ui.trailingNewline(
chalk.bold.yellow('Follow up definitions ignored, as functionality is under development'),
);
}

public followUpDetails(feedback: string): string {
return Ui.trailingNewline(['\n---------------------', feedback].join('\n'));
}
Expand Down
11 changes: 6 additions & 5 deletions ...s/genesys-web-messaging-tester-cli/src/commands/scriptedTest/createScriptedTestCommand.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
messageIdToConversationIdFactory,
MessageIdToConvoIdClient,
} from '../../genesysPlatform/messageIdToConversationIdFactory';
import { CommandExpectedlyFailedError } from '../CommandExpectedlyFailedError';

function parsePositiveInt(value: string) {
const parsedValue = parseInt(value, 10);
Expand Down Expand Up @@ -146,7 +147,7 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
const checkResult = await messageIdToConversationIdClient.preflightCheck();
if (!checkResult.ok) {
outputConfig.writeErr(ui.preflightCheckOfAssociateConvoIdFailed(checkResult));
throw new Error();
throw new CommandExpectedlyFailedError();
}
}
}
Expand All @@ -157,14 +158,14 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
testScriptFileContents = yamlFileReader(testScriptPath);
} catch (error) {
outputConfig.writeErr(ui.errorReadingTestScriptFile(error as Error));
throw new Error();
throw new CommandExpectedlyFailedError();
}

// 2. Validate Test Script
const testScriptValidationResults = validateTestScript(testScriptFileContents);
if (testScriptValidationResults.error) {
outputConfig.writeErr(ui.validatingTestScriptFailed(testScriptValidationResults.error));
throw new Error();
throw new CommandExpectedlyFailedError();
}

// 3. Merge session config from args and Test Script - args take priority
Expand All @@ -181,7 +182,7 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
outputConfig.writeErr(
ui.validatingSessionConfigFailed(sessionConfigValidationResults.error),
);
throw new Error();
throw new CommandExpectedlyFailedError();
}

// 5. Extract Scenarios from Test Script
Expand Down Expand Up @@ -292,7 +293,7 @@ GENESYSCLOUD_OAUTHCLIENT_SECRET`,
);

if (results.scenarioResults.some((r) => !r.hasPassed)) {
throw new Error();
throw new CommandExpectedlyFailedError();
}
},
);
Expand Down
3 changes: 2 additions & 1 deletion packages/genesys-web-messaging-tester-cli/src/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
#!/usr/bin/env node

import { createCli } from './createCli';
import { CommandExpectedlyFailedError } from './commands/CommandExpectedlyFailedError';

createCli()
.parseAsync(process.argv)
.then(() => process.exit(0))
.catch((error) => {
if (error) {
if (error && !(error instanceof CommandExpectedlyFailedError)) {
console.error(error);
}
process.exit(1);
Expand Down

0 comments on commit 2ddf250

Please sign in to comment.