Add output option to AI command (#177)

examples/cli-ai-tests/README.md

@@ -20,8 +20,21 @@ export $(cat .env | xargs) && ./chatgpt-run.sh
 export DEPLOYMENT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
 export REGION=xxxx.pure.cloud
 export GOOGLE_APPLICATION_CREDENTIALS=PATH/TO/JSON
+export VERTEX_AI_PROJECT=example-project
+export VERTEX_AI_LOCATION=example-location
 ```
 
 ```shell
 export $(cat .env | xargs) && ./google-vertex-ai-run.sh
 ```
+
+# Troubleshooting
+
+## Permission denied during script running
+
+If experiencing a permission denied error when running a script then try reinstalling the node version:
+
+```shell
+nvm uninstall <version in .nvm>
+nvm install <version in .nvm>
+```

examples/cli-ai-tests/google-vertex-ai-example.yml

@@ -4,8 +4,6 @@ config:
   ai:
     provider: google-vertex-ai
     config:
-      location: example-location
-      project: example-gcp-project
       modelVersion: "002"
       examples:
         - input: "What would you like to do today?"

packages/genesys-web-messaging-tester-cli/.gitignore

@@ -0,0 +1,2 @@
+test-outputs/
+test-outputs/*

packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/chatGptConfigLoads.spec.ts

@@ -0,0 +1,88 @@
+import { readFileSync } from 'fs';
+import { Command } from 'commander';
+import { createCli } from '../../../src/createCli';
+import { ChatCompletionClient } from '../../../src/commands/aiTest/chatCompletionClients/chatCompletionClient';
+
+describe('ChatGPT config', () => {
+  let fsReadFileSync: jest.MockedFunction<typeof readFileSync>;
+
+  let mockOpenAiChatCompletionClient: jest.Mocked<ChatCompletionClient>;
+
+  let cli: Command;
+  let capturedOutput: {
+    errOut: string[];
+    stdOut: string[];
+  };
+
+  beforeEach(() => {
+    fsReadFileSync = jest.fn();
+
+    capturedOutput = {
+      errOut: [],
+      stdOut: [],
+    };
+
+    const cliCommand = new Command().exitOverride(() => {
+      throw new Error('CLI Command errored');
+    });
+
+    const scenarioTestCommand = new Command()
+      .exitOverride(() => {
+        throw new Error('Scenario Test Command errored');
+      })
+      .configureOutput({
+        writeErr: (str) => {
+          console.error(str);
+          capturedOutput.errOut.push(str);
+        },
+        writeOut: (str) => {
+          console.log(str);
+          capturedOutput.stdOut.push(str);
+        },
+      });
+
+    cli = createCli(cliCommand, undefined, {
+      command: scenarioTestCommand,
+      fsReadFileSync,
+      fsAccessSync: jest.fn(),
+      fsWriteFileSync: jest.fn(),
+      webMessengerSessionFactory: jest.fn().mockReturnValue({ on: jest.fn(), close: jest.fn() }),
+      openAiCreateChatCompletionClient: () => mockOpenAiChatCompletionClient,
+      googleAiCreateChatCompletionClient: () => {
+        throw new Error('Not implemented');
+      },
+      conversationFactory: jest
+        .fn()
+        .mockReturnValue({ waitForConversationToStart: jest.fn(), sendText: jest.fn() }),
+      processEnv: { OPENAI_API_KEY: 'test' },
+    });
+  });
+
+  test('ChatGPT provider is loaded', async () => {
+    fsReadFileSync.mockReturnValue(`
+config:
+  deploymentId: test-deployment-id
+  region: test-region
+  origin: test-origin
+  ai:
+    provider: chatgpt
+scenarios:
+  Test:
+    setup:
+      prompt: Test prompt
+      terminatingPhrases:
+        pass: ["PASS"]
+        fail: ["FAIL"]
+`);
+    mockOpenAiChatCompletionClient = {
+      getProviderName: jest.fn().mockReturnValue('mock-chatgpt'),
+      predict: jest.fn().mockResolvedValue({ role: 'customer', content: 'PASS' }),
+      preflightCheck: jest.fn().mockResolvedValue({ ok: true }),
+    };
+
+    await cli.parseAsync([...['node', '/path/to/cli'], 'ai', ...['/test/path']]);
+
+    expect(mockOpenAiChatCompletionClient.preflightCheck).toHaveBeenCalled();
+    expect(mockOpenAiChatCompletionClient.predict).toHaveBeenCalled();
+  });
+});

packages/genesys-web-messaging-tester-cli/__tests__/commands/ai/savesOutputFiles.spec.ts

@@ -0,0 +1,123 @@
+import { readFileSync } from 'fs';
+import { Command } from 'commander';
+import { createCli } from '../../../src/createCli';
+import { writeFileSync } from 'node:fs';
+import stripAnsi from 'strip-ansi';
+
+const yamlFileContents = `
+config:
+  deploymentId: test-deployment-id-1
+  region: test-region-1
+  ai:
+    provider: chatgpt
+scenarios:
+  Test:
+    setup:
+      prompt: Test prompt
+      terminatingPhrases:
+        pass: ["PASS"]
+        fail: ["FAIL"]
+`;
+
+describe('AI saving output', () => {
+  let fsReadFileSync: jest.MockedFunction<typeof readFileSync>;
+  let fsWriteFileSync: jest.MockedFunction<typeof writeFileSync>;
+
+  let cli: Command;
+  let capturedOutput: {
+    errOut: string[];
+    stdOut: string[];
+  };
+
+  beforeEach(() => {
+    capturedOutput = {
+      errOut: [],
+      stdOut: [],
+    };
+
+    fsReadFileSync = jest.fn().mockReturnValue(yamlFileContents);
+    fsWriteFileSync = jest.fn();
+
+    const cliCommand = new Command().exitOverride(() => {
+      throw new Error('CLI Command errored');
+    });
+
+    const scenarioTestCommand = new Command()
+      .exitOverride(() => {
+        throw new Error('Scenario Test Command errored');
+      })
+      .configureOutput({
+        writeErr: (str) => {
+          console.error(str);
+          capturedOutput.errOut.push(str);
+        },
+        writeOut: (str) => {
+          console.log(str);
+          capturedOutput.stdOut.push(str);
+        },
+      });
+
+    cli = createCli(cliCommand, undefined, {
+      command: scenarioTestCommand,
+      fsAccessSync: jest.fn(),
+      fsReadFileSync,
+      fsWriteFileSync,
+      webMessengerSessionFactory: jest.fn().mockReturnValue({ on: jest.fn(), close: jest.fn() }),
+      openAiCreateChatCompletionClient: () => ({
+        getProviderName: jest.fn().mockReturnValue('mock-chatgpt'),
+        predict: jest.fn().mockResolvedValue({ role: 'customer', content: 'PASS' }),
+        preflightCheck: jest.fn().mockResolvedValue({ ok: true }),
+      }),
+      googleAiCreateChatCompletionClient: jest.fn(),
+      conversationFactory: jest
+        .fn()
+        .mockReturnValue({ waitForConversationToStart: jest.fn(), sendText: jest.fn() }),
+      processEnv: { OPENAI_API_KEY: 'test' },
+    });
+  });
+
+  test('Calls write operation with filename and conversation contents', async () => {
+    await cli.parseAsync([
+      ...['node', '/path/to/cli'],
+      'ai',
+      ...['/test/path', '--out-dir', '/test/dir/'],
+    ]);
+
+    expect(fsWriteFileSync).toHaveBeenCalledTimes(1);
+
+    const [filename, body] = fsWriteFileSync.mock.calls[0];
+    expect(filename).toMatch(/\/test\/dir\/test-\d+\.json/);
+    expect(JSON.parse(body as string)).toStrictEqual({
+      placeholderValues: {},
+      prompt: 'Test prompt',
+      reasonForEnd: {
+        type: 'pass',
+        description: "Terminating phrase found in response: 'PASS'",
+      },
+      transcript: [
+        {
+          role: 'customer',
+          content: 'PASS',
+        },
+      ],
+    });
+  });
+
+  test('Writes out errors saving output file ', async () => {
+    fsWriteFileSync.mockImplementation(() => {
+      throw new Error('dummy error');
+    });
+
+    await cli.parseAsync([
+      ...['node', '/path/to/cli'],
+      'ai',
+      ...['/test/path', '--out-dir', '/test/dir/'],
+    ]);
+
+    expect(fsWriteFileSync).toHaveBeenCalledTimes(1);
+    expect(capturedOutput.errOut.map(stripAnsi).join('')).toStrictEqual(`
+Failed to save output file:
+dummy error
+`);
+  });
+});