feat: simple tests (#9)

* feat: simple tests

* fix: style test

* fix: change files owner

* refactor: GPT 4 -> GTP-4

.github/workflows/test.yml

@@ -0,0 +1,63 @@
+name: Dream Tests
+
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: [self-hosted]
+    steps:
+
+    - uses: actions/checkout@v3
+
+    - name: Add .env_secret file
+      run: touch .env_secret
+
+    - name: Add google api keys
+      run: ./tests/test.sh MODE=add-google
+
+    - name: Build universal_prompted_assistant
+      run: ./tests/test.sh BOT=universal_prompted_assistant MODE=build
+
+    - name: Start universal_prompted_assistant
+      run: |
+        ./tests/test.sh BOT=universal_prompted_assistant MODE=clean
+        ./tests/test.sh BOT=universal_prompted_assistant MODE=start
+
+    - name: Test universal_prompted_assistant
+      run: |
+        ./tests/test.sh BOT=universal_prompted_assistant MODE=test
+        ./tests/test.sh BOT=universal_prompted_assistant MODE=clean
+
+    - name: Add openai api key
+      run: |
+        ./tests/test.sh MODE=add-openai
+
+    - name: Build multiskill_ai_assistant
+      run: ./tests/test.sh BOT=multiskill_ai_assistant MODE=build
+
+    - name: Start multiskill_ai_assistant
+      run: |
+        ./tests/test.sh BOT=multiskill_ai_assistant MODE=clean
+        ./tests/test.sh BOT=multiskill_ai_assistant MODE=start
+
+    - name: Test multiskill_ai_assistant
+      run: |
+        ./tests/test.sh BOT=multiskill_ai_assistant MODE=test
+        ./tests/test.sh BOT=multiskill_ai_assistant MODE=clean
+
+    - name: Logs
+      if: failure()
+      run: |
+        ./tests/test.sh BOT=multiskill_ai_assistant MODE=logs
+        ./tests/test.sh BOT=universal_prompted_assistant MODE=logs
+
+    - name: Cleanup
+      if: always()
+      run: |
+        ./tests/test.sh BOT=universal_prompted_assistant MODE=clean
+        ./tests/test.sh BOT=multiskill_ai_assistant MODE=clean
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+      GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}

dp/proxy.yml

@@ -0,0 +1,21 @@
+services:
+
+  combined-classification:
+    command: ["nginx", "-g", "daemon off;"]
+    build:
+      context: dp/proxy/
+      dockerfile: Dockerfile
+    environment:
+      - PROXY_PASS=3.95.137.219:8087
+      - PORT=8087
+
+  sentence-ranker:
+    command: [ "nginx", "-g", "daemon off;" ]
+    build:
+      context: dp/proxy/
+      dockerfile: Dockerfile
+    environment:
+      - PROXY_PASS=3.95.137.219:8128
+      - PORT=8128
+
+version: '3.7'

tests/Docker/requirements.txt

@@ -0,0 +1,4 @@
+click==8.1.6
+PyYAML==6.0.1
+python-dotenv==1.0.0
+python-on-whales==0.64.1

tests/Docker/test.py

@@ -0,0 +1,82 @@
+import logging
+import time
+from multiprocessing import Process
+
+import click
+import requests
+from python_on_whales import DockerClient
+
+logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def healthcheck(docker: DockerClient, timeout: float) -> None:
+    # TODO: add healthchecks to containers and look at this status.
+    end_time = time.time() + timeout
+    while True:
+        time.sleep(5)
+        containers = docker.compose.ps(all=True)
+
+        exited_containers = [c for c in containers if c.state.status == "exited"]
+        if exited_containers:
+            logger.error(f"Following containers are exited: {', '.join([c.name for c in exited_containers])}.")
+            for container in exited_containers:
+                logger.error(f"{container.name} output:")
+                logger.error(container.logs())
+            raise RuntimeError("Found exited containers")
+
+        not_running_containers = [c for c in containers if c.state.running is False]
+
+        if not_running_containers:
+            logger.info(f'Waiting {", ".join([c.name for c in not_running_containers])} to run.')
+        else:
+            logger.info("All containers are running.")
+        try:
+            resp = requests.get("http://0.0.0.0:4242/ping")
+            if resp.status_code == 200 and not not_running_containers:
+                return
+        except requests.exceptions.ConnectionError:
+            logger.info("Agent's ping is still unavailable")
+
+        if time.time() > end_time:
+            if not_running_containers:
+                logger.error(f"Failed to run " f'{", ".join([c.name for c in not_running_containers])}.')
+            raise TimeoutError("Failed to start.")
+
+
+@click.command()
+@click.option("--mode", type=click.Choice(["build", "up", "clean", "logs"]))
+@click.option(
+    "--compose-file",
+    "-f",
+    default=["docker-compose.yml"],
+    help="Path to docker compose file.",
+    multiple=True,
+)
+@click.option("--wait-timeout", default=480, help="How long to wait", type=float)
+def main(mode, compose_file, wait_timeout):
+    compose_file = list(compose_file)
+    docker = DockerClient(compose_files=compose_file, compose_project_name="test")
+    if mode == "build":
+        process = Process(target=docker.compose.build)
+        process.start()
+        process.join()
+        print(process.exitcode)
+        if process.exitcode:
+            raise SystemExit(f"Got {process.exitcode} exit code.")
+    elif mode == "up":
+        process = Process(target=docker.compose.up(detach=True))
+        process.start()
+        healthcheck(docker, wait_timeout)
+    elif mode == "clean":
+        docker.compose.kill()
+        docker.compose.down(remove_orphans=True, volumes=True)
+        docker.compose.rm(stop=True, volumes=True)
+    elif mode == "logs":
+        for c in docker.compose.ps(all=True):
+            logger.info(f"{c.name} logs:")
+            logger.info(c.logs())
+
+
+if __name__ == "__main__":
+    main()

tests/test.py

@@ -0,0 +1,78 @@
+import argparse
+import os
+import random
+from time import sleep
+
+import requests
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+
+
+def test_bot():
+    user_id = random.randint(0, 100)
+    res = requests.post(
+        "http://0.0.0.0:4242",
+        json={
+            "user_id": f"test-user-{user_id}",
+            "payload": "Who are you? who built you? what can you do?",
+        },
+    ).json()["active_skill"]
+    if "prompted" in res:
+        print("Success!")
+    else:
+        raise ValueError(f"\nERROR: assistant returned `{res}`\n")
+
+
+prompt = """TASK: Your name is Baby Sitting Assistant. You were made by Babies & Co.
+Help the human to get busy a baby. Do not discuss other topics. Respond with empathy.
+Ask open-ended questions to help the human understand what to do with a baby.
+"""
+
+LM_SERVICES_MAPPING = {
+    "Open-Assistant SFT-1 12B": "http://transformers-lm-oasst12b:8158/respond",
+    "GPT-JT 6B": "http://transformers-lm-gptjt:8161/respond",
+    "GPT-3.5": "http://openai-api-davinci3:8131/respond",
+    "ChatGPT": "http://openai-api-chatgpt:8145/respond",
+    "ChatGPT 16k": "http://openai-api-chatgpt-16k:8167/respond",
+    "GPT-4 32k": "http://openai-api-gpt4-32k:8160/respond",
+    "GPT-4": "http://openai-api-gpt4:8159/respond",
+}
+
+
+def check_universal_assistant(lm_services):
+    if OPENAI_API_KEY is None:
+        raise ValueError("OPENAI_API_KEY is None!")
+    for lm_service in lm_services:
+        print(f"Checking `Universal Assistant` with `{lm_service}`")
+
+        result = requests.post(
+            "http://0.0.0.0:4242",
+            json={
+                "user_id": f"test-user-{random.randint(100, 1000)}",
+                "payload": "I want an article about quantum physics for children.",
+                "prompt": prompt,
+                "lm_service_url": LM_SERVICES_MAPPING[lm_service],
+                "openai_api_key": OPENAI_API_KEY,
+            },
+        ).json()["active_skill"]
+
+        if "prompted" in result:
+            print("Success!")
+        else:
+            raise ValueError(f"\nERROR: `Universal Assistant` returned `{result}` with lm service {lm_service}\n")
+        sleep(5)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("assistant", help="assistant to test")
+    args = parser.parse_args()
+    assistant = args.assistant
+    if assistant == "multiskill_ai_assistant":
+        test_bot()
+    elif assistant == "universal_prompted_assistant":
+        check_universal_assistant(["ChatGPT", "GPT-3.5"])  # TODO: add "GPT-4 32k", "GPT-4" and "ChatGPT 16k"
+
+
+if __name__ == "__main__":
+    main()

tests/test.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+. /home/ubuntu/venv/bin/activate
+
+for ARGUMENT in "$@"; do
+
+    KEY=$(echo $ARGUMENT | cut -f1 -d=)
+    VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+    case "$KEY" in
+    DEVICE) DEVICE=${VALUE} ;;
+    MODE) MODE=${VALUE} ;;
+    BOT) BOT=${VALUE} ;;
+    *) ;;
+    esac
+done
+
+
+
+function wait_service() {
+    local timeout=${WAIT_TIMEOUT:-480}
+    local interval=${WAIT_INTERVAL:-10}
+    local url=$1
+    local reply_keyword=$2
+    while [[ $timeout -gt 0 ]]; do
+        local res=$(curl -s -XGET "$url" | grep "$reply_keyword")
+        if [ ! -z "$res" ]; then
+            echo FOUND service $url
+            echo REPLY: $res
+            return 0
+        fi
+        sleep $interval
+        ((timeout-=interval))
+        echo wait_service $url timeout in $timeout sec..
+    done
+    echo ERROR: $url is not responding
+    return 1
+}
+
+FILES="-f docker-compose.yml -f assistant_dists/$BOT/docker-compose.override.yml -f assistant_dists/$BOT/dev.yml -f dp/proxy.yml"
+
+function dockercompose_cmd() {
+    DOCKER_COMPOSE_CMD="docker compose --no-ansi -p test $FILES"
+    eval '$DOCKER_COMPOSE_CMD "$@"'
+    exit $?
+}
+
+function add_google_keys() {
+    echo 'GOOGLE_API_KEY=$GOOGLE_API_KEY' >> .env
+    echo 'GOOGLE_CSE_ID=$GOOGLE_CSE_ID' >> .env
+}
+
+function add_openai_key() {
+    echo 'OPENAI_API_KEY=$OPENAI_API_KEY' >> .env
+}
+
+if [[ "$MODE" == "add-google" ]]; then
+    add_google_keys
+fi
+
+if [[ "$MODE" == "add-openai" ]]; then
+    add_openai_key
+fi
+
+if [[ "$MODE" == "build" ]]; then
+  dockercompose_cmd build
+fi
+
+function cleanup() {
+    local exit_status=${1:-$?}
+    echo SHUTDOWN TESTING ENVIRONMENT..
+
+    docker run --rm -v $(pwd):/tmp -e UID=$(id -u) ubuntu:mantic chown -R $UID /tmp
+    python tests/Docker/test.py $FILES --mode clean
+    docker run --rm -v $(pwd):/tmp -e UID=$(id -u) ubuntu:mantic chown -R $UID /tmp
+
+    dockercompose_cmd down
+    dockercompose_cmd rm mongo
+    python tests/Docker/test.py $FILES --mode clean
+    echo EXIT $0 with STATUS: $exit_status
+}
+
+if [[ "$MODE" == "start" ]]; then
+  python tests/Docker/test.py $FILES --mode up
+fi
+
+if [[ "$MODE" == "clean" ]]; then
+    cleanup
+    exit 0
+fi
+
+if [[ "$MODE" == "test" ]]; then
+    dockercompose_cmd exec -T -u $(id -u) -e OPENAI_API_KEY=$OPENAI_API_KEY agent python3 tests/test.py $BOT
+    exit 0
+fi
+
+if [[ "$MODE" == "logs" ]]; then
+    python tests/Docker/test.py $FILES --mode logs
+fi