Merge pull request #10 from dbpunk-labs/feat/add_selfhost

feat: use self hosted runner

.github/workflows/ci.yaml

@@ -10,15 +10,13 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
+    #runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
           submodules: recursive
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Run tests
@@ -29,8 +27,6 @@ jobs:
           pytest tests/*
           cd ${WS_DIR}/agent
           pytest tests/*
-          cd ${WS_DIR}
-          bash clean_sandbox.sh
       - uses: actions/upload-artifact@v3
         if: failure()
         with:

agent/src/octopus_agent/agent_setup.py

@@ -14,32 +14,34 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""
-
-"""
+""" """
 import click
 import asyncio
 from octopus_agent.agent_sdk import AgentSDK
 
+
 async def add_kernel(endpoint, api_key, kernel_endpoint, kernel_api_key, workspace):
     sdk = AgentSDK(endpoint, api_key)
     sdk.connect()
     try:
         await sdk.add_kernel(kernel_api_key, kernel_endpoint, workspace)
-        print("add kernel %s done"%kernel_endpoint)
+        print("add kernel %s done" % kernel_endpoint)
     except Exception as ex:
-        print("add kernel %s failed"%kernel_endpoint)
+        print("add kernel %s failed" % kernel_endpoint)
+
 
 @click.command()
-@click.option('--kernel_endpoint', help='the endpoint of kernel')
-@click.option('--kernel_api_key', help="the api key of kernel")
-@click.option('--kernel_workspace', help="the workspace of kernel")
-@click.option('--agent_endpoint', help="the endpoint of agent")
-@click.option('--admin_key', help="the admin key of agent")
+@click.option("--kernel_endpoint", help="the endpoint of kernel")
+@click.option("--kernel_api_key", help="the api key of kernel")
+@click.option("--kernel_workspace", help="the workspace of kernel")
+@click.option("--agent_endpoint", help="the endpoint of agent")
+@click.option("--admin_key", help="the admin key of agent")
 def setup(kernel_endpoint, kernel_api_key, kernel_workspace, agent_endpoint, admin_key):
     if not kernel_endpoint or not kernel_api_key or not admin_key or not agent_endpoint:
         print("kernel_endpoint or kernel_api_key or admin key is empty")
         return
-    asyncio.run(add_kernel(agent_endpoint, admin_key, kernel_endpoint, kernel_api_key,
-        kernel_workspace))
-
+    asyncio.run(
+        add_kernel(
+            agent_endpoint, admin_key, kernel_endpoint, kernel_api_key, kernel_workspace
+        )
+    )

agent/src/octopus_agent/langchain_agent_builder.py

@@ -32,8 +32,6 @@ def build_openai_agent(llm, sdk, workspace, max_iterations, verbose):
     # init the agent
     tools = [
         ExecutePythonCodeTool(octopus_api=api),
-        ExecuteShellCodeTool(octopus_api=api),
-        ExecuteTypescriptCodeTool(octopus_api=api),
         PrintCodeTool(),
         PrintFinalAnswerTool(),
     ]

agent/src/octopus_agent/prompt.py

@@ -26,7 +26,7 @@
 
 Thirdly, the programming environment used to execute code  has the following capabilities
 * Internet connection: This allows the programming environment to access online resources, such as documentation, libraries, and code repositories.
-* IPython kernel: This allows the programming environment to execute Python, TypeScript, and shell scripts.
+* IPython kernel: This allows the programming environment to execute Python code
     * Filesystem: This allows the programming environment to open, write, and delete files in the workspace directory.
     * Lots of installed Python libraries:These includes the following popular libraries:
         * pandas:data analysis and manipulation tool

agent/tests/agent_sdk_tests.py

@@ -45,9 +45,9 @@ async def test_upload_smoke_test(agent_sdk):
     path = os.path.abspath(__file__)
     await sdk.upload_file(path, "agent_sdk_tests.py")
     try:
-       responds = []
-       async for respond in sdk.prompt("write a hello world in python"):
-           responds.append(respond)
-       assert len(responds) > 0, "no responds for the prompt"
+        responds = []
+        async for respond in sdk.prompt("write a hello world in python"):
+            responds.append(respond)
+        assert len(responds) > 0, "no responds for the prompt"
     except Exception as ex:
-       assert 0, str(ex)
+        assert 0, str(ex)

agent/tests/agent_tool_tests.py

@@ -30,7 +30,7 @@
 
 
 @pytest.fixture
-async def kernel_sdk():
+def kernel_sdk():
     sdk = KernelSDK(api_base, api_key)
     sdk.connect()
     yield sdk
@@ -40,7 +40,7 @@ async def kernel_sdk():
 @pytest.mark.asyncio
 async def test_async_smoke_test(kernel_sdk):
     code = "print('hello world!')"
-    sdk = await anext(kernel_sdk)
+    sdk = kernel_sdk
     api = OctopusAPIJsonOutput(sdk, api_data_dir)
     result = await api.arun(code)
     assert not result["result"]
@@ -49,7 +49,7 @@ async def test_async_smoke_test(kernel_sdk):
 
 @pytest.mark.asyncio
 async def test_get_result(kernel_sdk):
-    sdk = await anext(kernel_sdk)
+    sdk = kernel_sdk
     code = "5"
     api = OctopusAPIJsonOutput(sdk, api_data_dir)
     result = await api.arun(code)
@@ -62,7 +62,7 @@ async def test_get_result(kernel_sdk):
 
 @pytest.mark.asyncio
 async def test_sync_smoke_test_markdown(kernel_sdk):
-    sdk = await anext(kernel_sdk)
+    sdk = kernel_sdk
     code = "print('hello world!')"
     api = OctopusAPIMarkdownOutput(sdk, api_data_dir)
     result = await api.arun(code)
@@ -73,7 +73,7 @@ async def test_sync_smoke_test_markdown(kernel_sdk):
 
 @pytest.mark.asyncio
 async def test_get_result_markdown(kernel_sdk):
-    sdk = await anext(kernel_sdk)
+    sdk = kernel_sdk
     code = "5"
     api = OctopusAPIMarkdownOutput(sdk, api_data_dir)
     result = await api.arun(code)
@@ -83,7 +83,7 @@ async def test_get_result_markdown(kernel_sdk):
 
 @pytest.mark.asyncio
 async def test_display_result(kernel_sdk):
-    sdk = await anext(kernel_sdk)
+    sdk = kernel_sdk
     code = """
 import matplotlib.pyplot as plt 
 import numpy as np

kernel/tests/kernel_client_tests.py

@@ -113,28 +113,6 @@ async def test_stderr_occurs(kernel_manager):
     kernel_client.stop_client()
 
 
-@pytest.mark.asyncio
-async def test_ts_stdout_occurs(ts_kernel_manager):
-    """Test stdout occurs"""
-    kernel_client = KernelClient(ts_kernel_manager.config_path)
-    await kernel_client.start_client()
-    logger.info("is alive %s", await kernel_client.is_alive())
-    code = """console.log("hello world!")"""
-    kernel_client.execute(code)
-    messages = []
-    async for msg in kernel_client.read_response():
-        if not msg:
-            break
-        logger.info(msg)
-        messages.append(msg)
-    filtered = list(filter(lambda x: x["msg_type"] == "stream", messages))
-    assert len(filtered) > 0, messages
-    assert filtered[0]["content"]["name"] == "stdout"
-    await asyncio.sleep(2)
-    await kernel_client.stop_watch()
-    kernel_client.stop_client()
-
-
 @pytest.mark.asyncio
 async def test_stdout_occurs(kernel_manager):
     """Test stdout occurs"""

start_sandbox.sh

@@ -1,10 +1,10 @@
 #! /bin/sh
 #
 # start_local.sh
-ps -eu | grep python3 | grep -v grep | awk '{print $2}' | while read line; do kill -9 $line; done
+ps -ef | grep python3 | grep -v grep | awk '{print $2}' | while read line; do kill -9 $line; done
 WORKDIR=`pwd`
 
-cd ${WORKDIR}/proto && pip install .
+cd ${WORKDIR}/proto && make && pip install .
 cd ${WORKDIR}/kernel && pip install .
 cd ${WORKDIR}/agent && pip install .
 cd ${WORKDIR}/chat && pip install .