From 8ef6fdd0dae39f499dc3e3f7c363be103e0f175a Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Tue, 19 Dec 2023 18:45:02 +0900 Subject: [PATCH] doc: Add Client SDK examples (#1789) Co-authored-by: Kyujin Cho --- .editorconfig | 3 + changes/1789.fix.md | 1 + docs/README.md | 67 +++- docs/_static/css/customTheme.css | 171 +++++---- docs/client/dev/examples.rst | 485 +++++++++++++++++++------- docs/client/dev/index.rst | 2 +- docs/client/dev/session.rst | 3 + docs/conf.py | 14 +- src/ai/backend/client/func/session.py | 33 ++ 9 files changed, 548 insertions(+), 231 deletions(-) create mode 100644 changes/1789.fix.md diff --git a/.editorconfig b/.editorconfig index 14a336be19..ad23db07fc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -15,6 +15,9 @@ max_line_length = 100 indent_style = space indent_size = 4 +[*.md] +trim_trailing_whitespace = false + [*.rst] max_line_length = 0 indent_style = space diff --git a/changes/1789.fix.md b/changes/1789.fix.md new file mode 100644 index 0000000000..fec67bd0a6 --- /dev/null +++ b/changes/1789.fix.md @@ -0,0 +1 @@ +Add a missing `ComputeSession.start_service()` functional API in the client SDK with documentation updates diff --git a/docs/README.md b/docs/README.md index b0f4be0116..b405b6c1f7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,14 +21,10 @@ $ git clone https://github.com/lablup/backend.ai bai-dev $ cd ./bai-dev/docs $ pyenv local bai-docs $ pip install -U pip setuptools wheel -$ pip install -U \ - --find-links=https://dist.backend.ai/pypi/simple/grpcio \ - --find-links=https://dist.backend.ai/pypi/simple/grpcio-tools \ - --find-links=https://dist.backend.ai/pypi/simple/hiredis \ - --find-links=https://dist.backend.ai/pypi/simple/psycopg-binary \ - -r requirements.txt +$ pip install -U -r requirements.txt ``` + ## Building API Reference JSON file ```console $ ./py -m ai.backend.manager.openapi docs/manager/rest-reference/openapi.json @@ -36,9 +32,11 @@ $ ./py -m ai.backend.manager.openapi docs/manager/rest-reference/openapi.json This script must be executed on behalf of the virtual environment managed by pants, not by the venv for the sphinx. Generated OpenAPI JSON file will be located at under `manager/rest-reference/openapi.json`. + ## Building HTML document -> 📌 NOTE: Please ensure that you are inside the `docs` directory and the virtualenv is activated. +> [!NOTE] +> Please ensure that you are inside the `docs` directory and the virtualenv is activated. ### Make the html version @@ -49,15 +47,16 @@ $ make html The compiled documentation is under `_build/html/index.html`. You may serve it for local inspection using `python -m http.server --directory _build/html`. + ## Translation -#### Generate/update pot (Portable Object Template) files +### Generate/update pot (Portable Object Template) files ```console $ make gettext ``` -#### Build po (Portable Object) files using sphinx-intl +### Build po (Portable Object) files using sphinx-intl ```console $ sphinx-intl update -p _build/locale/ -l ko @@ -66,7 +65,7 @@ $ sphinx-intl update -p _build/locale/ -l ko The `.po` message files are under `locales/ko/LC_MESSAGES/`. Edit them by filling missing translations. -#### Build HTML files with translated version +### Build HTML files with translated version ```console $ sphinx-intl build @@ -84,7 +83,7 @@ The compiled documentation is under `_build/latex/BackendAIDoc.pdf`. Building PDF requires following libraries to be present on your system. -* TeX Live +* TeX Live - ko.TeX (texlive-lang-korean) - latexmk * ImageMagick @@ -93,12 +92,13 @@ Building PDF requires following libraries to be present on your system. ### Installing dependencies on macOS 1. Install MacTeX from [here](https://www.tug.org/mactex/). There are two types of MacTeX distributions; The BasicTeX one is more lightweight and MacTeX contains most of the libraries commonly used. 2. Follow [here](http://wiki.ktug.org/wiki/wiki.php/KtugPrivateRepository) (Korean) to set up KTUG repository. -3. Exceute following command to install missing dependencies. +3. Exceute following command to install missing dependencies. ```console sudo tlmgr install latexmk tex-gyre fncychap wrapfig capt-of framed needspace collection-langkorean collection-fontsrecommended tabulary varwidth titlesec ``` 4. Install both Pretendard (used for main font) and D2Coding (used to draw monospace characters) fonts on your system. + ## Advanced Settings ### Managing the hierarchy of toctree (Table of Contents) of documentation @@ -126,12 +126,43 @@ Example: Please ask the docs maintainer for help. -### Previewing built HTML documentation -Simply create a nginx server which serves `_build/html` folder. For example (docker required): -```bash -docker run --rm -it -v $(pwd)/_build/html:/usr/share/nginx/html -p 8000:80 nginx -``` -Executing the command above inside `docs` folder will serve the documentation page on port 8000 (http://localhost:8000). + +## Preview + +### The PR previews + +Our ReadTheDocs bot automatically builds the HTML preview for each commit of a PR that changes +the contents of the `docs` directory. +You may simply click the link in the PR comment written by the bot. + +### The HTML documentation + +You may open `_build/html/index.html` from the local filesystem directly on your browser, +but the REST API reference (as of 24.03) which uses a dedicated Javascript-based viewer won't work. + +To preview the full documentation including the REST API reference seamlessly, you need to run a local nginx server. + +1. Create a HTTP server which serves `_build/html` folder. For example: + ```bash + python -m http.server --directory _build/html 8000 + ``` +2. Executing the command above inside `docs` folder will serve the documentation page on port 8000 (http://localhost:8000). + +### Interactive GraphQL browser + +You may use [GraphiQL](https://github.com/graphql/graphiql/tree/main/packages/graphiql#graphiql) +to interact and inspect the Backend.AI Manager's GraphQL API. + +1. Ensure you have the access to the manager server. + The manager's *etcd* configuration should say `config/api/allow-graphql-schema-introspection` is true. +2. Run `backend.ai proxy` command of the client SDK. Depending on your setup, adjust `--bind` and `--port` options. + Use the client SDK version 21.03.7+ or 20.09.9+ at least to avoid unexpected CORS issues. +3. Copy `index.html` from https://gist.github.com/achimnol/dc9996aeffc7cf15e96478e635eb0699 +4. Replace `""` with the real address (host:port) of the proxy, which can be accessed from your browser as well. +5. Run `python -m http.server` command in the directory where `index.html` is located. +6. Open the page served by the HTTP server in the previous step in your web browser. + Enjoy auto-completion and schema introspection of Backend.AI admin API! + ## References for newcomers diff --git a/docs/_static/css/customTheme.css b/docs/_static/css/customTheme.css index 617bfc0347..6018cc9a86 100644 --- a/docs/_static/css/customTheme.css +++ b/docs/_static/css/customTheme.css @@ -7454,89 +7454,88 @@ section { } */ pre { line-height: 125%; } -td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } -td.linenos .special { color: #767676; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -span.linenos.special { color: #767676; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } -.highlight .hll { background-color: #ffffcc } -.highlight { - background: #1E1E1E; - color: #FFF; -} -.highlight .c { color: #8f5902; font-style: italic } /* Comment */ -.highlight .err { color: #a40000; border: 1px solid #ef2929 } /* Error */ -.highlight .g { color: #767676 } /* Generic */ -.highlight .k { color: #51B2E0; font-weight: bold } /* Keyword */ -.highlight .l { color: #767676 } /* Literal */ -.highlight .n { color: #767676 } /* Name */ -.highlight .o { color: #ce5c00; font-weight: bold } /* Operator */ -.highlight .x { color: #767676 } /* Other */ -.highlight .p { color: #767676; font-weight: bold } /* Punctuation */ -.highlight .ch { color: #8f5902; font-style: italic } /* Comment.Hashbang */ -.highlight .cm { color: #8f5902; font-style: italic } /* Comment.Multiline */ -.highlight .cp { color: #8f5902; font-style: italic } /* Comment.Preproc */ -.highlight .cpf { color: #8f5902; font-style: italic } /* Comment.PreprocFile */ -.highlight .c1 { color: #03B5E5; font-style: italic } /* Comment.Single */ -.highlight .cs { color: #8f5902; font-style: italic } /* Comment.Special */ -.highlight .gd { color: #a40000 } /* Generic.Deleted */ -.highlight .ge { color: #767676; font-style: italic } /* Generic.Emph */ -.highlight .ges { color: #767676; font-weight: bold; font-style: italic } /* Generic.EmphStrong */ -.highlight .gr { color: #ef2929 } /* Generic.Error */ -.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ -.highlight .gi { color: #00A000 } /* Generic.Inserted */ -.highlight .go { color: #767676; font-style: italic } /* Generic.Output */ -.highlight .gp { color: #FF9D00 } /* Generic.Prompt */ -.highlight .gs { color: #767676; font-weight: bold } /* Generic.Strong */ -.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ -.highlight .gt { color: #a40000; font-weight: bold } /* Generic.Traceback */ -.highlight .kc { color: #51B2E0; font-weight: bold } /* Keyword.Constant */ -.highlight .kd { color: #51B2E0; font-weight: bold } /* Keyword.Declaration */ -.highlight .kn { color: #51B2E0; font-weight: bold } /* Keyword.Namespace */ -.highlight .kp { color: #51B2E0; font-weight: bold } /* Keyword.Pseudo */ -.highlight .kr { color: #51B2E0; font-weight: bold } /* Keyword.Reserved */ -.highlight .kt { color: #51B2E0; font-weight: bold } /* Keyword.Type */ -.highlight .ld { color: #767676 } /* Literal.Date */ -.highlight .m { color: #0000cf; font-weight: bold } /* Literal.Number */ -.highlight .s { color: #75D100 } /* Literal.String */ -.highlight .na { color: #c4a000 } /* Name.Attribute */ -.highlight .nb { color: #51B2E0 } /* Name.Builtin */ -.highlight .nc { color: #767676 } /* Name.Class */ -.highlight .no { color: #767676 } /* Name.Constant */ -.highlight .nd { color: #5c35cc; font-weight: bold } /* Name.Decorator */ -.highlight .ni { color: #ce5c00 } /* Name.Entity */ -.highlight .ne { color: #cc0000; font-weight: bold } /* Name.Exception */ -.highlight .nf { color: #767676 } /* Name.Function */ -.highlight .nl { color: #f57900 } /* Name.Label */ -.highlight .nn { color: #767676 } /* Name.Namespace */ -.highlight .nx { color: #767676 } /* Name.Other */ -.highlight .py { color: #767676 } /* Name.Property */ -.highlight .nt { color: #51B2E0; font-weight: bold } /* Name.Tag */ -.highlight .nv { color: #767676 } /* Name.Variable */ -.highlight .ow { color: #51B2E0; font-weight: bold } /* Operator.Word */ -.highlight .pm { color: #767676; font-weight: bold } /* Punctuation.Marker */ -.highlight .w { color: #f8f8f8 } /* Text.Whitespace */ -.highlight .mb { color: #0000cf; font-weight: bold } /* Literal.Number.Bin */ -.highlight .mf { color: #0000cf; font-weight: bold } /* Literal.Number.Float */ -.highlight .mh { color: #0000cf; font-weight: bold } /* Literal.Number.Hex */ -.highlight .mi { color: #0000cf; font-weight: bold } /* Literal.Number.Integer */ -.highlight .mo { color: #0000cf; font-weight: bold } /* Literal.Number.Oct */ -.highlight .sa { color: #75D100 } /* Literal.String.Affix */ -.highlight .sb { color: #75D100 } /* Literal.String.Backtick */ -.highlight .sc { color: #75D100 } /* Literal.String.Char */ -.highlight .dl { color: #75D100 } /* Literal.String.Delimiter */ -.highlight .sd { color: #8f5902; font-style: italic } /* Literal.String.Doc */ -.highlight .s2 { color: #75D100 } /* Literal.String.Double */ -.highlight .se { color: #75D100 } /* Literal.String.Escape */ -.highlight .sh { color: #75D100 } /* Literal.String.Heredoc */ -.highlight .si { color: #75D100 } /* Literal.String.Interpol */ -.highlight .sx { color: #75D100 } /* Literal.String.Other */ -.highlight .sr { color: #75D100 } /* Literal.String.Regex */ -.highlight .s1 { color: #75D100 } /* Literal.String.Single */ -.highlight .ss { color: #75D100 } /* Literal.String.Symbol */ -.highlight .bp { color: #3465a4 } /* Name.Builtin.Pseudo */ -.highlight .fm { color: #767676 } /* Name.Function.Magic */ -.highlight .vc { color: #767676 } /* Name.Variable.Class */ -.highlight .vg { color: #767676 } /* Name.Variable.Global */ -.highlight .vi { color: #767676 } /* Name.Variable.Instance */ -.highlight .vm { color: #767676 } /* Name.Variable.Magic */ -.highlight .il { color: #0000cf; font-weight: bold } /* Literal.Number.Integer.Long */ +td.linenos .normal { color: #6e7681; background-color: #0d1117; padding-left: 5px; padding-right: 5px; } +span.linenos { color: #6e7681; background-color: #0d1117; padding-left: 5px; padding-right: 5px; } +td.linenos .special { color: #e6edf3; background-color: #6e7681; padding-left: 5px; padding-right: 5px; } +span.linenos.special { color: #e6edf3; background-color: #6e7681; padding-left: 5px; padding-right: 5px; } +.highlight .hll { background-color: #6e7681 } +.highlight { background: #0d1117; color: #e6edf3 } +.highlight .c { color: #8b949e; font-style: italic } /* Comment */ +.highlight .err { color: #f85149 } /* Error */ +.highlight .esc { color: #e6edf3 } /* Escape */ +.highlight .g { color: #e6edf3 } /* Generic */ +.highlight .k { color: #ff7b72 } /* Keyword */ +.highlight .l { color: #a5d6ff } /* Literal */ +.highlight .n { color: #e6edf3 } /* Name */ +.highlight .o { color: #ff7b72; font-weight: bold } /* Operator */ +.highlight .x { color: #e6edf3 } /* Other */ +.highlight .p { color: #e6edf3 } /* Punctuation */ +.highlight .ch { color: #8b949e; font-style: italic } /* Comment.Hashbang */ +.highlight .cm { color: #8b949e; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #8b949e; font-weight: bold; font-style: italic } /* Comment.Preproc */ +.highlight .cpf { color: #8b949e; font-style: italic } /* Comment.PreprocFile */ +.highlight .c1 { color: #8b949e; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #8b949e; font-weight: bold; font-style: italic } /* Comment.Special */ +.highlight .gd { color: #ffa198; background-color: #490202 } /* Generic.Deleted */ +.highlight .ge { color: #e6edf3; font-style: italic } /* Generic.Emph */ +.highlight .ges { color: #e6edf3; font-weight: bold; font-style: italic } /* Generic.EmphStrong */ +.highlight .gr { color: #ffa198 } /* Generic.Error */ +.highlight .gh { color: #79c0ff; font-weight: bold } /* Generic.Heading */ +.highlight .gi { color: #56d364; background-color: #0f5323 } /* Generic.Inserted */ +.highlight .go { color: #8b949e } /* Generic.Output */ +.highlight .gp { color: #8b949e } /* Generic.Prompt */ +.highlight .gs { color: #e6edf3; font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #79c0ff } /* Generic.Subheading */ +.highlight .gt { color: #ff7b72 } /* Generic.Traceback */ +.highlight .g-Underline { color: #e6edf3; text-decoration: underline } /* Generic.Underline */ +.highlight .kc { color: #79c0ff } /* Keyword.Constant */ +.highlight .kd { color: #ff7b72 } /* Keyword.Declaration */ +.highlight .kn { color: #ff7b72 } /* Keyword.Namespace */ +.highlight .kp { color: #79c0ff } /* Keyword.Pseudo */ +.highlight .kr { color: #ff7b72 } /* Keyword.Reserved */ +.highlight .kt { color: #ff7b72 } /* Keyword.Type */ +.highlight .ld { color: #79c0ff } /* Literal.Date */ +.highlight .m { color: #a5d6ff } /* Literal.Number */ +.highlight .s { color: #a5d6ff } /* Literal.String */ +.highlight .na { color: #e6edf3 } /* Name.Attribute */ +.highlight .nb { color: #e6edf3 } /* Name.Builtin */ +.highlight .nc { color: #f0883e; font-weight: bold } /* Name.Class */ +.highlight .no { color: #79c0ff; font-weight: bold } /* Name.Constant */ +.highlight .nd { color: #d2a8ff; font-weight: bold } /* Name.Decorator */ +.highlight .ni { color: #ffa657 } /* Name.Entity */ +.highlight .ne { color: #f0883e; font-weight: bold } /* Name.Exception */ +.highlight .nf { color: #d2a8ff; font-weight: bold } /* Name.Function */ +.highlight .nl { color: #79c0ff; font-weight: bold } /* Name.Label */ +.highlight .nn { color: #ff7b72 } /* Name.Namespace */ +.highlight .nx { color: #e6edf3 } /* Name.Other */ +.highlight .py { color: #79c0ff } /* Name.Property */ +.highlight .nt { color: #7ee787 } /* Name.Tag */ +.highlight .nv { color: #79c0ff } /* Name.Variable */ +.highlight .ow { color: #ff7b72; font-weight: bold } /* Operator.Word */ +.highlight .pm { color: #e6edf3 } /* Punctuation.Marker */ +.highlight .w { color: #6e7681 } /* Text.Whitespace */ +.highlight .mb { color: #a5d6ff } /* Literal.Number.Bin */ +.highlight .mf { color: #a5d6ff } /* Literal.Number.Float */ +.highlight .mh { color: #a5d6ff } /* Literal.Number.Hex */ +.highlight .mi { color: #a5d6ff } /* Literal.Number.Integer */ +.highlight .mo { color: #a5d6ff } /* Literal.Number.Oct */ +.highlight .sa { color: #79c0ff } /* Literal.String.Affix */ +.highlight .sb { color: #a5d6ff } /* Literal.String.Backtick */ +.highlight .sc { color: #a5d6ff } /* Literal.String.Char */ +.highlight .dl { color: #79c0ff } /* Literal.String.Delimiter */ +.highlight .sd { color: #a5d6ff } /* Literal.String.Doc */ +.highlight .s2 { color: #a5d6ff } /* Literal.String.Double */ +.highlight .se { color: #79c0ff } /* Literal.String.Escape */ +.highlight .sh { color: #79c0ff } /* Literal.String.Heredoc */ +.highlight .si { color: #a5d6ff } /* Literal.String.Interpol */ +.highlight .sx { color: #a5d6ff } /* Literal.String.Other */ +.highlight .sr { color: #79c0ff } /* Literal.String.Regex */ +.highlight .s1 { color: #a5d6ff } /* Literal.String.Single */ +.highlight .ss { color: #a5d6ff } /* Literal.String.Symbol */ +.highlight .bp { color: #e6edf3 } /* Name.Builtin.Pseudo */ +.highlight .fm { color: #d2a8ff; font-weight: bold } /* Name.Function.Magic */ +.highlight .vc { color: #79c0ff } /* Name.Variable.Class */ +.highlight .vg { color: #79c0ff } /* Name.Variable.Global */ +.highlight .vi { color: #79c0ff } /* Name.Variable.Instance */ +.highlight .vm { color: #79c0ff } /* Name.Variable.Magic */ +.highlight .il { color: #a5d6ff } /* Literal.Number.Integer.Long */ diff --git a/docs/client/dev/examples.rst b/docs/client/dev/examples.rst index 523dae645d..0f35381f32 100644 --- a/docs/client/dev/examples.rst +++ b/docs/client/dev/examples.rst @@ -1,41 +1,236 @@ Examples ======== -Synchronous-mode execution --------------------------- +Here are several examples to demonstrate the functional API usage. -Query mode -~~~~~~~~~~ +Initialization of the API Client +-------------------------------- -This is the minimal code to execute a code snippet with this client SDK. +Implicit configuration from environment variables +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. code-block:: python3 +.. code-block:: python - import sys - from ai.backend.client import Session - - with Session() as session: - kern = session.ComputeSession.get_or_create('python:3.6-ubuntu18.04') - code = 'print("hello world")' - mode = 'query' - run_id = None - while True: - result = kern.execute(run_id, code, mode=mode) - run_id = result['runId'] # keeps track of this particular run loop - for rec in result.get('console', []): - if rec[0] == 'stdout': - print(rec[1], end='', file=sys.stdout) - elif rec[0] == 'stderr': - print(rec[1], end='', file=sys.stderr) - else: - handle_media(rec) - sys.stdout.flush() - if result['status'] == 'finished': - break - else: - mode = 'continued' - code = '' - kern.destroy() + from ai.backend.client.session import Session + + def main(): + with Session() as api_session: + print(api_session.System.get_versions()) + + if __name__ == "__main__": + main() + +.. seealso:: :doc:`/client/gsg/config` + + +Explicit configuration +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from ai.backend.client.config import APIConfig + from ai.backend.client.session import Session + + def main(): + config = APIConfig( + endpoint="https://api.backend.ai.local", + endpoint_type="api", + domain="default", + group="default", # the default project name to use + ) + with Session(config=config) as api_session: + print(api_session.System.get_versions()) + + if __name__ == "__main__": + main() + +.. seealso:: :class:`ai.backend.client.config.APIConfig` + +Asyncio-native API session +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import asyncio + from ai.backend.client.session import AsyncSession + + async def main(): + async with AsyncSession() as api_session: + print(api_session.System.get_versions()) + + if __name__ == "__main__": + asyncio.run(main()) + +.. seealso:: The interface of API client session objects: :mod:`ai.backend.client.session` + + +Working with Compute Sessions +----------------------------- + +.. note:: + + From here, we omit the ``main()`` function structure in the sample codes. + +Listing currently running compute sessions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import functools + from ai.backend.client.session import Session + + with Session() as api_session: + fetch_func = functools.partial( + api_session.ComputeSession.paginated_list, + status="RUNNING", + ) + current_offset = 0 + while True: + result = fetch_func(page_offset=current_offset, page_size=20) + if result.total_count == 0: + # no items found + break + current_offset += len(result.items) + for item in result.items: + print(item) + if current_offset >= result.total_count: + # end of list + break + +Creating and destroying a compute session +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from ai.backend.client.session import Session + + with Session() as api_session: + my_session = api_session.ComputeSession.get_or_create( + "python:3.9-ubuntu20.04", # registered container image name + mounts=["mydata", "mymodel"], # vfolder names + resources={"cpu": 8, "mem": "32g", "cuda.device": 2}, + ) + print(my_session.id) + my_session.destroy() + + + +Accessing Container Applications +-------------------------------- + + Launchable apps may vary for sessions. From here we illustrate + an example to create a ttyd (web-based terminal) app, + which is available for all Backend.AI sessions. + +.. note:: + + This example is only applicable for the Backend.AI cluster with + AppProxy v2 enabled and configured. AppProxy v2 only ships with + enterprise version of Backend.AI. + + +The ``ComputeSession.start_service()`` API +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + import requests + + from ai.backend.client.request import Request + from ai.backend.client.session import Session + + app_name = "ttyd" + + with Session() as api_session: + sess = api_session.ComputeSession.get_or_create(...) + service_info = sess.start_service(app_name, login_session_token="dummy") + app_proxy_url = f"{service_info['wsproxy_addr']}/v2/proxy/{service_info['token']}/{sess.id}/add?app={app_name}" + resp = requests.get(app_proxy_url) + body = resp.json() + auth_url = body["url"] + print(auth_url) # opening this link from browser will navigate user to the terminal session + +.. versionadded:: 23.09.8 + + :meth:`ai.backend.client.func.session.ComputeSession.start_service()` + +Set the value ``login_session_token`` to a dummy string like ``"dummy"`` as it is a trace of the legacy interface, which is no longer used. + +Alternatively, in versions before 23.09.8, you may use the raw :class:`ai.backend.client.Request` to call the server-side ``start_service`` API. + +.. code-block:: python + + import asyncio + + import aiohttp + + from ai.backend.client.request import Request + from ai.backend.client.session import AsyncSession + + app_name = "ttyd" + + async def main(): + async with AsyncSession() as api_session: + sess = api_session.ComputeSession.get_or_create(...) + rqst = Request( + "POST", + f"/session/{sess.id}/start-service", + ) + rqst.set_json({"app": app_name, "login_session_token": "dummy"}) + async with rqst.fetch() as resp: + body = await resp.json() + app_proxy_url = f"{body['wsproxy_addr']}/v2/proxy/{body['token']}/{sess.id}/add?app={app_name}" + + async with aiohttp.ClientSession() as client: + async with client.get(app_proxy_url) as resp: + body = await resp.json() + auth_url = body["url"] + print(auth_url) # opening this link from browser will navigate user to the terminal session + + if __name__ == "__main__": + asyncio.run(main()) + + +Code Execution via API +---------------------- + +Synchronous mode +~~~~~~~~~~~~~~~~ + +Snippet execution (query mode) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is the minimal code to execute a code snippet with this client SDK. + +.. code-block:: python + + import sys + from ai.backend.client.session import Session + + with Session() as api_session: + my_session = api_session.ComputeSession.get_or_create("python:3.9-ubuntu20.04") + code = 'print("hello world")' + mode = "query" + run_id = None + try: + while True: + result = my_session.execute(run_id, code, mode=mode) + run_id = result["runId"] # keeps track of this particular run loop + for rec in result.get("console", []): + if rec[0] == "stdout": + print(rec[1], end="", file=sys.stdout) + elif rec[0] == "stderr": + print(rec[1], end="", file=sys.stderr) + else: + handle_media(rec) + sys.stdout.flush() + if result["status"] == "finished": + break + else: + mode = "continued" + code = "" + finally: + my_session.destroy() You need to take care of ``client_token`` because it determines whether to reuse kernel sessions or not. @@ -44,78 +239,80 @@ but within the timeout, any kernel creation requests with the same ``client_toke let Backend.AI cloud to reuse the kernel. -Batch mode -~~~~~~~~~~ +Script execution (batch mode) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ You first need to upload the files after creating the session and construct a ``opts`` struct. -.. code-block:: python3 - - import sys - from ai.backend.client import Session - - with Session() as session: - kern = session.ComputeSession.get_or_create('python:3.6-ubuntu18.04') - kern.upload(['mycode.py', 'setup.py']) - code = '' - mode = 'batch' - run_id = None - opts = { - 'build': '*', # calls "python setup.py install" - 'exec': 'python mycode.py arg1 arg2', - } - while True: - result = kern.execute(run_id, code, mode=mode, opts=opts) - opts.clear() - run_id = result['runId'] - for rec in result.get('console', []): - if rec[0] == 'stdout': - print(rec[1], end='', file=sys.stdout) - elif rec[0] == 'stderr': - print(rec[1], end='', file=sys.stderr) - else: - handle_media(rec) - sys.stdout.flush() - if result['status'] == 'finished': - break - else: - mode = 'continued' - code = '' - kern.destroy() +.. code-block:: python + + import sys + from ai.backend.client.session import Session + + with Session() as session: + compute_sess = session.ComputeSession.get_or_create("python:3.6-ubuntu18.04") + compute_sess.upload(["mycode.py", "setup.py"]) + code = "" + mode = "batch" + run_id = None + opts = { + "build": "*", # calls "python setup.py install" + "exec": "python mycode.py arg1 arg2", + } + try: + while True: + result = kern.execute(run_id, code, mode=mode, opts=opts) + opts.clear() + run_id = result["runId"] + for rec in result.get("console", []): + if rec[0] == "stdout": + print(rec[1], end="", file=sys.stdout) + elif rec[0] == "stderr": + print(rec[1], end="", file=sys.stderr) + else: + handle_media(rec) + sys.stdout.flush() + if result["status"] == "finished": + break + else: + mode = "continued" + code = "" + finally: + compute_sess.destroy() Handling user inputs -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ Inside the while-loop for ``kern.execute()`` above, change the if-block for ``result['status']`` as follows: -.. code:: python3 +.. code:: python ... - if result['status'] == 'finished': + if result["status"] == "finished": break - elif result['status'] == 'waiting-input': - mode = 'input' - if result['options'].get('is_password', False): + elif result["status"] == "waiting-input": + mode = "input" + if result["options"].get("is_password", False): code = getpass.getpass() else: code = input() else: - mode = 'continued' - code = '' + mode = "continued" + code = "" ... -A common gotcha is to miss setting ``mode = 'input'``. Be careful! +A common gotcha is to miss setting ``mode = "input"``. Be careful! Handling multi-media outputs -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The ``handle_media()`` function used above examples would look like: -.. code-block:: python3 +.. code-block:: python def handle_media(record): media_type = record[0] # MIME-Type string @@ -132,59 +329,109 @@ Currently the following behaviors are well-defined: browsers) -Asynchronous-mode Execution ---------------------------- +Asynchronous mode +~~~~~~~~~~~~~~~~~ The async version has all sync-version interfaces as coroutines but comes with additional features such as ``stream_execute()`` which streams the execution results via websockets and ``stream_pty()`` for interactive terminal streaming. -.. code-block:: python3 +.. code-block:: python import asyncio import json import sys import aiohttp - from ai.backend.client import AsyncSession + from ai.backend.client.session import AsyncSession async def main(): - async with AsyncSession() as session: - kern = await session.ComputeSession.get_or_create('python:3.6-ubuntu18.04', - client_token='mysession') + async with AsyncSession() as api_session: + compute_sess = await api_session.ComputeSession.get_or_create( + "python:3.6-ubuntu18.04", + client_token="mysession", + ) code = 'print("hello world")' - mode = 'query' - async with kern.stream_execute(code, mode=mode) as stream: - # no need for explicit run_id since WebSocket connection represents it! - async for result in stream: - if result.type != aiohttp.WSMsgType.TEXT: - continue - result = json.loads(result.data) - for rec in result.get('console', []): - if rec[0] == 'stdout': - print(rec[1], end='', file=sys.stdout) - elif rec[0] == 'stderr': - print(rec[1], end='', file=sys.stderr) - else: - handle_media(rec) - sys.stdout.flush() - if result['status'] == 'finished': - break - elif result['status'] == 'waiting-input': - mode = 'input' - if result['options'].get('is_password', False): - code = getpass.getpass() + mode = "query" + try: + async with compute_sess.stream_execute(code, mode=mode) as stream: + # no need for explicit run_id since WebSocket connection represents it! + async for result in stream: + if result.type != aiohttp.WSMsgType.TEXT: + continue + result = json.loads(result.data) + for rec in result.get("console", []): + if rec[0] == "stdout": + print(rec[1], end="", file=sys.stdout) + elif rec[0] == "stderr": + print(rec[1], end="", file=sys.stderr) + else: + handle_media(rec) + sys.stdout.flush() + if result["status"] == "finished": + break + elif result["status"] == "waiting-input": + mode = "input" + if result["options"].get("is_password", False): + code = getpass.getpass() + else: + code = input() + await stream.send_text(code) else: - code = input() - await stream.send_text(code) - else: - mode = 'continued' - code = '' - await kern.destroy() - - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(main()) - finally: - loop.stop() - -.. versionadded:: 1.5 + mode = "continued" + code = "" + finally: + await compute_sess.destroy() + + if __name__ == "__main__": + asyncio.run(main()) + +.. versionadded:: 19.03 + + +Working with model service +-------------------------- + +Along with working AppProxy v2 deployments, model service requires a resource group configured to accept the inference workload. + +Starting model service +~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from ai.backend.client.session import Session + + with Session() as api_session: + compute_sess = api_session.Service.create( + "python:3.6-ubuntu18.04", + "Llama2-70B", + 1, + service_name="Llama2-service", + resources={"cuda.shares": 2, "cpu": 8, "mem": "64g"}, + open_to_public=False, + ) + +If you set ``open_to_public=True``, the endpoint accepts anonymous traffic without the authentication token (see below). + + +Making request to model service endpoint +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + from ai.backend.client.session import Session + + with Session() as api_session: + compute_sess = api_session.Service.create(...) + service_info = compute_sess.info() + endpoint = service_info["url"] # this value can be None if no successful inference service deployment has been made + + token_info = compute_sess.generate_api_token("3600s") + token = token_info["token"] + headers = {"Authorization": f"BackendAI {token}"} # providing token is not required for public model services + resp = requests.get(f"{endpoint}/v1/models", headers=headers) + +The token returned by the ``generate_api_token()`` method is a JSON web token (JWT), which conveys all required information to authenticate the inference request. +Once generated, it cannot be revoked. A token may have its own expiration date/time. +The lifetime of a token is configured by the user who deploys the inference model, and currently there is no intrinsic minimum/maximum limits of the lifetime. + +.. versionadded:: 23.09 diff --git a/docs/client/dev/index.rst b/docs/client/dev/index.rst index dc3252deff..8c580bd173 100644 --- a/docs/client/dev/index.rst +++ b/docs/client/dev/index.rst @@ -2,7 +2,7 @@ Developer Guides ================ .. toctree:: - :maxdepth: 1 + :maxdepth: 3 :caption: Table of contents session diff --git a/docs/client/dev/session.rst b/docs/client/dev/session.rst index 63cb7e9ef9..690baa76e7 100644 --- a/docs/client/dev/session.rst +++ b/docs/client/dev/session.rst @@ -1,6 +1,9 @@ Client Session ============== +Client Session Objects +---------------------- + .. module:: ai.backend.client.session .. currentmodule:: ai.backend.client.session diff --git a/docs/conf.py b/docs/conf.py index be70d7228a..042dba9928 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -17,8 +17,6 @@ import sys from pathlib import Path -import sphinx_rtd_theme - root_path = Path(__file__).parent.parent on_rtd = os.environ.get("READTHEDOCS") == "True" @@ -62,6 +60,7 @@ # General information about the project. from datetime import date + project = "Backend.AI Documentation" copyright = f"2015-{date.today().year}, Lablup Inc." author = "Lablup Inc." @@ -127,10 +126,10 @@ numfig = True intersphinx_mapping = { - "python": ("http://docs.python.org/3", None), - "multidict": ("https://multidict.readthedocs.io/en/stable/", None), - "yarl": ("https://yarl.readthedocs.io/en/stable/", None), - "aiohttp": ("https://aiohttp.readthedocs.io/en/stable/", None), + "python": ("https://docs.python.org/3", None), + "multidict": ("https://multidict.aio-libs.org/en/stable/", None), + "yarl": ("https://yarl.aio-libs.org/en/stable/", None), + "aiohttp": ("https://docs.aiohttp.org/en/stable/", None), } @@ -241,4 +240,5 @@ html_js_files = [ 'js/custom.js', -] \ No newline at end of file +] + diff --git a/src/ai/backend/client/func/session.py b/src/ai/backend/client/func/session.py index 81b070a403..a503a04b0e 100644 --- a/src/ai/backend/client/func/session.py +++ b/src/ai/backend/client/func/session.py @@ -997,6 +997,39 @@ async def get_abusing_report(self): async with rqst.fetch() as resp: return await resp.json() + @api_function + async def start_service( + self, + app: str, + *, + port: int | Undefined = undefined, + envs: dict[str, Any] | Undefined = undefined, + arguments: dict[str, Any] | Undefined = undefined, + login_session_token: str | Undefined = undefined, + ) -> Mapping[str, Any]: + """ + Starts application from Backend.AI session and returns access credentials + to access AppProxy endpoint. + """ + body: dict[str, Any] = {"app": app} + if port is not undefined: + body["port"] = port + if envs is not undefined: + body["envs"] = json.dumps(envs) + if arguments is not undefined: + body["arguments"] = json.dumps(arguments) + if login_session_token is not undefined: + body["login_session_token"] = login_session_token + + prefix = get_naming(api_session.get().api_version, "path") + rqst = Request( + "POST", + f"/{prefix}/{self.name}/start-service", + ) + rqst.set_json(body) + async with rqst.fetch() as resp: + return await resp.json() + # only supported in AsyncAPISession def listen_events(self, scope: Literal["*", "session", "kernel"] = "*") -> SSEContextManager: """