diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml new file mode 100644 index 00000000..b7342eaa --- /dev/null +++ b/.github/workflows/code-quality.yml @@ -0,0 +1,36 @@ +name: Test code quality + +on: + push: + pull_request: + branches: + - master + +jobs: + code-quality: + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Workaround github issue https://github.com/actions/runner-images/issues/7192 + run: sudo echo RESET grub-efi/install_devices | sudo debconf-communicate grub-pc + + - name: Install pip and hatch + run: | + sudo apt-get install -y python3-pip + pip3 install hatch hatch-vcs + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-code-quality-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-code-quality- + + - name: Install required system packages only for Ubuntu Linux + run: sudo apt-get install -y libsecp256k1-dev + + - name: Run Hatch lint + run: hatch run linting:all diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 00000000..7355d533 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,51 @@ +name: Pytest and code Coverage + +on: + push: + branches: + - "*" + pull_request: + branches: + - main + schedule: + # Run every night at 04:00 (GitHub Actions timezone) + # in order to catch when unfrozen dependency updates + # break the use of the library. + - cron: '4 0 * * *' + +jobs: + pytest: + strategy: + fail-fast: false + matrix: + os: [ macos-12, macos-13, macos-14, ubuntu-20.04, ubuntu-22.04, ubuntu-24.04 ] + runs-on: ${{matrix.os}} + + steps: + - uses: actions/checkout@v4 + + - name: Install required system packages for macOS + run: sudo apt-get install -y python3-pip libsecp256k1-dev + if: startsWith(matrix.os, 'ubuntu-') + + - name: Install required system packages for macOS + if: startsWith(matrix.os, 'macos-') + run: | + brew update + brew tap cuber/homebrew-libsecp256k1 + brew install libsecp256k1 + brew install automake + + - run: python3 -m venv /tmp/venv + - run: /tmp/venv/bin/python -m pip install --upgrade pip hatch coverage + + # Only run coverage on one OS + - run: /tmp/venv/bin/hatch run testing:test + if: matrix.os != 'ubuntu-24.04' + - run: /tmp/venv/bin/hatch run testing:cov + if: matrix.os == 'ubuntu-24.04' + - uses: codecov/codecov-action@v4.0.1 + if: matrix.os == 'ubuntu-24.04' + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: aleph-im/aleph-sdk-python diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index a58d10f7..ac9a8d5c 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -9,10 +9,11 @@ on: - master jobs: - build: - strategy: + build-install-wheel: + strategy: + fail-fast: false matrix: - os: [macos-11, macos-12, ubuntu-20.04, ubuntu-22.04] + os: [macos-12, macos-13, macos-14, ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] runs-on: ${{matrix.os}} steps: @@ -35,23 +36,23 @@ jobs: with: python-version: 3.11 - - name: Install required system packages only for Ubuntu Linux if: startsWith(matrix.os, 'ubuntu-') run: | sudo apt-get update sudo apt-get -y upgrade sudo apt-get install -y libsecp256k1-dev - + - name: Install required Python packages run: | - python3 -m pip install --upgrade build - python3 -m pip install --user --upgrade twine + python3 -m venv /tmp/venv + /tmp/venv/bin/python3 -m pip install --upgrade hatch hatch-vcs - name: Build source and wheel packages run: | - python3 -m build + /tmp/venv/bin/hatch build - name: Install the Python wheel run: | - python3 -m pip install dist/aleph_client-*.whl + python3 -m venv /tmp/install-venv + /tmp/install-venv/bin/python3 -m pip install dist/aleph_client-*.whl diff --git a/.github/workflows/test-pytest.yml b/.github/workflows/test-docker.yml similarity index 97% rename from .github/workflows/test-pytest.yml rename to .github/workflows/test-docker.yml index 457975c8..45a9ef7f 100644 --- a/.github/workflows/test-pytest.yml +++ b/.github/workflows/test-docker.yml @@ -9,8 +9,8 @@ on: - master jobs: - build: - runs-on: ubuntu-20.04 + test-docker: + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 diff --git a/README.md b/README.md new file mode 100644 index 00000000..0fa843d5 --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ +# aleph-client + +Python Client for the [aleph.im network](https://www.aleph.im), next generation network of +decentralized big data applications. Developement follows the [Aleph +Whitepaper](https://github.com/aleph-im/aleph-whitepaper). + +## Documentation + +Documentation can be found on https://docs.aleph.im/tools/aleph-client/ + +## Requirements + +### Linux + +Some cryptographic functionalities use curve secp256k1 and require +installing [libsecp256k1](https://github.com/bitcoin-core/secp256k1). + +> apt-get install -y python3-pip libsecp256k1-dev + +### macOs + +> brew tap cuber/homebrew-libsecp256k1 +> brew install libsecp256k1 + +### Windows + +The software is not tested on Windows, but should work using +the Windows Subsystem for Linux (WSL). + +## Installation + +### From PyPI + +Using pip and [PyPI](https://pypi.org/project/aleph-client/): + +> pip install aleph-client + +### Using a container + +Use the Aleph client and it\'s CLI from within Docker or Podman with: + +> docker run --rm -ti -v $(pwd)/ ghcr.io/aleph-im/aleph-client/aleph-client:master --help + +Warning: This will use an ephemeral key pair that will be discarded when +stopping the container + +## Installation for development + +We recommend using [hatch](https://hatch.pypa.io/) for development. + +Hatch is a modern, extensible Python project manager. +It creates a virtual environment for each project and manages dependencies. + +> pip install hatch + +### Running tests + +> hatch test + +or + +> hatch run testing:cov + +### Formatting code + +> hatch run linting:format + +### Checking types + +> hatch run linting:typing + +## Publish to PyPI + +> hatch build +> hatch upload + +If you want NULS2 support you will need to install nuls2-python +(currently only available on github): + +> pip install aleph-sdk-python[nuls2] + +To install from source and still be able to modify the source code: + +> pip install -e . + diff --git a/README.rst b/README.rst deleted file mode 100644 index a1b09133..00000000 --- a/README.rst +++ /dev/null @@ -1,63 +0,0 @@ -============ -aleph-client -============ - -Python Client for the aleph.im network, next generation network of decentralized big data applications. -Developement follows the `Aleph Whitepaper `_. - -Documentation -============= - -Documentation (albeit still vastly incomplete as it is a work in progress) can be found at http://aleph-client.readthedocs.io/ or built from this repo with: - - $ python setup.py docs - - -Requirements -============ - -- Linux : - -Some cryptographic functionalities use curve secp256k1 and require installing -`libsecp256k1 `_. - - $ apt-get install -y python3-pip libsecp256k1-dev - -- macOs : - - $ brew tap cuber/homebrew-libsecp256k1 - $ brew install libsecp256k1 - - -Installation -============ - -Using pip and `PyPI `_: - - $ pip install aleph-client - - -Installation for development -============================ - -If you want NULS2 support you will need to install nuls2-python (currently only available on github): - - $ pip install git+https://github.com/aleph-im/nuls2-python.git - - -To install from source and still be able to modify the source code: - - $ pip install -e . - or - $ python setup.py develop - - - -Using Docker -============ - -Use the Aleph client and it's CLI from within Docker or Podman with: - - $ docker run --rm -ti -v $(pwd)/data:/data ghcr.io/aleph-im/aleph-client/aleph-client:master --help - -Warning: This will use an ephemeral key that will be discarded when stopping the container. diff --git a/docs/conf.py b/docs/conf.py index 1c78dc00..77c21566 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,9 +13,7 @@ import shutil import sys -__location__ = os.path.join( - os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe())) -) +__location__ = os.path.join(os.getcwd(), os.path.dirname(inspect.getfile(inspect.currentframe()))) # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..62045dcc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,222 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "aleph-client" +dynamic = ["version"] +description = "Python Client library for the Aleph.im network" +readme = "README.md" +readme-content-type = "text/x-rst; charset=UTF-8" +requires-python = ">=3.6" +license = { file = "LICENSE.txt" } +authors = [ + { name = "Aleph.im Team", email = "hello@aleph.im" }, +] +keywords = ["Aleph.im", "Client", "Library", "Python"] +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Environment :: Console", + "Framework :: aiohttp", + "License :: OSI Approved :: MIT License", + "Topic :: System :: Distributed Computing", +] + +dependencies = [ + "aleph-sdk-python>=1.0.0rc1", + "aleph-message>=0.4.6", + "aiohttp==3.9.5", + "typer==0.12.3", + "python-magic==0.4.27", + "pygments==2.18.0", + "rich==13.7.1", + "aiodns==3.2.0", +] +[project.optional-dependencies] +nuls2 = ["nuls2-sdk==0.1.0"] +ethereum = ["eth_account>=0.4.0"] +polkadot = ["substrate-interface==1.3.4"] +cosmos = ["cosmospy==6.0.0"] +solana = ["pynacl==1.5.0", "base58==2.1.1"] +tezos = ["pynacl==1.5.0", "aleph-pytezos==0.1.0"] +docs = ["sphinxcontrib-plantuml==0.27"] + +[project.urls] +Documentation = "https://docs.aleph.im/tools/aleph-client/" +Issues = "https://github.com/aleph-im/aleph-client/issues" +Source = "https://github.com/aleph-im/aleph-client" +Discussions = "https://community.aleph.im/" + +[project.scripts] +aleph = "aleph_client.__main__:app" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.sdist] +include = [ + "src/aleph_client", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/aleph_client"] + +[tool.hatch.version] +source = "vcs" + +[tool.hatch.envs.default] +platforms = ["linux", "macos"] +dependencies = [ + "pytest==8.2.2", + "pytest-asyncio==0.23.7", + "pytest-cov==5.0.0", + "mypy==1.10.0", + "base58==2.1.1", + "fastapi==0.98.0", + "httpx==0.27.0", + "types-requests==2.32.0.20240602", + "types-setuptools==70.0.0.20240524", + "typing_extensions==4.12.2", + "sphinxcontrib-plantuml==0.27" +] + +[tool.hatch.envs.testing] +type = "virtual" +dependencies = [ + "pytest==8.2.2", + "pytest-asyncio==0.23.7", + "pytest-cov==5.0.0", + "mypy==1.10.0", + "base58==2.1.1", + "fastapi==0.98.0", + "httpx==0.27.0", +] +[tool.hatch.envs.testing.scripts] +test = "pytest {args:} ./src/aleph_client/ ./tests/" +test-cov = "pytest --cov {args:} ./src/aleph_client/ ./tests/" +cov-report = [ + "coverage report", +] +cov = [ + "test-cov", + "cov-report", +] + +[[tool.hatch.envs.all.matrix]] +python = ["3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.linting] +dependencies = [ + "black==24.4.2", + "mypy==1.10.0", + "ruff==0.4.9", + "isort==5.13.2", + + "types-requests==2.32.0.20240602", + "types-setuptools==70.0.0.20240524", + "typing_extensions==4.12.2", +] +[tool.hatch.envs.linting.scripts] +typing = "mypy {args:} ./src/ ./tests/" +style = [ + # "ruff {args:}", + "black --check --diff {args:} ./src/ ./tests/", + "isort --check-only --profile black {args:} ./src/ ./tests/", +] +fmt = [ + "black {args:} ./src/ ./tests/", + # "ruff --fix {args:}", + "isort --profile black {args:} ./src/ ./tests/", + "style", +] +all = [ + "style", + "typing", +] + +[tool.pytest.ini_options] +pythonpath = [ + "src" +] +testpaths = [ + "tests" +] + +[tool.black] +line-length = 120 +target-version = ["py39"] + +[tool.mypy] +python_version = "3.9" +install_types = true +non_interactive = true +ignore_missing_imports = true +explicit_package_bases = true +check_untyped_defs = true + +[tool.ruff] +target-version = "py39" +line-length = 120 +select = [ + "A", + "ARG", + "B", + "C", + "DTZ", + "E", + "EM", + "F", + "FBT", + "I", + "ICN", + "ISC", + "N", + "PLC", + "PLE", + "PLR", + "PLW", + "Q", + "RUF", + "S", + "T", + "TID", + "UP", + "W", + "YTT", +] +ignore = [ +# # Allow non-abstract empty methods in abstract base classes +# "B027", +# # Allow boolean positional values in function calls, like `dict.get(... True)` +# "FBT003", +# # Ignore checks for possible passwords +# "S105", "S106", "S107", +# # Ignore complexity +# "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # Allow the use of assert statements + "S101" +] + +#[tool.ruff.isort] +#known-first-party = ["aleph_client"] + +[tool.coverage.run] +branch = true +parallel = true +source_pkgs = ["aleph_client", "tests"] + +[tool.coverage.paths] +aleph_client = ["src/aleph_client"] +tests = ["tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.spinx] +source-dir = "docs" +build-dir = "docs/_build" diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index fe2c4e5c..00000000 --- a/setup.cfg +++ /dev/null @@ -1,152 +0,0 @@ -# This file is used to configure your project. -# Read more about the various options under: -# http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files - -[metadata] -name = aleph-client -description = Lightweight Python Client library for the Aleph.im network -author = Aleph.im Team -author_email = hello@aleph.im -license = mit -long_description = file: README.rst -long_description_content_type = text/x-rst; charset=UTF-8 -url = https://github.com/aleph-im/aleph-client -project_urls = - Documentation = https://aleph.im/ -# Change if running only on Windows, Mac or Linux (comma-separated) -platforms = any -# Add here all kinds of additional classifiers as defined under -# https://pypi.python.org/pypi?%3Aaction=list_classifiers -classifiers = - Development Status :: 4 - Beta - Programming Language :: Python :: 3 - -[options] -zip_safe = False -packages = find: -include_package_data = True -package_dir = - =src -# DON'T CHANGE THE FOLLOWING LINE! IT WILL BE UPDATED BY PYSCAFFOLD! -setup_requires = pyscaffold>=3.2a0,<3.3a0 -# Add here dependencies of your project (semicolon/line-separated), e.g. -install_requires = - aleph-sdk-python~=0.9.1 - aleph-message>=0.4.3 - coincurve==17.0.0 - aiohttp==3.8.4 - eciespy==0.3.13 - typer==0.9.0 - eth_account==0.9.0 - python-magic==0.4.27 - pygments==2.16.1 - rich==13.6.0 - aiodns==3.1.1 -# The usage of test_requires is discouraged, see `Dependency Management` docs -# tests_require = pytest; pytest-cov -# Require a specific Python version, e.g. Python 2.7 or >= 3.4 -# python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.* - -[options.packages.find] -where = src -exclude = - tests - -[options.extras_require] -# Add here additional requirements for extra features, to install with: -# `pip install aleph-client[PDF]` like: -# PDF = ReportLab; RXP -# Add here test requirements (semicolon/line-separated) -testing = - pytest==7.4.2 - pytest-asyncio==0.21.1 - pytest-cov==4.1.0 - mypy==1.5.1 - secp256k1==0.14.0 - pynacl==1.5.0 - base58==2.1.1 - aleph-pytezos==0.1.0 - fastapi==0.98.0 - # httpx is required in tests by fastapi.testclient - httpx==0.25.0 - types-requests==2.31.0.10 - types-setuptools==68.2.0.0 - typing_extensions==4.5.0 -nuls2 = - aleph-nuls2==0.1.0 -ethereum = - eth_account>=0.4.0 -polkadot = - substrate-interface==1.3.4 -cosmos = - cosmospy==6.0.0 -solana = - pynacl==1.5.0 - base58==2.1.1 -tezos = - pynacl==1.5.0 - aleph-pytezos==0.1.0 -docs = - sphinxcontrib-plantuml==0.27 - -[options.entry_points] -# Add here console scripts like: -console_scripts = - aleph = aleph_client.__main__:app -# For example: -# console_scripts = -# fibonacci = aleph_client.skeleton:run -# And any other entry points, for example: -# pyscaffold.cli = -# awesome = pyscaffoldext.awesome.extension:AwesomeExtension - -[test] -# py.test options when running `python setup.py test` -# addopts = --verbose -extras = True - -[tool:pytest] -# Options for py.test: -# Specify command line options as you would do when invoking py.test directly. -# e.g. --cov-report html (or xml) for html/xml output or --junitxml junit.xml -# in order to write a coverage file that can be read by Jenkins. -addopts = - --cov aleph_client --cov-report term-missing - --verbose -norecursedirs = - dist - build - .tox -testpaths = tests - -[aliases] -dists = bdist_wheel - -[bdist_wheel] -# Use this option if your package is pure-python -universal = 0 - -[build_sphinx] -source_dir = docs -build_dir = build/sphinx - -[devpi:upload] -# Options for the devpi: PyPI server and packaging tool -# VCS export must be deactivated since we are using setuptools-scm -no-vcs = 1 -formats = bdist_wheel - -[flake8] -# Some sane defaults for the code style checker flake8 -exclude = - .tox - build - dist - .eggs - docs/conf.py - -[pyscaffold] -# PyScaffold's parameters when the project was created. -# This will be used when updating. Do not change! -version = 3.2.1 -package = aleph_client diff --git a/setup.py b/setup.py deleted file mode 100644 index 5ec256b1..00000000 --- a/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -""" - Setup file for aleph_client. - Use setup.cfg to configure your project. - - This file was generated with PyScaffold 3.2.1. - PyScaffold helps you to put up the scaffold of your new Python project. - Learn more under: https://pyscaffold.org/ -""" -import sys - -from pkg_resources import VersionConflict, require -from setuptools import setup - -try: - require("setuptools>=38.3") -except VersionConflict: - print("Error: version of setuptools is too old (<38.3)!") - sys.exit(1) - - -if __name__ == "__main__": - setup(use_pyscaffold=True) diff --git a/src/aleph_client/__init__.py b/src/aleph_client/__init__.py index 3b86c105..3e953a40 100644 --- a/src/aleph_client/__init__.py +++ b/src/aleph_client/__init__.py @@ -1,18 +1,17 @@ -from pkg_resources import DistributionNotFound, get_distribution +from importlib.metadata import PackageNotFoundError, version try: # Change here if project is renamed and does not equal the package name - dist_name = "aleph-client" - __version__ = get_distribution(dist_name).version -except DistributionNotFound: + __version__ = version("aleph-client") +except PackageNotFoundError: __version__ = "unknown" -finally: - del get_distribution, DistributionNotFound # Deprecation check -moved_types = ["AlephClient", "AuthenticatedAlephClient", "synchronous", "asynchronous"] +moved_types = ["__version__", "AlephClient", "AuthenticatedAlephClient", "synchronous", "asynchronous"] def __getattr__(name): if name in moved_types: - raise ImportError(f"The 'aleph_client.{name}' type is deprecated and has been removed from aleph_client. Please use `aleph.sdk.{name}` instead.") + raise ImportError( + f"The 'aleph_client.{name}' type is deprecated and has been removed from aleph_client. Please use `aleph.sdk.{name}` instead." + ) diff --git a/src/aleph_client/__main__.py b/src/aleph_client/__main__.py index e399f3c3..e0d5b685 100644 --- a/src/aleph_client/__main__.py +++ b/src/aleph_client/__main__.py @@ -2,8 +2,6 @@ Aleph Client command-line interface. """ -from aleph_client.utils import AsyncTyper - from aleph_client.commands import ( about, account, @@ -15,31 +13,24 @@ node, program, ) +from aleph_client.utils import AsyncTyper app = AsyncTyper(no_args_is_help=True) app.add_typer(account.app, name="account", help="Manage account") -app.add_typer( - aggregate.app, name="aggregate", help="Manage aggregate messages on aleph.im" -) -app.add_typer( - files.app, name="file", help="File uploading and pinning on IPFS and aleph.im" -) +app.add_typer(aggregate.app, name="aggregate", help="Manage aggregate messages on aleph.im") +app.add_typer(files.app, name="file", help="File uploading and pinning on IPFS and aleph.im") app.add_typer( message.app, name="message", help="Post, amend, watch and forget messages on aleph.im", ) -app.add_typer( - program.app, name="program", help="Upload and update programs on aleph.im VM" -) +app.add_typer(program.app, name="program", help="Upload and update programs on aleph.im VM") app.add_typer(about.app, name="about", help="Display the informations of Aleph CLI") app.add_typer(node.app, name="node", help="Get node info on aleph.im network") app.add_typer(domain.app, name="domain", help="Manage custom Domain (dns) on aleph.im") -app.add_typer( - instance.app, name="instance", help="Manage instances (VMs) on aleph.im network" -) +app.add_typer(instance.app, name="instance", help="Manage instances (VMs) on aleph.im network") if __name__ == "__main__": app() diff --git a/src/aleph_client/commands/about.py b/src/aleph_client/commands/about.py index 72f959d5..942312f3 100644 --- a/src/aleph_client/commands/about.py +++ b/src/aleph_client/commands/about.py @@ -1,5 +1,8 @@ +from __future__ import annotations + +from importlib.metadata import version as importlib_version + import typer -from pkg_resources import get_distribution from aleph_client.utils import AsyncTyper @@ -11,7 +14,7 @@ def get_version(value: bool): dist_name = "aleph-client" if value: try: - __version__ = get_distribution(dist_name).version + __version__ = importlib_version(dist_name) finally: typer.echo(f"Aleph CLI Version: {__version__}") raise typer.Exit(1) diff --git a/src/aleph_client/commands/account.py b/src/aleph_client/commands/account.py index ddb6c221..68995c48 100644 --- a/src/aleph_client/commands/account.py +++ b/src/aleph_client/commands/account.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import base64 import json @@ -27,9 +29,7 @@ @app.command() def create( private_key: Optional[str] = typer.Option(None, help=help_strings.PRIVATE_KEY), - private_key_file: Optional[Path] = typer.Option( - None, help=help_strings.PRIVATE_KEY_FILE - ), + private_key_file: Optional[Path] = typer.Option(None, help=help_strings.PRIVATE_KEY_FILE), replace: bool = False, debug: bool = False, ): @@ -38,11 +38,7 @@ def create( setup_logging(debug) if private_key_file is None: - private_key_file = Path( - typer.prompt( - "Enter file in which to save the key", sdk_settings.PRIVATE_KEY_FILE - ) - ) + private_key_file = Path(typer.prompt("Enter file in which to save the key", sdk_settings.PRIVATE_KEY_FILE)) if private_key_file.exists() and not replace: typer.secho(f"Error: key already exists: '{private_key_file}'", fg=RED) @@ -68,12 +64,8 @@ def create( @app.command() def address( - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), ): """ Display your public address. @@ -91,12 +83,8 @@ def address( @app.command() def export_private_key( - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), ): """ Display your private key. @@ -125,12 +113,8 @@ def path(): @app.command("sign-bytes") def sign_bytes( message: Optional[str] = typer.Option(None, help="Message to sign"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Sign a message using your private key.""" @@ -151,12 +135,8 @@ def sign_bytes( @app.command() async def balance( address: Optional[str] = typer.Option(None, help="Address"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), ): account: AccountFromPrivateKey = _load_account(private_key, private_key_file) @@ -173,10 +153,6 @@ async def balance( formatted_balance_data = json.dumps(balance_data, indent=4, default=extended_json_encoder) typer.echo(formatted_balance_data) else: - typer.echo( - f"Failed to retrieve balance for address {address}. Status code: {response.status}" - ) + typer.echo(f"Failed to retrieve balance for address {address}. Status code: {response.status}") else: - typer.echo( - "Error: Please provide either a private key, private key file, or an address." - ) + typer.echo("Error: Please provide either a private key, private key file, or an address.") diff --git a/src/aleph_client/commands/aggregate.py b/src/aleph_client/commands/aggregate.py index 0db9d39c..dd5d8a6e 100644 --- a/src/aleph_client/commands/aggregate.py +++ b/src/aleph_client/commands/aggregate.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json from pathlib import Path from typing import Optional @@ -9,7 +11,7 @@ from aleph.sdk.query.filters import MessageFilter from aleph.sdk.types import AccountFromPrivateKey from aleph.sdk.utils import extended_json_encoder -from aleph_message.models import MessageType +from aleph_message.models.base import MessageType from aleph_client.commands import help_strings from aleph_client.commands.utils import setup_logging @@ -21,16 +23,10 @@ @app.command() async def forget( key: str = typer.Argument(..., help="Aggregate item hash to be removed."), - reason: Optional[str] = typer.Option( - None, help="A description of why the messages are being forgotten" - ), + reason: Optional[str] = typer.Option(None, help="A description of why the messages are being forgotten"), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Forget all the messages composing an aggregate.""" @@ -39,17 +35,15 @@ async def forget( account: AccountFromPrivateKey = _load_account(private_key, private_key_file) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: message_response = await client.get_messages( message_filter=MessageFilter( addresses=[account.get_address()], - message_types=[MessageType.aggregate.value], + message_types=[MessageType.aggregate], content_keys=[key], ) ) - hash_list = [message["item_hash"] for message in message_response.messages] + hash_list = [message.item_hash for message in message_response.messages] await client.forget(hashes=hash_list, reason=reason, channel=channel) @@ -57,19 +51,13 @@ async def forget( @app.command() async def post( key: str = typer.Argument(..., help="Aggregate key to be created."), - content: str = typer.Argument( - ..., help="Aggregate content (ex : {'c': 3, 'd': 4})" - ), + content: str = typer.Argument(..., help="Aggregate content (ex : {'c': 3, 'd': 4})"), address: Optional[str] = typer.Option(default=None, help="address"), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - inline: Optional[bool] = typer.Option(False, help="inline"), - sync: Optional[bool] = typer.Option(False, help="Sync response"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + inline: bool = typer.Option(False, help="inline"), + sync: bool = typer.Option(False, help="Sync response"), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Create or Update aggregate""" @@ -84,9 +72,7 @@ async def post( typer.echo("Invalid JSON for content. Please provide valid JSON.") raise typer.Exit(1) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: message, _ = await client.create_aggregate( key=key, content=content_dict, @@ -103,12 +89,8 @@ async def post( async def get( key: str = typer.Argument(..., help="Aggregate key to be fetched."), address: Optional[str] = typer.Option(default=None, help="Address"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Fetch an aggregate by key and content.""" @@ -120,9 +102,7 @@ async def get( # if no address we load current account as a private key address = account.get_address() if address is None else address - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: aggregates = await client.fetch_aggregate(address=address, key=key) if aggregates: diff --git a/src/aleph_client/commands/domain.py b/src/aleph_client/commands/domain.py index ba0b47a1..47062d7e 100644 --- a/src/aleph_client/commands/domain.py +++ b/src/aleph_client/commands/domain.py @@ -1,6 +1,8 @@ +from __future__ import annotations + from pathlib import Path from time import sleep -from typing import Optional, cast +from typing import Dict, Optional, cast import typer from aleph.sdk.account import _load_account @@ -16,14 +18,16 @@ from aleph.sdk.exceptions import DomainConfigurationError from aleph.sdk.query.filters import MessageFilter from aleph.sdk.types import AccountFromPrivateKey -from aleph_client.commands import help_strings -from aleph_client.commands.utils import is_environment_interactive -from aleph_client.utils import AsyncTyper -from aleph_message.models import AggregateMessage, MessageType +from aleph_message.models import AggregateMessage +from aleph_message.models.base import MessageType from rich.console import Console from rich.prompt import Confirm, Prompt from rich.table import Table +from aleph_client.commands import help_strings +from aleph_client.commands.utils import is_environment_interactive +from aleph_client.utils import AsyncTyper + app = AsyncTyper(no_args_is_help=True) @@ -101,22 +105,24 @@ async def attach_resource( if (not interactive) or Confirm.ask("Continue"): """Create aggregate message""" - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: + + options: Optional[Dict] = None + if catch_all_path and catch_all_path.startswith("/"): + options = {"catch_all_path": catch_all_path} + aggregate_content = { - fqdn: { + str(fqdn): { "message_id": item_hash, "type": resource_type, # console page compatibility - "programType": resource_type + "programType": resource_type, + "options": options, } } if catch_all_path and catch_all_path.startswith("/"): - aggregate_content[fqdn]["options"] = { - "catch_all_path": catch_all_path - } + aggregate_content[fqdn]["options"] = {"catch_all_path": catch_all_path} aggregate_message, message_status = await client.create_aggregate( key="domains", content=aggregate_content, channel="ALEPH-CLOUDSOLUTIONS" @@ -128,9 +134,7 @@ async def attach_resource( ) -async def detach_resource( - account: AccountFromPrivateKey, fqdn: Hostname, interactive: Optional[bool] = None -): +async def detach_resource(account: AccountFromPrivateKey, fqdn: Hostname, interactive: Optional[bool] = None): domain_info = await get_aggregate_domain_info(account, fqdn) interactive = is_environment_interactive() if interactive is None else interactive @@ -147,19 +151,15 @@ async def detach_resource( current_resource = "null" resource_type = await get_target_type(fqdn) - table.add_row( - f"{current_resource[:16]}...{current_resource[-16:]}", "", resource_type - ) + table.add_row(f"{current_resource[:16]}...{current_resource[-16:]}", "", resource_type) console.print(table) if (not interactive) or Confirm.ask("Continue"): """Update aggregate message""" - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: - aggregate_content = {fqdn: None} + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: + aggregate_content = {str(fqdn): None} aggregate_message, message_status = await client.create_aggregate( key="domains", content=aggregate_content, channel="ALEPH-CLOUDSOLUTIONS" @@ -173,22 +173,12 @@ async def detach_resource( @app.command() async def add( - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), fqdn: str = typer.Argument(..., help=help_strings.CUSTOM_DOMAIN_NAME), - target: Optional[TargetType] = typer.Option( - None, help=help_strings.CUSTOM_DOMAIN_TARGET_TYPES - ), - item_hash: Optional[str] = typer.Option( - None, help=help_strings.CUSTOM_DOMAIN_ITEM_HASH - ), - owner: Optional[str] = typer.Option( - None, help=help_strings.CUSTOM_DOMAIN_OWNER_ADDRESS - ), + target: Optional[TargetType] = typer.Option(None, help=help_strings.CUSTOM_DOMAIN_TARGET_TYPES), + item_hash: Optional[str] = typer.Option(None, help=help_strings.CUSTOM_DOMAIN_ITEM_HASH), + owner: Optional[str] = typer.Option(None, help=help_strings.CUSTOM_DOMAIN_OWNER_ADDRESS), ask: bool = typer.Option(default=True, help=help_strings.ASK_FOR_CONFIRMATION), ): """Add and link a Custom Domain.""" @@ -199,11 +189,14 @@ async def add( domain_validator = DomainValidator() fqdn = hostname_from_url(fqdn) - if target is None: - target = Prompt.ask( - "Select a target resource type", - choices=[TargetType.IPFS, TargetType.PROGRAM, TargetType.INSTANCE], + while target is None: + target = TargetType( + Prompt.ask( + "Select a target resource type", + choices=[TargetType.IPFS, TargetType.PROGRAM, TargetType.INSTANCE], + ) ) + selected_target: TargetType = target table = Table(title=f"Required DNS entries for: {fqdn}") @@ -213,11 +206,9 @@ async def add( table.add_column("DNS VALUE", justify="right", style="green") owner = owner or account.get_address() - dns_rules = domain_validator.get_required_dns_rules(fqdn, target, owner) + dns_rules = domain_validator.get_required_dns_rules(fqdn, selected_target, owner) for rule_id, rule in enumerate(dns_rules): - table.add_row( - str(rule_id), rule.dns["type"], rule.dns["name"], rule.dns["value"] - ) + table.add_row(str(rule_id), rule.dns["type"], rule.dns["name"], rule.dns["value"]) console.print(table) @@ -266,16 +257,10 @@ async def add( @app.command() async def attach( - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), fqdn: str = typer.Argument(..., help=help_strings.CUSTOM_DOMAIN_NAME), - item_hash: Optional[str] = typer.Option( - None, help=help_strings.CUSTOM_DOMAIN_ITEM_HASH - ), + item_hash: Optional[str] = typer.Option(None, help=help_strings.CUSTOM_DOMAIN_ITEM_HASH), catch_all_path: str = typer.Option(default=None, help=help_strings.IPFS_CATCH_ALL_PATH), ask: bool = typer.Option(default=True, help=help_strings.ASK_FOR_CONFIRMATION), ): @@ -283,39 +268,33 @@ async def attach( account: AccountFromPrivateKey = _load_account(private_key, private_key_file) await attach_resource( - account, Hostname(fqdn), item_hash, interactive=False if (not ask) else None, catch_all_path=catch_all_path + account, + Hostname(fqdn), + item_hash, + interactive=False if (not ask) else None, + catch_all_path=catch_all_path, ) raise typer.Exit() @app.command() async def detach( - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), fqdn: str = typer.Argument(..., help=help_strings.CUSTOM_DOMAIN_NAME), ask: bool = typer.Option(default=True, help=help_strings.ASK_FOR_CONFIRMATION), ): """Unlink Custom Domain.""" account: AccountFromPrivateKey = _load_account(private_key, private_key_file) - await detach_resource( - account, Hostname(fqdn), interactive=False if (not ask) else None - ) + await detach_resource(account, Hostname(fqdn), interactive=False if (not ask) else None) raise typer.Exit() @app.command() async def info( - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), fqdn: str = typer.Argument(..., help=help_strings.CUSTOM_DOMAIN_NAME), ): """Show Custom Domain Details.""" diff --git a/src/aleph_client/commands/files.py b/src/aleph_client/commands/files.py index a698989e..53520e43 100644 --- a/src/aleph_client/commands/files.py +++ b/src/aleph_client/commands/files.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json as json_lib import logging from datetime import datetime @@ -10,7 +12,7 @@ from aleph.sdk.account import _load_account from aleph.sdk.conf import settings as sdk_settings from aleph.sdk.types import AccountFromPrivateKey, StorageEnum -from aleph_message.models import StoreMessage +from aleph_message.models import ItemHash, StoreMessage from aleph_message.status import MessageStatus from pydantic import BaseModel, Field from rich import box @@ -29,12 +31,8 @@ async def pin( item_hash: str = typer.Argument(..., help="IPFS hash to pin on aleph.im"), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), ref: Optional[str] = typer.Option(None, help=help_strings.REF), debug: bool = False, ): @@ -44,9 +42,7 @@ async def pin( account: AccountFromPrivateKey = _load_account(private_key, private_key_file) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: result: StoreMessage status: MessageStatus result, status = await client.create_store( @@ -63,12 +59,8 @@ async def pin( async def upload( path: Path = typer.Argument(..., help="Path of the file to upload"), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), ref: Optional[str] = typer.Option(None, help=help_strings.REF), debug: bool = False, ): @@ -78,9 +70,7 @@ async def upload( account: AccountFromPrivateKey = _load_account(private_key, private_key_file) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: if not path.is_file(): typer.echo(f"Error: File not found: '{path}'") raise typer.Exit(code=1) @@ -89,11 +79,7 @@ async def upload( logger.debug("Reading file") # TODO: Read in lazy mode instead of copying everything in memory file_content = fd.read() - storage_engine = ( - StorageEnum.ipfs - if len(file_content) > 4 * 1024 * 1024 - else StorageEnum.storage - ) + storage_engine = StorageEnum.ipfs if len(file_content) > 4 * 1024 * 1024 else StorageEnum.storage logger.debug("Uploading file") result: StoreMessage status: MessageStatus @@ -111,9 +97,7 @@ async def upload( @app.command() async def download( hash: str = typer.Argument(..., help="hash to download from aleph."), - use_ipfs: bool = typer.Option( - default=False, help="Download using IPFS instead of storage" - ), + use_ipfs: bool = typer.Option(default=False, help="Download using IPFS instead of storage"), output_path: Path = typer.Option(Path("."), help="Output directory path"), file_name: str = typer.Option(None, help="Output file name (without extension)"), file_extension: str = typer.Option(None, help="Output file extension"), @@ -146,12 +130,8 @@ async def forget( item_hash: str = typer.Argument(..., help="Hash to forget"), reason: str = typer.Argument(..., help="reason to forget"), channel: Optional[str] = typer.Option(None, help="channel"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """forget a file and his message on aleph.im.""" @@ -160,10 +140,8 @@ async def forget( account: AccountFromPrivateKey = _load_account(private_key, private_key_file) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: - value = await client.forget(hashes=[item_hash], reason=reason, channel=channel) + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: + value = await client.forget(hashes=[ItemHash(item_hash)], reason=reason, channel=channel) typer.echo(f"{value[0].json(indent=4)}") @@ -229,21 +207,15 @@ def _show_files(files_data: dict) -> None: @app.command() async def list( address: Optional[str] = typer.Option(None, help="Address"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), pagination: int = typer.Option(100, help="Maximum number of files to return."), page: int = typer.Option(1, help="Offset in pages."), sort_order: int = typer.Option( -1, help="Order in which files should be listed: -1 means most recent messages first, 1 means older messages first.", ), - json: bool = typer.Option( - default=False, help="Print as json instead of rich table" - ), + json: bool = typer.Option(default=False, help="Print as json instead of rich table"), ): """List all files for a given address""" account: AccountFromPrivateKey = _load_account(private_key, private_key_file) @@ -253,9 +225,7 @@ async def list( if address: # Build the query parameters - query_params = GetAccountFilesQueryParams( - pagination=pagination, page=page, sort_order=sort_order - ) + query_params = GetAccountFilesQueryParams(pagination=pagination, page=page, sort_order=sort_order) uri = f"{sdk_settings.API_HOST}/api/v0/addresses/{address}/files" async with aiohttp.ClientSession() as session: @@ -268,10 +238,6 @@ async def list( else: typer.echo(formatted_files_data) else: - typer.echo( - f"Failed to retrieve files for address {address}. Status code: {response.status}" - ) + typer.echo(f"Failed to retrieve files for address {address}. Status code: {response.status}") else: - typer.echo( - "Error: Please provide either a private key, private key file, or an address." - ) + typer.echo("Error: Please provide either a private key, private key file, or an address.") diff --git a/src/aleph_client/commands/instance.py b/src/aleph_client/commands/instance/__init__.py similarity index 84% rename from src/aleph_client/commands/instance.py rename to src/aleph_client/commands/instance/__init__.py index 84fdb755..3b0bf756 100644 --- a/src/aleph_client/commands/instance.py +++ b/src/aleph_client/commands/instance/__init__.py @@ -1,7 +1,9 @@ +from __future__ import annotations + import asyncio import logging from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import typer from aiohttp import ClientResponseError, ClientSession @@ -23,7 +25,9 @@ from rich.table import Table from aleph_client.commands import help_strings +from aleph_client.commands.node import NodeInfo, fetch_nodes from aleph_client.commands.utils import ( + colorful_message_json, get_or_prompt_volumes, setup_logging, validated_int_prompt, @@ -156,7 +160,10 @@ def validate_ssh_pubkey_file(file: Union[str, Path]) -> Path: ) memory = validated_int_prompt( - f"Maximum memory allocation on vm in MiB", memory, min_value=2000, max_value=8000 + f"Maximum memory allocation on vm in MiB", + memory, + min_value=2000, + max_value=8000, ) rootfs_size = validated_int_prompt( @@ -247,7 +254,7 @@ async def delete( typer.echo("You are not the owner of this instance") raise typer.Exit(code=1) - message, status = await client.forget(hashes=[item_hash], reason=reason) + message, status = await client.forget(hashes=[ItemHash(item_hash)], reason=reason) if print_message: typer.echo(f"{message.json(indent=4)}") @@ -256,20 +263,43 @@ async def delete( ) -async def _get_ipv6_address(message: InstanceMessage) -> Tuple[str, str]: +async def fetch_json(session: ClientSession, url: str) -> dict: + async with session.get(url) as resp: + resp.raise_for_status() + return await resp.json() + + +async def _get_ipv6_address( + message: InstanceMessage, node_list: NodeInfo +) -> Tuple[str, str]: async with ClientSession() as session: try: - resp = await session.get( - f"https://scheduler.api.aleph.cloud/api/v0/allocation/{message.item_hash}" - ) - resp.raise_for_status() - status = await resp.json() - return status["vm_hash"], status["vm_ipv6"] + if not message.content.payment: + # Fetch from the scheduler API directly if no payment + status = await fetch_json( + session, + f"https://scheduler.api.aleph.cloud/api/v0/allocation/{message.item_hash}", + ) + return status["vm_hash"], status["vm_ipv6"] + for node in node_list.nodes: + if node["stream_reward"] == message.content.payment.receiver: + + # Fetch from the CRN API if payment + executions = await fetch_json( + session, f"{node['address']}about/executions/list" + ) + if message.item_hash in executions: + ipv6_address = executions[message.item_hash]["networking"][ + "ipv6" + ] + return message.item_hash, ipv6_address + + return message.item_hash, "Not available (yet)" except ClientResponseError: return message.item_hash, "Not available (yet)" -async def _show_instances(messages: List[InstanceMessage]): +async def _show_instances(messages: List[InstanceMessage], node_list: NodeInfo): table = Table(box=box.SIMPLE_HEAVY) table.add_column("Item Hash", style="cyan") table.add_column("Vcpus", style="magenta") @@ -278,7 +308,9 @@ async def _show_instances(messages: List[InstanceMessage]): table.add_column("IPv6 address", style="yellow") scheduler_responses = dict( - await asyncio.gather(*[_get_ipv6_address(message) for message in messages]) + await asyncio.gather( + *[_get_ipv6_address(message, node_list) for message in messages] + ) ) for message in messages: @@ -330,4 +362,5 @@ async def list( if json: typer.echo(resp.json(indent=4)) else: - await _show_instances(resp.messages) + resource_nodes: NodeInfo = await fetch_nodes() + await _show_instances(resp.messages, resource_nodes) diff --git a/src/aleph_client/commands/instance/display.py b/src/aleph_client/commands/instance/display.py new file mode 100644 index 00000000..ec6a1b92 --- /dev/null +++ b/src/aleph_client/commands/instance/display.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import asyncio +import typing +from functools import partial +from typing import Callable, Literal, Set, Sized, Tuple, Union, cast + +from rich.console import Console +from rich.live import Live +from rich.progress import Progress +from rich.table import Table + +from aleph_client.commands.instance.network import fetch_and_queue_crn_info +from aleph_client.commands.node import NodeInfo, _fetch_nodes, _format_score +from aleph_client.models import MachineInfo + +if typing.TYPE_CHECKING: + from aleph_client.commands.instance.network import MachineInfoQueue + + +class ProgressTable: + """Display a progress bar and a table side by side.""" + + progress: Progress + table: Table + + def __init__(self, progress, table): + self.progress = progress + self.table = table + + def __rich_console__(self, console: Console, options): + yield self.progress + yield self.table + + +def create_crn_resource_table() -> Table: + """Prepare a table to display available resources on CRNs in order to schedule a PAYG instance on it.""" + table = Table(title="Compute Node Information") + table.add_column("Score", style="green", no_wrap=True, justify="center") + table.add_column("Name", style="#029AFF", justify="left") + table.add_column("Cores", style="green", justify="left") + table.add_column("RAM", style="green", justify="left") + table.add_column("HDD", style="green", justify="left") + table.add_column("Version", style="green", justify="center") + table.add_column("Reward Address", style="green", justify="center") + table.add_column("Address", style="green", justify="center") + return table + + +def create_progress_bar(sized_object: Sized) -> Tuple[Progress, Callable[[], None]]: + """Create a progress bar and a function to increment it. + + Args: + sized_object: Sized object to create a progress bar for, typically a list. + Returns: + The progress bar and a function to increment it. + """ + progress_bar = Progress() + progress_bar_task = progress_bar.add_task( + "[green]Fetching node info... It might take some time", + total=len(sized_object), + ) + # We use a partial function to create a function that increments the progress bar by 1 + # and can be called from within coroutines. + increment_progress_bar: Callable[[], None] = partial(progress_bar.update, progress_bar_task, advance=1) + return progress_bar, increment_progress_bar + + +def create_table_with_progress_bar( + sized_object: Sized, +) -> Tuple[ProgressTable, Callable[[], None]]: + """Create a table of CRNs together with a progress bar and a function to increment it. + + Args: + sized_object: Sized object to create a progress bar for, typically a list. + Returns: + The table and the function to increment the progress bar. + """ + progress_bar, increment_progress_bar = create_progress_bar(sized_object) + table = create_crn_resource_table() + return ProgressTable(progress_bar, table), increment_progress_bar + + +async def update_table( + queue: MachineInfoQueue, + table: Table, + increment_progress_bar: Callable[[], None], + valid_reward_addresses: Set[str], +) -> None: + """ + Updates table with MachineInfo objects from the queue, updates progress bar and valid reward addresses. + + Args: + queue: Asyncio queue that provides MachineInfo objects. + table: Rich Table object of CRN resources. + increment_progress_bar: Function to increment progress bar. + valid_reward_addresses: Set of valid reward addresses to update. + """ + while True: + data: Union[MachineInfo, None, Literal["END_OF_QUEUE"]] = await queue.get() + increment_progress_bar() + if data is None: + continue + elif data == "END_OF_QUEUE": + break + else: + data = cast(MachineInfo, data) + cpu, hdd, ram = convert_system_info_to_str(data) + table.add_row( + _format_score(data.score), + data.name, + cpu, + ram, + hdd, + data.version, + data.reward_address, + data.url, + ) + valid_reward_addresses.add(data.reward_address) + + +def convert_system_info_to_str(data: MachineInfo) -> Tuple[str, str, str]: + """ + Converts MachineInfo object that contains CPU, RAM and HDD information to a tuple of strings. + + Args: + data: Information obtained about the CRN. + + Returns: + CPU, RAM, and HDD information as strings. + """ + cpu: str = f"{data.machine_usage.cpu.count}" + hdd: str = f"{data.machine_usage.disk.available_kB / 1_000_000:.2f} GB" + ram: str = f"{data.machine_usage.mem.available_kB / 1_000_000:.2f} GB" + + return cpu, hdd, ram + + +async def fetch_crn_info() -> set: + """ + Fetches compute node information asynchronously. + Display and update a Live tab where CRN info will be displayed + + Returns: + List of valid reward addresses. + """ + + # Fetch node information from the API + node_info: NodeInfo = await _fetch_nodes() + + # Create the console and progress table + console = Console() + progress_table, increment_progress_bar = create_table_with_progress_bar(node_info.nodes) + valid_reward_addresses: Set[str] = set() + + # We use a queue in order to store retrieved data from the nodes not in order + queue: MachineInfoQueue = asyncio.Queue() + + # The Live context manager allows us to update the table and progress bar in real time + with Live(progress_table, console=console, refresh_per_second=2): + await asyncio.gather( + # Fetch CRN info from the nodes into the queue + fetch_and_queue_crn_info(node_info, queue), + # Update the table with the CRN info from the queue in parallel + update_table( + queue, + progress_table.table, + increment_progress_bar, + valid_reward_addresses, + ), + ) + + return valid_reward_addresses diff --git a/src/aleph_client/commands/instance/network.py b/src/aleph_client/commands/instance/network.py new file mode 100644 index 00000000..d2d3cb98 --- /dev/null +++ b/src/aleph_client/commands/instance/network.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import asyncio +import logging +import re +import typing +from json import JSONDecodeError +from typing import List, Literal, Optional, Tuple, Union +from urllib.parse import ParseResult, urlparse + +import aiohttp +from aiohttp import InvalidURL +from multidict import CIMultiDictProxy +from pydantic import ValidationError + +from aleph_client.commands.node import NodeInfo +from aleph_client.conf import settings +from aleph_client.models import MachineInfo, MachineUsage + +logger = logging.getLogger(__name__) + +# Some users had fun adding URLs that are obviously not CRNs. +# If you work for one of these companies, please send a large check to the Aleph team, +# and we may consider removing your domain from the blacklist. Or just use a subdomain. +FORBIDDEN_HOSTS = [ + "amazon.com", + "apple.com", + "facebook.com", + "google.com", + "google.es", + "microsoft.com", + "openai.com", + "twitter.com", + "x.com", + "youtube.com", +] + +# This type annotation is hidden behind typing.TYPE_CHECKING for compatibility with Python 3.8 +if typing.TYPE_CHECKING: + # A queue is used to pass the machine info from the coroutines that fetch it to the + # coroutine in charge of updating the table and progress bar. + # Invalid URLs are represented as None, and the end of the queue is marked with "END_OF_QUEUE". + MachineInfoQueue = asyncio.Queue[Union[MachineInfo, None, Literal["END_OF_QUEUE"]]] + + +def get_version(headers: CIMultiDictProxy[str]) -> Optional[str]: + """Extracts the version of the CRN from the headers of the response. + + Args: + headers: aiohttp response headers. + Returns: + Version of the CRN if found, None otherwise. + """ + if "Server" in headers: + for server in headers.getall("Server"): + version_match: List[str] = re.findall(r"^aleph-vm/(.*)$", server) + # Return the first match + if version_match and version_match[0]: + return version_match[0] + return None + + +async def fetch_crn_info(node_url: str) -> Tuple[Optional[MachineUsage], Optional[str]]: + """ + Fetches compute node usage information and version. + + Args: + node_url: URL of the compute node. + Returns: + Machine usage information and version. + """ + # Remove trailing slashes to avoid having // in the URL. + url: str = node_url.rstrip("/") + "/about/usage/system" + timeout = aiohttp.ClientTimeout(total=settings.HTTP_REQUEST_TIMEOUT) + try: + # A new session is created for each request since they each target a different host. + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(url) as resp: + resp.raise_for_status() + data_raw: dict = await resp.json() + version = get_version(resp.headers) + return MachineUsage.parse_obj(data_raw), version + except TimeoutError as e: + logger.debug(f"Timeout while fetching: {url}: {e}") + except aiohttp.ClientConnectionError as e: + logger.debug(f"Error on connection: {url}: {e}") + except aiohttp.ClientResponseError as e: + logger.debug(f"Error on response: {url}: {e}") + except JSONDecodeError as e: + logger.debug(f"Error decoding JSON: {url}: {e}") + except ValidationError as e: + logger.debug(f"Validation error when fetching: {url}: {e}") + except InvalidURL as e: + logger.debug(f"Invalid URL: {url}: {e}") + return None, None + + +def sanitize_url(url: str) -> str: + """Ensure that the URL is valid and not obviously irrelevant. + + Args: + url: URL to sanitize. + Returns: + Sanitized URL. + """ + if not url: + raise InvalidURL("Empty URL") + + # Use urllib3 to parse the URL. + # This should raise the same InvalidURL exception if the URL is invalid. + parsed_url: ParseResult = urlparse(url) + + if parsed_url.scheme not in ["http", "https"]: + raise InvalidURL(f"Invalid URL scheme: {parsed_url.scheme}") + + if parsed_url.hostname in FORBIDDEN_HOSTS: + logger.debug( + f"Invalid URL {url} hostname {parsed_url.hostname} is in the forbidden host list " + f"({', '.join(FORBIDDEN_HOSTS)})" + ) + raise InvalidURL("Invalid URL host") + + return url + + +async def fetch_crn_info_in_queue(node: dict, queue: MachineInfoQueue) -> None: + """Fetch the resource usage from a CRN and put it in the queue + + Args: + node: Information about the CRN from the 'corechannel' aggregate. + queue: Queue used to update the table live. + """ + # Skip nodes without an address or with an invalid address + try: + node_url = sanitize_url(node["address"]) + except InvalidURL: + logger.info(f"Invalid URL: {node['address']}") + await queue.put(None) + return + + # Skip nodes without a reward address + if not node["stream_reward"]: + await queue.put(None) + return + + # Fetch the machine usage and version from its HTTP API + machine_usage, version = await fetch_crn_info(node_url) + + if not machine_usage: + await queue.put(None) + return + + await queue.put( + MachineInfo.from_unsanitized_input( + machine_usage=machine_usage, + score=node["score"], + name=node["name"], + version=version, + reward_address=node["stream_reward"], + url=node["address"], + ) + ) + + +async def fetch_and_queue_crn_info( + node_info: NodeInfo, + queue: MachineInfoQueue, +): + """Fetch the resource usage of all CRNs in the node_info asynchronously + and put them in the queue. + + Fetches the resource usage and version of each node in parallel using a separate coroutine. + + Args: + node_info: Information about all CRNs from the 'corechannel' aggregate. + queue: Queue used to update the table live. + """ + coroutines = [fetch_crn_info_in_queue(node, queue) for node in node_info.nodes] + await asyncio.gather(*coroutines) + await queue.put("END_OF_QUEUE") diff --git a/src/aleph_client/commands/message.py b/src/aleph_client/commands/message.py index f4b52898..157ef2ee 100644 --- a/src/aleph_client/commands/message.py +++ b/src/aleph_client/commands/message.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import asyncio import json import os.path @@ -16,6 +18,10 @@ from aleph.sdk.query.responses import MessagesResponse from aleph.sdk.types import AccountFromPrivateKey, StorageEnum from aleph.sdk.utils import extended_json_encoder +from aleph_message.models import AlephMessage, ProgramMessage +from aleph_message.models.base import MessageType +from aleph_message.models.item_hash import ItemHash + from aleph_client.commands import help_strings from aleph_client.commands.utils import ( colorful_json, @@ -25,7 +31,6 @@ str_to_datetime, ) from aleph_client.utils import AsyncTyper -from aleph_message.models import AlephMessage, ItemHash, MessageType, ProgramMessage app = AsyncTyper(no_args_is_help=True) @@ -35,7 +40,7 @@ async def get( item_hash: str, ): async with AlephHttpClient(api_server=sdk_settings.API_HOST) as client: - message = await client.get_message(item_hash=ItemHash(item_hash)) + message: AlephMessage = await client.get_message(item_hash=ItemHash(item_hash)) typer.echo(colorful_message_json(message)) @@ -67,9 +72,7 @@ async def find( parsed_chains = chains.split(",") if chains else None message_types = ( - [MessageType(message_type) for message_type in parsed_message_types] - if parsed_message_types - else None + [MessageType(message_type) for message_type in parsed_message_types] if parsed_message_types else None ) start_time = str_to_datetime(start_date) @@ -106,12 +109,8 @@ async def post( type: str = typer.Option("test", help="Text representing the message object type"), ref: Optional[str] = typer.Option(None, help=help_strings.REF), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Post a message on aleph.im.""" @@ -128,29 +127,21 @@ async def post( raise typer.Exit(code=1) file_size = os.path.getsize(path) - storage_engine = ( - StorageEnum.ipfs if file_size > 4 * 1024 * 1024 else StorageEnum.storage - ) + storage_engine = StorageEnum.ipfs if file_size > 4 * 1024 * 1024 else StorageEnum.storage with open(path, "r") as fd: content = json.load(fd) else: content_raw = input_multiline() - storage_engine = ( - StorageEnum.ipfs - if len(content_raw) > 4 * 1024 * 1024 - else StorageEnum.storage - ) + storage_engine = StorageEnum.ipfs if len(content_raw) > 4 * 1024 * 1024 else StorageEnum.storage try: content = json.loads(content_raw) except json.decoder.JSONDecodeError: typer.echo("Not valid JSON") raise typer.Exit(code=2) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: result, status = await client.create_post( post_content=content, post_type=type, @@ -166,12 +157,8 @@ async def post( @app.command() async def amend( item_hash: str = typer.Argument(..., help="Hash reference of the message to amend"), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Amend an existing aleph.im message.""" @@ -209,9 +196,7 @@ async def amend( new_content.type = "amend" typer.echo(new_content) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: message, status, response = await client.submit( content=new_content.dict(), message_type=existing_message.type, @@ -222,31 +207,21 @@ async def amend( @app.command() async def forget( - hashes: str = typer.Argument( - ..., help="Comma separated list of hash references of messages to forget" - ), - reason: Optional[str] = typer.Option( - None, help="A description of why the messages are being forgotten." - ), + hashes: str = typer.Argument(..., help="Comma separated list of hash references of messages to forget"), + reason: Optional[str] = typer.Option(None, help="A description of why the messages are being forgotten."), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Forget an existing aleph.im message.""" setup_logging(debug) - hash_list: List[str] = hashes.split(",") + hash_list: List[ItemHash] = [ItemHash(h) for h in hashes.split(",")] account: AccountFromPrivateKey = _load_account(private_key, private_key_file) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: await client.forget(hashes=hash_list, reason=reason, channel=channel) @@ -263,9 +238,7 @@ async def watch( async with AlephHttpClient(api_server=sdk_settings.API_HOST) as client: original: AlephMessage = await client.get_message(item_hash=ref) async for message in client.watch_messages( - message_filter=MessageFilter( - refs=[ref], addresses=[original.content.address] - ) + message_filter=MessageFilter(refs=[ref], addresses=[original.content.address]) ): typer.echo(f"{message.json(indent=indent)}") @@ -273,12 +246,8 @@ async def watch( @app.command() def sign( message: Optional[str] = typer.Option(None, help=help_strings.SIGNABLE_MESSAGE), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), debug: bool = False, ): """Sign an aleph message with a private key. If no --message is provided, the message will be read from stdin.""" diff --git a/src/aleph_client/commands/node.py b/src/aleph_client/commands/node.py index 817b7806..1b5d4c33 100644 --- a/src/aleph_client/commands/node.py +++ b/src/aleph_client/commands/node.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import datetime import json as json_lib import logging import re import unicodedata -from typing import Dict, List, Optional +from typing import Optional import aiohttp import typer @@ -30,7 +32,7 @@ def __init__(self, **kwargs): self.core_node.sort(key=lambda x: x.get("score", 0), reverse=True) -async def _fetch_nodes() -> NodeInfo: +async def fetch_nodes() -> NodeInfo: """Fetch node aggregates and format it as NodeInfo""" async with aiohttp.ClientSession() as session: async with session.get(node_link) as resp: @@ -53,7 +55,7 @@ def _remove_ansi_escape(string: str) -> str: return ansi_escape.sub("", string) -def _format_score(score): +def _format_score(score: float) -> text.Text: if score < 0.5: return text.Text(f"{score:.2%}", style="red", justify="right") elif score < 0.75: @@ -75,12 +77,14 @@ def _show_compute(node_info): table.add_column("Creation Time", style="#029AFF", justify="center") table.add_column("Decentralization", style="green", justify="right") table.add_column("Status", style="green", justify="right") + table.add_column("Item Hash", style="green", justify="center") for node in node_info.nodes: # Prevent escaping with name node_name = node["name"] node_name = _escape_and_normalize(node_name) node_name = _remove_ansi_escape(node_name) + node_hash = node["hash"] # Format Value creation_time = datetime.datetime.fromtimestamp(node["time"]).strftime( @@ -89,13 +93,13 @@ def _show_compute(node_info): score = _format_score(node["score"]) decentralization = _format_score(node["decentralization"]) status = _format_status(node["status"]) - table.add_row( score, node_name, creation_time, decentralization, status, + node_hash, ) console = Console() @@ -164,7 +168,7 @@ async def compute( setup_logging(debug) - compute_info: NodeInfo = await _fetch_nodes() + compute_info: NodeInfo = await fetch_nodes() compute_info.nodes = _filter_node( core_info=compute_info.nodes, active=active, address=address ) @@ -189,7 +193,7 @@ async def core( """Get all core node on aleph""" setup_logging(debug) - core_info: NodeInfo = await _fetch_nodes() + core_info: NodeInfo = await fetch_nodes() core_info.core_node = _filter_node( core_info=core_info.core_node, active=active, address=address ) diff --git a/src/aleph_client/commands/program.py b/src/aleph_client/commands/program.py index 8eeb6907..74292bd2 100644 --- a/src/aleph_client/commands/program.py +++ b/src/aleph_client/commands/program.py @@ -1,16 +1,22 @@ +from __future__ import annotations + import json import logging from base64 import b16decode, b32encode from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Mapping, Optional from zipfile import BadZipFile import typer from aleph.sdk import AuthenticatedAlephHttpClient from aleph.sdk.account import _load_account from aleph.sdk.conf import settings as sdk_settings -from aleph.sdk.query.filters import MessageFilter from aleph.sdk.types import AccountFromPrivateKey, StorageEnum +from aleph_message.models import ProgramMessage, StoreMessage +from aleph_message.models.execution.program import ProgramContent +from aleph_message.models.item_hash import ItemHash +from aleph_message.status import MessageStatus + from aleph_client.commands import help_strings from aleph_client.commands.utils import ( get_or_prompt_volumes, @@ -20,14 +26,6 @@ ) from aleph_client.conf import settings from aleph_client.utils import AsyncTyper, create_archive -from aleph_message.models import ( - ItemHash, - MessagesResponse, - ProgramContent, - ProgramMessage, - StoreMessage, -) -from aleph_message.status import MessageStatus logger = logging.getLogger(__name__) app = AsyncTyper(no_args_is_help=True) @@ -38,22 +36,14 @@ async def upload( path: Path = typer.Argument(..., help="Path to your source code"), entrypoint: str = typer.Argument(..., help="Your program entrypoint"), channel: Optional[str] = typer.Option(default=None, help=help_strings.CHANNEL), - memory: int = typer.Option( - sdk_settings.DEFAULT_VM_MEMORY, help="Maximum memory allocation on vm in MiB" - ), - vcpus: int = typer.Option( - sdk_settings.DEFAULT_VM_VCPUS, help="Number of virtual cpus to allocate." - ), + memory: int = typer.Option(sdk_settings.DEFAULT_VM_MEMORY, help="Maximum memory allocation on vm in MiB"), + vcpus: int = typer.Option(sdk_settings.DEFAULT_VM_VCPUS, help="Number of virtual cpus to allocate."), timeout_seconds: float = typer.Option( sdk_settings.DEFAULT_VM_TIMEOUT, help="If vm is not called after [timeout_seconds] it will shutdown", ), - private_key: Optional[str] = typer.Option( - sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY - ), - private_key_file: Optional[Path] = typer.Option( - sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE - ), + private_key: Optional[str] = typer.Option(sdk_settings.PRIVATE_KEY_STRING, help=help_strings.PRIVATE_KEY), + private_key_file: Optional[Path] = typer.Option(sdk_settings.PRIVATE_KEY_FILE, help=help_strings.PRIVATE_KEY_FILE), print_messages: bool = typer.Option(False), print_code_message: bool = typer.Option(False), print_program_message: bool = typer.Option(False), @@ -67,12 +57,8 @@ async def upload( ), debug: bool = False, persistent: bool = False, - persistent_volume: Optional[List[str]] = typer.Option( - None, help=help_strings.PERSISTENT_VOLUME - ), - ephemeral_volume: Optional[List[str]] = typer.Option( - None, help=help_strings.EPHEMERAL_VOLUME - ), + persistent_volume: Optional[List[str]] = typer.Option(None, help=help_strings.PERSISTENT_VOLUME), + ephemeral_volume: Optional[List[str]] = typer.Option(None, help=help_strings.EPHEMERAL_VOLUME), immutable_volume: Optional[List[str]] = typer.Option( None, help=help_strings.IMMUATABLE_VOLUME, @@ -96,9 +82,7 @@ async def upload( account: AccountFromPrivateKey = _load_account(private_key, private_key_file) runtime = ( - runtime - or input(f"Ref of runtime ? [{sdk_settings.DEFAULT_RUNTIME_ID}] ") - or sdk_settings.DEFAULT_RUNTIME_ID + runtime or input(f"Ref of runtime ? [{sdk_settings.DEFAULT_RUNTIME_ID}] ") or sdk_settings.DEFAULT_RUNTIME_ID ) volumes = get_or_prompt_volumes( @@ -107,7 +91,7 @@ async def upload( immutable_volume=immutable_volume, ) - subscriptions: Optional[List[Dict]] + subscriptions: Optional[List[Mapping]] = None if beta and yes_no_input("Subscribe to messages ?", default=False): content_raw = input_multiline() try: @@ -118,19 +102,13 @@ async def upload( else: subscriptions = None - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: # Upload the source code with open(path_object, "rb") as fd: logger.debug("Reading file") # TODO: Read in lazy mode instead of copying everything in memory file_content = fd.read() - storage_engine = ( - StorageEnum.ipfs - if len(file_content) > 4 * 1024 * 1024 - else StorageEnum.storage - ) + storage_engine = StorageEnum.ipfs if len(file_content) > 4 * 1024 * 1024 else StorageEnum.storage logger.debug("Uploading file") user_code: StoreMessage status: MessageStatus @@ -166,9 +144,7 @@ async def upload( typer.echo(f"{message.json(indent=4)}") item_hash: ItemHash = message.item_hash - hash_base32 = ( - b32encode(b16decode(item_hash.upper())).strip(b"=").lower().decode() - ) + hash_base32 = b32encode(b16decode(item_hash.upper())).strip(b"=").lower().decode() typer.echo( f"Your program has been uploaded on aleph.im\n\n" @@ -196,16 +172,10 @@ async def update( account = _load_account(private_key, private_key_file) path = path.absolute() - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: - program_message: ProgramMessage = client.get_message( - item_hash=item_hash, message_type=ProgramMessage - ) + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: + program_message: ProgramMessage = await client.get_message(item_hash=item_hash, message_type=ProgramMessage) code_ref = program_message.content.code.ref - code_message: StoreMessage = client.get_message( - item_hash=code_ref, message_type=StoreMessage - ) + code_message: StoreMessage = await client.get_message(item_hash=code_ref, message_type=StoreMessage) try: path, encoding = create_archive(path) @@ -229,9 +199,10 @@ async def update( # TODO: Read in lazy mode instead of copying everything in memory file_content = fd.read() logger.debug("Uploading file") - message, status = client.create_store( + message: StoreMessage + message, status = await client.create_store( file_content=file_content, - storage_engine=code_message.content.item_type, + storage_engine=StorageEnum(code_message.content.item_type), channel=code_message.channel, guess_mime_type=True, ref=code_message.item_hash, @@ -254,15 +225,8 @@ async def unpersist( account = _load_account(private_key, private_key_file) - async with AuthenticatedAlephHttpClient( - account=account, api_server=sdk_settings.API_HOST - ) as client: - existing: MessagesResponse = await client.get_messages( - message_filter=MessageFilter( - hashes=[item_hash] - ) - ) - message: ProgramMessage = existing.messages[0] + async with AuthenticatedAlephHttpClient(account=account, api_server=sdk_settings.API_HOST) as client: + message: ProgramMessage = await client.get_message(item_hash=item_hash, message_type=ProgramMessage) content: ProgramContent = message.content.copy() content.on.persistent = False diff --git a/src/aleph_client/commands/utils.py b/src/aleph_client/commands/utils.py index 7c38082d..a016b39b 100644 --- a/src/aleph_client/commands/utils.py +++ b/src/aleph_client/commands/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import os import sys @@ -12,6 +14,8 @@ from rich.prompt import IntPrompt, Prompt, PromptError from typer import echo +logger = logging.getLogger(__name__) + def colorful_json(obj: str): """Render a JSON string with colors.""" @@ -96,11 +100,7 @@ def volume_to_dict(volume: List[str]) -> Optional[Dict[str, Union[str, int]]]: def get_or_prompt_volumes(ephemeral_volume, immutable_volume, persistent_volume): volumes = [] # Check if the volumes are empty - if ( - persistent_volume is None - or ephemeral_volume is None - or immutable_volume is None - ): + if persistent_volume is None or ephemeral_volume is None or immutable_volume is None: for volume in prompt_for_volumes(): volumes.append(volume) typer.echo("\n") diff --git a/src/aleph_client/conf.py b/src/aleph_client/conf.py index 36953ae7..7129a612 100644 --- a/src/aleph_client/conf.py +++ b/src/aleph_client/conf.py @@ -26,21 +26,13 @@ class Settings(BaseSettings): ADDRESS_TO_USE: Optional[str] = None DEFAULT_CHANNEL: str = "TEST" - DEFAULT_RUNTIME_ID: str = ( - "63f07193e6ee9d207b7d1fcf8286f9aee34e6f12f101d2ec77c1229f92964696" - ) - DEBIAN_11_ROOTFS_ID: str = ( - "887957042bb0e360da3485ed33175882ce72a70d79f1ba599400ff4802b7cee7" - ) - DEBIAN_12_ROOTFS_ID: str = ( - "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595" - ) - UBUNTU_22_ROOTFS_ID: str = ( - "77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27" - ) + DEFAULT_RUNTIME_ID: str = "63f07193e6ee9d207b7d1fcf8286f9aee34e6f12f101d2ec77c1229f92964696" + DEBIAN_11_ROOTFS_ID: str = "887957042bb0e360da3485ed33175882ce72a70d79f1ba599400ff4802b7cee7" + DEBIAN_12_ROOTFS_ID: str = "6e30de68c6cedfa6b45240c2b51e52495ac6fb1bd4b36457b3d5ca307594d595" + UBUNTU_22_ROOTFS_ID: str = "77fef271aa6ff9825efa3186ca2e715d19e7108279b817201c69c34cedc74c27" DEFAULT_ROOTFS_SIZE: int = 20_000 DEFAULT_INSTANCE_MEMORY: int = 2_048 - DEFAULT_HYPERVISOR: HypervisorType = HypervisorType.firecracker + DEFAULT_HYPERVISOR: HypervisorType = HypervisorType.qemu DEFAULT_VM_MEMORY: int = 128 DEFAULT_VM_VCPUS: int = 1 @@ -56,6 +48,8 @@ class Config: case_sensitive = False env_file = ".env" + HTTP_REQUEST_TIMEOUT = 5.0 + # Settings singleton settings = Settings() @@ -72,6 +66,4 @@ class Config: assert settings.CONFIG_HOME if str(settings.PRIVATE_KEY_FILE) == "ethereum.key": - settings.PRIVATE_KEY_FILE = Path( - settings.CONFIG_HOME, "private-keys", "ethereum.key" - ) + settings.PRIVATE_KEY_FILE = Path(settings.CONFIG_HOME, "private-keys", "ethereum.key") diff --git a/src/aleph_client/models.py b/src/aleph_client/models.py new file mode 100644 index 00000000..48c1eba4 --- /dev/null +++ b/src/aleph_client/models.py @@ -0,0 +1,95 @@ +from datetime import datetime +from typing import Optional + +from aleph_message.models.execution.environment import CpuProperties +from pydantic import BaseModel + +from aleph_client.commands.node import _escape_and_normalize, _remove_ansi_escape + +# This is a copy from aleph-vm + + +class LoadAverage(BaseModel): + load1: float + load5: float + load15: float + + +class CoreFrequencies(BaseModel): + min: float + max: float + + +class CpuUsage(BaseModel): + count: int + load_average: LoadAverage + core_frequencies: CoreFrequencies + + +class MemoryUsage(BaseModel): + total_kB: int + available_kB: int + + +class DiskUsage(BaseModel): + total_kB: int + available_kB: int + + +class UsagePeriod(BaseModel): + start_timestamp: datetime + duration_seconds: float + + +class MachineProperties(BaseModel): + cpu: CpuProperties + + +class MachineUsage(BaseModel): + cpu: CpuUsage + mem: MemoryUsage + disk: DiskUsage + period: UsagePeriod + properties: MachineProperties + active: bool = True + + +class MachineInfo(BaseModel): + machine_usage: MachineUsage + score: float + name: str + version: Optional[str] + reward_address: str + url: str + + @classmethod + def from_unsanitized_input( + cls, machine_usage: MachineUsage, score: float, name: str, version: Optional[str], reward_address: str, url: str + ) -> "MachineInfo": + """Create a MachineInfo instance from unsanitized input. + + User input from the account page or the API may contain malicious or unexpected data. + This method ensures that the input is sanitized before creating a MachineInfo object. + + Args: + machine_usage: MachineUsage object from the CRN API. + score: Score of the CRN. + name: Name of the CRN. + version: Version of the CRN. + reward_address: Reward address of the CRN. + url: URL of the CRN. + """ + node_name: str = _remove_ansi_escape(_escape_and_normalize(name)) + + # The version field is optional, so we need to handle it separately + raw_version: Optional[str] = version + version = _remove_ansi_escape(_escape_and_normalize(raw_version)) if raw_version else None + + return cls( + machine_usage=MachineUsage.parse_obj(machine_usage), + score=score, + name=node_name, + version=version, + reward_address=reward_address, + url=url, + ) diff --git a/src/aleph_client/utils.py b/src/aleph_client/utils.py index 5d367f7c..3aef21c0 100644 --- a/src/aleph_client/utils.py +++ b/src/aleph_client/utils.py @@ -10,8 +10,8 @@ import typer from aleph.sdk.types import GenericMessage -from aleph_message.models import MessageType -from aleph_message.models.execution.program import Encoding +from aleph_message.models.base import MessageType +from aleph_message.models.execution.base import Encoding from aleph_client.conf import settings @@ -49,9 +49,7 @@ def create_archive(path: Path) -> Tuple[Path, Encoding]: archive_path = Path(f"{path}.zip") return archive_path, Encoding.zip elif os.path.isfile(path): - if path.suffix == ".squashfs" or ( - magic and magic.from_file(path).startswith("Squashfs filesystem") - ): + if path.suffix == ".squashfs" or (magic and magic.from_file(path).startswith("Squashfs filesystem")): return path, Encoding.squashfs else: try_open_zip(Path(path)) diff --git a/tests/unit/test_commands.py b/tests/unit/test_commands.py index 91ef8148..acc186a1 100644 --- a/tests/unit/test_commands.py +++ b/tests/unit/test_commands.py @@ -27,27 +27,21 @@ def get_test_message(account: ETHAccount): def test_account_create(account_file: Path): old_key = account_file.read_bytes() - result = runner.invoke( - app, ["account", "create", "--replace", "--private-key-file", str(account_file)] - ) + result = runner.invoke(app, ["account", "create", "--replace", "--private-key-file", str(account_file)]) assert result.exit_code == 0, result.stdout new_key = account_file.read_bytes() assert new_key != old_key def test_account_address(account_file: Path): - result = runner.invoke( - app, ["account", "address", "--private-key-file", str(account_file)] - ) + result = runner.invoke(app, ["account", "address", "--private-key-file", str(account_file)]) assert result.exit_code == 0 assert result.stdout.startswith("0x") assert len(result.stdout.strip()) == 42 def test_account_export_private_key(account_file: Path): - result = runner.invoke( - app, ["account", "export-private-key", "--private-key-file", str(account_file)] - ) + result = runner.invoke(app, ["account", "export-private-key", "--private-key-file", str(account_file)]) assert result.exit_code == 0 assert result.stdout.startswith("0x") assert len(result.stdout.strip()) == 66 @@ -89,10 +83,7 @@ def test_message_find(): ) assert result.exit_code == 0 assert "0x101d8D16372dBf5f1614adaE95Ee5CCE61998Fc9" in result.stdout - assert ( - "bd79839bf96e595a06da5ac0b6ba51dea6f7e2591bb913deccded04d831d29f4" - in result.stdout - ) + assert "bd79839bf96e595a06da5ac0b6ba51dea6f7e2591bb913deccded04d831d29f4" in result.stdout def test_post_message(account_file): diff --git a/tests/unit/test_instance.py b/tests/unit/test_instance.py new file mode 100644 index 00000000..9d8d2e6a --- /dev/null +++ b/tests/unit/test_instance.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import asyncio +import typing +from datetime import datetime, timezone +from typing import Set +from unittest import mock + +import pytest +from aiohttp import InvalidURL +from aleph_message.models.execution.environment import CpuProperties +from multidict import CIMultiDict, CIMultiDictProxy + +from aleph_client.commands.instance.display import ( + ProgressTable, + create_table_with_progress_bar, + update_table, +) +from aleph_client.commands.instance.network import ( + FORBIDDEN_HOSTS, + fetch_crn_info, + get_version, + sanitize_url, +) +from aleph_client.models import ( + CoreFrequencies, + CpuUsage, + DiskUsage, + LoadAverage, + MachineInfo, + MachineProperties, + MachineUsage, + MemoryUsage, + UsagePeriod, +) + +if typing.TYPE_CHECKING: + from aleph_client.commands.instance.network import MachineInfoQueue + + +def dummy_machine_info() -> MachineInfo: + """Create a dummy MachineInfo object for testing purposes.""" + return MachineInfo( + machine_usage=MachineUsage( + cpu=CpuUsage( + count=8, + load_average=LoadAverage(load1=0.5, load5=0.4, load15=0.3), + core_frequencies=CoreFrequencies(min=1.0, max=2.0), + ), + mem=MemoryUsage( + total_kB=1_000_000, + available_kB=500_000, + ), + disk=DiskUsage( + total_kB=1_000_000, + available_kB=500_000, + ), + period=UsagePeriod( + start_timestamp=datetime.now(tz=timezone.utc), + duration_seconds=60, + ), + properties=MachineProperties( + cpu=CpuProperties( + architecture="x86_64", + vendor="AuthenticAMD", + ), + ), + ), + score=0.5, + name="CRN", + version="0.0.1", + reward_address="0xcafecafecafecafecafecafecafecafecafecafecafecafecafecafecafecafe", + url="https://example.com", + ) + + +def dict_to_ci_multi_dict_proxy(d: dict) -> CIMultiDictProxy: + """Return a read-only proxy to a case-insensitive multi-dict created from a dict.""" + return CIMultiDictProxy(CIMultiDict(d)) + + +def test_get_version() -> None: + + # No server field in headers + headers = dict_to_ci_multi_dict_proxy({}) + assert get_version(headers) is None + + # Server header but no aleph-vm + headers = dict_to_ci_multi_dict_proxy({"Server": "nginx"}) + assert get_version(headers) is None + + # Server header with aleph-vm + headers = dict_to_ci_multi_dict_proxy({"Server": "aleph-vm/0.1.0"}) + assert get_version(headers) == "0.1.0" + + # Server header multiple aleph-vm values + headers = dict_to_ci_multi_dict_proxy({"Server": "aleph-vm/0.1.0", "server": "aleph-vm/0.2.0"}) + assert get_version(headers) == "0.1.0" + + +def test_create_table_with_progress_bar(): + """Test the creation of a table with progress bar.""" + sized_object = [1, 2, 3] + table, increment_function = create_table_with_progress_bar(sized_object) + assert isinstance(table, ProgressTable) + + # Test that calling the increment function ends up with the progress bar + # being finished after `len(sized_objects)` calls. + for i in range(3): + # The progress bar should not be finished yet + assert table.progress.tasks[0].finished_time is None + increment_function() + + # The progress bar should be finished now + finished_time = table.progress.tasks[0].finished_time + assert finished_time is not None and finished_time > 0 + + +@pytest.mark.asyncio +async def test_update_table(): + queue: MachineInfoQueue = asyncio.Queue() + table = mock.Mock() + table.add_row = mock.Mock() + increment_progress_bar = mock.Mock() + valid_reward_addresses: Set[str] = set() + + async def populate_queue(): + assert table.add_row.call_count == 0 + # Put the data in the queue + await queue.put(dummy_machine_info()) + # End the test by putting an end of queue marker + await queue.put("END_OF_QUEUE") + + # Populate the queue and update the table concurrently. + await asyncio.gather( + populate_queue(), + update_table(queue, table, increment_progress_bar, valid_reward_addresses), + ) + + assert table.add_row.call_count == 1 + assert valid_reward_addresses == {dummy_machine_info().reward_address} + + +@pytest.mark.asyncio +async def test_fetch_crn_info() -> None: + # Test with valid node + # TODO: Mock the response from the node, don't rely on a real node + node_url = "https://ovh.staging.aleph.sh" + machine_usage, version = await fetch_crn_info(node_url) + assert machine_usage is not None + assert version is not None + assert isinstance(machine_usage, MachineUsage) + assert isinstance(version, str) + + # Test with invalid node + invalid_node_url = "https://coconut.example.org/" + assert await fetch_crn_info(invalid_node_url) == (None, None) + + # TODO: Test different error handling + + +def test_sanitize_url_with_empty_url(): + with pytest.raises(InvalidURL, match="Empty URL"): + sanitize_url("") + + +def test_sanitize_url_with_invalid_scheme(): + with pytest.raises(InvalidURL, match="Invalid URL scheme"): + sanitize_url("ftp://example.org") + + +def test_sanitize_url_with_forbidden_host(): + for host in FORBIDDEN_HOSTS: + with pytest.raises(InvalidURL, match="Invalid URL host"): + sanitize_url(f"http://{host}") + + +def test_sanitize_url_with_valid_url(): + url = "http://example.org" + assert sanitize_url(url) == url + + +def test_sanitize_url_with_https_scheme(): + url = "https://example.org" + assert sanitize_url(url) == url diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0671bf24..996a390d 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,11 +1,11 @@ from aleph_message.models import ( AggregateMessage, ForgetMessage, - MessageType, PostMessage, ProgramMessage, StoreMessage, ) +from aleph_message.models.base import MessageType from aleph_client.utils import get_message_type_value