diff --git a/.github/workflows/build-python-client.yml b/.github/workflows/build-python-client.yml new file mode 100644 index 00000000..1c699ca7 --- /dev/null +++ b/.github/workflows/build-python-client.yml @@ -0,0 +1,120 @@ +name: build-and-test-python-client +on: + workflow_dispatch: + push: + pull_request: +jobs: + build: + runs-on: ubuntu-latest + outputs: + osparc: ${{ steps.find-wheel.outputs.osparc-wheel }} + osparc_client: ${{ steps.find-wheel.outputs.osparc_client-wheel }} + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.client_payload.ref }} + fetch-depth: 0 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Generate version from tag + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + run: | + tag_version=$(echo ${{ github.ref }} | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+')) + bash scripts/validate_semantic_version.bash "${tag_version}" > clients/python/client/VERSION + - name: Generate client + run: | + git status + make devenv + source .venv/bin/activate + cd clients/python + make install-dev + make dist-ci + - name: Determine wheel artifact + id: find-wheel + run: | + cd clients/python/artifacts/dist + OSPARC_WHEEL=$(ls osparc-*.whl) + OSPARC_CLIENT_WHEEL=$(ls osparc_client*.whl) + echo "osparc-wheel=${OSPARC_WHEEL}" >> $GITHUB_OUTPUT + echo "osparc_client-wheel=${OSPARC_CLIENT_WHEEL}" >> $GITHUB_OUTPUT + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: osparc_python_wheels + path: clients/python/artifacts/dist/ + + test-20-04: + name: python ${{ matrix.python-version }} ubuntu-20.04 + runs-on: ubuntu-20.04 + needs: build + strategy: + max-parallel: 4 + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.client_payload.ref }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Pip cache + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-{{ matrix.python-version }} + restore-keys: | + ${{ runner.os }}-pip + - name: Download wheels + uses: actions/download-artifact@v3 + with: + name: osparc_python_wheels + path: clients/python/artifacts/dist/ + - name: Install and Test + run: | + python -m venv .venv + source .venv/bin/activate + python -m pip install pytest + python -m pip install clients/python/artifacts/dist/${{needs.build.outputs.osparc}} --find-links=clients/python/artifacts/dist + cd clients/python + make install-test + pytest -v --ignore=/artifacts/client --ignore=test/e2e + + test-latest: + name: python ${{ matrix.python-version }} ubuntu-latest + runs-on: ubuntu-latest + needs: build + strategy: + max-parallel: 4 + matrix: + python-version: [3.7, 3.8, 3.9] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Pip cache + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-{{ matrix.python-version }} + restore-keys: | + ${{ runner.os }}-pip + - name: Download wheels + uses: actions/download-artifact@v3 + with: + name: osparc_python_wheels + path: clients/python/artifacts/dist/ + - name: Install and Test + run: | + python -m venv .venv + source .venv/bin/activate + python -m pip install pytest + python -m pip install clients/python/artifacts/dist/${{needs.build.outputs.osparc}} --find-links=clients/python/artifacts/dist + cd clients/python + make install-test + pytest -v --ignore=/artifacts/client --ignore=test/e2e diff --git a/.github/workflows/publish-python-client.yml b/.github/workflows/publish-python-client.yml index d31167c1..a40b3c5c 100644 --- a/.github/workflows/publish-python-client.yml +++ b/.github/workflows/publish-python-client.yml @@ -1,123 +1,16 @@ -name: publish-and-test-python-client +name: publish-python-client on: - workflow_dispatch: - push: - pull_request: -jobs: - build-n-publish: - runs-on: ubuntu-latest - outputs: - osparc: ${{ steps.find-wheel.outputs.osparc-wheel }} - osparc_client: ${{ steps.find-wheel.outputs.osparc_client-wheel }} - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.client_payload.ref }} - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Generate client - run: | - git status - make devenv - source .venv/bin/activate - cd clients/python - make install-dev - make dist-ci - - name: Determine wheel artifact - id: find-wheel - run: | - cd clients/python/artifacts/dist - OSPARC_WHEEL=$(ls osparc-*.whl) - OSPARC_CLIENT_WHEEL=$(ls osparc_client*.whl) - echo "osparc-wheel=${OSPARC_WHEEL}" >> $GITHUB_OUTPUT - echo "osparc_client-wheel=${OSPARC_CLIENT_WHEEL}" >> $GITHUB_OUTPUT - - name: Upload wheels - uses: actions/upload-artifact@v3 - with: - name: osparc_python_wheels - path: clients/python/artifacts/dist/ + workflow_run: + workflows: [build-and-test-python-client] + branches: [master] + types: + - completed - test-20-04: - name: python ${{ matrix.python-version }} ubuntu-20.04 - runs-on: ubuntu-20.04 - needs: build-n-publish - strategy: - max-parallel: 4 - matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.client_payload.ref }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Pip cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-{{ matrix.python-version }} - restore-keys: | - ${{ runner.os }}-pip - - name: Download wheels - uses: actions/download-artifact@v3 - with: - name: osparc_python_wheels - path: clients/python/artifacts/dist/ - - name: Install and Test - run: | - python -m venv .venv - source .venv/bin/activate - python -m pip install pytest - python -m pip install clients/python/artifacts/dist/${{needs.build-n-publish.outputs.osparc}} --find-links=clients/python/artifacts/dist - cd clients/python - make install-test - pytest -v --ignore=/artifacts/client --ignore=test/e2e - - test-latest: - name: python ${{ matrix.python-version }} ubuntu-latest - runs-on: ubuntu-latest - needs: build-n-publish - strategy: - max-parallel: 4 - matrix: - python-version: [3.7, 3.8, 3.9] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Pip cache - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-{{ matrix.python-version }} - restore-keys: | - ${{ runner.os }}-pip - - name: Download wheels - uses: actions/download-artifact@v3 - with: - name: osparc_python_wheels - path: clients/python/artifacts/dist/ - - name: Install and Test - run: | - python -m venv .venv - source .venv/bin/activate - python -m pip install pytest - python -m pip install clients/python/artifacts/dist/${{needs.build-n-publish.outputs.osparc}} --find-links=clients/python/artifacts/dist - cd clients/python - make install-test - pytest -v --ignore=/artifacts/client --ignore=test/e2e - - publish-osparc_client-to-pypi: +jobs: + publish-osparc_client: name: Publish osparc_client wheel - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest - needs: [build-n-publish, test-20-04, test-latest] environment: name: pypi url: https://pypi.org/p/osparc_client @@ -125,29 +18,29 @@ jobs: id-token: write steps: - name: Download wheels - uses: actions/download-artifact@v3 - with: - name: osparc_python_wheels - path: dist/ + run: gh run download ${{ github.event.workflow_run.id }} - name: Remove osparc wheel - run: rm -f dist/${{needs.build-n-publish.outputs.osparc}} + run: | + osparc_wheel=$(ls osparc_python_wheels/osparc-*) + rm -f ${osparc_wheel} - name: Publish osparc_client to Test PyPI + if: github.event_name == 'push' uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ verbose: true - packages-dir: dist/ + packages-dir: osparc_python_wheels/ - name: Publish osparc_client to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: verbose: true - packages-dir: dist/ + packages-dir: osparc_python_wheels/ - publish-osparc-to-pypi: + publish-osparc: name: Publish osparc wheel - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest - needs: [build-n-publish, test-20-04, test-latest] environment: name: pypi url: https://pypi.org/p/osparc @@ -155,20 +48,21 @@ jobs: id-token: write steps: - name: Download wheels - uses: actions/download-artifact@v3 - with: - name: osparc_python_wheels - path: dist/ + run: gh run download ${{ github.event.workflow_run.id }} - name: Remove osparc_client wheel - run: rm -f dist/${{needs.build-n-publish.outputs.osparc_client}} + run: | + osparc_client_wheel=$(ls osparc_python_wheels/osparc_client*) + rm -f ${osparc_client_wheel} - name: Publish osparc to Test PyPI + if: github.event_name == 'push' uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ verbose: true - packages-dir: dist/ + packages-dir: osparc_python_wheels/ - name: Publish osparc to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 with: verbose: true - packages-dir: dist/ + packages-dir: osparc_python_wheels/ diff --git a/.gitignore b/.gitignore index aea0a878..c36af280 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,4 @@ tmp* /clients/python/client/build/* /clients/python/client/osparc/data/openapi.json /clients/python/test/e2e/pytest.ini +/clients/python/client/VERSION diff --git a/api/config.json b/api/config.json index 194f7b5c..b7d702c4 100644 --- a/api/config.json +++ b/api/config.json @@ -1,7 +1,3 @@ { - "homepage": "https://itisfoundation.github.io/osparc-simcore-clients/", - "python": - { - "version": "0.7.0" - } + "homepage": "https://itisfoundation.github.io/osparc-simcore-clients/" } diff --git a/clients/python/Makefile b/clients/python/Makefile index 376c0d3c..94bfe363 100644 --- a/clients/python/Makefile +++ b/clients/python/Makefile @@ -4,6 +4,19 @@ ARTIFACTS_DIR := $(PYTHON_DIR)/artifacts CONTAINER_API_JSON := /local/$(REL_API_JSON_PATH) GENERATOR := python UID := $(shell id -u) +ADDITIONAL_PROPS := \ + generateSourceCodeOnly=false\ + hideGenerationTimestamp=true\ + library=urllib3\ + packageName=osparc\ + packageUrl=$(shell bash $(SCRIPTS_DIR)/jq.bash -r .homepage $(REPO_ROOT)/api/config.json)\ + projectName=osparc_client +ADDITIONAL_PROPS := $(foreach prop,$(ADDITIONAL_PROPS),$(strip $(prop))) +VERSION_FILE := $(PYTHON_DIR)/client/VERSION + +.PHONY: generate_version +client/VERSION: + @bash $(SCRIPTS_DIR)/generate_version.bash > $(VERSION_FILE) .PHONY: artifacts_dir artifacts_dir: @@ -20,7 +33,8 @@ postprocess-build: # Generation of Python client .PHONY: python-client -python-client: validate-api-specification artifacts_dir ## generate python client +python-client: client/VERSION validate-api-specification artifacts_dir ## generate python client + $(eval VERSION := $(shell cat $(VERSION_FILE))) -rm -r $(ARTIFACTS_DIR)/client docker run --rm --user $(UID):$(UID)\ --volume "$(REPO_ROOT):/local" \ @@ -28,18 +42,19 @@ python-client: validate-api-specification artifacts_dir ## generate python clien --generator-name=$(GENERATOR) \ --git-user-id=$(GIT_USER_ID) \ --git-repo-id=$(GIT_CLIENT_REPO_ID) \ - --http-user-agent="osparc-api/$(APP_VERSION)/python" \ + --http-user-agent="osparc-api/$(VERSION)/python" \ --input-spec=$(CONTAINER_API_JSON) \ --output=/local/clients/python/artifacts/client \ - --additional-properties=$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS))) \ + --additional-properties=packageVersion=$(VERSION),$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS))) \ --package-name=osparc_client \ - --release-note="Updated to $(APP_VERSION)" + --release-note="Updated to $(VERSION)" $(MAKE) postprocess-build .PHONY: python-client-from-templates -python-client-from-templates: validate-api-specification artifacts_dir ## generate python client using templates in a specified directory (usage: 'make python-client-from-templates -e TEMPLATE_DIR=path/to/templates') +python-client-from-templates: client/VERSION validate-api-specification artifacts_dir ## generate python client using templates in a specified directory (usage: 'make python-client-from-templates -e TEMPLATE_DIR=path/to/templates') $(if $(TEMPLATE_DIR),,$(error The TEMPLATE_DIR environment variable must be set)) @echo "Using template-dir: $(TEMPLATE_DIR)" + $(eval VERSION := $(shell cat $(VERSION_FILE))) -rm -r $(ARTIFACTS_DIR)/client @docker run --rm --user $(UID):$(UID)\ --volume "$(REPO_ROOT):/local" \ @@ -48,12 +63,12 @@ python-client-from-templates: validate-api-specification artifacts_dir ## genera --generator-name=$(GENERATOR) \ --git-user-id=$(GIT_USER_ID) \ --git-repo-id=$(GIT_CLIENT_REPO_ID) \ - --http-user-agent="osparc-api/$(APP_VERSION)/python" \ + --http-user-agent="osparc-api/$(VERSION)/python" \ --input-spec=$(CONTAINER_API_JSON) \ --output=/local/clients/python/artifacts/client \ - --additional-properties=$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS))) \ + --additional-properties=packageVersion=$(VERSION),$(subst $(space),$(comma),$(strip $(ADDITIONAL_PROPS))) \ --package-name=osparc_client \ - --release-note="Updated to $(APP_VERSION)" \ + --release-note="Updated to $(VERSION)" \ --template-dir=/tmp/python_templates $(MAKE) postprocess-build @@ -167,48 +182,9 @@ e2e-shell: guard-GH_TOKEN ## shell for running e2e tests @echo "WARNING ##### $@ does not exist, cloning $< as $@ ############"; cp $< $@) .PHONY: image -image: ## builds image $(APP_NAME):$(APP_VERSION) - docker build -f Dockerfile -t $(APP_NAME):$(APP_VERSION) $(CURDIR) +image: client/VERSION ## builds image $(APP_NAME):version + docker build -f Dockerfile -t $(APP_NAME):$(shell cat $(VERSION_FILE)) $(CURDIR) .PHONY: shell -shell: ## runs container and opens bash shell - docker run -it $(APP_NAME):$(APP_VERSION) /bin/bash - - - -# RELEASE ------------------------------------------------------------------------------- - -prod_prefix := v -_git_get_current_branch = $(shell git rev-parse --abbrev-ref HEAD) - -# NOTE: be careful that GNU Make replaces newlines with space which is why this command cannot work using a Make function -_url_encoded_title = $(APP_VERSION) -_url_encoded_tag = $(prod_prefix)$(APP_VERSION) -_url_encoded_target = $(if $(git_sha),$(git_sha),master) -_prettify_logs = $$(git log \ - $$(git describe --match="$(prod_prefix)*" --abbrev=0 --tags)..$(if $(git_sha),$(git_sha),HEAD) \ - --pretty=format:"- %s") -define _url_encoded_logs -$(shell \ - scripts/url-encoder.bash \ - "$(_prettify_logs)"\ -) -endef -_git_get_repo_orga_name = $(shell git config --get remote.origin.url | \ - grep --perl-regexp --only-matching "((?<=git@github\.com:)|(?<=https:\/\/github\.com\/))(.*?)(?=.git)") - -.PHONY: .check-master-branch -.check-master-branch: - @if [ "$(_git_get_current_branch)" != "master" ]; then\ - echo -e "\e[91mcurrent branch is not master branch."; exit 1;\ - fi - -.PHONY: release -release pre-release: .check-master-branch ## Creates github release link. Usage: make release-prod git_sha=optional - # ensure tags are up-to-date - @git pull --tags - @echo -e "\e[33mOpen the following link to create a release:"; - @echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring pre-, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)"; - @echo -e "\e[33mOr open the following link to create a release and paste the logs:"; - @echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring pre-, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)"; - @echo -e "\e[34m$(_prettify_logs)" +shell: client/VERSION ## runs container and opens bash shell + docker run -it $(APP_NAME):$(shell cat $(VERSION_FILE)) /bin/bash diff --git a/clients/python/client/setup.py b/clients/python/client/setup.py index dc959373..3c38e0a2 100644 --- a/clients/python/client/setup.py +++ b/clients/python/client/setup.py @@ -1,15 +1,12 @@ -import json from pathlib import Path -from typing import Any, Dict from setuptools import find_packages, setup # noqa: H301 -repo_root: Path = (Path(__file__) / "../../../..").resolve() - -config: Dict[str, Any] = json.loads((repo_root / "api/config.json").read_text()) +VERSION_FILE: Path = Path(__file__).parent / "VERSION" +assert VERSION_FILE.is_file() NAME = "osparc" -VERSION = f"{config['python']['version']}" +VERSION = VERSION_FILE.read_text() # To install the library, run the following # # python setup.py install diff --git a/clients/python/test/e2e/ci/preprocess_client_config.bash b/clients/python/test/e2e/ci/preprocess_client_config.bash index b6af31a1..2dc0e89d 100644 --- a/clients/python/test/e2e/ci/preprocess_client_config.bash +++ b/clients/python/test/e2e/ci/preprocess_client_config.bash @@ -18,7 +18,7 @@ doc+="\tA json string which, when passed to install_osparc_python_client.bash in print_doc() { echo -e "$doc"; } [ $# -eq 0 ] && print_doc && exit 0 -client_workflow=publish-and-test-python-client +client_workflow=build-and-test-python-client client_config=$1 # extract keys from input json diff --git a/scripts/common.Makefile b/scripts/common.Makefile index 897fd994..0af5e252 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -11,7 +11,6 @@ VCS_URL := $(shell git config --get remote.origin.url) VCS_REF := $(shell git rev-parse --short HEAD) NOW_TIMESTAMP := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") APP_NAME := $(notdir $(CURDIR)) -APP_VERSION := $(shell bash $(SCRIPTS_DIR)/jq.bash -r .python.version $(REPO_ROOT)/api/config.json) # Specify which openapi generator should be used to generate the clients in this repo OPENAPI_GENERATOR_NAME := itisfoundation/openapi-generator-cli-openapi-generator-v4.2.3 @@ -26,15 +25,7 @@ GIT_USER_ID := ITISFoundation GIT_CLIENT_REPO_ID := osparc-simcore-clients GIT_OPENAPI_REPO_ID := openapi-generator GENERATOR_NAME := python -ADDITIONAL_PROPS := \ - generateSourceCodeOnly=false\ - hideGenerationTimestamp=true\ - library=urllib3\ - packageName=osparc\ - packageUrl=$(shell bash $(SCRIPTS_DIR)/jq.bash -r .homepage $(REPO_ROOT)/api/config.json)\ - packageVersion=$(APP_VERSION)\ - projectName=osparc_client -ADDITIONAL_PROPS := $(foreach prop,$(ADDITIONAL_PROPS),$(strip $(prop))) + null := space := $(null) # diff --git a/scripts/generate_version.bash b/scripts/generate_version.bash new file mode 100644 index 00000000..f0cf6b08 --- /dev/null +++ b/scripts/generate_version.bash @@ -0,0 +1,17 @@ +#!/bin/bash + +# Generate version + +# This is done by inspecting the tags on the git repo https://github.com/ITISFoundation/osparc-simcore-clients + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + +release_info=$(git ls-remote --tags --refs --sort=version:refname https://github.com/ITISFoundation/osparc-simcore-clients | tail -1) +release_version=$(echo "${release_info}" | grep -oP '(?<=refs/tags/v)\d+\.\d+\.\d+') +release_commit=$(echo "${release_info}" | grep -oE '^[[:alnum:]]+') +current_commit=$(git rev-parse HEAD) + +merge_base=$(git merge-base "${release_commit}" "${current_commit}") +n_commits_to_merge_base=$(git rev-list --count "${merge_base}".."${current_commit}") +echo -n "${release_version}.dev${n_commits_to_merge_base}" diff --git a/scripts/validate_semantic_version.bash b/scripts/validate_semantic_version.bash new file mode 100644 index 00000000..6bad36d5 --- /dev/null +++ b/scripts/validate_semantic_version.bash @@ -0,0 +1,17 @@ +#!/bin/bash + +# Validate that a string is a valid semantic version +# This uses the regex pattern here: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes + + +version=$1 +validated_version=$(echo "${version}" | grep -E "^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$") +if [ -z "${validated_version}" ]; then + echo "Received invalid semantic version: ${version}" + exit 1 +fi +echo -n "${validated_version}"