From 83960b0b6a5cda0251da98e719126427d52dc672 Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 23 Nov 2023 10:57:52 +0100 Subject: [PATCH 1/4] Create a local development stack --- dev/README.md | 66 +++++++++++++++++++++++++++++++++++ dev/create_worker.sh | 27 ++++++++++++++ dev/docker-compose.yml | 63 +++++++++++++++++++++++++++++++++ dev/start_first_req_task.sh | 31 ++++++++++++++++ dev/zimit_ui_dev/Dockerfile | 10 ++++++ dev/zimit_ui_dev/environ.json | 4 +++ 6 files changed, 201 insertions(+) create mode 100644 dev/README.md create mode 100755 dev/create_worker.sh create mode 100644 dev/docker-compose.yml create mode 100755 dev/start_first_req_task.sh create mode 100644 dev/zimit_ui_dev/Dockerfile create mode 100644 dev/zimit_ui_dev/environ.json diff --git a/dev/README.md b/dev/README.md new file mode 100644 index 0000000..82b2d2d --- /dev/null +++ b/dev/README.md @@ -0,0 +1,66 @@ +This is a docker-compose configuration to be used **only** for development purpose. There is +almost zero security in the stack configuration. + +It is composed of the Zimit frontend and API (of course), but also a local Zimfarm DB, +API and UI, so that you can test the whole integration locally. + +Zimit UI and API are not deployed as they would be in production to allow hot reload of +most modifications done to the source code. + +Zimfarm UI, API and DB are deployed with official production Docker images. + +## List of containers + +### zimit_ui + +This container is Zimit frontend web server (UI only) + +### zimit_api + +This container is Zimit API server (API only) + +## zimfarm_db + +This container is a local Zimfarm database + +## zimfarm_api + +This container is a local Zimfarm API + +## zimfarm_ui + +This container is a local Zimfarm UI + +## Instructions + +First start the Docker-Compose stack: + +```sh +cd dev +docker compose -p zimit up -d +``` + +If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB: + +```sh +dev/create_worker.sh +``` + +If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`. + +## Restart the backend + +Should the API process fail, you might restart it with: +```sh +docker restart zimit-zimit_ui-1 +``` + +## Browse the web UIs + +You might open following URLs in your favorite browser: + +- [Zimit UI](http://localhost:8001) +- [Zimfarm API](http://localhost:8002) +- [Zimfarm UI](http://localhost:8003) + +You can login into Zimfarm UI with username `admin` and password `admin`. \ No newline at end of file diff --git a/dev/create_worker.sh b/dev/create_worker.sh new file mode 100755 index 0000000..9b6940a --- /dev/null +++ b/dev/create_worker.sh @@ -0,0 +1,27 @@ +echo "Retrieving access token" + +ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \ + 'http://localhost:8002/v1/auth/authorize' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'username=admin&password=admin' \ + | jq -r '.access_token')" + +echo "Worker check-in (will create if missing)" + +curl -s -X 'PUT' \ + 'http://localhost:8002/v1/workers/worker/check-in' \ + -H 'accept: */*' \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \ + -d '{ + "username": "admin", + "cpu": 3, + "memory": 1024, + "disk": 0, + "offliners": [ + "zimit" + ] +}' + +echo "DONE" \ No newline at end of file diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml new file mode 100644 index 0000000..d52c088 --- /dev/null +++ b/dev/docker-compose.yml @@ -0,0 +1,63 @@ +services: + zimfarm_db: + image: postgres:15.2-bullseye + ports: + - 127.0.0.1:5432:5432 + volumes: + - zimfarm_data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=zimfarm + - POSTGRES_USER=zimfarm + - POSTGRES_PASSWORD=zimpass + zimfarm_api: + image: ghcr.io/openzim/zimfarm-dispatcher:latest + ports: + - 127.0.0.1:8004:80 + environment: + BINDING_HOST: 0.0.0.0 + JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw + POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@zimfarm_db:5432/zimfarm + ALEMBIC_UPGRADE_HEAD_ON_START: "1" + ZIMIT_USE_RELAXED_SCHEMA: "y" + depends_on: + - zimfarm_db + zimfarm-ui: + image: ghcr.io/openzim/zimfarm-ui:latest + ports: + - 127.0.0.1:8003:80 + environment: + ZIMFARM_WEBAPI: http://localhost:8002/v1 + depends_on: + - zimfarm_api + zimit_api: + build: .. + volumes: + - ../api/src:/app + command: python main.py + ports: + - 127.0.0.1:8002:8000 + environment: + BINDING_HOST: 0.0.0.0 + INTERNAL_ZIMFARM_WEBAPI: http://zimfarm_api:80/v1 + _ZIMFARM_USERNAME: admin + _ZIMFARM_PASSWORD: admin + TASK_WORKER: worker + depends_on: + - zimfarm_api + zimit_ui: + build: + dockerfile: ../dev/zimit_ui_dev/Dockerfile + context: ../ui + volumes: + - ../ui/src:/app/src + - ../ui/public:/app/public + - ../dev/zimit_ui_dev/environ.json:/app/public/environ.json + ports: + - 127.0.0.1:8001:80 + environment: + ZIMIT_API_URL: http://localhost:8002 + depends_on: + - zimit_api + +volumes: + zimfarm_data: diff --git a/dev/start_first_req_task.sh b/dev/start_first_req_task.sh new file mode 100755 index 0000000..56ef952 --- /dev/null +++ b/dev/start_first_req_task.sh @@ -0,0 +1,31 @@ +echo "Retrieving access token" + +ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \ + 'http://localhost:8002/v1/auth/authorize' \ + -H 'accept: application/json' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d 'username=admin&password=admin' \ + | jq -r '.access_token')" + +echo "Get last requested task" + +LAST_TASK_ID="$(curl -s -X 'GET' \ + 'http://localhost:8002/v1/requested-tasks/' \ + -H 'accept: application/json' \ + -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \ + | jq -r '.items[0]._id')" + +if [ "$LAST_TASK_ID" = "null" ]; then + echo "No pending requested task. Exiting script." + exit 1 +fi + +echo "Start task" + +curl -s -X 'POST' \ + "http://localhost:8002/v1/tasks/$LAST_TASK_ID?worker_name=worker" \ + -H 'accept: application/json' \ + -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \ + -d '' + +echo "DONE" \ No newline at end of file diff --git a/dev/zimit_ui_dev/Dockerfile b/dev/zimit_ui_dev/Dockerfile new file mode 100644 index 0000000..32a1994 --- /dev/null +++ b/dev/zimit_ui_dev/Dockerfile @@ -0,0 +1,10 @@ +FROM node:14-alpine + +RUN apk --no-cache add yarn +WORKDIR /app +COPY package.json yarn.lock /app/ +RUN yarn install && yarn cache clean +COPY *.js /app/ +COPY public /app/public +COPY src /app/src +CMD ["yarn", "serve", "--host", "0.0.0.0", "--port", "80"] \ No newline at end of file diff --git a/dev/zimit_ui_dev/environ.json b/dev/zimit_ui_dev/environ.json new file mode 100644 index 0000000..7619b85 --- /dev/null +++ b/dev/zimit_ui_dev/environ.json @@ -0,0 +1,4 @@ +{ + "ZIMFARM_WEBAPI": "http://localhost:8004/v1", + "ZIMIT_API_URL": "http://localhost:8002/api/v1" +} From c9c7b8508d013b659bf9b2e44ffde652dd002cfc Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 23 Nov 2023 14:15:10 +0100 Subject: [PATCH 2/4] Ensure error handling logic is displaying the issue on the UI --- api/src/routes/errors.py | 38 ++++++++++++++------------------ api/src/routes/requests.py | 24 +++++++++++++------- ui/src/components/NewRequest.vue | 2 +- ui/src/constants.js | 19 +++++++--------- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/api/src/routes/errors.py b/api/src/routes/errors.py index 861f5bf..ec62c3b 100644 --- a/api/src/routes/errors.py +++ b/api/src/routes/errors.py @@ -67,44 +67,40 @@ def handler_validationerror(e): return make_response(jsonify({"message": e.messages}), HTTPStatus.BAD_REQUEST) -# 400 -class BadRequest(Exception): +class ExceptionWithMessage(Exception): def __init__(self, message: str = None): self.message = message + @staticmethod + def handler(e, status: HTTPStatus): + if isinstance(e, ExceptionWithMessage) and e.message is not None: + return make_response(jsonify({"error": e.message}), status) + return Response(status=status) + + +# 400 +class BadRequest(ExceptionWithMessage): @staticmethod def handler(e): - if isinstance(e, BadRequest) and e.message is not None: - return make_response(jsonify({"error": e.message}), HTTPStatus.BAD_REQUEST) - return Response(status=HTTPStatus.BAD_REQUEST) + return super().handler(e, HTTPStatus.BAD_REQUEST) # 401 -class Unauthorized(Exception): - def __init__(self, message: str = None): - self.message = message - +class Unauthorized(ExceptionWithMessage): @staticmethod def handler(e): - if isinstance(e, Unauthorized) and e.message is not None: - return make_response(jsonify({"error": e.message}), HTTPStatus.UNAUTHORIZED) - return Response(status=HTTPStatus.UNAUTHORIZED) + return super().handler(e, HTTPStatus.UNAUTHORIZED) # 404 -class NotFound(Exception): - def __init__(self, message: str = None): - self.message = message - +class NotFound(ExceptionWithMessage): @staticmethod def handler(e): - if isinstance(e, NotFound) and e.message is not None: - return make_response(jsonify({"error": e.message}), HTTPStatus.NOT_FOUND) - return Response(status=HTTPStatus.NOT_FOUND) + return super().handler(e, HTTPStatus.NOT_FOUND) # 500 -class InternalError(Exception): +class InternalError(ExceptionWithMessage): @staticmethod def handler(e): - return Response(status=HTTPStatus.INTERNAL_SERVER_ERROR) + return super().handler(e, HTTPStatus.INTERNAL_SERVER_ERROR) diff --git a/api/src/routes/requests.py b/api/src/routes/requests.py index 704f651..226ad58 100644 --- a/api/src/routes/requests.py +++ b/api/src/routes/requests.py @@ -129,7 +129,14 @@ def post(self, *args, **kwargs): success, status, resp = query_api("POST", "/schedules/", payload=payload) if not success: logger.error(f"Unable to create schedule via HTTP {status}: {resp}") - raise InternalError(f"Unable to create schedule via HTTP {status}: {resp}") + message = f"Unable to create schedule via HTTP {status}: {resp}" + if status == http.HTTPStatus.BAD_REQUEST: + # if Zimfarm replied this is a bad request, then this is most probably + # a bad request due to user input so we can track it like a bad request + raise BadRequest(message) + else: + # otherwise, this is most probably an internal problem in our systems + raise InternalError(message) # request a task for that newly created schedule success, status, resp = query_api( @@ -138,23 +145,24 @@ def post(self, *args, **kwargs): payload={"schedule_names": [schedule_name], "worker": TASK_WORKER}, ) if not success: - logger.error(f"Unable to request {schedule_name} via HTTP {status}") - logger.debug(resp) - raise InternalError(f"Unable to request schedule via HTTP {status}: {resp}") + logger.error(f"Unable to request {schedule_name} via HTTP {status}: {resp}") + raise InternalError( + f"Unable to request schedule via HTTP {status}): {resp}" + ) try: task_id = resp.get("requested").pop() if not task_id: - raise ValueError("task_id is False") + raise InternalError("task_id is False") except Exception as exc: raise InternalError(f"Couldn't retrieve requested task id: {exc}") # remove newly created schedule (not needed anymore) success, status, resp = query_api("DELETE", f"/schedules/{schedule_name}") if not success: - logger.error(f"Unable to remove schedule {schedule_name} via HTTP {status}") - logger.debug(resp) - + logger.error( + f"Unable to remove schedule {schedule_name} via HTTP {status}: {resp}" + ) return make_response(jsonify({"id": str(task_id)}), http.HTTPStatus.CREATED) diff --git a/ui/src/components/NewRequest.vue b/ui/src/components/NewRequest.vue index 468a811..01fa8d7 100644 --- a/ui/src/components/NewRequest.vue +++ b/ui/src/components/NewRequest.vue @@ -233,7 +233,7 @@ throw "Didn't receive task_id"; }) .catch(function (error) { - parent.alertError("Unable to create schedule:\n" + Constants.standardHTTPError(error.response)); + parent.alertError("Unable to request ZIM creation:
" + Constants.standardHTTPError(error.response)); }) .then(function () { parent.toggleLoader(false); diff --git a/ui/src/constants.js b/ui/src/constants.js index 7c0f205..caaa85d 100644 --- a/ui/src/constants.js +++ b/ui/src/constants.js @@ -98,19 +98,16 @@ export default { 599: "Network Connect Timeout Error", }; - if (response === undefined) { // no response - //usually due to browser blocking failed OPTION preflight request - return "Cross-Origin Request Blocked: preflight request failed." + if (response === undefined) { + // no response is usually due to browser blocking due to CORS issue + return "Unknown response; probably CORS issue." } - let status_text = response.statusText ? response.statusText : statuses[response.status]; - if (response.status == 400) { - if (response.data && response.data.error) - status_text += "
" + JSON.stringify(response.data.error); - if (response.data && response.data.error_description) - status_text += "
" + JSON.stringify(response.data.error_description); - if (response.data && response.data.message) - status_text += "
" + JSON.stringify(response.data.message); + // If error is provided, display it (do not display error code since this is too technical) + if (response.data && response.data.error) { + return response.data.error; } + // Last resort, display only available information + let status_text = response.statusText ? response.statusText : statuses[response.status]; return response.status + ": " + status_text + "."; }, }; From ec365192a6a410978b30bbb896b4ce856f3c876e Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 23 Nov 2023 15:11:44 +0100 Subject: [PATCH 3/4] Automatize deployment on k8s + align CI with bootstrap conventions --- .github/workflows/Publish.yml | 33 ++++++++++++++++++++++++++++ .github/workflows/{ci.yml => QA.yml} | 18 ++++++++++----- .github/workflows/docker.yml | 23 ------------------- 3 files changed, 45 insertions(+), 29 deletions(-) create mode 100644 .github/workflows/Publish.yml rename .github/workflows/{ci.yml => QA.yml} (73%) delete mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml new file mode 100644 index 0000000..e870f29 --- /dev/null +++ b/.github/workflows/Publish.yml @@ -0,0 +1,33 @@ +name: Publish + +on: + push: + branches: + - main + +jobs: + build-and-deploy: + name: Build and deploy + + runs-on: ubuntu-22.04 + + steps: + - name: Retrieve source code + uses: actions/checkout@v3 + + - name: Build and publish Docker Image + uses: openzim/docker-publish-action@v10 + with: + image-name: openzim/zimit-ui + on-master: latest + restrict-to: openzim/zimit-frontend + registries: ghcr.io + credentials: GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} + + - name: Deploy Zimit frontend changes to youzim.it + uses: actions-hub/kubectl@master + env: + KUBE_CONFIG: ${{ secrets.ZIMIT_KUBE_CONFIG }} + with: + args: rollout restart deployments ui-deployment -n zimit diff --git a/.github/workflows/ci.yml b/.github/workflows/QA.yml similarity index 73% rename from .github/workflows/ci.yml rename to .github/workflows/QA.yml index 88b40de..cc81aac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/QA.yml @@ -1,23 +1,29 @@ -name: CI +name: QA -on: [push] +on: + pull_request: + push: + branches: + - main jobs: - code-formating: + check-qa: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - with: - fetch-depth: 1 + - name: Retrieve source code + uses: actions/checkout@v3 + - name: Set up Python 3.8 uses: actions/setup-python@v1 with: python-version: 3.8 + - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r api/requirements.txt + - name: black code formatting check run: | pip install -U "black==22.3.0" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 0615ca6..0000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Docker - -on: - push: - branches: - - main - -jobs: - build-and-push: - name: Deploy Docker Image - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3.4.0 - - name: Build and push - uses: openzim/docker-publish-action@v10 - with: - image-name: openzim/zimit-ui - on-master: latest - restrict-to: openzim/zimit-frontend - registries: ghcr.io - credentials: - GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} - GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} From 7165a2b90013ad59707b3dd0a8416612f0857ffb Mon Sep 17 00:00:00 2001 From: benoit74 Date: Thu, 23 Nov 2023 16:21:54 +0100 Subject: [PATCH 4/4] Align code for readability --- .github/workflows/Publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml index e870f29..84bd69a 100644 --- a/.github/workflows/Publish.yml +++ b/.github/workflows/Publish.yml @@ -22,7 +22,8 @@ jobs: on-master: latest restrict-to: openzim/zimit-frontend registries: ghcr.io - credentials: GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} + credentials: + GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }} GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }} - name: Deploy Zimit frontend changes to youzim.it