diff --git a/.github/workflows/Publish.yml b/.github/workflows/Publish.yml
new file mode 100644
index 0000000..84bd69a
--- /dev/null
+++ b/.github/workflows/Publish.yml
@@ -0,0 +1,34 @@
+name: Publish
+
+on:
+ push:
+ branches:
+ - main
+
+jobs:
+ build-and-deploy:
+ name: Build and deploy
+
+ runs-on: ubuntu-22.04
+
+ steps:
+ - name: Retrieve source code
+ uses: actions/checkout@v3
+
+ - name: Build and publish Docker Image
+ uses: openzim/docker-publish-action@v10
+ with:
+ image-name: openzim/zimit-ui
+ on-master: latest
+ restrict-to: openzim/zimit-frontend
+ registries: ghcr.io
+ credentials:
+ GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+ GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+
+ - name: Deploy Zimit frontend changes to youzim.it
+ uses: actions-hub/kubectl@master
+ env:
+ KUBE_CONFIG: ${{ secrets.ZIMIT_KUBE_CONFIG }}
+ with:
+ args: rollout restart deployments ui-deployment -n zimit
diff --git a/.github/workflows/ci.yml b/.github/workflows/QA.yml
similarity index 73%
rename from .github/workflows/ci.yml
rename to .github/workflows/QA.yml
index 88b40de..cc81aac 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/QA.yml
@@ -1,23 +1,29 @@
-name: CI
+name: QA
-on: [push]
+on:
+ pull_request:
+ push:
+ branches:
+ - main
jobs:
- code-formating:
+ check-qa:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v1
- with:
- fetch-depth: 1
+ - name: Retrieve source code
+ uses: actions/checkout@v3
+
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
+
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -r api/requirements.txt
+
- name: black code formatting check
run: |
pip install -U "black==22.3.0"
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
deleted file mode 100644
index 0615ca6..0000000
--- a/.github/workflows/docker.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: Docker
-
-on:
- push:
- branches:
- - main
-
-jobs:
- build-and-push:
- name: Deploy Docker Image
- runs-on: ubuntu-22.04
- steps:
- - uses: actions/checkout@v3.4.0
- - name: Build and push
- uses: openzim/docker-publish-action@v10
- with:
- image-name: openzim/zimit-ui
- on-master: latest
- restrict-to: openzim/zimit-frontend
- registries: ghcr.io
- credentials:
- GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
- GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
diff --git a/api/src/routes/errors.py b/api/src/routes/errors.py
index 861f5bf..ec62c3b 100644
--- a/api/src/routes/errors.py
+++ b/api/src/routes/errors.py
@@ -67,44 +67,40 @@ def handler_validationerror(e):
return make_response(jsonify({"message": e.messages}), HTTPStatus.BAD_REQUEST)
-# 400
-class BadRequest(Exception):
+class ExceptionWithMessage(Exception):
def __init__(self, message: str = None):
self.message = message
+ @staticmethod
+ def handler(e, status: HTTPStatus):
+ if isinstance(e, ExceptionWithMessage) and e.message is not None:
+ return make_response(jsonify({"error": e.message}), status)
+ return Response(status=status)
+
+
+# 400
+class BadRequest(ExceptionWithMessage):
@staticmethod
def handler(e):
- if isinstance(e, BadRequest) and e.message is not None:
- return make_response(jsonify({"error": e.message}), HTTPStatus.BAD_REQUEST)
- return Response(status=HTTPStatus.BAD_REQUEST)
+ return super().handler(e, HTTPStatus.BAD_REQUEST)
# 401
-class Unauthorized(Exception):
- def __init__(self, message: str = None):
- self.message = message
-
+class Unauthorized(ExceptionWithMessage):
@staticmethod
def handler(e):
- if isinstance(e, Unauthorized) and e.message is not None:
- return make_response(jsonify({"error": e.message}), HTTPStatus.UNAUTHORIZED)
- return Response(status=HTTPStatus.UNAUTHORIZED)
+ return super().handler(e, HTTPStatus.UNAUTHORIZED)
# 404
-class NotFound(Exception):
- def __init__(self, message: str = None):
- self.message = message
-
+class NotFound(ExceptionWithMessage):
@staticmethod
def handler(e):
- if isinstance(e, NotFound) and e.message is not None:
- return make_response(jsonify({"error": e.message}), HTTPStatus.NOT_FOUND)
- return Response(status=HTTPStatus.NOT_FOUND)
+ return super().handler(e, HTTPStatus.NOT_FOUND)
# 500
-class InternalError(Exception):
+class InternalError(ExceptionWithMessage):
@staticmethod
def handler(e):
- return Response(status=HTTPStatus.INTERNAL_SERVER_ERROR)
+ return super().handler(e, HTTPStatus.INTERNAL_SERVER_ERROR)
diff --git a/api/src/routes/requests.py b/api/src/routes/requests.py
index 704f651..226ad58 100644
--- a/api/src/routes/requests.py
+++ b/api/src/routes/requests.py
@@ -129,7 +129,14 @@ def post(self, *args, **kwargs):
success, status, resp = query_api("POST", "/schedules/", payload=payload)
if not success:
logger.error(f"Unable to create schedule via HTTP {status}: {resp}")
- raise InternalError(f"Unable to create schedule via HTTP {status}: {resp}")
+ message = f"Unable to create schedule via HTTP {status}: {resp}"
+ if status == http.HTTPStatus.BAD_REQUEST:
+ # if Zimfarm replied this is a bad request, then this is most probably
+ # a bad request due to user input so we can track it like a bad request
+ raise BadRequest(message)
+ else:
+ # otherwise, this is most probably an internal problem in our systems
+ raise InternalError(message)
# request a task for that newly created schedule
success, status, resp = query_api(
@@ -138,23 +145,24 @@ def post(self, *args, **kwargs):
payload={"schedule_names": [schedule_name], "worker": TASK_WORKER},
)
if not success:
- logger.error(f"Unable to request {schedule_name} via HTTP {status}")
- logger.debug(resp)
- raise InternalError(f"Unable to request schedule via HTTP {status}: {resp}")
+ logger.error(f"Unable to request {schedule_name} via HTTP {status}: {resp}")
+ raise InternalError(
+ f"Unable to request schedule via HTTP {status}): {resp}"
+ )
try:
task_id = resp.get("requested").pop()
if not task_id:
- raise ValueError("task_id is False")
+ raise InternalError("task_id is False")
except Exception as exc:
raise InternalError(f"Couldn't retrieve requested task id: {exc}")
# remove newly created schedule (not needed anymore)
success, status, resp = query_api("DELETE", f"/schedules/{schedule_name}")
if not success:
- logger.error(f"Unable to remove schedule {schedule_name} via HTTP {status}")
- logger.debug(resp)
-
+ logger.error(
+ f"Unable to remove schedule {schedule_name} via HTTP {status}: {resp}"
+ )
return make_response(jsonify({"id": str(task_id)}), http.HTTPStatus.CREATED)
diff --git a/dev/README.md b/dev/README.md
new file mode 100644
index 0000000..82b2d2d
--- /dev/null
+++ b/dev/README.md
@@ -0,0 +1,66 @@
+This is a docker-compose configuration to be used **only** for development purpose. There is
+almost zero security in the stack configuration.
+
+It is composed of the Zimit frontend and API (of course), but also a local Zimfarm DB,
+API and UI, so that you can test the whole integration locally.
+
+Zimit UI and API are not deployed as they would be in production to allow hot reload of
+most modifications done to the source code.
+
+Zimfarm UI, API and DB are deployed with official production Docker images.
+
+## List of containers
+
+### zimit_ui
+
+This container is Zimit frontend web server (UI only)
+
+### zimit_api
+
+This container is Zimit API server (API only)
+
+## zimfarm_db
+
+This container is a local Zimfarm database
+
+## zimfarm_api
+
+This container is a local Zimfarm API
+
+## zimfarm_ui
+
+This container is a local Zimfarm UI
+
+## Instructions
+
+First start the Docker-Compose stack:
+
+```sh
+cd dev
+docker compose -p zimit up -d
+```
+
+If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB:
+
+```sh
+dev/create_worker.sh
+```
+
+If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`.
+
+## Restart the backend
+
+Should the API process fail, you might restart it with:
+```sh
+docker restart zimit-zimit_ui-1
+```
+
+## Browse the web UIs
+
+You might open following URLs in your favorite browser:
+
+- [Zimit UI](http://localhost:8001)
+- [Zimfarm API](http://localhost:8002)
+- [Zimfarm UI](http://localhost:8003)
+
+You can login into Zimfarm UI with username `admin` and password `admin`.
\ No newline at end of file
diff --git a/dev/create_worker.sh b/dev/create_worker.sh
new file mode 100755
index 0000000..9b6940a
--- /dev/null
+++ b/dev/create_worker.sh
@@ -0,0 +1,27 @@
+echo "Retrieving access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+ 'http://localhost:8002/v1/auth/authorize' \
+ -H 'accept: application/json' \
+ -H 'Content-Type: application/x-www-form-urlencoded' \
+ -d 'username=admin&password=admin' \
+ | jq -r '.access_token')"
+
+echo "Worker check-in (will create if missing)"
+
+curl -s -X 'PUT' \
+ 'http://localhost:8002/v1/workers/worker/check-in' \
+ -H 'accept: */*' \
+ -H 'Content-Type: application/json' \
+ -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+ -d '{
+ "username": "admin",
+ "cpu": 3,
+ "memory": 1024,
+ "disk": 0,
+ "offliners": [
+ "zimit"
+ ]
+}'
+
+echo "DONE"
\ No newline at end of file
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
new file mode 100644
index 0000000..d52c088
--- /dev/null
+++ b/dev/docker-compose.yml
@@ -0,0 +1,63 @@
+services:
+ zimfarm_db:
+ image: postgres:15.2-bullseye
+ ports:
+ - 127.0.0.1:5432:5432
+ volumes:
+ - zimfarm_data:/var/lib/postgresql/data
+ environment:
+ - POSTGRES_DB=zimfarm
+ - POSTGRES_USER=zimfarm
+ - POSTGRES_PASSWORD=zimpass
+ zimfarm_api:
+ image: ghcr.io/openzim/zimfarm-dispatcher:latest
+ ports:
+ - 127.0.0.1:8004:80
+ environment:
+ BINDING_HOST: 0.0.0.0
+ JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
+ POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@zimfarm_db:5432/zimfarm
+ ALEMBIC_UPGRADE_HEAD_ON_START: "1"
+ ZIMIT_USE_RELAXED_SCHEMA: "y"
+ depends_on:
+ - zimfarm_db
+ zimfarm-ui:
+ image: ghcr.io/openzim/zimfarm-ui:latest
+ ports:
+ - 127.0.0.1:8003:80
+ environment:
+ ZIMFARM_WEBAPI: http://localhost:8002/v1
+ depends_on:
+ - zimfarm_api
+ zimit_api:
+ build: ..
+ volumes:
+ - ../api/src:/app
+ command: python main.py
+ ports:
+ - 127.0.0.1:8002:8000
+ environment:
+ BINDING_HOST: 0.0.0.0
+ INTERNAL_ZIMFARM_WEBAPI: http://zimfarm_api:80/v1
+ _ZIMFARM_USERNAME: admin
+ _ZIMFARM_PASSWORD: admin
+ TASK_WORKER: worker
+ depends_on:
+ - zimfarm_api
+ zimit_ui:
+ build:
+ dockerfile: ../dev/zimit_ui_dev/Dockerfile
+ context: ../ui
+ volumes:
+ - ../ui/src:/app/src
+ - ../ui/public:/app/public
+ - ../dev/zimit_ui_dev/environ.json:/app/public/environ.json
+ ports:
+ - 127.0.0.1:8001:80
+ environment:
+ ZIMIT_API_URL: http://localhost:8002
+ depends_on:
+ - zimit_api
+
+volumes:
+ zimfarm_data:
diff --git a/dev/start_first_req_task.sh b/dev/start_first_req_task.sh
new file mode 100755
index 0000000..56ef952
--- /dev/null
+++ b/dev/start_first_req_task.sh
@@ -0,0 +1,31 @@
+echo "Retrieving access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+ 'http://localhost:8002/v1/auth/authorize' \
+ -H 'accept: application/json' \
+ -H 'Content-Type: application/x-www-form-urlencoded' \
+ -d 'username=admin&password=admin' \
+ | jq -r '.access_token')"
+
+echo "Get last requested task"
+
+LAST_TASK_ID="$(curl -s -X 'GET' \
+ 'http://localhost:8002/v1/requested-tasks/' \
+ -H 'accept: application/json' \
+ -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+ | jq -r '.items[0]._id')"
+
+if [ "$LAST_TASK_ID" = "null" ]; then
+ echo "No pending requested task. Exiting script."
+ exit 1
+fi
+
+echo "Start task"
+
+curl -s -X 'POST' \
+ "http://localhost:8002/v1/tasks/$LAST_TASK_ID?worker_name=worker" \
+ -H 'accept: application/json' \
+ -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
+ -d ''
+
+echo "DONE"
\ No newline at end of file
diff --git a/dev/zimit_ui_dev/Dockerfile b/dev/zimit_ui_dev/Dockerfile
new file mode 100644
index 0000000..32a1994
--- /dev/null
+++ b/dev/zimit_ui_dev/Dockerfile
@@ -0,0 +1,10 @@
+FROM node:14-alpine
+
+RUN apk --no-cache add yarn
+WORKDIR /app
+COPY package.json yarn.lock /app/
+RUN yarn install && yarn cache clean
+COPY *.js /app/
+COPY public /app/public
+COPY src /app/src
+CMD ["yarn", "serve", "--host", "0.0.0.0", "--port", "80"]
\ No newline at end of file
diff --git a/dev/zimit_ui_dev/environ.json b/dev/zimit_ui_dev/environ.json
new file mode 100644
index 0000000..7619b85
--- /dev/null
+++ b/dev/zimit_ui_dev/environ.json
@@ -0,0 +1,4 @@
+{
+ "ZIMFARM_WEBAPI": "http://localhost:8004/v1",
+ "ZIMIT_API_URL": "http://localhost:8002/api/v1"
+}
diff --git a/ui/src/components/NewRequest.vue b/ui/src/components/NewRequest.vue
index 468a811..01fa8d7 100644
--- a/ui/src/components/NewRequest.vue
+++ b/ui/src/components/NewRequest.vue
@@ -233,7 +233,7 @@
throw "Didn't receive task_id";
})
.catch(function (error) {
- parent.alertError("Unable to create schedule:\n" + Constants.standardHTTPError(error.response));
+ parent.alertError("Unable to request ZIM creation:
" + Constants.standardHTTPError(error.response));
})
.then(function () {
parent.toggleLoader(false);
diff --git a/ui/src/constants.js b/ui/src/constants.js
index 7c0f205..caaa85d 100644
--- a/ui/src/constants.js
+++ b/ui/src/constants.js
@@ -98,19 +98,16 @@ export default {
599: "Network Connect Timeout Error",
};
- if (response === undefined) { // no response
- //usually due to browser blocking failed OPTION preflight request
- return "Cross-Origin Request Blocked: preflight request failed."
+ if (response === undefined) {
+ // no response is usually due to browser blocking due to CORS issue
+ return "Unknown response; probably CORS issue."
}
- let status_text = response.statusText ? response.statusText : statuses[response.status];
- if (response.status == 400) {
- if (response.data && response.data.error)
- status_text += "
" + JSON.stringify(response.data.error);
- if (response.data && response.data.error_description)
- status_text += "
" + JSON.stringify(response.data.error_description);
- if (response.data && response.data.message)
- status_text += "
" + JSON.stringify(response.data.message);
+ // If error is provided, display it (do not display error code since this is too technical)
+ if (response.data && response.data.error) {
+ return response.data.error;
}
+ // Last resort, display only available information
+ let status_text = response.statusText ? response.statusText : statuses[response.status];
return response.status + ": " + status_text + ".";
},
};