Merge pull request #15 from kiwix/dev

Improve basic repo architecture

.github/workflows/backend-Publish.yaml

@@ -0,0 +1,31 @@
+name: Publish backend Docker image
+
+on:
+  push:
+    paths:
+      - 'backend/**'
+    branches:
+      - main
+
+jobs:
+
+  publish-backend:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Retrieve source code
+        uses: actions/checkout@v4
+
+      - name: Build and publish Docker Image
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/mirrors-qa-backend
+          tag-pattern: /^v([0-9.]+)$/
+          latest-on-tag: true
+          restrict-to: openzim/mirrors-qa
+          context: backend
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          repo_description: auto
+          repo_overview: auto

.github/workflows/backend-QA.yaml

@@ -3,13 +3,15 @@ name: Backend QA
 on:
   pull_request:
   push:
+    paths:
+      - 'backend/**'
     branches:
       - main
 
 jobs:
+
   check-qa:
     runs-on: ubuntu-22.04
-
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/worker-QA.yaml

@@ -3,13 +3,15 @@ name: Worker QA
 on:
   pull_request:
   push:
+    paths:
+      - 'worker/**'
     branches:
       - main
 
 jobs:
+
   check-qa:
     runs-on: ubuntu-22.04
-
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/worker-manager-Publish.yaml

@@ -0,0 +1,32 @@
+name: Publish worker-manager Docker image
+
+on:
+  push:
+    paths:
+      - 'worker/**'
+    branches:
+      - main
+
+jobs:
+
+  publish-worker-manager:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Retrieve source code
+        uses: actions/checkout@v4
+
+      - name: Build and publish Docker Image
+        uses: openzim/docker-publish-action@v10
+        with:
+          image-name: openzim/mirrors-qa-worker-manager
+          latest-on-tag: true
+          tag-pattern: /^v([0-9.]+)$/
+          restrict-to: openzim/mirrors-qa
+          context: worker
+          dockerfile: manager.Dockerfile
+          registries: ghcr.io
+          credentials:
+            GHCRIO_USERNAME=${{ secrets.GHCR_USERNAME }}
+            GHCRIO_TOKEN=${{ secrets.GHCR_TOKEN }}
+          repo_description: auto
+          repo_overview: auto

backend/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.11-slim
+LABEL org.opencontainers.image.source=https://github.com/kiwix/mirrors-qa
+# Copy code
+COPY src /src/src
+# Copy pyproject.toml and its dependencies
+COPY pyproject.toml README.md /src/
+
+# Install + cleanup
+RUN pip install --no-cache-dir /src \
+ && rm -rf /src
+
+EXPOSE 80
+
+CMD ["uvicorn", "mirrors_qa_backend.main:app", "--host", "0.0.0.0", "--port", "80"]

backend/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "mirrors_qa_backend"
-requires-python = ">=3.12,<3.13"
+requires-python = ">=3.11,<3.13"
 description = "mirrors-qa Backend API"
 readme = "README.md"
 authors = [
@@ -21,7 +21,7 @@ dependencies = [
 license = {text = "GPL-3.0-or-later"}
 classifiers = [
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.11",
     "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
 ]
 
@@ -54,19 +54,16 @@ dev = [
   "mirrors_qa_backend[check]",
 ]
 
-[project.scripts]
-mirrors-qa-backend = "backend:entrypoint"
-
 [tool.hatch.version]
-path = "src/backend/__about__.py"
+path = "src/mirrors_qa_backend/__about__.py"
 
 [tool.hatch.build]
 exclude = [
   "/.github",
 ]
 
 [tool.hatch.build.targets.wheel]
-packages = ["src/backend"]
+packages = ["src/mirrors_qa_backend"]
 
 [tool.hatch.envs.default]
 features = ["dev"]
@@ -188,7 +185,7 @@ unfixable = [
 ]
 
 [tool.ruff.lint.isort]
-known-first-party = ["backend"]
+known-first-party = ["mirrors_qa_backend"]
 
 [tool.ruff.lint.flake8-bugbear]
 # add exceptions to B008 for fastapi.
@@ -207,15 +204,15 @@ testpaths = ["tests"]
 pythonpath = [".", "src"]
 
 [tool.coverage.paths]
-backend = ["src/backend"]
+mirrors_qa_backend = ["src/mirrors_qa_backend"]
 tests = ["tests"]
 
 [tool.coverage.run]
-source_pkgs = ["backend"]
+source_pkgs = ["mirrors_qa_backend"]
 branch = true
 parallel = true
 omit = [
-  "src/backend/__about__.py",
+  "src/mirrors_qa_backend/__about__.py",
 ]
 
 [tool.coverage.report]
@@ -229,6 +226,6 @@ exclude_lines = [
 include = ["src", "tests", "tasks.py"]
 exclude = [".env/**", ".venv/**"]
 extraPaths = ["src"]
-pythonVersion = "3.12"
+pythonVersion = "3.11"
 typeCheckingMode="strict"
 disableBytesTypePromotions = true

backend/src/mirrors_qa_backend/__init__.py

@@ -0,0 +1,10 @@
+import logging
+import os
+
+logger = logging.getLogger("backend")
+
+if not logger.hasHandlers():
+    logger.setLevel(logging.DEBUG if bool(os.getenv("DEBUG")) else logging.INFO)
+    handler = logging.StreamHandler()
+    handler.setFormatter(logging.Formatter("[%(asctime)s: %(levelname)s] %(message)s"))
+    logger.addHandler(handler)

backend/alembic.ini → backend/src/mirrors_qa_backend/alembic.ini

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = src/backend/migrations
+script_location = migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
@@ -36,10 +36,10 @@ prepend_sys_path = .
 # sourceless = false
 
 # version location specification; This defaults
-# to src/backend/migirations/versions.  When using multiple version
+# to migirations/versions.  When using multiple version
 # directories, initial revisions must be specified with --version-path.
 # The path separator used here should be the separator specified by "version_path_separator" below.
-# version_locations = %(here)s/bar:%(here)s/bat:src/backend/migirations/versions
+# version_locations = %(here)s/bar:%(here)s/bat:migirations/versions
 
 # version path separator; As mentioned above, this is the character used to split
 # version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.

backend/src/mirrors_qa_backend/db/__init__.py

@@ -0,0 +1,46 @@
+import subprocess
+from collections.abc import Generator
+from pathlib import Path
+
+from sqlalchemy import SelectBase, create_engine, func, select
+from sqlalchemy.orm import Session as OrmSession
+from sqlalchemy.orm import sessionmaker
+
+from mirrors_qa_backend import logger
+from mirrors_qa_backend.db.models import Mirror
+from mirrors_qa_backend.settings import Settings
+
+Session = sessionmaker(
+    bind=create_engine(url=Settings.database_url, echo=False),
+    expire_on_commit=False,
+)
+
+
+def gen_dbsession() -> Generator[OrmSession, None, None]:
+    """FastAPI's Depends() compatible helper to provide a begin DB Session"""
+    with Session.begin() as session:
+        yield session
+
+
+def upgrade_db_schema():
+    """Checks if Alembic schema has been applied to the DB"""
+    src_dir = Path(__file__).parent.parent
+    logger.info(f"Upgrading database schema with config in {src_dir}")
+    subprocess.check_output(args=["alembic", "upgrade", "head"], cwd=src_dir)
+
+
+def count_from_stmt(session: OrmSession, stmt: SelectBase) -> int:
+    """Count all records returned by any statement `stmt` passed as parameter"""
+    return session.execute(
+        select(func.count()).select_from(stmt.subquery())
+    ).scalar_one()
+
+
+def initialize_mirrors() -> None:
+    with Session.begin() as session:
+        count = count_from_stmt(session, select(Mirror))
+        if count == 0:
+            logger.info("No mirrors exist in database.")
+            # TODO: update mirrors from https://download.kiwix.org/mirrors.html
+        else:
+            logger.info(f"Found {count} mirrors in database.")

backend/src/backend/db/models.py → backend/src/mirrors_qa_backend/db/models.py

@@ -15,7 +15,7 @@
 )
 from sqlalchemy.sql.schema import MetaData
 
-from backend.enums import StatusEnum
+from mirrors_qa_backend.enums import StatusEnum
 
 
 class Base(MappedAsDataclass, DeclarativeBase):

backend/src/mirrors_qa_backend/main.py

@@ -0,0 +1,23 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from mirrors_qa_backend import db
+from mirrors_qa_backend.routes import auth, tests
+
+
+@asynccontextmanager
+async def lifespan(_: FastAPI):
+    db.upgrade_db_schema()
+    db.initialize_mirrors()
+    yield
+
+
+def create_app(*, debug: bool = True):
+    app = FastAPI(debug=debug, docs_url="/", lifespan=lifespan)
+    app.include_router(router=tests.router)
+    app.include_router(router=auth.router)
+    return app
+
+
+app = create_app()

backend/src/backend/migrations/env.py → backend/src/mirrors_qa_backend/migrations/env.py

@@ -4,7 +4,7 @@
 from alembic import context
 from sqlalchemy import create_engine
 
-from backend.db.models import Base
+from mirrors_qa_backend.db.models import Base
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

backend/src/mirrors_qa_backend/routes/__init__.py

@@ -0,0 +1,8 @@
+from typing import Annotated
+
+from fastapi import Depends
+from sqlalchemy.orm import Session
+
+from mirrors_qa_backend.db import gen_dbsession
+
+DbSession = Annotated[Session, Depends(gen_dbsession)]

backend/src/mirrors_qa_backend/routes/auth.py

@@ -0,0 +1,9 @@
+from fastapi import APIRouter, Response, status
+from fastapi.responses import JSONResponse
+
+router = APIRouter(prefix="/auth", tags=["auth"])
+
+
+@router.post("/authenticate")
+def authenticate_user() -> Response:
+    return JSONResponse(content={"token": "token"}, status_code=status.HTTP_200_OK)

backend/src/mirrors_qa_backend/routes/tests.py

@@ -0,0 +1,57 @@
+from fastapi import APIRouter, Response, status
+from fastapi.responses import JSONResponse
+
+router = APIRouter(prefix="/tests", tags=["tests"])
+
+
+@router.get(
+    "",
+    status_code=status.HTTP_200_OK,
+    responses={
+        status.HTTP_200_OK: {"description": "Returns the list of tests."},
+    },
+)
+def list_tests() -> Response:
+    return JSONResponse(
+        content={
+            "tests": [],
+            "metadata": {
+                "currentPage": None,
+                "pageSize": 10,
+                "firstPage": None,
+                "lastPage": None,
+                "nextPage": None,
+                "totalRecords": 20,
+            },
+        },
+        status_code=status.HTTP_200_OK,
+    )
+
+
+@router.get(
+    "/{test_id}",
+    status_code=status.HTTP_200_OK,
+    responses={
+        status.HTTP_200_OK: {"description": "Returns the details of a test."},
+        status.HTTP_404_NOT_FOUND: {"description": "Test with id does not exist."},
+    },
+)
+def get_test(test_id: str) -> Response:
+    return JSONResponse(
+        content={"id": test_id},
+        status_code=status.HTTP_200_OK,
+    )
+
+
+@router.patch(
+    "/{test_id}",
+    status_code=status.HTTP_200_OK,
+    responses={
+        status.HTTP_200_OK: {"description": "Update the details of a test."},
+    },
+)
+def update_test(test_id: str) -> Response:
+    return JSONResponse(
+        content={"id": test_id},
+        status_code=status.HTTP_200_OK,
+    )