🗃️(api) add 'statique' materialized view

Having a SQL materialized view for "statique" data will ease recurrent
database queries joining the same tables frequently. Moreover this will
speed up database queries and response time for API endpoints using
them.

Makefile

@@ -3,7 +3,7 @@ SHELL := /bin/bash
 
 # -- Docker
 COMPOSE                    	 = bin/compose
-COMPOSE_UP                 	 = $(COMPOSE) up -d
+COMPOSE_UP                 	 = $(COMPOSE) up -d --remove-orphans
 COMPOSE_RUN                	 = $(COMPOSE) run --rm --no-deps
 COMPOSE_RUN_API            	 = $(COMPOSE_RUN) api
 COMPOSE_RUN_API_PIPENV     	 = $(COMPOSE_RUN_API) pipenv run
@@ -333,6 +333,10 @@ reset-db: \
   reset-dashboard-db
 .PHONY: reset-db
 
+refresh-api-static: ## Refresh the API Statique Materialized View
+	$(COMPOSE) exec api pipenv run python -m qualicharge refresh-static
+.PHONY: refresh-api-static
+
 reset-api-db: ## Reset the PostgreSQL API database
 	$(COMPOSE) stop
 	$(COMPOSE) down postgresql

src/api/CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to
   `SENTRY_PROFILES_SAMPLE_RATE` configuration
 - Set request's user (`username`) in Sentry's context
 - Add `Localisation.coordonneesXY` unique contraint [BC] 💥
+- Implement `Statique` materialized view
 
 ### Changed
 

src/api/Pipfile

@@ -26,6 +26,7 @@ questionary = "==2.1.0"
 sentry-sdk = {extras = ["fastapi"], version = "==2.20.0"}
 setuptools = "==75.8.0"
 sqlalchemy-timescaledb = "==0.4.1"
+sqlalchemy-utils = "==0.41.2"
 sqlmodel = "==0.0.22"
 typer = "==0.15.1"
 uvicorn = {extras = ["standard"] }

src/api/cron.json

@@ -0,0 +1,7 @@
+{
+  "jobs": [
+    {
+      "command": "*/10 * * * * python -m qualicharge refresh-static"
+    }
+  ]
+}

src/api/pyproject.toml

@@ -74,5 +74,6 @@ exclude = [
 [[tool.mypy.overrides]]
 module = [
   "shapely.*",
+  "sqlalchemy_utils.*"
 ]
 ignore_missing_imports = true

src/api/qualicharge/cli.py

@@ -14,6 +14,7 @@
 from rich.table import Table
 from sqlalchemy import Column as SAColumn
 from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError
+from sqlalchemy_utils import refresh_materialized_view
 from sqlmodel import Session as SMSession
 from sqlmodel import select
 
@@ -23,7 +24,7 @@
 from .db import get_session
 from .exceptions import IntegrityError as QCIntegrityError
 from .fixtures.operational_units import prefixes
-from .schemas.core import OperationalUnit
+from .schemas.core import STATIQUE_MV_TABLE_NAME, OperationalUnit
 from .schemas.sql import StatiqueImporter
 
 logging.basicConfig(
@@ -456,6 +457,18 @@ def import_static(ctx: typer.Context, input_file: Path):
     console.log("Saved (or updated) all entries successfully.")
 
 
+@app.command()
+def refresh_static(ctx: typer.Context, concurrently: bool = False):
+    """Refresh the Statique materialized view."""
+    session: SMSession = ctx.obj
+
+    # Refresh the database
+    console.log("Refreshing database…")
+    refresh_materialized_view(
+        session, STATIQUE_MV_TABLE_NAME, concurrently=concurrently
+    )
+
+
 @app.callback()
 def main(ctx: typer.Context):
     """Attach database session to the context object."""

src/api/qualicharge/migrations/env.py

@@ -3,9 +3,11 @@
 from logging.config import fileConfig
 
 from alembic import context
+from geoalchemy2.alembic_helpers import create_geospatial_index
 from sqlalchemy import engine_from_config, pool
 from sqlmodel import SQLModel
 
+
 from qualicharge.conf import settings
 
 # Nota bene: be sure to import all models that need to be migrated here

src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py

@@ -0,0 +1,47 @@
+"""Add statique materialized view
+
+Revision ID: 4b99d15436b0
+Revises: 9ae109e209c9
+Create Date: 2025-01-16 15:02:04.004411
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+from geoalchemy2.functions import ST_GeomFromEWKB
+from sqlalchemy_utils.view import CreateView, DropView
+
+from qualicharge.schemas.core import StatiqueMV, _StatiqueMV
+
+# revision identifiers, used by Alembic.
+revision: str = "4b99d15436b0"
+down_revision: Union[str, None] = "9ae109e209c9"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Create the Statique Materialized view and related indexes."""
+    op.execute(
+        CreateView(
+            _StatiqueMV.__table__.fullname, _StatiqueMV.selectable, materialized=True
+        )
+    )
+    op.create_geospatial_index(
+        "idx_statique_coordonneesXY",
+        _StatiqueMV.__table__.fullname,
+        [ST_GeomFromEWKB(StatiqueMV.coordonneesXY)],
+        unique=False,
+        postgresql_using="gist",
+    )
+    for idx in _StatiqueMV.__table__.indexes:
+        idx.create(op.get_bind())
+
+
+def downgrade() -> None:
+    """Delete the Statique Materialized View."""
+
+    op.execute(
+        DropView(_StatiqueMV.__table__.fullname, materialized=True, cascade=True)
+    )

src/api/qualicharge/schemas/core.py

@@ -1,7 +1,7 @@
 """QualiCharge core statique and dynamique schemas."""
 
 from enum import IntEnum
-from typing import TYPE_CHECKING, List, Optional, Union, cast
+from typing import TYPE_CHECKING, ClassVar, List, Optional, Union, cast
 from uuid import UUID, uuid4
 
 from geoalchemy2.shape import to_shape
@@ -19,11 +19,15 @@
 )
 from pydantic_extra_types.coordinate import Coordinate
 from shapely.geometry import mapping
-from sqlalchemy import event
+from sqlalchemy import Select, event
+from sqlalchemy import cast as SA_cast
 from sqlalchemy.dialects.postgresql import ENUM as PgEnum
+from sqlalchemy.orm import registry
 from sqlalchemy.schema import Column as SAColumn
+from sqlalchemy.schema import Index
 from sqlalchemy.types import Date, DateTime, String
-from sqlmodel import Field, Relationship, UniqueConstraint, select
+from sqlalchemy_utils import create_materialized_view
+from sqlmodel import Field, Relationship, SQLModel, UniqueConstraint, select
 from sqlmodel import Session as SMSession
 from sqlmodel.main import SQLModelConfig
 
@@ -44,12 +48,17 @@
     ImplantationStationEnum,
     NotFutureDate,
     RaccordementEnum,
+    Statique,
 )
 from . import BaseTimestampedSQLModel
 
 if TYPE_CHECKING:
     from qualicharge.auth.schemas import Group
 
+mapper_registry = registry()
+
+STATIQUE_MV_TABLE_NAME: str = "statique"
+
 
 class OperationalUnitTypeEnum(IntEnum):
     """Operational unit types."""
@@ -472,3 +481,93 @@ class Status(BaseTimestampedSQLModel, StatusBase, table=True):
     def id_pdc_itinerance(self) -> str:
         """Return the PointDeCharge.id_pdc_itinerance (used for serialization only)."""
         return self.point_de_charge.id_pdc_itinerance
+
+
+class StatiqueMV(Statique, SQLModel):
+    """Statique Materialized View."""
+
+    __tablename__ = STATIQUE_MV_TABLE_NAME
+
+    model_config = SQLModel.model_config
+
+
+class _StatiqueMV(SQLModel):
+    """Statique Materialized view.
+
+    NOTE: This is an internal model used **ONLY** for creating the materialized view.
+    """
+
+    selectable: ClassVar[Select] = (
+        select(  # type: ignore[call-overload, misc]
+            Amenageur.nom_amenageur,
+            Amenageur.siren_amenageur,
+            Amenageur.contact_amenageur,
+            Operateur.nom_operateur,
+            Operateur.contact_operateur,
+            Operateur.telephone_operateur,
+            Enseigne.nom_enseigne,
+            Station.id_station_itinerance,
+            Station.id_station_local,
+            Station.nom_station,
+            Station.implantation_station,
+            Localisation.adresse_station,
+            Localisation.code_insee_commune,
+            SA_cast(
+                Localisation.coordonneesXY,
+                Geometry(
+                    geometry_type="POINT",
+                    # WGS84 coordinates system
+                    srid=4326,
+                    spatial_index=False,
+                ),
+            ).label("coordonneesXY"),
+            Station.nbre_pdc,
+            PointDeCharge.id_pdc_itinerance,
+            PointDeCharge.id_pdc_local,
+            PointDeCharge.puissance_nominale,
+            PointDeCharge.prise_type_ef,
+            PointDeCharge.prise_type_2,
+            PointDeCharge.prise_type_combo_ccs,
+            PointDeCharge.prise_type_chademo,
+            PointDeCharge.prise_type_autre,
+            PointDeCharge.gratuit,
+            PointDeCharge.paiement_acte,
+            PointDeCharge.paiement_cb,
+            PointDeCharge.paiement_autre,
+            PointDeCharge.tarification,
+            Station.condition_acces,
+            PointDeCharge.reservation,
+            Station.horaires,
+            PointDeCharge.accessibilite_pmr,
+            PointDeCharge.restriction_gabarit,
+            Station.station_deux_roues,
+            Station.raccordement,
+            Station.num_pdl,
+            Station.date_mise_en_service,
+            PointDeCharge.observations,
+            Station.date_maj,
+            PointDeCharge.cable_t2_attache,
+        )
+        .select_from(PointDeCharge)
+        .join(Station)
+        .join(Amenageur)
+        .join(Operateur)
+        .join(Enseigne)
+        .join(Localisation)
+    )
+
+    __table__ = create_materialized_view(
+        name=STATIQUE_MV_TABLE_NAME,
+        selectable=selectable,
+        metadata=SQLModel.metadata,
+        indexes=[
+            Index("idx_statique_id_pdc_itinerance", "id_pdc_itinerance", unique=True),
+            Index(
+                "idx_statique_code_insee_commune",
+                "code_insee_commune",
+            ),
+        ],
+    )
+
+
+mapper_registry.map_imperatively(StatiqueMV, _StatiqueMV.__table__)

src/api/tests/schemas/test_static.py

@@ -5,9 +5,11 @@
 
 import pytest
 from geoalchemy2.shape import to_shape
+from geoalchemy2.types import WKBElement
 from pydantic_extra_types.coordinate import Coordinate
 from shapely.geometry import mapping
 from sqlalchemy.exc import IntegrityError
+from sqlalchemy_utils import refresh_materialized_view
 from sqlmodel import select
 
 from qualicharge.factories.static import (
@@ -18,8 +20,16 @@
     OperationalUnitFactory,
     PointDeChargeFactory,
     StationFactory,
+    StatiqueFactory,
 )
-from qualicharge.schemas.core import Amenageur, Localisation, OperationalUnit, Station
+from qualicharge.schemas.core import (
+    Amenageur,
+    Localisation,
+    OperationalUnit,
+    Station,
+    StatiqueMV,
+)
+from qualicharge.schemas.utils import save_statiques
 
 
 @pytest.mark.parametrize(
@@ -392,3 +402,19 @@ def test_operational_unit_create_stations_fk(db_session):
         select(Station).where(Station.operational_unit_id == operational_unit.id)
     ).all()
     assert len(stations) == n_stations + extra_stations
+
+
+def test_statique_materialized_view(db_session):
+    """Test the StatiqueMV schema."""
+    n_pdc = 4
+    statiques = StatiqueFactory.batch(n_pdc)
+    save_statiques(db_session, statiques)
+    refresh_materialized_view(db_session, "statique")
+
+    db_statiques = db_session.exec(select(StatiqueMV)).all()
+    assert len(db_statiques) == n_pdc
+
+    assert {s.id_pdc_itinerance for s in statiques} == {
+        s.id_pdc_itinerance for s in db_statiques
+    }
+    assert isinstance(db_statiques[0].coordonneesXY, WKBElement)