From 023973c0a75d293db7800bd46e07b1e69f073217 Mon Sep 17 00:00:00 2001 From: Julien Maupetit Date: Wed, 18 Dec 2024 19:26:01 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=97=83=EF=B8=8F(api)=20add=20'statique'?= =?UTF-8?q?=20materialized=20view?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Having a SQL materialized view for "statique" data will ease recurrent database queries joining the same tables frequently. Moreover this will speed up database queries and response time for API endpoints using them. --- Makefile | 6 +- src/api/CHANGELOG.md | 1 + src/api/Pipfile | 1 + src/api/Pipfile.lock | 17 ++- src/api/cron.json | 7 ++ src/api/pyproject.toml | 1 + src/api/qualicharge/cli.py | 15 ++- src/api/qualicharge/migrations/env.py | 2 + ...d15436b0_add_statique_materialized_view.py | 47 ++++++++ src/api/qualicharge/schemas/core.py | 105 +++++++++++++++++- src/api/tests/schemas/test_static.py | 28 ++++- 11 files changed, 220 insertions(+), 10 deletions(-) create mode 100644 src/api/cron.json create mode 100644 src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py diff --git a/Makefile b/Makefile index c891153c..1ae4c79a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SHELL := /bin/bash # -- Docker COMPOSE = bin/compose -COMPOSE_UP = $(COMPOSE) up -d +COMPOSE_UP = $(COMPOSE) up -d --remove-orphans COMPOSE_RUN = $(COMPOSE) run --rm --no-deps COMPOSE_RUN_API = $(COMPOSE_RUN) api COMPOSE_RUN_API_PIPENV = $(COMPOSE_RUN_API) pipenv run @@ -333,6 +333,10 @@ reset-db: \ reset-dashboard-db .PHONY: reset-db +refresh-api-static: ## Refresh the API Statique Materialized View + $(COMPOSE) exec api pipenv run python -m qualicharge refresh-static +.PHONY: refresh-api-static + reset-api-db: ## Reset the PostgreSQL API database $(COMPOSE) stop $(COMPOSE) down postgresql diff --git a/src/api/CHANGELOG.md b/src/api/CHANGELOG.md index 792eed96..6a88f767 100644 --- a/src/api/CHANGELOG.md +++ b/src/api/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to `SENTRY_PROFILES_SAMPLE_RATE` configuration - Set request's user (`username`) in Sentry's context - Add `Localisation.coordonneesXY` unique contraint [BC] šŸ’„ +- Implement `Statique` materialized view ### Changed diff --git a/src/api/Pipfile b/src/api/Pipfile index 18b1c43e..455edd9b 100644 --- a/src/api/Pipfile +++ b/src/api/Pipfile @@ -26,6 +26,7 @@ questionary = "==2.1.0" sentry-sdk = {extras = ["fastapi"], version = "==2.20.0"} setuptools = "==75.8.0" sqlalchemy-timescaledb = "==0.4.1" +sqlalchemy-utils = "==0.41.2" sqlmodel = "==0.0.22" typer = "==0.15.1" uvicorn = {extras = ["standard"] } diff --git a/src/api/Pipfile.lock b/src/api/Pipfile.lock index b31fbfbf..3c96292f 100644 --- a/src/api/Pipfile.lock +++ b/src/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6c812bc177a6f0da310185aa53a02103338be8ddc24295c1aa061820f9a30352" + "sha256": "8dda9b1e97aa4c777e193b31fa0da4b39e75107360a5a73d5b167170d1b744af" }, "pipfile-spec": 6, "requires": { @@ -1353,6 +1353,15 @@ "index": "pypi", "version": "==0.4.1" }, + "sqlalchemy-utils": { + "hashes": [ + "sha256:85cf3842da2bf060760f955f8467b87983fb2e30f1764fd0e24a48307dc8ec6e", + "sha256:bc599c8c3b3319e53ce6c5c3c471120bd325d0071fb6f38a10e924e3d07b9990" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==0.41.2" + }, "sqlmodel": { "hashes": [ "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e", @@ -2109,11 +2118,11 @@ }, "faker": { "hashes": [ - "sha256:49dde3b06a5602177bc2ad013149b6f60a290b7154539180d37b6f876ae79b20", - "sha256:ac4cf2f967ce02c898efa50651c43180bd658a7707cfd676fcc5410ad1482c03" + "sha256:42f2da8cf561e38c72b25e9891168b1e25fec42b6b0b5b0b6cd6041da54af885", + "sha256:926d2301787220e0554c2e39afc4dc535ce4b0a8d0a089657137999f66334ef4" ], "markers": "python_version >= '3.8'", - "version": "==33.3.1" + "version": "==35.0.0" }, "flask": { "hashes": [ diff --git a/src/api/cron.json b/src/api/cron.json new file mode 100644 index 00000000..443ca356 --- /dev/null +++ b/src/api/cron.json @@ -0,0 +1,7 @@ +{ + "jobs": [ + { + "command": "*/10 * * * * python -m qualicharge refresh-static" + } + ] +} diff --git a/src/api/pyproject.toml b/src/api/pyproject.toml index 187ee919..7c60d7aa 100644 --- a/src/api/pyproject.toml +++ b/src/api/pyproject.toml @@ -74,5 +74,6 @@ exclude = [ [[tool.mypy.overrides]] module = [ "shapely.*", + "sqlalchemy_utils.*" ] ignore_missing_imports = true diff --git a/src/api/qualicharge/cli.py b/src/api/qualicharge/cli.py index 8b021447..bb986f06 100644 --- a/src/api/qualicharge/cli.py +++ b/src/api/qualicharge/cli.py @@ -14,6 +14,7 @@ from rich.table import Table from sqlalchemy import Column as SAColumn from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError +from sqlalchemy_utils import refresh_materialized_view from sqlmodel import Session as SMSession from sqlmodel import select @@ -23,7 +24,7 @@ from .db import get_session from .exceptions import IntegrityError as QCIntegrityError from .fixtures.operational_units import prefixes -from .schemas.core import OperationalUnit +from .schemas.core import STATIQUE_MV_TABLE_NAME, OperationalUnit from .schemas.sql import StatiqueImporter logging.basicConfig( @@ -456,6 +457,18 @@ def import_static(ctx: typer.Context, input_file: Path): console.log("Saved (or updated) all entries successfully.") +@app.command() +def refresh_static(ctx: typer.Context, concurrently: bool = False): + """Refresh the Statique materialized view.""" + session: SMSession = ctx.obj + + # Refresh the database + console.log("Refreshing databaseā€¦") + refresh_materialized_view( + session, STATIQUE_MV_TABLE_NAME, concurrently=concurrently + ) + + @app.callback() def main(ctx: typer.Context): """Attach database session to the context object.""" diff --git a/src/api/qualicharge/migrations/env.py b/src/api/qualicharge/migrations/env.py index f00bba8d..cbd9bf79 100644 --- a/src/api/qualicharge/migrations/env.py +++ b/src/api/qualicharge/migrations/env.py @@ -3,9 +3,11 @@ from logging.config import fileConfig from alembic import context +from geoalchemy2.alembic_helpers import create_geospatial_index from sqlalchemy import engine_from_config, pool from sqlmodel import SQLModel + from qualicharge.conf import settings # Nota bene: be sure to import all models that need to be migrated here diff --git a/src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py b/src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py new file mode 100644 index 00000000..82e37aec --- /dev/null +++ b/src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py @@ -0,0 +1,47 @@ +"""Add statique materialized view + +Revision ID: 4b99d15436b0 +Revises: 9ae109e209c9 +Create Date: 2025-01-16 15:02:04.004411 + +""" + +from typing import Sequence, Union + +from alembic import op +from geoalchemy2.functions import ST_GeomFromEWKB +from sqlalchemy_utils.view import CreateView, DropView + +from qualicharge.schemas.core import StatiqueMV, _StatiqueMV + +# revision identifiers, used by Alembic. +revision: str = "4b99d15436b0" +down_revision: Union[str, None] = "9ae109e209c9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the Statique Materialized view and related indexes.""" + op.execute( + CreateView( + _StatiqueMV.__table__.fullname, _StatiqueMV.selectable, materialized=True + ) + ) + op.create_geospatial_index( + "idx_statique_coordonneesXY", + _StatiqueMV.__table__.fullname, + [ST_GeomFromEWKB(StatiqueMV.coordonneesXY)], + unique=False, + postgresql_using="gist", + ) + for idx in _StatiqueMV.__table__.indexes: + idx.create(op.get_bind()) + + +def downgrade() -> None: + """Delete the Statique Materialized View.""" + + op.execute( + DropView(_StatiqueMV.__table__.fullname, materialized=True, cascade=True) + ) diff --git a/src/api/qualicharge/schemas/core.py b/src/api/qualicharge/schemas/core.py index 7d1b38fe..30fc9b84 100644 --- a/src/api/qualicharge/schemas/core.py +++ b/src/api/qualicharge/schemas/core.py @@ -1,7 +1,7 @@ """QualiCharge core statique and dynamique schemas.""" from enum import IntEnum -from typing import TYPE_CHECKING, List, Optional, Union, cast +from typing import TYPE_CHECKING, ClassVar, List, Optional, Union, cast from uuid import UUID, uuid4 from geoalchemy2.shape import to_shape @@ -19,11 +19,15 @@ ) from pydantic_extra_types.coordinate import Coordinate from shapely.geometry import mapping -from sqlalchemy import event +from sqlalchemy import Select, event +from sqlalchemy import cast as SA_cast from sqlalchemy.dialects.postgresql import ENUM as PgEnum +from sqlalchemy.orm import registry from sqlalchemy.schema import Column as SAColumn +from sqlalchemy.schema import Index from sqlalchemy.types import Date, DateTime, String -from sqlmodel import Field, Relationship, UniqueConstraint, select +from sqlalchemy_utils import create_materialized_view +from sqlmodel import Field, Relationship, SQLModel, UniqueConstraint, select from sqlmodel import Session as SMSession from sqlmodel.main import SQLModelConfig @@ -44,12 +48,17 @@ ImplantationStationEnum, NotFutureDate, RaccordementEnum, + Statique, ) from . import BaseTimestampedSQLModel if TYPE_CHECKING: from qualicharge.auth.schemas import Group +mapper_registry = registry() + +STATIQUE_MV_TABLE_NAME: str = "statique" + class OperationalUnitTypeEnum(IntEnum): """Operational unit types.""" @@ -472,3 +481,93 @@ class Status(BaseTimestampedSQLModel, StatusBase, table=True): def id_pdc_itinerance(self) -> str: """Return the PointDeCharge.id_pdc_itinerance (used for serialization only).""" return self.point_de_charge.id_pdc_itinerance + + +class StatiqueMV(Statique, SQLModel): + """Statique Materialized View.""" + + __tablename__ = STATIQUE_MV_TABLE_NAME + + model_config = SQLModel.model_config + + +class _StatiqueMV(SQLModel): + """Statique Materialized view. + + NOTE: This is an internal model used **ONLY** for creating the materialized view. + """ + + selectable: ClassVar[Select] = ( + select( # type: ignore[call-overload, misc] + Amenageur.nom_amenageur, + Amenageur.siren_amenageur, + Amenageur.contact_amenageur, + Operateur.nom_operateur, + Operateur.contact_operateur, + Operateur.telephone_operateur, + Enseigne.nom_enseigne, + Station.id_station_itinerance, + Station.id_station_local, + Station.nom_station, + Station.implantation_station, + Localisation.adresse_station, + Localisation.code_insee_commune, + SA_cast( + Localisation.coordonneesXY, + Geometry( + geometry_type="POINT", + # WGS84 coordinates system + srid=4326, + spatial_index=False, + ), + ).label("coordonneesXY"), + Station.nbre_pdc, + PointDeCharge.id_pdc_itinerance, + PointDeCharge.id_pdc_local, + PointDeCharge.puissance_nominale, + PointDeCharge.prise_type_ef, + PointDeCharge.prise_type_2, + PointDeCharge.prise_type_combo_ccs, + PointDeCharge.prise_type_chademo, + PointDeCharge.prise_type_autre, + PointDeCharge.gratuit, + PointDeCharge.paiement_acte, + PointDeCharge.paiement_cb, + PointDeCharge.paiement_autre, + PointDeCharge.tarification, + Station.condition_acces, + PointDeCharge.reservation, + Station.horaires, + PointDeCharge.accessibilite_pmr, + PointDeCharge.restriction_gabarit, + Station.station_deux_roues, + Station.raccordement, + Station.num_pdl, + Station.date_mise_en_service, + PointDeCharge.observations, + Station.date_maj, + PointDeCharge.cable_t2_attache, + ) + .select_from(PointDeCharge) + .join(Station) + .join(Amenageur) + .join(Operateur) + .join(Enseigne) + .join(Localisation) + ) + + __table__ = create_materialized_view( + name=STATIQUE_MV_TABLE_NAME, + selectable=selectable, + metadata=SQLModel.metadata, + indexes=[ + Index("idx_statique_id_pdc_itinerance", "id_pdc_itinerance", unique=True), + Index( + "idx_statique_code_insee_commune", + "code_insee_commune", + ), + ], + ) + + +mapper_registry.map_imperatively(StatiqueMV, _StatiqueMV.__table__) diff --git a/src/api/tests/schemas/test_static.py b/src/api/tests/schemas/test_static.py index cef105f4..86e816ba 100644 --- a/src/api/tests/schemas/test_static.py +++ b/src/api/tests/schemas/test_static.py @@ -5,9 +5,11 @@ import pytest from geoalchemy2.shape import to_shape +from geoalchemy2.types import WKBElement from pydantic_extra_types.coordinate import Coordinate from shapely.geometry import mapping from sqlalchemy.exc import IntegrityError +from sqlalchemy_utils import refresh_materialized_view from sqlmodel import select from qualicharge.factories.static import ( @@ -18,8 +20,16 @@ OperationalUnitFactory, PointDeChargeFactory, StationFactory, + StatiqueFactory, ) -from qualicharge.schemas.core import Amenageur, Localisation, OperationalUnit, Station +from qualicharge.schemas.core import ( + Amenageur, + Localisation, + OperationalUnit, + Station, + StatiqueMV, +) +from qualicharge.schemas.utils import save_statiques @pytest.mark.parametrize( @@ -392,3 +402,19 @@ def test_operational_unit_create_stations_fk(db_session): select(Station).where(Station.operational_unit_id == operational_unit.id) ).all() assert len(stations) == n_stations + extra_stations + + +def test_statique_materialized_view(db_session): + """Test the StatiqueMV schema.""" + n_pdc = 4 + statiques = StatiqueFactory.batch(n_pdc) + save_statiques(db_session, statiques) + refresh_materialized_view(db_session, "statique") + + db_statiques = db_session.exec(select(StatiqueMV)).all() + assert len(db_statiques) == n_pdc + + assert {s.id_pdc_itinerance for s in statiques} == { + s.id_pdc_itinerance for s in db_statiques + } + assert isinstance(db_statiques[0].coordonneesXY, WKBElement)