Merge branch 'pblottiere:main' into main

CHANGELOG.md

@@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## [1.1.0] - 2024-09-24
+
+### Added
+
+- Add Python files after installation of dependencies in Dockerfile : https://github.com/pblottiere/QSA/pull/68
+- Add basic cache entrypoints : https://github.com/pblottiere/QSA/pull/74
+- Add data type and band counts metadata for raster layers : https://github.com/pblottiere/QSA/pull/80
+
+### Fixed
+
+- Always create parent directory for internal sqlite database : https://github.com/pblottiere/QSA/pull/73
+- Clear MapProxy legends cache : https://github.com/pblottiere/QSA/pull/78
+
+
 ## [1.0.0] - 2024-07-31
 
 ### Added

README.md

@@ -1,6 +1,6 @@
 # QGIS Server Administration
 
-[![Release](https://img.shields.io/badge/release-1.0.0-green.svg)](https://github.com/pblottiere/QSA/releases)
+[![Release](https://img.shields.io/badge/release-1.1.0-green.svg)](https://github.com/pblottiere/QSA/releases)
 [![CI](https://img.shields.io/github/actions/workflow/status/pblottiere/QSA/tests.yml)](https://github.com/pblottiere/QSA/actions)
 [![Documentation](https://img.shields.io/badge/docs-Book-informational)](https://pblottiere.github.io/QSA/)
 

docs/src/qsa-api/endpoints/projects.md

@@ -175,3 +175,28 @@ $ curl "http://localhost:5000/api/projects/my_project/styles" \
     }
   }'
 ````
+
+## Cache
+
+| Method  |                      URL                         |         Description                                                                                                          |
+|---------|--------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|
+| GET     | `/api/projects/{project}/cache`                  | Return metadata about the cache                                                                                              |
+| POST    | `/api/projects/{project}/cache/reset`            | Clear cached data and reset cache configuration                                                                              |
+
+Example:
+
+```` shell
+$ curl "http://localhost:5000/api/projects/my_project/cache"
+{
+  "valid":true,
+  "storage":"filesystem"
+}
+````
+
+<div class="warning">
+Reset cache
+
+When a QGIS project is created manually without QSA, the cache is not
+initialized. This method allows to create the MapProxy configuration file
+accordingly.
+</div>

docs/src/qsa-api/installation.md

@@ -15,9 +15,9 @@ $ . venv/bin/activate
 A prebuilt image can be found on `ghcr.io/pblottiere/qsa`:
 
 ```` shell
-$ docker pull ghcr.io/pblottiere/qsa:1.0.0
+$ docker pull ghcr.io/pblottiere/qsa:1.1.0
 ````
 
 Otherwise the image can manually be built using:
-`docker build -t my-custom-qsa-image .`. See [Sandbox](sandbox/) for details
-how to use it.
+`docker build -t my-custom-qsa-image .`. See [Sandbox](../sandbox/index.html)
+for details how to use it.

docs/src/sandbox/raster/layers.md

@@ -23,8 +23,26 @@ true
 ### List layers and get metadata
 
 ```` shell
+# list layers
 $ curl "http://localhost:5000/api/projects/my_project/layers?schema=my_schema"
 ["polygons","dem"]
+
+# get metadata
+$ curl "http://localhost:5000/api/projects/my_project/layers/dem?schema=my_schema"
+{
+  "bands": 1,
+  "bbox": "18.6662979442000001 45.77670143760000343, 18.70359794419999844 45.81170143760000002",
+  "crs": "EPSG:4326",
+  "current_style": "default",
+  "data_type": "float32",
+  "name": "dem",
+  "source": "/dem.tif",
+  "styles": [
+    "default"
+  ],
+  "type": "raster",
+  "valid": true
+}
 ````
 
 

qsa-api/qsa_api/api/projects.py

@@ -436,3 +436,39 @@ def project_del_layer(name, layer_name):
     except Exception as e:
         logger().exception(str(e))
         return {"error": "internal server error"}, 415
+
+
+@projects.get("/<name>/cache")
+def project_cache(name):
+    log_request()
+    try:
+        psql_schema = request.args.get("schema", default="public")
+        project = QSAProject(name, psql_schema)
+        if project.exists():
+            cache_infos, err = project.cache_metadata()
+            if err:
+                return {"error": err}, 415
+            return jsonify(cache_infos), 201
+        else:
+            return {"error": "Project does not exist"}, 415
+    except Exception as e:
+        logger().exception(str(e))
+        return {"error": "internal server error"}, 415
+
+
+@projects.post("/<name>/cache/reset")
+def project_cache_reset(name):
+    log_request()
+    try:
+        psql_schema = request.args.get("schema", default="public")
+        project = QSAProject(name, psql_schema)
+        if project.exists():
+            rc, err = project.cache_reset()
+            if err:
+                return {"error": err}, 415
+            return jsonify(rc), 201
+        else:
+            return {"error": "Project does not exist"}, 415
+    except Exception as e:
+        logger().exception(str(e))
+        return {"error": "internal server error"}, 415

qsa-api/qsa_api/mapproxy/mapproxy.py

@@ -30,7 +30,12 @@ def write(self) -> None:
         with open(self._mapproxy_project, "w") as file:
             yaml.safe_dump(self.cfg, file, sort_keys=False)
 
-    def read(self) -> bool:
+    def read(self) -> (bool, str):
+        # if a QGIS project is created manually without QSA, the MapProxy
+        # configuration file may not be created at this point.
+        if not self._mapproxy_project.exists():
+            self.create()
+
         try:
             with open(self._mapproxy_project, "r") as file:
                 self.cfg = yaml.safe_load(file)
@@ -48,6 +53,21 @@ def read(self) -> bool:
 
         return True, ""
 
+    def metadata(self) -> dict:
+        md = {}
+
+        md["storage"] = ""
+        md["valid"] = False
+
+        if self._mapproxy_project.exists():
+            md["valid"] = True
+
+            md["storage"] = "filesystem"
+            if config().mapproxy_cache_s3_bucket:
+                md["storage"] = "s3"
+
+        return md
+
     def clear_cache(self, layer_name: str) -> None:
         if config().mapproxy_cache_s3_bucket:
             bucket_name = config().mapproxy_cache_s3_bucket
@@ -64,10 +84,14 @@ def clear_cache(self, layer_name: str) -> None:
             bucket.objects.filter(Prefix=cache_dir).delete()
         else:
             cache_dir = self._mapproxy_project.parent / "cache_data"
-            self.debug(f"Clear cache '{cache_dir}'")
+            self.debug(f"Clear tiles cache '{cache_dir}'")
             for d in cache_dir.glob(f"{layer_name}_cache_*"):
                 shutil.rmtree(d)
 
+            cache_dir = self._mapproxy_project.parent / "cache_data" / "legends"
+            self.debug(f"Clear legends cache '{cache_dir}'")
+            shutil.rmtree(cache_dir, ignore_errors=True)
+
     def add_layer(
         self,
         name: str,

qsa-api/qsa_api/project.py

@@ -129,8 +129,44 @@ def metadata(self) -> dict:
 
         if StorageBackend.type() == StorageBackend.POSTGRESQL:
             m["schema"] = self.schema
+
+        m["cache"] = "disabled"
+        if self._mapproxy_enabled:
+            m["cache"] = "mapproxy"
+
         return m
 
+    def cache_metadata(self) -> (dict, str):
+        if self._mapproxy_enabled:
+            return QSAMapProxy(self.name).metadata(), ""
+        return {}, "Cache is disabled"
+
+    def cache_reset(self) -> (bool, str):
+        if self._mapproxy_enabled:
+            mp = QSAMapProxy(self.name)
+            rc, err = mp.read()
+            if not rc:
+                return False, err
+
+            p = QgsProject()
+            p.read(self._qgis_project_uri)
+
+            for layer in p.mapLayers().values():
+                t = layer.type()
+                bbox = QSAProject._layer_bbox(layer)
+                epsg_code = QSAProject._layer_epsg_code(layer)
+
+                mp.remove_layer(layer.name())
+                mp.add_layer(
+                    layer.name(), bbox, epsg_code, t == Qgis.LayerType.Raster, None
+                )
+
+                mp.write()
+
+            return True, ""
+
+        return False, "Cache is disabled"
+
     def style_default(self, geometry: str) -> bool:
         con = sqlite3.connect(self.sqlite_db.as_posix())
         cur = con.cursor()
@@ -182,6 +218,9 @@ def layer(self, name: str) -> dict:
 
             if layer.type() == Qgis.LayerType.Vector:
                 infos["geometry"] = QgsWkbTypes.displayString(layer.wkbType())
+            elif layer.type() == Qgis.LayerType.Raster:
+                infos["bands"] = layer.bandCount()
+                infos["data_type"] = layer.dataProvider().dataType(1).name.lower()
 
             infos["source"] = layer.source()
             infos["crs"] = layer.crs().authid()
@@ -245,7 +284,7 @@ def layer_update_style(
     def layer_exists(self, name: str) -> bool:
         return bool(self.layer(name))
 
-    def remove_layer(self, name: str) -> None:
+    def remove_layer(self, name: str) -> bool:
         # remove layer in qgis project
         project = QgsProject()
         project.read(self._qgis_project_uri, Qgis.ProjectReadFlag.DontResolveLayers)
@@ -260,7 +299,11 @@ def remove_layer(self, name: str) -> None:
         # remove layer in mapproxy config
         if self._mapproxy_enabled:
             mp = QSAMapProxy(self.name)
-            mp.read()
+            rc, err = mp.read()
+            if not rc:
+                self.debug(err)
+                return False
+
             mp.remove_layer(name)
             mp.write()
 
@@ -280,6 +323,10 @@ def exists(self) -> bool:
             )
             projects = storage.listProjects(uri)
 
+            # necessary step if the project has been created without QSA
+            if self.name in projects:
+                self._qgis_projects_dir().mkdir(parents=True, exist_ok=True)
+
             return self.name in projects and self._qgis_projects_dir().exists()
 
     def create(self, author: str) -> (bool, str):
@@ -319,8 +366,13 @@ def remove(self) -> None:
         for layer in self.layers:
             self.remove_layer(layer)
 
+        # remove mapproxy config file
+        if self._mapproxy_enabled:
+            mp = QSAMapProxy(self.name)
+            mp.remove()
+
         # remove qsa projects dir
-        shutil.rmtree(self._qgis_project_dir)
+        shutil.rmtree(self._qgis_project_dir, ignore_errors=True)
 
         # remove remove qgis prohect in db if necessary
         if StorageBackend.type() == StorageBackend.POSTGRESQL:
@@ -424,20 +476,10 @@ def set_published_wfs_layers(qgis_project_instance, layer_ids):
         if self._mapproxy_enabled:
             self.debug("Update MapProxy configuration file")
 
-            bbox = list(
-                map(
-                    float,
-                    lyr.extent()
-                    .asWktCoordinates()
-                    .replace(",", "")
-                    .split(" "),
-                )
-            )
-
-            authid_items = lyr.crs().authid().split(":")
-            if len(authid_items) < 2:
+            bbox = QSAProject._layer_bbox(lyr)
+            epsg_code = QSAProject._layer_epsg_code(lyr)
+            if epsg_code < 0:
                 return False, f"Invalid CRS {lyr.crs().authid()}"
-            epsg_code = int(authid_items[1])
 
             self.debug(f"EPSG code {epsg_code}")
 
@@ -716,6 +758,25 @@ def _layer_provider(layer_type: Qgis.LayerType, datasource: str) -> str:
             provider = "gdal"
         return provider
 
+    @staticmethod
+    def _layer_epsg_code(lyr) -> int:
+        authid_items = lyr.crs().authid().split(":")
+        if len(authid_items) < 2:
+            return -1
+        return int(authid_items[1])
+
+    @staticmethod
+    def _layer_bbox(lyr) -> list:
+        return list(
+            map(
+                float,
+                lyr.extent()
+                .asWktCoordinates()
+                .replace(",", "")
+                .split(" "),
+            )
+        )
+
     @property
     def _qgis_project_uri(self) -> str:
         if StorageBackend.type() == StorageBackend.POSTGRESQL: