Dac 571 mount user specific wpsoutputs

CHANGES.rst

@@ -10,7 +10,7 @@ Changes
 Features / Changes
 ~~~~~~~~~~~~~~~~~~~~~
 * Add monitoring to the ``FileSystem`` handler to watch wps outputs data.
-* Synchronize public wps outputs data to the user workspaces folder with hardlinks for user access.
+* Synchronize both public and user wps outputs data to the workspace folder with hardlinks for user access.
 * Add resync endpoint to trigger a handler's resync operations. Only the ``FileSystem`` handler is implemented for now,
   regenerating hardlinks associated to wps outputs public data.
 

cowbird/handlers/impl/geoserver.py

@@ -11,7 +11,6 @@
 from magpie.models import Layer, Workspace
 from magpie.permissions import Access, Scope
 
-from cowbird.constants import DEFAULT_ADMIN_GID, DEFAULT_ADMIN_UID
 from cowbird.handlers.handler import HANDLER_URL_PARAM, HANDLER_WORKSPACE_DIR_PARAM, Handler
 from cowbird.handlers.handler_factory import HandlerFactory
 from cowbird.handlers.impl.magpie import GEOSERVER_READ_PERMISSIONS, GEOSERVER_WRITE_PERMISSIONS
@@ -20,7 +19,7 @@
 from cowbird.permissions_synchronizer import Permission
 from cowbird.request_task import RequestTask
 from cowbird.typedefs import JSON, SettingsType
-from cowbird.utils import CONTENT_TYPE_JSON, get_logger, update_filesystem_permissions
+from cowbird.utils import CONTENT_TYPE_JSON, apply_default_path_ownership, apply_new_path_permissions, get_logger
 
 GeoserverType: TypeAlias = "Geoserver"  # need a reference for the decorator before it gets defined
 
@@ -207,39 +206,6 @@ def get_shapefile_list(self, workspace_name: str, shapefile_name: str) -> List[s
         base_filename = self._shapefile_folder_dir(workspace_name) + "/" + shapefile_name
         return [base_filename + ext for ext in SHAPEFILE_ALL_EXTENSIONS]
 
-    @staticmethod
-    def _apply_new_path_permissions(path: str, is_readable: bool, is_writable: bool, is_executable: bool) -> None:
-        """
-        Applies new permissions to a path, if required.
-        """
-        # Only use the 3 last octal digits
-        previous_perms = os.stat(path)[stat.ST_MODE] & 0o777
-
-        new_perms = update_filesystem_permissions(previous_perms,
-                                                  is_readable=is_readable,
-                                                  is_writable=is_writable,
-                                                  is_executable=is_executable)
-        # Only apply chmod if there is an actual change, to avoid looping events between Magpie and Cowbird
-        if new_perms != previous_perms:
-            try:
-                os.chmod(path, new_perms)
-            except PermissionError as exc:
-                LOGGER.warning("Failed to change permissions on the %s path: %s", path, exc)
-
-    @staticmethod
-    def _apply_default_path_ownership(path: str) -> None:
-        """
-        Applies default ownership to a path, if required.
-        """
-        path_stat = os.stat(path)
-        # Only apply chown if there is an actual change, to avoid looping events between Magpie and Cowbird
-        if path_stat.st_uid != DEFAULT_ADMIN_UID or path_stat.st_gid != DEFAULT_ADMIN_GID:
-            try:
-                # This operation only works as root.
-                os.chown(path, DEFAULT_ADMIN_UID, DEFAULT_ADMIN_GID)
-            except PermissionError as exc:
-                LOGGER.warning("Failed to change ownership of the %s path: %s", path, exc)
-
     def _update_resource_paths_permissions(self,
                                            resource_type: str,
                                            permission: Permission,
@@ -277,8 +243,8 @@ def _update_resource_paths_permissions(self,
                 if path.endswith(tuple(SHAPEFILE_REQUIRED_EXTENSIONS)):
                     LOGGER.warning("%s could not be found and its permissions could not be updated.", path)
                 continue
-            self._apply_new_path_permissions(path, is_readable, is_writable, is_executable)
-            self._apply_default_path_ownership(path)
+            apply_new_path_permissions(path, is_readable, is_writable, is_executable)
+            apply_default_path_ownership(path)
 
     def _update_resource_paths_permissions_recursive(self,
                                                      resource: JSON,
@@ -530,7 +496,7 @@ def _update_magpie_workspace_permissions(self, workspace_name: str) -> None:
 
         datastore_dir_path = self._shapefile_folder_dir(workspace_name)
         # Make sure the directory has the right ownership
-        self._apply_default_path_ownership(datastore_dir_path)
+        apply_default_path_ownership(datastore_dir_path)
 
         workspace_status = os.stat(datastore_dir_path)[stat.ST_MODE]
         is_readable = bool(workspace_status & stat.S_IROTH and workspace_status & stat.S_IXOTH)
@@ -661,11 +627,11 @@ def _normalize_shapefile_permissions(self,
         """
         for shapefile in self.get_shapefile_list(workspace_name, shapefile_name):
             if os.path.exists(shapefile):
-                self._apply_default_path_ownership(shapefile)
-                self._apply_new_path_permissions(shapefile,
-                                                 is_readable=is_readable,
-                                                 is_writable=is_writable,
-                                                 is_executable=False)
+                apply_default_path_ownership(shapefile)
+                apply_new_path_permissions(shapefile,
+                                           is_readable=is_readable,
+                                           is_writable=is_writable,
+                                           is_executable=False)
 
     def remove_shapefile(self, workspace_name: str, filename: str) -> None:
         """

cowbird/handlers/impl/magpie.py

@@ -216,6 +216,24 @@ def get_geoserver_layer_res_id(self, workspace_name: str, layer_name: str, creat
                 parent_id=workspace_res_id)
         return layer_res_id
 
+    def get_user_list(self) -> List[str]:
+        """
+        Returns the list of all Magpie usernames.
+        """
+        resp = self._send_request(method="GET", url=f"{self.url}/users", params={"detail": False})
+        if resp.status_code != 200:
+            raise MagpieHttpError(f"Could not find the list of users. HttpError {resp.status_code} : {resp.text}")
+        return resp.json()["user_names"]
+
+    def get_user_id_from_user_name(self, user_name: str) -> int:
+        """
+        Finds the id of a user from his username.
+        """
+        resp = self._send_request(method="GET", url=f"{self.url}/users/{user_name}")
+        if resp.status_code != 200:
+            raise MagpieHttpError(f"Could not find the user `{user_name}`. HttpError {resp.status_code} : {resp.text}")
+        return resp.json()["user"]["user_id"]
+
     def get_user_name_from_user_id(self, user_id: int) -> str:
         """
         Finds the name of a user from his user id.
@@ -226,7 +244,7 @@ def get_user_name_from_user_id(self, user_id: int) -> str:
         for user_info in resp.json()["users"]:
             if "user_id" in user_info and user_info["user_id"] == user_id:
                 return user_info["user_name"]
-        raise MagpieHttpError(f"Could not find any user with the id {user_id}.")
+        raise MagpieHttpError(f"Could not find any user with the id `{user_id}`.")
 
     def get_user_permissions(self, user: str) -> Dict[str, JSON]:
         """
@@ -246,6 +264,16 @@ def get_user_permissions_by_res_id(self, user: str, res_id: int, effective: bool
                                   f"HttpError {resp.status_code} : {resp.text}")
         return resp.json()
 
+    def get_user_names_by_group_name(self, grp_name: str) -> List[str]:
+        """
+        Returns the list of Magpie usernames from a group.
+        """
+        resp = self._send_request(method="GET", url=f"{self.url}/groups/{grp_name}/users")
+        if resp.status_code != 200:
+            raise MagpieHttpError(f"Could not find the list of usernames from group `{grp_name}`. "
+                                  f"HttpError {resp.status_code} : {resp.text}")
+        return resp.json()["user_names"]
+
     def get_group_permissions(self, grp: str) -> Dict[str, JSON]:
         """
         Gets all group resource permissions.

cowbird/utils.py

@@ -28,7 +28,7 @@
 from webob.headers import EnvironHeaders, ResponseHeaders
 
 from cowbird import __meta__
-from cowbird.constants import get_constant, validate_required
+from cowbird.constants import DEFAULT_ADMIN_GID, DEFAULT_ADMIN_UID, get_constant, validate_required
 from cowbird.typedefs import (
     JSON,
     AnyHeadersType,
@@ -588,6 +588,25 @@
                             raise_missing=False, raise_not_set=False))
 
 
+def apply_new_path_permissions(path: str, is_readable: bool, is_writable: bool, is_executable: bool) -> None:
+    """
+    Applies new permissions to a path, if required.
+    """
+    # Only use the 3 last octal digits
+    previous_perms = os.stat(path)[stat.ST_MODE] & 0o777
+
+    new_perms = update_filesystem_permissions(previous_perms,
+                                              is_readable=is_readable,
+                                              is_writable=is_writable,
+                                              is_executable=is_executable)
+    # Only apply chmod if there is an actual change, to avoid looping events between Magpie and Cowbird
+    if new_perms != previous_perms:
+        try:
+            os.chmod(path, new_perms)
+        except PermissionError as exc:
+            LOGGER.warning("Failed to change permissions on the %s path: %s", path, exc)
+
+
 def update_filesystem_permissions(permission: int, is_readable: bool, is_writable: bool, is_executable: bool) -> int:
     """
     Applies/remove read, write and execute permissions (user only) to the input file system permissions.
@@ -598,3 +617,17 @@
     # Only use the 3 last octal digits
     permission = permission & 0o777
     return permission
+
+
+def apply_default_path_ownership(path: str) -> None:
+    """
+    Applies default ownership to a path, if required.
+    """
+    path_stat = os.stat(path)
+    # Only apply chown if there is an actual change, to avoid looping events between Magpie and Cowbird
+    if path_stat.st_uid != DEFAULT_ADMIN_UID or path_stat.st_gid != DEFAULT_ADMIN_GID:
+        try:
+            # This operation only works as root.
+            os.chown(path, DEFAULT_ADMIN_UID, DEFAULT_ADMIN_GID)
+        except PermissionError as exc:
+            LOGGER.warning("Failed to change ownership of the %s path: %s", path, exc)

docs/components.rst

@@ -63,8 +63,23 @@ user workspaces location (e.g.:``/user_workspaces/public/wpsoutputs``). When a `
 `birdhouse`, the folder containing the hardlinks will be mounted as volume, so that the users have access
 via their `JupyterLab` instance. The volume will be made read-only to prevent a user from modifying the public data.
 
-..
-    TODO: add section on user data when implemented
+The user wps outputs data is made accessible by generating hardlinks from a wps outputs data directory containing user
+data to a subdirectory found in the related user's workspace. For example, with a source path
+``/wpsoutputs/<bird-name>/users/<user-id>/<job-id>/<output-file>``, a hardlink is generated at the path
+``/user_workspaces/<user-name>/wpsoutputs/<bird-name>/<job-id>/<output-file>``. The hardlink path uses a similar
+structure as found in the source path, but removes the redundant ``users`` and ``<user-id>`` path segments. The
+hardlink files will be automatically available to the user on a `JupyterLab` instance since the workspace is mounted as
+a volume. Any file that is found under a directory ``/wpsoutputs/<bird-name>/users/<user-id>/`` is considered to be
+user data and any outside file is considered public.
+
+The permissions found on the files are synchronized with the permissions found on `Magpie`. If `Magpie` uses a
+`secure-data-proxy` service, this service handles the permissions of those files. If a file does not have a
+corresponding route on the `secure-data-proxy` service, it will use the closest parent permissions. Note that the route
+resources found under the `secure-data-proxy` service must match exactly a path on the filesystem, starting with the
+directory name ``wpsoutputs``, and following with the desired children directories/file names. If no `secure-data-proxy`
+service is found, the user files are assumed to be fully available with read and write permissions for the user. Note
+that if the file does not have any read or write permissions, the hardlink will not be available in the user's
+workspace.
 
 Note that different design choices were made to respect the constraints of the file system and to prevent the user from
 accessing forbidden data: