From ea47b48b850049e0fecbef182dd56e7791810cde Mon Sep 17 00:00:00 2001 From: dscc-admin-ch <> Date: Mon, 19 Aug 2024 12:57:20 +0000 Subject: [PATCH] Update from https://github.com/dscc-admin-ch/lomas/commit/a990333054a709dc6195d2a9956592e71c1e176a --- .../en/_modules/lomas_client/client.html | 10 +++++--- .../private_dataset/private_dataset.html | 9 ++++--- .../lomas_server/routes/routes_admin.html | 23 ++++++++++++++---- html/develop/en/genindex.html | 2 ++ html/develop/en/lomas_server.html | 1 + .../en/lomas_server.private_dataset.html | 18 ++++++++++++++ html/develop/en/lomas_server.routes.html | 10 +++++--- html/develop/en/objects.inv | Bin 12429 -> 12463 bytes html/develop/en/searchindex.js | 2 +- .../en/_modules/lomas_client/client.html | 10 +++++--- .../private_dataset/private_dataset.html | 9 ++++--- .../lomas_server/routes/routes_admin.html | 23 ++++++++++++++---- html/v0.1.0/en/genindex.html | 2 ++ html/v0.1.0/en/lomas_server.html | 1 + .../en/lomas_server.private_dataset.html | 18 ++++++++++++++ html/v0.1.0/en/lomas_server.routes.html | 10 +++++--- html/v0.1.0/en/objects.inv | Bin 12322 -> 12369 bytes html/v0.1.0/en/searchindex.js | 2 +- 18 files changed, 120 insertions(+), 30 deletions(-) diff --git a/html/develop/en/_modules/lomas_client/client.html b/html/develop/en/_modules/lomas_client/client.html index a46626f8..e8d5dfd6 100644 --- a/html/develop/en/_modules/lomas_client/client.html +++ b/html/develop/en/_modules/lomas_client/client.html @@ -94,7 +94,6 @@

Source code for lomas_client.client

 import base64
 import json
 from enum import StrEnum
-from io import StringIO
 from typing import Dict, List, Optional, Union
 
 import opendp as dp
@@ -219,8 +218,13 @@ 
Source code for lomas_client.client
 
         if res.status_code == HTTP_200_OK:
             data = res.content.decode("utf8")
-            df = pd.read_csv(StringIO(data))
-            return df
+            response = json.loads(data)
+            dummy_df = pd.DataFrame(response["dummy_dict"])
+            dummy_df = dummy_df.astype(response["dtypes"])
+            for col in response["datetime_columns"]:
+                dummy_df[col] = pd.to_datetime(dummy_df[col])
+            return dummy_df
+
         print(error_message(res))
         return None

 
diff --git a/html/develop/en/_modules/lomas_server/private_dataset/private_dataset.html b/html/develop/en/_modules/lomas_server/private_dataset/private_dataset.html
index 759f3a7c..06576022 100644
--- a/html/develop/en/_modules/lomas_server/private_dataset/private_dataset.html
+++ b/html/develop/en/_modules/lomas_server/private_dataset/private_dataset.html
@@ -118,7 +118,7 @@ 
Source code for lomas_server.private_dataset.private_dataset
self.metadata: dict = metadata
         self.dataset_observers: List[PrivateDatasetObserver] = []
 
-        dtypes, datetime_columns = _get_dtypes(metadata)
+        dtypes, datetime_columns = get_column_dtypes(metadata)
         self.dtypes: Dict[str, str] = dtypes
         self.datetime_columns: List[str] = datetime_columns
 
@@ -179,7 +179,9 @@ 
Source code for lomas_server.private_dataset.private_dataset
def _get_dtypes(metadata: dict) -> Tuple[Dict[str, str], List[str]]:
+

+[docs]
+def get_column_dtypes(metadata: dict) -> Tuple[Dict[str, str], List[str]]:
     """Extract and return the column types from the metadata.
 
     Args:
@@ -197,7 +199,8 @@ 
Source code for lomas_server.private_dataset.private_dataset
datetime_columns.append(col_name)
         else:
             dtypes[col_name] = data["type"]
-    return dtypes, datetime_columns
+    return dtypes, datetime_columns

+
diff --git a/html/develop/en/_modules/lomas_server/routes/routes_admin.html b/html/develop/en/_modules/lomas_server/routes/routes_admin.html index f9330115..c1163273 100644 --- a/html/develop/en/_modules/lomas_server/routes/routes_admin.html +++ b/html/develop/en/_modules/lomas_server/routes/routes_admin.html @@ -92,10 +92,11 @@

Source code for lomas_server.routes.routes_admin

 from fastapi import APIRouter, Body, Depends, Header, Request
-from fastapi.responses import JSONResponse, RedirectResponse, StreamingResponse
+from fastapi.responses import JSONResponse, RedirectResponse
 
 from dp_queries.dummy_dataset import make_dummy_dataset
-from routes.utils import server_live, stream_dataframe
+from private_dataset.private_dataset import get_column_dtypes
+from routes.utils import server_live
 from utils.error_handler import (
     KNOWN_EXCEPTIONS,
     InternalServerException,
@@ -243,7 +244,7 @@ 
Source code for lomas_server.routes.routes_admin

     request: Request,
     query_json: GetDummyDataset = Body(example_get_dummy_dataset),
     user_name: str = Header(None),
-) -> StreamingResponse:
+) -> JSONResponse:
     """
     Generates and returns a dummy dataset.
 
@@ -263,7 +264,8 @@ 
Source code for lomas_server.routes.routes_admin

         InternalServerException: For any other unforseen exceptions.
 
     Returns:
-        StreamingResponse: a pd.DataFrame representing the dummy dataset.
+        JSONResponse: a dict with the dataframe as a dict, the column types
+            and the list of datetime columns.
     """
     app = request.app
 
@@ -279,16 +281,27 @@ 
Source code for lomas_server.routes.routes_admin

         ds_metadata = app.state.admin_database.get_dataset_metadata(
             query_json.dataset_name
         )
+        dtypes, datetime_columns = get_column_dtypes(ds_metadata)
 
         dummy_df = make_dummy_dataset(
             ds_metadata, query_json.dummy_nb_rows, query_json.dummy_seed
         )
+
+        for col in datetime_columns:
+            dummy_df[col] = dummy_df[col].dt.strftime("%Y-%m-%dT%H:%M:%S")
+
     except KNOWN_EXCEPTIONS as e:
         raise e
     except Exception as e:
         raise InternalServerException(e) from e
 
-    return stream_dataframe(dummy_df)

+    return JSONResponse(
+        content={
+            "dummy_dict": dummy_df.to_dict(orient="records"),
+            "dtypes": dtypes,
+            "datetime_columns": datetime_columns,
+        }
+    )

 
 
 
diff --git a/html/develop/en/genindex.html b/html/develop/en/genindex.html
index 159b5590..fb6404fa 100644
--- a/html/develop/en/genindex.html
+++ b/html/develop/en/genindex.html
@@ -395,6 +395,8 @@ 
G

       
  • get_archives_of_user() (in module lomas_server.mongodb_admin)
 
    • 
       
  • get_collection() (in module lomas_server.mongodb_admin)
+
    • 
+      
  • get_column_dtypes() (in module lomas_server.private_dataset.private_dataset)
 
    • 
       
  • get_config() (in module lomas_server.utils.config)
 
diff --git a/html/develop/en/lomas_server.html b/html/develop/en/lomas_server.html
index 1e7c35c6..5f263b7a 100644
--- a/html/develop/en/lomas_server.html
+++ b/html/develop/en/lomas_server.html
@@ -294,6 +294,7 @@ 
    SubpackagesPrivateDataset.subscribe_for_memory_usage_updates()
    • 
 
 
+
  • get_column_dtypes()
    • 
 
 
 
  • lomas_server.private_dataset.s3_dataset module
    • 
diff --git a/html/v0.1.0/en/_modules/lomas_server/routes/routes_admin.html b/html/v0.1.0/en/_modules/lomas_server/routes/routes_admin.html
index 1b0c7cd8..0887181a 100644
--- a/html/v0.1.0/en/_modules/lomas_server/routes/routes_admin.html
+++ b/html/v0.1.0/en/_modules/lomas_server/routes/routes_admin.html
@@ -92,10 +92,11 @@
              
   
    Source code for lomas_server.routes.routes_admin
     from fastapi import APIRouter, Body, Depends, Header, Request
-from fastapi.responses import JSONResponse, RedirectResponse, StreamingResponse
+from fastapi.responses import JSONResponse, RedirectResponse
 
 from dp_queries.dummy_dataset import make_dummy_dataset
-from routes.utils import server_live, stream_dataframe
+from private_dataset.private_dataset import get_column_dtypes
+from routes.utils import server_live
 from utils.error_handler import (
     KNOWN_EXCEPTIONS,
     InternalServerException,
@@ -243,7 +244,7 @@ 
    Source code for lomas_server.routes.routes_admin
    
     request: Request,
     query_json: GetDummyDataset = Body(example_get_dummy_dataset),
     user_name: str = Header(None),
-) -> StreamingResponse:
+) -> JSONResponse:
     """
     Generates and returns a dummy dataset.
 
@@ -263,7 +264,8 @@ 
    Source code for lomas_server.routes.routes_admin
    
         InternalServerException: For any other unforseen exceptions.
 
     Returns:
-        StreamingResponse: a pd.DataFrame representing the dummy dataset.
+        JSONResponse: a dict with the dataframe as a dict, the column types
+            and the list of datetime columns.
     """
     app = request.app
 
@@ -279,16 +281,27 @@ 
    Source code for lomas_server.routes.routes_admin
    
         ds_metadata = app.state.admin_database.get_dataset_metadata(
             query_json.dataset_name
         )
+        dtypes, datetime_columns = get_column_dtypes(ds_metadata)
 
         dummy_df = make_dummy_dataset(
             ds_metadata, query_json.dummy_nb_rows, query_json.dummy_seed
         )
+
+        for col in datetime_columns:
+            dummy_df[col] = dummy_df[col].dt.strftime("%Y-%m-%dT%H:%M:%S")
+
     except KNOWN_EXCEPTIONS as e:
         raise e
     except Exception as e:
         raise InternalServerException(e) from e
 
-    return stream_dataframe(dummy_df)
    
+    return JSONResponse(
+        content={
+            "dummy_dict": dummy_df.to_dict(orient="records"),
+            "dtypes": dtypes,
+            "datetime_columns": datetime_columns,
+        }
+    )
    
 
 
 
diff --git a/html/v0.1.0/en/genindex.html b/html/v0.1.0/en/genindex.html
index 7f4b1d27..0b0d1a7d 100644
--- a/html/v0.1.0/en/genindex.html
+++ b/html/v0.1.0/en/genindex.html
@@ -395,6 +395,8 @@ 
    G
    
       
  • get_archives_of_user() (in module lomas_server.mongodb_admin)
 
    • 
       
  • get_collection() (in module lomas_server.mongodb_admin)
+
    • 
+      
  • get_column_dtypes() (in module lomas_server.private_dataset.private_dataset)
 
    • 
       
  • get_config() (in module lomas_server.utils.config)
 
diff --git a/html/v0.1.0/en/lomas_server.html b/html/v0.1.0/en/lomas_server.html
index 90ef3925..a1368a76 100644
--- a/html/v0.1.0/en/lomas_server.html
+++ b/html/v0.1.0/en/lomas_server.html
@@ -294,6 +294,7 @@ 
    SubpackagesPrivateDataset.subscribe_for_memory_usage_updates()
    • 
 
 
+
  • get_column_dtypes()
    • 
 
 
 
  • lomas_server.private_dataset.s3_dataset module