Update from dscc-admin-ch/lomas@a990333

html/develop/en/_modules/lomas_client/client.html

@@ -94,7 +94,6 @@ <h1>Source code for lomas_client.client</h1><div class="highlight"><pre>
 <span></span><span class="kn">import</span> <span class="nn">base64</span>
 <span class="kn">import</span> <span class="nn">json</span>
 <span class="kn">from</span> <span class="nn">enum</span> <span class="kn">import</span> <span class="n">StrEnum</span>
-<span class="kn">from</span> <span class="nn">io</span> <span class="kn">import</span> <span class="n">StringIO</span>
 <span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">Dict</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Union</span>
 
 <span class="kn">import</span> <span class="nn">opendp</span> <span class="k">as</span> <span class="nn">dp</span>
@@ -219,8 +218,13 @@ <h1>Source code for lomas_client.client</h1><div class="highlight"><pre>
 
         <span class="k">if</span> <span class="n">res</span><span class="o">.</span><span class="n">status_code</span> <span class="o">==</span> <span class="n">HTTP_200_OK</span><span class="p">:</span>
             <span class="n">data</span> <span class="o">=</span> <span class="n">res</span><span class="o">.</span><span class="n">content</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="s2">&quot;utf8&quot;</span><span class="p">)</span>
-            <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">read_csv</span><span class="p">(</span><span class="n">StringIO</span><span class="p">(</span><span class="n">data</span><span class="p">))</span>
-            <span class="k">return</span> <span class="n">df</span>
+            <span class="n">response</span> <span class="o">=</span> <span class="n">json</span><span class="o">.</span><span class="n">loads</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
+            <span class="n">dummy_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;dummy_dict&quot;</span><span class="p">])</span>
+            <span class="n">dummy_df</span> <span class="o">=</span> <span class="n">dummy_df</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="n">response</span><span class="p">[</span><span class="s2">&quot;dtypes&quot;</span><span class="p">])</span>
+            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">response</span><span class="p">[</span><span class="s2">&quot;datetime_columns&quot;</span><span class="p">]:</span>
+                <span class="n">dummy_df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">to_datetime</span><span class="p">(</span><span class="n">dummy_df</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
+            <span class="k">return</span> <span class="n">dummy_df</span>
+
         <span class="nb">print</span><span class="p">(</span><span class="n">error_message</span><span class="p">(</span><span class="n">res</span><span class="p">))</span>
         <span class="k">return</span> <span class="kc">None</span></div>
 

html/develop/en/_modules/lomas_server/private_dataset/private_dataset.html

@@ -118,7 +118,7 @@ <h1>Source code for lomas_server.private_dataset.private_dataset</h1><div class=
         <span class="bp">self</span><span class="o">.</span><span class="n">metadata</span><span class="p">:</span> <span class="nb">dict</span> <span class="o">=</span> <span class="n">metadata</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dataset_observers</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">PrivateDatasetObserver</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
 
-        <span class="n">dtypes</span><span class="p">,</span> <span class="n">datetime_columns</span> <span class="o">=</span> <span class="n">_get_dtypes</span><span class="p">(</span><span class="n">metadata</span><span class="p">)</span>
+        <span class="n">dtypes</span><span class="p">,</span> <span class="n">datetime_columns</span> <span class="o">=</span> <span class="n">get_column_dtypes</span><span class="p">(</span><span class="n">metadata</span><span class="p">)</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">dtypes</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">dtypes</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">datetime_columns</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="n">datetime_columns</span>
 
@@ -179,7 +179,9 @@ <h1>Source code for lomas_server.private_dataset.private_dataset</h1><div class=
 
 
 
-<span class="k">def</span> <span class="nf">_get_dtypes</span><span class="p">(</span><span class="n">metadata</span><span class="p">:</span> <span class="nb">dict</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
+<div class="viewcode-block" id="get_column_dtypes">
+<a class="viewcode-back" href="../../../lomas_server.private_dataset.html#lomas_server.private_dataset.private_dataset.get_column_dtypes">[docs]</a>
+<span class="k">def</span> <span class="nf">get_column_dtypes</span><span class="p">(</span><span class="n">metadata</span><span class="p">:</span> <span class="nb">dict</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;Extract and return the column types from the metadata.</span>
 
 <span class="sd">    Args:</span>
@@ -197,7 +199,8 @@ <h1>Source code for lomas_server.private_dataset.private_dataset</h1><div class=
             <span class="n">datetime_columns</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">col_name</span><span class="p">)</span>
         <span class="k">else</span><span class="p">:</span>
             <span class="n">dtypes</span><span class="p">[</span><span class="n">col_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">data</span><span class="p">[</span><span class="s2">&quot;type&quot;</span><span class="p">]</span>
-    <span class="k">return</span> <span class="n">dtypes</span><span class="p">,</span> <span class="n">datetime_columns</span>
+    <span class="k">return</span> <span class="n">dtypes</span><span class="p">,</span> <span class="n">datetime_columns</span></div>
+
 </pre></div>
 
            </div>

html/develop/en/_modules/lomas_server/routes/routes_admin.html

@@ -92,10 +92,11 @@
 
   <h1>Source code for lomas_server.routes.routes_admin</h1><div class="highlight"><pre>
 <span></span><span class="kn">from</span> <span class="nn">fastapi</span> <span class="kn">import</span> <span class="n">APIRouter</span><span class="p">,</span> <span class="n">Body</span><span class="p">,</span> <span class="n">Depends</span><span class="p">,</span> <span class="n">Header</span><span class="p">,</span> <span class="n">Request</span>
-<span class="kn">from</span> <span class="nn">fastapi.responses</span> <span class="kn">import</span> <span class="n">JSONResponse</span><span class="p">,</span> <span class="n">RedirectResponse</span><span class="p">,</span> <span class="n">StreamingResponse</span>
+<span class="kn">from</span> <span class="nn">fastapi.responses</span> <span class="kn">import</span> <span class="n">JSONResponse</span><span class="p">,</span> <span class="n">RedirectResponse</span>
 
 <span class="kn">from</span> <span class="nn">dp_queries.dummy_dataset</span> <span class="kn">import</span> <span class="n">make_dummy_dataset</span>
-<span class="kn">from</span> <span class="nn">routes.utils</span> <span class="kn">import</span> <span class="n">server_live</span><span class="p">,</span> <span class="n">stream_dataframe</span>
+<span class="kn">from</span> <span class="nn">private_dataset.private_dataset</span> <span class="kn">import</span> <span class="n">get_column_dtypes</span>
+<span class="kn">from</span> <span class="nn">routes.utils</span> <span class="kn">import</span> <span class="n">server_live</span>
 <span class="kn">from</span> <span class="nn">utils.error_handler</span> <span class="kn">import</span> <span class="p">(</span>
     <span class="n">KNOWN_EXCEPTIONS</span><span class="p">,</span>
     <span class="n">InternalServerException</span><span class="p">,</span>
@@ -243,7 +244,7 @@ <h1>Source code for lomas_server.routes.routes_admin</h1><div class="highlight">
     <span class="n">request</span><span class="p">:</span> <span class="n">Request</span><span class="p">,</span>
     <span class="n">query_json</span><span class="p">:</span> <span class="n">GetDummyDataset</span> <span class="o">=</span> <span class="n">Body</span><span class="p">(</span><span class="n">example_get_dummy_dataset</span><span class="p">),</span>
     <span class="n">user_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="n">Header</span><span class="p">(</span><span class="kc">None</span><span class="p">),</span>
-<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">StreamingResponse</span><span class="p">:</span>
+<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">JSONResponse</span><span class="p">:</span>
 <span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">    Generates and returns a dummy dataset.</span>
 
@@ -263,7 +264,8 @@ <h1>Source code for lomas_server.routes.routes_admin</h1><div class="highlight">
 <span class="sd">        InternalServerException: For any other unforseen exceptions.</span>
 
 <span class="sd">    Returns:</span>
-<span class="sd">        StreamingResponse: a pd.DataFrame representing the dummy dataset.</span>
+<span class="sd">        JSONResponse: a dict with the dataframe as a dict, the column types</span>
+<span class="sd">            and the list of datetime columns.</span>
 <span class="sd">    &quot;&quot;&quot;</span>
     <span class="n">app</span> <span class="o">=</span> <span class="n">request</span><span class="o">.</span><span class="n">app</span>
 
@@ -279,16 +281,27 @@ <h1>Source code for lomas_server.routes.routes_admin</h1><div class="highlight">
         <span class="n">ds_metadata</span> <span class="o">=</span> <span class="n">app</span><span class="o">.</span><span class="n">state</span><span class="o">.</span><span class="n">admin_database</span><span class="o">.</span><span class="n">get_dataset_metadata</span><span class="p">(</span>
             <span class="n">query_json</span><span class="o">.</span><span class="n">dataset_name</span>
         <span class="p">)</span>
+        <span class="n">dtypes</span><span class="p">,</span> <span class="n">datetime_columns</span> <span class="o">=</span> <span class="n">get_column_dtypes</span><span class="p">(</span><span class="n">ds_metadata</span><span class="p">)</span>
 
         <span class="n">dummy_df</span> <span class="o">=</span> <span class="n">make_dummy_dataset</span><span class="p">(</span>
             <span class="n">ds_metadata</span><span class="p">,</span> <span class="n">query_json</span><span class="o">.</span><span class="n">dummy_nb_rows</span><span class="p">,</span> <span class="n">query_json</span><span class="o">.</span><span class="n">dummy_seed</span>
         <span class="p">)</span>
+
+        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">datetime_columns</span><span class="p">:</span>
+            <span class="n">dummy_df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dummy_df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">dt</span><span class="o">.</span><span class="n">strftime</span><span class="p">(</span><span class="s2">&quot;%Y-%m-</span><span class="si">%d</span><span class="s2">T%H:%M:%S&quot;</span><span class="p">)</span>
+
     <span class="k">except</span> <span class="n">KNOWN_EXCEPTIONS</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
         <span class="k">raise</span> <span class="n">e</span>
     <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
         <span class="k">raise</span> <span class="n">InternalServerException</span><span class="p">(</span><span class="n">e</span><span class="p">)</span> <span class="kn">from</span> <span class="nn">e</span>
 
-    <span class="k">return</span> <span class="n">stream_dataframe</span><span class="p">(</span><span class="n">dummy_df</span><span class="p">)</span></div>
+    <span class="k">return</span> <span class="n">JSONResponse</span><span class="p">(</span>
+        <span class="n">content</span><span class="o">=</span><span class="p">{</span>
+            <span class="s2">&quot;dummy_dict&quot;</span><span class="p">:</span> <span class="n">dummy_df</span><span class="o">.</span><span class="n">to_dict</span><span class="p">(</span><span class="n">orient</span><span class="o">=</span><span class="s2">&quot;records&quot;</span><span class="p">),</span>
+            <span class="s2">&quot;dtypes&quot;</span><span class="p">:</span> <span class="n">dtypes</span><span class="p">,</span>
+            <span class="s2">&quot;datetime_columns&quot;</span><span class="p">:</span> <span class="n">datetime_columns</span><span class="p">,</span>
+        <span class="p">}</span>
+    <span class="p">)</span></div>
 
 
 

html/develop/en/genindex.html

@@ -395,6 +395,8 @@ <h2 id="G">G</h2>
       <li><a href="lomas_server.html#lomas_server.mongodb_admin.get_archives_of_user">get_archives_of_user() (in module lomas_server.mongodb_admin)</a>
 </li>
       <li><a href="lomas_server.html#lomas_server.mongodb_admin.get_collection">get_collection() (in module lomas_server.mongodb_admin)</a>
+</li>
+      <li><a href="lomas_server.private_dataset.html#lomas_server.private_dataset.private_dataset.get_column_dtypes">get_column_dtypes() (in module lomas_server.private_dataset.private_dataset)</a>
 </li>
       <li><a href="lomas_server.utils.html#lomas_server.utils.config.get_config">get_config() (in module lomas_server.utils.config)</a>
 

html/develop/en/lomas_server.html

@@ -294,6 +294,7 @@ <h2>Subpackages<a class="headerlink" href="#subpackages" title="Link to this hea
 <li class="toctree-l4"><a class="reference internal" href="lomas_server.private_dataset.html#lomas_server.private_dataset.private_dataset.PrivateDataset.subscribe_for_memory_usage_updates"><code class="docutils literal notranslate"><span class="pre">PrivateDataset.subscribe_for_memory_usage_updates()</span></code></a></li>
 </ul>
 </li>
+<li class="toctree-l3"><a class="reference internal" href="lomas_server.private_dataset.html#lomas_server.private_dataset.private_dataset.get_column_dtypes"><code class="docutils literal notranslate"><span class="pre">get_column_dtypes()</span></code></a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="lomas_server.private_dataset.html#module-lomas_server.private_dataset.s3_dataset">lomas_server.private_dataset.s3_dataset module</a><ul>

html/develop/en/lomas_server.private_dataset.html

@@ -236,6 +236,24 @@ <h2>Submodules<a class="headerlink" href="#submodules" title="Link to this headi
 
 </dd></dl>
 
+<dl class="py function">
+<dt class="sig sig-object py" id="lomas_server.private_dataset.private_dataset.get_column_dtypes">
+<span class="sig-prename descclassname"><span class="pre">lomas_server.private_dataset.private_dataset.</span></span><span class="sig-name descname"><span class="pre">get_column_dtypes</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">metadata</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">dict</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">Tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">Dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">List</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span></span></span><a class="reference internal" href="_modules/lomas_server/private_dataset/private_dataset.html#get_column_dtypes"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#lomas_server.private_dataset.private_dataset.get_column_dtypes" title="Link to this definition"></a></dt>
+<dd><p>Extract and return the column types from the metadata.</p>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters<span class="colon">:</span></dt>
+<dd class="field-odd"><p><strong>metadata</strong> (<em>dict</em>) – The metadata dictionary.</p>
+</dd>
+<dt class="field-even">Returns<span class="colon">:</span></dt>
+<dd class="field-even"><p>The dictionary of the column type.
+list: The list of columns of datetime type</p>
+</dd>
+<dt class="field-odd">Return type<span class="colon">:</span></dt>
+<dd class="field-odd"><p>dict</p>
+</dd>
+</dl>
+</dd></dl>
+
 </section>
 <section id="module-lomas_server.private_dataset.s3_dataset">
 <span id="lomas-server-private-dataset-s3-dataset-module"></span><h2>lomas_server.private_dataset.s3_dataset module<a class="headerlink" href="#module-lomas_server.private_dataset.s3_dataset" title="Link to this heading"></a></h2>