Add Flux variables CSV and enhance micromet functionality

Added a comprehensive CSV file for Flux variables, enhancing data tracking. Updated multiple scripts in the micromet package to improve configuration handling, CSV path management, and file compilation. Moreover, introduced new dependencies in the environment file and pyproject.toml to support enhanced data visualization and statistical analysis features.

Notebooks/DL_test.ipynb

@@ -10,11 +10,11 @@
      "start_time": "2024-07-25T21:02:10.115012Z"
     },
     "execution": {
-     "iopub.execute_input": "2024-10-01T13:19:17.027678Z",
-     "iopub.status.busy": "2024-10-01T13:19:17.027678Z",
-     "iopub.status.idle": "2024-10-01T13:19:19.493694Z",
-     "shell.execute_reply": "2024-10-01T13:19:19.493694Z",
-     "shell.execute_reply.started": "2024-10-01T13:19:17.027678Z"
+     "iopub.execute_input": "2024-10-01T20:04:24.166323Z",
+     "iopub.status.busy": "2024-10-01T20:04:24.166323Z",
+     "iopub.status.idle": "2024-10-01T20:04:27.390335Z",
+     "shell.execute_reply": "2024-10-01T20:04:27.390335Z",
+     "shell.execute_reply.started": "2024-10-01T20:04:24.166323Z"
     }
    },
    "outputs": [],
@@ -42,19 +42,19 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 2,
    "id": "61e3a29165852b08",
    "metadata": {
     "ExecuteTime": {
      "end_time": "2024-07-25T19:13:24.857446Z",
      "start_time": "2024-07-25T19:13:24.854024Z"
     },
     "execution": {
-     "iopub.execute_input": "2024-10-01T14:07:14.269179Z",
-     "iopub.status.busy": "2024-10-01T14:07:14.269179Z",
-     "iopub.status.idle": "2024-10-01T14:07:14.275904Z",
-     "shell.execute_reply": "2024-10-01T14:07:14.274897Z",
-     "shell.execute_reply.started": "2024-10-01T14:07:14.269179Z"
+     "iopub.execute_input": "2024-10-01T20:04:27.394344Z",
+     "iopub.status.busy": "2024-10-01T20:04:27.391341Z",
+     "iopub.status.idle": "2024-10-01T20:04:28.485295Z",
+     "shell.execute_reply": "2024-10-01T20:04:28.485295Z",
+     "shell.execute_reply.started": "2024-10-01T20:04:27.394344Z"
     }
    },
    "outputs": [],
@@ -66,30 +66,6 @@
     "%matplotlib inline"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 6,
-   "id": "47cdb112fc572e94",
-   "metadata": {
-    "ExecuteTime": {
-     "end_time": "2024-07-25T19:13:28.808731Z",
-     "start_time": "2024-07-25T19:13:28.716582Z"
-    },
-    "execution": {
-     "iopub.execute_input": "2024-10-01T14:07:16.887368Z",
-     "iopub.status.busy": "2024-10-01T14:07:16.887368Z",
-     "iopub.status.idle": "2024-10-01T14:07:16.891733Z",
-     "shell.execute_reply": "2024-10-01T14:07:16.890728Z",
-     "shell.execute_reply.started": "2024-10-01T14:07:16.887368Z"
-    }
-   },
-   "outputs": [],
-   "source": [
-    "#lgin = micromet.login\n",
-    "#pwrd = quote(micromet.passwrd)\n",
-    "#engine = create_engine(f\"postgresql+psycopg2://{lgin}:{pwrd}@{micromet.ip}:5432/groundwater\")\n"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 7,
@@ -141,8 +117,6 @@
     "                'US-UTN':'Juab'\n",
     "                }\n",
     "\n",
-    "site_folders = {'US-UTP':'Phrag',\n",
-    "                }\n",
     "\n",
     "compdf = {}\n",
     "\n",

micromet/converter.py

@@ -462,7 +462,7 @@ def ssitc_scale(self):
         ssitc_columns = ['FC_SSITC_TEST', 'LE_SSITC_TEST', 'ET_SSITC_TEST', 'H_SSITC_TEST', 'TAU_SSITC_TEST']
         for column in ssitc_columns:
             if column in self.et_data.columns:
-                if (self.et_data[column] > 3).any() > 2:
+                if (self.et_data[column] > 3).any():
                     self.et_data[column] = self.scale_and_convert(self.et_data[column])
 
     @staticmethod
@@ -479,7 +479,7 @@ def rating(x):
         three rating categories. For values less than or equal to 3, the method returns 0. For values
         between 4 and 6 (inclusive), the method returns 1. For all other values, the method returns 2.
         """
-        if x <= 3:
+        if 0 <= x <= 3:
             x = 0
         elif 4 <= x <= 6:
             x = 1
@@ -656,6 +656,14 @@ def load_data():
     df = pd.read_csv('./data/FP_variable_20220810.csv')
     return df
 
+def outfile(df, stationname, out_dir):
+    """Generates a file name based on the ameriflux file nameing standards
+
+    """
+    first_index = pd.to_datetime(df['TIMESTAMP_START'][0], format ='%Y%m%d%H%M')
+    last_index = pd.to_datetime(df['TIMESTAMP_END'][-1], format ='%Y%m%d%H%M')#df.index[-1], format ='%Y%m%d%H%M')
+    filename = stationname + f"HH{first_index.strftime('%Y%m%d%H%M')}_{last_index.strftime('%Y%m%d%H%M')}.csv" #{last_index_plus_30min.strftime('%Y%m%d%H%M')}.csv"
+    df.to_csv(out_dir + stationname + "/" + filename)
 
 if __name__ == '__main__':
     data = load_data()

micromet/graphs.py

@@ -1,5 +1,7 @@
 import plotly.graph_objects as go
 import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
 
 def energy_sankey(df, date_text="2024-06-19 12:00"):
     """
@@ -117,4 +119,20 @@ def energy_sankey(df, date_text="2024-06-19 12:00"):
 
     # Show the figure
     # fig.show()
-    return fig
+    return fig
+
+def bland_altman_plot(data1, data2, *args, **kwargs):
+    data1     = np.asarray(data1)
+    data2     = np.asarray(data2)
+    mean      = np.mean([data1, data2], axis=0)
+    diff      = data1 - data2                   # Difference between data1 and data2
+    md        = np.mean(diff)                   # Mean of the difference
+    sd        = np.std(diff, axis=0)            # Standard deviation of the difference
+    CI_low    = md - 1.96*sd
+    CI_high   = md + 1.96*sd
+
+    plt.scatter(mean, diff, *args, **kwargs)
+    plt.axhline(md,           color='black', linestyle='-')
+    plt.axhline(md + 1.96*sd, color='gray', linestyle='--')
+    plt.axhline(md - 1.96*sd, color='gray', linestyle='--')
+    return md, sd, mean, CI_low, CI_high

station_config/US-UTW.ini

@@ -1,5 +1,5 @@
 [METADATA]
-climate_file_path = station_data/US-UTW_HH_202105200830_202406191230.csv
+climate_file_path = ../station_data/US-UTW_HH_202105200830_202409151300.csv
 station_latitude = 39.4453
 station_longitude = -110.7288
 station_elevation = 1682