From 1e5bea23404fdebe93b7dce979072b4c846f0d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mirko=20M=C3=A4licke?= Date: Wed, 31 Jul 2024 10:28:43 +0200 Subject: [PATCH] move logger --- Dockerfile | 5 ++++- docker-compose.yml | 1 - src/aggregator.py | 2 +- src/clip.py | 2 +- src/ingestor.py | 2 +- src/loader.py | 30 +++++++++++++++--------------- src/logger.py | 35 ----------------------------------- src/reporter.py | 2 +- src/run.py | 2 +- src/utils.py | 2 +- src/writer.py | 2 +- 11 files changed, 26 insertions(+), 59 deletions(-) delete mode 100644 src/logger.py diff --git a/Dockerfile b/Dockerfile index 1cab446..20a4ab7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.10.13 # install the toolbox runner tools -RUN pip install json2args==0.6.1 +RUN pip install json2args==0.7.0 # Install GDAL which will be used by geopandas RUN pip install --upgrade pip @@ -45,6 +45,9 @@ RUN mkdir /src COPY ./src /src RUN mv /whitebox/WhiteboxTools_linux_amd64/WBT /src/WBT +# copy the citation file - looks funny to make COPY not fail if the file is not there +COPY ./CITATION.cf[f] /src/CITATION.cff + # download a precompiled binary of duckdb # first line checks the architecture, and replaces x86_64 with amd64, which is what duckdb uses RUN arch=$(uname -m | sed s/x86_64/amd64/) && \ diff --git a/docker-compose.yml b/docker-compose.yml index c0cd9b9..1bdf8cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.8' services: db: image: postgis/postgis:15-3.4 diff --git a/src/aggregator.py b/src/aggregator.py index e4815ff..7101e0e 100644 --- a/src/aggregator.py +++ b/src/aggregator.py @@ -7,7 +7,7 @@ import polars as pl from param import load_params -from logger import logger +from json2args.logger import logger from writer import dispatch_result_saver class AggregationMacros(TypedDict): diff --git a/src/clip.py b/src/clip.py index 3f944d0..ad385cc 100644 --- a/src/clip.py +++ b/src/clip.py @@ -9,7 +9,7 @@ from pyproj import CRS from geocube.api.core import make_geocube -from logger import logger +from json2args.logger import logger from param import load_params diff --git a/src/ingestor.py b/src/ingestor.py index 0931fd9..adba101 100644 --- a/src/ingestor.py +++ b/src/ingestor.py @@ -9,7 +9,7 @@ import duckdb from metacatalog.models import Entry -from logger import logger +from json2args.logger import logger from param import load_params SPATIAL_DIMENSIONS = ('lon', 'lat', 'z') diff --git a/src/loader.py b/src/loader.py index bb93ce8..f03ab5b 100644 --- a/src/loader.py +++ b/src/loader.py @@ -11,7 +11,7 @@ import geopandas as gpd import rasterio as rio -from logger import logger +from json2args.logger import logger from writer import dispatch_save_file, entry_metadata_saver from param import load_params, Params from utils import whitebox_log_handler @@ -315,22 +315,22 @@ def error_handler(future): tiles = [future.result() for future in futures if future.result() is not None] # run the merge function and delete the other files - if len(tiles) > 1: - logger.debug('Starting WhitboxTools mosaic operation...') - _wbt_merge_raster(dataset_base_path, f"{entry.variable.name.replace(' ', '_')}_{entry.id}.tif") + # if len(tiles) > 1: + # logger.debug('Starting WhitboxTools mosaic operation...') + # _wbt_merge_raster(dataset_base_path, f"{entry.variable.name.replace(' ', '_')}_{entry.id}.tif") - # remove the tiles - for tile in tiles: - Path(tile).unlink() + # # remove the tiles + # for tile in tiles: + # Path(tile).unlink() - # check if there is exactly one tile - elif len(tiles) == 1: - # rename the file - new_name = dataset_base_path / f"{entry.variable.name.replace(' ', '_')}_{entry.id}.tif" - Path(tiles[0]).rename(new_name) - tiles = [str(new_name)] - else: - logger.warning(f'No tiles were clipped for the reference area. It might not be covered by dataset ') + # # check if there is exactly one tile + # elif len(tiles) == 1: + # # rename the file + # new_name = dataset_base_path / f"{entry.variable.name.replace(' ', '_')}_{entry.id}.tif" + # Path(tiles[0]).rename(new_name) + # tiles = [str(new_name)] + # else: + # logger.warning(f'No tiles were clipped for the reference area. It might not be covered by dataset ') # save the metadata metafile_name = str(params.dataset_path / f"{entry.variable.name.replace(' ', '_')}_{entry.id}.metadata.json") diff --git a/src/logger.py b/src/logger.py deleted file mode 100644 index 4d3e4a0..0000000 --- a/src/logger.py +++ /dev/null @@ -1,35 +0,0 @@ -import logging -import os - -# create a new logger -logger = logging.getLogger('vforwater') - -# set the log level to debug -logger.setLevel(logging.DEBUG) - -# create a file handler for all messages -processing_handler = logging.FileHandler('/out/processing.log') - -# create a file handler for warnings and errors -error_handler = logging.FileHandler('/out/errors.log') -error_handler.setLevel(logging.WARNING) - -# create a console handler for all messages -console_handler = logging.StreamHandler() -console_handler.setLevel(getattr(logging, os.environ.get('LOG_LEVEL', 'INFO'))) - -# create a formatter for the file handlers -formatter = logging.Formatter('[%(asctime)s] - [%(levelname)s] - [%(message)s]') - -# create a formatter to the console -console_formatter = logging.Formatter('[%(levelname)s]: %(message)s') - -# set the formatters -processing_handler.setFormatter(formatter) -error_handler.setFormatter(formatter) -console_handler.setFormatter(console_formatter) - -# add the handlers to the logger -logger.addHandler(processing_handler) -logger.addHandler(error_handler) -logger.addHandler(console_handler) \ No newline at end of file diff --git a/src/reporter.py b/src/reporter.py index 834bb02..d063a94 100644 --- a/src/reporter.py +++ b/src/reporter.py @@ -5,7 +5,7 @@ from ydata_profiling import ProfileReport from param import load_params -from logger import logger +from json2args.logger import logger def generate_profile_report(file_name: str) -> None: diff --git a/src/run.py b/src/run.py index 07a8119..dcf6fc5 100644 --- a/src/run.py +++ b/src/run.py @@ -13,7 +13,7 @@ from param import load_params, Integrations from loader import load_entry_data -from logger import logger +from json2args.logger import logger import ingestor import aggregator import reporter diff --git a/src/utils.py b/src/utils.py index 806bbf2..bd2065f 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1,4 +1,4 @@ -from logger import logger +from json2args.logger import logger # define a handler for whiteboxgis tools verbose output def whitebox_log_handler(msg: str): diff --git a/src/writer.py b/src/writer.py index 08a0da8..108ba86 100644 --- a/src/writer.py +++ b/src/writer.py @@ -13,7 +13,7 @@ import pandas as pd import xarray as xr -from logger import logger +from json2args.logger import logger # create a union of all supported Dataframe types