From a546b279e6f7c134534864c16c15f2a54dc97832 Mon Sep 17 00:00:00 2001 From: bnb32 Date: Fri, 19 Jul 2024 13:01:55 -0600 Subject: [PATCH] linting --- .github/workflows/linter.yml | 2 +- .pre-commit-config.yaml | 2 -- sup3r/__init__.py | 15 ------------ sup3r/bias/base.py | 3 +-- sup3r/bias/bias_calc_vortex.py | 2 +- sup3r/models/abstract.py | 2 +- sup3r/models/base.py | 2 +- sup3r/models/conditional.py | 2 +- sup3r/pipeline/forward_pass_cli.py | 6 +---- sup3r/postprocessing/writers/base.py | 4 ++-- sup3r/qa/qa.py | 2 +- sup3r/utilities/__init__.py | 27 ++++++++++++++++++++++ tests/data_handlers/test_dh_nc_cc.py | 4 ++-- tests/extracters/test_extracter_caching.py | 6 +++-- 14 files changed, 43 insertions(+), 36 deletions(-) diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index 10507dd3f..a6089ac06 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -17,7 +17,7 @@ jobs: with: fetch-depth: 0 - name: Lint Code Base - uses: super-linter/super-linter/slim@v6.2.0 + uses: super-linter/super-linter@v4 env: VALIDATE_ALL_CODEBASE: false VALIDATE_PYTHON_BLACK: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f7668d868..0f091995c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,8 +18,6 @@ repos: [ --rcfile, .github/linters/.python-lint, - --ignore-paths, - tests/, ] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.5.3 diff --git a/sup3r/__init__.py b/sup3r/__init__.py index f3734719f..f683bacef 100644 --- a/sup3r/__init__.py +++ b/sup3r/__init__.py @@ -25,18 +25,3 @@ SUP3R_DIR = os.path.dirname(os.path.realpath(__file__)) CONFIG_DIR = os.path.join(SUP3R_DIR, 'configs') TEST_DATA_DIR = os.path.join(os.path.dirname(SUP3R_DIR), 'tests', 'data') - - -VERSION_RECORD = { - 'sup3r': __version__, - 'tensorflow': tf.__version__, - 'sklearn': sklearn.__version__, - 'pandas': pd.__version__, - 'numpy': np.__version__, - 'nrel-phygnn': phygnn.__version__, - 'nrel-rex': rex.__version__, - 'python': sys.version, - 'xarray': xarray.__version__, - 'h5netcdf': h5netcdf.__version__, - 'dask': dask.__version__, -} diff --git a/sup3r/bias/base.py b/sup3r/bias/base.py index 81d42fe6e..c2fbe8941 100644 --- a/sup3r/bias/base.py +++ b/sup3r/bias/base.py @@ -15,10 +15,9 @@ from scipy.spatial import KDTree import sup3r.preprocessing -from sup3r import VERSION_RECORD from sup3r.preprocessing import DataHandlerNC as DataHandler from sup3r.preprocessing.utilities import _compute_if_dask, expand_paths -from sup3r.utilities import ModuleName +from sup3r.utilities import VERSION_RECORD, ModuleName from sup3r.utilities.cli import BaseCLI logger = logging.getLogger(__name__) diff --git a/sup3r/bias/bias_calc_vortex.py b/sup3r/bias/bias_calc_vortex.py index 3220f9312..eac18d8d1 100644 --- a/sup3r/bias/bias_calc_vortex.py +++ b/sup3r/bias/bias_calc_vortex.py @@ -16,8 +16,8 @@ from rex import Resource from scipy.interpolate import interp1d -from sup3r import VERSION_RECORD from sup3r.postprocessing import OutputHandler, RexOutputs +from sup3r.utilities import VERSION_RECORD logger = logging.getLogger(__name__) diff --git a/sup3r/models/abstract.py b/sup3r/models/abstract.py index 696f0311a..640c99b88 100644 --- a/sup3r/models/abstract.py +++ b/sup3r/models/abstract.py @@ -19,9 +19,9 @@ from tensorflow.keras import optimizers import sup3r.utilities.loss_metrics -from sup3r import VERSION_RECORD from sup3r.preprocessing.data_handlers.base import ExoData from sup3r.preprocessing.utilities import _numpy_if_tensor +from sup3r.utilities import VERSION_RECORD from sup3r.utilities.utilities import Timer logger = logging.getLogger(__name__) diff --git a/sup3r/models/base.py b/sup3r/models/base.py index 8fb597b67..959d287f9 100644 --- a/sup3r/models/base.py +++ b/sup3r/models/base.py @@ -12,7 +12,7 @@ import tensorflow as tf from tensorflow.keras import optimizers -from sup3r import VERSION_RECORD +from sup3r.utilities import VERSION_RECORD from .abstract import AbstractInterface, AbstractSingleModel diff --git a/sup3r/models/conditional.py b/sup3r/models/conditional.py index 539263a6e..a5025419e 100644 --- a/sup3r/models/conditional.py +++ b/sup3r/models/conditional.py @@ -10,7 +10,7 @@ import tensorflow as tf from tensorflow.keras import optimizers -from sup3r import VERSION_RECORD +from sup3r.utilities import VERSION_RECORD from .abstract import AbstractInterface, AbstractSingleModel diff --git a/sup3r/pipeline/forward_pass_cli.py b/sup3r/pipeline/forward_pass_cli.py index 436447c45..eaf380dd1 100644 --- a/sup3r/pipeline/forward_pass_cli.py +++ b/sup3r/pipeline/forward_pass_cli.py @@ -46,11 +46,7 @@ def main(ctx, verbose): ) @click.pass_context def from_config(ctx, config_file, verbose=False, pipeline_step=None): - """Run sup3r forward pass from a config file. - - TODO: Can we figure out how to remove the first ForwardPassStrategy - initialization here, so that its only initialized once for each node? - """ + """Run sup3r forward pass from a config file.""" config = BaseCLI.from_config_preflight( ModuleName.FORWARD_PASS, ctx, config_file, verbose diff --git a/sup3r/postprocessing/writers/base.py b/sup3r/postprocessing/writers/base.py index e7259a902..eafe6706b 100644 --- a/sup3r/postprocessing/writers/base.py +++ b/sup3r/postprocessing/writers/base.py @@ -12,8 +12,8 @@ from rex.outputs import Outputs as BaseRexOutputs from scipy.interpolate import griddata -from sup3r import VERSION_RECORD, __version__ from sup3r.preprocessing.derivers.utilities import parse_feature +from sup3r.utilities import VERSION_RECORD from sup3r.utilities.utilities import pd_date_range logger = logging.getLogger(__name__) @@ -293,7 +293,7 @@ def full_version_record(self): def set_version_attr(self): """Set the version attribute to the h5 file.""" - self.h5.attrs['version'] = __version__ + self.h5.attrs['version'] = VERSION_RECORD['sup3r'] self.h5.attrs['full_version_record'] = json.dumps( self.full_version_record ) diff --git a/sup3r/qa/qa.py b/sup3r/qa/qa.py index f766a1c8e..d8a86e35c 100644 --- a/sup3r/qa/qa.py +++ b/sup3r/qa/qa.py @@ -367,7 +367,7 @@ def get_node_cmd(cls, config): log_arg_str += f', log_file="{log_file}"' cmd = ( - f"python -c '{import_str};\n" + f"python -c '{import_str};\n" 't0 = time.time();\n' f'logger = init_logger({log_arg_str});\n' f'qa = {qa_init_str};\n' diff --git a/sup3r/utilities/__init__.py b/sup3r/utilities/__init__.py index 2d9f0583b..c80532a96 100644 --- a/sup3r/utilities/__init__.py +++ b/sup3r/utilities/__init__.py @@ -1,8 +1,35 @@ """Sup3r utilities""" +import os import sys from enum import Enum +import dask +import h5netcdf +import numpy as np +import pandas as pd +import phygnn +import rex +import sklearn +import tensorflow as tf +import xarray + +from .._version import __version__ + +VERSION_RECORD = { + 'sup3r': __version__, + 'tensorflow': tf.__version__, + 'sklearn': sklearn.__version__, + 'pandas': pd.__version__, + 'numpy': np.__version__, + 'nrel-phygnn': phygnn.__version__, + 'nrel-rex': rex.__version__, + 'python': sys.version, + 'xarray': xarray.__version__, + 'h5netcdf': h5netcdf.__version__, + 'dask': dask.__version__, +} + class ModuleName(str, Enum): """A collection of the module names available in sup3r. diff --git a/tests/data_handlers/test_dh_nc_cc.py b/tests/data_handlers/test_dh_nc_cc.py index ce41aa441..b49db9d89 100644 --- a/tests/data_handlers/test_dh_nc_cc.py +++ b/tests/data_handlers/test_dh_nc_cc.py @@ -33,8 +33,8 @@ def test_get_just_coords_nc(): assert np.array_equal( handler.lat_lon[-1, 0, :], ( - handler.extracter[Dimension.LATITUDE].min(), - handler.extracter[Dimension.LONGITUDE].min(), + handler.extracter.data[Dimension.LATITUDE].min(), + handler.extracter.data[Dimension.LONGITUDE].min(), ), ) assert not handler.data_vars diff --git a/tests/extracters/test_extracter_caching.py b/tests/extracters/test_extracter_caching.py index 4e1ee53a8..f8bfe1eee 100644 --- a/tests/extracters/test_extracter_caching.py +++ b/tests/extracters/test_extracter_caching.py @@ -61,11 +61,13 @@ def test_data_caching(input_files, ext, shape, target, features): assert extracter.data.dtype == np.dtype(np.float32) loader = Loader(cacher.out_files) assert np.array_equal( - loader[features, ...].compute(), extracter[features, ...].compute() + loader.data[features, ...].compute(), + extracter.data[features, ...].compute(), ) # make sure full domain can be loaded with extracters extracter = Extracter(cacher.out_files) assert np.array_equal( - loader[features, ...].compute(), extracter[features, ...].compute() + loader.data[features, ...].compute(), + extracter.data[features, ...].compute(), )