removed all manual test dirs and replaced with tmp_path

tests/test_bathymetry.py

@@ -12,17 +12,15 @@
 
 
 @pytest.mark.unittest
-def test_read_write_asc():
+def test_read_write_asc(tmp_path):
     lat_range = np.arange(-8,2,0.25)
     lon_range = np.arange(-10,1,0.25)
     data = np.cos(lon_range) * lat_range[np.newaxis].T
-    file_asc = './dummy.asc'
+    file_asc = os.path.join(tmp_path, 'dummy.asc')
     dfmt.write_bathy_toasc(file_asc, lon_sel_ext=lon_range, lat_sel_ext=lat_range, elev_sel_ext=data, asc_fmt='%14.9f',nodata_val=-999)
 
     ds_asc = dfmt.read_asc(file_asc)
 
     assert (np.abs(ds_asc['lon'].to_numpy() - lon_range) < 1e-9).all()
     assert (np.abs(ds_asc['lat'].to_numpy() - lat_range) < 1e-9).all()
     assert (np.abs(ds_asc['data'].to_numpy() - data) < 1e-9).all()
-
-    os.remove(file_asc)

tests/test_download.py

@@ -5,7 +5,6 @@
 @author: veenstra
 """
 
-import shutil
 import os
 import pytest
 import pandas as pd
@@ -34,71 +33,58 @@ def test_copernicusmarine_credentials():
 #TODO: properly set environment variables in github would prevent localness
 @pytest.mark.requireslocaldata
 @pytest.mark.unittest
-def test_download_era5():
+def test_download_era5(tmp_path):
     date_min = '2010-01-01'
     date_max = '2010-01-02'
     longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
     variables_era5 = ['msl']#'v10n'] # check variables_dict in dfmt.download_ERA5() for valid names
-    dir_output = 'era5_temp'
     for varkey in variables_era5:
-        os.makedirs(dir_output, exist_ok=True)
-
         dfmt.download_ERA5(varkey, 
                            longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
                            date_min=date_min, date_max=date_max,
-                           dir_output=dir_output, overwrite=True)
-    #clean up
-    shutil.rmtree(dir_output)
+                           dir_output=tmp_path, overwrite=True)
 
 
 #TODO: properly set environment variables in github would prevent localness
 @pytest.mark.requireslocaldata
 @pytest.mark.unittest
-def test_download_cmems_my():
+def test_download_cmems_my(tmp_path):
     date_min = '2010-01-01'
     date_max = '2010-01-02'
     longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
     varlist_cmems = ['bottomT','no3'] # avaliable variables differ per product, examples are ['bottomT','mlotst','siconc','sithick','so','thetao','uo','vo','usi','vsi','zos','no3']. More info on https://data.marine.copernicus.eu/products
-    dir_output = 'cmems_temp_my'
     for varkey in varlist_cmems:
         file_prefix = 'cmems_'
         dfmt.download_CMEMS(varkey=varkey,
                             longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
                             date_min=date_min, date_max=date_max,
-                            dir_output=dir_output, file_prefix=file_prefix, overwrite=True)
-    #clean up
-    shutil.rmtree(dir_output)
+                            dir_output=tmp_path, file_prefix=file_prefix, overwrite=True)
 
 
 #TODO: properly set environment variables in github would prevent localness
 @pytest.mark.requireslocaldata
 @pytest.mark.unittest
-def test_download_cmems_forecast():
+def test_download_cmems_forecast(tmp_path):
     date_min = pd.Timestamp.today()
     date_max = pd.Timestamp.today() + pd.Timedelta(days=1)
     longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
     varlist_cmems = ['tob','no3'] # avaliable variables differ per product, examples are ['bottomT','mlotst','siconc','sithick','so','thetao','uo','vo','usi','vsi','zos','no3']. More info on https://data.marine.copernicus.eu/products
-    dir_output = 'cmems_temp_forecast'
     for varkey in varlist_cmems:
         file_prefix = 'cmems_'
         dfmt.download_CMEMS(varkey=varkey,
                             longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
                             date_min=date_min, date_max=date_max,
-                            dir_output=dir_output, file_prefix=file_prefix, overwrite=True)
-    #clean up
-    shutil.rmtree(dir_output)
+                            dir_output=tmp_path, file_prefix=file_prefix, overwrite=True)
 
 
 @pytest.mark.unittest
-def test_download_hycom():
+def test_download_hycom(tmp_path):
     # domain
     longitude_min, longitude_max, latitude_min, latitude_max =    2,   3,  51, 52 #test domain
     date_min = '2010-01-01'
     date_max = '2010-01-02'
     varlist_hycom = ['surf_el']#'water_temp'] #['tau','water_u','water_v','water_temp','salinity','surf_el']
 
-    dir_output = 'hycom_temp'
-    os.makedirs(dir_output, exist_ok=True)
     for varkey in varlist_hycom:
         # Path(dir_output).mkdir(parents=True, exist_ok=True)
         period_range_years = pd.period_range(date_min,date_max,freq='Y')
@@ -108,6 +94,4 @@ def test_download_hycom():
                               varkey=varkey,
                               longitude_min=longitude_min, longitude_max=longitude_max, latitude_min=latitude_min, latitude_max=latitude_max,
                               date_min=date_min, date_max=date_max,
-                              dir_output=dir_output, file_prefix=file_prefix, overwrite=True)
-    #clean up
-    shutil.rmtree(dir_output)
+                              dir_output=tmp_path, file_prefix=file_prefix, overwrite=True)

tests/test_external_packages.py

@@ -5,7 +5,6 @@
 @author: veenstra
 """
 
-import os
 import pytest
 import xarray as xr
 import xugrid as xu
@@ -119,7 +118,7 @@ def test_xarray_pandas_resample():
 
 
 @pytest.mark.unittest
-def test_xarray_decode_default_fillvals():
+def test_xarray_decode_default_fillvals(tmp_path):
     """
     This test will fail as soon as xarray handles default fillvalues: https://github.com/Deltares/dfm_tools/issues/490
     After that, the minimum xarray requirement can be updated
@@ -145,7 +144,7 @@ def test_xarray_decode_default_fillvals():
     ds[varn_fnc] = ds[varn_fnc].where(ds[varn_fnc]!=fill_value,fill_value_default)
 
     #write file
-    file_out = 'temp_fnc_default_fillvals_map.nc'
+    file_out = tmp_path / 'temp_fnc_default_fillvals_map.nc'
     ds.to_netcdf(file_out)
 
     #open dataset with decode_fillvals
@@ -165,11 +164,3 @@ def test_xarray_decode_default_fillvals():
     fnc_new = uds2.grid.face_node_connectivity
 
     assert fill_value_default in fnc_new
-
-    # cleanup
-    # del ds
-    # del uds2
-    # del fnc_new
-    # PermissionError: [WinError 32] The process cannot access the file because it is being used by another process: 'temp_fnc_default_fillvals_map.nc'
-    # os.remove(file_out)
-

tests/test_hydrolib_helpers.py

@@ -5,7 +5,6 @@
 @author: veenstra
 """
 
-import os
 import pytest
 import dfm_tools as dfmt
 import hydrolib.core.dflowfm as hcdfm
@@ -27,12 +26,12 @@ def test_geodataframe_with_Polygon_to_PolyFile():
 
 
 @pytest.mark.unittest
-def test_geodataframe_with_LineString_to_PolyFile():
+def test_geodataframe_with_LineString_to_PolyFile(tmp_path):
     """
     converting a geodataframe with Linestring geometries to hcdfm.PolyFile
     """
     # write polygon file
-    file_pol = 'temp_coastlines.pol'
+    file_pol = tmp_path / 'temp_coastlines.pol'
     lon_min, lon_max, lat_min, lat_max = -68.45, -68.1, 12, 12.35
     bbox = (lon_min, lat_min, lon_max, lat_max)
     coastlines_gdf = dfmt.get_coastlines_gdb(bbox=bbox, res='h')
@@ -46,6 +45,4 @@ def test_geodataframe_with_LineString_to_PolyFile():
     gdf_polyfile = dfmt.PolyFile_to_geodataframe_linestrings(polyfile_object,crs=None)
 
     polyfile = dfmt.geodataframe_to_PolyFile(gdf_polyfile)
-
     assert isinstance(polyfile, hcdfm.PolyFile)
-    os.remove(file_pol)

tests/test_interpolate_grid2bnd.py

@@ -97,8 +97,6 @@ def test_check_time_extent():
 
     # on ds timerange
     check_time_extent(ds, tstart="2019-12-31 12:00:00", tstop="2020-01-03 12:00:00")
-
-    del ds
 
 
 @pytest.mark.unittest
@@ -239,13 +237,13 @@ def test_plipointsDataset_to_ForcingModel_drop_allnan_points():
 
 
 @pytest.mark.systemtest
-def test_open_dataset_extra_correctdepths():
+def test_open_dataset_extra_correctdepths(tmp_path):
     """
     to validate open_dataset_extra behaviour for depths, in the past the depth values got lost and replaced by depth idx
     """
 
     ds_moretime = cmems_dataset_4times()
-    file_nc = 'temp_cmems_dummydata.nc'
+    file_nc = tmp_path / 'temp_cmems_dummydata.nc'
     ds_moretime.to_netcdf(file_nc)
 
     ds_moretime_import = dfmt.open_dataset_extra(dir_pattern=file_nc, quantity='salinitybnd', tstart='2020-01-01 12:00:00', tstop='2020-01-02 12:00:00')
@@ -257,14 +255,10 @@ def test_open_dataset_extra_correctdepths():
 
     assert (np.abs(depth_actual - depth_expected) < 1e-9).all()
     assert len(ds_moretime_import.time) == 2
-
-    # cleanup
-    del ds_moretime_import
-    os.remove(file_nc)
 
 
 @pytest.mark.unittest
-def test_open_dataset_extra_slightly_different_latlons():
+def test_open_dataset_extra_slightly_different_latlons(tmp_path):
     """
     to check whether an error is raised when trying to combine datasets with slightly 
     different coordinates: https://github.com/Deltares/dfm_tools/issues/574
@@ -278,8 +272,8 @@ def test_open_dataset_extra_slightly_different_latlons():
     ds_lon[1] += 1e-8
     ds2['longitude'] = xr.DataArray(ds_lon,dims='longitude')
 
-    file_nc1 = 'temp_cmems_2day_p1.nc'
-    file_nc2 = 'temp_cmems_2day_p2.nc'
+    file_nc1 = tmp_path / 'temp_cmems_2day_p1.nc'
+    file_nc2 = tmp_path / 'temp_cmems_2day_p2.nc'
     ds1.to_netcdf(file_nc1)
     ds2.to_netcdf(file_nc2)
 
@@ -292,12 +286,6 @@ def test_open_dataset_extra_slightly_different_latlons():
         # ValueError: cannot align objects with join='exact' where index/labels/sizes are not equal along these coordinates (dimensions): 'longitude' ('longitude',)
         pass # this is expected, so pass
 
-    # cleanup
-    del ds1
-    del ds2
-    os.remove(file_nc1)
-    os.remove(file_nc2)
-
 
 @pytest.mark.unittest
 def test_interp_regularnc_to_plipointsDataset():

tests/test_meshkernel_helpers.py

@@ -5,7 +5,6 @@
 @author: veenstra
 """
 
-import os
 import pytest
 import xugrid as xu
 import dfm_tools as dfmt
@@ -14,7 +13,6 @@
 import numpy as np
 import geopandas as gpd
 from shapely.geometry import Polygon
-import glob
 from dfm_tools.meshkernel_helpers import (geographic_to_meshkernel_projection, 
                                           uds_add_crs_attrs,
                                           crs_to_isgeographic
@@ -88,9 +86,9 @@ def test_meshkernel_delete_withcoastlines():
 
 
 @pytest.mark.unittest
-def test_meshkernel_delete_withshp():
+def test_meshkernel_delete_withshp(tmp_path):
     # write shapefile from coords
-    file_shp = 'mk_delete_test.shp'
+    file_shp = tmp_path / 'mk_delete_test.shp'
     points_x = np.array([-68.40156631636155, -68.36143523088661, -68.28392176442131, -68.26413109213229,  
                          -68.20915700244058, -68.1965129618115, -68.20860726154366, -68.199811407193,    
                          -68.23059689742034, -68.23389534280184, -68.26303161033846, -68.29436684146273, 
@@ -118,10 +116,6 @@ def test_meshkernel_delete_withshp():
     dfmt.meshkernel_delete_withshp(mk=mk, coastlines_shp=file_shp)
 
     assert len(mk.mesh2d_get().face_nodes) == 17272
-
-    # delete shapefile
-    shp_list = glob.glob(file_shp.replace('.shp','.*'))
-    [os.remove(x) for x in shp_list]
 
 
 @pytest.mark.unittest
@@ -168,7 +162,7 @@ def test_meshkernel_to_UgridDataset_geographic_mismatch():
 
 
 @pytest.mark.systemtest
-def test_meshkernel_to_UgridDataset():
+def test_meshkernel_to_UgridDataset(tmp_path):
     """
     generate grid with meshkernel. Then convert with `dfmt.meshkernel_to_UgridDataset()` from 0-based to 1-based indexing to make FM-compatible network.
     assert if _FillValue, start_index, min and max are the expected values, this ensures FM-compatibility
@@ -198,7 +192,7 @@ def test_meshkernel_to_UgridDataset():
 
     #convert to xugrid and write to netcdf
     xu_grid_uds = dfmt.meshkernel_to_UgridDataset(mk=mk, crs=crs)
-    netfile = 'test_startindex_net.nc'
+    netfile = tmp_path / 'test_startindex_net.nc'
     xu_grid_uds.ugrid.to_netcdf(netfile)
 
     # plot
@@ -210,8 +204,6 @@ def test_meshkernel_to_UgridDataset():
 
     #assert output grid
     ds_out = xr.open_dataset(netfile,decode_cf=False).load()
-    ds_out.close()
-    os.remove(netfile)
     assert ds_out.mesh2d_face_nodes.attrs['_FillValue'] == -1
     assert ds_out.mesh2d_face_nodes.attrs['start_index'] == 1
     assert 0 not in ds_out.mesh2d_face_nodes.to_numpy()