Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

finish water mass notebook #28

Merged
merged 6 commits into from
May 1, 2024
Merged

finish water mass notebook #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
81a3b6b
flexible in chunk size for local and opendap
chiaweh2 Apr 24, 2024
8265de2
change to new io structure
chiaweh2 Apr 24, 2024
19fd75c
water mass calculation finished
chiaweh2 Apr 30, 2024
8eee5b4
explicit opendap data source for pytest
chiaweh2 Apr 30, 2024
fa59b2b
remove debug code
chiaweh2 Apr 30, 2024
f034e76
Merge pull request #27 from NOAA-PSL/feature-index
chiaweh2 Apr 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/gha_pytest_pr.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ jobs:

- name: pytesting
shell: bash -l {0}
run: pytest
run: pytest --location opendap
2 changes: 1 addition & 1 deletion .github/workflows/gha_pytest_push.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,4 @@ jobs:

- name: pytesting
shell: bash -l {0}
run: pytest
run: pytest --location opendap
33 changes: 22 additions & 11 deletions mom6/mom6_module/mom6_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,21 +195,24 @@ def get_all(self) -> xr.Dataset:
raise OSError('for raw grid please input the path to grid file')
else:
ds_static = MOM6Static.get_grid(self.static_relative_dir)
# setup chuck
io_chunk = {}
elif self.source == 'opendap':
file_list = OpenDapStore(grid=self.grid,data_type='forecast').get_catalog()
for file in file_list:
var_flag = 'static' in file
if var_flag :
ds_static = xr.open_dataset(file)
io_chunk = {'init': 4,'member':1,'lead':-1}

file_read = [file for file in file_list if self.var in file]
file_read = [file for file in file_list if f'{self.var}_' in file]

# merge the static field with the variables
ds = xr.open_mfdataset(
file_read,
combine='nested',
concat_dim='init',
chunks={'init': 4,'member':1,'lead':-1}
chunks=io_chunk
).sortby('init')
ds = xr.merge([ds_static,ds])
# ds = ds.isel(init=slice(1,None)) # exclude the 1980 empty field due to merge
Expand All @@ -232,14 +235,16 @@ def get_all(self) -> xr.Dataset:
else:
mom6_dir = os.path.join(DATA_PATH,self.data_relative_dir)
file_list = glob.glob(f'{mom6_dir}/*.nc')
io_chunk = {}
elif self.source == 'opendap':
file_list = OpenDapStore(grid=self.grid,data_type='forecast').get_catalog()
io_chunk = {'init': 1,'member':1,'lead':-1}

file_read = [file for file in file_list if self.var in file]
file_read = [file for file in file_list if f'{self.var}_' in file]
ds = xr.open_mfdataset(
file_read,combine='nested',
concat_dim='init',
chunks={'init': 1,'member':1,'lead':1}
chunks=io_chunk
).sortby('init')

# test if accident read raw file
Expand Down Expand Up @@ -288,15 +293,17 @@ def get_tercile(
raise OSError('for raw grid please input the path to grid file')
else:
ds_static = MOM6Static.get_grid(self.static_relative_dir)
io_chunk = {}
elif self.source == 'opendap':
file_list = OpenDapStore(grid=self.grid,data_type='forecast').get_catalog()
for file in file_list:
var_flag = 'static' in file
if var_flag :
ds_static = xr.open_dataset(file)
io_chunk = {'init': 4,'member':1,'lead':-1}

# refine based on var name
file_read = [file for file in file_list if self.var in file]
file_read = [file for file in file_list if f'{self.var}_' in file]
# refine based on region
if average_type == 'grid':
file_read = [file for file in file_read if '.region.' not in file]
Expand All @@ -308,7 +315,7 @@ def get_tercile(
file_read,
combine='nested',
concat_dim='init',
chunks={'init': 4,'member':1,'lead':-1}
chunks=io_chunk
).sortby('init')
ds = xr.merge([ds_static,ds])
# ds = ds.isel(init=slice(1,None)) # exclude the 1980 empty field due to merge
Expand All @@ -331,10 +338,12 @@ def get_tercile(
else:
mom6_dir = os.path.join(DATA_PATH,self.tercile_relative_dir)
file_list = glob.glob(f'{mom6_dir}/*.nc')
io_chunk = {}
elif self.source == 'opendap':
file_list = OpenDapStore(grid=self.grid,data_type='forecast').get_catalog()
io_chunk = {'init': 4,'member':1,'lead':-1}

file_read = [file for file in file_list if self.var in file]
file_read = [file for file in file_list if f'{self.var}_' in file]

# refine based on region
if average_type == 'grid':
Expand All @@ -345,7 +354,7 @@ def get_tercile(
ds = xr.open_mfdataset(
file_read,combine='nested',
concat_dim='init',
chunks={'init': 1,'member':1,'lead':1}
chunks=io_chunk
).sortby('init')

# test if accident read raw file
Expand Down Expand Up @@ -538,20 +547,22 @@ def get_all(self) -> xr.Dataset:
raise IOError('for raw grid please input the path to grid file')
else:
ds_static = MOM6Static.get_grid(self.static_relative_dir)
io_chunk = {}
elif self.source == 'opendap':
file_list = OpenDapStore(grid=self.grid,data_type='historical').get_catalog()
for file in file_list:
var_flag = 'static' in file
if var_flag :
ds_static = xr.open_dataset(file)
io_chunk = {'time': 100}

file_read = [file for file in file_list if self.var in file]
file_read = [file for file in file_list if f'.{self.var}.' in file]

# merge the static field with the variables
ds = xr.open_mfdataset(
file_read,combine='nested',
concat_dim='time',
chunks={'time': 100}
chunks=io_chunk
).sortby('time')
ds = xr.merge([ds_static,ds])
ds = ds.isel(time=slice(1,None)) # exclude the 1980 empty field due to merge
Expand All @@ -577,7 +588,7 @@ def get_all(self) -> xr.Dataset:
elif self.source == 'opendap':
file_list = OpenDapStore(grid=self.grid,data_type='historical').get_catalog()

file_read = [file for file in file_list if self.var in file]
file_read = [file for file in file_list if f'.{self.var}.' in file]
ds = xr.open_mfdataset(
file_read,
combine='nested',
Expand Down
2,350 changes: 118 additions & 2,232 deletions mom6/notebook/gulf_stream_index.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

557 changes: 557 additions & 0 deletions mom6/notebook/nechannel_index.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

Loading