From 7df39f80a1196259a737490fb7db9a09294fae30 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 29 Jan 2024 21:06:16 -0500 Subject: [PATCH 01/17] add docs for editing NWB files --- docs/gallery/advanced_io/plot_editing.py | 158 +++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 docs/gallery/advanced_io/plot_editing.py diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py new file mode 100644 index 000000000..bf1b83ab3 --- /dev/null +++ b/docs/gallery/advanced_io/plot_editing.py @@ -0,0 +1,158 @@ +""" +Editing NWB files +================= + +This tutorial demonstrates how to edit NWB files. How and whether it is possible to edit +an NWB file depends on the storage backend and the type of edit. Here, we go through the +common types of edits for HDF5 files. Keep in mind that any edit to an existing NWB file +make it no longer a valid NWB file. We highly recommend making a copy before +editing and running a validation check on the file after editing it. + +In-place editing with h5py +--------------------------- + +Editing a dataset value +~~~~~~~~~~~~~~~~~~~~~~~ +You can change the value(s) of a dataset using :py:mod:`h5py`. + +First, let's create an NWB file with data: +""" +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from datetime import datetime +from dateutil.tz import tzlocal +import numpy as np + +nwbfile = NWBFile( + session_description="my first synthetic recording", + identifier="EXAMPLE_ID", + session_start_time=datetime.now(tzlocal()), + session_id="LONELYMTN", +) + +nwbfile.add_acquisition( + TimeSeries( + name="synthetic_timeseries", + description="Random values", + data=np.random.randn(100, 100), + unit="m", + rate=10e3, + ) +) + +with NWBHDF5IO("test_edit.nwb", "w") as io: + io.write(nwbfile) + + +############################################## +# Now, let's try to edit the values of the dataset: +import h5py + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"]["synthetic_timeseries"]["data"][:10] = 0.0 + +############################################## +# This will edit the dataset in-place, and should work for all datasets. You can also +# edit attributes in-place: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"]["synthetic_timeseries"]["data"].attrs["unit"] = "volts" + + +# Changing the shape of dataset +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Whether it is possible to change the shape of a dataset depends on how the dataset was +# created. If the dataset was created with a flexible shape, then it is possible to +# change in-place. Creating a dataset with a flexible shape is done by specifying the +# ``maxshape`` argument of the :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` class +# constructor. Using a ``None`` value for ``maxshape`` allows the dataset to be reset +# arbitrarily long in that dimension. Chunking is required for datasets with flexible +# shapes. Setting ``maxshape`` automatically sets chunking to ``True`, if not specified. +# +# First, let's create an NWB file with a dataset with a flexible shape: + +from pynwb import NWBHDF5IO, NWBFile, TimeSeries +from datetime import datetime +from dateutil.tz import tzlocal +import numpy as np +from hdmf.backends.hdf5.h5_utils import H5DataIO + +nwbfile = NWBFile( + session_description="my first synthetic recording", + identifier="EXAMPLE_ID", + session_start_time=datetime.now(tzlocal()), + session_id="LONELYMTN", +) + +data_io = H5DataIO(data=np.random.randn(100, 100), maxshape=(None, 100)) + +nwbfile.add_acquisition( + TimeSeries( + name="synthetic_timeseries", + description="Random values", + data=data_io, + unit="m", + rate=10e3, + ) +) + +with NWBHDF5IO("test_edit2.nwb", "w") as io: + io.write(nwbfile) + +############################################## +# The ``None`` in ``maxshape`` means that the dataset has an unlimited shape. You can +# also use an integer to specify a fixed ``maxshape``. If you do not specify a +# ``maxshape``, then the dataset will have a fixed shape. You can change the shape of +# this dataset. + +import h5py + +with h5py.File("test_edit2.nwb", "r+") as f: + f["acquisition"]["synthetic_timeseries"]["data"].resize((200, 100)) + +############################################## +# This will change the shape of the dataset in-place. If you try to change the shape of +# a dataset with a fixed shape, you will get an error: +# +# .. code-block:: python +# import h5py +# +# with h5py.File("test_edit.nwb", "r+") as f: +# f["acquisition"]["synthetic_timeseries"]["data"].resize((200, 100)) +# +# ValueError: Unable to resize dataset (no object or chunk storage) +# +# Replacing a dataset in h5py +# ---------------------------- +# There are several types of dataset edits that cannot be done in-place. +# +# * Changing the shape of a dataset with a fixed shape +# * Changing the datatype of a dataset +# * Changing the compression of a dataset +# * Changing the chunking of a dataset +# * Changing the max-shape of a dataset +# * Changing the fill-value of a dataset +# +# For any of these, you will need to create a new dataset with the new shape, copying +# the data from the old dataset to the new dataset, and deleting the old dataset. + +with h5py.File("test_edit2.nwb", "r+") as f: + data = f["acquisition"]["synthetic_timeseries"]["data"][:] + del f["acquisition"]["synthetic_timeseries"]["data"] + f["acquisition"]["synthetic_timeseries"].create_dataset( + name="data", + data=data, + maxshape=(None, 100), + chunks=(100, 100), + compression="gzip", + compression_opts=3, + fillvalue=0.0, + dtype=np.float64, + ) + +############################################## +# .. note:: +# Because of the way HDF5 works, the ``del`` action will not actually free up any +# space in the HDF5 file. To free up space in the file, you will need to run the +# ``h5repack`` command line tool. See the `h5repack documentation +# `_ for more +# information. From 3a837feac45875d4010b09921ab97cd4b602e900 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 29 Jan 2024 21:10:02 -0500 Subject: [PATCH 02/17] rmv unnecessary imports --- docs/gallery/advanced_io/plot_editing.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index bf1b83ab3..5bba45684 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -70,10 +70,6 @@ # # First, let's create an NWB file with a dataset with a flexible shape: -from pynwb import NWBHDF5IO, NWBFile, TimeSeries -from datetime import datetime -from dateutil.tz import tzlocal -import numpy as np from hdmf.backends.hdf5.h5_utils import H5DataIO nwbfile = NWBFile( From 48ec49aaea19c8964e7f024eef1443a8c8e138b7 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 29 Jan 2024 21:13:44 -0500 Subject: [PATCH 03/17] fixing formatting --- docs/gallery/advanced_io/plot_editing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 5bba45684..3b91a4e57 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -58,6 +58,7 @@ f["acquisition"]["synthetic_timeseries"]["data"].attrs["unit"] = "volts" +############################################## # Changing the shape of dataset # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Whether it is possible to change the shape of a dataset depends on how the dataset was From b3a060f495ff8ea9e5f29b2a50110dbe19c22950 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 29 Jan 2024 21:17:23 -0500 Subject: [PATCH 04/17] minor edits --- docs/gallery/advanced_io/plot_editing.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 3b91a4e57..b4f484f6a 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -10,6 +10,8 @@ In-place editing with h5py --------------------------- +The :py:mod:`h5py` package allows for in-place editing of HDF5 files. Where possible, +this is the recommended way to edit HDF5 files. Editing a dataset value ~~~~~~~~~~~~~~~~~~~~~~~ @@ -120,7 +122,7 @@ # # Replacing a dataset in h5py # ---------------------------- -# There are several types of dataset edits that cannot be done in-place. +# There are several types of dataset edits that cannot be done in-place: # # * Changing the shape of a dataset with a fixed shape # * Changing the datatype of a dataset @@ -133,11 +135,11 @@ # the data from the old dataset to the new dataset, and deleting the old dataset. with h5py.File("test_edit2.nwb", "r+") as f: - data = f["acquisition"]["synthetic_timeseries"]["data"][:] + data_values = f["acquisition"]["synthetic_timeseries"]["data"][:] del f["acquisition"]["synthetic_timeseries"]["data"] f["acquisition"]["synthetic_timeseries"].create_dataset( name="data", - data=data, + data=data_values, maxshape=(None, 100), chunks=(100, 100), compression="gzip", From 387a4a1e44e32e447edf48dce190895a73236aa4 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Mon, 29 Jan 2024 21:19:16 -0500 Subject: [PATCH 05/17] remove python code that is not rendering --- docs/gallery/advanced_io/plot_editing.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index b4f484f6a..dc3b2c61d 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -110,15 +110,7 @@ ############################################## # This will change the shape of the dataset in-place. If you try to change the shape of -# a dataset with a fixed shape, you will get an error: -# -# .. code-block:: python -# import h5py -# -# with h5py.File("test_edit.nwb", "r+") as f: -# f["acquisition"]["synthetic_timeseries"]["data"].resize((200, 100)) -# -# ValueError: Unable to resize dataset (no object or chunk storage) +# a dataset with a fixed shape, you will get an error. # # Replacing a dataset in h5py # ---------------------------- From 2798be0ad372a9f3b1f483f634370ed51ab4a4fe Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 30 Jan 2024 10:54:38 -0500 Subject: [PATCH 06/17] improve editing tutorial --- docs/gallery/advanced_io/plot_editing.py | 32 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index dc3b2c61d..e74b69fdf 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -5,16 +5,17 @@ This tutorial demonstrates how to edit NWB files. How and whether it is possible to edit an NWB file depends on the storage backend and the type of edit. Here, we go through the common types of edits for HDF5 files. Keep in mind that any edit to an existing NWB file -make it no longer a valid NWB file. We highly recommend making a copy before -editing and running a validation check on the file after editing it. +make it no longer a valid NWB file. We call this "doing surgery" on the NWB file. +We highly recommend making a copy before editing and running a validation check on the +file after editing it. In-place editing with h5py --------------------------- The :py:mod:`h5py` package allows for in-place editing of HDF5 files. Where possible, this is the recommended way to edit HDF5 files. -Editing a dataset value -~~~~~~~~~~~~~~~~~~~~~~~ +Editing values of datasets and attributes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can change the value(s) of a dataset using :py:mod:`h5py`. First, let's create an NWB file with data: @@ -59,6 +60,27 @@ with h5py.File("test_edit.nwb", "r+") as f: f["acquisition"]["synthetic_timeseries"]["data"].attrs["unit"] = "volts" +############################################## +# This also works for attributes of groups: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"]["synthetic_timeseries"].attrs["description"] = "Random values in volts" + +############################################## +# .. warning:: +# You can edit values that will bring the file out of compliance with +# the NWB specification. For example: +# +# with h5py.File("test_edit.nwb", "r+") as f: +# f["acquisition"]["synthetic_timeseries"]["neurodata_type"] = "TimeSeries" +# +# Renaming groups and datasets +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Rename groups and datasets in-place using the ``move`` method. For example, to rename +# the ``"synthetic_timeseries"`` group: + +with h5py.File("test_edit2.nwb", "r+") as f: + f["acquisition"].move("synthetic_timeseries", "synthetic_timeseries_renamed") ############################################## # Changing the shape of dataset @@ -147,3 +169,5 @@ # ``h5repack`` command line tool. See the `h5repack documentation # `_ for more # information. + + From cff1f6ec82e1b1b673ff65e83747ada17f076e77 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 30 Jan 2024 17:15:20 -0500 Subject: [PATCH 07/17] fixing up tutorial --- docs/gallery/advanced_io/plot_editing.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index e74b69fdf..a7dd0d244 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -7,7 +7,7 @@ common types of edits for HDF5 files. Keep in mind that any edit to an existing NWB file make it no longer a valid NWB file. We call this "doing surgery" on the NWB file. We highly recommend making a copy before editing and running a validation check on the -file after editing it. +file after editing it. See :ref:`validating`. In-place editing with h5py --------------------------- @@ -69,19 +69,24 @@ ############################################## # .. warning:: # You can edit values that will bring the file out of compliance with -# the NWB specification. For example: -# -# with h5py.File("test_edit.nwb", "r+") as f: -# f["acquisition"]["synthetic_timeseries"]["neurodata_type"] = "TimeSeries" +# the NWB specification. # # Renaming groups and datasets # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Rename groups and datasets in-place using the ``move`` method. For example, to rename # the ``"synthetic_timeseries"`` group: -with h5py.File("test_edit2.nwb", "r+") as f: +with h5py.File("test_edit.nwb", "r+") as f: f["acquisition"].move("synthetic_timeseries", "synthetic_timeseries_renamed") +############################################## +# You can use this same technique to move a group or dataset to a different location in +# the file. For example, to move the ``"synthetic_timeseries_renamed"`` group to the +# ``"analysis"`` group: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"].move("synthetic_timeseries_renamed", "/analysis/synthetic_timeseries_renamed") + ############################################## # Changing the shape of dataset # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -91,7 +96,7 @@ # ``maxshape`` argument of the :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` class # constructor. Using a ``None`` value for ``maxshape`` allows the dataset to be reset # arbitrarily long in that dimension. Chunking is required for datasets with flexible -# shapes. Setting ``maxshape`` automatically sets chunking to ``True`, if not specified. +# shapes. Setting ``maxshape`` automatically sets chunking to ``True``, if not specified. # # First, let's create an NWB file with a dataset with a flexible shape: From baaf77bd44c10ff55cab7897f8b5cbbc4ab6d454 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 30 Jan 2024 17:33:54 -0500 Subject: [PATCH 08/17] Update docs/gallery/advanced_io/plot_editing.py Co-authored-by: Oliver Ruebel --- docs/gallery/advanced_io/plot_editing.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index a7dd0d244..d555c4a9b 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -2,12 +2,17 @@ Editing NWB files ================= -This tutorial demonstrates how to edit NWB files. How and whether it is possible to edit -an NWB file depends on the storage backend and the type of edit. Here, we go through the -common types of edits for HDF5 files. Keep in mind that any edit to an existing NWB file -make it no longer a valid NWB file. We call this "doing surgery" on the NWB file. -We highly recommend making a copy before editing and running a validation check on the -file after editing it. See :ref:`validating`. +This tutorial demonstrates how to edit NWB files manually (i.e., without using PyNWB). +How and whether it is possible to edit an NWB file depends on the storage backend +and the type of edit. Here, we go through the common types of manual edits for HDF5 files +using :py:mod:`h5py`. We call this "doing surgery" on the NWB file. + +.. warning:: + + Manually editing an existing NWB file can make the file invalid if you are not careful. + We highly recommend making a copy before editing and running a validation check on the + file after editing it. See :ref:`validating`. + In-place editing with h5py --------------------------- From 48f8c675acffad37d6b157fb3dd2f11047c8f2ad Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Tue, 30 Jan 2024 18:04:50 -0500 Subject: [PATCH 09/17] using pynwb where possible --- docs/gallery/advanced_io/plot_editing.py | 49 +++++++++++++----------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index d555c4a9b..b7dff1bd0 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -14,14 +14,11 @@ file after editing it. See :ref:`validating`. -In-place editing with h5py ---------------------------- -The :py:mod:`h5py` package allows for in-place editing of HDF5 files. Where possible, -this is the recommended way to edit HDF5 files. - -Editing values of datasets and attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -You can change the value(s) of a dataset using :py:mod:`h5py`. +Editing datasets in place +-------------------------- +When reading an HDF5 NWB file, PyNWB exposes :py:class:`h5py.Dataset` objects, which can +be edited in place. For this to work, you must open the file in read/write mode +(``"r+"`` or ``"a"``). First, let's create an NWB file with data: """ @@ -50,23 +47,29 @@ with NWBHDF5IO("test_edit.nwb", "w") as io: io.write(nwbfile) - ############################################## -# Now, let's try to edit the values of the dataset: -import h5py +# Editing values of datasets +# ~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Now, let's edit the values of the dataset + +with NWBHDF5IO("test_edit.nwb", "r+") as io: + nwbfile = io.read() + nwbfile.acquisition["synthetic_timeseries"].data[:10] = 0.0 -with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"]["synthetic_timeseries"]["data"][:10] = 0.0 ############################################## -# This will edit the dataset in-place, and should work for all datasets. You can also -# edit attributes in-place: +# You can also edit the attributes of that dataset through the ``attrs`` attribute: -with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"]["synthetic_timeseries"]["data"].attrs["unit"] = "volts" +with NWBHDF5IO("test_edit.nwb", "r+") as io: + nwbfile = io.read() + nwbfile.acquisition["synthetic_timeseries"].data.attrs["unit"] = "volts" ############################################## -# This also works for attributes of groups: +# If you want to edit the attributes of a group, you will need to open the file and edit +# it using :py:mod:`h5py`: + +import h5py with h5py.File("test_edit.nwb", "r+") as f: f["acquisition"]["synthetic_timeseries"].attrs["description"] = "Random values in volts" @@ -135,17 +138,17 @@ # ``maxshape``, then the dataset will have a fixed shape. You can change the shape of # this dataset. -import h5py -with h5py.File("test_edit2.nwb", "r+") as f: - f["acquisition"]["synthetic_timeseries"]["data"].resize((200, 100)) +with NWBHDF5IO("test_edit2.nwb", "r+") as io: + nwbfile = io.read() + nwbfile.acquisition["synthetic_timeseries"].resize((200, 100)) ############################################## # This will change the shape of the dataset in-place. If you try to change the shape of # a dataset with a fixed shape, you will get an error. # -# Replacing a dataset in h5py -# ---------------------------- +# Replacing a dataset using h5py +# ------------------------------ # There are several types of dataset edits that cannot be done in-place: # # * Changing the shape of a dataset with a fixed shape From fc232fa2c7558ccb3b2359172596aa242bf6801b Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 09:54:41 -0500 Subject: [PATCH 10/17] improve the editing tutorial and have it cross-reference with the add_remove_containers tutorial --- docs/gallery/advanced_io/plot_editing.py | 83 ++++++------------- docs/gallery/general/add_remove_containers.py | 28 ++----- 2 files changed, 32 insertions(+), 79 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index b7dff1bd0..6fbfee283 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -1,21 +1,23 @@ """ +.. _editing: + Editing NWB files ================= -This tutorial demonstrates how to edit NWB files manually (i.e., without using PyNWB). -How and whether it is possible to edit an NWB file depends on the storage backend -and the type of edit. Here, we go through the common types of manual edits for HDF5 files -using :py:mod:`h5py`. We call this "doing surgery" on the NWB file. +This tutorial demonstrates how to edit NWB files in-place. For information on how to add +or remove containers from an NWB file, see :ref:`modifying_data`. How and whether it is +possible to edit an NWB file depends on the storage backend +and the type of edit. .. warning:: - Manually editing an existing NWB file can make the file invalid if you are not careful. - We highly recommend making a copy before editing and running a validation check on the - file after editing it. See :ref:`validating`. + Manually editing an existing NWB file can make the file invalid if you are not + careful. We highly recommend making a copy before editing and running a validation + check on the file after editing it. See :ref:`validating`. -Editing datasets in place --------------------------- +Editing datasets +---------------- When reading an HDF5 NWB file, PyNWB exposes :py:class:`h5py.Dataset` objects, which can be edited in place. For this to work, you must open the file in read/write mode (``"r+"`` or ``"a"``). @@ -48,9 +50,6 @@ io.write(nwbfile) ############################################## -# Editing values of datasets -# ~~~~~~~~~~~~~~~~~~~~~~~~~~ -# # Now, let's edit the values of the dataset with NWBHDF5IO("test_edit.nwb", "r+") as io: @@ -66,8 +65,9 @@ nwbfile.acquisition["synthetic_timeseries"].data.attrs["unit"] = "volts" ############################################## -# If you want to edit the attributes of a group, you will need to open the file and edit -# it using :py:mod:`h5py`: +# Editing groups +# -------------- +# To edit the attributes of a group, open the file and edit it using :py:mod:`h5py`: import h5py @@ -76,12 +76,12 @@ ############################################## # .. warning:: -# You can edit values that will bring the file out of compliance with -# the NWB specification. +# You can edit values that will bring the file out of compliance with the NWB +# specification. # # Renaming groups and datasets # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Rename groups and datasets in-place using the ``move`` method. For example, to rename +# Rename groups and datasets in-place using the :py:meth:`~h5py.Group.move` method. For example, to rename # the ``"synthetic_timeseries"`` group: with h5py.File("test_edit.nwb", "r+") as f: @@ -93,7 +93,10 @@ # ``"analysis"`` group: with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"].move("synthetic_timeseries_renamed", "/analysis/synthetic_timeseries_renamed") + f["acquisition"].move( + "synthetic_timeseries_renamed", + "/analysis/synthetic_timeseries_renamed", + ) ############################################## # Changing the shape of dataset @@ -141,46 +144,14 @@ with NWBHDF5IO("test_edit2.nwb", "r+") as io: nwbfile = io.read() - nwbfile.acquisition["synthetic_timeseries"].resize((200, 100)) + nwbfile.acquisition["synthetic_timeseries"].data.resize((200, 100)) ############################################## # This will change the shape of the dataset in-place. If you try to change the shape of # a dataset with a fixed shape, you will get an error. # -# Replacing a dataset using h5py -# ------------------------------ -# There are several types of dataset edits that cannot be done in-place: -# -# * Changing the shape of a dataset with a fixed shape -# * Changing the datatype of a dataset -# * Changing the compression of a dataset -# * Changing the chunking of a dataset -# * Changing the max-shape of a dataset -# * Changing the fill-value of a dataset -# -# For any of these, you will need to create a new dataset with the new shape, copying -# the data from the old dataset to the new dataset, and deleting the old dataset. - -with h5py.File("test_edit2.nwb", "r+") as f: - data_values = f["acquisition"]["synthetic_timeseries"]["data"][:] - del f["acquisition"]["synthetic_timeseries"]["data"] - f["acquisition"]["synthetic_timeseries"].create_dataset( - name="data", - data=data_values, - maxshape=(None, 100), - chunks=(100, 100), - compression="gzip", - compression_opts=3, - fillvalue=0.0, - dtype=np.float64, - ) - -############################################## -# .. note:: -# Because of the way HDF5 works, the ``del`` action will not actually free up any -# space in the HDF5 file. To free up space in the file, you will need to run the -# ``h5repack`` command line tool. See the `h5repack documentation -# `_ for more -# information. - - +# There are several types of dataset edits that cannot be done in-place: changing the +# shape of a dataset with a fixed shape, or changing the datatype, compression, +# chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using +# the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See +# :ref:`modifying_data` for more information. diff --git a/docs/gallery/general/add_remove_containers.py b/docs/gallery/general/add_remove_containers.py index 80c5cb032..0e4c545aa 100644 --- a/docs/gallery/general/add_remove_containers.py +++ b/docs/gallery/general/add_remove_containers.py @@ -70,31 +70,13 @@ # file path, and it is not possible to remove objects from an NWB file. You can use the # :py:meth:`NWBHDF5IO.export ` method, detailed below, to modify an NWB file in these ways. # -# .. warning:: -# -# NWB datasets that have been written to disk are read as :py:class:`h5py.Dataset ` objects. -# Directly modifying the data in these :py:class:`h5py.Dataset ` objects immediately -# modifies the data on disk -# (the :py:meth:`NWBHDF5IO.write ` method does not need to be called and the -# :py:class:`~pynwb.NWBHDF5IO` instance does not need to be closed). Directly modifying datasets in this way -# can lead to files that do not validate or cannot be opened, so exercise caution when using this method. -# Note: only chunked datasets or datasets with ``maxshape`` set can be resized. -# See the `h5py chunked storage documentation `_ -# for more details. - -############################################################################### -# .. note:: -# -# It is not possible to modify the attributes (fields) of an NWB container in memory. - -############################################################################### # Exporting a written NWB file to a new file path -# --------------------------------------------------- +# ----------------------------------------------- # Use the :py:meth:`NWBHDF5IO.export ` method to read data from an existing NWB file, # modify the data, and write the modified data to a new file path. Modifications to the data can be additions or # removals of objects, such as :py:class:`~pynwb.base.TimeSeries` objects. This is especially useful if you -# have raw data and processed data in the same NWB file and you want to create a new NWB file with all of the -# contents of the original file except for the raw data for sharing with collaborators. +# have raw data and processed data in the same NWB file and you want to create a new NWB file with all the contents of +# the original file except for the raw data for sharing with collaborators. # # To remove existing containers, use the :py:class:`~hdmf.utils.LabelledDict.pop` method on any # :py:class:`~hdmf.utils.LabelledDict` object, such as ``NWBFile.acquisition``, ``NWBFile.processing``, @@ -200,7 +182,7 @@ export_io.export(src_io=read_io, nwbfile=read_nwbfile) ############################################################################### -# More information about export -# --------------------------------- # For more information about the export functionality, see :ref:`export` # and the PyNWB documentation for :py:meth:`NWBHDF5IO.export `. +# +# For more options about editing a file in place, see :ref:`editing`. From 233e0cd1b8079fdce4764df7146c990f4e70b1ff Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 09:56:02 -0500 Subject: [PATCH 11/17] add note formatting --- docs/gallery/advanced_io/plot_editing.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 6fbfee283..7e4b3a73d 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -150,8 +150,9 @@ # This will change the shape of the dataset in-place. If you try to change the shape of # a dataset with a fixed shape, you will get an error. # -# There are several types of dataset edits that cannot be done in-place: changing the -# shape of a dataset with a fixed shape, or changing the datatype, compression, -# chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using -# the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See -# :ref:`modifying_data` for more information. +# .. note:: +# There are several types of dataset edits that cannot be done in-place: changing the +# shape of a dataset with a fixed shape, or changing the datatype, compression, +# chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using +# the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See +# :ref:`modifying_data` for more information. From b8e77063dd2e01f49f32aa8afe32ae386d233e0d Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 10:02:53 -0500 Subject: [PATCH 12/17] refactoring --- docs/gallery/advanced_io/plot_editing.py | 77 ++++++++++++------------ 1 file changed, 38 insertions(+), 39 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 7e4b3a73d..7a1f0f539 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -4,10 +4,10 @@ Editing NWB files ================= -This tutorial demonstrates how to edit NWB files in-place. For information on how to add -or remove containers from an NWB file, see :ref:`modifying_data`. How and whether it is -possible to edit an NWB file depends on the storage backend -and the type of edit. +This tutorial demonstrates how to edit NWB files in-place to make small changes to +existing containers. To add or remove containers from an NWB file, see +:ref:`modifying_data`. How and whether it is possible to edit an NWB file depends on the +storage backend and the type of edit. .. warning:: @@ -58,46 +58,12 @@ ############################################## -# You can also edit the attributes of that dataset through the ``attrs`` attribute: +# You can edit the attributes of that dataset through the ``attrs`` attribute: with NWBHDF5IO("test_edit.nwb", "r+") as io: nwbfile = io.read() nwbfile.acquisition["synthetic_timeseries"].data.attrs["unit"] = "volts" -############################################## -# Editing groups -# -------------- -# To edit the attributes of a group, open the file and edit it using :py:mod:`h5py`: - -import h5py - -with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"]["synthetic_timeseries"].attrs["description"] = "Random values in volts" - -############################################## -# .. warning:: -# You can edit values that will bring the file out of compliance with the NWB -# specification. -# -# Renaming groups and datasets -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# Rename groups and datasets in-place using the :py:meth:`~h5py.Group.move` method. For example, to rename -# the ``"synthetic_timeseries"`` group: - -with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"].move("synthetic_timeseries", "synthetic_timeseries_renamed") - -############################################## -# You can use this same technique to move a group or dataset to a different location in -# the file. For example, to move the ``"synthetic_timeseries_renamed"`` group to the -# ``"analysis"`` group: - -with h5py.File("test_edit.nwb", "r+") as f: - f["acquisition"].move( - "synthetic_timeseries_renamed", - "/analysis/synthetic_timeseries_renamed", - ) - ############################################## # Changing the shape of dataset # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -156,3 +122,36 @@ # chunking, max-shape, or fill-value of a dataset. For any of these, we recommend using # the :py:class:`pynwb.NWBHDF5IO.export` method to export the data to a new file. See # :ref:`modifying_data` for more information. +# +# Editing groups +# -------------- +# To edit the attributes of a group, open the file and edit it using :py:mod:`h5py`: + +import h5py + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"]["synthetic_timeseries"].attrs["description"] = "Random values in volts" + +############################################## +# .. warning:: +# Be careful not to edit values that will bring the file out of compliance with the +# NWB specification. +# +# Renaming groups and datasets +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Rename groups and datasets in-place using the :py:meth:`~h5py.Group.move` method. For example, to rename +# the ``"synthetic_timeseries"`` group: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"].move("synthetic_timeseries", "synthetic_timeseries_renamed") + +############################################## +# You can use this same technique to move a group or dataset to a different location in +# the file. For example, to move the ``"synthetic_timeseries_renamed"`` group to the +# ``"analysis"`` group: + +with h5py.File("test_edit.nwb", "r+") as f: + f["acquisition"].move( + "synthetic_timeseries_renamed", + "/analysis/synthetic_timeseries_renamed", + ) From 0982847b66fdec6c0a2ad7ad8171dc4038f273d4 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 10:04:07 -0500 Subject: [PATCH 13/17] Update docs/gallery/general/add_remove_containers.py --- docs/gallery/general/add_remove_containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/gallery/general/add_remove_containers.py b/docs/gallery/general/add_remove_containers.py index 0e4c545aa..90ed8f324 100644 --- a/docs/gallery/general/add_remove_containers.py +++ b/docs/gallery/general/add_remove_containers.py @@ -185,4 +185,4 @@ # For more information about the export functionality, see :ref:`export` # and the PyNWB documentation for :py:meth:`NWBHDF5IO.export `. # -# For more options about editing a file in place, see :ref:`editing`. +# For more information about editing a file in place, see :ref:`editing`. From ab79f28e05d9e8fb6ab389a507cabe9e7b6641b7 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 14:40:09 -0500 Subject: [PATCH 14/17] Update docs/gallery/advanced_io/plot_editing.py Co-authored-by: Oliver Ruebel --- docs/gallery/advanced_io/plot_editing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 7a1f0f539..6ed3a6754 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -71,9 +71,10 @@ # created. If the dataset was created with a flexible shape, then it is possible to # change in-place. Creating a dataset with a flexible shape is done by specifying the # ``maxshape`` argument of the :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` class -# constructor. Using a ``None`` value for ``maxshape`` allows the dataset to be reset -# arbitrarily long in that dimension. Chunking is required for datasets with flexible -# shapes. Setting ``maxshape`` automatically sets chunking to ``True``, if not specified. +# constructor. Using a ``None`` value for a component of the ``maxshape`` tuple allows +# the size of the corresponding dimension to grow, such that is can be be reset arbitrarily long +# in that dimension. Chunking is required for datasets with flexible shapes. Setting ``maxshape``, +# hence, automatically sets chunking to ``True``, if not specified. # # First, let's create an NWB file with a dataset with a flexible shape: From 43fba1bae87897e26496219255d333bdc2c327ae Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 14:40:27 -0500 Subject: [PATCH 15/17] Update docs/gallery/advanced_io/plot_editing.py Co-authored-by: Oliver Ruebel --- docs/gallery/advanced_io/plot_editing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 6ed3a6754..5373b979f 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -126,6 +126,7 @@ # # Editing groups # -------------- +# Editing of groups is not yet supported in PyNWB. # To edit the attributes of a group, open the file and edit it using :py:mod:`h5py`: import h5py From b4f91809ae3f6cbc4bac3180e98e5d9f493dae78 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 14:40:54 -0500 Subject: [PATCH 16/17] Update docs/gallery/advanced_io/plot_editing.py Co-authored-by: Oliver Ruebel --- docs/gallery/advanced_io/plot_editing.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index 5373b979f..af3337d13 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -103,10 +103,12 @@ io.write(nwbfile) ############################################## -# The ``None`` in ``maxshape`` means that the dataset has an unlimited shape. You can -# also use an integer to specify a fixed ``maxshape``. If you do not specify a -# ``maxshape``, then the dataset will have a fixed shape. You can change the shape of -# this dataset. +# The ``None``value in the first component of ``maxshape`` means that the +# the first dimension of the dataset is unlimited. By setting the second dimension +# of ``maxshape`` to ``100``, that dimension is fixed to be no larger than ``100``. +# If you do not specify a``maxshape``, then the shape of the dataset will be fixed +# to the shape that the dataset was created with. Here, you can change the shape of +# the first dimension of this dataset. with NWBHDF5IO("test_edit2.nwb", "r+") as io: From 31fb46509ae50452014f971bd086a34fd80388e4 Mon Sep 17 00:00:00 2001 From: Ben Dichter Date: Wed, 31 Jan 2024 14:42:57 -0500 Subject: [PATCH 17/17] flake8 --- docs/gallery/advanced_io/plot_editing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/gallery/advanced_io/plot_editing.py b/docs/gallery/advanced_io/plot_editing.py index af3337d13..e45e3b887 100644 --- a/docs/gallery/advanced_io/plot_editing.py +++ b/docs/gallery/advanced_io/plot_editing.py @@ -71,8 +71,8 @@ # created. If the dataset was created with a flexible shape, then it is possible to # change in-place. Creating a dataset with a flexible shape is done by specifying the # ``maxshape`` argument of the :py:class:`~hdmf.backends.hdf5.h5_utils.H5DataIO` class -# constructor. Using a ``None`` value for a component of the ``maxshape`` tuple allows -# the size of the corresponding dimension to grow, such that is can be be reset arbitrarily long +# constructor. Using a ``None`` value for a component of the ``maxshape`` tuple allows +# the size of the corresponding dimension to grow, such that is can be be reset arbitrarily long # in that dimension. Chunking is required for datasets with flexible shapes. Setting ``maxshape``, # hence, automatically sets chunking to ``True``, if not specified. # @@ -105,7 +105,7 @@ ############################################## # The ``None``value in the first component of ``maxshape`` means that the # the first dimension of the dataset is unlimited. By setting the second dimension -# of ``maxshape`` to ``100``, that dimension is fixed to be no larger than ``100``. +# of ``maxshape`` to ``100``, that dimension is fixed to be no larger than ``100``. # If you do not specify a``maxshape``, then the shape of the dataset will be fixed # to the shape that the dataset was created with. Here, you can change the shape of # the first dimension of this dataset. @@ -128,7 +128,7 @@ # # Editing groups # -------------- -# Editing of groups is not yet supported in PyNWB. +# Editing of groups is not yet supported in PyNWB. # To edit the attributes of a group, open the file and edit it using :py:mod:`h5py`: import h5py