diff --git a/.codespellrc b/.codespellrc index ad57c4b81..a38689dfe 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,7 +1,8 @@ [codespell] # in principle .ipynb can be corrected -- a good number of typos there # nwb-schema -- excluding since submodule, should have its own fixes/checks -skip = .git,*.pdf,*.svg,venvs,env,*.ipynb,nwb-schema +skip = .git,*.pdf,*.svg,venvs,env,nwb-schema +ignore-regex = ^\s*"image/\S+": ".* # it is optin in a url # potatos - demanded to be left alone, autogenerated ignore-words-list = optin,potatos diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 7a1e8dc04..7aa79c9e7 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -16,4 +16,4 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Codespell - uses: codespell-project/actions-codespell@v1 + uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 781822258..c47941c21 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -108,6 +108,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Python uses: actions/setup-python@v4 @@ -151,6 +152,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Conda uses: conda-incubator/setup-miniconda@v2 @@ -194,9 +196,9 @@ jobs: fail-fast: false matrix: include: - - { name: linux-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } - - { name: windows-python3.11-ros3, python-ver: "3.11", os: windows-latest } - - { name: macos-python3.11-ros3 , python-ver: "3.11", os: macos-latest } + - { name: conda-linux-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-windows-python3.11-ros3, python-ver: "3.11", os: windows-latest } + - { name: conda-macos-python3.11-ros3 , python-ver: "3.11", os: macos-latest } steps: - name: Cancel non-latest runs uses: styfle/cancel-workflow-action@0.11.0 @@ -207,6 +209,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Conda uses: conda-incubator/setup-miniconda@v2 @@ -240,9 +243,9 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } - - { name: windows-gallery-python3.11-ros3, python-ver: "3.11", os: windows-latest } - - { name: macos-gallery-python3.11-ros3 , python-ver: "3.11", os: macos-latest } + - { name: conda-linux-gallery-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-windows-gallery-python3.11-ros3, python-ver: "3.11", os: windows-latest } + - { name: conda-macos-gallery-python3.11-ros3 , python-ver: "3.11", os: macos-latest } steps: - name: Cancel non-latest runs uses: styfle/cancel-workflow-action@0.11.0 @@ -253,6 +256,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Conda uses: conda-incubator/setup-miniconda@v2 @@ -278,4 +282,4 @@ jobs: - name: Run gallery ros3 tests run: | - python test.py --example-ros3 \ No newline at end of file + python test.py --example-ros3 diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index a465676bc..acbc3bd05 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -71,7 +71,7 @@ jobs: - name: Run integration tests and generate coverage report run: | - python -m coverage run -p test.py --integration --backwards + python -m coverage run -p test.py --integration --validation-module --backwards # validation CLI tests generate separate .coverage files that need to be merged python -m coverage combine python -m coverage xml # codecov uploader requires xml format diff --git a/.github/workflows/run_dandi_read_tests.yml b/.github/workflows/run_dandi_read_tests.yml index ec8cc2e84..7148d209e 100644 --- a/.github/workflows/run_dandi_read_tests.yml +++ b/.github/workflows/run_dandi_read_tests.yml @@ -1,15 +1,15 @@ name: Run DANDI read tests on: - schedule: - - cron: '0 6 * * *' # once per day at 1am ET + # NOTE this is disabled until we can run this systematically instead of randomly + # so we don't get constant error notifications and waste compute cycles + # See https://github.com/NeurodataWithoutBorders/pynwb/issues/1804 + # schedule: + # - cron: '0 6 * * *' # once per day at 1am ET workflow_dispatch: jobs: run-tests: runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} # necessary for conda steps: - name: Cancel non-latest runs uses: styfle/cancel-workflow-action@0.11.0 @@ -22,19 +22,14 @@ jobs: submodules: 'recursive' fetch-depth: 0 # tags are required for versioneer to determine the version - - name: Set up Conda - uses: conda-incubator/setup-miniconda@v2 + - name: Set up Python + uses: actions/setup-python@v4 with: - auto-update-conda: true - activate-environment: ros3 - environment-file: environment-ros3.yml - python-version: "3.11" - channels: conda-forge - auto-activate-base: false + python-version: '3.11' - name: Install run dependencies run: | - python -m pip install dandi pytest + python -m pip install dandi fsspec requests aiohttp pytest python -m pip uninstall -y pynwb # uninstall pynwb python -m pip install -e . python -m pip list @@ -47,4 +42,4 @@ jobs: - name: Run DANDI read tests run: | - pytest -rP tests/read_dandi/ + python tests/read_dandi/read_dandi.py diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d2a492496..e4479a554 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -92,6 +92,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Python uses: actions/setup-python@v4 @@ -173,7 +174,7 @@ jobs: fail-fast: false matrix: include: - - { name: linux-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } steps: - name: Cancel non-latest runs uses: styfle/cancel-workflow-action@0.11.0 @@ -184,6 +185,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Conda uses: conda-incubator/setup-miniconda@v2 @@ -217,7 +219,7 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-gallery-python3.11-ros3 , python-ver: "3.11", os: ubuntu-latest } steps: - name: Cancel non-latest runs uses: styfle/cancel-workflow-action@0.11.0 @@ -228,6 +230,7 @@ jobs: - uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Conda uses: conda-incubator/setup-miniconda@v2 @@ -271,6 +274,7 @@ jobs: uses: actions/checkout@v3 with: submodules: 'recursive' + fetch-depth: 0 # tags are required for versioneer to determine the version - name: Set up Python uses: actions/setup-python@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fa808a77..2ddcdda7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # PyNWB Changelog +## PyNWB 2.6.0 (Upcoming) + +### Enhancements and minor changes +- For `NWBHDF5IO()`, change the default of arg `load_namespaces` from `False` to `True`. @bendichter [#1748](https://github.com/NeurodataWithoutBorders/pynwb/pull/1748) +- Add `NWBHDF5IO.can_read()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703) +- Add `pynwb.get_nwbfile_version()`. @bendichter [#1703](https://github.com/NeurodataWithoutBorders/pynwb/pull/1703) +- Updated timeseries data checks to warn instead of error when reading invalid files. @stephprince [#1793](https://github.com/NeurodataWithoutBorders/pynwb/pull/1793) and [#1809](https://github.com/NeurodataWithoutBorders/pynwb/pull/1809) +- Expose the offset, conversion and channel conversion parameters in `mock_ElectricalSeries`. @h-mayorquin [#1796](https://github.com/NeurodataWithoutBorders/pynwb/pull/1796) +- Expose `starting_time` in `mock_ElectricalSeries`. @h-mayorquin [#1805](https://github.com/NeurodataWithoutBorders/pynwb/pull/1805) +- Enhance `get_data_in_units()` to work with objects that have a `channel_conversion` attribute like the `ElectricalSeries`. @h-mayorquin [#1806](https://github.com/NeurodataWithoutBorders/pynwb/pull/1806) +- Refactor validation CLI tests to use `{sys.executable} -m coverage` to use the same Python version and run correctly on Debian systems. @yarikoptic [#1811](https://github.com/NeurodataWithoutBorders/pynwb/pull/1811) + +### Bug fixes +- Fix bug where namespaces were loaded in "w-" mode. @h-mayorquin [#1795](https://github.com/NeurodataWithoutBorders/pynwb/pull/1795) +- Fix bug where pynwb version was reported as "unknown" to readthedocs @stephprince [#1810](https://github.com/NeurodataWithoutBorders/pynwb/pull/1810) + +### Documentation and tutorial enhancements +- Add RemFile to streaming tutorial. @bendichter [#1761](https://github.com/NeurodataWithoutBorders/pynwb/pull/1761) +- Fix typos and improve clarify throughout tutorials. @zm711 [#1825](https://github.com/NeurodataWithoutBorders/pynwb/pull/1825) + +## PyNWB 2.5.0 (August 18, 2023) + +### Enhancements and minor changes +- Add `TimeSeries.get_timestamps()`. @bendichter [#1741](https://github.com/NeurodataWithoutBorders/pynwb/pull/1741) +- Add `TimeSeries.get_data_in_units()`. @bendichter [#1745](https://github.com/NeurodataWithoutBorders/pynwb/pull/1745) +- Updated `ExternalResources` name change to `HERD`, along with HDMF 3.9.0 being the new minimum. @mavaylon1 [#1754](https://github.com/NeurodataWithoutBorders/pynwb/pull/1754) + +### Documentation and tutorial enhancements +- Updated streaming tutorial to ensure code is run on tests and clarify text. @bendichter [#1760](https://github.com/NeurodataWithoutBorders/pynwb/pull/1760) @oruebel [#1762](https://github.com/NeurodataWithoutBorders/pynwb/pull/1762) +- Fixed minor documentation build warnings and broken links to `basic_trials` tutorial @oruebel [#1762](https://github.com/NeurodataWithoutBorders/pynwb/pull/1762) + +## PyNWB 2.4.1 (July 26, 2023) +- Stop running validation tests as part of integration tests. They cause issues in CI and can be run separately. @rly [#1740](https://github.com/NeurodataWithoutBorders/pynwb/pull/1740) + ## PyNWB 2.4.0 (July 23, 2023) ### Enhancements and minor changes diff --git a/docs/gallery/advanced_io/linking_data.py b/docs/gallery/advanced_io/linking_data.py index 082aa3c51..2f79d1488 100644 --- a/docs/gallery/advanced_io/linking_data.py +++ b/docs/gallery/advanced_io/linking_data.py @@ -6,57 +6,50 @@ PyNWB supports linking between files using external links. -""" +Example Use Case: Integrating data from multiple files +--------------------------------------------------------- -#################### -# Example Use Case: Integrating data from multiple files -# --------------------------------------------------------- -# -# NBWContainer classes (e.g., :py:class:`~pynwb.base.TimeSeries`) support the integration of data stored in external -# HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use -# case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external -# links allows us to save each data stream to an external HDF5 files during data acquisition and to -# afterwards link the data into a single NWB:N file. In this case, each recording becomes represented by a -# separate file-system object that can be set as read-only once the experiment is done. In the following -# we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other -# NWBContainers as well. -# +NBWContainer classes (e.g., :py:class:`~pynwb.base.TimeSeries`) support the integration of data stored in external +HDF5 files with NWB data files via external links. To make things more concrete, let's look at the following use +case. We want to simultaneously record multiple data streams during data acquisition. Using the concept of external +links allows us to save each data stream to an external HDF5 files during data acquisition and to +afterwards link the data into a single NWB file. In this case, each recording becomes represented by a +separate file-system object that can be set as read-only once the experiment is done. In the following +we are using :py:meth:`~pynwb.base.TimeSeries` as an example, but the same approach works for other +NWBContainers as well. -#################### -# .. tip:: -# -# The same strategies we use here for creating External Links also apply to Soft Links. -# The main difference between soft and external links is that soft links point to other -# objects within the same file while external links point to objects in external files. -# +.. tip:: -#################### -# .. tip:: -# -# In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition -# system can be stored (or linked) in the *sync* group. In the NWB:N format, hardware-recorded time data -# must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before -# it can be included in the *timestamps* of the *TimeSeries*. This means, in the case -# of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible -# timestamps in the same file when using external links. -# + The same strategies we use here for creating External Links also apply to Soft Links. + The main difference between soft and external links is that soft links point to other + objects within the same file while external links point to objects in external files. -#################### -# .. warning:: -# -# External links can become stale/break. Since external links are pointing to data in other files -# external links may become invalid any time files are modified on the file system, e.g., renamed, -# moved or access permissions are changed. -# + .. tip:: -#################### -# Creating test data -# --------------------------- -# -# In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file. -# We then show how we can integrate these files into a single NWBFile. + In the case of :py:meth:`~pynwb.base.TimeSeries`, the uncorrected timestamps generated by the acquisition + system can be stored (or linked) in the *sync* group. In the NWB format, hardware-recorded time data + must then be corrected to a common time base (e.g., timestamps from all hardware sources aligned) before + it can be included in the *timestamps* of the *TimeSeries*. This means, in the case + of :py:meth:`~pynwb.base.TimeSeries` we need to be careful that we are not including data with incompatible + timestamps in the same file when using external links. + + +.. warning:: + + External links can become stale/break. Since external links are pointing to data in other files + external links may become invalid any time files are modified on the file system, e.g., renamed, + moved or access permissions are changed. + + +Creating test data +--------------------------- + +In the following we are creating two :py:meth:`~pynwb.base.TimeSeries` each written to a separate file. +We then show how we can integrate these files into a single NWBFile. +""" # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_linking_data.png' + from datetime import datetime from uuid import uuid4 @@ -228,7 +221,7 @@ # Step 2: Add the container to another NWBFile # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # To integrate both :py:meth:`~pynwb.base.TimeSeries` into a single file we simply create a new -# :py:meth:`~pynwb.file.NWBFile` and our existing :py:meth:`~pynwb.base.TimeSeries` to it. PyNWB's +# :py:meth:`~pynwb.file.NWBFile` and add our existing :py:meth:`~pynwb.base.TimeSeries` to it. PyNWB's # :py:class:`~pynwb.NWBHDF5IO` backend then automatically detects that the TimeSeries have already # been written to another file and will create external links for us. # diff --git a/docs/gallery/advanced_io/plot_iterative_write.py b/docs/gallery/advanced_io/plot_iterative_write.py index c461cddf8..958981a0b 100644 --- a/docs/gallery/advanced_io/plot_iterative_write.py +++ b/docs/gallery/advanced_io/plot_iterative_write.py @@ -17,7 +17,7 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # # In the typical write process, datasets are created and written as a whole. In contrast, -# iterative data write refers to the writing of the content of a dataset in an incremental, +# iterative data write refers to the writing of the contents of a dataset in an incremental, # iterative fashion. #################### @@ -32,10 +32,10 @@ # to avoid this problem by writing the data one-subblock-at-a-time, so that we only need to hold # a small subset of the array in memory at any given time. # * **Data streaming** In the context of streaming data we are faced with several issues: -# **1)** data is not available in memory but arrives in subblocks as the stream progresses +# **1)** data is not available in-memory but arrives in subblocks as the stream progresses # **2)** caching the data of a stream in-memory is often prohibitively expensive and volatile # **3)** the total size of the data is often unknown ahead of time. -# Iterative data write allows us to address issues 1) and 2) by enabling us to save data to +# Iterative data write allows us to address issues 1) and 2) by enabling us to save data to a # file incrementally as it arrives from the data stream. Issue 3) is addressed in the HDF5 # storage backend via support for chunking, enabling the creation of resizable arrays. # @@ -44,7 +44,7 @@ # data source. # # * **Sparse data arrays** In order to reduce storage size of sparse arrays a challenge is that while -# the data array (e.g., a matrix) may be large, only few values are set. To avoid storage overhead +# the data array (e.g., a matrix) may be large, only a few values are set. To avoid storage overhead # for storing the full array we can employ (in HDF5) a combination of chunking, compression, and # and iterative data write to significantly reduce storage cost for sparse data. # @@ -161,7 +161,7 @@ def write_test_file(filename, data, close_io=True): # # Here we use a simple data generator but PyNWB does not make any assumptions about what happens # inside the generator. Instead of creating data programmatically, you may hence, e.g., receive -# data from an acquisition system (or other source). We can, hence, use the same approach to write streaming data. +# data from an acquisition system (or other source). We can use the same approach to write streaming data. #################### # Step 1: Define the data generator @@ -208,7 +208,7 @@ def iter_sin(chunk_length=10, max_chunks=100): #################### # Discussion # ^^^^^^^^^^ -# Note, we here actually do not know how long our timeseries will be. +# Note, here we don't actually know how long our timeseries will be. print( "maxshape=%s, recommended_data_shape=%s, dtype=%s" @@ -218,7 +218,7 @@ def iter_sin(chunk_length=10, max_chunks=100): #################### # As we can see :py:class:`~hdmf.data_utils.DataChunkIterator` automatically recommends # in its ``maxshape`` that the first dimensions of our array should be unlimited (``None``) and the second -# dimension be ``10`` (i.e., the length of our chunk. Since :py:class:`~hdmf.data_utils.DataChunkIterator` +# dimension should be ``10`` (i.e., the length of our chunk. Since :py:class:`~hdmf.data_utils.DataChunkIterator` # has no way of knowing the minimum size of the array it automatically recommends the size of the first # chunk as the minimum size (i.e, ``(1, 10)``) and also infers the data type automatically from the first chunk. # To further customize this behavior we may also define the ``maxshape``, ``dtype``, and ``buffer_size`` when @@ -227,8 +227,8 @@ def iter_sin(chunk_length=10, max_chunks=100): # .. tip:: # # We here used :py:class:`~hdmf.data_utils.DataChunkIterator` to conveniently wrap our data stream. -# :py:class:`~hdmf.data_utils.DataChunkIterator` assumes that our generators yields in **consecutive order** -# **single** complete element along the **first dimension** of our a array (i.e., iterate over the first +# :py:class:`~hdmf.data_utils.DataChunkIterator` assumes that our generator yields in **consecutive order** +# a **single** complete element along the **first dimension** of our array (i.e., iterate over the first # axis and yield one-element-at-a-time). This behavior is useful in many practical cases. However, if # this strategy does not match our needs, then using :py:class:`~hdmf.data_utils.GenericDataChunkIterator` # or implementing your own derived :py:class:`~hdmf.data_utils.AbstractDataChunkIterator` may be more @@ -266,7 +266,7 @@ def __next__(self): """ Return in each iteration a fully occupied data chunk of self.chunk_shape values at a random location within the matrix. Chunks are non-overlapping. REMEMBER: h5py does not support all - fancy indexing that numpy does so we need to make sure our selection can be + the fancy indexing that numpy does so we need to make sure our selection can be handled by the backend. """ if self.__chunks_created < self.num_chunks: @@ -289,7 +289,7 @@ def __next__(self): next = __next__ def recommended_chunk_shape(self): - # Here we can optionally recommend what a good chunking should be. + # Here we can optionally recommend what a good chunking could be. return self.chunk_shape def recommended_data_shape(self): @@ -379,7 +379,7 @@ def maxshape(self): # Now lets check out the size of our data file and compare it against the expected full size of our matrix import os -expected_size = xsize * ysize * 8 # This is the full size of our matrix in byte +expected_size = xsize * ysize * 8 # This is the full size of our matrix in bytes occupied_size = num_values * 8 # Number of non-zero values in out matrix file_size = os.stat( "basic_sparse_iterwrite_example.nwb" @@ -420,14 +420,14 @@ def maxshape(self): # A slight overhead (here 0.08MB) is expected because our file contains also the additional objects from # the NWBFile, plus some overhead for managing all the HDF5 metadata for all objects. # * **3) vs 2):** Adding compression does not yield any improvement here. This is expected, because, again we -# selected the chunking here in a way that we already allocated the minimum amount of storage to represent our data +# selected the chunking here in a way that we already allocated the minimum amount of storage to represent our data # and lossless compression of random data is not efficient. # * **4) vs 2):** When we increase our chunk size to ``(100,100)`` (i.e., ``100x`` larger than the chunks produced by -# our matrix generator) we observe an according roughly ``100x`` increase in file size. This is expected +# our matrix generator) we observe an accordingly roughly ``100x`` increase in file size. This is expected # since our chunks now do not align perfectly with the occupied data and each occupied chunk is allocated fully. # * **5) vs 4):** When using compression for the larger chunks we see a significant reduction # in file size (``1.14MB`` vs. ``80MB``). This is because the allocated chunks now contain in addition to the random -# values large areas of constant fillvalues, which compress easily. +# values large areas of constant fill values, which compress easily. # # **Advantages:** # @@ -435,12 +435,12 @@ def maxshape(self): # * Only the data chunks in the HDF5 file that contain non-default values are ever being allocated # * The overall size of our file is reduced significantly # * Reduced I/O load -# * On read users can use the array as usual +# * On read, users can use the array as usual # # .. tip:: # -# With great power comes great responsibility **!** I/O and storage cost will depend among others on the chunk size, -# compression options, and the write pattern, i.e., the number and structure of the +# With great power comes great responsibility **!** I/O and storage cost will depend, among other factors, +# on the chunk size, compression options, and the write pattern, i.e., the number and structure of the # :py:class:`~hdmf.data_utils.DataChunk` objects written. For example, using ``(1,1)`` chunks and writing them # one value at a time would result in poor I/O performance in most practical cases, because of the large number of # chunks and large number of small I/O operations required. @@ -471,7 +471,7 @@ def maxshape(self): # # When converting large data files, a typical problem is that it is often too expensive to load all the data # into memory. This example is very similar to the data generator example only that instead of generating -# data on-the-fly in memory we are loading data from a file one-chunk-at-a-time in our generator. +# data on-the-fly in-memory we are loading data from a file one-chunk-at-a-time in our generator. # #################### @@ -568,7 +568,7 @@ def iter_largearray(filename, shape, dtype="float64"): # In practice, data from recording devices may be distributed across many files, e.g., one file per time range # or one file per recording channel. Using iterative data write provides an elegant solution to this problem # as it allows us to process large arrays one-subarray-at-a-time. To make things more interesting we'll show -# this for the case where each recording channel (i.e, the second dimension of our ``TimeSeries``) is broken up +# this for the case where each recording channel (i.e., the second dimension of our ``TimeSeries``) is broken up # across files. #################### diff --git a/docs/gallery/advanced_io/streaming.py b/docs/gallery/advanced_io/streaming.py index b3800584a..760e2da71 100644 --- a/docs/gallery/advanced_io/streaming.py +++ b/docs/gallery/advanced_io/streaming.py @@ -23,113 +23,152 @@ Now you can get the url of a particular NWB file using the dandiset ID and the path of that file within the dandiset. -.. code-block:: python - - from dandi.dandiapi import DandiAPIClient - - dandiset_id = '000006' # ephys dataset from the Svoboda Lab - filepath = 'sub-anm372795/sub-anm372795_ses-20170718.nwb' # 450 kB file - with DandiAPIClient() as client: - asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) - s3_url = asset.get_content_url(follow_redirects=1, strip_query=True) - - -Streaming Method 1: fsspec --------------------------- -fsspec is another data streaming approach that is quite flexible and has several performance advantages. This library -creates a virtual filesystem for remote stores. With this approach, a virtual file is created for the file and -the virtual filesystem layer takes care of requesting data from the S3 bucket whenever data is -read from the virtual file. Note that this implementation is completely unaware of internals of the HDF5 format -and thus can work for **any** file, not only for the purpose of use with H5PY and PyNWB. - -First install ``fsspec`` and the dependencies of the :py:class:`~fsspec.implementations.http.HTTPFileSystem`: - -.. code-block:: bash - - pip install fsspec requests aiohttp - -Then in Python: - -.. code-block:: python - - import fsspec - import pynwb - import h5py - from fsspec.implementations.cached import CachingFileSystem - - # first, create a virtual filesystem based on the http protocol - fs=fsspec.filesystem("http") - - # create a cache to save downloaded data to disk (optional) - fs = CachingFileSystem( - fs=fs, - cache_storage="nwb-cache", # Local folder for the cache - ) - - # next, open the file - with fs.open(s3_url, "rb") as f: - with h5py.File(f) as file: - with pynwb.NWBHDF5IO(file=file, load_namespaces=True) as io: - nwbfile = io.read() - print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:]) - - -fsspec is a library that can be used to access a variety of different store formats, including (at the time of -writing): - -.. code-block:: python - - from fsspec.registry import known_implementations - known_implementations.keys() - -file, memory, dropbox, http, https, zip, tar, gcs, gs, gdrive, sftp, ssh, ftp, hdfs, arrow_hdfs, webhdfs, s3, s3a, wandb, oci, adl, abfs, az, cached, blockcache, filecache, simplecache, dask, dbfs, github, git, smb, jupyter, jlab, libarchive, reference - -The S3 backend, in particular, may provide additional functionality for accessing data on DANDI. See the -`fsspec documentation on known implementations `_ -for a full updated list of supported store formats. - -Streaming Method 2: ROS3 ------------------------- -ROS3 is one of the supported methods for reading data from a remote store. ROS3 stands for "read only S3" and is a -driver created by the HDF5 Group that allows HDF5 to read HDF5 files stored remotely in s3 buckets. Using this method -requires that your HDF5 library is installed with the ROS3 driver enabled. This is not the default configuration, -so you will need to make sure you install the right version of ``h5py`` that has this advanced configuration enabled. -You can install HDF5 with the ROS3 driver from `conda-forge `_ using ``conda``. You may -first need to uninstall a currently installed version of ``h5py``. - -.. code-block:: bash - - pip uninstall h5py - conda install -c conda-forge "h5py>=3.2" - -Now instantiate a :py:class:`~pynwb.NWBHDF5IO` object with the S3 URL and specify the driver as "ros3". This -will download metadata about the file from the S3 bucket to memory. The values of datasets are accessed lazily, -just like when reading an NWB file stored locally. So, slicing into a dataset will require additional time to -download the sliced data (and only the sliced data) to memory. - -.. code-block:: python - - from pynwb import NWBHDF5IO - - with NWBHDF5IO(s3_url, mode='r', load_namespaces=True, driver='ros3') as io: - nwbfile = io.read() - print(nwbfile) - print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:]) - -Which streaming method to choose? ---------------------------------- - -fsspec has many advantages over ros3: - -1. fsspec is easier to install -2. fsspec supports caching, which will dramatically speed up repeated requests for the - same region of data -3. fsspec automatically retries when s3 fails to return. -4. fsspec works with other storage backends and -5. fsspec works with other types of files. -6. In our hands, fsspec is faster out-of-the-box. - -For these reasons, we would recommend use fsspec for most Python users. """ # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_streaming.png' + +from dandi.dandiapi import DandiAPIClient + +dandiset_id = '000006' # ephys dataset from the Svoboda Lab +filepath = 'sub-anm372795/sub-anm372795_ses-20170718.nwb' # 450 kB file +with DandiAPIClient() as client: + asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) + s3_url = asset.get_content_url(follow_redirects=1, strip_query=True) + +############################################## +# Streaming Method 1: fsspec +# -------------------------- +# fsspec is another data streaming approach that is quite flexible and has several performance advantages. This library +# creates a virtual filesystem for remote stores. With this approach, a virtual file is created for the file and +# the virtual filesystem layer takes care of requesting data from the S3 bucket whenever data is +# read from the virtual file. Note that this implementation is completely unaware of internals of the HDF5 format +# and thus can work for **any** file, not only for the purpose of use with H5PY and PyNWB. +# +# First install ``fsspec`` and the dependencies of the :py:class:`~fsspec.implementations.http.HTTPFileSystem`: +# +# .. code-block:: bash +# +# pip install fsspec requests aiohttp +# +# Then in Python: + +import fsspec +import pynwb +import h5py +from fsspec.implementations.cached import CachingFileSystem + +# first, create a virtual filesystem based on the http protocol +fs = fsspec.filesystem("http") + +# create a cache to save downloaded data to disk (optional) +fs = CachingFileSystem( + fs=fs, + cache_storage="nwb-cache", # Local folder for the cache +) + +# next, open the file +with fs.open(s3_url, "rb") as f: + with h5py.File(f) as file: + with pynwb.NWBHDF5IO(file=file) as io: + nwbfile = io.read() + print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:]) + +################################## +# fsspec is a library that can be used to access a variety of different store formats, including (at the time of +# writing): +# +# .. code-block:: python +# +# from fsspec.registry import known_implementations +# known_implementations.keys() +# +# file, memory, dropbox, http, https, zip, tar, gcs, gs, gdrive, sftp, ssh, ftp, hdfs, arrow_hdfs, webhdfs, s3, s3a, +# wandb, oci, adl, abfs, az, cached, blockcache, filecache, simplecache, dask, dbfs, github, git, smb, jupyter, jlab, +# libarchive, reference +# +# The S3 backend, in particular, may provide additional functionality for accessing data on DANDI. See the +# `fsspec documentation on known implementations `_ +# for a full updated list of supported store formats. +# +# One downside of this fsspec method is that fsspec is not optimized for reading HDF5 files, and so streaming data +# using this method can be slow. A faster alternative is ``remfile`` described below. +# +# Streaming Method 2: ROS3 +# ------------------------ +# ROS3 stands for "read only S3" and is a driver created by the HDF5 Group that allows HDF5 to read HDF5 files stored +# remotely in s3 buckets. Using this method requires that your HDF5 library is installed with the ROS3 driver enabled. +# With ROS3 support enabled in h5py, we can instantiate a :py:class:`~pynwb.NWBHDF5IO` object with the S3 URL and +# specify the driver as "ros3". + +from pynwb import NWBHDF5IO + +with NWBHDF5IO(s3_url, mode='r', driver='ros3') as io: + nwbfile = io.read() + print(nwbfile) + print(nwbfile.acquisition['lick_times'].time_series['lick_left_times'].data[:]) + +################################## +# This will download metadata about the file from the S3 bucket to memory. The values of datasets are accessed lazily, +# just like when reading an NWB file stored locally. So, slicing into a dataset will require additional time to +# download the sliced data (and only the sliced data) to memory. +# +# .. note:: +# +# Pre-built h5py packages on PyPI do not include this S3 support. If you want this feature, you could use packages +# from conda-forge, or build h5py from source against an HDF5 build with S3 support. You can install HDF5 with +# the ROS3 driver from `conda-forge `_ using ``conda``. You may +# first need to uninstall a currently installed version of ``h5py``. +# +# .. code-block:: bash +# +# pip uninstall h5py +# conda install -c conda-forge "h5py>=3.2" +# +# Besides the extra burden of installing h5py from a non-PyPI source, one downside of this ROS3 method is that +# this method does not support automatic retries in case the connection fails. + + +################################################## +# Method 3: remfile +# ----------------- +# ``remfile`` is another library that enables indexing and streaming of files in s3. remfile is simple and fast, +# especially for the initial load of the nwb file and for accessing small pieces of data. The caveats of ``remfile`` +# are that it is a very new project that has not been tested in a variety of use-cases and caching options are +# limited compared to ``fsspec``. `remfile` is a simple, lightweight dependency with a very small codebase. +# +# You can install ``remfile`` with pip: +# +# .. code-block:: bash +# +# pip install remfile +# + +import h5py +from pynwb import NWBHDF5IO +import remfile + +rem_file = remfile.File(s3_url) + +with h5py.File(rem_file, "r") as h5py_file: + with NWBHDF5IO(file=h5py_file, load_namespaces=True) as io: + nwbfile = io.read() + print(nwbfile.acquisition["lick_times"].time_series["lick_left_times"].data[:]) + +################################################## +# Which streaming method to choose? +# --------------------------------- +# +# From a user perspective, once opened, the :py:class:`~pynwb.file.NWBFile` works the same with +# fsspec, ros3, or remfile. However, in general, we currently recommend using fsspec for streaming +# NWB files because it is more performant and reliable than ros3 and more widely tested than remfile. +# However, if you are experiencing long wait times for the initial file load on your network, you +# may want to try remfile. +# +# Advantages of fsspec include: +# +# 1. supports caching, which will dramatically speed up repeated requests for the +# same region of data, +# 2. automatically retries when s3 fails to return, which helps avoid errors when accessing data due to +# intermittent errors in connections with S3 (remfile does this as well), +# 3. works also with other storage backends (e.g., GoogleDrive or Dropbox, not just S3) and file formats, and +# 4. in our experience appears to provide faster out-of-the-box performance than the ros3 driver. diff --git a/docs/gallery/domain/plot_behavior.py b/docs/gallery/domain/plot_behavior.py index 3436be89e..8f341bea1 100644 --- a/docs/gallery/domain/plot_behavior.py +++ b/docs/gallery/domain/plot_behavior.py @@ -100,7 +100,7 @@ # .. note:: # :py:class:`~pynwb.behavior.SpatialSeries` data should be stored as one continuous stream, # as it is acquired, not by trial as is often reshaped for analysis. -# Data can be trial-aligned on-the-fly using the trials table. See the :ref:`basic_trials` tutorial +# Data can be trial-aligned on-the-fly using the trials table. See the :ref:`time_intervals` tutorial # for further information. # # For position data ``reference_frame`` indicates the zero-position, e.g. diff --git a/docs/gallery/general/add_remove_containers.py b/docs/gallery/general/add_remove_containers.py index 26708f639..80c5cb032 100644 --- a/docs/gallery/general/add_remove_containers.py +++ b/docs/gallery/general/add_remove_containers.py @@ -77,7 +77,7 @@ # modifies the data on disk # (the :py:meth:`NWBHDF5IO.write ` method does not need to be called and the # :py:class:`~pynwb.NWBHDF5IO` instance does not need to be closed). Directly modifying datasets in this way -# can lead to files that do not validate or cannot be opened, so take caution when using this method. +# can lead to files that do not validate or cannot be opened, so exercise caution when using this method. # Note: only chunked datasets or datasets with ``maxshape`` set can be resized. # See the `h5py chunked storage documentation `_ # for more details. diff --git a/docs/gallery/general/extensions.py b/docs/gallery/general/extensions.py index fa4f4cbb7..4ec8f4749 100644 --- a/docs/gallery/general/extensions.py +++ b/docs/gallery/general/extensions.py @@ -100,7 +100,7 @@ # Using extensions # ----------------------------------------------------- # -# After an extension has been created, it can be used by downstream codes for reading and writing data. +# After an extension has been created, it can be used by downstream code for reading and writing data. # There are two main mechanisms for reading and writing extension data with PyNWB. # The first involves defining new :py:class:`~pynwb.core.NWBContainer` classes that are then mapped # to the neurodata types in the extension. @@ -167,7 +167,7 @@ def __init__(self, **kwargs): # By default, extensions are cached to file so that your NWB file will carry the extensions needed to read the file # with it. # -# To demonstrate this, first we will make some fake data using our extensions. +# To demonstrate this, first we will make some simulated data using our extensions. from datetime import datetime @@ -248,16 +248,11 @@ def __init__(self, **kwargs): # .. note:: # # For more information on writing NWB files, see :ref:`basic_writing`. - -#################### -# By default, PyNWB does not use the namespaces cached in a file--you must -# explicitly specify this. This behavior is enabled by the *load_namespaces* -# argument to the :py:class:`~pynwb.NWBHDF5IO` constructor. - -with NWBHDF5IO("cache_spec_example.nwb", mode="r", load_namespaces=True) as io: - nwbfile = io.read() - -#################### +# +# By default, if a namespace is not already loaded, PyNWB loads the namespace cached in +# the file. To disable this, set ``load_namespaces=False`` in the +# :py:class:`~pynwb.NWBHDF5IO` constructor. +# # .. _MultiContainerInterface: # # Creating and using a custom MultiContainerInterface @@ -375,17 +370,17 @@ class PotatoSack(MultiContainerInterface): nwb = io.read() print(nwb.get_processing_module()["potato_sack"].get_potato("big_potato").weight) # note: you can call get_processing_module() with or without the module name as -# an argument. however, if there is more than one module, the name is required. -# here, there is more than one potato, so the name of the potato is required as -# an argument to get get_potato +# an argument. However, if there is more than one module, the name is required. +# Here, there is more than one potato, so the name of the potato is required as +# an argument to get_potato #################### # Example: Cortical Surface Mesh # ----------------------------------------------------- # # Here we show how to create extensions by creating a data class for a -# cortical surface mesh. This data type is particularly important for ECoG data, we need to know where each electrode is -# with respect to the gyri and sulci. Surface mesh objects contain two types of data: +# cortical surface mesh. This data type is particularly important for ECoG data, since we need to know where +# each electrode is with respect to the gyri and sulci. Surface mesh objects contain two types of data: # # 1. `vertices`, which is an (n, 3) matrix of floats that represents points in 3D space # diff --git a/docs/gallery/general/object_id.py b/docs/gallery/general/object_id.py index 481cbb36a..206142715 100644 --- a/docs/gallery/general/object_id.py +++ b/docs/gallery/general/object_id.py @@ -32,7 +32,7 @@ session_start_time=start_time, ) -# make some fake data +# make some simulated data timestamps = np.linspace(0, 100, 1024) data = ( np.sin(0.333 * timestamps) diff --git a/docs/gallery/general/plot_file.py b/docs/gallery/general/plot_file.py index 6ac9ee250..beead22f6 100644 --- a/docs/gallery/general/plot_file.py +++ b/docs/gallery/general/plot_file.py @@ -33,7 +33,7 @@ ^^^^^^^^^^ :py:class:`~pynwb.base.TimeSeries` objects store time series data and correspond to the *TimeSeries* specifications -provided by the `NWB Format`_ . Like the NWB specification, :py:class:`~pynwb.base.TimeSeries` Python objects +provided by the `NWB Format`_. Like the NWB specification, :py:class:`~pynwb.base.TimeSeries` Python objects follow an object-oriented inheritance pattern, i.e., the class :py:class:`~pynwb.base.TimeSeries` serves as the base class for all other :py:class:`~pynwb.base.TimeSeries` types, such as, :py:class:`~pynwb.ecephys.ElectricalSeries`, which itself may have further subtypes, e.g., @@ -122,17 +122,20 @@ NWB organizes data into different groups depending on the type of data. Groups can be thought of as folders within the file. Here are some of the groups within an :py:class:`~pynwb.file.NWBFile` and the types of data they are intended to store: - * **acquisition**: raw, acquired data that should never change - * **processing**: processed data, typically the results of preprocessing algorithms and could change - * **analysis**: results of data analysis - * **stimuli**: stimuli used in the experiment (e.g., images, videos, light pulses) + +* **acquisition**: raw, acquired data that should never change +* **processing**: processed data, typically the results of preprocessing algorithms and could change +* **analysis**: results of data analysis +* **stimuli**: stimuli used in the experiment (e.g., images, videos, light pulses) The following examples will reference variables that may not be defined within the block they are used in. For clarity, we define them here: + """ -from datetime import datetime # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_file.png' + +from datetime import datetime from uuid import uuid4 import numpy as np @@ -156,7 +159,8 @@ # occurred exactly at the session start time. # # Create an :py:class:`~pynwb.file.NWBFile` object with the required fields -# (``session_description``, ``identifier``, ``session_start_time``) and additional metadata. +# (:py:attr:`~pynwb.file.NWBFile.session_description`, :py:attr:`~pynwb.file.NWBFile.identifier`, +# :py:attr:`~pynwb.file.NWBFile.session_start_time`) and additional metadata. # # .. note:: # Use keyword arguments when constructing :py:class:`~pynwb.file.NWBFile` objects. diff --git a/docs/gallery/general/read_basics.py b/docs/gallery/general/plot_read_basics.py similarity index 62% rename from docs/gallery/general/read_basics.py rename to docs/gallery/general/plot_read_basics.py index 03a698e4f..c4a829d75 100644 --- a/docs/gallery/general/read_basics.py +++ b/docs/gallery/general/plot_read_basics.py @@ -24,28 +24,30 @@ The following examples will reference variables that may not be defined within the block they are used in. For clarity, we define them here: """ -import matplotlib.pyplot as plt # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnails_read_basics.png' + +import matplotlib.pyplot as plt import numpy as np -from dandi.dandiapi import DandiAPIClient from pynwb import NWBHDF5IO #################### -# Read the data -# ------------- -# We will use the `DANDI `_ neurophysiology data archive -# to access an NWB File. We will use data from one session of an experiment by -# `Chandravadia et al. (2020) `_, where +# We will access NWB data on the `DANDI Archive `_, +# and demonstrate reading one session of an experiment by +# `Chandravadia et al. (2020) `_. In this study, # the authors recorded single neuron activity from the medial temporal lobes of human subjects # while they performed a recognition memory task. # # Download the data -# ^^^^^^^^^^^^^^^^^ +# ----------------- # First, we will demonstrate how to download an NWB data file from `DANDI `_ # to your machine. # +# Download using the DANDI Web UI +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# You can download files directly from the DANDI website. +# # 1. Go to the DANDI archive and open `this `_ dataset # 2. List the files in this dataset by clicking the "Files" button in Dandiset Actions (top right column of the page). # @@ -69,62 +71,60 @@ # :alt: selecting a folder on dandi # :align: center # -# Stream the data -# ^^^^^^^^^^^^^^^ -# -# Next, we will demonstrate how to stream the data from the DANDI archive without -# having to download it to your machine. -# Streaming data requires having HDF5 installed with the ROS3 (read-only S3) driver. -# You can install from `conda-forge `_ using ``conda``. -# You might need to first uninstall a currently installed version of ``h5py``. -# -# .. code-block:: bash -# -# $ pip uninstall h5py -# $ conda install -c conda-forge "h5py>=3.2" -# -# We can access the data stored in an S3 bucket using the DANDI API, -# which can be installed from pip: +# Downloading data programmatically +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +# Alternatively, you can download data using the `dandi` Python module. + +from dandi.download import download + +download("https://api.dandiarchive.org/api/assets/0f57f0b0-f021-42bb-8eaa-56cd482e2a29/download/", ".") + +###################################################### +# .. seealso:: # -# .. code-block:: bash +# Learn about all the different ways you can download data from the DANDI Archive +# `here `_ # -# $ pip install -U dandi +# .. seealso:: Streaming data # -# .. seealso:: -# You can learn more about streaming data in the :ref:`streaming` tutorial. +# Instead of downloading data, another approach is to stream data directly from an archive. Streaming data allows you +# to download only the data you want from a file, so it can be a much better approach when the desired data files +# contain a lot of data you don't care about. There are several approaches to streaming NWB files, outlined in +# :ref:`streaming`. # -# Then, we will use the :py:class:`~dandi.dandiapi.DandiAPIClient` to obtain the S3 URL that points to the NWB File -# stored in S3. We will need the identifier of the dataset (``dandiset_id``) and the path -# to the NWB File. -# We can read these from the DANDI archive URL where ``dandiset_id`` is "000004" and -# file is located in "sub-P11HMH" folder. - - -dandiset_id = "000004" -filepath = "sub-P11HMH/sub-P11HMH_ses-20061101_ecephys+image.nwb" -with DandiAPIClient() as client: - asset = client.get_dandiset(dandiset_id, "draft").get_asset_by_path(filepath) - s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) - -#################### -# Using NWBHDF5IO -# --------------- +# Opening an NWB file with NWBHDF5IO +# ---------------------------------- # # Reading and writing NWB data is carried out using the :py:class:`~pynwb.NWBHDF5IO` class. # :py:class:`~pynwb.NWBHDF5IO` reads NWB data that is in the `HDF5 `_ # storage format, a popular, hierarchical format for storing large-scale scientific data. # -# The first argument to the constructor of :py:class:`~pynwb.NWBHDF5IO` is the ``file_path`` - -# this can be the path that points to the downloaded file on your computer or -# it can be an S3 URL. -# -# Use the ``read`` method to read the data into a :py:class:`~pynwb.file.NWBFile` object. +# The first argument to the constructor of :py:class:`~pynwb.NWBHDF5IO` is the ``file_path``. Use the ``read`` method to +# read the data into a :py:class:`~pynwb.file.NWBFile` object. -# Open the file in read mode "r", and specify the driver as "ros3" for S3 files -io = NWBHDF5IO(s3_path, mode="r", driver="ros3") +filepath = "sub-P11HMH_ses-20061101_ecephys+image.nwb" +# Open the file in read mode "r", +io = NWBHDF5IO(filepath, mode="r") nwbfile = io.read() +nwbfile -#################### +####################################### +# :py:class:`~pynwb.NWBHDF5IO` can also be used as a context manager: + +with NWBHDF5IO(filepath, mode="r") as io2: + nwbfile2 = io2.read() + + # data accessible here + +# data not accessible here + +###################################### +# The advantage of using a context manager is that the file is closed automatically when the context finishes +# successfully or if there is an error. Be aware that if you use this method, closing the context (unindenting the code) +# will automatically close the :py:class:`~pynwb.NWBHDF5IO` object and the corresponding h5py File object. The data not +# already read from the NWB file will then be inaccessible, so any code that reads data must be placed within the +# context. +# # Access stimulus data # -------------------- # @@ -135,39 +135,18 @@ #################### # ``NWBFile.stimulus`` is a dictionary that can contain PyNWB objects representing -# different types of data; such as images (grayscale, RGB) or time series of images. +# different types of data, such as images (grayscale, RGB) or time series of images. # In this file, ``NWBFile.stimulus`` contains a single key "StimulusPresentation" with an # :py:class:`~pynwb.image.OpticalSeries` object representing what images were shown to the subject and at what times. nwbfile.stimulus["StimulusPresentation"] #################### -# .. code-block:: none -# -# {'StimulusPresentation': StimulusPresentation pynwb.image.OpticalSeries at 0x140385583638560 -# Fields: -# comments: no comments -# conversion: 1.0 -# data: -# description: no description -# dimension: -# distance: 0.7 -# field_of_view: -# format: raw -# interval: 1 -# orientation: lower left -# resolution: -1.0 -# timestamps: -# timestamps_unit: seconds -# unit: pixels -# } -# -# # Lazy loading of datasets # ------------------------ # Data arrays are read passively from the NWB file. # Accessing the ``data`` attribute of the :py:class:`~pynwb.image.OpticalSeries` object -# does not read the data values, but presents an HDF5 object that can be indexed to read data. +# does not read the data values, but presents an :py:class:`h5py.Dataset` object that can be indexed to read data. # You can use the ``[:]`` operator to read the entire data array into memory. stimulus_presentation = nwbfile.stimulus["StimulusPresentation"] @@ -181,10 +160,6 @@ stimulus_presentation.data.shape #################### -# .. code-block:: none -# -# (200, 400, 300, 3) -# # This :py:class:`~pynwb.image.OpticalSeries` data contains 200 images of size 400x300 pixels with three channels # (red, green, and blue). # @@ -201,13 +176,6 @@ plt.imshow(image, aspect="auto") #################### -# -# .. image:: ../../_static/demo_nwbfile_stimulus_plot_1.png -# :width: 500 -# :alt: NWBFile stimulus image -# :align: center -# -# # Access single unit data # ----------------------- # Data and metadata about sorted single units are stored in :py:class:`~pynwb.misc.Units` @@ -224,7 +192,7 @@ # We can view the single unit data as a :py:class:`~pandas.DataFrame`. units_df = units.to_dataframe() -units_df +units_df.head() #################### # To access the spike times of the first single unit, index :py:class:`~pynwb.file.NWBFile.units` with the column @@ -234,12 +202,6 @@ units["spike_times"][0] #################### -# .. code-block:: none -# -# array([5932.811644, 6081.077044, 6091.982364, 6093.127644, 6095.068204, -# 6097.438244, 6116.694804, 6129.827604, 6134.825004, 6142.583924, ...]) -# -# # Visualize spiking activity relative to stimulus onset # ----------------------------------------------------- # We can look at when these single units spike relative to when image stimuli were presented to the subject. @@ -276,12 +238,6 @@ axs[1].axvline(0, color=[0.5, 0.5, 0.5]) #################### -# -# .. image:: ../../_static/demo_nwbfile_units_plot.png -# :width: 500 -# :alt: NWBFile units visualization -# :align: center -# # Access Trials # ------------- # Trials are stored as :py:class:`~pynwb.epoch.TimeIntervals` object which is a subclass @@ -290,16 +246,16 @@ # and additional metadata. # # .. seealso:: -# You can learn more about trials in the :ref:`basic_trials` tutorial section. +# You can learn more about trials in the :ref:`time_intervals` tutorial. # # Similarly to :py:class:`~pynwb.misc.Units`, we can view trials as a :py:class:`pandas.DataFrame`. trials_df = nwbfile.trials.to_dataframe() -trials_df +trials_df.head() #################### -# The :py:class:`~pynwb.file.NWBFile.stimulus` can be mapped one-to-one to each row (trial) -# of :py:class:`~pynwb.file.NWBFile.trials` based on the ``stim_on_time`` column. +# The stimulus can be mapped one-to-one to each row (trial) of +# :py:class:`~pynwb.file.NWBFile.trials` based on the ``stim_on_time`` column. assert np.all(stimulus_presentation.timestamps[:] == trials_df.stim_on_time[:]) @@ -309,7 +265,7 @@ stim_on_times_landscapes = trials_df[ trials_df.category_name == "landscapes" ].stim_on_time -for time in stim_on_times_landscapes[:3]: +for time in stim_on_times_landscapes.iloc[:3]: img = np.squeeze( stimulus_presentation.data[ np.where(stimulus_presentation.timestamps[:] == time) @@ -321,59 +277,9 @@ plt.imshow(img, aspect="auto") #################### -# -# .. image:: ../../_static/demo_nwbfile_stimulus_plot_2.png -# :width: 500 -# :alt: NWBFile landscapes stimuli image -# :align: center -# # Exploring the NWB file # ---------------------- # So far we have explored the NWB file by printing the :py:class:`~pynwb.file.NWBFile` # object and accessing its attributes, but it may be useful to explore the data in a -# more interactive, visual way. -# -# You can use `NWBWidgets `_, -# a package containing interactive widgets for visualizing NWB data, -# or you can use the `HDFView `_ -# tool, which can open any generic HDF5 file, which an NWB file is. -# -# NWBWidgets -# ^^^^^^^^^^ -# Install NWBWidgets using pip install: -# -# .. code-block:: bash -# -# $ pip install -U nwbwidgets -# -# Then import the ``nwbwidgets`` package and run the ``nwb2widget()`` function on -# the :py:class:`~pynwb.file.NWBFile` object. - -##################### -# -# .. code-block:: python -# -# from nwbwidgets import nwb2widget -# -# nwb2widget(nwbfile) -# - -#################### -# -# .. image:: ../../_static/demo_nwbwidgets.png -# :width: 700 -# :alt: inspect nwb file with nwbwidgets -# :align: center -# -# -# HDFView -# ^^^^^^^ -# To use `HDFView `_ to inspect and explore the NWB file, -# download and install HDFView from `here `_ -# and then open the NWB file using the application. -# -# .. image:: ../../_static/demo_hdfview.png -# :width: 700 -# :alt: inspect nwb file with hdfview -# :align: center -# +# more interactive, visual way. See :ref:`analysistools-explore` for an updated list of programs for +# exploring NWB files. diff --git a/docs/gallery/general/scratch.py b/docs/gallery/general/scratch.py index 50d97339a..0e00c5e96 100644 --- a/docs/gallery/general/scratch.py +++ b/docs/gallery/general/scratch.py @@ -12,7 +12,7 @@ .. note:: The scratch space is explicitly for non-standardized data that is not intended for reuse - by others. Standard NWB:N types, and extension if required, should always be used for any data that you + by others. Standard NWB types, and extension if required, should always be used for any data that you intend to share. As such, published data should not include scratch data and a user should be able to ignore any data stored in scratch to use a file. @@ -127,7 +127,7 @@ # # You may end up wanting to store results from some one-off analysis, and writing an extension # to get your data into an NWBFile is too much over head. This is facilitated by the scratch space -# in NWB:N. [#]_ +# in NWB. [#]_ # # First, lets read our processed data and then make a copy diff --git a/docs/notebooks/convert-crcns-ret-1-meisterlab-compare-nwb-1.0.6.ipynb b/docs/notebooks/convert-crcns-ret-1-meisterlab-compare-nwb-1.0.6.ipynb index c348c4bd0..65dc34188 100644 --- a/docs/notebooks/convert-crcns-ret-1-meisterlab-compare-nwb-1.0.6.ipynb +++ b/docs/notebooks/convert-crcns-ret-1-meisterlab-compare-nwb-1.0.6.ipynb @@ -39,7 +39,7 @@ "source": [ "This notebook uses the convert script and API for NWB v.1.0.6 (not the current NWB 2.0 and PyNWB) to generate NWB v1.0.6 data files and compare with the current format. This notebook is mainly for comparison purposes. The corresponding notebook for converting the MeisterLab example data to NWB 2.x is available here: https://github.com/NeurodataWithoutBorders/pynwb/blob/dev/docs/notebooks/convert-crcns-ret-1-meisterlab.ipynb .\n", "\n", - "This example is based on https://github.com/NeurodataWithoutBorders/api-python/blob/master/examples/create_scripts/crcns_ret-1.py from H5Gate (i.e., the orignal write API for NWB v1.x). A tar file with the example data is available for download from: https://portal.nersc.gov/project/crcns/download/nwb-1/example_script_data/source_data_2.tar.gz Please download and uncompress the data file and update the paths in the *Settings* section if you want to run the notebook. " + "This example is based on https://github.com/NeurodataWithoutBorders/api-python/blob/master/examples/create_scripts/crcns_ret-1.py from H5Gate (i.e., the original write API for NWB v1.x). A tar file with the example data is available for download from: https://portal.nersc.gov/project/crcns/download/nwb-1/example_script_data/source_data_2.tar.gz Please download and uncompress the data file and update the paths in the *Settings* section if you want to run the notebook. " ] }, { @@ -71,7 +71,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# 3 Exectute convert using the original H5Gate API" + "# 3 Execute convert using the original H5Gate API" ] }, { @@ -1259,7 +1259,7 @@ "source": [ "Compared to the convert using NWB v1.0.x shown above, the NWB 2 convert example makes the following main changes:\n", "\n", - "* NWB 2.x uses the extension mechanism to add custom data fields rather than adding unspecified custom data directly to the file, i.e., all objects (datasets, attributes, groups etc.) are governed by a formal specification. E.g., in the original script for NWB 1.0.x, pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. For NWB 2 we create an extensions MeisterImageSeries which extens ImageSeries and stores those values as attributes pixel_size, x, y, dx, dy. For NWB 2 we chosse attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", + "* NWB 2.x uses the extension mechanism to add custom data fields rather than adding unspecified custom data directly to the file, i.e., all objects (datasets, attributes, groups etc.) are governed by a formal specification. E.g., in the original script for NWB 1.0.x, pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. For NWB 2 we create an extensions MeisterImageSeries which extens ImageSeries and stores those values as attributes pixel_size, x, y, dx, dy. For NWB 2 we chose attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", "* Change si_unit attribute to unit for compliance with the spec of ImageSeries \n", "* Moved 'source' attribute from the Module to the Interface as source is not defined in the spec for modules but only for Interface\n", "* Added missing 'source' for SpikeUnit\n", @@ -1269,7 +1269,7 @@ "* NWBContainer is now a base type of all core neurodata_types and as such `help` and `source` attributes have been added to all core types\n", "* The original script reused iterator variables in nested loops. We have updated those occurrence to avoid consusion and avoid possible errors. \n", "* The following custom metadata fields---i.e., datasets that were originally added to the file without being part of the NWB specification and without creation of corresponding extensions---have not yet been integrated with the NWB files:\n", - " * /general custom metdata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", + " * /general custom metadata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", " * SpikeUnit custom datasets with additional copies of the per-stimulus spike times (i.e., /processing/Cells/UnitTimes/cell_*/stim_* in the original version). This will require an extension for SpikeUnit.\n", " * /subject, subject/genotype, subject/species : See Issue https://github.com/NeurodataWithoutBorders/pynwb/issues/45 support for subject metadata is upcoming in PyNWB \n", " * /specifications, /specifications/nwb_core.py : See Issue hssue https://github.com/NeurodataWithoutBorders/pynwb/issues/44 will be added by PyNWB automatically" diff --git a/docs/notebooks/convert-crcns-ret-1-meisterlab.ipynb b/docs/notebooks/convert-crcns-ret-1-meisterlab.ipynb index d66a0644e..0107d42aa 100644 --- a/docs/notebooks/convert-crcns-ret-1-meisterlab.ipynb +++ b/docs/notebooks/convert-crcns-ret-1-meisterlab.ipynb @@ -109,7 +109,7 @@ "This example is based on https://github.com/NeurodataWithoutBorders/api-python/blob/master/examples/create_scripts/crcns_ret-1.py from H5Gate. \n", "\n", "Compared to the NWB files generated by the original example we here use the extension mechanism to add custom data fields rather than adding unspecified custom data directly to the file, i.e., all objects (datasets, attributes, groups etc.) are governed by a formal specification.\n", - "* Previously pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. Here we create an extensions MeisterImageSeries which extens ImageSeries and stores that values as attributes pixel_size, x, y, dx, dy. We here chosse attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", + "* Previously pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. Here we create an extensions MeisterImageSeries which extens ImageSeries and stores that values as attributes pixel_size, x, y, dx, dy. We here chose attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", "\n", "Compared to the NWB files generated by the original example the files generated here contain the following additional main changes:\n", "\n", @@ -123,7 +123,7 @@ "* NWBContainer is now a base type of all core neurodata_types and as such `help` and `source` attributes have been added to all core types\n", "* The original script reused iterator variables in nested loops. We have updated those occurrence to avoid consusion and avoid possible errors. \n", "* The following custom metadata fields---i.e., datasets that were originally added to the file without being part of the NWB specification and without creation of corresponding extensions---have not yet been integrated with the NWB files:\n", - " * /general custom metdata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", + " * /general custom metadata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", " * SpikeUnit custom datasets with additional copies of the per-stimulus spike times (i.e., /processing/Cells/UnitTimes/cell_*/stim_* in the original version). This will require an extension for SpikeUnit.\n", " * /subject, subject/genotype, subject/species : See Issue https://github.com/NeurodataWithoutBorders/pynwb/issues/45 support for subject metadata is upcoming in PyNWB \n", " * /specifications, /specifications/nwb_core.py : See Issue https://github.com/NeurodataWithoutBorders/pynwb/issues/44 will be added by PyNWB automatically\n", @@ -591,7 +591,7 @@ "# Build the namespace\n", "ns_builder = NWBNamespaceBuilder('Extension for use in my Lab', ns_name)\n", "\n", - "# Create a custom ImageSeries to add our custom attributes and add our extenions to the namespace\n", + "# Create a custom ImageSeries to add our custom attributes and add our extensions to the namespace\n", "mis_ext = NWBGroupSpec('A custom ImageSeries to add MeisterLab custom metadata',\n", " attributes=[NWBAttributeSpec('x' , 'int', 'meister x', required=False),\n", " NWBAttributeSpec('y' , 'int', 'meister y', required=False),\n", @@ -697,7 +697,7 @@ "metadata": {}, "source": [ "We can now inspect our container class using the usual mechanisms, e.g., help. For illustration purposes, let's call help on our class. Here we can see that:\n", - "* Our custom attributes have been added to the constructor with approbriate documention describing the type and purpose we indicated in the spec for our attributes\n", + "* Our custom attributes have been added to the constructor with approbriate documentation describing the type and purpose we indicated in the spec for our attributes\n", "* From the \"Method resolution order\" documentationw we can see that our MeisterImageSeries inherits from pynwb.image.ImageSeries so that interaction mechanism from the base class are also available in our class" ] }, @@ -746,7 +746,7 @@ " | bits_per_pixel (int): Number of bit per image pixel\n", " | dimension (Iterable): Number of pixels on x, y, (and z) axes.\n", " | resolution (float): The smallest meaningful difference (in specified unit) between values in data\n", - " | conversion (float): Scalar to multiply each element by to conver to volts\n", + " | conversion (float): Scalar to multiply each element by to convert to volts\n", " | timestamps (ndarray or list or tuple or Dataset or DataChunkIterator or DataIO or TimeSeries): Timestamps for samples stored in data\n", " | starting_time (float): The timestamp of the first sample\n", " | rate (float): Sampling rate in Hz\n", @@ -885,7 +885,7 @@ "def convert_single_file(file_stimulus_data, file_meta, spike_units, electrode_meta):\n", " import h5py\n", " #########################################\n", - " # Create the NWBFile containter\n", + " # Create the NWBFile container\n", " ##########################################\n", " nwbfile = NWBFile(session_description=file_meta['description'],\n", " identifier=file_meta['identifier'],\n", @@ -1004,7 +1004,7 @@ "source": [ "## Step 5.2: Convert all files\n", "\n", - "Convert all the files by iteating over the files and calling `convert_single_file` function for each of the file" + "Convert all the files by iterating over the files and calling `convert_single_file` function for each of the file" ] }, { diff --git a/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions-and-external-stimulus.ipynb b/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions-and-external-stimulus.ipynb index 73090873a..4d081f7ab 100644 --- a/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions-and-external-stimulus.ipynb +++ b/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions-and-external-stimulus.ipynb @@ -100,7 +100,7 @@ "This example is based on https://github.com/NeurodataWithoutBorders/api-python/blob/master/examples/create_scripts/crcns_ret-1.py from H5Gate. \n", "\n", "Compared to the NWB files generated by the original example we here use the extension mechanism to add custom data fields rather than adding unspecified custom data directly to the file, i.e., all objects (datasets, attributes, groups etc.) are governed by a formal specification.\n", - "* Previously pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. Here we create an extensions MeisterImageSeries which extens ImageSeries and stores that values as attributes pixel_size, x, y, dx, dy. We here chosse attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", + "* Previously pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. Here we create an extensions MeisterImageSeries which extens ImageSeries and stores that values as attributes pixel_size, x, y, dx, dy. We here chose attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", "\n", "Compared to the NWB files generated by the original example the files generated here contain the following additional main changes:\n", "\n", @@ -113,12 +113,12 @@ "* NWBContainer is now a base type of all core neurodata_types and as such `help` and `source` attributes have been added to all core types\n", "* The original script reused iterator variables in nested loops. We have updated those occurrence to avoid consusion and avoid possible errors. \n", "* The following custom metadata fields---i.e., datasets that were originally added to the file without being part of the NWB specification and without creation of corresponding extensions---have not yet been integrated with the NWB files:\n", - " * /general custom metdata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", + " * /general custom metadata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", " * SpikeUnit custom datasets with additional copies of the per-stimulus spike times (i.e., /processing/Cells/UnitTimes/cell_*/stim_* in the original version). This will require an extension for SpikeUnit.\n", " * /subject, subject/genotype, subject/species : See Issue https://bitbucket.org/lblneuro/pynwb/issues/45 support for subject metadata is upcoming in PyNWB \n", " * /specifications, /specifications/nwb_core.py : See Issue https://bitbucket.org/lblneuro/pynwb/issues/44 will be added by PyNWB automatically\n", "\n", - "For readability and to ease comparison, we include in Sectoin 6 the original example scrip from H5Gate. Note, the files generated by the original script are omitting a few required datasets/attributes and as such do not actually validate. " + "For readability and to ease comparison, we include in Sectoin 6 the original example script from H5Gate. Note, the files generated by the original script are omitting a few required datasets/attributes and as such do not actually validate. " ] }, { @@ -570,7 +570,7 @@ "# Build the namespace\n", "ns_builder = NWBNamespaceBuilder('Extension for use in my Lab', ns_name)\n", "\n", - "# Create a custom ImageSeries to add our custom attributes and add our extenions to the namespace\n", + "# Create a custom ImageSeries to add our custom attributes and add our extensions to the namespace\n", "mis_ext = NWBGroupSpec('A custom ImageSeries to add MeisterLab custom metadata',\n", " attributes=[NWBAttributeSpec('x' , 'int', 'meister x', required=False),\n", " NWBAttributeSpec('y' , 'int', 'meister y', required=False),\n", @@ -605,7 +605,7 @@ "metadata": {}, "source": [ "We can now inspect our container class using the usual mechanisms, e.g., help. For illustration purposes, let's call help on our class. Here we can see that:\n", - "* Our custom attributes have been added to the constructor with approbriate documention describing the type and purpose we indicated in the spec for our attributes\n", + "* Our custom attributes have been added to the constructor with approbriate documentation describing the type and purpose we indicated in the spec for our attributes\n", "* From the \"Method resolution order\" documentationw we can see that our MeisterImageSeries inherits from pynwb.image.ImageSeries so that interaction mechanism from the base class are also available in our class" ] }, @@ -652,7 +652,7 @@ " | bits_per_pixel (int): Number of bit per image pixel\n", " | dimension (Iterable): Number of pixels on x, y, (and z) axes.\n", " | resolution (float): The smallest meaningful difference (in specified unit) between values in data\n", - " | conversion (float): Scalar to multiply each element by to conver to volts\n", + " | conversion (float): Scalar to multiply each element by to convert to volts\n", " | timestamps (list or ndarray or TimeSeries): Timestamps for samples stored in data\n", " | starting_time (float): The timestamp of the first sample\n", " | rate (float): Sampling rate in Hz\n", @@ -887,7 +887,7 @@ "def convert_single_file(file_stimulus_data, file_meta, spike_units, electrode_meta):\n", " import h5py\n", " #########################################\n", - " # Create the NWBFile containter\n", + " # Create the NWBFile container\n", " ##########################################\n", " nwbfile = NWBFile(file_name=file_meta['output_filename'],\n", " session_description=file_meta['description'],\n", @@ -1001,7 +1001,7 @@ "source": [ "## Step 5.3: Convert all files\n", "\n", - "Convert all the files by iteating over the files and calling `convert_single_file` function for each of the file" + "Convert all the files by iterating over the files and calling `convert_single_file` function for each of the file" ] }, { diff --git a/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions.ipynb b/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions.ipynb index de032a1df..d612c11c9 100644 --- a/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions.ipynb +++ b/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-with-custom-extensions.ipynb @@ -100,7 +100,7 @@ "This example is based on https://github.com/NeurodataWithoutBorders/api-python/blob/master/examples/create_scripts/crcns_ret-1.py from H5Gate. \n", "\n", "Compared to the NWB files generated by the original example we here use the extension mechanism to add custom data fields rather than adding unspecified custom data directly to the file, i.e., all objects (datasets, attributes, groups etc.) are governed by a formal specification.\n", - "* Previously pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. Here we create an extensions MeisterImageSeries which extens ImageSeries and stores that values as attributes pixel_size, x, y, dx, dy. We here chosse attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", + "* Previously pixle_size, meister_x, meister_y, meister_dx, meister_dy were stored as custom datasets in ImageSeries. Here we create an extensions MeisterImageSeries which extens ImageSeries and stores that values as attributes pixel_size, x, y, dx, dy. We here chose attributes instead of datasets simply because these are small, single int and float metadata values for which attributes are more approbirate.\n", "\n", "Compared to the NWB files generated by the original example the files generated here contain the following additional main changes:\n", "\n", @@ -110,16 +110,16 @@ "* Added missing tags and description for epochs\n", "* Added /general/devices/... to describe the device\n", "* Added neurodata_type and namespace attributes for format compliance\n", - "* Instead of storing stimulus data in external HDF5 files we here store all data in the same NWB file. Added stimulus data to ImageSeries diretly and added corresponding conversion, resolution, unit, etc. attributes to ensure format compliance\n", - "* /general/extracellular_ephys has been resturctured so that alldata about the probe is now in /general/extracellular_ephys/61-channel_probe/device\n", + "* Instead of storing stimulus data in external HDF5 files we here store all data in the same NWB file. Added stimulus data to ImageSeries directly and added corresponding conversion, resolution, unit, etc. attributes to ensure format compliance\n", + "* /general/extracellular_ephys has been restructured so that alldata about the probe is now in /general/extracellular_ephys/61-channel_probe/device\n", "* The original script reused iterator variables in nested loops. We have updated those occurrence to avoid consusion and avoid possible errors. \n", "* The following custom metadata fields---i.e., datasets that were originally added to the file without being part of the NWB specification and without creation of corresponding extensions---have not yet been integrated with the NWB files:\n", - " * /general custom metdata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", + " * /general custom metadata: /notes, /random_number_generation, /related_publications. This will require extension of NWBFile to extend the spec of /general. Improvements to make this easier have been proposed for discussion at the upcoming hackathon.\n", " * SpikeUnit custom datasets with additional copies of the per-stimulus spike times (i.e., /processing/Cells/UnitTimes/cell_*/stim_* in the original version). This will require an extension for SpikeUnit.\n", " * /subject, subject/genotype, subject/species : See Issue https://bitbucket.org/lblneuro/pynwb/issues/45 support for subject metadata is upcoming in PyNWB \n", " * /specifications, /specifications/nwb_core.py : See Issue https://bitbucket.org/lblneuro/pynwb/issues/44 will be added by PyNWB automatically\n", "\n", - "For readability and to ease comparison, we include in Sectoin 6 the original example scrip from H5Gate. Note, the files generated by the original script are omitting a few required datasets/attributes and as such do not actually validate. " + "For readability and to ease comparison, we include in Sectoin 6 the original example script from H5Gate. Note, the files generated by the original script are omitting a few required datasets/attributes and as such do not actually validate. " ] }, { @@ -555,7 +555,7 @@ "# Build the namespace\n", "ns_builder = NWBNamespaceBuilder('Extension for use in my Lab', ns_name)\n", "\n", - "# Create a custom ImageSeries to add our custom attributes and add our extenions to the namespace\n", + "# Create a custom ImageSeries to add our custom attributes and add our extensions to the namespace\n", "mis_ext = NWBGroupSpec('A custom ImageSeries to add MeisterLab custom metadata',\n", " attributes=[NWBAttributeSpec('x' , 'int', 'meister x', required=False),\n", " NWBAttributeSpec('y' , 'int', 'meister y', required=False),\n", @@ -590,7 +590,7 @@ "metadata": {}, "source": [ "We can now inspect our container class using the usual mechanisms, e.g., help. For illustration purposes, let's call help on our class. Here we can see that:\n", - "* Our custom attributes have been added to the constructor with approbriate documention describing the type and purpose we indicated in the spec for our attributes\n", + "* Our custom attributes have been added to the constructor with approbriate documentation describing the type and purpose we indicated in the spec for our attributes\n", "* From the \"Method resolution order\" documentationw we can see that our MeisterImageSeries inherits from pynwb.image.ImageSeries so that interaction mechanism from the base class are also available in our class" ] }, @@ -637,7 +637,7 @@ " | bits_per_pixel (int): Number of bit per image pixel\n", " | dimension (Iterable): Number of pixels on x, y, (and z) axes.\n", " | resolution (float): The smallest meaningful difference (in specified unit) between values in data\n", - " | conversion (float): Scalar to multiply each element by to conver to volts\n", + " | conversion (float): Scalar to multiply each element by to convert to volts\n", " | timestamps (list or ndarray or TimeSeries): Timestamps for samples stored in data\n", " | starting_time (float): The timestamp of the first sample\n", " | rate (float): Sampling rate in Hz\n", @@ -871,7 +871,7 @@ "source": [ "def convert_single_file(file_stimulus_data, file_meta, spike_units, electrode_meta):\n", " #########################################\n", - " # Create the NWBFile containter\n", + " # Create the NWBFile container\n", " ##########################################\n", " nwbfile = NWBFile(file_name=file_meta['output_filename'],\n", " session_description=file_meta['description'],\n", @@ -974,7 +974,7 @@ "source": [ "## Step 5.3: Convert all files\n", "\n", - "Convert all the files by iteating over the files and calling `convert_single_file` function for each of the file" + "Convert all the files by iterating over the files and calling `convert_single_file` function for each of the file" ] }, { diff --git a/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-without-custom-extensions.ipynb b/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-without-custom-extensions.ipynb index fc7147686..a03f26ff0 100644 --- a/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-without-custom-extensions.ipynb +++ b/docs/notebooks/convert-crcns-ret-1-old/convert-crcns-ret-1-meisterlab-without-custom-extensions.ipynb @@ -105,15 +105,15 @@ "* Added missing tags and description for epochs\n", "* Added /general/devices/... to describe the device\n", "* Added neurodata_type and namespace attributes for format compliance\n", - "* Instead of storing stimulus data in external HDF5 files we here store all data in the same NWB file. Added stimulus data to ImageSeries diretly and added corresponding conversion, resolution, unit, etc. attributes to ensure format compliance\n", - "* /general/extracellular_ephys has been resturctured so that alldata about the probe is now in /general/extracellular_ephys/61-channel_probe/device\n", + "* Instead of storing stimulus data in external HDF5 files we here store all data in the same NWB file. Added stimulus data to ImageSeries directly and added corresponding conversion, resolution, unit, etc. attributes to ensure format compliance\n", + "* /general/extracellular_ephys has been restructured so that alldata about the probe is now in /general/extracellular_ephys/61-channel_probe/device\n", "* The original script reused iterator variables in nested loops. We have updated those occurrence to avoid consusion and avoid possible errors. \n", "* This notebook, currently does not store custom metadata fields (i.e., datasets that were added to the file without being part of the NWB specification and without creation of corresponding extensions). The main omitted objects are:\n", " * ImageSeries: pixle_size, meister_x, meister_y, meister_dx, meister_dy (the data of those variables is available in this notebook as part of the stimulus_data dict / curr_stimulus)\n", - " * /general custom metdata: /subject, subject/genotype, subject/species, /specifications, /specifications/nwb_core.py, /notes, /random_number_generation, /related_publications, \n", + " * /general custom metadata: /subject, subject/genotype, subject/species, /specifications, /specifications/nwb_core.py, /notes, /random_number_generation, /related_publications, \n", " * SpikeUnit custom datasets with additional copies of the per-stimulus spike times: /processing/Cells/UnitTimes/cell_*/stim_*\n", "\n", - "For readability and to ease comparison, we include in Sectoin 6 the original example scrip from H5Gate. Note, the files generated by the original script are omitting a few required datasets/attributes and as such do not actually validate. " + "For readability and to ease comparison, we include in Sectoin 6 the original example script from H5Gate. Note, the files generated by the original script are omitting a few required datasets/attributes and as such do not actually validate. " ] }, { @@ -552,7 +552,7 @@ "source": [ "def convert_single_file(file_stimulus_data, file_meta, spike_units, electrode_meta):\n", " #########################################\n", - " # Create the NWBFile containter\n", + " # Create the NWBFile container\n", " ##########################################\n", " nwbfile = NWBFile(file_name=file_meta['output_filename'],\n", " session_description=file_meta['description'],\n", @@ -649,7 +649,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Convert all the files by iteating over the files and calling `convert_single_file` function for each of the file" + "Convert all the files by iterating over the files and calling `convert_single_file` function for each of the file" ] }, { diff --git a/docs/notebooks/read-Allen-Brain-Oservatory.ipynb b/docs/notebooks/read-Allen-Brain-Oservatory.ipynb index 7f5825a66..84c5288c7 100644 --- a/docs/notebooks/read-Allen-Brain-Oservatory.ipynb +++ b/docs/notebooks/read-Allen-Brain-Oservatory.ipynb @@ -185,7 +185,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Each of these three \"ophys experiments\" corresponds to a single 2-photon microscopy recording session. In each container, the particular visual stimulus that was presented is indicated by the `session_type`. Each `session_type` contains several diffferent stimulus sets; for more information, take a look at the [whitepaper](http://help.brain-map.org/display/observatory/Data+-+Visual+Coding).\n", + "Each of these three \"ophys experiments\" corresponds to a single 2-photon microscopy recording session. In each container, the particular visual stimulus that was presented is indicated by the `session_type`. Each `session_type` contains several different stimulus sets; for more information, take a look at the [whitepaper](http://help.brain-map.org/display/observatory/Data+-+Visual+Coding).\n", "\n", "Lets use pynwb to load the data from `three_session_B`. If this is the first time through this notebook this might take a minute to download these files, and will require approximately 300Mb of total disk space for the three nwb files: " ] @@ -215,7 +215,7 @@ "metadata": {}, "source": [ "# Reading legacy nwb files with pynwb:\n", - "Now that we have downloaded the nwb files using the `BrainObservatoryCache`, we can use pynwb to load the data and take a peek inside. Because this file was created from [version 1.0 of the NWB Schema](https://github.com/NeurodataWithoutBorders/specification/blob/master/version_1.0.6/NWB_file_format_specification_1.0.6.pdf>), we have to use a object called a type map to help the [NWB 2.0 schema](http://nwb-schema.readthedocs.io/en/latest/format.html) interpret the data in the \"legacy\" file. Alot has changed in the transition from NWB 1.0 to 2.0, including a more modular software architecture, a simplified (and extended) specification language, a mechanism for easy creation of custom schema extensions ([Click here for more information](http://www.nwb.org/2017/09/06/what-is-new-in-nwbn-v2-0/))." + "Now that we have downloaded the nwb files using the `BrainObservatoryCache`, we can use pynwb to load the data and take a peek inside. Because this file was created from [version 1.0 of the NWB Schema](https://github.com/NeurodataWithoutBorders/specification/blob/master/version_1.0.6/NWB_file_format_specification_1.0.6.pdf>), we have to use a object called a type map to help the [NWB 2.0 schema](http://nwb-schema.readthedocs.io/en/latest/format.html) interpret the data in the \"legacy\" file. A lot has changed in the transition from NWB 1.0 to 2.0, including a more modular software architecture, a simplified (and extended) specification language, a mechanism for easy creation of custom schema extensions ([Click here for more information](http://www.nwb.org/2017/09/06/what-is-new-in-nwbn-v2-0/))." ] }, { @@ -423,7 +423,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Lets use the allensdk to plot one of these image templates from one of the natural movies that was shown during this session. The template is has an original shape of 304x608 pixels, however this source image is stretched to fit on a 1200x1920 monitor, and warped so that scene appears flat from the perspective of the viewer, whos eye is close to the screen. For more information about the Brain Observatory Stimulus, check out the [stimulus whitepaper](http://help.brain-map.org/download/attachments/10616846/VisualCoding_VisualStimuli.pdf)" + "Lets use the allensdk to plot one of these image templates from one of the natural movies that was shown during this session. The template is has an original shape of 304x608 pixels, however this source image is stretched to fit on a 1200x1920 monitor, and warped so that scene appears flat from the perspective of the viewer, whose eye is close to the screen. For more information about the Brain Observatory Stimulus, check out the [stimulus whitepaper](http://help.brain-map.org/download/attachments/10616846/VisualCoding_VisualStimuli.pdf)" ] }, { diff --git a/docs/source/_static/demo_hdfview.png b/docs/source/_static/demo_hdfview.png deleted file mode 100644 index e87750ab0..000000000 Binary files a/docs/source/_static/demo_hdfview.png and /dev/null differ diff --git a/docs/source/_static/demo_nwbfile_stimulus_plot_1.png b/docs/source/_static/demo_nwbfile_stimulus_plot_1.png deleted file mode 100644 index 708240366..000000000 Binary files a/docs/source/_static/demo_nwbfile_stimulus_plot_1.png and /dev/null differ diff --git a/docs/source/_static/demo_nwbfile_stimulus_plot_2.png b/docs/source/_static/demo_nwbfile_stimulus_plot_2.png deleted file mode 100644 index 6bf9e2ced..000000000 Binary files a/docs/source/_static/demo_nwbfile_stimulus_plot_2.png and /dev/null differ diff --git a/docs/source/_static/demo_nwbfile_units_plot.png b/docs/source/_static/demo_nwbfile_units_plot.png deleted file mode 100644 index d51e0f413..000000000 Binary files a/docs/source/_static/demo_nwbfile_units_plot.png and /dev/null differ diff --git a/docs/source/_static/demo_nwbwidgets.png b/docs/source/_static/demo_nwbwidgets.png deleted file mode 100644 index edb844402..000000000 Binary files a/docs/source/_static/demo_nwbwidgets.png and /dev/null differ diff --git a/docs/source/conf.py b/docs/source/conf.py index f45bc01f8..143d9d2c6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -146,6 +146,8 @@ def __call__(self, filename): 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), 'dandi': ('https://dandi.readthedocs.io/en/stable/', None), 'fsspec': ("https://filesystem-spec.readthedocs.io/en/latest/", None), + 'nwbwidgets': ("https://nwb-widgets.readthedocs.io/en/latest/", None), + 'nwb-overview': ("https://nwb-overview.readthedocs.io/en/latest/", None), } extlinks = { diff --git a/docs/source/export.rst b/docs/source/export.rst index 44a7a3a4b..490cd346e 100644 --- a/docs/source/export.rst +++ b/docs/source/export.rst @@ -109,7 +109,7 @@ How do I write a newly instantiated ``NWBFile`` to two different file paths? ----------------------------------------------------------------------------------------------------------------------- PyNWB does not support writing an :py:class:`~pynwb.file.NWBFile` that was not read from a file to two different files. For example, if you instantiate :py:class:`~pynwb.file.NWBFile` A and write it to file path 1, you cannot also write it -to file path 2. However, you can first write the :py:class:`~pynwb.file.NWBFile`` to file path 1, read the +to file path 2. However, you can first write the :py:class:`~pynwb.file.NWBFile` to file path 1, read the :py:class:`~pynwb.file.NWBFile` from file path 1, and then export it to file path 2. .. code-block:: python diff --git a/docs/source/install_users.rst b/docs/source/install_users.rst index 6e33c2035..368ab7bd0 100644 --- a/docs/source/install_users.rst +++ b/docs/source/install_users.rst @@ -34,7 +34,7 @@ This will automatically install the following required dependencies: Install release from Conda-forge -------------------------------- -`Conda-forge `_ is a community led collection of recipes, build infrastructure +`Conda-forge `_ is a community led collection of recipes, build infrastructure and distributions for the `conda `_ package manager. To install or update PyNWB distribution from conda-forge using conda simply run: diff --git a/docs/source/software_process.rst b/docs/source/software_process.rst index f2ccb335d..07fd97246 100644 --- a/docs/source/software_process.rst +++ b/docs/source/software_process.rst @@ -17,7 +17,7 @@ tested on all supported operating systems and python distributions. That way, as a contributor, you know if you introduced regressions or coding style inconsistencies. -There are badges in the :pynwb:`README <#readme>` file which shows +There are badges in the :pynwb:`README ` file which shows the current condition of the dev branch. -------- @@ -25,7 +25,7 @@ Coverage -------- Code coverage is computed and reported using the coverage_ tool. There are two coverage-related -badges in the :pynwb:`README <#readme>` file. One shows the status of the :pynwb:`GitHub Action workflow ` which runs the coverage_ tool and uploads the report to +badges in the :pynwb:`README ` file. One shows the status of the :pynwb:`GitHub Action workflow ` which runs the coverage_ tool and uploads the report to codecov_, and the other badge shows the percentage coverage reported from codecov_. A detailed report can be found on codecov_, which shows line by line which lines are covered by the tests. diff --git a/docs/source/testing/mock.rst b/docs/source/testing/mock.rst index fe0f761a3..c640c2c14 100644 --- a/docs/source/testing/mock.rst +++ b/docs/source/testing/mock.rst @@ -7,9 +7,9 @@ neurodata objects. However, this can be quite laborious for some types. For inst :py:class:`~hdmf.common.table.DynamicTableRegion` of a :py:class:`~pynwb.ophys.PlaneSegmentation` table with the appropriate rows. This object in turn requires input of an :py:class:`~pynwb.ophys.ImageSegmentation` object, which in turn requires a :py:class:`~pynwb.device.Device` and an :py:class:`~pynwb.ophys.OpticalChannel` object. In -the end, creating a single neurodata object in this case requires the creation of 5 other objects. ``testing.mock`` -is a module that creates boilerplate objects with a single line of code that can be used for testing. In this case, you -could simply run +the end, creating a single neurodata object in this case requires the creation of 5 other objects. +:py:mod:`.testing.mock` is a module that creates boilerplate objects with a single line of code that can be used for +testing. In this case, you could simply run .. code-block:: python @@ -28,6 +28,44 @@ necessary neurodata types. You can customize any of these fields just as you wou roi_response_series = mock_RoiResponseSeries(data=[[1,2,3], [1,2,3]]) + +If you want to create objects and automatically add them to an :py:class:`~pynwb.file.NWBFile`, create an +:py:class:`~pynwb.file.NWBFile` and pass it into the mock function: + +.. code-block:: python + + from pynwb.testing.mock.file import mock_NWBFile + from pynwb.testing.mock.ophys import mock_RoiResponseSeries + + nwbfile = mock_NWBFile() + mock_RoiResponseSeries(nwbfile=nwbfile) + +Now this NWBFile contains an :py:class:`~pynwb.ophys.RoiResponseSeries` and all the upstream classes: + +.. code-block:: + + >>> print(nwbfile) + + root pynwb.file.NWBFile at 0x4335131760 + Fields: + devices: { + Device , + Device2 + } + file_create_date: [datetime.datetime(2023, 6, 26, 21, 56, 44, 322249, tzinfo=tzlocal())] + identifier: 3c13e816-a50f-49a9-85ec-93b9944c3e79 + imaging_planes: { + ImagingPlane , + ImagingPlane2 + } + processing: { + ophys + } + session_description: session_description + session_start_time: 1970-01-01 00:00:00-05:00 + timestamps_reference_time: 1970-01-01 00:00:00-05:00 + + Name generator -------------- Two neurodata objects stored in the same location within an NWB file must have unique names. This can cause an error diff --git a/docs/source/validation.rst b/docs/source/validation.rst index 8cc32a3f7..73c138127 100644 --- a/docs/source/validation.rst +++ b/docs/source/validation.rst @@ -11,7 +11,7 @@ The validator can be invoked like so: python -m pynwb.validate test.nwb If the file contains no NWB extensions, then this command will validate the file ``test.nwb`` against the -*core* NWB specification. On success, the output will is: +*core* NWB specification. On success, the output will be: .. code-block:: text diff --git a/environment-ros3.yml b/environment-ros3.yml index 83e9d37d2..c84b4c090 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -12,4 +12,10 @@ dependencies: - pandas==2.0.0 - python-dateutil==2.8.2 - setuptools - - dandi==0.52.0 # NOTE: dandi does not support osx-arm64 + - dandi==0.59.0 # NOTE: dandi does not support osx-arm64 + - fsspec==2023.6.0 + - requests==2.28.1 + - aiohttp==3.8.3 + - pip + - pip: + - remfile==0.1.9 diff --git a/requirements-doc.txt b/requirements-doc.txt index e8594dd32..2050f4439 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -11,4 +11,4 @@ sphinx-copybutton dataframe_image # used to render large dataframe as image in the sphinx gallery to improve html display lxml # used by dataframe_image when using the matplotlib backend hdf5plugin - +dandi>=0.46.6 diff --git a/requirements-min.txt b/requirements-min.txt index 662133bd5..8f52348f1 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # minimum versions of package dependencies for installing PyNWB h5py==2.10 # support for selection of datasets with list of indices added in 2.10 -hdmf==3.8.0 +hdmf==3.9.0 numpy==1.18 pandas==1.1.5 python-dateutil==2.7.3 diff --git a/requirements.txt b/requirements.txt index 4c0cf18fd..2ad7b813e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # pinned dependencies to reproduce an entire development environment to use PyNWB h5py==3.8.0 -hdmf==3.8.0 +hdmf==3.9.0 numpy==1.24.2 pandas==2.0.0 python-dateutil==2.8.2 diff --git a/setup.cfg b/setup.cfg index 3c492df25..d44fcc2b1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ VCS = git versionfile_source = src/pynwb/_version.py versionfile_build = pynwb/_version.py tag_prefix = '' +style = pep440-pre [flake8] max-line-length = 120 @@ -28,6 +29,7 @@ per-file-ignores = tests/integration/__init__.py:F401 src/pynwb/testing/__init__.py:F401 src/pynwb/validate.py:T201 + tests/read_dandi/read_first_nwb_asset.py:T201 setup.py:T201 test.py:T201 scripts/*:T201 diff --git a/setup.py b/setup.py index 8a5010d6d..90aebf55f 100755 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ reqs = [ 'h5py>=2.10', - 'hdmf>=3.8.0', + 'hdmf>=3.9.0', 'numpy>=1.16', 'pandas>=1.1.5', 'python-dateutil>=2.7.3', diff --git a/src/pynwb/__init__.py b/src/pynwb/__init__.py index 181079970..6e3b3104f 100644 --- a/src/pynwb/__init__.py +++ b/src/pynwb/__init__.py @@ -4,7 +4,6 @@ import os.path from pathlib import Path from copy import deepcopy -from warnings import warn import h5py from hdmf.spec import NamespaceCatalog @@ -148,6 +147,33 @@ def _dec(cls): _dec(container_cls) +def get_nwbfile_version(h5py_file: h5py.File): + """ + Get the NWB version of the file if it is an NWB file. + :returns: Tuple consisting of: 1) the original version string as stored in the file and + 2) a tuple with the parsed components of the version string, consisting of integers + and strings, e.g., (2, 5, 1, beta). (None, None) will be returned if the file is not a valid NWB file + or the nwb_version is missing, e.g., in the case when no data has been written to the file yet. + """ + # Get the version string for the NWB file + try: + nwb_version_string = h5py_file.attrs['nwb_version'] + # KeyError occurs when the file is empty (e.g., when creating a new file nothing has been written) + # or when the HDF5 file is not a valid NWB file + except KeyError: + return None, None + # Other system may have written nwb_version as a fixed-length string, resulting in a numpy.bytes_ object + # on read, rather than a variable-length string. To address this, decode the bytes if necessary. + if not isinstance(nwb_version_string, str): + nwb_version_string = nwb_version_string.decode() + + # Parse the version string + nwb_version_parts = nwb_version_string.replace("-", ".").replace("_", ".").split(".") + nwb_version = tuple([int(i) if i.isnumeric() else i + for i in nwb_version_parts]) + return nwb_version_string, nwb_version + + # a function to register an object mapper for a container class @docval({"name": "container_cls", "type": type, "doc": "the Container class for which the given ObjectMapper class gets used"}, @@ -201,13 +227,25 @@ def get_sum(self, a, b): class NWBHDF5IO(_HDF5IO): + @staticmethod + def can_read(path: str): + """Determine whether a given path is readable by this class""" + if not os.path.isfile(path): # path is file that exists + return False + try: + with h5py.File(path, "r") as file: # path is HDF5 file + return get_nwbfile_version(file)[1][0] >= 2 # Major version of NWB >= 2 + except IOError: + return False + @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, {'name': 'mode', 'type': str, 'doc': 'the mode to open the HDF5 file with, one of ("w", "r", "r+", "a", "w-", "x")', 'default': 'r'}, {'name': 'load_namespaces', 'type': bool, - 'doc': 'whether or not to load cached namespaces from given path - not applicable in write mode', - 'default': False}, + 'doc': ('whether or not to load cached namespaces from given path - not applicable in write mode ' + 'or when `manager` is not None or when `extensions` is not None'), + 'default': True}, {'name': 'manager', 'type': BuildManager, 'doc': 'the BuildManager to use for I/O', 'default': None}, {'name': 'extensions', 'type': (str, TypeMap, list), 'doc': 'a path to a namespace, a TypeMap, or a list consisting paths to namespaces and TypeMaps', @@ -216,22 +254,18 @@ class NWBHDF5IO(_HDF5IO): {'name': 'comm', 'type': "Intracomm", 'doc': 'the MPI communicator to use for parallel I/O', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, - {'name': 'external_resources_path', 'type': str, 'doc': 'The path to the ExternalResources', + {'name': 'herd_path', 'type': str, 'doc': 'The path to the HERD', 'default': None},) def __init__(self, **kwargs): - path, mode, manager, extensions, load_namespaces, file_obj, comm, driver, external_resources_path =\ + path, mode, manager, extensions, load_namespaces, file_obj, comm, driver, herd_path =\ popargs('path', 'mode', 'manager', 'extensions', 'load_namespaces', - 'file', 'comm', 'driver', 'external_resources_path', kwargs) + 'file', 'comm', 'driver', 'herd_path', kwargs) # Define the BuildManager to use - if load_namespaces: - if manager is not None: - warn("loading namespaces from file - ignoring 'manager'") - if extensions is not None: - warn("loading namespaces from file - ignoring 'extensions' argument") - # namespaces are not loaded when creating an NWBHDF5IO object in write mode - if 'w' in mode or mode == 'x': - raise ValueError("cannot load namespaces from file when writing to it") + io_modes_that_create_file = ['w', 'w-', 'x'] + if mode in io_modes_that_create_file or manager is not None or extensions is not None: + load_namespaces = False + if load_namespaces: tm = get_type_map() super().load_namespaces(tm, path, file=file_obj, driver=driver) manager = BuildManager(tm) @@ -251,7 +285,7 @@ def __init__(self, **kwargs): manager = get_manager() # Open the file super().__init__(path, manager=manager, mode=mode, file=file_obj, comm=comm, - driver=driver, external_resources_path=external_resources_path) + driver=driver, herd_path=herd_path) @property def nwb_version(self): @@ -263,23 +297,7 @@ def nwb_version(self): and strings, e.g., (2, 5, 1, beta). (None, None) will be returned if the nwb_version is missing, e.g., in the case when no data has been written to the file yet. """ - # Get the version string for the NWB file - try: - nwb_version_string = self._file.attrs['nwb_version'] - # KeyError occurs when the file is empty (e.g., when creating a new file nothing has been written) - # or when the HDF5 file is not a valid NWB file - except KeyError: - return None, None - # Other system may have written nwb_version as a fixed-length string, resulting in a numpy.bytes_ object - # on read, rather than a variable-length string. To address this, decode the bytes if necessary. - if not isinstance(nwb_version_string, str): - nwb_version_string = nwb_version_string.decode() - - # Parse the version string - nwb_version_parts = nwb_version_string.replace("-", ".").replace("_", ".").split(".") - nwb_version = tuple([int(i) if i.isnumeric() else i - for i in nwb_version_parts]) - return nwb_version_string, nwb_version + return get_nwbfile_version(self._file) @docval(*get_docval(_HDF5IO.read), {'name': 'skip_version_check', 'type': bool, 'doc': 'skip checking of NWB version', 'default': False}) diff --git a/src/pynwb/_version.py b/src/pynwb/_version.py index 57dfeb9fc..bf16355e1 100644 --- a/src/pynwb/_version.py +++ b/src/pynwb/_version.py @@ -44,7 +44,7 @@ def get_config(): cfg = VersioneerConfig() cfg.VCS = "git" cfg.style = "pep440-pre" - cfg.tag_prefix = "*.*.*" + cfg.tag_prefix = "" cfg.parentdir_prefix = "None" cfg.versionfile_source = "src/pynwb/_version.py" cfg.verbose = False diff --git a/src/pynwb/base.py b/src/pynwb/base.py index 41f163c70..42f7b7ff3 100644 --- a/src/pynwb/base.py +++ b/src/pynwb/base.py @@ -174,15 +174,25 @@ def __init__(self, **kwargs): timestamps = args_to_process['timestamps'] if timestamps is not None: if self.rate is not None: - raise ValueError('Specifying rate and timestamps is not supported.') + self._error_on_new_warn_on_construct( + error_msg='Specifying rate and timestamps is not supported.' + ) if self.starting_time is not None: - raise ValueError('Specifying starting_time and timestamps is not supported.') + self._error_on_new_warn_on_construct( + error_msg='Specifying starting_time and timestamps is not supported.' + ) self.fields['timestamps'] = timestamps self.timestamps_unit = self.__time_unit self.interval = 1 if isinstance(timestamps, TimeSeries): timestamps.__add_link('timestamp_link', self) elif self.rate is not None: + if self.rate < 0: + self._error_on_new_warn_on_construct( + error_msg='Rate must not be a negative value.' + ) + elif self.rate == 0.0 and get_data_shape(data)[0] > 1: + warn('Timeseries has a rate of 0.0 Hz, but the length of the data is greater than 1.') if self.starting_time is None: # override default if rate is provided but not starting time self.starting_time = 0.0 self.starting_time_unit = self.__time_unit @@ -281,6 +291,19 @@ def __add_link(self, links_key, link): def time_unit(self): return self.__time_unit + def get_timestamps(self): + if self.fields.get('timestamps'): + return self.timestamps + else: + return np.arange(len(self.data)) / self.rate + self.starting_time + + def get_data_in_units(self): + if "channel_conversion" in self.fields: + scale_factor = self.conversion * self.channel_conversion[:, np.newaxis] + else: + scale_factor = self.conversion + return np.asarray(self.data) * scale_factor + self.offset + @register_class('Image', CORE_NAMESPACE) class Image(NWBData): diff --git a/src/pynwb/file.py b/src/pynwb/file.py index aa816c940..e5e56ad26 100644 --- a/src/pynwb/file.py +++ b/src/pynwb/file.py @@ -8,7 +8,7 @@ import pandas as pd from hdmf.common import DynamicTableRegion, DynamicTable -from hdmf.container import ExternalResourcesManager +from hdmf.container import HERDManager from hdmf.utils import docval, getargs, get_docval, popargs, popargs_to_dict, AllowPositional from . import register_class, CORE_NAMESPACE @@ -150,7 +150,7 @@ def __init__(self, **kwargs): @register_class('NWBFile', CORE_NAMESPACE) -class NWBFile(MultiContainerInterface, ExternalResourcesManager): +class NWBFile(MultiContainerInterface, HERDManager): """ A representation of an NWB file. """ diff --git a/src/pynwb/misc.py b/src/pynwb/misc.py index 098fce1de..4d977b4f2 100644 --- a/src/pynwb/misc.py +++ b/src/pynwb/misc.py @@ -75,7 +75,7 @@ def __init__(self, **kwargs): {'name': 'features', 'type': (list, np.ndarray), 'doc': 'the feature values for this time point'}) def add_features(self, **kwargs): time, features = getargs('time', 'features', kwargs) - if type(self.timestamps) == list and type(self.data) is list: + if isinstance(self.timestamps, list) and isinstance(self.data, list): self.timestamps.append(time) self.data.append(features) else: diff --git a/src/pynwb/resources.py b/src/pynwb/resources.py index bfb8a6ba2..acdc22b12 100644 --- a/src/pynwb/resources.py +++ b/src/pynwb/resources.py @@ -1,10 +1,14 @@ -from hdmf.common import ExternalResources as hdmf_ExternalResources +from hdmf.common import HERD as hdmf_HERD from . import get_type_map as tm from hdmf.utils import docval, get_docval -class ExternalResources(hdmf_ExternalResources): - @docval(*get_docval(hdmf_ExternalResources.__init__)) +class HERD(hdmf_HERD): + """ + HDMF External Resources Data Structure. + A table for mapping user terms (i.e. keys) to resource entities. + """ + @docval(*get_docval(hdmf_HERD.__init__)) def __init__(self, **kwargs): kwargs['type_map'] = tm() super().__init__(**kwargs) diff --git a/src/pynwb/testing/mock/base.py b/src/pynwb/testing/mock/base.py index 37271dd8f..45b95fc08 100644 --- a/src/pynwb/testing/mock/base.py +++ b/src/pynwb/testing/mock/base.py @@ -1,25 +1,30 @@ +from typing import Optional + import numpy as np +from ... import NWBFile from ...base import TimeSeries from .utils import name_generator def mock_TimeSeries( - name=None, + name: Optional[str] = None, data=None, - unit="volts", - resolution=-1.0, - conversion=1.0, + unit: str = "volts", + resolution: float = -1.0, + conversion: float = 1.0, timestamps=None, - starting_time=None, - rate=10.0, - comments="no comments", - description="no description", + starting_time: Optional[float] = None, + rate: Optional[float] = 10.0, + comments: str = "no comments", + description: str = "no description", control=None, control_description=None, continuity=None, -): - return TimeSeries( + nwbfile: Optional[NWBFile] = None, + offset=0., +) -> TimeSeries: + time_series = TimeSeries( name=name or name_generator("TimeSeries"), data=data if data is not None else np.array([1, 2, 3, 4]), unit=unit, @@ -33,4 +38,10 @@ def mock_TimeSeries( control=control, control_description=control_description, continuity=continuity, + offset=offset, ) + + if nwbfile is not None: + nwbfile.add_acquisition(time_series) + + return time_series diff --git a/src/pynwb/testing/mock/behavior.py b/src/pynwb/testing/mock/behavior.py index b0aba5eb1..b76d21dc8 100644 --- a/src/pynwb/testing/mock/behavior.py +++ b/src/pynwb/testing/mock/behavior.py @@ -1,5 +1,8 @@ +from typing import Optional + import numpy as np +from ... import NWBFile, TimeSeries from ...behavior import ( PupilTracking, Position, @@ -11,21 +14,22 @@ def mock_SpatialSeries( - name=None, + name: Optional[str] = None, data=None, - reference_frame="lower left is (0, 0)", - unit="meters", + reference_frame: str = "lower left is (0, 0)", + unit: str = "meters", conversion=1.0, resolution=-1.0, timestamps=None, - starting_time=None, - rate=10.0, - comments="no comments", - description="no description", + starting_time: Optional[float] = None, + rate: Optional[float] = 10.0, + comments: str = "no comments", + description: str = "no description", control=None, control_description=None, -): - return SpatialSeries( + nwbfile: Optional[NWBFile] = None, +) -> SpatialSeries: + spatial_series = SpatialSeries( name=name or name_generator("SpatialSeries"), data=data if data is not None else np.array([1, 2, 3, 4]), reference_frame=reference_frame, @@ -41,21 +45,48 @@ def mock_SpatialSeries( control_description=control_description, ) + if nwbfile is not None: + nwbfile.add_acquisition(spatial_series) + + return spatial_series + def mock_Position( - name=None, spatial_series=None, -): - return Position(name=name or name_generator("Position"), spatial_series=spatial_series or [mock_SpatialSeries()]) + name: Optional[str] = None, spatial_series: Optional[SpatialSeries] = None, nwbfile: Optional[NWBFile] = None, +) -> Position: + + position = Position( + name=name or name_generator("Position"), spatial_series=spatial_series or [mock_SpatialSeries(nwbfile=nwbfile)] + ) + + if nwbfile is not None: + nwbfile.add_acquisition(position) + return position def mock_PupilTracking( - name=None, time_series=None, -): - return PupilTracking(name=name or name_generator("PupilTracking"), time_series=time_series or [mock_TimeSeries()]) + name: Optional[str] = None, time_series: Optional[TimeSeries] = None, nwbfile: Optional[NWBFile] = None +) -> PupilTracking: + pupil_tracking = PupilTracking( + name=name or name_generator("PupilTracking"), time_series=time_series or [mock_TimeSeries(nwbfile=nwbfile)] + ) + + if nwbfile is not None: + nwbfile.add_acquisition(pupil_tracking) + + return pupil_tracking -def mock_CompassDirection(name=None, spatial_series=None): - return CompassDirection( +def mock_CompassDirection( + name: Optional[str] = None, spatial_series: Optional[SpatialSeries] = None, nwbfile: Optional[NWBFile] = None +) -> CompassDirection: + + compass_direction = CompassDirection( name=name or name_generator("CompassDirection"), - spatial_series=spatial_series or [mock_SpatialSeries(unit="radians")], + spatial_series=spatial_series or [mock_SpatialSeries(unit="radians", nwbfile=nwbfile)], ) + + if nwbfile is not None: + nwbfile.add_acquisition(compass_direction) + + return compass_direction diff --git a/src/pynwb/testing/mock/device.py b/src/pynwb/testing/mock/device.py index b1ad960cc..06ac628e8 100644 --- a/src/pynwb/testing/mock/device.py +++ b/src/pynwb/testing/mock/device.py @@ -1,15 +1,24 @@ +from typing import Optional + +from ... import NWBFile from ...device import Device from .utils import name_generator def mock_Device( - name=None, - description="description", - manufacturer=None, -): - return Device( + name: Optional[str] = None, + description: str = "description", + manufacturer: Optional[str] = None, + nwbfile: Optional[NWBFile] = None, +) -> Device: + device = Device( name=name or name_generator("Device"), description=description, manufacturer=manufacturer, ) + + if nwbfile is not None: + nwbfile.add_device(device) + + return device diff --git a/src/pynwb/testing/mock/ecephys.py b/src/pynwb/testing/mock/ecephys.py index 3e9e2d960..54edf7680 100644 --- a/src/pynwb/testing/mock/ecephys.py +++ b/src/pynwb/testing/mock/ecephys.py @@ -1,42 +1,62 @@ +from typing import Optional + import numpy as np -from hdmf.common.table import DynamicTableRegion +from hdmf.common.table import DynamicTableRegion, DynamicTable -from ...file import ElectrodeTable +from ...device import Device +from ...file import ElectrodeTable, NWBFile from ...ecephys import ElectricalSeries, ElectrodeGroup, SpikeEventSeries from .device import mock_Device from .utils import name_generator def mock_ElectrodeGroup( - name=None, - description="description", - location="location", - device=None, - position=None, -): - return ElectrodeGroup( + name: Optional[str] = None, + description: str = "description", + location: str = "location", + device: Optional[Device] = None, + position: Optional[str] = None, + nwbfile: Optional[NWBFile] = None, +) -> ElectrodeGroup: + + electrode_group = ElectrodeGroup( name=name or name_generator("ElectrodeGroup"), description=description, location=location, - device=device or mock_Device(), + device=device or mock_Device(nwbfile=nwbfile), position=position, ) + if nwbfile is not None: + nwbfile.add_electrode_group(electrode_group) + + return electrode_group -def mock_ElectrodeTable(n_rows=5, group=None): - table = ElectrodeTable() - group = group if group is not None else mock_ElectrodeGroup() + +def mock_ElectrodeTable( + n_rows: int = 5, group: Optional[ElectrodeGroup] = None, nwbfile: Optional[NWBFile] = None +) -> DynamicTable: + electrodes_table = ElectrodeTable() + group = group if group is not None else mock_ElectrodeGroup(nwbfile=nwbfile) for i in range(n_rows): - table.add_row( + electrodes_table.add_row( location="CA1", group=group, group_name=group.name, ) - return table + + if nwbfile is not None: + nwbfile.electrodes = electrodes_table + + return electrodes_table -def mock_electrodes(n_electrodes=5, table=mock_ElectrodeTable(n_rows=5)): +def mock_electrodes( + n_electrodes: int = 5, table: Optional[DynamicTable] = None, nwbfile: Optional[NWBFile] = None +) -> DynamicTableRegion: + + table = table or mock_ElectrodeTable(n_rows=5, nwbfile=nwbfile) return DynamicTableRegion( name="electrodes", data=list(range(n_electrodes)), @@ -46,36 +66,56 @@ def mock_electrodes(n_electrodes=5, table=mock_ElectrodeTable(n_rows=5)): def mock_ElectricalSeries( - name=None, - description="description", + name: Optional[str] = None, + description: str = "description", data=None, - rate=30000.0, + rate: float = 30000.0, timestamps=None, - electrodes=None, - filtering="filtering", -): - return ElectricalSeries( + starting_time: Optional[float] = None, + electrodes: Optional[DynamicTableRegion] = None, + filtering: str = "filtering", + nwbfile: Optional[NWBFile] = None, + channel_conversion: Optional[np.ndarray] = None, + conversion: float = 1.0, + offset: float = 0., +) -> ElectricalSeries: + electrical_series = ElectricalSeries( name=name or name_generator("ElectricalSeries"), description=description, data=data if data is not None else np.ones((10, 5)), rate=rate, + starting_time=starting_time, timestamps=timestamps, - electrodes=electrodes or mock_electrodes(), + electrodes=electrodes or mock_electrodes(nwbfile=nwbfile), filtering=filtering, + conversion=conversion, + offset=offset, + channel_conversion=channel_conversion, ) + if nwbfile is not None: + nwbfile.add_acquisition(electrical_series) + + return electrical_series + def mock_SpikeEventSeries( - name=None, - description="description", + name: Optional[str] = None, + description: str = "description", data=None, timestamps=np.arange(10).astype(float), - electrodes=None, -): - return SpikeEventSeries( + electrodes: Optional[DynamicTableRegion] = None, + nwbfile: Optional[NWBFile] = None, +) -> SpikeEventSeries: + spike_event_series = SpikeEventSeries( name=name or name_generator("SpikeEventSeries"), description=description, data=data if data is not None else np.ones((10, 5)), timestamps=timestamps if timestamps is not None else np.arange(10).astype(float), - electrodes=electrodes if electrodes is not None else mock_electrodes(), + electrodes=electrodes if electrodes is not None else mock_electrodes(nwbfile=nwbfile), ) + + if nwbfile is not None: + nwbfile.add_acquisition(spike_event_series) + + return spike_event_series diff --git a/src/pynwb/testing/mock/file.py b/src/pynwb/testing/mock/file.py index 1447b97cc..943f86dcb 100644 --- a/src/pynwb/testing/mock/file.py +++ b/src/pynwb/testing/mock/file.py @@ -1,3 +1,4 @@ +from typing import Optional from uuid import uuid4 from datetime import datetime from dateutil.tz import tzlocal @@ -7,31 +8,35 @@ def mock_NWBFile( - session_description='session_description', - identifier=None, - session_start_time=datetime(1970, 1, 1, tzinfo=tzlocal()), - subject=None, + session_description: str = 'session_description', + identifier: Optional[str] = None, + session_start_time: datetime = datetime(1970, 1, 1, tzinfo=tzlocal()), **kwargs ): return NWBFile( session_description=session_description, identifier=identifier or str(uuid4()), session_start_time=session_start_time, - subject=subject or mock_Subject(), **kwargs ) def mock_Subject( - age="P50D", - description="this is a mock mouse.", - sex="F", - subject_id=None, + age: Optional[str] = "P50D", + description: str = "this is a mock mouse.", + sex: Optional[str] = "F", + subject_id: Optional[str] = None, + nwbfile: Optional[NWBFile] = None, ): - return Subject( + subject = Subject( age=age, description=description, sex=sex, subject_id=subject_id or name_generator("subject"), ) + + if nwbfile is not None: + nwbfile.subject = subject + + return subject diff --git a/src/pynwb/testing/mock/icephys.py b/src/pynwb/testing/mock/icephys.py index bd3a0f1b5..2f13323e8 100644 --- a/src/pynwb/testing/mock/icephys.py +++ b/src/pynwb/testing/mock/icephys.py @@ -1,3 +1,5 @@ +from typing import Optional + import numpy as np from pynwb.icephys import ( @@ -12,90 +14,113 @@ from .utils import name_generator from .device import mock_Device +from ... import NWBFile +from ...device import Device def mock_IntracellularElectrode( - name=None, description="description", device=None, -): - return IntracellularElectrode( + name: Optional[str] = None, + description: str = "description", + device: Optional[Device] = None, + nwbfile: Optional[NWBFile] = None, +) -> IntracellularElectrode: + intracellular_electrode = IntracellularElectrode( name=name or name_generator("IntracellularElectrode"), description=description, - device=device or mock_Device(), + device=device or mock_Device(nwbfile=nwbfile), ) + if nwbfile is not None: + nwbfile.add_icephys_electrode(intracellular_electrode) + + return intracellular_electrode + def mock_VoltageClampStimulusSeries( - name=None, + name: Optional[str] = None, data=None, - rate=100_000., - electrode=None, - gain=0.02, + rate: float = 100_000., + electrode: Optional[IntracellularElectrode] = None, + gain: float = 0.02, timestamps=None, - starting_time=None, -): - return VoltageClampStimulusSeries( + starting_time: Optional[float] = None, + nwbfile: Optional[NWBFile] = None, +) -> VoltageClampStimulusSeries: + voltage_clamp_stimulus_series = VoltageClampStimulusSeries( name=name or name_generator("VoltageClampStimulusSeries"), data=data or np.ones((30,)), rate=None if timestamps else rate, - electrode=electrode or mock_IntracellularElectrode(), + electrode=electrode or mock_IntracellularElectrode(nwbfile=nwbfile), gain=gain, timestamps=timestamps, starting_time=starting_time, ) + if nwbfile is not None: + nwbfile.add_stimulus(voltage_clamp_stimulus_series) + + return voltage_clamp_stimulus_series + def mock_VoltageClampSeries( - name=None, + name: Optional[str] = None, data=None, - conversion=1.0, - resolution=np.nan, - starting_time=None, - rate=100_000.0, - electrode=None, - gain=0.02, - capacitance_slow=100e-12, - resistance_comp_correction=70.0, -): - return VoltageClampSeries( + conversion: float = 1.0, + resolution: float = np.nan, + starting_time: Optional[float] = None, + rate: Optional[float] = 100_000.0, + electrode: Optional[IntracellularElectrode] = None, + gain: float = 0.02, + capacitance_slow: float = 100e-12, + resistance_comp_correction: float = 70.0, + nwbfile: Optional[NWBFile] = None, +) -> VoltageClampSeries: + voltage_clamp_series = VoltageClampSeries( name=name if name is not None else name_generator("VoltageClampSeries"), data=data if data is not None else np.ones((30,)), conversion=conversion, resolution=resolution, starting_time=starting_time, rate=rate, - electrode=electrode or mock_IntracellularElectrode(), + electrode=electrode or mock_IntracellularElectrode(nwbfile=nwbfile), gain=gain, capacitance_slow=capacitance_slow, resistance_comp_correction=resistance_comp_correction, ) + if nwbfile is not None: + nwbfile.add_acquisition(voltage_clamp_series) + + return voltage_clamp_series + def mock_CurrentClampSeries( - name=None, + name: Optional[str] = None, data=None, - electrode=None, - gain=0.02, - stimulus_description="N/A", + electrode: Optional[IntracellularElectrode] = None, + gain: float = 0.02, + stimulus_description: str = "N/A", bias_current=None, bridge_balance=None, capacitance_compensation=None, resolution=-1.0, conversion=1.0, timestamps=None, - starting_time=None, - rate=100_000., - comments="no comments", - description="no description", + starting_time: Optional[float] = None, + rate: Optional[float] = 100_000., + comments: str = "no comments", + description: str = "no description", control=None, control_description=None, sweep_number=None, offset=0.0, - unit="volts", -): - return CurrentClampSeries( + unit: str = "volts", + nwbfile: Optional[NWBFile] = None, +) -> CurrentClampSeries: + current_clamp_series = CurrentClampSeries( name=name if name is not None else name_generator("CurrentClampSeries"), data=data if data is not None else np.ones((30,)), - electrode=electrode or mock_IntracellularElectrode(), + electrode=electrode or mock_IntracellularElectrode(nwbfile=nwbfile), gain=gain, stimulus_description=stimulus_description, bias_current=bias_current, @@ -115,9 +140,14 @@ def mock_CurrentClampSeries( unit=unit, ) + if nwbfile is not None: + nwbfile.add_acquisition(current_clamp_series) + + return current_clamp_series + def mock_CurrentClampStimulusSeries( - name=None, + name: Optional[str] = None, data=None, electrode=None, gain=0.02, @@ -134,11 +164,12 @@ def mock_CurrentClampStimulusSeries( sweep_number=None, offset=0.0, unit="amperes", -): - return CurrentClampStimulusSeries( + nwbfile: Optional[NWBFile] = None, +) -> CurrentClampStimulusSeries: + current_clamp_stimulus_series = CurrentClampStimulusSeries( name=name or name_generator("CurrentClampStimulusSeries"), data=data if data is not None else np.ones((30,)), - electrode=electrode or mock_IntracellularElectrode(), + electrode=electrode or mock_IntracellularElectrode(nwbfile=nwbfile), gain=gain, stimulus_description=stimulus_description, resolution=resolution, @@ -155,11 +186,16 @@ def mock_CurrentClampStimulusSeries( unit=unit, ) + if nwbfile is not None: + nwbfile.add_stimulus(current_clamp_stimulus_series) + + return current_clamp_stimulus_series + def mock_IZeroClampSeries( - name=None, + name: Optional[str] = None, data=None, - electrode=None, + electrode: Optional[IntracellularElectrode] = None, gain=.02, stimulus_description="N/A", resolution=-1.0, @@ -174,11 +210,12 @@ def mock_IZeroClampSeries( sweep_number=None, offset=0.0, unit="volts", -): - return IZeroClampSeries( + nwbfile: Optional[NWBFile] = None, +) -> IZeroClampSeries: + izero_clamp_series = IZeroClampSeries( name=name or name_generator("IZeroClampSeries"), data=data if data is not None else np.ones((30,)), - electrode=electrode or mock_IntracellularElectrode(), + electrode=electrode or mock_IntracellularElectrode(nwbfile=nwbfile), gain=gain, stimulus_description=stimulus_description, resolution=resolution, @@ -195,13 +232,25 @@ def mock_IZeroClampSeries( unit=unit, ) + if nwbfile is not None: + nwbfile.add_acquisition(izero_clamp_series) -def mock_IntracellularRecordingsTable(n_rows=5): + return izero_clamp_series + + +def mock_IntracellularRecordingsTable( + n_rows: int = 5, nwbfile: Optional[NWBFile] = None +) -> IntracellularRecordingsTable: irt = IntracellularRecordingsTable() for _ in range(n_rows): - electrode = mock_IntracellularElectrode() + electrode = mock_IntracellularElectrode(nwbfile=nwbfile) irt.add_recording( electrode=electrode, - stimulus=mock_VoltageClampStimulusSeries(electrode=electrode), - response=mock_VoltageClampSeries(electrode=electrode), + stimulus=mock_VoltageClampStimulusSeries(electrode=electrode, nwbfile=nwbfile), + response=mock_VoltageClampSeries(electrode=electrode, nwbfile=nwbfile), ) + + if nwbfile is not None: + nwbfile.intracellular_recordings = irt + + return irt diff --git a/src/pynwb/testing/mock/ogen.py b/src/pynwb/testing/mock/ogen.py index ceed36a43..43155d800 100644 --- a/src/pynwb/testing/mock/ogen.py +++ b/src/pynwb/testing/mock/ogen.py @@ -1,5 +1,9 @@ +from typing import Optional + import numpy as np +from ... import NWBFile +from ...device import Device from ...ogen import OptogeneticStimulusSite, OptogeneticSeries from .device import mock_Device @@ -7,39 +11,46 @@ def mock_OptogeneticStimulusSite( - name=None, - device=None, - description="optogenetic stimulus site", - excitation_lambda=500., - location="part of the brain", -): - return OptogeneticStimulusSite( + name: Optional[str] = None, + device: Optional[Device] = None, + description: str = "optogenetic stimulus site", + excitation_lambda: float = 500., + location: str = "part of the brain", + nwbfile: Optional[NWBFile] = None, +) -> OptogeneticStimulusSite: + optogenetic_stimulus_site = OptogeneticStimulusSite( name=name or name_generator("OptogeneticStimulusSite"), - device=device or mock_Device(), + device=device or mock_Device(nwbfile=nwbfile), description=description, excitation_lambda=excitation_lambda, location=location ) + if nwbfile is not None: + nwbfile.add_ogen_site(optogenetic_stimulus_site) + + return optogenetic_stimulus_site + def mock_OptogeneticSeries( - name=None, + name: Optional[str] = None, data=None, - site=None, - resolution=-1.0, - conversion=1.0, + site: Optional[OptogeneticStimulusSite] = None, + resolution: float = -1.0, + conversion: float = 1.0, timestamps=None, - starting_time=None, - rate=10.0, - comments="no comments", - description="no description", + starting_time: Optional[float] = None, + rate: Optional[float] = 10.0, + comments: str = "no comments", + description: str = "no description", control=None, control_description=None, -): - return OptogeneticSeries( + nwbfile: Optional[NWBFile] = None, +) -> OptogeneticSeries: + optogenetic_series = OptogeneticSeries( name=name or name_generator("OptogeneticSeries"), data=data if data is not None else np.array([1, 2, 3, 4]), - site=site or mock_OptogeneticStimulusSite(), + site=site or mock_OptogeneticStimulusSite(nwbfile=nwbfile), resolution=resolution, conversion=conversion, timestamps=timestamps, @@ -50,3 +61,8 @@ def mock_OptogeneticSeries( control=control, control_description=control_description, ) + + if nwbfile is not None: + nwbfile.add_acquisition(optogenetic_series) + + return optogenetic_series diff --git a/src/pynwb/testing/mock/ophys.py b/src/pynwb/testing/mock/ophys.py index f35d19720..cd99d5957 100644 --- a/src/pynwb/testing/mock/ophys.py +++ b/src/pynwb/testing/mock/ophys.py @@ -1,6 +1,11 @@ +from typing import Optional, Sequence + import numpy as np from hdmf.common.table import DynamicTableRegion + +from ... import NWBFile, ProcessingModule +from ...device import Device from ...ophys import ( RoiResponseSeries, OpticalChannel, @@ -17,38 +22,45 @@ def mock_OpticalChannel( - name=None, - description="description", - emission_lambda=500.0, -): - return OpticalChannel( + name: Optional[str] = None, + description: str = "description", + emission_lambda: float = 500.0, + nwbfile: Optional[NWBFile] = None, +) -> OpticalChannel: + optical_channel = OpticalChannel( name=name or name_generator("OpticalChannel"), description=description, emission_lambda=emission_lambda, ) + if nwbfile is not None: + mock_ImagingPlane(nwbfile=nwbfile, optical_channel=optical_channel) + + return optical_channel + def mock_ImagingPlane( - name=None, - optical_channel=None, - description="description", - device=None, - excitation_lambda=500.0, - indicator="indicator", - location="unknown", - imaging_rate=30.0, + name: Optional[str] = None, + optical_channel: Optional[OpticalChannel] = None, + description: str = "description", + device: Optional[Device] = None, + excitation_lambda: float = 500.0, + indicator: str = "indicator", + location: str = "unknown", + imaging_rate: float = 30.0, manifold=None, - conversion=1.0, - unit="meters", + conversion: float = 1.0, + unit: str = "meters", reference_frame=None, origin_coords=None, - origin_coords_unit="meters", + origin_coords_unit: str = "meters", grid_spacing=None, - grid_spacing_unit="meters", -): - return ImagingPlane( + grid_spacing_unit: str = "meters", + nwbfile: Optional[NWBFile] = None, +) -> ImagingPlane: + imaging_plane = ImagingPlane( name=name or name_generator("ImagingPlane"), - optical_channel=optical_channel or mock_OpticalChannel(), + optical_channel=optical_channel or mock_OpticalChannel(nwbfile=nwbfile), description=description, - device=device or mock_Device(), + device=device or mock_Device(nwbfile=nwbfile), excitation_lambda=excitation_lambda, indicator=indicator, location=location, @@ -63,13 +75,20 @@ def mock_ImagingPlane( grid_spacing_unit=grid_spacing_unit, ) + if nwbfile is not None: + if "ophys" not in nwbfile.processing: + nwbfile.create_processing_module("ophys", "ophys") + nwbfile.add_imaging_plane(imaging_plane) + + return imaging_plane + def mock_OnePhotonSeries( - name=None, - imaging_plane=None, + name: Optional[str] = None, + imaging_plane: Optional[ImagingPlane] = None, data=None, - rate=50.0, - unit="n.a.", + rate: Optional[float] = 50.0, + unit: str = "n.a.", exposure_time=None, binning=None, power=None, @@ -91,10 +110,11 @@ def mock_OnePhotonSeries( control=None, control_description=None, device=None, -): - return OnePhotonSeries( + nwbfile: Optional[NWBFile] = None, +) -> OnePhotonSeries: + one_photon_series = OnePhotonSeries( name=name if name is not None else name_generator("OnePhotonSeries"), - imaging_plane=imaging_plane or mock_ImagingPlane(), + imaging_plane=imaging_plane or mock_ImagingPlane(nwbfile=nwbfile), data=data if data is not None else np.ones((20, 5, 5)), unit=unit, exposure_time=exposure_time, @@ -112,6 +132,7 @@ def mock_OnePhotonSeries( conversion=conversion, timestamps=timestamps, starting_time=starting_time, + offset=offset, rate=rate, comments=comments, description=description, @@ -120,10 +141,15 @@ def mock_OnePhotonSeries( device=device, ) + if nwbfile is not None: + nwbfile.add_acquisition(one_photon_series) + + return one_photon_series + def mock_TwoPhotonSeries( - name=None, - imaging_plane=None, + name: Optional[str] = None, + imaging_plane: Optional[ImagingPlane] = None, data=None, rate=50.0, unit="n.a.", @@ -137,6 +163,7 @@ def mock_TwoPhotonSeries( dimension=None, resolution=-1.0, conversion=1.0, + offset=0.0, timestamps=None, starting_time=None, comments="no comments", @@ -144,10 +171,11 @@ def mock_TwoPhotonSeries( control=None, control_description=None, device=None, -): - return TwoPhotonSeries( + nwbfile: Optional[NWBFile] = None, +) -> TwoPhotonSeries: + two_photon_series = TwoPhotonSeries( name=name if name is not None else name_generator("TwoPhotonSeries"), - imaging_plane=imaging_plane or mock_ImagingPlane(), + imaging_plane=imaging_plane or mock_ImagingPlane(nwbfile=nwbfile), data=data if data is not None else np.ones((20, 5, 5)), unit=unit, format=format, @@ -168,19 +196,26 @@ def mock_TwoPhotonSeries( control=control, control_description=control_description, device=device, + offset=offset, ) + if nwbfile is not None: + nwbfile.add_acquisition(two_photon_series) + + return two_photon_series + def mock_PlaneSegmentation( - description="no description", - imaging_plane=None, - name=None, + description: str = "no description", + imaging_plane: Optional[ImagingPlane] = None, + name: Optional[str] = None, reference_images=None, - n_rois=5, -): + n_rois: int = 5, + nwbfile: Optional[NWBFile] = None, +) -> PlaneSegmentation: plane_segmentation = PlaneSegmentation( description=description, - imaging_plane=imaging_plane or mock_ImagingPlane(), + imaging_plane=imaging_plane or mock_ImagingPlane(nwbfile=nwbfile), name=name if name is not None else name_generator("PlaneSegmentation"), reference_images=reference_images, ) @@ -188,22 +223,37 @@ def mock_PlaneSegmentation( for _ in range(n_rois): plane_segmentation.add_roi(image_mask=np.zeros((10, 10))) + if nwbfile is not None: + if "ophys" not in nwbfile.processing: + nwbfile.create_processing_module("ophys", "ophys") + nwbfile.processing["ophys"].add(plane_segmentation) + return plane_segmentation def mock_ImageSegmentation( - plane_segmentations=None, name=None, -): - return ImageSegmentation( - plane_segmentations=plane_segmentations or [mock_PlaneSegmentation()], + plane_segmentations: Optional[Sequence[PlaneSegmentation]] = None, + name: Optional[str] = None, + nwbfile: Optional[NWBFile] = None +) -> ImageSegmentation: + image_segmentation = ImageSegmentation( + plane_segmentations=plane_segmentations or [mock_PlaneSegmentation(nwbfile=nwbfile)], name=name or name_generator("ImageSegmentation"), ) + if nwbfile is not None: + if "ophys" not in nwbfile.processing: + nwbfile.create_processing_module("ophys", "ophys") + + nwbfile.processing["ophys"].add(image_segmentation) + + return image_segmentation + def mock_RoiResponseSeries( - name=None, + name: Optional[str] = None, data=None, - unit="n.a.", + unit: str = "n.a.", rois=None, resolution=-1.0, conversion=1.0, @@ -215,7 +265,9 @@ def mock_RoiResponseSeries( control=None, control_description=None, n_rois=None, -): + plane_segmentation: Optional[PlaneSegmentation] = None, + nwbfile: Optional[NWBFile] = None, +) -> RoiResponseSeries: if data is not None: if n_rois is not None and n_rois != data.shape[1]: raise ValueError("Argument conflict: n_rois does not match second dimension of data.") @@ -223,7 +275,9 @@ def mock_RoiResponseSeries( else: n_rois = 5 - return RoiResponseSeries( + plane_seg = plane_segmentation or mock_PlaneSegmentation(n_rois=n_rois, nwbfile=nwbfile) + + roi_response_series = RoiResponseSeries( name=name if name is not None else name_generator("RoiResponseSeries"), data=data if data is not None else np.ones((30, n_rois)), unit=unit, @@ -231,7 +285,7 @@ def mock_RoiResponseSeries( or DynamicTableRegion( name="rois", description="rois", - table=mock_PlaneSegmentation(n_rois=n_rois), + table=plane_seg, data=list(range(n_rois)), ), resolution=resolution, @@ -245,16 +299,67 @@ def mock_RoiResponseSeries( control_description=control_description, ) + if nwbfile is not None: + if "ophys" not in nwbfile.processing: + nwbfile.create_processing_module("ophys", "ophys") + + if plane_seg.name not in nwbfile.processing["ophys"].data_interfaces: + nwbfile.processing["ophys"].add(plane_seg) + + nwbfile.processing["ophys"].add(roi_response_series) -def mock_DfOverF(roi_response_series=None, name=None): - return DfOverF( - roi_response_series=roi_response_series or [mock_RoiResponseSeries()], + return roi_response_series + + +def mock_DfOverF( + roi_response_series: Optional[RoiResponseSeries] = None, + name: Optional[str] = None, + nwbfile: Optional[NWBFile] = None +) -> DfOverF: + df_over_f = DfOverF( name=name if name is not None else name_generator("DfOverF"), ) + plane_seg = mock_PlaneSegmentation(nwbfile=nwbfile) + + if nwbfile is not None: + if "ophys" not in nwbfile.processing: + nwbfile.create_processing_module("ophys", "ophys") + nwbfile.processing["ophys"].add(df_over_f) + + else: + pm = ProcessingModule(name="ophys", description="ophys") + pm.add(plane_seg) + pm.add(df_over_f) -def mock_Fluorescence(roi_response_series=None, name=None): - return Fluorescence( - roi_response_series=roi_response_series or [mock_RoiResponseSeries()], + df_over_f.add_roi_response_series( + roi_response_series or mock_RoiResponseSeries(nwbfile=nwbfile, plane_segmentation=plane_seg) + ) + return df_over_f + + +def mock_Fluorescence( + roi_response_series: Optional[Sequence[RoiResponseSeries]] = None, + name: Optional[str] = None, + nwbfile: Optional[NWBFile] = None, +) -> Fluorescence: + fluorescence = Fluorescence( name=name if name is not None else name_generator("Fluorescence"), ) + plane_seg = mock_PlaneSegmentation(nwbfile=nwbfile) + + if nwbfile is not None: + if "ophys" not in nwbfile.processing: + nwbfile.create_processing_module("ophys", "ophys") + + nwbfile.processing["ophys"].add(fluorescence) + else: + pm = ProcessingModule(name="ophys", description="ophys") + pm.add(plane_seg) + pm.add(fluorescence) + + fluorescence.add_roi_response_series( + roi_response_series or mock_RoiResponseSeries(nwbfile=nwbfile, plane_segmentation=plane_seg) + ) + + return fluorescence diff --git a/src/pynwb/validate.py b/src/pynwb/validate.py index 23b3aee6f..62aa41426 100644 --- a/src/pynwb/validate.py +++ b/src/pynwb/validate.py @@ -156,6 +156,7 @@ def validate(**kwargs): file=sys.stderr, ) else: + io_kwargs.update(load_namespaces=False) namespaces_to_validate = [CORE_NAMESPACE] if namespace is not None: diff --git a/test.py b/test.py index ba9eebe47..16191ae3f 100755 --- a/test.py +++ b/test.py @@ -12,7 +12,16 @@ import traceback import unittest -flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5, 'validation': 6, 'ros3': 7, 'example-ros3': 8} +flags = { + 'pynwb': 2, + 'integration': 3, + 'example': 4, + 'backwards': 5, + 'validate-examples': 6, + 'ros3': 7, + 'example-ros3': 8, + 'validation-module': 9 +} TOTAL = 0 FAILURES = 0 @@ -154,7 +163,7 @@ def validate_nwbs(): def get_namespaces(nwbfile): comp = run(["python", "-m", "pynwb.validate", - "--list-namespaces", "--cached-namespace", nwb], + "--list-namespaces", nwbfile], stdout=PIPE, stderr=STDOUT, universal_newlines=True, timeout=30) if comp.returncode != 0: @@ -170,14 +179,13 @@ def get_namespaces(nwbfile): cmds = [] cmds += [["python", "-m", "pynwb.validate", nwb]] - cmds += [["python", "-m", "pynwb.validate", "--cached-namespace", nwb]] cmds += [["python", "-m", "pynwb.validate", "--no-cached-namespace", nwb]] for ns in namespaces: # for some reason, this logging command is necessary to correctly printing the namespace in the # next logging command logging.info("Namespace found: %s" % ns) - cmds += [["python", "-m", "pynwb.validate", "--cached-namespace", "--ns", ns, nwb]] + cmds += [["python", "-m", "pynwb.validate", "--ns", ns, nwb]] for cmd in cmds: logging.info("Validating with \"%s\"." % (" ".join(cmd[:-1]))) @@ -233,9 +241,6 @@ def run_integration_tests(verbose=True): run_test_suite("tests/integration/utils", "integration utils tests", verbose=verbose) - # also test the validation script - run_test_suite("tests/validation", "validation tests", verbose=verbose) - def clean_up_tests(): # remove files generated from running example files @@ -298,18 +303,21 @@ def main(): help='run example tests with ros3 streaming') parser.add_argument('-b', '--backwards', action='append_const', const=flags['backwards'], dest='suites', help='run backwards compatibility tests') - parser.add_argument('-w', '--validation', action='append_const', const=flags['validation'], dest='suites', - help='run example tests and validation tests on example NWB files') + parser.add_argument('-w', '--validate-examples', action='append_const', const=flags['validate-examples'], + dest='suites', help='run example tests and validation tests on example NWB files') parser.add_argument('-r', '--ros3', action='append_const', const=flags['ros3'], dest='suites', help='run ros3 streaming tests') + parser.add_argument('-x', '--validation-module', action='append_const', const=flags['validation-module'], + dest='suites', help='run tests on pynwb.validate') args = parser.parse_args() if not args.suites: args.suites = list(flags.values()) # remove from test suites run by default args.suites.pop(args.suites.index(flags['example'])) args.suites.pop(args.suites.index(flags['example-ros3'])) - args.suites.pop(args.suites.index(flags['validation'])) + args.suites.pop(args.suites.index(flags['validate-examples'])) args.suites.pop(args.suites.index(flags['ros3'])) + args.suites.pop(args.suites.index(flags['validation-module'])) # set up logger root = logging.getLogger() @@ -332,8 +340,10 @@ def main(): run_test_suite("tests/unit", "pynwb unit tests", verbose=args.verbosity) # Run example tests - if flags['example'] in args.suites or flags['validation'] in args.suites: + is_run_example_tests = False + if flags['example'] in args.suites or flags['validate-examples'] in args.suites: run_example_tests() + is_run_example_tests = True # Run example tests with ros3 streaming examples # NOTE this requires h5py to be built with ROS3 support and the dandi package to be installed @@ -342,13 +352,17 @@ def main(): run_example_ros3_tests() # Run validation tests on the example NWB files generated above - if flags['validation'] in args.suites: + if flags['validate-examples'] in args.suites: validate_nwbs() # Run integration tests if flags['integration'] in args.suites: run_integration_tests(verbose=args.verbosity) + # Run validation module tests, requires coverage to be installed + if flags['validation-module'] in args.suites: + run_test_suite("tests/validation", "validation tests", verbose=args.verbosity) + # Run backwards compatibility tests if flags['backwards'] in args.suites: run_test_suite("tests/back_compat", "pynwb backwards compatibility tests", verbose=args.verbosity) @@ -358,7 +372,7 @@ def main(): run_test_suite("tests/integration/ros3", "pynwb ros3 streaming tests", verbose=args.verbosity) # Delete files generated from running example tests above - if flags['example'] in args.suites or flags['validation'] in args.suites: + if is_run_example_tests: clean_up_tests() final_message = 'Ran %s tests' % TOTAL diff --git a/tests/back_compat/test_import_structure.py b/tests/back_compat/test_import_structure.py index e5f931f5d..79d4f6ad0 100644 --- a/tests/back_compat/test_import_structure.py +++ b/tests/back_compat/test_import_structure.py @@ -82,7 +82,6 @@ def test_outer_import_structure(self): "spec", "testing", "validate", - "warn", ] for member in expected_structure: self.assertIn(member=member, container=current_structure) diff --git a/tests/back_compat/test_read.py b/tests/back_compat/test_read.py index 919ae6bde..792d26e7a 100644 --- a/tests/back_compat/test_read.py +++ b/tests/back_compat/test_read.py @@ -31,6 +31,16 @@ class TestReadOldVersions(TestCase): "- expected an array of shape '[None]', got non-array data 'one publication'")], } + def get_io(self, path): + """Get an NWBHDF5IO object for the given path.""" + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"Ignoring cached namespace .*", + category=UserWarning, + ) + return NWBHDF5IO(str(path), 'r') + def test_read(self): """Test reading and validating all NWB files in the same folder as this file. @@ -43,7 +53,7 @@ def test_read(self): with self.subTest(file=f.name): with warnings.catch_warnings(record=True) as warnings_on_read: warnings.simplefilter("always") - with NWBHDF5IO(str(f), 'r', load_namespaces=True) as io: + with self.get_io(f) as io: errors = validate(io) io.read() for w in warnings_on_read: @@ -69,28 +79,28 @@ def test_read(self): def test_read_timeseries_no_data(self): """Test that a TimeSeries written without data is read with data set to the default value.""" f = Path(__file__).parent / '1.5.1_timeseries_no_data.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() np.testing.assert_array_equal(read_nwbfile.acquisition['test_timeseries'].data, TimeSeries.DEFAULT_DATA) def test_read_timeseries_no_unit(self): """Test that an ImageSeries written without unit is read with unit set to the default value.""" f = Path(__file__).parent / '1.5.1_timeseries_no_unit.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertEqual(read_nwbfile.acquisition['test_timeseries'].unit, TimeSeries.DEFAULT_UNIT) def test_read_imageseries_no_data(self): """Test that an ImageSeries written without data is read with data set to the default value.""" f = Path(__file__).parent / '1.5.1_imageseries_no_data.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() np.testing.assert_array_equal(read_nwbfile.acquisition['test_imageseries'].data, ImageSeries.DEFAULT_DATA) def test_read_imageseries_no_unit(self): """Test that an ImageSeries written without unit is read with unit set to the default value.""" f = Path(__file__).parent / '1.5.1_imageseries_no_unit.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertEqual(read_nwbfile.acquisition['test_imageseries'].unit, ImageSeries.DEFAULT_UNIT) @@ -100,7 +110,7 @@ def test_read_imageseries_non_external_format(self): f = Path(__file__).parent / fbase expected_warning = self.expected_warnings[fbase][0] with self.assertWarnsWith(UserWarning, expected_warning): - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertEqual(read_nwbfile.acquisition['test_imageseries'].format, "tiff") @@ -110,13 +120,13 @@ def test_read_imageseries_nonmatch_starting_frame(self): f = Path(__file__).parent / fbase expected_warning = self.expected_warnings[fbase][0] with self.assertWarnsWith(UserWarning, expected_warning): - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() np.testing.assert_array_equal(read_nwbfile.acquisition['test_imageseries'].starting_frame, [1, 2, 3]) def test_read_subject_no_age__reference(self): """Test that reading a Subject without an age__reference set with NWB schema 2.5.0 sets the value to None""" f = Path(__file__).parent / '2.2.0_subject_no_age__reference.nwb' - with NWBHDF5IO(str(f), 'r') as io: + with self.get_io(f) as io: read_nwbfile = io.read() self.assertIsNone(read_nwbfile.subject.age__reference) diff --git a/tests/integration/hdf5/test_ecephys.py b/tests/integration/hdf5/test_ecephys.py index 9d810270c..ff67d27c9 100644 --- a/tests/integration/hdf5/test_ecephys.py +++ b/tests/integration/hdf5/test_ecephys.py @@ -1,10 +1,21 @@ from hdmf.common import DynamicTableRegion - -from pynwb.ecephys import ElectrodeGroup, ElectricalSeries, FilteredEphys, LFP, Clustering, ClusterWaveforms,\ - SpikeEventSeries, EventWaveform, EventDetection, FeatureExtraction +from pynwb import NWBFile + +from pynwb.ecephys import ( + ElectrodeGroup, + ElectricalSeries, + FilteredEphys, + LFP, + Clustering, + ClusterWaveforms, + SpikeEventSeries, + EventWaveform, + EventDetection, + FeatureExtraction, +) from pynwb.device import Device from pynwb.file import ElectrodeTable as get_electrode_table -from pynwb.testing import NWBH5IOMixin, AcquisitionH5IOMixin, TestCase +from pynwb.testing import NWBH5IOMixin, AcquisitionH5IOMixin, NWBH5IOFlexMixin, TestCase class TestElectrodeGroupIO(NWBH5IOMixin, TestCase): @@ -28,27 +39,36 @@ def getContainer(self, nwbfile): return nwbfile.get_electrode_group(self.container.name) -class TestElectricalSeriesIO(AcquisitionH5IOMixin, TestCase): +def setup_electrode_table(): + table = get_electrode_table() + dev1 = Device(name='dev1') + group = ElectrodeGroup( + name='tetrode1', + description='tetrode description', + location='tetrode location', + device=dev1 + ) + for i in range(4): + table.add_row(location='CA1', group=group, group_name='tetrode1') + return table, group, dev1 - @staticmethod - def make_electrode_table(self): - """ Make an electrode table, electrode group, and device """ - self.table = get_electrode_table() - self.dev1 = Device(name='dev1') - self.group = ElectrodeGroup(name='tetrode1', - description='tetrode description', - location='tetrode location', - device=self.dev1) - for i in range(4): - self.table.add_row(location='CA1', group=self.group, group_name='tetrode1') - def setUpContainer(self): - """ Return the test ElectricalSeries to read/write """ - self.make_electrode_table(self) +class TestElectricalSeriesIO(NWBH5IOFlexMixin, TestCase): + + def getContainerType(self): + return "ElectricalSeries" + + def addContainer(self): + """ Add the test ElectricalSeries and related objects to the given NWBFile """ + table, group, dev1 = setup_electrode_table() + self.nwbfile.add_device(dev1) + self.nwbfile.add_electrode_group(group) + self.nwbfile.set_electrode_table(table) + region = DynamicTableRegion(name='electrodes', data=[0, 2], description='the first and third electrodes', - table=self.table) + table=table) data = list(zip(range(10), range(10, 20))) timestamps = list(map(lambda x: x/10., range(10))) channel_conversion = [1., 2., 3., 4.] @@ -61,14 +81,11 @@ def setUpContainer(self): filtering=filtering, timestamps=timestamps ) - return es - def addContainer(self, nwbfile): - """ Add the test ElectricalSeries and related objects to the given NWBFile """ - nwbfile.add_device(self.dev1) - nwbfile.add_electrode_group(self.group) - nwbfile.set_electrode_table(self.table) - nwbfile.add_acquisition(self.container) + self.nwbfile.add_acquisition(es) + + def getContainer(self, nwbfile: NWBFile): + return nwbfile.acquisition['test_eS'] def test_eg_ref(self): """ @@ -82,58 +99,70 @@ def test_eg_ref(self): self.assertIsInstance(row2.iloc[0]['group'], ElectrodeGroup) -class MultiElectricalSeriesIOMixin(AcquisitionH5IOMixin): - """ - Mixin class for methods to run a roundtrip test writing an NWB file with multiple ElectricalSeries. +class TestLFPIO(NWBH5IOFlexMixin, TestCase): - The abstract method setUpContainer needs to be implemented by classes that include this mixin. - def setUpContainer(self): - # return a test Container to read/write - """ + def getContainerType(self): + return "LFP" + + def addContainer(self): + table, group, dev1 = setup_electrode_table() + self.nwbfile.add_device(dev1) + self.nwbfile.add_electrode_group(group) + self.nwbfile.set_electrode_table(table) - def setUpTwoElectricalSeries(self): - """ Return two test ElectricalSeries to read/write """ - TestElectricalSeriesIO.make_electrode_table(self) region1 = DynamicTableRegion(name='electrodes', data=[0, 2], description='the first and third electrodes', - table=self.table) + table=table) region2 = DynamicTableRegion(name='electrodes', data=[1, 3], description='the second and fourth electrodes', - table=self.table) + table=table) data1 = list(zip(range(10), range(10, 20))) data2 = list(zip(reversed(range(10)), reversed(range(10, 20)))) timestamps = list(map(lambda x: x/10., range(10))) es1 = ElectricalSeries(name='test_eS1', data=data1, electrodes=region1, timestamps=timestamps) es2 = ElectricalSeries(name='test_eS2', data=data2, electrodes=region2, channel_conversion=[4., .4], timestamps=timestamps) - return es1, es2 + lfp = LFP() + self.nwbfile.add_acquisition(lfp) + lfp.add_electrical_series([es1, es2]) - def addContainer(self, nwbfile): - """ Add the test ElectricalSeries and related objects to the given NWBFile """ - nwbfile.add_device(self.dev1) - nwbfile.add_electrode_group(self.group) - nwbfile.set_electrode_table(self.table) - nwbfile.add_acquisition(self.container) + def getContainer(self, nwbfile: NWBFile): + return nwbfile.acquisition['LFP'] -class TestLFPIO(MultiElectricalSeriesIOMixin, TestCase): +class TestFilteredEphysIO(NWBH5IOFlexMixin, TestCase): - def setUpContainer(self): - """ Return a test LFP to read/write """ - es = self.setUpTwoElectricalSeries() - lfp = LFP(es) - return lfp + def getContainerType(self): + return "FilteredEphys" + def addContainer(self): + table, group, dev1 = setup_electrode_table() + self.nwbfile.add_device(dev1) + self.nwbfile.add_electrode_group(group) + self.nwbfile.set_electrode_table(table) -class TestFilteredEphysIO(MultiElectricalSeriesIOMixin, TestCase): + region1 = DynamicTableRegion(name='electrodes', + data=[0, 2], + description='the first and third electrodes', + table=table) + region2 = DynamicTableRegion(name='electrodes', + data=[1, 3], + description='the second and fourth electrodes', + table=table) + data1 = list(zip(range(10), range(10, 20))) + data2 = list(zip(reversed(range(10)), reversed(range(10, 20)))) + timestamps = list(map(lambda x: x/10., range(10))) + es1 = ElectricalSeries(name='test_eS1', data=data1, electrodes=region1, timestamps=timestamps) + es2 = ElectricalSeries(name='test_eS2', data=data2, electrodes=region2, channel_conversion=[4., .4], + timestamps=timestamps) + fe = FilteredEphys() + self.nwbfile.add_acquisition(fe) + fe.add_electrical_series([es1, es2]) - def setUpContainer(self): - """ Return a test FilteredEphys to read/write """ - es = self.setUpTwoElectricalSeries() - fe = FilteredEphys(es) - return fe + def getContainer(self, nwbfile: NWBFile): + return nwbfile.acquisition['FilteredEphys'] class TestClusteringIO(AcquisitionH5IOMixin, TestCase): @@ -155,28 +184,35 @@ def roundtripExportContainer(self, cache_spec=False): return super().roundtripExportContainer(cache_spec) -class EventWaveformConstructor(AcquisitionH5IOMixin, TestCase): +class EventWaveformConstructor(NWBH5IOFlexMixin, TestCase): + + def getContainerType(self): + return "SpikeEventSeries" + + def addContainer(self): + """ Add the test SpikeEventSeries and related objects to the given NWBFile """ + table, group, dev1 = setup_electrode_table() + self.nwbfile.add_device(dev1) + self.nwbfile.add_electrode_group(group) + self.nwbfile.set_electrode_table(table) - def setUpContainer(self): - """ Return a test EventWaveform to read/write """ - TestElectricalSeriesIO.make_electrode_table(self) region = DynamicTableRegion(name='electrodes', data=[0, 2], description='the first and third electrodes', - table=self.table) - sES = SpikeEventSeries(name='test_sES', - data=((1, 1), (2, 2), (3, 3)), - timestamps=[0., 1., 2.], - electrodes=region) - ew = EventWaveform(sES) - return ew + table=table) + ses = SpikeEventSeries( + name='test_sES', + data=((1, 1), (2, 2), (3, 3)), + timestamps=[0., 1., 2.], + electrodes=region + ) - def addContainer(self, nwbfile): - """ Add the test EventWaveform and related objects to the given NWBFile """ - nwbfile.add_device(self.dev1) - nwbfile.add_electrode_group(self.group) - nwbfile.set_electrode_table(self.table) - nwbfile.add_acquisition(self.container) + ew = EventWaveform() + self.nwbfile.add_acquisition(ew) + ew.add_spike_event_series(ses) + + def getContainer(self, nwbfile: NWBFile): + return nwbfile.acquisition['EventWaveform'] class ClusterWaveformsConstructor(AcquisitionH5IOMixin, TestCase): @@ -210,51 +246,66 @@ def roundtripExportContainer(self, cache_spec=False): return super().roundtripExportContainer(cache_spec) -class FeatureExtractionConstructor(AcquisitionH5IOMixin, TestCase): +class FeatureExtractionConstructor(NWBH5IOFlexMixin, TestCase): + + def getContainerType(self): + return "FeatureExtraction" + + def addContainer(self): + """ Add the test FeatureExtraction and related objects to the given NWBFile """ + table, group, dev1 = setup_electrode_table() + self.nwbfile.add_device(dev1) + self.nwbfile.add_electrode_group(group) + self.nwbfile.set_electrode_table(table) - def setUpContainer(self): - """ Return a test FeatureExtraction to read/write """ event_times = [1.9, 3.5] - TestElectricalSeriesIO.make_electrode_table(self) region = DynamicTableRegion(name='electrodes', data=[0, 2], description='the first and third electrodes', - table=self.table) + table=table) description = ['desc1', 'desc2', 'desc3'] features = [[[0., 1., 2.], [3., 4., 5.]], [[6., 7., 8.], [9., 10., 11.]]] fe = FeatureExtraction(electrodes=region, description=description, times=event_times, features=features) - return fe - def addContainer(self, nwbfile): - """ Add the test FeatureExtraction and related objects to the given NWBFile """ - nwbfile.add_device(self.dev1) - nwbfile.add_electrode_group(self.group) - nwbfile.set_electrode_table(self.table) - nwbfile.add_acquisition(self.container) + self.nwbfile.add_acquisition(fe) + def getContainer(self, nwbfile: NWBFile): + return nwbfile.acquisition['FeatureExtraction'] -class EventDetectionConstructor(AcquisitionH5IOMixin, TestCase): - def setUpContainer(self): - """ Return a test EventDetection to read/write """ - TestElectricalSeriesIO.make_electrode_table(self) +class EventDetectionConstructor(NWBH5IOFlexMixin, TestCase): + + def getContainerType(self): + return "EventDetection" + + def addContainer(self): + """ Add the test EventDetection and related objects to the given NWBFile """ + table, group, dev1 = setup_electrode_table() + self.nwbfile.add_device(dev1) + self.nwbfile.add_electrode_group(group) + self.nwbfile.set_electrode_table(table) + region = DynamicTableRegion(name='electrodes', data=[0, 2], description='the first and third electrodes', - table=self.table) + table=table) data = list(range(10)) ts = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] - self.eS = ElectricalSeries(name='test_eS', data=data, electrodes=region, timestamps=ts) - eD = EventDetection(detection_method='detection_method', - source_electricalseries=self.eS, - source_idx=(1, 2, 3), - times=(0.1, 0.2, 0.3)) - return eD + eS = ElectricalSeries( + name='test_eS', + data=data, + electrodes=region, + timestamps=ts + ) + eD = EventDetection( + detection_method='detection_method', + source_electricalseries=eS, + source_idx=(1, 2, 3), + times=(0.1, 0.2, 0.3) + ) - def addContainer(self, nwbfile): - """ Add the test EventDetection and related objects to the given NWBFile """ - nwbfile.add_device(self.dev1) - nwbfile.add_electrode_group(self.group) - nwbfile.set_electrode_table(self.table) - nwbfile.add_acquisition(self.eS) - nwbfile.add_acquisition(self.container) + self.nwbfile.add_acquisition(eS) + self.nwbfile.add_acquisition(eD) + + def getContainer(self, nwbfile: NWBFile): + return nwbfile.acquisition['EventDetection'] diff --git a/tests/integration/hdf5/test_io.py b/tests/integration/hdf5/test_io.py index 0fd790073..d68334c89 100644 --- a/tests/integration/hdf5/test_io.py +++ b/tests/integration/hdf5/test_io.py @@ -3,6 +3,7 @@ import numpy as np from h5py import File from pathlib import Path +import tempfile from pynwb import NWBFile, TimeSeries, get_manager, NWBHDF5IO, validate @@ -14,6 +15,7 @@ from pynwb.spec import NWBGroupSpec, NWBDatasetSpec, NWBNamespace from pynwb.ecephys import ElectricalSeries, LFP from pynwb.testing import remove_test_file, TestCase +from pynwb.testing.mock.file import mock_NWBFile class TestHDF5Writer(TestCase): @@ -122,6 +124,19 @@ def test_write_no_cache_spec(self): with File(self.path, 'r') as f: self.assertNotIn('specifications', f) + def test_file_creation_io_modes(self): + io_modes_that_create_file = ["w", "w-", "x"] + + with tempfile.TemporaryDirectory() as temp_dir: + temp_dir = Path(temp_dir) + for io_mode in io_modes_that_create_file: + file_path = temp_dir / f"test_io_mode={io_mode}.nwb" + + # Test file creation + nwbfile = mock_NWBFile() + with NWBHDF5IO(str(file_path), io_mode) as io: + io.write(nwbfile) + class TestHDF5WriterWithInjectedFile(TestCase): diff --git a/tests/read_dandi/read_first_nwb_asset.py b/tests/read_dandi/read_first_nwb_asset.py new file mode 100644 index 000000000..895dbb1c2 --- /dev/null +++ b/tests/read_dandi/read_first_nwb_asset.py @@ -0,0 +1,71 @@ +"""Test reading NWB files from the DANDI Archive using fsspec.""" +from dandi.dandiapi import DandiAPIClient +import fsspec +import h5py +import random +import sys +import traceback + +from pynwb import NWBHDF5IO + + +# NOTE: do not name the function with "test_" prefix, otherwise pytest +# will try to run it as a test + +# TODO read dandisets systematically, not randomly +# see https://github.com/NeurodataWithoutBorders/pynwb/issues/1804 + +def read_first_nwb_asset(): + """Test reading the first NWB asset from a random selection of 2 dandisets that uses NWB.""" + num_dandisets_to_read = 2 + client = DandiAPIClient() + dandisets = list(client.get_dandisets()) + random.shuffle(dandisets) + dandisets_to_read = dandisets[:num_dandisets_to_read] + print("Reading NWB files from the following dandisets:") + print([d.get_raw_metadata()["identifier"] for d in dandisets_to_read]) + + fs = fsspec.filesystem("http") + + failed_reads = dict() + for i, dandiset in enumerate(dandisets_to_read): + dandiset_metadata = dandiset.get_raw_metadata() + + # skip any dandisets that do not use NWB + if not any( + data_standard["identifier"] == "RRID:SCR_015242" # this is the RRID for NWB + for data_standard in dandiset_metadata["assetsSummary"].get("dataStandard", []) + ): + continue + + dandiset_identifier = dandiset_metadata["identifier"] + print("--------------") + print(f"{i}: {dandiset_identifier}") + + # iterate through assets until we get an NWB file (it could be MP4) + assets = dandiset.get_assets() + first_asset = next(assets) + while first_asset.path.split(".")[-1] != "nwb": + first_asset = next(assets) + if first_asset.path.split(".")[-1] != "nwb": + print("No NWB files?!") + continue + + s3_url = first_asset.get_content_url(follow_redirects=1, strip_query=True) + + try: + with fs.open(s3_url, "rb") as f: + with h5py.File(f) as file: + with NWBHDF5IO(file=file) as io: + io.read() + except Exception as e: + print(traceback.format_exc()) + failed_reads[dandiset] = e + + if failed_reads: + print(failed_reads) + sys.exit(1) + + +if __name__ == "__main__": + read_first_nwb_asset() diff --git a/tests/read_dandi/test_read_dandi.py b/tests/read_dandi/test_read_dandi.py deleted file mode 100644 index 84e9f3f62..000000000 --- a/tests/read_dandi/test_read_dandi.py +++ /dev/null @@ -1,52 +0,0 @@ -from dandi.dandiapi import DandiAPIClient -import sys -import traceback - -from pynwb import NWBHDF5IO -from pynwb.testing import TestCase - - -class TestReadNWBDandisets(TestCase): - """Test reading NWB files from the DANDI Archive using ROS3.""" - - def test_read_first_nwb_asset(self): - """Test reading the first NWB asset from each dandiset that uses NWB.""" - client = DandiAPIClient() - dandisets = client.get_dandisets() - - failed_reads = dict() - for i, dandiset in enumerate(dandisets): - dandiset_metadata = dandiset.get_raw_metadata() - - # skip any dandisets that do not use NWB - if not any( - data_standard["identifier"] == "RRID:SCR_015242" # this is the RRID for NWB - for data_standard in dandiset_metadata["assetsSummary"].get("dataStandard", []) - ): - continue - - dandiset_identifier = dandiset_metadata["identifier"] - print("--------------") - print(f"{i}: {dandiset_identifier}") - - # iterate through assets until we get an NWB file (it could be MP4) - assets = dandiset.get_assets() - first_asset = next(assets) - while first_asset.path.split(".")[-1] != "nwb": - first_asset = next(assets) - if first_asset.path.split(".")[-1] != "nwb": - print("No NWB files?!") - continue - - s3_url = first_asset.get_content_url(follow_redirects=1, strip_query=True) - - try: - with NWBHDF5IO(path=s3_url, load_namespaces=True, driver="ros3") as io: - io.read() - except Exception as e: - print(traceback.format_exc()) - failed_reads[dandiset] = e - - if failed_reads: - print(failed_reads) - sys.exit(1) diff --git a/tests/unit/test_base.py b/tests/unit/test_base.py index a27a90e96..ad4ce6739 100644 --- a/tests/unit/test_base.py +++ b/tests/unit/test_base.py @@ -1,4 +1,5 @@ import numpy as np +from numpy.testing import assert_array_equal from pynwb.base import ( ProcessingModule, @@ -10,6 +11,7 @@ ImageReferences ) from pynwb.testing import TestCase +from pynwb.testing.mock.base import mock_TimeSeries from hdmf.data_utils import DataChunkIterator from hdmf.backends.hdf5 import H5DataIO @@ -386,6 +388,82 @@ def test_dimension_warning(self): timestamps=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8], ) + def test_get_timestamps(self): + time_series = mock_TimeSeries(data=[1, 2, 3], rate=40.0, starting_time=30.0) + assert_array_equal(time_series.get_timestamps(), [30, 30+1/40, 30+2/40]) + + time_series = mock_TimeSeries(data=[1, 2, 3], timestamps=[3, 4, 5], rate=None) + assert_array_equal(time_series.get_timestamps(), [3, 4, 5]) + + def test_get_data_in_units(self): + ts = mock_TimeSeries(data=[1., 2., 3.], conversion=2., offset=3.) + assert_array_equal(ts.get_data_in_units(), [5., 7., 9.]) + + ts = mock_TimeSeries(data=[1., 2., 3.], conversion=2.) + assert_array_equal(ts.get_data_in_units(), [2., 4., 6.]) + + ts = mock_TimeSeries(data=[1., 2., 3.]) + assert_array_equal(ts.get_data_in_units(), [1., 2., 3.]) + + def test_non_positive_rate(self): + with self.assertRaisesWith(ValueError, 'Rate must not be a negative value.'): + TimeSeries(name='test_ts', data=list(), unit='volts', rate=-1.0) + + with self.assertWarnsWith(UserWarning, + 'Timeseries has a rate of 0.0 Hz, but the length of the data is greater than 1.'): + TimeSeries(name='test_ts1', data=[1, 2, 3], unit='volts', rate=0.0) + + def test_file_with_non_positive_rate_in_construct_mode(self): + """Test that UserWarning is raised when rate is 0 or negative + while being in construct mode (i.e,. on data read).""" + obj = TimeSeries.__new__(TimeSeries, + container_source=None, + parent=None, + object_id="test", + in_construct_mode=True) + with self.assertWarnsWith(warn_type=UserWarning, exc_msg='Rate must not be a negative value.'): + obj.__init__( + name="test_ts", + data=list(), + unit="volts", + rate=-1.0 + ) + + def test_file_with_rate_and_timestamps_in_construct_mode(self): + """Test that UserWarning is raised when rate and timestamps are both specified + while being in construct mode (i.e,. on data read).""" + obj = TimeSeries.__new__(TimeSeries, + container_source=None, + parent=None, + object_id="test", + in_construct_mode=True) + with self.assertWarnsWith(warn_type=UserWarning, exc_msg='Specifying rate and timestamps is not supported.'): + obj.__init__( + name="test_ts", + data=[11, 12, 13, 14, 15], + unit="volts", + rate=1.0, + timestamps=[1, 2, 3, 4, 5] + ) + + def test_file_with_starting_time_and_timestamps_in_construct_mode(self): + """Test that UserWarning is raised when starting_time and timestamps are both specified + while being in construct mode (i.e,. on data read).""" + obj = TimeSeries.__new__(TimeSeries, + container_source=None, + parent=None, + object_id="test", + in_construct_mode=True) + with self.assertWarnsWith(warn_type=UserWarning, + exc_msg='Specifying starting_time and timestamps is not supported.'): + obj.__init__( + name="test_ts", + data=[11, 12, 13, 14, 15], + unit="volts", + starting_time=1.0, + timestamps=[1, 2, 3, 4, 5] + ) + class TestImage(TestCase): def test_init(self): diff --git a/tests/unit/test_ecephys.py b/tests/unit/test_ecephys.py index 6cdfcd59e..f81b61f84 100644 --- a/tests/unit/test_ecephys.py +++ b/tests/unit/test_ecephys.py @@ -2,11 +2,23 @@ import numpy as np -from pynwb.ecephys import ElectricalSeries, SpikeEventSeries, EventDetection, Clustering, EventWaveform,\ - ClusterWaveforms, LFP, FilteredEphys, FeatureExtraction, ElectrodeGroup +from pynwb.base import ProcessingModule +from pynwb.ecephys import ( + ElectricalSeries, + SpikeEventSeries, + EventDetection, + Clustering, + EventWaveform, + ClusterWaveforms, + LFP, + FilteredEphys, + FeatureExtraction, + ElectrodeGroup, +) from pynwb.device import Device from pynwb.file import ElectrodeTable from pynwb.testing import TestCase +from pynwb.testing.mock.ecephys import mock_ElectricalSeries from hdmf.common import DynamicTableRegion @@ -104,6 +116,24 @@ def test_dimensions_warning(self): "but instead the first does. Data is oriented incorrectly and should be transposed." ) in str(w[-1].message) + def test_get_data_in_units(self): + + data = np.asarray([[1, 1, 1, 1, 1], [1, 1, 1, 1, 1]]) + conversion = 1.0 + offset = 3.0 + channel_conversion = np.asarray([2.0, 2.0]) + electrical_series = mock_ElectricalSeries( + data=data, + conversion=conversion, + offset=offset, + channel_conversion=channel_conversion, + ) + + data_in_units = electrical_series.get_data_in_units() + expected_data = data * conversion * channel_conversion[:, np.newaxis] + offset + + np.testing.assert_almost_equal(data_in_units, expected_data) + class SpikeEventSeriesConstructor(TestCase): @@ -207,7 +237,11 @@ def test_init(self): table, region = self._create_table_and_region() sES = SpikeEventSeries('test_sES', list(range(10)), list(range(10)), region) - ew = EventWaveform(sES) + pm = ProcessingModule(name='test_module', description='a test module') + ew = EventWaveform() + pm.add(table) + pm.add(ew) + ew.add_spike_event_series(sES) self.assertEqual(ew.spike_event_series['test_sES'], sES) self.assertEqual(ew['test_sES'], ew.spike_event_series['test_sES']) @@ -264,10 +298,25 @@ def _create_table_and_region(self): ) return table, region + def test_init(self): + _, region = self._create_table_and_region() + eS = ElectricalSeries('test_eS', [0, 1, 2, 3], region, timestamps=[0.1, 0.2, 0.3, 0.4]) + msg = ( + "The linked table for DynamicTableRegion 'electrodes' does not share " + "an ancestor with the DynamicTableRegion." + ) + with self.assertWarnsRegex(UserWarning, msg): + lfp = LFP(eS) + self.assertEqual(lfp.electrical_series.get('test_eS'), eS) + self.assertEqual(lfp['test_eS'], lfp.electrical_series.get('test_eS')) + def test_add_electrical_series(self): lfp = LFP() table, region = self._create_table_and_region() eS = ElectricalSeries('test_eS', [0, 1, 2, 3], region, timestamps=[0.1, 0.2, 0.3, 0.4]) + pm = ProcessingModule(name='test_module', description='a test module') + pm.add(table) + pm.add(lfp) lfp.add_electrical_series(eS) self.assertEqual(lfp.electrical_series.get('test_eS'), eS) @@ -285,16 +334,24 @@ def _create_table_and_region(self): return table, region def test_init(self): - table, region = self._create_table_and_region() + _, region = self._create_table_and_region() eS = ElectricalSeries('test_eS', [0, 1, 2, 3], region, timestamps=[0.1, 0.2, 0.3, 0.4]) - fe = FilteredEphys(eS) + msg = ( + "The linked table for DynamicTableRegion 'electrodes' does not share " + "an ancestor with the DynamicTableRegion." + ) + with self.assertWarnsRegex(UserWarning, msg): + fe = FilteredEphys(eS) self.assertEqual(fe.electrical_series.get('test_eS'), eS) self.assertEqual(fe['test_eS'], fe.electrical_series.get('test_eS')) def test_add_electrical_series(self): - fe = FilteredEphys() table, region = self._create_table_and_region() eS = ElectricalSeries('test_eS', [0, 1, 2, 3], region, timestamps=[0.1, 0.2, 0.3, 0.4]) + pm = ProcessingModule(name='test_module', description='a test module') + fe = FilteredEphys() + pm.add(table) + pm.add(fe) fe.add_electrical_series(eS) self.assertEqual(fe.electrical_series.get('test_eS'), eS) self.assertEqual(fe['test_eS'], fe.electrical_series.get('test_eS')) diff --git a/tests/unit/test_file.py b/tests/unit/test_file.py index bb5c9c1e1..756009ff3 100644 --- a/tests/unit/test_file.py +++ b/tests/unit/test_file.py @@ -527,6 +527,7 @@ def test_subject_age_duration(self): class TestCacheSpec(TestCase): + """Test whether the file can be written and read when caching the spec.""" def setUp(self): self.path = 'unittest_cached_spec.nwb' @@ -535,18 +536,20 @@ def tearDown(self): remove_test_file(self.path) def test_simple(self): - nwbfile = NWBFile(' ', ' ', + nwbfile = NWBFile('sess_desc', 'identifier', datetime.now(tzlocal()), file_create_date=datetime.now(tzlocal()), institution='University of California, San Francisco', lab='Chang Lab') with NWBHDF5IO(self.path, 'w') as io: io.write(nwbfile) - with NWBHDF5IO(self.path, 'r', load_namespaces=True) as reader: + with NWBHDF5IO(self.path, 'r') as reader: nwbfile = reader.read() + assert nwbfile.session_description == "sess_desc" class TestNoCacheSpec(TestCase): + """Test whether the file can be written and read when not caching the spec.""" def setUp(self): self.path = 'unittest_cached_spec.nwb' @@ -555,7 +558,7 @@ def tearDown(self): remove_test_file(self.path) def test_simple(self): - nwbfile = NWBFile(' ', ' ', + nwbfile = NWBFile('sess_desc', 'identifier', datetime.now(tzlocal()), file_create_date=datetime.now(tzlocal()), institution='University of California, San Francisco', @@ -563,9 +566,9 @@ def test_simple(self): with NWBHDF5IO(self.path, 'w') as io: io.write(nwbfile, cache_spec=False) - with self.assertWarnsWith(UserWarning, "No cached namespaces found in %s" % self.path): - with NWBHDF5IO(self.path, 'r', load_namespaces=True) as reader: - nwbfile = reader.read() + with NWBHDF5IO(self.path, 'r') as reader: + nwbfile = reader.read() + assert nwbfile.session_description == "sess_desc" class TestTimestampsRefDefault(TestCase): diff --git a/tests/unit/test_mock.py b/tests/unit/test_mock.py index 6f59c2007..d24e47551 100644 --- a/tests/unit/test_mock.py +++ b/tests/unit/test_mock.py @@ -1,3 +1,5 @@ +from pynwb import NWBHDF5IO + from pynwb.testing.mock.file import mock_Subject, mock_NWBFile from pynwb.testing.mock.base import mock_TimeSeries @@ -49,45 +51,57 @@ from pynwb.testing.mock.utils import name_generator, name_generator_registry +mock_functions = [ + mock_ImagingPlane, + mock_OnePhotonSeries, + mock_TwoPhotonSeries, + mock_RoiResponseSeries, + mock_PlaneSegmentation, + mock_OpticalChannel, + mock_Fluorescence, + mock_DfOverF, + mock_ImageSegmentation, + mock_OptogeneticStimulusSite, + mock_OptogeneticSeries, + mock_Device, + mock_Position, + mock_PupilTracking, + mock_CompassDirection, + mock_SpatialSeries, + mock_ElectrodeGroup, + mock_ElectrodeTable, + mock_ElectricalSeries, + mock_SpikeEventSeries, + mock_Subject, + mock_NWBFile, + mock_TimeSeries, + mock_CurrentClampSeries, + mock_IZeroClampSeries, + mock_VoltageClampSeries, + mock_VoltageClampStimulusSeries, + mock_IntracellularElectrode, + mock_CurrentClampStimulusSeries, + mock_IntracellularRecordingsTable, +] -@pytest.mark.parametrize( - "mock_function", [ - mock_ImagingPlane, - mock_OnePhotonSeries, - mock_TwoPhotonSeries, - mock_RoiResponseSeries, - mock_PlaneSegmentation, - mock_OpticalChannel, - mock_Fluorescence, - mock_DfOverF, - mock_ImageSegmentation, - mock_OptogeneticStimulusSite, - mock_OptogeneticSeries, - mock_Device, - mock_Position, - mock_PupilTracking, - mock_CompassDirection, - mock_SpatialSeries, - mock_ElectrodeGroup, - mock_ElectrodeTable, - mock_ElectricalSeries, - mock_SpikeEventSeries, - mock_Subject, - mock_NWBFile, - mock_TimeSeries, - mock_CurrentClampSeries, - mock_IZeroClampSeries, - mock_VoltageClampSeries, - mock_VoltageClampStimulusSeries, - mock_IntracellularElectrode, - mock_CurrentClampStimulusSeries, - mock_IntracellularRecordingsTable, - ], -) + +@pytest.mark.parametrize("mock_function", mock_functions) def test_mock(mock_function): mock_function() +@pytest.mark.parametrize("mock_function", mock_functions) +def test_mock_write(mock_function, tmp_path): + if mock_function is mock_NWBFile: + return + nwbfile = mock_NWBFile() + assert mock_function(nwbfile=nwbfile) is not None + + test_file = tmp_path / (mock_function.__name__ + ".nwb") + with NWBHDF5IO(test_file, "w") as io: + io.write(nwbfile) + + def test_name_generator(): name_generator_registry.clear() # reset registry diff --git a/tests/unit/test_ophys.py b/tests/unit/test_ophys.py index 1ebb7c640..88bd24535 100644 --- a/tests/unit/test_ophys.py +++ b/tests/unit/test_ophys.py @@ -2,7 +2,7 @@ import numpy as np -from pynwb.base import TimeSeries +from pynwb.base import TimeSeries, ProcessingModule from pynwb.device import Device from pynwb.image import ImageSeries from pynwb.ophys import ( @@ -398,9 +398,15 @@ def test_warnings(self): class DfOverFConstructor(TestCase): def test_init(self): + pm = ProcessingModule(name='ophys', description="Optical physiology") + ps = create_plane_segmentation() - rt_region = ps.create_roi_table_region(description='the second ROI', region=[1]) + pm.add(ps) + + dof = DfOverF() + pm.add(dof) + rt_region = ps.create_roi_table_region(description='the second ROI', region=[1]) rrs = RoiResponseSeries( name='test_ts', data=[1, 2, 3], @@ -408,26 +414,32 @@ def test_init(self): unit='unit', timestamps=[0.1, 0.2, 0.3] ) + dof.add_roi_response_series(rrs) - dof = DfOverF(rrs) self.assertEqual(dof.roi_response_series['test_ts'], rrs) class FluorescenceConstructor(TestCase): def test_init(self): + pm = ProcessingModule(name='ophys', description="Optical physiology") + ps = create_plane_segmentation() - rt_region = ps.create_roi_table_region(description='the second ROI', region=[1]) + pm.add(ps) - ts = RoiResponseSeries( + ff = Fluorescence() + pm.add(ff) + + rt_region = ps.create_roi_table_region(description='the second ROI', region=[1]) + rrs = RoiResponseSeries( name='test_ts', data=[1, 2, 3], rois=rt_region, unit='unit', timestamps=[0.1, 0.2, 0.3] ) + ff.add_roi_response_series(rrs) - ff = Fluorescence(ts) - self.assertEqual(ff.roi_response_series['test_ts'], ts) + self.assertEqual(ff.roi_response_series['test_ts'], rrs) class ImageSegmentationConstructor(TestCase): diff --git a/tests/unit/test_resources.py b/tests/unit/test_resources.py index cfb598b7b..108a7fd84 100644 --- a/tests/unit/test_resources.py +++ b/tests/unit/test_resources.py @@ -1,4 +1,6 @@ -from pynwb.resources import ExternalResources +import warnings + +from pynwb.resources import HERD from pynwb.testing import TestCase @@ -7,5 +9,11 @@ def test_constructor(self): """ Test constructor """ - er = ExternalResources() - self.assertIsInstance(er, ExternalResources) + with warnings.catch_warnings(record=True): + warnings.filterwarnings( + "ignore", + message=r"HERD is experimental .*", + category=UserWarning, + ) + er = HERD() + self.assertIsInstance(er, HERD) diff --git a/tests/validation/test_validate.py b/tests/validation/test_validate.py index 813f8d4e3..c2829ee1f 100644 --- a/tests/validation/test_validate.py +++ b/tests/validation/test_validate.py @@ -1,36 +1,45 @@ import subprocess import re +import sys from unittest.mock import patch from io import StringIO +import warnings from pynwb.testing import TestCase from pynwb import validate, NWBHDF5IO +# NOTE we use "coverage run -m pynwb.validate" instead of "python -m pynwb.validate" +# so that we can both test pynwb.validate and compute code coverage from that test. +# NOTE we also use "coverage run -p" which will generate a .coverage file with the +# machine name, process id, and a random number appended to the filename to +# simplify collecting and merging coverage data from multiple subprocesses. if "-p" +# is not used, then each "coverage run" will overwrite the .coverage file from a +# previous "coverage run". +# NOTE we run "coverage" as "{sys.executable} -m coverage" to 1. make sure to use +# the same python version, and on Debian systems executable is "python3-coverage", not +# just "coverage". +# NOTE the run_coverage.yml GitHub Action runs "python -m coverage combine" to +# combine the individual coverage reports into one .coverage file. +def run_coverage(extra_args: list[str]): + return subprocess.run( + [sys.executable, "-m", "coverage", "run", "-p", "-m", "pynwb.validate"] + + extra_args, + capture_output=True + ) + + class TestValidateCLI(TestCase): # 1.0.2_nwbfile.nwb has no cached specifications # 1.0.3_nwbfile.nwb has cached "core" specification # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications - # NOTE we use "coverage run -m pynwb.validate" instead of "python -m pynwb.validate" - # so that we can both test pynwb.validate and compute code coverage from that test. - # NOTE we also use "coverage run -p" which will generate a .coverage file with the - # machine name, process id, and a random number appended to the filename to - # simplify collecting and merging coverage data from multiple subprocesses. if "-p" - # is not used, then each "coverage run" will overwrite the .coverage file from a - # previous "coverage run". - # NOTE the run_coverage.yml GitHub Action runs "python -m coverage combine" to - # combine the individual coverage reports into one .coverage file. - def test_validate_file_no_cache(self): """Test that validating a file with no cached spec against the core namespace succeeds.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", - "tests/back_compat/1.0.2_nwbfile.nwb"], capture_output=True) + result = run_coverage(["tests/back_compat/1.0.2_nwbfile.nwb"]) stderr_regex = re.compile( - r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*" - r"warnings.warn\(msg\)\s*" r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " r"Falling back to PyNWB namespace information\.\s*" ) @@ -43,12 +52,9 @@ def test_validate_file_no_cache(self): def test_validate_file_no_cache_bad_ns(self): """Test that validating a file with no cached spec against a specified, unknown namespace fails.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.0.2_nwbfile.nwb", - "--ns", "notfound"], capture_output=True) + result = run_coverage(["tests/back_compat/1.0.2_nwbfile.nwb", "--ns", "notfound"]) stderr_regex = re.compile( - r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*" - r"warnings.warn\(msg\)\s*" r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " r"Falling back to PyNWB namespace information\.\s*" r"The namespace 'notfound' could not be found in PyNWB namespace information as only " @@ -60,8 +66,7 @@ def test_validate_file_no_cache_bad_ns(self): def test_validate_file_cached(self): """Test that validating a file with cached spec against its cached namespace succeeds.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", - "tests/back_compat/1.1.2_nwbfile.nwb"], capture_output=True) + result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb"]) self.assertEqual(result.stderr.decode('utf-8'), '') @@ -72,8 +77,7 @@ def test_validate_file_cached(self): def test_validate_file_cached_bad_ns(self): """Test that validating a file with cached spec against a specified, unknown namespace fails.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", - "tests/back_compat/1.1.2_nwbfile.nwb", "--ns", "notfound"], capture_output=True) + result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--ns", "notfound"]) stderr_regex = re.compile( r"The namespace 'notfound' could not be found in cached namespace information as only " @@ -85,8 +89,7 @@ def test_validate_file_cached_bad_ns(self): def test_validate_file_cached_extension(self): """Test that validating a file with cached spec against the cached namespaces succeeds.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", - "tests/back_compat/2.1.0_nwbfile_with_extension.nwb"], capture_output=True) + result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb"]) self.assertEqual(result.stderr.decode('utf-8'), '') @@ -97,9 +100,7 @@ def test_validate_file_cached_extension(self): def test_validate_file_cached_extension_pass_ns(self): """Test that validating a file with cached spec against the extension namespace succeeds.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", - "tests/back_compat/2.1.0_nwbfile_with_extension.nwb", - "--ns", "ndx-testextension"], capture_output=True) + result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb", "--ns", "ndx-testextension"]) self.assertEqual(result.stderr.decode('utf-8'), '') @@ -110,9 +111,7 @@ def test_validate_file_cached_extension_pass_ns(self): def test_validate_file_cached_core(self): """Test that validating a file with cached spec against the core namespace succeeds.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", - "tests/back_compat/2.1.0_nwbfile_with_extension.nwb", - "--ns", "core"], capture_output=True) + result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb", "--ns", "core"]) stdout_regex = re.compile( r"The namespace 'core' is included by the namespace 'ndx-testextension'. " @@ -122,8 +121,7 @@ def test_validate_file_cached_core(self): def test_validate_file_cached_hdmf_common(self): """Test that validating a file with cached spec against the hdmf-common namespace fails.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb", - "--ns", "hdmf-common"], capture_output=True) + result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--ns", "hdmf-common"]) stderr_regex = re.compile( r"The namespace 'hdmf-common' is included by the namespace 'core'\. Please validate against that " @@ -133,8 +131,7 @@ def test_validate_file_cached_hdmf_common(self): def test_validate_file_cached_ignore(self): """Test that validating a file with cached spec against the core namespace succeeds.""" - result = subprocess.run(["coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb", - "--no-cached-namespace"], capture_output=True) + result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--no-cached-namespace"]) self.assertEqual(result.stderr.decode('utf-8'), '') @@ -145,13 +142,7 @@ def test_validate_file_cached_ignore(self): def test_validate_file_invalid(self): """Test that validating an invalid file outputs errors.""" - result = subprocess.run( - [ - "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.0.2_str_experimenter.nwb", - "--no-cached-namespace" - ], - capture_output=True - ) + result = run_coverage(["tests/back_compat/1.0.2_str_experimenter.nwb", "--no-cached-namespace"]) stderr_regex = re.compile( r" - found the following errors:\s*" @@ -167,13 +158,7 @@ def test_validate_file_invalid(self): def test_validate_file_list_namespaces_core(self): """Test listing namespaces from a file""" - result = subprocess.run( - [ - "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/1.1.2_nwbfile.nwb", - "--list-namespaces" - ], - capture_output=True - ) + result = run_coverage(["tests/back_compat/1.1.2_nwbfile.nwb", "--list-namespaces"]) self.assertEqual(result.stderr.decode('utf-8'), '') @@ -182,13 +167,7 @@ def test_validate_file_list_namespaces_core(self): def test_validate_file_list_namespaces_extension(self): """Test listing namespaces from a file with an extension""" - result = subprocess.run( - [ - "coverage", "run", "-p", "-m", "pynwb.validate", "tests/back_compat/2.1.0_nwbfile_with_extension.nwb", - "--list-namespaces" - ], - capture_output=True - ) + result = run_coverage(["tests/back_compat/2.1.0_nwbfile_with_extension.nwb", "--list-namespaces"]) self.assertEqual(result.stderr.decode('utf-8'), '') @@ -202,33 +181,43 @@ class TestValidateFunction(TestCase): # 1.0.3_nwbfile.nwb has cached "core" specification # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specificaitions + def get_io(self, path): + """Get an NWBHDF5IO object for the given path, ignoring the warning about ignoring cached namespaces.""" + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=r"Ignoring cached namespace .*", + category=UserWarning, + ) + return NWBHDF5IO(str(path), 'r') + def test_validate_io_no_cache(self): """Test that validating a file with no cached spec against the core namespace succeeds.""" - with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.0.2_nwbfile.nwb') as io: errors = validate(io) self.assertEqual(errors, []) def test_validate_io_no_cache_bad_ns(self): """Test that validating a file with no cached spec against a specified, unknown namespace fails.""" - with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.0.2_nwbfile.nwb') as io: with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): validate(io, 'notfound') def test_validate_io_cached(self): """Test that validating a file with cached spec against its cached namespace succeeds.""" - with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.1.2_nwbfile.nwb') as io: errors = validate(io) self.assertEqual(errors, []) def test_validate_io_cached_extension(self): """Test that validating a file with cached spec against its cached namespaces succeeds.""" - with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r', load_namespaces=True) as io: + with self.get_io('tests/back_compat/2.1.0_nwbfile_with_extension.nwb') as io: errors = validate(io) self.assertEqual(errors, []) def test_validate_io_cached_extension_pass_ns(self): """Test that validating a file with cached extension spec against the extension namespace succeeds.""" - with NWBHDF5IO('tests/back_compat/2.1.0_nwbfile_with_extension.nwb', 'r', load_namespaces=True) as io: + with self.get_io('tests/back_compat/2.1.0_nwbfile_with_extension.nwb') as io: errors = validate(io, 'ndx-testextension') self.assertEqual(errors, []) @@ -237,9 +226,7 @@ def test_validate_io_cached_core_with_io(self): For back-compatability, test that validating a file with cached extension spec against the core namespace succeeds when using the `io` + `namespace` keywords. """ - with NWBHDF5IO( - path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb', mode='r', load_namespaces=True - ) as io: + with self.get_io(path='tests/back_compat/2.1.0_nwbfile_with_extension.nwb') as io: results = validate(io=io, namespace="core") self.assertEqual(results, []) @@ -295,13 +282,13 @@ def test_validate_file_cached_no_cache_bad_ns(self): def test_validate_io_cached_bad_ns(self): """Test that validating a file with cached spec against a specified, unknown namespace fails.""" - with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.1.2_nwbfile.nwb') as io: with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): validate(io, 'notfound') def test_validate_io_cached_hdmf_common(self): """Test that validating a file with cached spec against the hdmf-common namespace fails.""" - with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.get_io('tests/back_compat/1.1.2_nwbfile.nwb') as io: # TODO this error should not be different from the error when using the validate script above msg = "builder must have data type defined with attribute 'data_type'" with self.assertRaisesWith(ValueError, msg):