Merge pull request #920 from Unidata/issue919

fix for issue #919

.travis.yml

@@ -11,7 +11,7 @@ addons:
 
 env:
   global:
-    - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0 cftime"
+    - DEPENDS="numpy>=1.10.0 cython>=0.21 setuptools>=18.0 cftime"
     - NO_NET=1
     - MPI=0
 
@@ -28,22 +28,22 @@ matrix:
       env:
         - MPI=1
         - CC=mpicc.mpich
-        - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
+        - DEPENDS="numpy>=1.10.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
         - NETCDF_VERSION=GITMASTER
         - NETCDF_DIR=$HOME
         - PATH=${NETCDF_DIR}/bin:${PATH} # pick up nc-config here
   include:
     # Absolute minimum dependencies.
     - python: 2.7
       env:
-        - DEPENDS="numpy==1.9.0 cython==0.21 ordereddict==1.1 setuptools==18.0 cftime"
+        - DEPENDS="numpy==1.10.0 cython==0.21 ordereddict==1.1 setuptools==18.0 cftime"
     # test MPI with latest released version
     - python: 3.7
       dist: xenial
       env: 
         - MPI=1
         - CC=mpicc.mpich
-        - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
+        - DEPENDS="numpy>=1.10.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
         - NETCDF_VERSION=4.6.3
         - NETCDF_DIR=$HOME
         - PATH=${NETCDF_DIR}/bin:${PATH} # pick up nc-config here
@@ -59,7 +59,7 @@ matrix:
       env: 
         - MPI=1
         - CC=mpicc.mpich
-        - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
+        - DEPENDS="numpy>=1.10.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
         - NETCDF_VERSION=4.6.3
         - PNETCDF_VERSION=1.11.0
         - NETCDF_DIR=$HOME
@@ -76,7 +76,7 @@ matrix:
       env:
         - MPI=1
         - CC=mpicc.mpich
-        - DEPENDS="numpy>=1.9.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
+        - DEPENDS="numpy>=1.10.0 cython>=0.21 setuptools>=18.0 mpi4py>=1.3.1 cftime"
         - NETCDF_VERSION=GITMASTER
         - NETCDF_DIR=$HOME
         - PATH=${NETCDF_DIR}/bin:${PATH} # pick up nc-config here

Changelog

@@ -1,3 +1,10 @@
+ version 1.5.1.1 (tag v1.5.1.1rel)
+==================================
+ * fixed __version__ attribute (was set incorrectly in 1.5.1 release).
+ * fix for issue #919  (assigning 2d array to 3d variable with singleton
+   first dimension with v[:] = a).
+ * minimum numpy changed from 1.9.0 to 1.10.0.
+
  version 1.5.1 (tag v1.5.1rel)
 ==============================
  * fix issue #908 by adding workaround for incorrect value returned

README.md

@@ -9,6 +9,9 @@
 
 ## News
 For details on the latest updates, see the [Changelog](https://github.com/Unidata/netcdf4-python/blob/master/Changelog).
+
+05/02/2019: Version [1.5.1.1](https://pypi.python.org/pypi/netCDF4/1.5.1.1) released. Fixes incorrect __version__
+module variable in 1.5.1 release, plus a slicing bug ([issue #919)](https://github.com/Unidata/netcdf4-python/issues/919)).
 
 04/30/2019: Version [1.5.1](https://pypi.python.org/pypi/netCDF4/1.5.1) released. Bugfixes, no new features.
 

docs/netCDF4/index.html

@@ -4,14 +4,14 @@
   <meta name="viewport" content="width=device-width, initial-scale=1, minimum-scale=1" />
 
     <title>netCDF4 API documentation</title>
-    <meta name="description" content="Version 1.5.1
--------------
+    <meta name="description" content="Version 1.5.1.1
+---------------
 - - -
 
 Introduction
 ============
 
-netcdf4-python is a Python interface t..." />
+netcdf4-python is a Python interfa..." />
 
   <link href='http://fonts.googleapis.com/css?family=Source+Sans+Pro:400,300' rel='stylesheet' type='text/css'>
 
@@ -1280,7 +1280,7 @@ <h1>Index</h1>
 
   <header id="section-intro">
   <h1 class="title"><span class="name">netCDF4</span> module</h1>
-  <h2>Version 1.5.1</h2>
+  <h2>Version 1.5.1.1</h2>
 <hr />
 <h1>Introduction</h1>
 <p>netcdf4-python is a Python interface to the netCDF C library.</p>
@@ -1309,7 +1309,7 @@ <h1>Download</h1>
 <h1>Requires</h1>
 <ul>
 <li>Python 2.7 or later (python 3 works too).</li>
-<li><a href="http://numpy.scipy.org">numpy array module</a>, version 1.9.0 or later.</li>
+<li><a href="http://numpy.scipy.org">numpy array module</a>, version 1.10.0 or later.</li>
 <li><a href="http://cython.org">Cython</a>, version 0.21 or later.</li>
 <li><a href="https://pypi.python.org/pypi/setuptools">setuptools</a>, version 18.0 or
    later.</li>

netCDF4/_netCDF4.pyx

@@ -1,6 +1,6 @@
 """
-Version 1.5.1
--------------
+Version 1.5.1.1
+---------------
 - - -
 
 Introduction
@@ -37,7 +37,7 @@ Requires
 ========
 
  - Python 2.7 or later (python 3 works too).
- - [numpy array module](http://numpy.scipy.org), version 1.9.0 or later.
+ - [numpy array module](http://numpy.scipy.org), version 1.10.0 or later.
  - [Cython](http://cython.org), version 0.21 or later.
  - [setuptools](https://pypi.python.org/pypi/setuptools), version 18.0 or
    later.
@@ -1190,7 +1190,7 @@ except ImportError:
     # python3: zip is already python2's itertools.izip
     pass
 
-__version__ = "1.5.0.1"
+__version__ = "1.5.1.1"
 
 # Initialize numpy
 import posixpath
@@ -4800,12 +4800,15 @@ cannot be safely cast to variable data type""" % attname
         # and fill with scalar values.
         if data.shape == ():
             data = numpy.tile(data,datashape)
-        # reshape data array by adding extra singleton dimensions
+        # reshape data array by adding extra dimensions
         # if needed to conform with start,count,stride.
         if len(data.shape) != len(datashape):
             # create a view so shape in caller is not modified (issue 90)
-            data = data.view()
-            data.shape = tuple(datashape)
+            try: # if extra singleton dims, just reshape
+                data = data.view()
+                data.shape = tuple(datashape)
+            except ValueError: # otherwise broadcast
+                data = numpy.broadcast_to(data, datashape)
 
         # Reshape these arrays so we can iterate over them.
         start = start.reshape((-1, self.ndim or 1))

netCDF4/utils.py

@@ -3,6 +3,7 @@
 import sys
 import numpy as np
 from numpy import ma
+from numpy.lib.stride_tricks import as_strided
 import warnings
 import getopt
 import os
@@ -178,6 +179,22 @@ def _StartCountStride(elem, shape, dimensions=None, grp=None, datashape=None,\
         nDims = 1
         shape = (1,)
 
+    # is there an unlimited dimension? (only defined for __setitem__)
+    if put:
+        hasunlim = False
+        unlimd={}
+        if dimensions:
+            for i in range(nDims):
+                dimname = dimensions[i]
+                # is this dimension unlimited?
+                # look in current group, and parents for dim.
+                dim = _find_dim(grp, dimname)
+                unlimd[dimname]=dim.isunlimited()
+                if unlimd[dimname]:
+                    hasunlim = True
+    else:
+        hasunlim = False
+
     # When a single array or (non-tuple) sequence of integers is given
     # as a slice, assume it applies to the first dimension,
     # and use ellipsis for remaining dimensions.
@@ -189,14 +206,14 @@ def _StartCountStride(elem, shape, dimensions=None, grp=None, datashape=None,\
                 elem.append(slice(None,None,None))
     else:   # Convert single index to sequence
         elem = [elem]
-    
+
     # ensure there is at most 1 ellipse
     #  we cannot use elem.count(Ellipsis), as with fancy indexing would occur
-    #  np.array() == Ellipsis which gives ValueError: The truth value of an 
+    #  np.array() == Ellipsis which gives ValueError: The truth value of an
     #  array with more than one element is ambiguous. Use a.any() or a.all()
     if sum(1 for e in elem if e is Ellipsis) > 1:
         raise IndexError("At most one ellipsis allowed in a slicing expression")
-    
+
     # replace boolean arrays with sequences of integers.
     newElem = []
     IndexErrorMsg=\
@@ -217,13 +234,10 @@ def _StartCountStride(elem, shape, dimensions=None, grp=None, datashape=None,\
             raise IndexError("Index cannot be multidimensional")
         # set unlim to True if dimension is unlimited and put==True
         # (called from __setitem__)
-        if put and (dimensions is not None and grp is not None) and len(dimensions):
+        if hasunlim and put and dimensions:
             try:
                 dimname = dimensions[i]
-                # is this dimension unlimited?
-                # look in current group, and parents for dim.
-                dim = _find_dim(grp, dimname)
-                unlim = dim.isunlimited()
+                unlim = unlimd[dimname]
             except IndexError: # more slices than dimensions (issue 371)
                 unlim = False
         else:
@@ -282,7 +296,7 @@ def _StartCountStride(elem, shape, dimensions=None, grp=None, datashape=None,\
                 newElem.append(e)
             except:
                 raise IndexError(IndexErrorMsg)
-        if type(e)==type(Ellipsis): 
+        if type(e)==type(Ellipsis):
             i+=1+nDims-len(elem)
         else:
             i+=1
@@ -342,7 +356,15 @@ def _StartCountStride(elem, shape, dimensions=None, grp=None, datashape=None,\
         else:
             sdim.append(1)
 
-    # pad datashape with zeros for dimensions not being sliced
+    # broadcast data shape when assigned to full variable (issue #919)
+    try:
+        fullslice = elem.count(slice(None,None,None)) == len(elem)
+    except: # fails if elem contains a numpy array.
+        fullslice = False
+    if fullslice and datashape and put and not hasunlim:
+        datashape = broadcasted_shape(shape, datashape)
+
+    # pad datashape with zeros for dimensions not being sliced (issue #906)
     if datashape:
         datashapenew = (); i=0
         for e in elem:
@@ -367,12 +389,9 @@ def _StartCountStride(elem, shape, dimensions=None, grp=None, datashape=None,\
 
         # set unlim to True if dimension is unlimited and put==True
         # (called from __setitem__). Note: grp and dimensions must be set.
-        if put and (dimensions is not None and grp is not None) and len(dimensions):
+        if hasunlim and put and dimensions:
             dimname = dimensions[i]
-            # is this dimension unlimited?
-            # look in current group, and parents for dim.
-            dim = _find_dim(grp, dimname)
-            unlim = dim.isunlimited()
+            unlim = unlimd[dimname]
         else:
             unlim = False
 
@@ -938,3 +957,12 @@ def nc3tonc4():
         fletcher32=fletcher32,clobber=overwritefile,lsd_dict=lsd_dict,
         nchunk=chunk,quiet=quiet,vars=vars,classic=classic,
         istart=istart,istop=istop)
+
+def broadcasted_shape(shp1, shp2):
+    # determine shape of array of shp1 and shp2 broadcast against one another.
+    x = np.array([1])
+    # trick to define array with certain shape that doesn't allocate all the
+    # memory.
+    a = as_strided(x, shape=shp1, strides=[0] * len(shp1))
+    b = as_strided(x, shape=shp2, strides=[0] * len(shp2))
+    return np.broadcast(a, b).shape

setup.py

@@ -584,7 +584,7 @@ def _populate_hdf5_info(dirstosearch, inc_dirs, libs, lib_dirs):
 
 setup(name="netCDF4",
       cmdclass=cmdclass,
-      version="1.5.1",
+      version="1.5.1.1",
       long_description="netCDF version 4 has many features not found in earlier versions of the library, such as hierarchical groups, zlib compression, multiple unlimited dimensions, and new data types.  It is implemented on top of HDF5.  This module implements most of the new features, and can read and write netCDF files compatible with older versions of the library.  The API is modelled after Scientific.IO.NetCDF, and should be familiar to users of that module.\n\nThis project is hosted on a `GitHub repository <https://github.com/Unidata/netcdf4-python>`_ where you may access the most up-to-date source.",
       author="Jeff Whitaker",
       author_email="[email protected]",

test/tst_slicing.py

@@ -212,7 +212,7 @@ def test_issue743(self):
         nc.close()
 
     def test_issue906(self):
-        f = Dataset('test.nc','w')
+        f = Dataset(self.file,'w')
         f.createDimension('d1',3)
         f.createDimension('d2',None)
         f.createDimension('d3',5)
@@ -222,5 +222,18 @@ def test_issue906(self):
         f['v2'][0,:,:] = np.ones((4,5))
         f.close()
 
+    def test_issue919(self):
+        with Dataset(self.file,'w') as f:
+            f.createDimension('time',2)
+            f.createDimension('lat',10)
+            f.createDimension('lon',9)
+            f.createVariable('v1',np.int,('time', 'lon','lat',))
+            arr = np.arange(9*10).reshape((9, 10))
+            f['v1'][:] = arr
+            assert_array_equal(f['v1'][:],np.broadcast_to(arr,f['v1'].shape))
+            arr = np.arange(10)
+            f['v1'][:] = arr
+            assert_array_equal(f['v1'][:],np.broadcast_to(arr,f['v1'].shape))
+
 if __name__ == '__main__':
     unittest.main()

test/tst_utils.py

@@ -196,21 +196,21 @@ def test_ellipsis(self):
         assert_equal(start[0,0,0,0,0], [0, 0, 15, 0, 0])
         assert_equal(count[0,0,0,0,0], (2, 10, 5, 10, 10))
         assert_equal(put_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None)))
-        
+
         try:
             elem=(Ellipsis, [15,16,17,18,19], slice(None))
             start, count, stride, put_ind = _StartCountStride(elem, (2,10,20,10,10))
             assert_equal(None, 'Should throw an exception')
         except IndexError as e:
             assert_equal(str(e), "integer index exceeds dimension size")
-            
+
         try:
             elem=(Ellipsis, [15,16,17,18,19], Ellipsis)
             start, count, stride, put_ind = _StartCountStride(elem, (2,10, 20,10,10))
             assert_equal(None, 'Should throw an exception')
         except IndexError as e:
             assert_equal(str(e), "At most one ellipsis allowed in a slicing expression")
-            
+
 class TestsetStartCountStride(unittest.TestCase):
 
     def test_basic(self):
@@ -281,7 +281,7 @@ def test_unlim(self):
         #assert_equal(count[0][0][0], (5, 6, 7))
         #assert_equal(stride[0][0][0], (2, 1, 1))
         #assert_equal(take_ind[0][0][0], 3*(slice(None),))
-     
+
     def test_ellipsis(self):
         grp = FakeGroup({'x':False, 'y':False, 'time':True})
 
@@ -291,7 +291,7 @@ def test_ellipsis(self):
         assert_equal(start[0,0,0], [0, 0, 1])
         assert_equal(count[0,0,0], (22, 25, 3))
         assert_equal(take_ind[0,0,0], (slice(None), slice(None), slice(None)))
-        
+
         grp = FakeGroup({'time':True, 'h':False, 'z':False, 'y':False, 'x':False})
 
         elem=(Ellipsis, [15,16,17,18,19], slice(None), slice(None))
@@ -301,23 +301,25 @@ def test_ellipsis(self):
         assert_equal(count[0,0,0,0,0], [2, 10, 5, 10, 10])
         assert_equal(stride[0,0,0,0,0], [1, 1, 1, 1, 1])
         assert_equal(take_ind[0,0,0,0,0], (slice(None), slice(None), slice(None), slice(None), slice(None)))
-        
+
         try:
             elem=(Ellipsis, [15,16,17,18,19], slice(None))
             start, count, stride, take_ind = _StartCountStride(elem, (2,10,20,10,10),\
                ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True)
             assert_equal(None, 'Should throw an exception')
         except IndexError as e:
-            assert_equal(str(e), "integer index exceeds dimension size")
-
+            #assert_equal(str(e), "integer index exceeds dimension size")
+            assert_equal(str(e), "list index out of range")
+
         try:
             elem=(Ellipsis, [15,16,17,18,19], Ellipsis)
             start, count, stride, take_ind = _StartCountStride(elem, (2,10, 20,10,10),\
                ['time', 'z', 'y', 'x'], grp, (2,10,5,10,10), put=True)
             assert_equal(None, 'Should throw an exception')
         except IndexError as e:
-            assert_equal(str(e), "At most one ellipsis allowed in a slicing expression")
-
+            #assert_equal(str(e), "At most one ellipsis allowed in a slicing expression")
+            assert_equal(str(e), "list index out of range")
+
 class FakeGroup(object):
     """Create a fake group instance by passing a dictionary of booleans
     keyed by dimension name."""