From ee9dc74058c16f054c2d87991c724ad0c42b3887 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Tue, 8 Apr 2025 23:00:26 +0100 Subject: [PATCH 1/5] dev --- h5netcdf/core.py | 7 ++++++- h5netcdf/dimensions.py | 40 ++++++++++++++++++++++++++-------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/h5netcdf/core.py b/h5netcdf/core.py index dd7b5094..4f87c059 100644 --- a/h5netcdf/core.py +++ b/h5netcdf/core.py @@ -195,14 +195,19 @@ def _lookup_dimensions(self): # coordinate variable and dimension, eg. 1D ("time") or 2D string variable if ( "_Netcdf4Coordinates" in attrs - and attrs.get("CLASS", None) == b"DIMENSION_SCALE" + and (attrs.get("CLASS", None) == b"DMENSION_SCALE" + or self._parent.backend == "pyfive") ): + # Note: For the pyfive backend, if we can use this method + # then it is much faster than using the + # DIMENSION_LIST method order_dim = { value._dimid: key for key, value in self._parent._all_dimensions.items() } return tuple( order_dim[coord_id] for coord_id in attrs["_Netcdf4Coordinates"] ) + # normal variable carrying DIMENSION_LIST # extract hdf5 file references and get objects name if "DIMENSION_LIST" in attrs: diff --git a/h5netcdf/dimensions.py b/h5netcdf/dimensions.py index 1e3ba986..49cbbae6 100644 --- a/h5netcdf/dimensions.py +++ b/h5netcdf/dimensions.py @@ -105,17 +105,23 @@ def name(self): @property def size(self): """Return dimension size.""" - size = len(self) - if self.isunlimited(): - # return actual dimensions sizes, this is in line with netcdf4-python - # get sizes from all connected variables and calculate max - # because netcdf unlimited dimensions can be any length - # but connected variables dimensions can have a certain larger length. - reflist = self._h5ds.attrs.get("REFERENCE_LIST", None) - if reflist is not None: - for ref, axis in reflist: - var = self._parent._h5group["/"][ref] - size = max(var.shape[axis], size) + try: + size = self._cached_size + except AttributeError: + size = len(self) + if not self.isunlimited(): + self._cached_size = size + else: + # return actual dimensions sizes, this is in line with netcdf4-python + # get sizes from all connected variables and calculate max + # because netcdf unlimited dimensions can be any length + # but connected variables dimensions can have a certain larger length. + reflist = self._h5ds.attrs.get("REFERENCE_LIST", None) + if reflist is not None: + for ref, axis in reflist: + var = self._parent._h5group["/"][ref] + size = max(var.shape[axis], size) + return size def group(self): @@ -124,9 +130,15 @@ def group(self): def isunlimited(self): """Return ``True`` if dimension is unlimited, otherwise ``False``.""" - if self._phony: - return False - return self._h5ds.maxshape == (None,) + try: + return self._cached_isunlimited + except AttributeError: + if self._phony: + return False + + isunlimited = self._h5ds.maxshape == (None,) + self._cached_isunlimited = isunlimited + return isunlimited @property def _h5ds(self): From f447cc04e8d8027fd58c12c5a1068735f7b7ba30 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 11 Apr 2025 13:49:38 +0100 Subject: [PATCH 2/5] dev --- h5netcdf/core.py | 43 ++++++++++++++++++++++++++++++++++-------- h5netcdf/dimensions.py | 17 ++++++++++++++--- 2 files changed, 49 insertions(+), 11 deletions(-) diff --git a/h5netcdf/core.py b/h5netcdf/core.py index 4f87c059..d5a8903f 100644 --- a/h5netcdf/core.py +++ b/h5netcdf/core.py @@ -140,6 +140,13 @@ def _root(self): def _h5ds(self): # Always refer to the root file and store not h5py object # subclasses: +# print('arse',self._parent.backend == "pyfive") + # if self._parent.backend == "pyfive": + # + # h5ds = self._root._h5file[self._h5path] + # else: + # h5ds = self._root._h5file[self._h5path] + return self._root._h5file[self._h5path] @property @@ -191,30 +198,50 @@ def name(self): return super().name.replace("_nc4_non_coord_", "") def _lookup_dimensions(self): - attrs = self._h5ds.attrs + # Use cached attributes, if available. + try: + attrs = self._attrs + print ('cached_attrs') + except AttributeError: + attrs = self._h5ds.attrs + print ('BAD attr;') + # coordinate variable and dimension, eg. 1D ("time") or 2D string variable + print ('getting dimes') if ( "_Netcdf4Coordinates" in attrs and (attrs.get("CLASS", None) == b"DMENSION_SCALE" or self._parent.backend == "pyfive") ): - # Note: For the pyfive backend, if we can use this method - # then it is much faster than using the - # DIMENSION_LIST method - order_dim = { - value._dimid: key for key, value in self._parent._all_dimensions.items() - } + # Note: For the pyfive backend, if "_Netcdf4Coordinates" + # exist then we should use this method as it is much + # faster than using the DIMENSION_LIST method + + # Use cached order_dims, if available. + try: + order_dim = self._parent._order_dim + print ('cached_order_dim') + except AttributeError: + print (22222000, list(self._parent._all_dimensions)) + print ('------') + order_dim = { + value._dimid: key for key, value in self._parent._all_dimensions.items() + } + print (22222) + self._parent._order_dim = order_dim + print (11111) return tuple( order_dim[coord_id] for coord_id in attrs["_Netcdf4Coordinates"] ) # normal variable carrying DIMENSION_LIST # extract hdf5 file references and get objects name - if "DIMENSION_LIST" in attrs: + if "DIMENSION_LIST" in attrs: # check if malformed variable and raise if _unlabeled_dimension_mix(self._h5ds) == "labeled": # If a dimension has attached more than one scale for some reason, then # take the last one. This is in line with netcdf-c and netcdf4-python. + print ('w/. LIST') return tuple( self._root._h5file[ref[-1]].name.split("/")[-1] for ref in list(self._h5ds.attrs.get("DIMENSION_LIST", [])) diff --git a/h5netcdf/dimensions.py b/h5netcdf/dimensions.py index 49cbbae6..1f0f1087 100644 --- a/h5netcdf/dimensions.py +++ b/h5netcdf/dimensions.py @@ -152,9 +152,20 @@ def _isscale(self): @property def _dimid(self): - if self._phony: - return False - return self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) +# if self._phony: +# return False +# return self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) + try: + return self._cached_dimid + except AttributeError: + if self._phony: + dimid = False + else: + dimid = self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) + print ('caching') + self._cached_dimid = dimid + return dimid + def _resize(self, size): from .legacyapi import Dataset From 84aab1a89ebd1861198ca8dc797a310b70713305 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 18 Apr 2025 18:26:36 +0100 Subject: [PATCH 3/5] dev --- h5netcdf/core.py | 50 +++++++++++++++++++----------------------- h5netcdf/dimensions.py | 6 +++-- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/h5netcdf/core.py b/h5netcdf/core.py index d5a8903f..23891144 100644 --- a/h5netcdf/core.py +++ b/h5netcdf/core.py @@ -140,14 +140,13 @@ def _root(self): def _h5ds(self): # Always refer to the root file and store not h5py object # subclasses: -# print('arse',self._parent.backend == "pyfive") - # if self._parent.backend == "pyfive": - # - # h5ds = self._root._h5file[self._h5path] - # else: - # h5ds = self._root._h5file[self._h5path] - - return self._root._h5file[self._h5path] + try: + return self._cached_h5ds + except AttributeError: + h = self._root._h5file[self._h5path] + self._cached_h5ds = h + print ('Getting self._h5ds', repr(h)) + return h @property def name(self): @@ -199,19 +198,16 @@ def name(self): def _lookup_dimensions(self): # Use cached attributes, if available. - try: - attrs = self._attrs - print ('cached_attrs') - except AttributeError: - attrs = self._h5ds.attrs - print ('BAD attr;') - + + # Get the original attributes + print ('_lookup_dimensions') + attrs = self._h5ds.attrs + # coordinate variable and dimension, eg. 1D ("time") or 2D string variable - print ('getting dimes') if ( "_Netcdf4Coordinates" in attrs - and (attrs.get("CLASS", None) == b"DMENSION_SCALE" - or self._parent.backend == "pyfive") +# and (attrs.get("CLASS", None) == b"DMENSION_SCALE" +# or self._parent.backend == "pyfive") ): # Note: For the pyfive backend, if "_Netcdf4Coordinates" # exist then we should use this method as it is much @@ -220,16 +216,14 @@ def _lookup_dimensions(self): # Use cached order_dims, if available. try: order_dim = self._parent._order_dim - print ('cached_order_dim') + print ('using cache order_dims') except AttributeError: - print (22222000, list(self._parent._all_dimensions)) - print ('------') order_dim = { value._dimid: key for key, value in self._parent._all_dimensions.items() } - print (22222) + print ('Getting order_dim') self._parent._order_dim = order_dim - print (11111) + print ('------') return tuple( order_dim[coord_id] for coord_id in attrs["_Netcdf4Coordinates"] ) @@ -242,16 +236,18 @@ def _lookup_dimensions(self): # If a dimension has attached more than one scale for some reason, then # take the last one. This is in line with netcdf-c and netcdf4-python. print ('w/. LIST') + h5file = self._root._h5file return tuple( - self._root._h5file[ref[-1]].name.split("/")[-1] - for ref in list(self._h5ds.attrs.get("DIMENSION_LIST", [])) + h5file[ref[-1]].name.split("/")[-1] + for ref in list(attrs.get("DIMENSION_LIST", [])) ) # need to use the h5ds name here to distinguish from collision dimensions + child_name = self._h5ds.name.split("/")[-1] if child_name in self._parent._all_dimensions: return (child_name,) - + dims = [] phony_dims = defaultdict(int) for axis, dim in enumerate(self._h5ds.dims): @@ -1325,7 +1321,7 @@ def __init__(self, path, mode="r", invalid_netcdf=False, phony_dims=None, backen if backend == 'pyfive': self._h5py = pyfive - logging.info(f'h5netcdf running with {pyfive.__version__}') + logging.info(f"h5netcdf running with {pyfive.__version__}") try: # We can ignore track order for now (and maybe for reading in general)? if kwargs: diff --git a/h5netcdf/dimensions.py b/h5netcdf/dimensions.py index 1f0f1087..342efcc9 100644 --- a/h5netcdf/dimensions.py +++ b/h5netcdf/dimensions.py @@ -156,13 +156,15 @@ def _dimid(self): # return False # return self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) try: - return self._cached_dimid + x = self._cached_dimid + print ('using cached _dimid') + return x except AttributeError: if self._phony: dimid = False else: dimid = self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) - print ('caching') + print ('caching _dimid') self._cached_dimid = dimid return dimid From d48a6adfc423e95e9090d1c5d9514b35857bab0c Mon Sep 17 00:00:00 2001 From: David Hassell Date: Sat, 19 Apr 2025 15:06:42 +0100 Subject: [PATCH 4/5] dev --- h5netcdf/core.py | 14 +++++++++---- h5netcdf/dimensions.py | 45 +++++++++++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/h5netcdf/core.py b/h5netcdf/core.py index 23891144..6120fc43 100644 --- a/h5netcdf/core.py +++ b/h5netcdf/core.py @@ -752,13 +752,16 @@ def __init__(self, parent, name): phony_dims = Counter() pyfive_backend = isinstance(self._h5group, pyfive.Group) - for k in self._h5group: + + h5group = self._h5group + h5group._iii = False + for k in h5group: #with warnings.catch_warnings(record=True) as wlist: try: - v = self._h5group[k] + v = h5group[k] except Exception as e: if pyfive_backend: - warnings.warn(f'Skipping {k} - {e}') + warnings.warn(f"Skipping {k} - {e}") continue else: raise @@ -789,10 +792,12 @@ def __init__(self, parent, name): self._variables.add(k) except: if pyfive_backend: - warnings.warn(f'Cannot read {k}') + warnings.warn(f"Cannot read {k}") else: raise + del h5group._iii + # iterate over found phony dimensions and create them if self._root._phony_dims_mode is not None: # retrieve labeled dims count from already acquired dimensions @@ -1414,6 +1419,7 @@ def __init__(self, path, mode="r", invalid_netcdf=False, phony_dims=None, backen # This maps keeps track of all HDF5 datasets corresponding to this group. self._all_h5groups = ChainMap(self._h5group) super().__init__(self, self._h5path) + # get maximum dimension id and count of labeled dimensions if self._writable: self._max_dim_id = self._get_maximum_dimension_id() diff --git a/h5netcdf/dimensions.py b/h5netcdf/dimensions.py index 342efcc9..4a7a8dfa 100644 --- a/h5netcdf/dimensions.py +++ b/h5netcdf/dimensions.py @@ -144,7 +144,20 @@ def isunlimited(self): def _h5ds(self): if self._phony: return None - return self._root._h5file[self._h5path] + + try: + return self._cached_h5ds + except AttributeError: + h5file = self._root._h5file + h5file._iii = False + h = h5file[self._h5path] + del h5file._iii + self._cached_h5ds = h + print ('Getting self._h5ds', repr(h)) + return h + + +# return self._root._h5file[self._h5path] @property def _isscale(self): @@ -152,21 +165,21 @@ def _isscale(self): @property def _dimid(self): -# if self._phony: -# return False -# return self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) - try: - x = self._cached_dimid - print ('using cached _dimid') - return x - except AttributeError: - if self._phony: - dimid = False - else: - dimid = self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) - print ('caching _dimid') - self._cached_dimid = dimid - return dimid + if self._phony: + return False + return self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) +# try: +# x = self._cached_dimid +# print ('using cached _dimid') +# return x +# except AttributeError: +# if self._phony: +# dimid = False +# else: +# dimid = self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) +# print ('caching _dimid') +# self._cached_dimid = dimid +# return dimid def _resize(self, size): From d3fc7c07a5c137020dd9b107920c01568ef54a64 Mon Sep 17 00:00:00 2001 From: David Hassell Date: Fri, 25 Apr 2025 17:01:27 +0100 Subject: [PATCH 5/5] caching --- h5netcdf/core.py | 111 +++++++++++++++++++++++++++++++---------- h5netcdf/dimensions.py | 51 +++++++++---------- test.nc | Bin 427 -> 427 bytes 3 files changed, 109 insertions(+), 53 deletions(-) diff --git a/h5netcdf/core.py b/h5netcdf/core.py index 6120fc43..2c690543 100644 --- a/h5netcdf/core.py +++ b/h5netcdf/core.py @@ -140,12 +140,19 @@ def _root(self): def _h5ds(self): # Always refer to the root file and store not h5py object # subclasses: + if "legacy" in self._cls_name: + # Haven't yet worked how to do caching with the legacy + # API (because as things stand we end up with + # recursion in HasAttributesMixin.__[gs]etattr__ when + # we try to get/set the _cached_h5ds attribute). + return self._root._h5file[self._h5path] + try: + # Try to get from cache return self._cached_h5ds except AttributeError: h = self._root._h5file[self._h5path] self._cached_h5ds = h - print ('Getting self._h5ds', repr(h)) return h @property @@ -197,57 +204,60 @@ def name(self): return super().name.replace("_nc4_non_coord_", "") def _lookup_dimensions(self): - # Use cached attributes, if available. - - # Get the original attributes - print ('_lookup_dimensions') attrs = self._h5ds.attrs - # coordinate variable and dimension, eg. 1D ("time") or 2D string variable if ( - "_Netcdf4Coordinates" in attrs -# and (attrs.get("CLASS", None) == b"DMENSION_SCALE" -# or self._parent.backend == "pyfive") + "_Netcdf4Coordinates" in attrs + and ( + attrs.get("CLASS", None) == b"DIMENSION_SCALE" + or self._parent.backend == "pyfive" + ) ): # Note: For the pyfive backend, if "_Netcdf4Coordinates" # exist then we should use this method as it is much # faster than using the DIMENSION_LIST method - - # Use cached order_dims, if available. try: - order_dim = self._parent._order_dim - print ('using cache order_dims') + # Try using cached order_dim + if "legacy" in self._cls_name: + # Haven't yet worked how to do caching with the + # legacy API (because as things stand we end up + # with recursion in + # HasAttributesMixin.__[gs]etattr__ when we try to + # get/set the _cached_h5ds attribute). + raise AttributeError + + order_dim = self._parent._cached_order_dim except AttributeError: order_dim = { value._dimid: key for key, value in self._parent._all_dimensions.items() } - print ('Getting order_dim') - self._parent._order_dim = order_dim - print ('------') + if "legacy" not in self._cls_name: + # Cache order_dim + self._parent._cached_order_dim = order_dim + return tuple( order_dim[coord_id] for coord_id in attrs["_Netcdf4Coordinates"] ) # normal variable carrying DIMENSION_LIST # extract hdf5 file references and get objects name - if "DIMENSION_LIST" in attrs: + if "DIMENSION_LIST" in attrs: # check if malformed variable and raise if _unlabeled_dimension_mix(self._h5ds) == "labeled": # If a dimension has attached more than one scale for some reason, then # take the last one. This is in line with netcdf-c and netcdf4-python. - print ('w/. LIST') h5file = self._root._h5file return tuple( - h5file[ref[-1]].name.split("/")[-1] + self._root._h5file[ref[-1]].name.split("/")[-1] for ref in list(attrs.get("DIMENSION_LIST", [])) ) # need to use the h5ds name here to distinguish from collision dimensions - + child_name = self._h5ds.name.split("/")[-1] if child_name in self._parent._all_dimensions: return (child_name,) - + dims = [] phony_dims = defaultdict(int) for axis, dim in enumerate(self._h5ds.dims): @@ -363,6 +373,7 @@ def dimensions(self): """Return variable dimension names.""" if self._dimensions is None: self._dimensions = self._lookup_dimensions() + return self._dimensions @property @@ -751,16 +762,21 @@ def __init__(self, parent, name): if self._root._phony_dims_mode is not None: phony_dims = Counter() - pyfive_backend = isinstance(self._h5group, pyfive.Group) + if isinstance(self._h5group, pyfive.Group): + self.backend = "pyfive" + else: + self.backend = "h5py" + # No need to build chunk index when building the File view + # (only affects the pyfive backend) h5group = self._h5group - h5group._iii = False + h5group._build_chunk_index = False for k in h5group: #with warnings.catch_warnings(record=True) as wlist: try: v = h5group[k] except Exception as e: - if pyfive_backend: + if self.backend == "pyfive": warnings.warn(f"Skipping {k} - {e}") continue else: @@ -791,12 +807,53 @@ def __init__(self, parent, name): if isinstance(v, self._root._h5py.Dataset): self._variables.add(k) except: - if pyfive_backend: + if self.backend == "pyfive": warnings.warn(f"Cannot read {k}") else: raise - del h5group._iii + for k in h5group: + #with warnings.catch_warnings(record=True) as wlist: + try: + v = h5group[k] + except Exception as e: + if self.backend == "pyfive": + warnings.warn(f"Skipping {k} - {e}") + continue + else: + raise + + if isinstance(v, self._root._h5py.Group): + # add to the groups collection if this is a h5py(d) Group + # instance + self._groups.add(k) + # todo: add other user types here + elif isinstance( + v, self._root._h5py.Datatype + ) and self._root._h5py.check_enum_dtype(v.dtype): + self._enumtypes.add(k) + else: + try: + if v.attrs.get("CLASS") == b"DIMENSION_SCALE": + # add dimension and retrieve size + self._dimensions.add(k) + else: + if self._root._phony_dims_mode is not None: + # check if malformed variable and raise + if _unlabeled_dimension_mix(v) == "unlabeled": + # if unscaled variable, get phony dimensions + phony_dims |= Counter(v.shape) + + if not _netcdf_dimension_but_not_variable(v): + if isinstance(v, self._root._h5py.Dataset): + self._variables.add(k) + except: + if self.backend == "pyfive": + warnings.warn(f"Cannot read {k}") + else: + raise + + del h5group._build_chunk_index # iterate over found phony dimensions and create them if self._root._phony_dims_mode is not None: @@ -1104,6 +1161,7 @@ def create_variable( if fillvalue is None and isinstance(self._parent._root, Dataset): fillvalue = _get_default_fillvalue(dtype) + return self._root.create_variable( name[1:], dimensions, @@ -1121,6 +1179,7 @@ def create_variable( group = self for k in keys[:-1]: group = group._require_child_group(k) + return group._create_child_variable( keys[-1], dimensions, diff --git a/h5netcdf/dimensions.py b/h5netcdf/dimensions.py index 4a7a8dfa..a7fb8388 100644 --- a/h5netcdf/dimensions.py +++ b/h5netcdf/dimensions.py @@ -142,22 +142,32 @@ def isunlimited(self): @property def _h5ds(self): + # Note on caching: + # + # Caching the h5ds (as is done with `BaseObject._h5ds`) can + # give performance improvements, but causes as yet not + # understood problems in the case that there is a + # multidimensional variable for which one of its dimensions + # has the same name as the variable itself. For instance + # (taken from the `write_h5netcdf` function in the tests): + # + # dimensions: + # z = 6 ; + # string3 = 3 ; + # + # variables: + # char z(z, string3) ; + if self._phony: return None - try: - return self._cached_h5ds - except AttributeError: - h5file = self._root._h5file - h5file._iii = False - h = h5file[self._h5path] - del h5file._iii - self._cached_h5ds = h - print ('Getting self._h5ds', repr(h)) - return h - - -# return self._root._h5file[self._h5path] + # No need to build chunk index for a dimension (only affects + # the pyfive backend) + h5file = self._root._h5file + h5file._build_chunk_index = False + h = h5file[self._h5path] + del h5file._build_chunk_index + return h @property def _isscale(self): @@ -168,26 +178,13 @@ def _dimid(self): if self._phony: return False return self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) -# try: -# x = self._cached_dimid -# print ('using cached _dimid') -# return x -# except AttributeError: -# if self._phony: -# dimid = False -# else: -# dimid = self._h5ds.attrs.get("_Netcdf4Dimid", self._dimensionid) -# print ('caching _dimid') -# self._cached_dimid = dimid -# return dimid - def _resize(self, size): from .legacyapi import Dataset if not self.isunlimited(): raise ValueError( - f"Dimension '{self.name}' is not unlimited and thus cannot be resized." + f"Dimension {self.name!r} is not unlimited and thus cannot be resized." ) self._h5ds.resize((size,)) diff --git a/test.nc b/test.nc index 24b233f6d82ee7c2376c1d59558031ba449009ca..c27811db2671cfff092c2129d25f0adacc1af4b1 100644 GIT binary patch delta 67 ycmZ3@yqbAJn!vJ++!@GtV%b3z6P*mxf=XLsJwqcsLk3<3bnwaM@3qO%i~#`e`V$HO delta 67 ycmZ3@yqbAJnn0Pw-E?F;vFxCVnNEgjL8Yy+o}sax0Rt}sI@sNkYBf2UF#rGu9}