From 447e5a3d16764a880387d33d0b5938393e167817 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Tue, 4 Jun 2024 11:23:56 -0600 Subject: [PATCH] Speed up netCDF4, h5netcdf backends (#9067) * Speed up netCDF4 backend. xref #9058 Accessing `.shape` on a netCDF4 variable is ~20-40ms. This can add up for large numbers of variables, e.g.: https://github.com/pydata/xarray/discussions/9058 We already request the shape when creating NetCDF4ArrayWrapper, so we can reuse that. * Update h5netcdf backend too --- doc/whats-new.rst | 6 ++++++ xarray/backends/h5netcdf_.py | 2 +- xarray/backends/netCDF4_.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 49d39927f2b..6a97ceaff00 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -23,6 +23,12 @@ v2024.05.1 (unreleased) New Features ~~~~~~~~~~~~ +Performance +~~~~~~~~~~~ + +- Small optimization to the netCDF4 and h5netcdf backends (:issue:`9058`, :pull:`9067`). + By `Deepak Cherian `_. + Breaking changes ~~~~~~~~~~~~~~~~ diff --git a/xarray/backends/h5netcdf_.py b/xarray/backends/h5netcdf_.py index 71463193939..1993d5b19de 100644 --- a/xarray/backends/h5netcdf_.py +++ b/xarray/backends/h5netcdf_.py @@ -221,7 +221,7 @@ def open_store_variable(self, name, var): # save source so __repr__ can detect if it's local or not encoding["source"] = self._filename - encoding["original_shape"] = var.shape + encoding["original_shape"] = data.shape vlen_dtype = h5py.check_dtype(vlen=var.dtype) if vlen_dtype is str: diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index ae86c4ce384..1edf57c176e 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -454,7 +454,7 @@ def open_store_variable(self, name: str, var): pop_to(attributes, encoding, "least_significant_digit") # save source so __repr__ can detect if it's local or not encoding["source"] = self._filename - encoding["original_shape"] = var.shape + encoding["original_shape"] = data.shape return Variable(dimensions, data, attributes, encoding)