From c81731a957b17b23bd1604c5f673f3ca01886f4d Mon Sep 17 00:00:00 2001 From: Dugal Harris Date: Sat, 10 Jun 2023 16:22:24 +0200 Subject: [PATCH 1/6] add username --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index bf6a142..55066a0 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,6 +1,6 @@ # These are supported funding model platforms -github: dugalh # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +github: [leftfield-geospatial, dugalh] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] patreon: # Replace with a single Patreon username open_collective: # Replace with a single Open Collective username ko_fi: # Replace with a single Ko-fi username From 1e162f8c500dafbca30cc51f9f13e7f0c6b1200a Mon Sep 17 00:00:00 2001 From: Dugal Harris Date: Sat, 10 Jun 2023 16:22:53 +0200 Subject: [PATCH 2/6] bump version --- geedim/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geedim/version.py b/geedim/version.py index 48c2f6b..8adfee4 100644 --- a/geedim/version.py +++ b/geedim/version.py @@ -1 +1 @@ -__version__ = '1.7.1' +__version__ = '1.7.2' From b2192e312e94a38d76a44b008f585560cf6ba4d4 Mon Sep 17 00:00:00 2001 From: Dugal Harris Date: Sat, 10 Jun 2023 16:36:34 +0200 Subject: [PATCH 3/6] build overviews without GDAL_NUM_THREADS='ALL_CPUs' --- geedim/download.py | 37 ++++++++++++++++++++++--------------- geedim/enums.py | 1 + 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/geedim/download.py b/geedim/download.py index 9ce8e8b..a10a1a1 100644 --- a/geedim/download.py +++ b/geedim/download.py @@ -613,19 +613,24 @@ def _get_tile_shape( tile_shape = tuple(tile_shape.tolist()) return tile_shape, num_tiles - @staticmethod - def _build_overviews(dataset: rio.io.DatasetWriter, max_num_levels: int = 8, min_ovw_pixels: int = 256): - """ Build internal overviews, downsampled by successive powers of 2, for an open rasterio dataset. """ - if dataset.closed: - raise IOError('Image dataset is closed') - - # limit overviews so that the highest level has at least 2**8=256 pixels along the shortest dimension, - # and so there are no more than 8 levels. - max_ovw_levels = int(np.min(np.log2(dataset.shape))) - min_level_shape_pow2 = int(np.log2(min_ovw_pixels)) - num_ovw_levels = np.min([max_num_levels, max_ovw_levels - min_level_shape_pow2]) - ovw_levels = [2**m for m in range(1, num_ovw_levels + 1)] - dataset.build_overviews(ovw_levels, RioResampling.average) + def _build_overviews(self, filename: Union[str, pathlib.Path], max_num_levels: int = 8, min_ovw_pixels: int = 256): + """ Build internal overviews, downsampled by successive powers of 2, for a given filename. """ + + # TODO: revisit multi-threaded overviews on rio/gdal update + # build overviews in a single threaded environment (currently rio/gdal reports errors when building overviews + # with GDAL_NUM_THREADS='ALL_CPUs') + env_dict = dict(GTIFF_FORCE_RGBA=False, COMPRESS_OVERVIEW='DEFLATE') + if self.size >= 4e9: + env_dict.update(BIGTIFF_OVERVIEW=True) + + with rio.Env(**env_dict), rio.open(filename, 'r+') as ds: + # limit overviews so that the highest level has at least 2**8=256 pixels along the shortest dimension, + # and so there are no more than 8 levels. + max_ovw_levels = int(np.min(np.log2(ds.shape))) + min_level_shape_pow2 = int(np.log2(min_ovw_pixels)) + num_ovw_levels = np.min([max_num_levels, max_ovw_levels - min_level_shape_pow2]) + ovw_levels = [2**m for m in range(1, num_ovw_levels + 1)] + ds.build_overviews(ovw_levels, RioResampling.average) def _write_metadata(self, dataset: rio.io.DatasetWriter): """ Write Earth Engine and STAC metadata to an open rasterio dataset. """ @@ -949,6 +954,8 @@ def download_tile(tile): raise ex bar.update(bar.total - bar.n) # ensure the bar reaches 100% - # populate GeoTIFF metadata and build overviews + # populate GeoTIFF metadata exp_image._write_metadata(out_ds) - BaseImage._build_overviews(out_ds) + + # build overviews + exp_image._build_overviews(filename) diff --git a/geedim/enums.py b/geedim/enums.py index 88dc417..71e0288 100644 --- a/geedim/enums.py +++ b/geedim/enums.py @@ -82,6 +82,7 @@ class ExportType(str, Enum): cloud = 'cloud' """ Export to Google Cloud Storage. """ + class SpectralDistanceMetric(str, Enum): """ Enumeration for the spectral distance metric. """ sam = 'sam' From 58b6b393211f11c37aa66862e2deae5f7fab3866 Mon Sep 17 00:00:00 2001 From: Dugal Harris Date: Sat, 10 Jun 2023 16:56:40 +0200 Subject: [PATCH 4/6] bump version --- .github/workflows/install-test-conda-forge.yml | 2 +- .github/workflows/run-integration-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/install-test-conda-forge.yml b/.github/workflows/install-test-conda-forge.yml index 8c19b90..9fe902e 100644 --- a/.github/workflows/install-test-conda-forge.yml +++ b/.github/workflows/install-test-conda-forge.yml @@ -27,7 +27,7 @@ jobs: run: | mamba info mamba list - mamba install -c conda-forge geedim>=1.7.1 + mamba install -c conda-forge geedim>=1.7.2 mamba list - name: Test geedim timeout-minutes: 5 diff --git a/.github/workflows/run-integration-tests.yml b/.github/workflows/run-integration-tests.yml index 415dc03..b4e612f 100644 --- a/.github/workflows/run-integration-tests.yml +++ b/.github/workflows/run-integration-tests.yml @@ -27,7 +27,7 @@ jobs: - name: Install geedim run: | mamba info - mamba install -c conda-forge geedim=1.7.1 pytest + mamba install -c conda-forge geedim=1.7.2 pytest # force an update to the latest versions mamba update -c conda-forge geedim --force-reinstall mamba list From 0aec16ae6f0e977da77d4278d737d4d1adcb0c91 Mon Sep 17 00:00:00 2001 From: Dugal Harris Date: Sat, 10 Jun 2023 20:21:03 +0200 Subject: [PATCH 5/6] add logging level argument suppress_rio_logs --- geedim/tile.py | 2 +- geedim/utils.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/geedim/tile.py b/geedim/tile.py index b37e940..e80289b 100644 --- a/geedim/tile.py +++ b/geedim/tile.py @@ -123,7 +123,7 @@ def download(self, session=None, response=None, bar: tqdm = None): # read the geotiff with a rasterio memory file env = rio.Env(GDAL_NUM_THREADS='ALL_CPUs', GTIFF_FORCE_RGBA=False) - with utils.suppress_rio_warn_logs(), env, MemoryFile(ext_buffer) as mem_file: + with utils.suppress_rio_logs(), env, MemoryFile(ext_buffer) as mem_file: with mem_file.open() as ds: array = ds.read() if (array.dtype == np.dtype('float32')) or (array.dtype == np.dtype('float64')): diff --git a/geedim/utils.py b/geedim/utils.py index f8182fd..34149c1 100644 --- a/geedim/utils.py +++ b/geedim/utils.py @@ -109,13 +109,13 @@ def split_id(image_id: str) -> Tuple[str, str]: @contextmanager -def suppress_rio_warn_logs(): - """ A context manager that sets the `rasterio` logging level to ERROR, then returns it to its original value. """ +def suppress_rio_logs(level: int = logging.ERROR): + """ A context manager that sets the `rasterio` logging level, then returns it to its original value. """ try: # GEE sets GeoTIFF `colorinterp` tags incorrectly. This suppresses `rasterio` warning relating to this: # 'Sum of Photometric type-related color channels and ExtraSamples doesn't match SamplesPerPixel' rio_level = logging.getLogger('rasterio').getEffectiveLevel() - logging.getLogger('rasterio').setLevel(logging.ERROR) + logging.getLogger('rasterio').setLevel(level) yield finally: logging.getLogger('rasterio').setLevel(rio_level) @@ -137,7 +137,7 @@ def get_bounds(filename: pathlib.Path, expand: float = 5): dict Geojson polygon. """ - with suppress_rio_warn_logs(), rio.Env(GTIFF_FORCE_RGBA=False), rio.open(filename) as im: + with suppress_rio_logs(), rio.Env(GTIFF_FORCE_RGBA=False), rio.open(filename) as im: bbox = im.bounds if (im.crs.linear_units == "metre") and (expand > 0): # expand the bounding box expand_x = (bbox.right - bbox.left) * expand / 100.0 From fdfd5ec1b90e1f596950f1ba2aef8a1db3b1f60d Mon Sep 17 00:00:00 2001 From: Dugal Harris Date: Sat, 10 Jun 2023 20:22:04 +0200 Subject: [PATCH 6/6] reference gdal issue --- geedim/download.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/geedim/download.py b/geedim/download.py index a10a1a1..063e109 100644 --- a/geedim/download.py +++ b/geedim/download.py @@ -616,9 +616,9 @@ def _get_tile_shape( def _build_overviews(self, filename: Union[str, pathlib.Path], max_num_levels: int = 8, min_ovw_pixels: int = 256): """ Build internal overviews, downsampled by successive powers of 2, for a given filename. """ - # TODO: revisit multi-threaded overviews on rio/gdal update - # build overviews in a single threaded environment (currently rio/gdal reports errors when building overviews - # with GDAL_NUM_THREADS='ALL_CPUs') + # TODO: revisit multi-threaded overviews on gdal update + # build overviews in a single threaded environment (currently gdal reports errors when building overviews + # with GDAL_NUM_THREADS='ALL_CPUs' - see https://github.com/OSGeo/gdal/issues/7921) env_dict = dict(GTIFF_FORCE_RGBA=False, COMPRESS_OVERVIEW='DEFLATE') if self.size >= 4e9: env_dict.update(BIGTIFF_OVERVIEW=True) @@ -630,7 +630,8 @@ def _build_overviews(self, filename: Union[str, pathlib.Path], max_num_levels: i min_level_shape_pow2 = int(np.log2(min_ovw_pixels)) num_ovw_levels = np.min([max_num_levels, max_ovw_levels - min_level_shape_pow2]) ovw_levels = [2**m for m in range(1, num_ovw_levels + 1)] - ds.build_overviews(ovw_levels, RioResampling.average) + with utils.suppress_rio_logs(logging.CRITICAL): + ds.build_overviews(ovw_levels, RioResampling.average) def _write_metadata(self, dataset: rio.io.DatasetWriter): """ Write Earth Engine and STAC metadata to an open rasterio dataset. """