Merge pull request #19 from leftfield-geospatial/fix-overview_error

Work around overview error

.github/FUNDING.yml

@@ -1,6 +1,6 @@
 # These are supported funding model platforms
 
-github: dugalh # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+github: [leftfield-geospatial, dugalh] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
 patreon: # Replace with a single Patreon username
 open_collective: # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username

.github/workflows/install-test-conda-forge.yml

@@ -27,7 +27,7 @@ jobs:
       run: |
         mamba info
         mamba list
-        mamba install -c conda-forge geedim>=1.7.1
+        mamba install -c conda-forge geedim>=1.7.2
         mamba list
     - name: Test geedim
       timeout-minutes: 5

.github/workflows/run-integration-tests.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Install geedim
         run: |
           mamba info
-          mamba install -c conda-forge geedim=1.7.1 pytest
+          mamba install -c conda-forge geedim=1.7.2 pytest
           # force an update to the latest versions
           mamba update -c conda-forge geedim --force-reinstall
           mamba list

geedim/download.py

@@ -613,19 +613,25 @@ def _get_tile_shape(
         tile_shape = tuple(tile_shape.tolist())
         return tile_shape, num_tiles
 
-    @staticmethod
-    def _build_overviews(dataset: rio.io.DatasetWriter, max_num_levels: int = 8, min_ovw_pixels: int = 256):
-        """ Build internal overviews, downsampled by successive powers of 2, for an open rasterio dataset. """
-        if dataset.closed:
-            raise IOError('Image dataset is closed')
-
-        # limit overviews so that the highest level has at least 2**8=256 pixels along the shortest dimension,
-        # and so there are no more than 8 levels.
-        max_ovw_levels = int(np.min(np.log2(dataset.shape)))
-        min_level_shape_pow2 = int(np.log2(min_ovw_pixels))
-        num_ovw_levels = np.min([max_num_levels, max_ovw_levels - min_level_shape_pow2])
-        ovw_levels = [2**m for m in range(1, num_ovw_levels + 1)]
-        dataset.build_overviews(ovw_levels, RioResampling.average)
+    def _build_overviews(self, filename: Union[str, pathlib.Path], max_num_levels: int = 8, min_ovw_pixels: int = 256):
+        """ Build internal overviews, downsampled by successive powers of 2, for a given filename. """
+
+        # TODO: revisit multi-threaded overviews on gdal update
+        # build overviews in a single threaded environment (currently gdal reports errors when building overviews
+        # with GDAL_NUM_THREADS='ALL_CPUs' - see https://github.com/OSGeo/gdal/issues/7921)
+        env_dict = dict(GTIFF_FORCE_RGBA=False, COMPRESS_OVERVIEW='DEFLATE')
+        if self.size >= 4e9:
+            env_dict.update(BIGTIFF_OVERVIEW=True)
+
+        with rio.Env(**env_dict), rio.open(filename, 'r+') as ds:
+            # limit overviews so that the highest level has at least 2**8=256 pixels along the shortest dimension,
+            # and so there are no more than 8 levels.
+            max_ovw_levels = int(np.min(np.log2(ds.shape)))
+            min_level_shape_pow2 = int(np.log2(min_ovw_pixels))
+            num_ovw_levels = np.min([max_num_levels, max_ovw_levels - min_level_shape_pow2])
+            ovw_levels = [2**m for m in range(1, num_ovw_levels + 1)]
+            with utils.suppress_rio_logs(logging.CRITICAL):
+                ds.build_overviews(ovw_levels, RioResampling.average)
 
     def _write_metadata(self, dataset: rio.io.DatasetWriter):
         """ Write Earth Engine and STAC metadata to an open rasterio dataset. """
@@ -949,6 +955,8 @@ def download_tile(tile):
                     raise ex
 
             bar.update(bar.total - bar.n)   # ensure the bar reaches 100%
-            # populate GeoTIFF metadata and build overviews
+            # populate GeoTIFF metadata
             exp_image._write_metadata(out_ds)
-            BaseImage._build_overviews(out_ds)
+
+        # build overviews
+        exp_image._build_overviews(filename)

geedim/enums.py

@@ -82,6 +82,7 @@ class ExportType(str, Enum):
     cloud = 'cloud'
     """ Export to Google Cloud Storage. """
 
+
 class SpectralDistanceMetric(str, Enum):
     """ Enumeration for the spectral distance metric. """
     sam = 'sam'

geedim/tile.py

@@ -123,7 +123,7 @@ def download(self, session=None, response=None, bar: tqdm = None):
 
         # read the geotiff with a rasterio memory file
         env = rio.Env(GDAL_NUM_THREADS='ALL_CPUs', GTIFF_FORCE_RGBA=False)
-        with utils.suppress_rio_warn_logs(), env, MemoryFile(ext_buffer) as mem_file:
+        with utils.suppress_rio_logs(), env, MemoryFile(ext_buffer) as mem_file:
             with mem_file.open() as ds:
                 array = ds.read()
                 if (array.dtype == np.dtype('float32')) or (array.dtype == np.dtype('float64')):

geedim/utils.py

@@ -109,13 +109,13 @@ def split_id(image_id: str) -> Tuple[str, str]:
 
 
 @contextmanager
-def suppress_rio_warn_logs():
-    """ A context manager that sets the `rasterio` logging level to ERROR, then returns it to its original value. """
+def suppress_rio_logs(level: int = logging.ERROR):
+    """ A context manager that sets the `rasterio` logging level, then returns it to its original value. """
     try:
         # GEE sets GeoTIFF `colorinterp` tags incorrectly. This suppresses `rasterio` warning relating to this:
         # 'Sum of Photometric type-related color channels and ExtraSamples doesn't match SamplesPerPixel'
         rio_level = logging.getLogger('rasterio').getEffectiveLevel()
-        logging.getLogger('rasterio').setLevel(logging.ERROR)
+        logging.getLogger('rasterio').setLevel(level)
         yield
     finally:
         logging.getLogger('rasterio').setLevel(rio_level)
@@ -137,7 +137,7 @@ def get_bounds(filename: pathlib.Path, expand: float = 5):
     dict
         Geojson polygon.
     """
-    with suppress_rio_warn_logs(), rio.Env(GTIFF_FORCE_RGBA=False), rio.open(filename) as im:
+    with suppress_rio_logs(), rio.Env(GTIFF_FORCE_RGBA=False), rio.open(filename) as im:
         bbox = im.bounds
         if (im.crs.linear_units == "metre") and (expand > 0):  # expand the bounding box
             expand_x = (bbox.right - bbox.left) * expand / 100.0

geedim/version.py

@@ -1 +1 @@
-__version__ = '1.7.1'
+__version__ = '1.7.2'