From 1515e3e5916dc424b05251cc92778db019c68a1f Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Tue, 7 Jan 2025 12:57:59 -0500 Subject: [PATCH 1/2] replace copyfileobj with iter_chunks --- earthaccess/store.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/earthaccess/store.py b/earthaccess/store.py index 58ac9f59..7bf54c60 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -1,6 +1,5 @@ import datetime import logging -import shutil import traceback from functools import lru_cache from itertools import chain @@ -670,16 +669,13 @@ def _download_file(self, url: str, directory: Path) -> str: if not path.exists(): try: session = self.auth.get_session() - with session.get( - url, - stream=True, - allow_redirects=True, - ) as r: + with session.get(url, stream=True, allow_redirects=True) as r: r.raise_for_status() with open(path, "wb") as f: - # This is to cap memory usage for large files at 1MB per write to disk per thread + # Cap memory usage for large files at 1MB per write to disk per thread # https://docs.python-requests.org/en/latest/user/quickstart/#raw-response-content - shutil.copyfileobj(r.raw, f, length=1024 * 1024) + for chunk in r.iter_content(chunk_size=1024 * 1024): + f.write(chunk) except Exception: logger.exception(f"Error while downloading the file {local_filename}") raise Exception From db0fa4edbaa1b7a402d8ae6d3292c4d6582f549c Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Tue, 7 Jan 2025 14:32:59 -0500 Subject: [PATCH 2/2] add CHANGELOG entry --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec21ddb6..3c13adeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] +### Fixed + +- `earthaccess.download` will let requests automatically decode compressed content + ([#887](https://github.com/nsidc/earthaccess/issues/887)) + ([**@itcarroll**](https://github.com/itcarroll)) + ## [v0.12.0] - 2024-11-13 ### Changed