From f3f879496964082bf1463b0a5b1cf8596ba69c03 Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Tue, 6 Oct 2026 23:19:25 +0200 Subject: [PATCH 1/2] CLIs now have --max_downlaod_tiles --- elevation/__init__.py | 2 ++ elevation/__main__.py | 24 ++++++++++++++++++++++-- elevation/datasource.py | 11 +++++++++-- tests/test_40_main.py | 24 ++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/elevation/__init__.py b/elevation/__init__.py index a8fafc7..47fcc2b 100644 --- a/elevation/__init__.py +++ b/elevation/__init__.py @@ -27,6 +27,7 @@ DEFAULT_OUTPUT, DEFAULT_PRODUCT, MARGIN, + MAX_DOWNLOAD_TILES, PRODUCTS, RETIRED_PRODUCTS, ProductRetiredError, @@ -45,6 +46,7 @@ "DEFAULT_OUTPUT", "DEFAULT_PRODUCT", "MARGIN", + "MAX_DOWNLOAD_TILES", "PRODUCTS", "RETIRED_PRODUCTS", "TOOLS", diff --git a/elevation/__main__.py b/elevation/__main__.py index 8b46bd4..14824f2 100644 --- a/elevation/__main__.py +++ b/elevation/__main__.py @@ -127,8 +127,18 @@ def seed( "--margin", help="Decimal degree margin added to the bounds. Use '%' for percent margin.", ), + max_download_tiles: int = typer.Option( + elevation.MAX_DOWNLOAD_TILES, + "--max_download_tiles", + help="Maximum number of tiles to process.", + ), ) -> None: - elevation.seed(**ctx.obj, bounds=bounds, margin=margin) + elevation.seed( + **ctx.obj, + bounds=bounds, + margin=margin, + max_download_tiles=max_download_tiles, + ) @app.command(short_help="Clip the DEM to given bounds.") @@ -166,6 +176,11 @@ def clip( "--gdal-options", help="GDAL creation options of the output file, e.g. '-co COMPRESS=LZW'.", ), + max_download_tiles: int = typer.Option( + elevation.MAX_DOWNLOAD_TILES, + "--max_download_tiles", + help="Maximum number of tiles to process.", + ), ) -> None: if bounds is None and reference is None: raise typer.BadParameter( @@ -176,7 +191,12 @@ def clip( assert reference is not None bounds = spatial.import_bounds(reference) elevation.clip( - bounds, output=output, margin=margin, gdal_options=gdal_options, **ctx.obj + bounds, + output=output, + margin=margin, + gdal_options=gdal_options, + max_download_tiles=max_download_tiles, + **ctx.obj, ) diff --git a/elevation/datasource.py b/elevation/datasource.py index 0f528c1..884a81c 100644 --- a/elevation/datasource.py +++ b/elevation/datasource.py @@ -25,6 +25,7 @@ DEFAULT_OUTPUT = "out.tif" DEFAULT_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co PREDICTOR=2" MARGIN = "0" +MAX_DOWNLOAD_TILES = 25 # NOTE: # 0.0001388888889 == 0.5" is half pixel for DEMs with 1" spacing (DTED L2) @@ -324,7 +325,7 @@ def seed( product: str = DEFAULT_PRODUCT, bounds: tuple[float, float, float, float] | None = None, margin: str = MARGIN, - max_download_tiles: int = 9, + max_download_tiles: int = MAX_DOWNLOAD_TILES, ) -> tuple[Path, tuple[float, float, float, float]]: """Seed the DEM to given bounds. @@ -395,6 +396,7 @@ def clip( cache_dir: str | Path | None = None, product: str = DEFAULT_PRODUCT, gdal_options: str = DEFAULT_GDAL_OPTIONS, + max_download_tiles: int = MAX_DOWNLOAD_TILES, ) -> None: """Clip the DEM to given bounds. @@ -404,10 +406,15 @@ def clip( :param cache_dir: Root of the DEM cache folder. :param product: DEM product choice. :param gdal_options: GDAL creation options of the output file. + :param max_download_tiles: Maximum number of tiles to process. """ output = Path(output).resolve() datasource_root, bounds = seed( - cache_dir=cache_dir, product=product, bounds=bounds, margin=margin + cache_dir=cache_dir, + product=product, + bounds=bounds, + margin=margin, + max_download_tiles=max_download_tiles, ) left, bottom, right, top = bounds options = f"-q {gdal_options} -projwin {left} {top} {right} {bottom}" diff --git a/tests/test_40_main.py b/tests/test_40_main.py index 721542a..c76e6c8 100644 --- a/tests/test_40_main.py +++ b/tests/test_40_main.py @@ -118,6 +118,18 @@ def test_eio_seed_margin(mocker: MockerFixture, tmp_path: Path) -> None: assert mock_seed.call_args.kwargs["margin"] == "1" +def test_eio_seed_max_download_tiles(mocker: MockerFixture, tmp_path: Path) -> None: + root = tmp_path / "root" + runner = typer.testing.CliRunner() + mock_seed = mocker.patch("elevation.seed") + options = ( + f"--cache_dir {root!s} seed --bounds 12.5 42 12.5 42 --max_download_tiles 3" + ) + result = runner.invoke(__main__.app, options.split()) + assert not result.exception + assert mock_seed.call_args.kwargs["max_download_tiles"] == 3 + + def test_eio_clip(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" runner = typer.testing.CliRunner() @@ -168,6 +180,18 @@ def test_eio_clip_gdal_options(mocker: MockerFixture, tmp_path: Path) -> None: assert "COMPRESS=LZW" in mock_translate.call_args.kwargs["options"] +def test_eio_clip_max_download_tiles(mocker: MockerFixture, tmp_path: Path) -> None: + root = tmp_path / "root" + runner = typer.testing.CliRunner() + mock_clip = mocker.patch("elevation.clip") + options = ( + f"--cache_dir {root!s} clip --bounds 12.5 42 12.5 42 --max_download_tiles 3" + ) + result = runner.invoke(__main__.app, options.split()) + assert not result.exception + assert mock_clip.call_args.kwargs["max_download_tiles"] == 3 + + def test_eio_clean(tmp_path: Path) -> None: root = tmp_path / "root" (root / "MAPZEN" / "spool").mkdir(parents=True) From c664d2dd594474534f085dc286ada023a1cbd63a Mon Sep 17 00:00:00 2001 From: Alessandro Amici Date: Tue, 6 Oct 2026 23:32:38 +0200 Subject: [PATCH 2/2] Count only the tiles to download against max_download_tiles Only the tiles missing from the cache are counted and locked, and the seed and clip descriptions are synced in the CLI help, the API docstrings and the README, that also gains the eio seed --help block. closes #16 #39 #45 and #63 --- README.md | 76 ++++++++++++++++++++++++++----------- elevation/__main__.py | 6 +-- elevation/cache.py | 11 +++++- elevation/datasource.py | 17 +++++---- tests/test_20_datasource.py | 23 +++++++++++ 5 files changed, 98 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index cbd89a8..174b641 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,8 @@ subsequent accesses to the same and nearby areas are much faster. The `GLO-30` and `GLO-90` products are the exception: they are read in place from the Earth Data Hub and cached one Zarr chunk at a time. -The `clip` sub-command doesn't allow automatic download of a large amount of DEM tiles, +The `seed` and `clip` sub-commands refuse to download more than `--max_download_tiles` +(`25` by default) tiles at a time to prevent bulk downloads, please refer to the upstream providers' websites to learn the preferred procedures for bulk download. To show the STAC metadata of the datasets use: @@ -157,7 +158,7 @@ $ eio --help │ --cache_dir Root of the DEM cache folder. │ │ [env var: EIO_CACHE_DIR] │ │ [default: │ -│ /Users/amici/Library/Caches/elevation] │ +│ /Users/amici/Library/Caches/elevation] │ │ --help Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────╮ @@ -179,27 +180,56 @@ $ eio clip --help Usage: eio clip [OPTIONS] ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ -│ --output -o Path to output file. Existing files will │ -│ be overwritten. │ -│ [env var: EIO_CLIP_OUTPUT] │ -│ [default: out.tif] │ -│ --bounds ... Output bounds in 'left bottom right top' │ -│ order. │ -│ [env var: EIO_CLIP_BOUNDS] │ -│ --margin -m Decimal degree margin added to the bounds. │ -│ Use '%' for percent margin. │ -│ [env var: EIO_CLIP_MARGIN] │ -│ [default: 0] │ -│ --reference -r Use the extent of a reference GDAL/OGR │ -│ data source as output bounds. │ -│ [env var: EIO_CLIP_REFERENCE] │ -│ --gdal-options GDAL creation options of the output file, │ -│ e.g. '-co COMPRESS=LZW'. │ -│ [env var: EIO_CLIP_GDAL_OPTIONS] │ -│ [default: -co TILED=YES -co │ -│ COMPRESS=DEFLATE -co ZLEVEL=9 -co │ -│ PREDICTOR=2] │ -│ --help Show this message and exit. │ +│ --output -o Path to output file. Existing files │ +│ will be overwritten. │ +│ [env var: EIO_CLIP_OUTPUT] │ +│ [default: out.tif] │ +│ --bounds ... Output bounds in 'left bottom right │ +│ top' order. │ +│ [env var: EIO_CLIP_BOUNDS] │ +│ --margin -m Decimal degree margin added to the │ +│ bounds. Use '%' for percent margin. │ +│ [env var: EIO_CLIP_MARGIN] │ +│ [default: 0] │ +│ --reference -r Use the extent of a reference │ +│ GDAL/OGR data source as output │ +│ bounds. │ +│ [env var: EIO_CLIP_REFERENCE] │ +│ --gdal-options GDAL creation options of the output │ +│ file, e.g. '-co COMPRESS=LZW'. │ +│ [env var: EIO_CLIP_GDAL_OPTIONS] │ +│ [default: -co TILED=YES -co │ +│ COMPRESS=DEFLATE -co ZLEVEL=9 -co │ +│ PREDICTOR=2] │ +│ --max_download_tiles Maximum number of tiles to download. │ +│ [env var: │ +│ EIO_CLIP_MAX_DOWNLOAD_TILES] │ +│ [default: 25] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +The `seed` sub-command downloads and caches the tiles that cover the bounds +without producing any output file: + +```text +$ eio seed --help + + Usage: eio seed [OPTIONS] + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --bounds ... Output bounds in 'left bottom right │ +│ top' order. │ +│ [env var: EIO_SEED_BOUNDS] │ +│ --margin -m Decimal degree margin added to the │ +│ bounds. Use '%' for percent margin. │ +│ [env var: EIO_SEED_MARGIN] │ +│ [default: 0] │ +│ --max_download_tiles Maximum number of tiles to download. │ +│ [env var: │ +│ EIO_SEED_MAX_DOWNLOAD_TILES] │ +│ [default: 25] │ +│ --help Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` diff --git a/elevation/__main__.py b/elevation/__main__.py index 14824f2..94c2eba 100644 --- a/elevation/__main__.py +++ b/elevation/__main__.py @@ -119,7 +119,7 @@ def seed( bounds: tuple[float, float, float, float] | None = typer.Option( None, "--bounds", - help="Output bounds: left bottom right top.", + help="Output bounds in 'left bottom right top' order.", ), margin: str = typer.Option( elevation.MARGIN, @@ -130,7 +130,7 @@ def seed( max_download_tiles: int = typer.Option( elevation.MAX_DOWNLOAD_TILES, "--max_download_tiles", - help="Maximum number of tiles to process.", + help="Maximum number of tiles to download.", ), ) -> None: elevation.seed( @@ -179,7 +179,7 @@ def clip( max_download_tiles: int = typer.Option( elevation.MAX_DOWNLOAD_TILES, "--max_download_tiles", - help="Maximum number of tiles to process.", + help="Maximum number of tiles to download.", ), ) -> None: if bounds is None and reference is None: diff --git a/elevation/cache.py b/elevation/cache.py index 4b0f3b7..3cb2a7e 100644 --- a/elevation/cache.py +++ b/elevation/cache.py @@ -67,6 +67,12 @@ def lock_tiles(datasource_root: Path, tile_names: list[str]) -> Generator[None]: lock.release() +def is_cached(root: Path, tile_name: str) -> bool: + """Return whether *tile_name* is cached, i.e. present and not empty.""" + cached = root / "cache" / tile_name + return cached.exists() and cached.stat().st_size > 0 + + def ensure_tiles( root: Path, tiles: list[Tile], @@ -81,10 +87,11 @@ def ensure_tiles( URL and goes through the spool. """ for (ilon, ilat), tile_name in tiles: - cached = root / "cache" / tile_name - if cached.exists() and cached.stat().st_size > 0: + if is_cached(root, tile_name): continue + cached = root / "cache" / tile_name + # prepare the data if GDAL cannot download it / read it as it is source, spooled = prepare_tile( tile_name=tile_name, spool=root / "spool", ilat=ilat, ilon=ilon, **kwargs diff --git a/elevation/datasource.py b/elevation/datasource.py index 884a81c..dd481fc 100644 --- a/elevation/datasource.py +++ b/elevation/datasource.py @@ -336,7 +336,7 @@ def seed( :param product: DEM product choice. :param bounds: Output bounds in 'left bottom right top' order. :param margin: Decimal degree margin added to the bounds. Use '%' for percent margin. - :param max_download_tiles: Maximum number of tiles to process. + :param max_download_tiles: Maximum number of tiles to download. :return: The datasource root and the bounds with the margin applied. """ if bounds is None: @@ -346,19 +346,22 @@ def seed( cached_tiles = spec["cached_tiles"] cached_tiles_kwargs = spec.get("cached_tiles_kwargs", {}) tiles = list(cached_tiles(*bounds, **cached_tiles_kwargs)) + downloads = [ + tile for tile in tiles if not cache.is_cached(datasource_root, tile[1]) + ] # FIXME: emergency hack to enforce the no-bulk-download policy - if len(tiles) > max_download_tiles: + if len(downloads) > max_download_tiles: raise RuntimeError( - f"Too many tiles: {len(tiles)}. Please consult the " + f"Too many tiles to download: {len(downloads)}. Please consult the " "providers' websites for how to bulk download tiles." ) prepare_tile = spec["prepare_tile"] prepare_tile_kwargs = spec.get("prepare_tile_kwargs", {}) - with cache.lock_tiles(datasource_root, [name for _, name in tiles]): + with cache.lock_tiles(datasource_root, [name for _, name in downloads]): cache.ensure_tiles( datasource_root, - tiles, + downloads, prepare_tile=prepare_tile, gdal_options=spec.get("tile_gdal_options", spatial.INT_TILE_GDAL_OPTIONS), **prepare_tile_kwargs, @@ -405,8 +408,8 @@ def clip( :param margin: Decimal degree margin added to the bounds. Use '%' for percent margin. :param cache_dir: Root of the DEM cache folder. :param product: DEM product choice. - :param gdal_options: GDAL creation options of the output file. - :param max_download_tiles: Maximum number of tiles to process. + :param gdal_options: GDAL creation options of the output file, e.g. '-co COMPRESS=LZW'. + :param max_download_tiles: Maximum number of tiles to download. """ output = Path(output).resolve() datasource_root, bounds = seed( diff --git a/tests/test_20_datasource.py b/tests/test_20_datasource.py index cf3f6ac..6288453 100644 --- a/tests/test_20_datasource.py +++ b/tests/test_20_datasource.py @@ -313,6 +313,29 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: datasource.seed(cache_dir=root) +def test_seed_max_download_tiles_counts_only_downloads( + mocker: MockerFixture, tmp_path: Path +) -> None: + root = tmp_path / "root" + cached = root / "SRTM1_GEOID" / "cache" + cached.mkdir(parents=True) + (cached / "N43E013.tif").write_bytes(b"cached") + (cached / "N43E014.tif").write_bytes(b"cached") + mock_fetch = mocker.patch("elevation.datasource.fetch_tile") + mocker.patch("elevation.spatial.call_gdal_translate") + mocker.patch("subprocess.check_call") + + datasource_root, _ = datasource.seed( + cache_dir=root, + product="SRTM1_GEOID", + bounds=(13.1, 43.1, 14.9, 43.9), + max_download_tiles=1, + ) + + assert datasource_root == root / "SRTM1_GEOID" + mock_fetch.assert_not_called() + + def test_seed_remote(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" mock_check_call = mocker.patch("subprocess.check_call")