diff --git a/README.md b/README.md index cbd89a8..174b641 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,8 @@ subsequent accesses to the same and nearby areas are much faster. The `GLO-30` and `GLO-90` products are the exception: they are read in place from the Earth Data Hub and cached one Zarr chunk at a time. -The `clip` sub-command doesn't allow automatic download of a large amount of DEM tiles, +The `seed` and `clip` sub-commands refuse to download more than `--max_download_tiles` +(`25` by default) tiles at a time to prevent bulk downloads, please refer to the upstream providers' websites to learn the preferred procedures for bulk download. To show the STAC metadata of the datasets use: @@ -157,7 +158,7 @@ $ eio --help │ --cache_dir Root of the DEM cache folder. │ │ [env var: EIO_CACHE_DIR] │ │ [default: │ -│ /Users/amici/Library/Caches/elevation] │ +│ /Users/amici/Library/Caches/elevation] │ │ --help Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────╮ @@ -179,27 +180,56 @@ $ eio clip --help Usage: eio clip [OPTIONS] ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ -│ --output -o Path to output file. Existing files will │ -│ be overwritten. │ -│ [env var: EIO_CLIP_OUTPUT] │ -│ [default: out.tif] │ -│ --bounds ... Output bounds in 'left bottom right top' │ -│ order. │ -│ [env var: EIO_CLIP_BOUNDS] │ -│ --margin -m Decimal degree margin added to the bounds. │ -│ Use '%' for percent margin. │ -│ [env var: EIO_CLIP_MARGIN] │ -│ [default: 0] │ -│ --reference -r Use the extent of a reference GDAL/OGR │ -│ data source as output bounds. │ -│ [env var: EIO_CLIP_REFERENCE] │ -│ --gdal-options GDAL creation options of the output file, │ -│ e.g. '-co COMPRESS=LZW'. │ -│ [env var: EIO_CLIP_GDAL_OPTIONS] │ -│ [default: -co TILED=YES -co │ -│ COMPRESS=DEFLATE -co ZLEVEL=9 -co │ -│ PREDICTOR=2] │ -│ --help Show this message and exit. │ +│ --output -o Path to output file. Existing files │ +│ will be overwritten. │ +│ [env var: EIO_CLIP_OUTPUT] │ +│ [default: out.tif] │ +│ --bounds ... Output bounds in 'left bottom right │ +│ top' order. │ +│ [env var: EIO_CLIP_BOUNDS] │ +│ --margin -m Decimal degree margin added to the │ +│ bounds. Use '%' for percent margin. │ +│ [env var: EIO_CLIP_MARGIN] │ +│ [default: 0] │ +│ --reference -r Use the extent of a reference │ +│ GDAL/OGR data source as output │ +│ bounds. │ +│ [env var: EIO_CLIP_REFERENCE] │ +│ --gdal-options GDAL creation options of the output │ +│ file, e.g. '-co COMPRESS=LZW'. │ +│ [env var: EIO_CLIP_GDAL_OPTIONS] │ +│ [default: -co TILED=YES -co │ +│ COMPRESS=DEFLATE -co ZLEVEL=9 -co │ +│ PREDICTOR=2] │ +│ --max_download_tiles Maximum number of tiles to download. │ +│ [env var: │ +│ EIO_CLIP_MAX_DOWNLOAD_TILES] │ +│ [default: 25] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +The `seed` sub-command downloads and caches the tiles that cover the bounds +without producing any output file: + +```text +$ eio seed --help + + Usage: eio seed [OPTIONS] + +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ +│ --bounds ... Output bounds in 'left bottom right │ +│ top' order. │ +│ [env var: EIO_SEED_BOUNDS] │ +│ --margin -m Decimal degree margin added to the │ +│ bounds. Use '%' for percent margin. │ +│ [env var: EIO_SEED_MARGIN] │ +│ [default: 0] │ +│ --max_download_tiles Maximum number of tiles to download. │ +│ [env var: │ +│ EIO_SEED_MAX_DOWNLOAD_TILES] │ +│ [default: 25] │ +│ --help Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` diff --git a/elevation/__init__.py b/elevation/__init__.py index a8fafc7..47fcc2b 100644 --- a/elevation/__init__.py +++ b/elevation/__init__.py @@ -27,6 +27,7 @@ DEFAULT_OUTPUT, DEFAULT_PRODUCT, MARGIN, + MAX_DOWNLOAD_TILES, PRODUCTS, RETIRED_PRODUCTS, ProductRetiredError, @@ -45,6 +46,7 @@ "DEFAULT_OUTPUT", "DEFAULT_PRODUCT", "MARGIN", + "MAX_DOWNLOAD_TILES", "PRODUCTS", "RETIRED_PRODUCTS", "TOOLS", diff --git a/elevation/__main__.py b/elevation/__main__.py index 8b46bd4..94c2eba 100644 --- a/elevation/__main__.py +++ b/elevation/__main__.py @@ -119,7 +119,7 @@ def seed( bounds: tuple[float, float, float, float] | None = typer.Option( None, "--bounds", - help="Output bounds: left bottom right top.", + help="Output bounds in 'left bottom right top' order.", ), margin: str = typer.Option( elevation.MARGIN, @@ -127,8 +127,18 @@ def seed( "--margin", help="Decimal degree margin added to the bounds. Use '%' for percent margin.", ), + max_download_tiles: int = typer.Option( + elevation.MAX_DOWNLOAD_TILES, + "--max_download_tiles", + help="Maximum number of tiles to download.", + ), ) -> None: - elevation.seed(**ctx.obj, bounds=bounds, margin=margin) + elevation.seed( + **ctx.obj, + bounds=bounds, + margin=margin, + max_download_tiles=max_download_tiles, + ) @app.command(short_help="Clip the DEM to given bounds.") @@ -166,6 +176,11 @@ def clip( "--gdal-options", help="GDAL creation options of the output file, e.g. '-co COMPRESS=LZW'.", ), + max_download_tiles: int = typer.Option( + elevation.MAX_DOWNLOAD_TILES, + "--max_download_tiles", + help="Maximum number of tiles to download.", + ), ) -> None: if bounds is None and reference is None: raise typer.BadParameter( @@ -176,7 +191,12 @@ def clip( assert reference is not None bounds = spatial.import_bounds(reference) elevation.clip( - bounds, output=output, margin=margin, gdal_options=gdal_options, **ctx.obj + bounds, + output=output, + margin=margin, + gdal_options=gdal_options, + max_download_tiles=max_download_tiles, + **ctx.obj, ) diff --git a/elevation/cache.py b/elevation/cache.py index 4b0f3b7..3cb2a7e 100644 --- a/elevation/cache.py +++ b/elevation/cache.py @@ -67,6 +67,12 @@ def lock_tiles(datasource_root: Path, tile_names: list[str]) -> Generator[None]: lock.release() +def is_cached(root: Path, tile_name: str) -> bool: + """Return whether *tile_name* is cached, i.e. present and not empty.""" + cached = root / "cache" / tile_name + return cached.exists() and cached.stat().st_size > 0 + + def ensure_tiles( root: Path, tiles: list[Tile], @@ -81,10 +87,11 @@ def ensure_tiles( URL and goes through the spool. """ for (ilon, ilat), tile_name in tiles: - cached = root / "cache" / tile_name - if cached.exists() and cached.stat().st_size > 0: + if is_cached(root, tile_name): continue + cached = root / "cache" / tile_name + # prepare the data if GDAL cannot download it / read it as it is source, spooled = prepare_tile( tile_name=tile_name, spool=root / "spool", ilat=ilat, ilon=ilon, **kwargs diff --git a/elevation/datasource.py b/elevation/datasource.py index 0f528c1..dd481fc 100644 --- a/elevation/datasource.py +++ b/elevation/datasource.py @@ -25,6 +25,7 @@ DEFAULT_OUTPUT = "out.tif" DEFAULT_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co PREDICTOR=2" MARGIN = "0" +MAX_DOWNLOAD_TILES = 25 # NOTE: # 0.0001388888889 == 0.5" is half pixel for DEMs with 1" spacing (DTED L2) @@ -324,7 +325,7 @@ def seed( product: str = DEFAULT_PRODUCT, bounds: tuple[float, float, float, float] | None = None, margin: str = MARGIN, - max_download_tiles: int = 9, + max_download_tiles: int = MAX_DOWNLOAD_TILES, ) -> tuple[Path, tuple[float, float, float, float]]: """Seed the DEM to given bounds. @@ -335,7 +336,7 @@ def seed( :param product: DEM product choice. :param bounds: Output bounds in 'left bottom right top' order. :param margin: Decimal degree margin added to the bounds. Use '%' for percent margin. - :param max_download_tiles: Maximum number of tiles to process. + :param max_download_tiles: Maximum number of tiles to download. :return: The datasource root and the bounds with the margin applied. """ if bounds is None: @@ -345,19 +346,22 @@ def seed( cached_tiles = spec["cached_tiles"] cached_tiles_kwargs = spec.get("cached_tiles_kwargs", {}) tiles = list(cached_tiles(*bounds, **cached_tiles_kwargs)) + downloads = [ + tile for tile in tiles if not cache.is_cached(datasource_root, tile[1]) + ] # FIXME: emergency hack to enforce the no-bulk-download policy - if len(tiles) > max_download_tiles: + if len(downloads) > max_download_tiles: raise RuntimeError( - f"Too many tiles: {len(tiles)}. Please consult the " + f"Too many tiles to download: {len(downloads)}. Please consult the " "providers' websites for how to bulk download tiles." ) prepare_tile = spec["prepare_tile"] prepare_tile_kwargs = spec.get("prepare_tile_kwargs", {}) - with cache.lock_tiles(datasource_root, [name for _, name in tiles]): + with cache.lock_tiles(datasource_root, [name for _, name in downloads]): cache.ensure_tiles( datasource_root, - tiles, + downloads, prepare_tile=prepare_tile, gdal_options=spec.get("tile_gdal_options", spatial.INT_TILE_GDAL_OPTIONS), **prepare_tile_kwargs, @@ -395,6 +399,7 @@ def clip( cache_dir: str | Path | None = None, product: str = DEFAULT_PRODUCT, gdal_options: str = DEFAULT_GDAL_OPTIONS, + max_download_tiles: int = MAX_DOWNLOAD_TILES, ) -> None: """Clip the DEM to given bounds. @@ -403,11 +408,16 @@ def clip( :param margin: Decimal degree margin added to the bounds. Use '%' for percent margin. :param cache_dir: Root of the DEM cache folder. :param product: DEM product choice. - :param gdal_options: GDAL creation options of the output file. + :param gdal_options: GDAL creation options of the output file, e.g. '-co COMPRESS=LZW'. + :param max_download_tiles: Maximum number of tiles to download. """ output = Path(output).resolve() datasource_root, bounds = seed( - cache_dir=cache_dir, product=product, bounds=bounds, margin=margin + cache_dir=cache_dir, + product=product, + bounds=bounds, + margin=margin, + max_download_tiles=max_download_tiles, ) left, bottom, right, top = bounds options = f"-q {gdal_options} -projwin {left} {top} {right} {bottom}" diff --git a/tests/test_20_datasource.py b/tests/test_20_datasource.py index cf3f6ac..6288453 100644 --- a/tests/test_20_datasource.py +++ b/tests/test_20_datasource.py @@ -313,6 +313,29 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: datasource.seed(cache_dir=root) +def test_seed_max_download_tiles_counts_only_downloads( + mocker: MockerFixture, tmp_path: Path +) -> None: + root = tmp_path / "root" + cached = root / "SRTM1_GEOID" / "cache" + cached.mkdir(parents=True) + (cached / "N43E013.tif").write_bytes(b"cached") + (cached / "N43E014.tif").write_bytes(b"cached") + mock_fetch = mocker.patch("elevation.datasource.fetch_tile") + mocker.patch("elevation.spatial.call_gdal_translate") + mocker.patch("subprocess.check_call") + + datasource_root, _ = datasource.seed( + cache_dir=root, + product="SRTM1_GEOID", + bounds=(13.1, 43.1, 14.9, 43.9), + max_download_tiles=1, + ) + + assert datasource_root == root / "SRTM1_GEOID" + mock_fetch.assert_not_called() + + def test_seed_remote(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" mock_check_call = mocker.patch("subprocess.check_call") diff --git a/tests/test_40_main.py b/tests/test_40_main.py index 721542a..c76e6c8 100644 --- a/tests/test_40_main.py +++ b/tests/test_40_main.py @@ -118,6 +118,18 @@ def test_eio_seed_margin(mocker: MockerFixture, tmp_path: Path) -> None: assert mock_seed.call_args.kwargs["margin"] == "1" +def test_eio_seed_max_download_tiles(mocker: MockerFixture, tmp_path: Path) -> None: + root = tmp_path / "root" + runner = typer.testing.CliRunner() + mock_seed = mocker.patch("elevation.seed") + options = ( + f"--cache_dir {root!s} seed --bounds 12.5 42 12.5 42 --max_download_tiles 3" + ) + result = runner.invoke(__main__.app, options.split()) + assert not result.exception + assert mock_seed.call_args.kwargs["max_download_tiles"] == 3 + + def test_eio_clip(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" runner = typer.testing.CliRunner() @@ -168,6 +180,18 @@ def test_eio_clip_gdal_options(mocker: MockerFixture, tmp_path: Path) -> None: assert "COMPRESS=LZW" in mock_translate.call_args.kwargs["options"] +def test_eio_clip_max_download_tiles(mocker: MockerFixture, tmp_path: Path) -> None: + root = tmp_path / "root" + runner = typer.testing.CliRunner() + mock_clip = mocker.patch("elevation.clip") + options = ( + f"--cache_dir {root!s} clip --bounds 12.5 42 12.5 42 --max_download_tiles 3" + ) + result = runner.invoke(__main__.app, options.split()) + assert not result.exception + assert mock_clip.call_args.kwargs["max_download_tiles"] == 3 + + def test_eio_clean(tmp_path: Path) -> None: root = tmp_path / "root" (root / "MAPZEN" / "spool").mkdir(parents=True)