Expose max_download_tiles in the python API and command line

README.md

@@ -122,7 +122,8 @@ subsequent accesses to the same and nearby areas are much faster.
 The `GLO-30` and `GLO-90` products are the exception: they are read in place
 from the Earth Data Hub and cached one Zarr chunk at a time.
 
-The `clip` sub-command doesn't allow automatic download of a large amount of DEM tiles,
+The `seed` and `clip` sub-commands refuse to download more than `--max_download_tiles`
+(`25` by default) tiles at a time to prevent bulk downloads,
 please refer to the upstream providers' websites to learn the preferred procedures for bulk download.
 
 To show the STAC metadata of the datasets use:
@@ -157,7 +158,7 @@ $ eio --help
 │ --cache_dir        <directory>                            Root of the DEM cache folder.          │
 │                                                           [env var: EIO_CACHE_DIR]               │
 │                                                           [default:                              │
-│                                                           /Users/amici/Library/Caches/elevation] │
+│                                                           /Users/amici/Library/Caches/elevation]  │
 │ --help                                                    Show this message and exit.            │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────╮
@@ -179,27 +180,56 @@ $ eio clip --help
  Usage: eio clip [OPTIONS]
 
 ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
-│ --output        -o      <file>                        Path to output file. Existing files will   │
-│                                                       be overwritten.                            │
-│                                                       [env var: EIO_CLIP_OUTPUT]                 │
-│                                                       [default: out.tif]                         │
-│ --bounds                <float float float float>...  Output bounds in 'left bottom right top'   │
-│                                                       order.                                     │
-│                                                       [env var: EIO_CLIP_BOUNDS]                 │
-│ --margin        -m      <str>                         Decimal degree margin added to the bounds. │
-│                                                       Use '%' for percent margin.                │
-│                                                       [env var: EIO_CLIP_MARGIN]                 │
-│                                                       [default: 0]                               │
-│ --reference     -r      <path>                        Use the extent of a reference GDAL/OGR     │
-│                                                       data source as output bounds.              │
-│                                                       [env var: EIO_CLIP_REFERENCE]              │
-│ --gdal-options          <str>                         GDAL creation options of the output file,  │
-│                                                       e.g. '-co COMPRESS=LZW'.                   │
-│                                                       [env var: EIO_CLIP_GDAL_OPTIONS]           │
-│                                                       [default: -co TILED=YES -co                │
-│                                                       COMPRESS=DEFLATE -co ZLEVEL=9 -co          │
-│                                                       PREDICTOR=2]                               │
-│ --help                                                Show this message and exit.                │
+│ --output              -o      <file>                        Path to output file. Existing files  │
+│                                                             will be overwritten.                 │
+│                                                             [env var: EIO_CLIP_OUTPUT]           │
+│                                                             [default: out.tif]                   │
+│ --bounds                      <float float float float>...  Output bounds in 'left bottom right  │
+│                                                             top' order.                          │
+│                                                             [env var: EIO_CLIP_BOUNDS]           │
+│ --margin              -m      <str>                         Decimal degree margin added to the   │
+│                                                             bounds. Use '%' for percent margin.  │
+│                                                             [env var: EIO_CLIP_MARGIN]           │
+│                                                             [default: 0]                         │
+│ --reference           -r      <path>                        Use the extent of a reference        │
+│                                                             GDAL/OGR data source as output       │
+│                                                             bounds.                              │
+│                                                             [env var: EIO_CLIP_REFERENCE]        │
+│ --gdal-options                <str>                         GDAL creation options of the output  │
+│                                                             file, e.g. '-co COMPRESS=LZW'.       │
+│                                                             [env var: EIO_CLIP_GDAL_OPTIONS]     │
+│                                                             [default: -co TILED=YES -co          │
+│                                                             COMPRESS=DEFLATE -co ZLEVEL=9 -co    │
+│                                                             PREDICTOR=2]                         │
+│ --max_download_tiles          <int>                         Maximum number of tiles to download. │
+│                                                             [env var:                            │
+│                                                             EIO_CLIP_MAX_DOWNLOAD_TILES]         │
+│                                                             [default: 25]                        │
+│ --help                                                      Show this message and exit.          │
+╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
+```
+
+The `seed` sub-command downloads and caches the tiles that cover the bounds
+without producing any output file:
+
+```text
+$ eio seed --help
+
+ Usage: eio seed [OPTIONS]
+
+╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
+│ --bounds                      <float float float float>...  Output bounds in 'left bottom right  │
+│                                                             top' order.                          │
+│                                                             [env var: EIO_SEED_BOUNDS]           │
+│ --margin              -m      <str>                         Decimal degree margin added to the   │
+│                                                             bounds. Use '%' for percent margin.  │
+│                                                             [env var: EIO_SEED_MARGIN]           │
+│                                                             [default: 0]                         │
+│ --max_download_tiles          <int>                         Maximum number of tiles to download. │
+│                                                             [env var:                            │
+│                                                             EIO_SEED_MAX_DOWNLOAD_TILES]         │
+│                                                             [default: 25]                        │
+│ --help                                                      Show this message and exit.          │
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
 

elevation/__init__.py

@@ -27,6 +27,7 @@
     DEFAULT_OUTPUT,
     DEFAULT_PRODUCT,
     MARGIN,
+    MAX_DOWNLOAD_TILES,
     PRODUCTS,
     RETIRED_PRODUCTS,
     ProductRetiredError,
@@ -45,6 +46,7 @@
     "DEFAULT_OUTPUT",
     "DEFAULT_PRODUCT",
     "MARGIN",
+    "MAX_DOWNLOAD_TILES",
     "PRODUCTS",
     "RETIRED_PRODUCTS",
     "TOOLS",

elevation/__main__.py

@@ -119,16 +119,26 @@ def seed(
     bounds: tuple[float, float, float, float] | None = typer.Option(
         None,
         "--bounds",
-        help="Output bounds: left bottom right top.",
+        help="Output bounds in 'left bottom right top' order.",
     ),
     margin: str = typer.Option(
         elevation.MARGIN,
         "-m",
         "--margin",
         help="Decimal degree margin added to the bounds. Use '%' for percent margin.",
     ),
+    max_download_tiles: int = typer.Option(
+        elevation.MAX_DOWNLOAD_TILES,
+        "--max_download_tiles",
+        help="Maximum number of tiles to download.",
+    ),
 ) -> None:
-    elevation.seed(**ctx.obj, bounds=bounds, margin=margin)
+    elevation.seed(
+        **ctx.obj,
+        bounds=bounds,
+        margin=margin,
+        max_download_tiles=max_download_tiles,
+    )
 
 
 @app.command(short_help="Clip the DEM to given bounds.")
@@ -166,6 +176,11 @@ def clip(
         "--gdal-options",
         help="GDAL creation options of the output file, e.g. '-co COMPRESS=LZW'.",
     ),
+    max_download_tiles: int = typer.Option(
+        elevation.MAX_DOWNLOAD_TILES,
+        "--max_download_tiles",
+        help="Maximum number of tiles to download.",
+    ),
 ) -> None:
     if bounds is None and reference is None:
         raise typer.BadParameter(
@@ -176,7 +191,12 @@ def clip(
         assert reference is not None
         bounds = spatial.import_bounds(reference)
     elevation.clip(
-        bounds, output=output, margin=margin, gdal_options=gdal_options, **ctx.obj
+        bounds,
+        output=output,
+        margin=margin,
+        gdal_options=gdal_options,
+        max_download_tiles=max_download_tiles,
+        **ctx.obj,
     )
 
 

elevation/cache.py

@@ -67,6 +67,12 @@ def lock_tiles(datasource_root: Path, tile_names: list[str]) -> Generator[None]:
         lock.release()
 
 
+def is_cached(root: Path, tile_name: str) -> bool:
+    """Return whether *tile_name* is cached, i.e. present and not empty."""
+    cached = root / "cache" / tile_name
+    return cached.exists() and cached.stat().st_size > 0
+
+
 def ensure_tiles(
     root: Path,
     tiles: list[Tile],
@@ -81,10 +87,11 @@ def ensure_tiles(
     URL and goes through the spool.
     """
     for (ilon, ilat), tile_name in tiles:
-        cached = root / "cache" / tile_name
-        if cached.exists() and cached.stat().st_size > 0:
+        if is_cached(root, tile_name):
             continue
 
+        cached = root / "cache" / tile_name
+
         # prepare the data if GDAL cannot download it / read it as it is
         source, spooled = prepare_tile(
             tile_name=tile_name, spool=root / "spool", ilat=ilat, ilon=ilon, **kwargs

elevation/datasource.py

@@ -25,6 +25,7 @@
 DEFAULT_OUTPUT = "out.tif"
 DEFAULT_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co PREDICTOR=2"
 MARGIN = "0"
+MAX_DOWNLOAD_TILES = 25
 
 # NOTE:
 #   0.0001388888889 == 0.5" is half pixel for DEMs with 1" spacing (DTED L2)
@@ -324,7 +325,7 @@ def seed(
     product: str = DEFAULT_PRODUCT,
     bounds: tuple[float, float, float, float] | None = None,
     margin: str = MARGIN,
-    max_download_tiles: int = 9,
+    max_download_tiles: int = MAX_DOWNLOAD_TILES,
 ) -> tuple[Path, tuple[float, float, float, float]]:
     """Seed the DEM to given bounds.
 
@@ -335,7 +336,7 @@ def seed(
     :param product: DEM product choice.
     :param bounds: Output bounds in 'left bottom right top' order.
     :param margin: Decimal degree margin added to the bounds. Use '%' for percent margin.
-    :param max_download_tiles: Maximum number of tiles to process.
+    :param max_download_tiles: Maximum number of tiles to download.
     :return: The datasource root and the bounds with the margin applied.
     """
     if bounds is None:
@@ -345,19 +346,22 @@ def seed(
     cached_tiles = spec["cached_tiles"]
     cached_tiles_kwargs = spec.get("cached_tiles_kwargs", {})
     tiles = list(cached_tiles(*bounds, **cached_tiles_kwargs))
+    downloads = [
+        tile for tile in tiles if not cache.is_cached(datasource_root, tile[1])
+    ]
     # FIXME: emergency hack to enforce the no-bulk-download policy
-    if len(tiles) > max_download_tiles:
+    if len(downloads) > max_download_tiles:
         raise RuntimeError(
-            f"Too many tiles: {len(tiles)}. Please consult the "
+            f"Too many tiles to download: {len(downloads)}. Please consult the "
             "providers' websites for how to bulk download tiles."
         )
 
     prepare_tile = spec["prepare_tile"]
     prepare_tile_kwargs = spec.get("prepare_tile_kwargs", {})
-    with cache.lock_tiles(datasource_root, [name for _, name in tiles]):
+    with cache.lock_tiles(datasource_root, [name for _, name in downloads]):
         cache.ensure_tiles(
             datasource_root,
-            tiles,
+            downloads,
             prepare_tile=prepare_tile,
             gdal_options=spec.get("tile_gdal_options", spatial.INT_TILE_GDAL_OPTIONS),
             **prepare_tile_kwargs,
@@ -395,6 +399,7 @@ def clip(
     cache_dir: str | Path | None = None,
     product: str = DEFAULT_PRODUCT,
     gdal_options: str = DEFAULT_GDAL_OPTIONS,
+    max_download_tiles: int = MAX_DOWNLOAD_TILES,
 ) -> None:
     """Clip the DEM to given bounds.
 
@@ -403,11 +408,16 @@ def clip(
     :param margin: Decimal degree margin added to the bounds. Use '%' for percent margin.
     :param cache_dir: Root of the DEM cache folder.
     :param product: DEM product choice.
-    :param gdal_options: GDAL creation options of the output file.
+    :param gdal_options: GDAL creation options of the output file, e.g. '-co COMPRESS=LZW'.
+    :param max_download_tiles: Maximum number of tiles to download.
     """
     output = Path(output).resolve()
     datasource_root, bounds = seed(
-        cache_dir=cache_dir, product=product, bounds=bounds, margin=margin
+        cache_dir=cache_dir,
+        product=product,
+        bounds=bounds,
+        margin=margin,
+        max_download_tiles=max_download_tiles,
     )
     left, bottom, right, top = bounds
     options = f"-q {gdal_options} -projwin {left} {top} {right} {bottom}"

tests/test_20_datasource.py

@@ -313,6 +313,29 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None:
         datasource.seed(cache_dir=root)
 
 
+def test_seed_max_download_tiles_counts_only_downloads(
+    mocker: MockerFixture, tmp_path: Path
+) -> None:
+    root = tmp_path / "root"
+    cached = root / "SRTM1_GEOID" / "cache"
+    cached.mkdir(parents=True)
+    (cached / "N43E013.tif").write_bytes(b"cached")
+    (cached / "N43E014.tif").write_bytes(b"cached")
+    mock_fetch = mocker.patch("elevation.datasource.fetch_tile")
+    mocker.patch("elevation.spatial.call_gdal_translate")
+    mocker.patch("subprocess.check_call")
+
+    datasource_root, _ = datasource.seed(
+        cache_dir=root,
+        product="SRTM1_GEOID",
+        bounds=(13.1, 43.1, 14.9, 43.9),
+        max_download_tiles=1,
+    )
+
+    assert datasource_root == root / "SRTM1_GEOID"
+    mock_fetch.assert_not_called()
+
+
 def test_seed_remote(mocker: MockerFixture, tmp_path: Path) -> None:
     root = tmp_path / "root"
     mock_check_call = mocker.patch("subprocess.check_call")

tests/test_40_main.py

@@ -118,6 +118,18 @@ def test_eio_seed_margin(mocker: MockerFixture, tmp_path: Path) -> None:
     assert mock_seed.call_args.kwargs["margin"] == "1"
 
 
+def test_eio_seed_max_download_tiles(mocker: MockerFixture, tmp_path: Path) -> None:
+    root = tmp_path / "root"
+    runner = typer.testing.CliRunner()
+    mock_seed = mocker.patch("elevation.seed")
+    options = (
+        f"--cache_dir {root!s} seed --bounds 12.5 42 12.5 42 --max_download_tiles 3"
+    )
+    result = runner.invoke(__main__.app, options.split())
+    assert not result.exception
+    assert mock_seed.call_args.kwargs["max_download_tiles"] == 3
+
+
 def test_eio_clip(mocker: MockerFixture, tmp_path: Path) -> None:
     root = tmp_path / "root"
     runner = typer.testing.CliRunner()
@@ -168,6 +180,18 @@ def test_eio_clip_gdal_options(mocker: MockerFixture, tmp_path: Path) -> None:
     assert "COMPRESS=LZW" in mock_translate.call_args.kwargs["options"]
 
 
+def test_eio_clip_max_download_tiles(mocker: MockerFixture, tmp_path: Path) -> None:
+    root = tmp_path / "root"
+    runner = typer.testing.CliRunner()
+    mock_clip = mocker.patch("elevation.clip")
+    options = (
+        f"--cache_dir {root!s} clip --bounds 12.5 42 12.5 42 --max_download_tiles 3"
+    )
+    result = runner.invoke(__main__.app, options.split())
+    assert not result.exception
+    assert mock_clip.call_args.kwargs["max_download_tiles"] == 3
+
+
 def test_eio_clean(tmp_path: Path) -> None:
     root = tmp_path / "root"
     (root / "MAPZEN" / "spool").mkdir(parents=True)