diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f7cd34c..d423d5f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,11 @@ - Download the original SRTM GL1 data from OpenTopography for the ``SRTM1_GEOID`` product. - Add the ``MAPZEN`` product for the global Mapzen terrain tiles mosaic and make it the default product. +- Add the ``GLO-30`` and ``GLO-90`` products for the Copernicus DEM global 30m and 90m + DSM on the EGM2008 geoid. They are distributed by the Earth Data Hub as a cloud-hosted + Zarr store that is read in place, one chunk at a time, and cached as GeoTIFF tiles + like the other products, so the credentials in ``~/.netrc`` are used and GDAL 3.8 or + later is required. - Retire the ``SRTM1`` product name: requesting it raises an error pointing to the migration notes. - Accept ``str`` or ``Path`` for all path-valued arguments. diff --git a/README.md b/README.md index e4e190e..e9d9f2e 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ Elevation provides easy download, cache and access of the global datasets: hosted on OpenTopography, 30m heights on the WGS84 ellipsoid. - `SRTM3`: [SRTM global 90m v4.1](https://bigdata.cgiar.org/srtm-90m-digital-elevation-database/) produced by CGIAR-CSI. +- `GLO-30`: [Copernicus DEM global 30m (2021)](https://doi.org/10.5270/ESA-c5d3d65) + produced by ESA and the European Union, 30m heights on the EGM2008 geoid. +- `GLO-90`: [Copernicus DEM global 90m (2021)](https://doi.org/10.5270/ESA-c5d3d65) + the 90m companion of `GLO-30`. Note that any download policies and attribution requirements of the respective providers apply. @@ -69,6 +73,26 @@ For the SRTM global 90m v4.1 DEM use: $ eio --product SRTM3 clip -o Rome-SRTM3-DEM.tif --bounds 12.35 41.8 12.65 42 ``` +For the Copernicus DEM global 30m or 90m DEMs use: + +```console +$ eio --product GLO-30 clip -o Rome-GLO-30-DEM.tif --bounds 12.35 41.8 12.65 42 +$ eio --product GLO-90 clip -o Rome-GLO-90-DEM.tif --bounds 12.35 41.8 12.65 42 +``` + +The `GLO-30` and `GLO-90` products are distributed by the Earth Data Hub as a +single cloud-hosted Zarr store that is read in place and cached one chunk at a +time instead of downloading whole tiles, and the credentials of the account +stored in `~/.netrc` are used to access it: + +```console +machine data.earthdatahub.destine.eu + login + password +``` + +Reading the store needs GDAL 3.8 or later. + The `--bounds` option accepts latitude and longitude coordinates (more precisely in geodetic coordinates in the WGS84 reference system EPSG:4326 for those who care) given as `left bottom right top` similarly to the `rio` command form `rasterio`. @@ -91,6 +115,8 @@ The first time an area is accessed Elevation downloads the data tiles from the AWS S3, CGIAR-CSI or OpenTopography servers and caches them in GeoTIFF compressed formats, subsequent accesses to the same and nearby areas are much faster. +The `GLO-30` and `GLO-90` products are the exception: they are read in place +from the Earth Data Hub and cached one Zarr chunk at a time. The `clip` sub-command doesn't allow automatic download of a large amount of DEM tiles, please refer to the upstream providers' websites to learn the preferred procedures for bulk download. @@ -121,8 +147,8 @@ $ eio --help ╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮ │ --version Show the version and exit. │ │ [env var: EIO_VERSION] │ -│ --product [MAPZEN|SRTM1_GEOID|SRTM1_ELLIP|SRTM3 DEM product choice. │ -│ ] [env var: EIO_PRODUCT] │ +│ --product [MAPZEN|GLO-30|GLO-90|SRTM1_GEOID|SRT DEM product choice. │ +│ M1_ELLIP|SRTM3] [env var: EIO_PRODUCT] │ │ [default: MAPZEN] │ │ --cache_dir Root of the DEM cache folder. │ │ [env var: EIO_CACHE_DIR] │ diff --git a/docs/index.md b/docs/index.md index 9f52225..0cba758 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,7 +1,7 @@ # elevation Download, cache and clip global terrain digital elevation models: Mapzen terrain -tiles, SRTM 30m and 90m DEMs. +tiles, SRTM 30m and 90m DEMs, Copernicus DEM 30m and 90m DSMs. If you have any feedback or you want to help out head over to our main repository: https://github.com/bopen/elevation diff --git a/elevation/datasets/elevation.yaml b/elevation/datasets/elevation.yaml index a8911d5..946eec5 100644 --- a/elevation/datasets/elevation.yaml +++ b/elevation/datasets/elevation.yaml @@ -1,6 +1,6 @@ id: elevation title: elevation datasets -description: 'Download, cache and clip global terrain digital elevation models: Mapzen terrain tiles, SRTM 30m and 90m DEMs.' +description: 'Download, cache and clip global terrain digital elevation models: Mapzen terrain tiles, SRTM 30m and 90m DEMs, Copernicus DEM 30m and 90m DSMs.' type: Catalog stac_version: 1.1.0 links: diff --git a/elevation/datasource.py b/elevation/datasource.py index 975c23d..d872678 100644 --- a/elevation/datasource.py +++ b/elevation/datasource.py @@ -20,7 +20,7 @@ from collections.abc import Callable, Iterator, Sequence from importlib import resources from pathlib import Path -from typing import TypedDict +from typing import Any, NotRequired, TypedDict import appdirs @@ -47,10 +47,20 @@ CACHE_DIR: str = appdirs.user_cache_dir("elevation", "bopen") DEFAULT_OUTPUT = "out.tif" DEFAULT_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co PREDICTOR=2" -# the VRT mosaic reads the cache tiles whole, the options only trade size for speed -TILE_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co PREDICTOR=2" +CACHE_EXT = ".tif" +TILE_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9" +INT_TILE_GDAL_OPTIONS = TILE_GDAL_OPTIONS + " -co PREDICTOR=2" +FLOAT_TILE_GDAL_OPTIONS = TILE_GDAL_OPTIONS + " -co PREDICTOR=3" MARGIN = "0" +# NOTE: +# 0.0001388888889 == 0.5" is half pixel for DEMs with 1" spacing (DTED L2) +# 0.0004166666667 == 1.5" is half pixel for DEMs with 3" spacing (DTED L1) +EDH_L2_CHUNK_INDECES_TRANSFORM = (-180.0001388888889, 1.0, 90.00013888888888, -0.5) +EDH_L1_CHUNK_INDECES_TRANSFORM = (-180.00041666666667, 2.0, 90.00041666666667, -2.0) +DTED_L2_TILE_INDECES_TRANSFORM = (-0.0001388888889, 1.0, -0.0001388888889, 1.0) +CGIAR_L1_TILE_INDECES_TRANSFORM = (-185.0004166666667, 5.0, 65.0004166666667, -5.0) + def resolve_cache_dir(cache_dir: str | Path | None) -> Path: """Return the DEM cache folder to use, as an absolute path. @@ -63,24 +73,27 @@ def resolve_cache_dir(cache_dir: str | Path | None) -> Path: return Path(cache_dir).resolve() -def srtm1_tile_ilonlat(lon: float, lat: float) -> tuple[int, int]: - return math.floor(lon), math.floor(lat) +def latlon_to_indeces( + transform: tuple[float, float, float, float], lon: float, lat: float +) -> tuple[int, int]: + lon_start, lon_step, lat_start, lat_step = transform + ilon = math.floor((lon - lon_start) / lon_step) + ilat = math.floor((lat - lat_start) / lat_step) + return ilon, ilat -def srtm3_tile_ilonlat(lon: float, lat: float) -> tuple[int, int]: - ilon, ilat = srtm1_tile_ilonlat(lon, lat) - return (ilon + 180) // 5 + 1, (64 - ilat) // 5 +Tile = tuple[tuple[int, int], str] -def srtm1_tiles_names( +def dted_l2_tiles( left: float, bottom: float, right: float, top: float, tile_name_template: str = "{slat}{slon}.tif", -) -> Iterator[str]: - ileft, itop = srtm1_tile_ilonlat(left, top) - iright, ibottom = srtm1_tile_ilonlat(right, bottom) +) -> Iterator[Tile]: + ileft, itop = latlon_to_indeces(DTED_L2_TILE_INDECES_TRANSFORM, left, top) + iright, ibottom = latlon_to_indeces(DTED_L2_TILE_INDECES_TRANSFORM, right, bottom) # special case often used *integer* top and right to avoid downloading unneeded tiles if isinstance(top, int) or top.is_integer(): itop -= 1 @@ -90,34 +103,38 @@ def srtm1_tiles_names( slon = f"{'E' if ilon >= 0 else 'W'}{abs(ilon):03d}" for ilat in range(ibottom, itop + 1): slat = f"{'N' if ilat >= 0 else 'S'}{abs(ilat):02d}" - yield tile_name_template.format(**locals()) + yield (ilon, ilat), tile_name_template.format(**locals()) -def srtm3_tiles_names( +def cgiar_l1_tiles( left: float, bottom: float, right: float, top: float, tile_template: str = "srtm_{ilon:02d}_{ilat:02d}.tif", -) -> Iterator[str]: - ileft, itop = srtm3_tile_ilonlat(left, top) - iright, ibottom = srtm3_tile_ilonlat(right, bottom) +) -> Iterator[Tile]: + ileft, itop = latlon_to_indeces(CGIAR_L1_TILE_INDECES_TRANSFORM, left, top) + iright, ibottom = latlon_to_indeces(CGIAR_L1_TILE_INDECES_TRANSFORM, right, bottom) for ilon in range(ileft, iright + 1): for ilat in range(itop, ibottom + 1): if ilon > 0 and ilat > 0: - yield tile_template.format(**locals()) + yield (ilon, ilat), tile_template.format(**locals()) -def srtm_ellip_tiles_names( +def srtm_ellip_tiles( left: float, bottom: float, right: float, top: float, tile_name_template: str = "{slat}{slon}_wgs84.tif", -) -> Iterator[str]: - ileft, itop = srtm1_tile_ilonlat(left, top) - iright, ibottom = srtm1_tile_ilonlat(right, bottom) - +) -> Iterator[Tile]: + ileft, itop = latlon_to_indeces(DTED_L2_TILE_INDECES_TRANSFORM, left, top) + iright, ibottom = latlon_to_indeces(DTED_L2_TILE_INDECES_TRANSFORM, right, bottom) + # special case often used *integer* top and right to avoid downloading unneeded tiles + if isinstance(top, int) or top.is_integer(): + itop -= 1 + if isinstance(right, int) or right.is_integer(): + iright -= 1 for ilon in range(ileft, iright + 1): slon = f"{'E' if ilon >= 0 else 'W'}{abs(ilon):03d}" for ilat in range(ibottom, itop + 1): @@ -127,59 +144,136 @@ def srtm_ellip_tiles_names( fname = tile_name_template.format(**locals()) if ilat >= 0: - yield f"{subdir}/{north_subdir}/{fname}" + yield (ilon, ilat), f"{subdir}/{north_subdir}/{fname}" else: - yield f"{subdir}/{fname}" + yield (ilon, ilat), f"{subdir}/{fname}" + +def prepare_tile_download_uncompress( + tile_name: str, + spool: Path, + datasource_url: str, + ilon: int, + ilat: int, + **kwargs: Any, +) -> tuple[list[str], Path | None]: + source, spool_name, member = tile_source(datasource_url, tile_name, **kwargs) + spooled = spool / spool_name + fetch_tile(source, spooled, member=member) + return [str(spooled)], spooled -def mapzen_tiles_names( - left: float, bottom: float, right: float, top: float -) -> Iterator[str]: - yield from srtm1_tiles_names(left, bottom, right, top, "{slat}/{slat}{slon}.tif") + +def zarr_tiles( + left: float, + bottom: float, + right: float, + top: float, + transform: tuple[float, float, float, float], +) -> Iterator[Tile]: + ileft, itop = latlon_to_indeces(transform, left, top) + iright, ibottom = latlon_to_indeces(transform, right, bottom) + for ilon in range(ileft, iright + 1): + for ilat in range(itop, ibottom + 1): + if ilon >= 0 and ilat >= 0: + yield (ilon, ilat), f"{ilat}/{ilon}.tif" + + +def prepare_tile_zarr( + tile_name: str, + spool: Path, + datasource_url: str, + ilat: int, + ilon: int, + chunks: tuple[int, int], + **kwargs: Any, +) -> tuple[list[str], Path | None]: + srcwin = [ilon * chunks[0], ilat * chunks[1], chunks[0], chunks[1]] + gdal_source = [ + "-srcwin", + *map(str, srcwin), + f'ZARR:"/vsicurl/{datasource_url}":/dsm', + ] + return gdal_source, None class DatasourceSpec(TypedDict): - folders: tuple[str, ...] - datasource_url: str - tile_ext: str - compressed_ext: str - tile_names: Callable[..., Iterator[str]] + # a local product has one URL per tile (``tiles``), a remote one is a + # single chunked source (``grid``): the key tells the two apart + cached_tiles: Callable[..., Iterator[Tile]] + # keyword arguments for ``cached_tiles``, e.g. the tile name template + # of a product that keeps its tiles in subfolders + cached_tiles_kwargs: NotRequired[dict[str, Any]] + # prepare the tile for GDAL, downloading it or reading the window of the + # chunked source, next to the spool file to remove once it is cached + prepare_tile: Callable[..., tuple[list[str], Path | None]] + # keyword arguments for ``prepare_tile``: the datasource URL, the source + # extension, the archive the provider serves it in, the chunk size + prepare_tile_kwargs: dict[str, Any] + tile_gdal_options: NotRequired[str] MAPZEN_SPEC: DatasourceSpec = { - "folders": ("spool", "cache"), - "datasource_url": "https://s3.amazonaws.com/elevation-tiles-prod/skadi", - "tile_ext": ".hgt", - "compressed_ext": ".hgt.gz", - "tile_names": mapzen_tiles_names, + "prepare_tile": prepare_tile_download_uncompress, + "prepare_tile_kwargs": { + "datasource_url": "https://s3.amazonaws.com/elevation-tiles-prod/skadi", + "tile_ext": ".hgt", + "compressed_ext": ".hgt.gz", + }, + "cached_tiles": dted_l2_tiles, + "cached_tiles_kwargs": {"tile_name_template": "{slat}/{slat}{slon}.tif"}, } SRTM1_GEOID_SPEC: DatasourceSpec = { - "folders": ("spool", "cache"), - "datasource_url": "https://opentopography.s3.sdsc.edu/raster/SRTM_GL1/SRTM_GL1_srtm", - "tile_ext": ".tif", - "compressed_ext": "", - "tile_names": srtm1_tiles_names, + "prepare_tile": prepare_tile_download_uncompress, + "prepare_tile_kwargs": { + "datasource_url": "https://opentopography.s3.sdsc.edu/raster/SRTM_GL1/SRTM_GL1_srtm", + }, + "cached_tiles": dted_l2_tiles, } SRTM1_ELLIP_SPEC: DatasourceSpec = { - "folders": ("spool", "cache"), - "datasource_url": "https://opentopography.s3.sdsc.edu/raster/SRTM_GL1_Ellip/SRTM_GL1_Ellip_srtm", - "tile_ext": ".tif", - "compressed_ext": "", - "tile_names": srtm_ellip_tiles_names, + "prepare_tile": prepare_tile_download_uncompress, + "prepare_tile_kwargs": { + "datasource_url": "https://opentopography.s3.sdsc.edu/raster/SRTM_GL1_Ellip/SRTM_GL1_Ellip_srtm", + }, + "cached_tiles": srtm_ellip_tiles, } SRTM3_SPEC: DatasourceSpec = { - "folders": ("spool", "cache"), - "datasource_url": "https://srtm.csi.cgiar.org/wp-content/uploads/files/srtm_5x5/TIFF", - "tile_ext": ".tif", - "compressed_ext": ".zip", - "tile_names": srtm3_tiles_names, + "prepare_tile_kwargs": { + "datasource_url": "https://srtm.csi.cgiar.org/wp-content/uploads/files/srtm_5x5/TIFF", + "compressed_ext": ".zip", + }, + "cached_tiles": cgiar_l1_tiles, + "prepare_tile": prepare_tile_download_uncompress, +} + +GLO_30_SPEC: DatasourceSpec = { + "prepare_tile": prepare_tile_zarr, + "prepare_tile_kwargs": { + "datasource_url": "https://data.earthdatahub.destine.eu/copernicus-dem/GLO-30-v1.zarr", + "chunks": (3600, 1800), + }, + "tile_gdal_options": FLOAT_TILE_GDAL_OPTIONS, + "cached_tiles": zarr_tiles, + "cached_tiles_kwargs": {"transform": EDH_L2_CHUNK_INDECES_TRANSFORM}, +} + +GLO_90_SPEC: DatasourceSpec = { + "prepare_tile": prepare_tile_zarr, + "prepare_tile_kwargs": { + "datasource_url": "https://data.earthdatahub.destine.eu/copernicus-dem/GLO-90-v1.zarr", + "chunks": (2400, 2400), + }, + "tile_gdal_options": FLOAT_TILE_GDAL_OPTIONS, + "cached_tiles": zarr_tiles, + "cached_tiles_kwargs": {"transform": EDH_L1_CHUNK_INDECES_TRANSFORM}, } PRODUCTS_SPECS: dict[str, DatasourceSpec] = { "MAPZEN": MAPZEN_SPEC, + "GLO-30": GLO_30_SPEC, + "GLO-90": GLO_90_SPEC, "SRTM1_GEOID": SRTM1_GEOID_SPEC, "SRTM1_ELLIP": SRTM1_ELLIP_SPEC, "SRTM3": SRTM3_SPEC, @@ -210,10 +304,12 @@ def __str__(self) -> str: } -CACHE_EXT = ".tif" - - -def tile_source(spec: DatasourceSpec, tile_name: str) -> tuple[str, str, str | None]: +def tile_source( + datasource_url: str, + tile_name: str, + tile_ext: str = ".tif", + compressed_ext: str | None = None, +) -> tuple[str, str, str | None]: """Return the ``(url, spool_name, member)`` of the tile for *tile_name*. *tile_name* is the cache tile name, always a ``.tif``; the spool name is the @@ -222,11 +318,10 @@ def tile_source(spec: DatasourceSpec, tile_name: str) -> tuple[str, str, str | N to read inside a ``.zip`` archive and ``None`` otherwise. """ stem = tile_name.removesuffix(CACHE_EXT) - spool_name = f"{stem}{spec['tile_ext']}" - compressed_ext = spec["compressed_ext"] - remote = spool_name if not compressed_ext else f"{stem}{compressed_ext}" + spool_name = f"{stem}{tile_ext}" + remote = spool_name if compressed_ext is None else f"{stem}{compressed_ext}" member = Path(spool_name).name if compressed_ext == ".zip" else None - return f"{spec['datasource_url']}/{remote}", spool_name, member + return f"{datasource_url}/{remote}", spool_name, member def fetch_tile(source: str, destination: Path, *, member: str | None = None) -> None: @@ -259,10 +354,9 @@ def fetch_tile(source: str, destination: Path, *, member: str | None = None) -> def write_cache_tile( - source: str | Path, + gdal_source: Sequence[str], destination: Path, *, - srcwin: Sequence[int] | None = None, gdal_options: str = TILE_GDAL_OPTIONS, ) -> list[str]: """Write *source* to *destination* as the internal compressed GeoTIFF tile. @@ -278,30 +372,49 @@ def write_cache_tile( :return: The command arguments. """ destination.parent.mkdir(parents=True, exist_ok=True) - window = [] if srcwin is None else ["-srcwin", *map(str, srcwin)] cmd = [ "gdal_translate", "-q", *gdal_options.split(), - *window, - str(source), + *gdal_source, str(destination), ] subprocess.check_call(cmd) return cmd -def ensure_tiles(root: Path, spec: DatasourceSpec, tile_names: Sequence[str]) -> None: - """Fetch and cache *tile_names*, skipping the tiles already in the cache.""" - for tile_name in tile_names: +def ensure_tiles( + root: Path, + tiles: Sequence[Tile], + prepare_tile: Callable[..., tuple[list[str], Path | None]], + gdal_options: str = TILE_GDAL_OPTIONS, + **kwargs: Any, +) -> None: + """Fetch and cache *tiles*, skipping the tiles already in the cache. + + A tile is a ``(name, window)`` pair: a tile with a window is read in place + from ``datasource_url``, a tile without one is downloaded whole from its own + URL and goes through the spool. + """ + for (ilon, ilat), tile_name in tiles: cached = root / "cache" / tile_name if cached.exists() and cached.stat().st_size > 0: continue - source, spool_name, member = tile_source(spec, tile_name) - spooled = root / "spool" / spool_name - fetch_tile(source, spooled, member=member) - write_cache_tile(spooled, cached) - spooled.unlink(missing_ok=True) + + # prepare the data if GDAL cannot download it / read it as it is + gdal_source, spooled = prepare_tile( + tile_name, root / "spool", ilat=ilat, ilon=ilon, **kwargs + ) + + # convert the data to the internal cache format + ready = root / "spool/ready" / tile_name + write_cache_tile(gdal_source, ready, gdal_options=gdal_options) + if spooled is not None: + spooled.unlink(missing_ok=True) + + # finally move the data inside the cache. The move is atomic in most cases + cached.parent.mkdir(parents=True, exist_ok=True) + shutil.move(ready, cached) def build_vrt(root: Path, product: str) -> list[str]: @@ -327,7 +440,7 @@ def ensure_setup( raise ProductRetiredError(RETIRED_PRODUCTS[product]) datasource_root = resolve_cache_dir(cache_dir) / product spec = PRODUCTS_SPECS[product] - util.ensure_setup(datasource_root, folders=spec["folders"]) + util.ensure_setup(datasource_root) return datasource_root, spec @@ -354,6 +467,9 @@ def seed( ) -> Path: """Seed the DEM to given bounds. + A remote product is not downloaded whole: only the chunks of the store that + cover the bounds are read in place and cached. + :param cache_dir: Root of the DEM cache folder. :param product: DEM product choice. :param bounds: Output bounds in 'left bottom right top' order. @@ -362,19 +478,30 @@ def seed( if bounds is None: raise TypeError("bounds must be supplied") datasource_root, spec = ensure_setup(cache_dir, product) - ensure_tiles_names = list(spec["tile_names"](*bounds)) + cached_tiles = spec["cached_tiles"] + cached_tiles_kwargs = spec.get("cached_tiles_kwargs", {}) + tiles = list(cached_tiles(*bounds, **cached_tiles_kwargs)) # FIXME: emergency hack to enforce the no-bulk-download policy - if len(ensure_tiles_names) > max_download_tiles: + if len(tiles) > max_download_tiles: raise RuntimeError( - f"Too many tiles: {len(ensure_tiles_names)}. Please consult the " + f"Too many tiles: {len(tiles)}. Please consult the " "providers' websites for how to bulk download tiles." ) - with util.lock_tiles(datasource_root, ensure_tiles_names): - ensure_tiles(datasource_root, spec, ensure_tiles_names) + prepare_tile = spec["prepare_tile"] + prepare_tile_kwargs = spec.get("prepare_tile_kwargs", {}) + with util.lock_tiles(datasource_root, [name for _, name in tiles]): + ensure_tiles( + datasource_root, + tiles, + prepare_tile=prepare_tile, + gdal_options=spec.get("tile_gdal_options", TILE_GDAL_OPTIONS), + **prepare_tile_kwargs, + ) with util.lock_vrt(datasource_root, product): build_vrt(datasource_root, product) + return datasource_root @@ -388,12 +515,13 @@ def build_bounds( margin_lat = (top - bottom) * margin_percent / 100 else: margin_lon = margin_lat = float(margin) - return ( + bounds = ( left - margin_lon, bottom - margin_lat, right + margin_lon, top + margin_lat, ) + return bounds def clip( diff --git a/elevation/util.py b/elevation/util.py index 49531b0..e224485 100644 --- a/elevation/util.py +++ b/elevation/util.py @@ -65,12 +65,11 @@ def lock_vrt(datasource_root: Path, product: str) -> Generator[None]: yield -def ensure_setup(root: Path, folders: Iterable[str] = ()) -> list[Path]: - """Create *root* and the *folders* in it, returning the created folders.""" +def ensure_setup(root: Path) -> None: + """Create the product folder and its ``cache`` subfolder. + + The ``spool`` folder is created on demand by the tile download. + """ with fasteners.InterProcessLock(root / FOLDER_LOCKFILE_NAME): - created_folders = [] - for path in [root] + [root / p for p in folders]: - if not path.exists(): - path.mkdir(parents=True) - created_folders.append(path) - return created_folders + for path in (root, root / "cache"): + path.mkdir(parents=True, exist_ok=True) diff --git a/tests/conftest.py b/tests/conftest.py index 2c2cf11..e504a82 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -107,7 +107,7 @@ def integration_data( def integrate( product: str, name: str, bounds: tuple[float, float, float, float] ) -> None: - reference = REFERENCE_DATA_DIR / product.lower() / f"{name}.tif" + reference = REFERENCE_DATA_DIR / product / f"{name}.tif" if not update and not reference.exists(): pytest.skip(f"missing {reference}: run --update-integration-data") output = tmp_path / f"{name}.tif" diff --git a/tests/data/GLO-30/ne_rome.tif b/tests/data/GLO-30/ne_rome.tif new file mode 100644 index 0000000..a24fa0c Binary files /dev/null and b/tests/data/GLO-30/ne_rome.tif differ diff --git a/tests/data/GLO-30/nw_san_francisco.tif b/tests/data/GLO-30/nw_san_francisco.tif new file mode 100644 index 0000000..722e66e Binary files /dev/null and b/tests/data/GLO-30/nw_san_francisco.tif differ diff --git a/tests/data/GLO-30/se_sydney.tif b/tests/data/GLO-30/se_sydney.tif new file mode 100644 index 0000000..79a3843 Binary files /dev/null and b/tests/data/GLO-30/se_sydney.tif differ diff --git a/tests/data/GLO-30/sw_santiago.tif b/tests/data/GLO-30/sw_santiago.tif new file mode 100644 index 0000000..7564a0a Binary files /dev/null and b/tests/data/GLO-30/sw_santiago.tif differ diff --git a/tests/data/GLO-90/ne_rome.tif b/tests/data/GLO-90/ne_rome.tif new file mode 100644 index 0000000..7cf720d Binary files /dev/null and b/tests/data/GLO-90/ne_rome.tif differ diff --git a/tests/data/GLO-90/nw_san_francisco.tif b/tests/data/GLO-90/nw_san_francisco.tif new file mode 100644 index 0000000..58d7aac Binary files /dev/null and b/tests/data/GLO-90/nw_san_francisco.tif differ diff --git a/tests/data/GLO-90/se_sydney.tif b/tests/data/GLO-90/se_sydney.tif new file mode 100644 index 0000000..7183209 Binary files /dev/null and b/tests/data/GLO-90/se_sydney.tif differ diff --git a/tests/data/GLO-90/sw_santiago.tif b/tests/data/GLO-90/sw_santiago.tif new file mode 100644 index 0000000..5deb6b0 Binary files /dev/null and b/tests/data/GLO-90/sw_santiago.tif differ diff --git a/tests/data/mapzen/ne_rome.tif b/tests/data/MAPZEN/ne_rome.tif similarity index 100% rename from tests/data/mapzen/ne_rome.tif rename to tests/data/MAPZEN/ne_rome.tif diff --git a/tests/data/mapzen/nw_iceland.tif b/tests/data/MAPZEN/nw_iceland.tif similarity index 100% rename from tests/data/mapzen/nw_iceland.tif rename to tests/data/MAPZEN/nw_iceland.tif diff --git a/tests/data/mapzen/nw_san_francisco.tif b/tests/data/MAPZEN/nw_san_francisco.tif similarity index 100% rename from tests/data/mapzen/nw_san_francisco.tif rename to tests/data/MAPZEN/nw_san_francisco.tif diff --git a/tests/data/mapzen/se_sydney.tif b/tests/data/MAPZEN/se_sydney.tif similarity index 100% rename from tests/data/mapzen/se_sydney.tif rename to tests/data/MAPZEN/se_sydney.tif diff --git a/tests/data/mapzen/sw_santiago.tif b/tests/data/MAPZEN/sw_santiago.tif similarity index 100% rename from tests/data/mapzen/sw_santiago.tif rename to tests/data/MAPZEN/sw_santiago.tif diff --git a/tests/data/srtm1_ellip/ne_rome.tif b/tests/data/SRTM1_ELLIP/ne_rome.tif similarity index 100% rename from tests/data/srtm1_ellip/ne_rome.tif rename to tests/data/SRTM1_ELLIP/ne_rome.tif diff --git a/tests/data/srtm1_ellip/nw_bogota.tif b/tests/data/SRTM1_ELLIP/nw_bogota.tif similarity index 100% rename from tests/data/srtm1_ellip/nw_bogota.tif rename to tests/data/SRTM1_ELLIP/nw_bogota.tif diff --git a/tests/data/srtm1_ellip/nw_san_francisco.tif b/tests/data/SRTM1_ELLIP/nw_san_francisco.tif similarity index 100% rename from tests/data/srtm1_ellip/nw_san_francisco.tif rename to tests/data/SRTM1_ELLIP/nw_san_francisco.tif diff --git a/tests/data/srtm1_ellip/se_sydney.tif b/tests/data/SRTM1_ELLIP/se_sydney.tif similarity index 100% rename from tests/data/srtm1_ellip/se_sydney.tif rename to tests/data/SRTM1_ELLIP/se_sydney.tif diff --git a/tests/data/srtm1_ellip/sw_santiago.tif b/tests/data/SRTM1_ELLIP/sw_santiago.tif similarity index 100% rename from tests/data/srtm1_ellip/sw_santiago.tif rename to tests/data/SRTM1_ELLIP/sw_santiago.tif diff --git a/tests/data/srtm1_geoid/ne_rome.tif b/tests/data/SRTM1_GEOID/ne_rome.tif similarity index 100% rename from tests/data/srtm1_geoid/ne_rome.tif rename to tests/data/SRTM1_GEOID/ne_rome.tif diff --git a/tests/data/srtm1_geoid/nw_san_francisco.tif b/tests/data/SRTM1_GEOID/nw_san_francisco.tif similarity index 100% rename from tests/data/srtm1_geoid/nw_san_francisco.tif rename to tests/data/SRTM1_GEOID/nw_san_francisco.tif diff --git a/tests/data/srtm1_geoid/se_sydney.tif b/tests/data/SRTM1_GEOID/se_sydney.tif similarity index 100% rename from tests/data/srtm1_geoid/se_sydney.tif rename to tests/data/SRTM1_GEOID/se_sydney.tif diff --git a/tests/data/srtm1_geoid/sw_santiago.tif b/tests/data/SRTM1_GEOID/sw_santiago.tif similarity index 100% rename from tests/data/srtm1_geoid/sw_santiago.tif rename to tests/data/SRTM1_GEOID/sw_santiago.tif diff --git a/tests/data/srtm3/ne_rome.tif b/tests/data/SRTM3/ne_rome.tif similarity index 100% rename from tests/data/srtm3/ne_rome.tif rename to tests/data/SRTM3/ne_rome.tif diff --git a/tests/data/srtm3/nw_san_francisco.tif b/tests/data/SRTM3/nw_san_francisco.tif similarity index 100% rename from tests/data/srtm3/nw_san_francisco.tif rename to tests/data/SRTM3/nw_san_francisco.tif diff --git a/tests/data/srtm3/se_sydney.tif b/tests/data/SRTM3/se_sydney.tif similarity index 100% rename from tests/data/srtm3/se_sydney.tif rename to tests/data/SRTM3/se_sydney.tif diff --git a/tests/data/srtm3/sw_santiago.tif b/tests/data/SRTM3/sw_santiago.tif similarity index 100% rename from tests/data/srtm3/sw_santiago.tif rename to tests/data/SRTM3/sw_santiago.tif diff --git a/tests/integration_glo_30.py b/tests/integration_glo_30.py new file mode 100644 index 0000000..d1283c0 --- /dev/null +++ b/tests/integration_glo_30.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2016-2026 B-Open Solutions srl - https://bopen.eu +# + +"""Integration tests for the ``GLO-30`` product, i.e. Copernicus DEM 30m. + +The product is a north-up Zarr store on the Earth Data Hub that is read in place +instead of being downloaded, so these tests also cover the Earth Data Hub +credentials in ``~/.netrc`` and the ``ZARR:`` GDAL connection string. +The regions are the same as the ones of ``GLO-90`` and ``SRTM3``, so the three +reference datasets cover the same areas and can be cross-checked. +Each test clips a ~100x100 pixel DEM and compares it with the reference GeoTIFF +committed in ``tests/data/GLO-30``, see ``CONTRIBUTING.rst`` to regenerate them. +""" + +from collections.abc import Callable + +SIZE = 100 / 3600 + +IntegrationData = Callable[[str, str, tuple[float, float, float, float]], None] + + +def test_ne_rome(integration_data: IntegrationData) -> None: + bounds = (12.4, 41.8, 12.4 + SIZE, 41.8 + SIZE) + integration_data("GLO-30", "ne_rome", bounds) + + +def test_nw_san_francisco(integration_data: IntegrationData) -> None: + bounds = (-122.44, 37.74, -122.44 + SIZE, 37.74 + SIZE) + integration_data("GLO-30", "nw_san_francisco", bounds) + + +def test_se_sydney(integration_data: IntegrationData) -> None: + bounds = (151.16, -33.9, 151.16 + SIZE, -33.9 + SIZE) + integration_data("GLO-30", "se_sydney", bounds) + + +def test_sw_santiago(integration_data: IntegrationData) -> None: + bounds = (-70.6, -33.42, -70.6 + SIZE, -33.42 + SIZE) + integration_data("GLO-30", "sw_santiago", bounds) diff --git a/tests/integration_glo_90.py b/tests/integration_glo_90.py new file mode 100644 index 0000000..7d9e049 --- /dev/null +++ b/tests/integration_glo_90.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2016-2026 B-Open Solutions srl - https://bopen.eu +# + +"""Integration tests for the ``GLO-90`` product, i.e. Copernicus DEM 90m. + +The product is a north-up Zarr store on the Earth Data Hub that is read in place +instead of being downloaded, so these tests also cover the Earth Data Hub +credentials in ``~/.netrc`` and the ``ZARR:`` GDAL connection string. +The regions are the same as the ones of ``SRTM3``, so the two reference datasets +cover the same areas and can be cross-checked, even though the two grids are half +a pixel apart. +Each test clips a ~100x100 pixel DEM and compares it with the reference GeoTIFF +committed in ``tests/data/GLO-90``, see ``CONTRIBUTING.rst`` to regenerate them. +""" + +from collections.abc import Callable + +SIZE = 100 / 1200 + +IntegrationData = Callable[[str, str, tuple[float, float, float, float]], None] + + +def test_ne_rome(integration_data: IntegrationData) -> None: + bounds = (12.4, 41.8, 12.4 + SIZE, 41.8 + SIZE) + integration_data("GLO-90", "ne_rome", bounds) + + +def test_nw_san_francisco(integration_data: IntegrationData) -> None: + bounds = (-122.44, 37.74, -122.44 + SIZE, 37.74 + SIZE) + integration_data("GLO-90", "nw_san_francisco", bounds) + + +def test_se_sydney(integration_data: IntegrationData) -> None: + bounds = (151.16, -33.9, 151.16 + SIZE, -33.9 + SIZE) + integration_data("GLO-90", "se_sydney", bounds) + + +def test_sw_santiago(integration_data: IntegrationData) -> None: + bounds = (-70.6, -33.42, -70.6 + SIZE, -33.42 + SIZE) + integration_data("GLO-90", "sw_santiago", bounds) diff --git a/tests/integration_mapzen.py b/tests/integration_mapzen.py index e4d4c3c..f61e6b7 100644 --- a/tests/integration_mapzen.py +++ b/tests/integration_mapzen.py @@ -8,7 +8,7 @@ chosen to cover plain SRTM data, the USGS 3DEP data in the United States and the high latitudes that no SRTM product covers. Each test clips a ~100x100 pixel DEM and compares it with the reference GeoTIFF -committed in ``tests/data/mapzen``, see ``CONTRIBUTING.rst``. +committed in ``tests/data/MAPZEN``, see ``CONTRIBUTING.rst``. """ from collections.abc import Callable diff --git a/tests/integration_srtm1_ellip.py b/tests/integration_srtm1_ellip.py index e13cb66..11974c6 100644 --- a/tests/integration_srtm1_ellip.py +++ b/tests/integration_srtm1_ellip.py @@ -7,7 +7,7 @@ The tile names of this product are grouped in ``North/North_30_60``, ``North/North_0_29`` and ``South`` folders, all of them are covered here. Each test clips a ~100x100 pixel DEM and compares it with the reference GeoTIFF -committed in ``tests/data/srtm1_ellip``, see ``CONTRIBUTING.rst`` to regenerate them. +committed in ``tests/data/SRTM1_ELLIP``, see ``CONTRIBUTING.rst`` to regenerate them. """ from collections.abc import Callable diff --git a/tests/integration_srtm1_geoid.py b/tests/integration_srtm1_geoid.py index 4cb8dd4..b5bf5d7 100644 --- a/tests/integration_srtm1_geoid.py +++ b/tests/integration_srtm1_geoid.py @@ -5,7 +5,7 @@ """Integration tests for the ``SRTM1_GEOID`` product, i.e. SRTM GL1 on OpenTopography. Each test clips a ~100x100 pixel DEM and compares it with the reference GeoTIFF -committed in ``tests/data/srtm1_geoid``, see ``CONTRIBUTING.rst`` to regenerate them. +committed in ``tests/data/SRTM1_GEOID``, see ``CONTRIBUTING.rst`` to regenerate them. """ from collections.abc import Callable diff --git a/tests/integration_srtm3.py b/tests/integration_srtm3.py index b08a1f6..80dfa6c 100644 --- a/tests/integration_srtm3.py +++ b/tests/integration_srtm3.py @@ -5,7 +5,7 @@ """Integration tests for the ``SRTM3`` product, i.e. CGIAR-CSI SRTM 90m. Each test clips a ~100x100 pixel DEM and compares it with the reference GeoTIFF -committed in ``tests/data/srtm3``, see ``CONTRIBUTING.rst`` to regenerate them. +committed in ``tests/data/SRTM3``, see ``CONTRIBUTING.rst`` to regenerate them. """ from collections.abc import Callable diff --git a/tests/test_10_util.py b/tests/test_10_util.py index bb8606b..f520d97 100644 --- a/tests/test_10_util.py +++ b/tests/test_10_util.py @@ -27,17 +27,9 @@ def test_lock_vrt(tmp_path: Path) -> None: def test_ensure_setup(tmp_path: Path) -> None: root = tmp_path / "root" - created_folders = util.ensure_setup(root) - assert len(created_folders) == 0 - assert len(list(tmp_path.iterdir())) == 1 - - folders = ["etc", "lib"] - created_folders = util.ensure_setup(root, folders=folders) - assert len(created_folders) == 2 - assert created_folders[0].name == "etc" - assert created_folders[1].name == "lib" - assert len(list(root.iterdir())) == 3 - - created_folders = util.ensure_setup(root, folders=folders) - assert len(created_folders) == 0 - assert len(list(root.iterdir())) == 3 + + util.ensure_setup(root) + + assert (root / "cache").is_dir() + # the spool folder is created on demand by the tile download + assert not (root / "spool").exists() diff --git a/tests/test_20_datasource.py b/tests/test_20_datasource.py index 1f46130..371d04c 100644 --- a/tests/test_20_datasource.py +++ b/tests/test_20_datasource.py @@ -30,101 +30,246 @@ def gdalinfo_json(path: Path) -> dict[str, Any]: return info -def test_srtm3_tile_ilonlat() -> None: - # values from https://srtm.csi.cgiar.org/SELECTION/inputCoord.asp - assert datasource.srtm3_tile_ilonlat(-177.5, 52.5) == (1, 2) - assert datasource.srtm3_tile_ilonlat(177.5, -47.5) == (72, 22) - assert datasource.srtm3_tile_ilonlat(10.1, 44.9) == (39, 4) - assert datasource.srtm3_tile_ilonlat(14.9, 44.9) == (39, 4) - assert datasource.srtm3_tile_ilonlat(10.1, 40.1) == (39, 4) - assert datasource.srtm3_tile_ilonlat(14.9, 40.1) == (39, 4) +def write_ready_tile(source: str | Path, ready: Path, **kwargs: Any) -> None: + """Stand in for the mocked cache write: leave a tile for the cache move.""" + ready.parent.mkdir(parents=True, exist_ok=True) + ready.write_bytes(b"tile") -def test_srtm1_tiles_names() -> None: - assert list(datasource.srtm1_tiles_names(10.1, 44.9, 10.1, 44.9)) == ["N44E010.tif"] +def test_latlon_to_indeces_CGIAR_L1_TILE_INDECES_TRANSFORM() -> None: + transform = datasource.CGIAR_L1_TILE_INDECES_TRANSFORM + # values from https://srtm.csi.cgiar.org/SELECTION/inputCoord.asp + assert datasource.latlon_to_indeces(transform, -177.5, 52.5) == (1, 2) + assert datasource.latlon_to_indeces(transform, 177.5, -47.5) == (72, 22) + assert datasource.latlon_to_indeces(transform, 10.1, 44.9) == (39, 4) + assert datasource.latlon_to_indeces(transform, 14.9, 44.9) == (39, 4) + assert datasource.latlon_to_indeces(transform, 10.1, 40.1) == (39, 4) + assert datasource.latlon_to_indeces(transform, 14.9, 40.1) == (39, 4) + + +def test_latlon_to_indeces_DTED_L2_TILE_INDECES_TRANSFORM() -> None: + transform = datasource.DTED_L2_TILE_INDECES_TRANSFORM + # the 1 degree tiles of the SRTM1 products, e.g. N44E010 covers 10E-11E + assert datasource.latlon_to_indeces(transform, 10.1, 44.9) == (10, 44) + assert datasource.latlon_to_indeces(transform, -73.99, 7.056) == (-74, 7) + assert datasource.latlon_to_indeces(transform, 15.931, -19.194) == (15, -20) + # a whole degree is a tile node, shared by the two tiles that meet there, + # and the half pixel makes it belong to the one that starts at the node + assert datasource.latlon_to_indeces(transform, 10.0, 44.0) == (10, 44) + + +def test_latlon_to_indeces_EDH_L2_CHUNK_INDECES_TRANSFORM() -> None: + transform = datasource.EDH_L2_CHUNK_INDECES_TRANSFORM + # the chunks of the Copernicus store are 1 by 0.5 degrees and 192_96 is + # the Rome region of the integration tests + assert datasource.latlon_to_indeces(transform, 12.4, 41.9) == (192, 96) + assert datasource.latlon_to_indeces(transform, 12.4, 41.4) == (192, 97) + # the chunks at the north west and the south east corner of the store + assert datasource.latlon_to_indeces(transform, -180.0, 90.0) == (0, 0) + assert datasource.latlon_to_indeces(transform, 179.9, -89.9) == (359, 359) + # the chunk boundaries are half a pixel outside the whole degrees, so a + # bound on a whole degree falls in the chunk that starts there + assert datasource.latlon_to_indeces(transform, 13.0, 41.5) == (193, 97) + + +def test_latlon_to_indeces_EDH_L1_CHUNK_INDECES_TRANSFORM() -> None: + transform = datasource.EDH_L1_CHUNK_INDECES_TRANSFORM + # the chunks of the Copernicus store are 2 by 2 degrees at 3 arc seconds, + # 96_24 is the Rome region of the integration tests + assert datasource.latlon_to_indeces(transform, 12.4, 41.9) == (96, 24) + assert datasource.latlon_to_indeces(transform, 15.0, 39.9) == (97, 25) + # the chunks at the north west and the south east corner of the store + assert datasource.latlon_to_indeces(transform, -180.0, 90.0) == (0, 0) + assert datasource.latlon_to_indeces(transform, 179.9, -89.9) == (179, 89) + # an even degree is a chunk boundary, half a pixel west of it, so a bound + # that lands on one, like 14.0, reaches the chunk that starts there + assert datasource.latlon_to_indeces(transform, 14.0, 41.9) == (97, 24) + + +def test_dted_l2_tiles() -> None: + assert list(datasource.dted_l2_tiles(10.1, 44.9, 10.1, 44.9)) == [ + ((10, 44), "N44E010.tif") + ] # NOTE this also tests int (not float) input - assert list(datasource.srtm1_tiles_names(10, 44, 11, 45)) == ["N44E010.tif"] + assert list(datasource.dted_l2_tiles(10, 44, 11, 45)) == [((10, 44), "N44E010.tif")] -def test_mapzen_tiles_names() -> None: - assert list(datasource.mapzen_tiles_names(10.1, 44.9, 10.1, 44.9)) == [ - "N44/N44E010.tif" +def test_mapzen_tiles() -> None: + # MAPZEN is the DTED L2 lattice with a subfolder in the tile name + spec = datasource.MAPZEN_SPEC + tiles = spec["cached_tiles"] + kwargs = spec["cached_tiles_kwargs"] + + assert list(tiles(10.1, 44.9, 10.1, 44.9, **kwargs)) == [ + ((10, 44), "N44/N44E010.tif") ] # NOTE this also tests int (not float) input - assert list(datasource.mapzen_tiles_names(10, 44, 11, 45)) == ["N44/N44E010.tif"] + assert list(tiles(10, 44, 11, 45, **kwargs)) == [((10, 44), "N44/N44E010.tif")] -def test_srtm3_tiles_names() -> None: - assert next(datasource.srtm3_tiles_names(10.1, 44.9, 10.1, 44.9)).endswith( - "srtm_39_04.tif" +def test_cgiar_l1_tiles() -> None: + assert next(datasource.cgiar_l1_tiles(10.1, 44.9, 10.1, 44.9)) == ( + (39, 4), + "srtm_39_04.tif", ) - assert next(datasource.srtm3_tiles_names(25.50, 58.40, 27.67, 60.06)).endswith( - "srtm_42_01.tif" + assert next(datasource.cgiar_l1_tiles(25.50, 58.40, 27.67, 60.06)) == ( + (42, 1), + "srtm_42_01.tif", ) - assert len(list(datasource.srtm3_tiles_names(9.9, 39.1, 15.1, 45.1))) == 9 + assert len(list(datasource.cgiar_l1_tiles(9.9, 39.1, 15.1, 45.1))) == 9 + + +def test_srtm_ellip_tiles() -> None: + ds1 = [((10, 44), "North/North_30_60/N44E010_wgs84.tif")] + ds2 = [((-74, 7), "North/North_0_29/N07W074_wgs84.tif")] + ds3 = [((15, -20), "South/S20E015_wgs84.tif")] + assert list(datasource.srtm_ellip_tiles(10.1, 44.9, 10.1, 44.9)) == ds1 + assert list(datasource.srtm_ellip_tiles(-73.99, 7.056, -73.90, 7.660)) == ds2 + assert list(datasource.srtm_ellip_tiles(15.931, -19.194, 15.329, -19.961)) == ds3 + # the tiles share their edge row and column, so a bound on a whole degree + # does not reach the tiles that start there + assert list(datasource.srtm_ellip_tiles(10.1, 44.1, 12.0, 46.0)) == [ + ((10, 44), "North/North_30_60/N44E010_wgs84.tif"), + ((10, 45), "North/North_30_60/N45E010_wgs84.tif"), + ((11, 44), "North/North_30_60/N44E011_wgs84.tif"), + ((11, 45), "North/North_30_60/N45E011_wgs84.tif"), + ] -def test_srtm_ellip_tiles_names() -> None: - ds1 = ["North/North_30_60/N44E010_wgs84.tif"] - ds2 = ["North/North_0_29/N07W074_wgs84.tif"] - ds3 = ["South/S20E015_wgs84.tif"] - assert list(datasource.srtm_ellip_tiles_names(10.1, 44.9, 10.1, 44.9)) == ds1 - assert list(datasource.srtm_ellip_tiles_names(-73.99, 7.056, -73.90, 7.660)) == ds2 - assert ( - list(datasource.srtm_ellip_tiles_names(15.931, -19.194, 15.329, -19.961)) == ds3 +def test_tile_source() -> None: + # a product that serves plain tiles declares only its datasource URL + spec = datasource.SRTM1_GEOID_SPEC + kwargs = dict(spec["prepare_tile_kwargs"]) + datasource_url = kwargs.pop("datasource_url") + assert kwargs == {} + url, spooled, member = datasource.tile_source( + datasource_url, "N41E012.tif", **kwargs ) + assert url == f"{datasource_url}/N41E012.tif" + assert spooled == "N41E012.tif" + assert member is None - -def test_tile_source() -> None: + # MAPZEN serves the DTED tiles gunzipped, so the spool name drops the .gz + spec = datasource.MAPZEN_SPEC + kwargs = dict(spec["prepare_tile_kwargs"]) + datasource_url = kwargs.pop("datasource_url") + assert kwargs == {"tile_ext": ".hgt", "compressed_ext": ".hgt.gz"} url, spooled, member = datasource.tile_source( - datasource.MAPZEN_SPEC, "N41/N41E012.tif" + datasource_url, "N41/N41E012.tif", **kwargs ) - assert url.endswith("/skadi/N41/N41E012.hgt.gz") + assert url == f"{datasource_url}/N41/N41E012.hgt.gz" assert spooled == "N41/N41E012.hgt" assert member is None + # the SRTM3 tiles are served inside a .zip, that the spec has to declare or + # seed asks for a plain .tif that the provider does not have + spec = datasource.SRTM3_SPEC + kwargs = dict(spec["prepare_tile_kwargs"]) + datasource_url = kwargs.pop("datasource_url") + assert kwargs == {"compressed_ext": ".zip"} url, spooled, member = datasource.tile_source( - datasource.SRTM3_SPEC, "srtm_39_04.tif" + datasource_url, "srtm_39_04.tif", **kwargs ) - assert url.endswith("/srtm_39_04.zip") + assert url == f"{datasource_url}/srtm_39_04.zip" assert spooled == "srtm_39_04.tif" assert member == "srtm_39_04.tif" + # the SRTM1 ellipsoidal tiles keep the name and the subfolder of the cache + spec = datasource.SRTM1_ELLIP_SPEC + kwargs = dict(spec["prepare_tile_kwargs"]) + datasource_url = kwargs.pop("datasource_url") url, spooled, member = datasource.tile_source( - datasource.SRTM1_ELLIP_SPEC, "North/North_30_60/N44E010_wgs84.tif" + datasource_url, "North/North_30_60/N44E010_wgs84.tif" ) - assert url.endswith("/North/North_30_60/N44E010_wgs84.tif") + assert url == f"{datasource_url}/North/North_30_60/N44E010_wgs84.tif" assert spooled == "North/North_30_60/N44E010_wgs84.tif" assert member is None def test_ensure_tiles(mocker: MockerFixture, tmp_path: Path) -> None: + spec = datasource.SRTM1_GEOID_SPEC mock_fetch = mocker.patch("elevation.datasource.fetch_tile") - mock_write = mocker.patch("elevation.datasource.write_cache_tile") + mock_write = mocker.patch( + "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + ) - datasource.ensure_tiles(tmp_path, datasource.SRTM1_GEOID_SPEC, ["N41E012.tif"]) + datasource.ensure_tiles( + tmp_path, + [((12, 41), "N41E012.tif")], + prepare_tile=spec["prepare_tile"], + **spec["prepare_tile_kwargs"], + ) mock_fetch.assert_called_once_with( - f"{datasource.SRTM1_GEOID_SPEC['datasource_url']}/N41E012.tif", + f"{spec['prepare_tile_kwargs']['datasource_url']}/N41E012.tif", tmp_path / "spool" / "N41E012.tif", member=None, ) mock_write.assert_called_once_with( - tmp_path / "spool" / "N41E012.tif", tmp_path / "cache" / "N41E012.tif" + [str(tmp_path / "spool" / "N41E012.tif")], + tmp_path / "spool" / "ready" / "N41E012.tif", + gdal_options=datasource.TILE_GDAL_OPTIONS, ) + # the tile reaches the cache only once it has been written in the spool + assert (tmp_path / "cache" / "N41E012.tif").read_bytes() == b"tile" def test_ensure_tiles_skips_cached(mocker: MockerFixture, tmp_path: Path) -> None: + spec = datasource.SRTM1_GEOID_SPEC cached = tmp_path / "cache" / "N41E012.tif" cached.parent.mkdir(parents=True) cached.write_bytes(b"cached") mock_fetch = mocker.patch("elevation.datasource.fetch_tile") mock_write = mocker.patch("elevation.datasource.write_cache_tile") - datasource.ensure_tiles(tmp_path, datasource.SRTM1_GEOID_SPEC, ["N41E012.tif"]) + datasource.ensure_tiles( + tmp_path, + [((12, 41), "N41E012.tif")], + prepare_tile=spec["prepare_tile"], + **spec["prepare_tile_kwargs"], + ) mock_fetch.assert_not_called() mock_write.assert_not_called() + assert cached.read_bytes() == b"cached" + + +def test_ensure_tiles_remote(mocker: MockerFixture, tmp_path: Path) -> None: + spec = datasource.GLO_90_SPEC + mock_fetch = mocker.patch("elevation.datasource.fetch_tile") + mock_write = mocker.patch( + "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + ) + # the chunk the Rome bounds of the integration tests fall in + tiles = list( + spec["cached_tiles"](12.4, 41.8, 12.4, 41.8, **spec["cached_tiles_kwargs"]) + ) + assert tiles == [((96, 24), "24/96.tif")] + + datasource.ensure_tiles( + tmp_path, + tiles, + prepare_tile=spec["prepare_tile"], + gdal_options=spec["tile_gdal_options"], + **spec["prepare_tile_kwargs"], + ) + + # a store is read in place: no download, one window per chunk, and the + # connection string keeps the CRS in the store, so no ``:/dsm`` suffix + mock_fetch.assert_not_called() + mock_write.assert_called_once_with( + [ + "-srcwin", + "230400", + "57600", + "2400", + "2400", + 'ZARR:"/vsicurl/https://data.earthdatahub.destine.eu/copernicus-dem/GLO-90-v1.zarr":/dsm', + ], + tmp_path / "spool" / "ready" / "24/96.tif", + gdal_options=datasource.FLOAT_TILE_GDAL_OPTIONS, + ) + assert (tmp_path / "cache" / "24/96.tif").read_bytes() == b"tile" def test_fetch_tile(tmp_path: Path) -> None: @@ -156,19 +301,15 @@ def test_fetch_tile_zip(tmp_path: Path) -> None: def test_write_cache_tile_command(tmp_path: Path, mocker: MockerFixture) -> None: check_call = mocker.patch("subprocess.check_call") destination = tmp_path / "cache" / "destination.tif" + gdal_source = ["-srcwin", "0", "0", "1", "1", str(REFERENCE)] - cmd = datasource.write_cache_tile(REFERENCE, destination, srcwin=(0, 0, 1, 1)) + cmd = datasource.write_cache_tile(gdal_source, destination) assert cmd == [ "gdal_translate", "-q", *datasource.TILE_GDAL_OPTIONS.split(), - "-srcwin", - "0", - "0", - "1", - "1", - str(REFERENCE), + *gdal_source, str(destination), ] check_call.assert_called_once_with(cmd) @@ -179,7 +320,7 @@ def test_write_cache_tile_command(tmp_path: Path, mocker: MockerFixture) -> None def test_write_cache_tile(tmp_path: Path) -> None: destination = tmp_path / "cache" / "destination.tif" - datasource.write_cache_tile(REFERENCE, destination) + datasource.write_cache_tile([str(REFERENCE)], destination) source = gdalinfo_json(REFERENCE) tile = gdalinfo_json(destination) @@ -248,9 +389,12 @@ def test_do_clip_gdal_options(mocker: MockerFixture, tmp_path: Path) -> None: def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" bounds = (13.1, 43.1, 13.9, 43.9) + spec = datasource.SRTM1_GEOID_SPEC mock_check_call = mocker.patch("subprocess.check_call") mock_fetch = mocker.patch("elevation.datasource.fetch_tile") - mock_write = mocker.patch("elevation.datasource.write_cache_tile") + mock_write = mocker.patch( + "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + ) datasource_root = datasource.seed( cache_dir=root, product="SRTM1_GEOID", bounds=bounds @@ -258,13 +402,14 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: assert datasource_root == root / "SRTM1_GEOID" mock_fetch.assert_called_once_with( - f"{datasource.SRTM1_GEOID_SPEC['datasource_url']}/N43E013.tif", + f"{spec['prepare_tile_kwargs']['datasource_url']}/N43E013.tif", datasource_root / "spool" / "N43E013.tif", member=None, ) mock_write.assert_called_once_with( - datasource_root / "spool" / "N43E013.tif", - datasource_root / "cache" / "N43E013.tif", + [str(datasource_root / "spool" / "N43E013.tif")], + datasource_root / "spool" / "ready" / "N43E013.tif", + gdal_options=datasource.TILE_GDAL_OPTIONS, ) assert mock_check_call.call_args[0][0][0] == "gdalbuildvrt" @@ -275,6 +420,45 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: datasource.seed(cache_dir=root) +def test_seed_remote(mocker: MockerFixture, tmp_path: Path) -> None: + root = tmp_path / "root" + mock_check_call = mocker.patch("subprocess.check_call") + mock_fetch = mocker.patch("elevation.datasource.fetch_tile") + mock_write = mocker.patch( + "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + ) + + datasource_root = datasource.seed( + cache_dir=root, + product="GLO-30", + bounds=(12.4, 41.8, 12.4 + 100 / 3600, 41.8 + 100 / 3600), + ) + + assert datasource_root == root / "GLO-30" + # the store is read in place, one window per chunk of the 1 by 0.5 degrees + # grid, and the tile name mirrors the chunk layout + mock_fetch.assert_not_called() + mock_write.assert_called_once_with( + [ + "-srcwin", + "691200", + "172800", + "3600", + "1800", + 'ZARR:"/vsicurl/https://data.earthdatahub.destine.eu/copernicus-dem/GLO-30-v1.zarr":/dsm', + ], + datasource_root / "spool" / "ready" / "96/192.tif", + gdal_options=datasource.FLOAT_TILE_GDAL_OPTIONS, + ) + assert (datasource_root / "cache" / "96/192.tif").read_bytes() == b"tile" + assert mock_check_call.call_args[0][0][0] == "gdalbuildvrt" + + with pytest.raises(RuntimeError): + datasource.seed( + cache_dir=root, product="GLO-30", bounds=(0.0, -100.0, 100.0, 0.0) + ) + + def test_build_bounds() -> None: raw_bounds = (13.1, 43.1, 13.9, 43.9) assert datasource.build_bounds(raw_bounds, margin="0") == raw_bounds @@ -298,7 +482,7 @@ def test_clip(mocker: MockerFixture, tmp_path: Path) -> None: bounds = (13.1, 43.1, 14.9, 44.9) mock_check_call = mocker.patch("subprocess.check_call") mocker.patch("elevation.datasource.fetch_tile") - mocker.patch("elevation.datasource.write_cache_tile") + mocker.patch("elevation.datasource.write_cache_tile", side_effect=write_ready_tile) datasource.clip(cache_dir=root, bounds=bounds, output="out.tif") @@ -372,9 +556,9 @@ def test_info(tmp_path: Path) -> None: def test_dataset() -> None: assert "id: SRTM3\n" in elevation.dataset("SRTM3") + assert "id: GLO-30\n" in elevation.dataset("GLO-30") text = elevation.dataset() assert text.count("id: ") == len(elevation.PRODUCTS) - assert "GLO-30" not in text assert text.endswith("\n") assert text.count("\n---\n") == len(elevation.PRODUCTS) - 1 assert "\n\n---\nid: SRTM1_GEOID\n" in text diff --git a/tests/test_40_main.py b/tests/test_40_main.py index 64b545d..d248050 100644 --- a/tests/test_40_main.py +++ b/tests/test_40_main.py @@ -33,7 +33,6 @@ def test_eio_dataset() -> None: assert not result.exception for dataset in elevation.PRODUCTS: assert f"id: {dataset}\n" in result.output - assert "GLO-30" not in result.output def test_eio_dataset_one() -> None: