diff --git a/elevation/__init__.py b/elevation/__init__.py index cb00df7..a7a30a7 100644 --- a/elevation/__init__.py +++ b/elevation/__init__.py @@ -38,7 +38,7 @@ resolve_cache_dir, seed, ) -from .util import TOOLS, selfcheck +from .spatial import TOOLS, selfcheck __all__ = [ "CACHE_DIR", diff --git a/elevation/datasource.py b/elevation/datasource.py index 6288d9b..51083a9 100644 --- a/elevation/datasource.py +++ b/elevation/datasource.py @@ -16,7 +16,6 @@ import math import os import shutil -import subprocess from collections.abc import Callable, Iterator, Sequence from importlib import resources from pathlib import Path @@ -24,7 +23,7 @@ import appdirs -from . import util +from . import spatial, util __all__ = [ "CACHE_DIR", @@ -48,9 +47,6 @@ DEFAULT_OUTPUT = "out.tif" DEFAULT_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9 -co PREDICTOR=2" CACHE_EXT = ".tif" -TILE_GDAL_OPTIONS = "-co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9" -INT_TILE_GDAL_OPTIONS = TILE_GDAL_OPTIONS + " -co PREDICTOR=2" -FLOAT_TILE_GDAL_OPTIONS = TILE_GDAL_OPTIONS + " -co PREDICTOR=3" MARGIN = "0" # NOTE: @@ -156,13 +152,13 @@ def prepare_tile_download_uncompress( tile_ext: str = ".tif", compressed_ext: str | None = None, **kwargs: Any, -) -> tuple[list[str], Path | None]: +) -> tuple[str, Path | None]: source, spool_name, member = tile_source( datasource_url, tile_name, tile_ext, compressed_ext ) spooled = spool / spool_name fetch_tile(source, spooled, member=member) - return [str(spooled)], spooled + return str(spooled), spooled def zarr_tiles( @@ -187,13 +183,12 @@ def prepare_tile_zarr( ilon: int, chunks: tuple[int, int], **kwargs: Any, -) -> tuple[list[str], Path | None]: +) -> tuple[str, Path | None]: srcwin = [ilon * chunks[0], ilat * chunks[1], chunks[0], chunks[1]] - gdal_source = [ - "-srcwin", - *map(str, srcwin), - f'ZARR:"/vsicurl/{datasource_url}":{variable_path}', - ] + gdal_source = ( + f"-srcwin {' '.join(map(str, srcwin))} " + + f'ZARR:"/vsicurl/{datasource_url}":{variable_path}' + ) return gdal_source, None @@ -206,7 +201,7 @@ class DatasourceSpec(TypedDict): cached_tiles_kwargs: NotRequired[dict[str, Any]] # prepare the tile for GDAL, downloading it or reading the window of the # chunked source, next to the spool file to remove once it is cached - prepare_tile: Callable[..., tuple[list[str], Path | None]] + prepare_tile: Callable[..., tuple[str, Path | None]] # keyword arguments for ``prepare_tile``: the datasource URL and the path # of the variable in the store, the source extension, the archive the # provider serves it in, the chunk size @@ -257,7 +252,7 @@ class DatasourceSpec(TypedDict): "variable_path": "/dsm", "chunks": (3600, 1800), }, - "tile_gdal_options": FLOAT_TILE_GDAL_OPTIONS, + "tile_gdal_options": spatial.FLOAT_TILE_GDAL_OPTIONS, "cached_tiles": zarr_tiles, "cached_tiles_kwargs": {"transform": EDH_L2_CHUNK_INDECES_TRANSFORM}, } @@ -269,7 +264,7 @@ class DatasourceSpec(TypedDict): "variable_path": "/dsm", "chunks": (2400, 2400), }, - "tile_gdal_options": FLOAT_TILE_GDAL_OPTIONS, + "tile_gdal_options": spatial.FLOAT_TILE_GDAL_OPTIONS, "cached_tiles": zarr_tiles, "cached_tiles_kwargs": {"transform": EDH_L1_CHUNK_INDECES_TRANSFORM}, } @@ -357,41 +352,11 @@ def fetch_tile(source: str, destination: Path, *, member: str | None = None) -> temporary.unlink(missing_ok=True) -def write_cache_tile( - gdal_source: Sequence[str], - destination: Path, - *, - gdal_options: str = TILE_GDAL_OPTIONS, -) -> list[str]: - """Write *source* to *destination* as the internal compressed GeoTIFF tile. - - The data, its dtype, its nodata value, its georeferencing and its metadata are - preserved unchanged, only compression is added. ``PREDICTOR=2`` in the default - options suits the integer products, pass ``PREDICTOR=3`` for float ones. - - :param source: Any GDAL readable raster, local or remote. - :param destination: Path of the cache GeoTIFF, parent folders are created. - :param srcwin: Window of *source* to write, e.g. a single ``Zarr`` chunk. - :param gdal_options: GDAL creation options of the cache tile. - :return: The command arguments. - """ - destination.parent.mkdir(parents=True, exist_ok=True) - cmd = [ - "gdal_translate", - "-q", - *gdal_options.split(), - *gdal_source, - str(destination), - ] - subprocess.check_call(cmd) - return cmd - - def ensure_tiles( root: Path, tiles: Sequence[Tile], - prepare_tile: Callable[..., tuple[list[str], Path | None]], - gdal_options: str = TILE_GDAL_OPTIONS, + prepare_tile: Callable[..., tuple[str, Path | None]], + gdal_options: str = spatial.INT_TILE_GDAL_OPTIONS, **kwargs: Any, ) -> None: """Fetch and cache *tiles*, skipping the tiles already in the cache. @@ -406,13 +371,13 @@ def ensure_tiles( continue # prepare the data if GDAL cannot download it / read it as it is - gdal_source, spooled = prepare_tile( + source, spooled = prepare_tile( tile_name=tile_name, spool=root / "spool", ilat=ilat, ilon=ilon, **kwargs ) # convert the data to the internal cache format ready = root / "spool/ready" / tile_name - write_cache_tile(gdal_source, ready, gdal_options=gdal_options) + spatial.call_gdal_translate(source, ready, options=gdal_options) if spooled is not None: spooled.unlink(missing_ok=True) @@ -423,17 +388,12 @@ def ensure_tiles( def build_vrt(root: Path, product: str) -> list[str]: """Build the ``.vrt`` mosaic over the non empty cache tiles.""" - tiles = sorted( - tile for tile in (root / "cache").rglob("*.tif") if tile.stat().st_size > 0 - ) - cmd = [ - "gdalbuildvrt", - "-q", - "-overwrite", - str(root / f"{product}.vrt"), - *map(str, tiles), - ] - subprocess.check_call(cmd) + tiles = [] + for tile in (root / "cache").rglob("*.tif"): + if tile.stat().st_size > 0: + tiles.append(str(tile)) + options = "-q -overwrite" + cmd = spatial.call_gdalbuildvrt(sorted(tiles), root / f"{product}.vrt", options) return cmd @@ -456,10 +416,10 @@ def do_clip( gdal_options: str = DEFAULT_GDAL_OPTIONS, ) -> list[str]: left, bottom, right, top = bounds - options = f"gdal_translate -q {gdal_options} -projwin {left} {top} {right} {bottom}" - cmd = [*options.split(), str(path / f"{product}.vrt"), str(output)] + options = f"-q {gdal_options} -projwin {left} {top} {right} {bottom}" + source = str(path / f"{product}.vrt") with util.lock_vrt(path, product): - subprocess.check_call(cmd) + cmd = spatial.call_gdal_translate(source, output, options=options) return cmd @@ -499,7 +459,7 @@ def seed( datasource_root, tiles, prepare_tile=prepare_tile, - gdal_options=spec.get("tile_gdal_options", TILE_GDAL_OPTIONS), + gdal_options=spec.get("tile_gdal_options", spatial.INT_TILE_GDAL_OPTIONS), **prepare_tile_kwargs, ) diff --git a/elevation/spatial.py b/elevation/spatial.py index 806f61f..d31c670 100644 --- a/elevation/spatial.py +++ b/elevation/spatial.py @@ -18,42 +18,119 @@ from pathlib import Path from typing import Any -CORNERS = ("upperLeft", "lowerLeft", "upperRight", "lowerRight") +DEFAULT_GDAL_OPTIONS = "-q" +TILE_GDAL_OPTIONS = ( + DEFAULT_GDAL_OPTIONS + " -co TILED=YES -co COMPRESS=DEFLATE -co ZLEVEL=9" +) +INT_TILE_GDAL_OPTIONS = TILE_GDAL_OPTIONS + " -co PREDICTOR=2" +FLOAT_TILE_GDAL_OPTIONS = TILE_GDAL_OPTIONS + " -co PREDICTOR=3" +TOOLS: list[tuple[str, str]] = [ + ("gdal_translate", "gdal_translate --version"), + ("gdalbuildvrt", "gdalbuildvrt --version"), + ("gdalinfo", "gdalinfo --version"), + ("ogrinfo", "ogrinfo --version"), +] -def gdal_report(cmd: list[str]) -> Any: - """Run the *cmd* GDAL command and return its JSON report, or ``None``.""" - try: - output = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) - return json.loads(output) - except (OSError, subprocess.CalledProcessError, json.JSONDecodeError): - return None +def selfcheck( + tools: dict[str, str] | list[tuple[str, str]] = TOOLS, + verbose: bool = False, +) -> str: + """Audit the system for issues. + + :param tools: Tools description, defaults to TOOLS. + :param verbose: Report each tool as it is tested. + """ + report = [] + issues = [] + for tool_name, check_cli in dict(tools).items(): + if verbose: + report.append(f"Checking {tool_name!r} ...") + try: + subprocess.check_output(check_cli, shell=True, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError: + issues.append(f"{tool_name!r} not found or not usable.") + report.append("\n".join(issues) if issues else "Your system is ready.") + result = "\n".join(report) + return result + +def gdal_json(cmd: str, destination: str) -> Any: + """Run the *cmd* GDAL command on *destination* and return its JSON report.""" + output = subprocess.check_output([*cmd.split(), destination]) + return json.loads(output) + + +def call_gdal_translate( + source: str, + destination: Path, + options: str = DEFAULT_GDAL_OPTIONS, +) -> list[str]: + """Write *source* to *destination* calling the gdal_translate binary. + + :param source: Any GDAL readable raster, local or remote, may include selection options. + :param destination: Path of the destination, parent folders are created. + :param options: GDAL creation options, the default is good for caching tiles. + """ + destination.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + "gdal_translate", + *options.split(), + *source.split(), + str(destination), + ] + subprocess.check_call(cmd) + return cmd -def raster_bounds(reference: str) -> tuple[float, float, float, float] | None: + +def call_gdalbuildvrt( + sources: list[str], + destination: Path, + options: str = DEFAULT_GDAL_OPTIONS, +) -> list[str]: + """Build the *destination* ``.vrt`` mosaic over *sources*. + + :param sources: Paths of the raster tiles to mosaic. + :param destination: Path of the destination, parent folders are created. + :param options: GDAL options placed before the destination. + """ + destination.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + "gdalbuildvrt", + *options.split(), + str(destination), + *sources, + ] + subprocess.check_call(cmd) + return cmd + + +def raster_bounds(reference: str) -> tuple[float, float, float, float]: """Return the bounds of the raster *reference*, ``None`` if it is not a raster.""" - report = gdal_report(["gdalinfo", "-json", "-nomd", "-norat", "-noct", reference]) + corners_keys = ("upperLeft", "lowerLeft", "upperRight", "lowerRight") + + report = gdal_json("gdalinfo -json -nomd -norat -noct", reference) if not isinstance(report, dict) or "cornerCoordinates" not in report: - return None + raise TypeError("'cornerCoordinates' not found") corners = report["cornerCoordinates"] # all four corners make the bounds of a rotated raster exact - xs = [corners[key][0] for key in CORNERS] - ys = [corners[key][1] for key in CORNERS] + xs = [corners[key][0] for key in corners_keys] + ys = [corners[key][1] for key in corners_keys] return min(xs), min(ys), max(xs), max(ys) -def vector_bounds(reference: str) -> tuple[float, float, float, float] | None: +def vector_bounds(reference: str) -> tuple[float, float, float, float]: """Return the bounds of the vector *reference*, ``None`` if it is not a vector.""" - report = gdal_report(["ogrinfo", "-json", "-al", "-so", reference]) + report = gdal_json("ogrinfo -json -al -so", reference) if not isinstance(report, dict) or not report.get("layers"): - return None + raise TypeError("'layers' not found") layer = report["layers"][0] fields = layer.get("geometryFields") or [{}] # GDAL >= 3.6 reports the extent of each geometry field as a list, the older # versions report a single extent of the layer as an object extent = fields[0].get("extent") or layer.get("extent") if extent is None: - return None + raise TypeError("'extent' not found") if isinstance(extent, dict): return extent["xmin"], extent["ymin"], extent["xmax"], extent["ymax"] left, bottom, right, top = extent @@ -72,7 +149,11 @@ def import_bounds(reference: str | Path) -> tuple[float, float, float, float]: """ # ASSUMPTION: the bounds are given in geodetic WGS84 crs reference = str(reference) - bounds = raster_bounds(reference) or vector_bounds(reference) - if bounds is None: - raise RuntimeError(f"Reference datasource could not be opened {reference!r}.") + try: + bounds = raster_bounds(reference) + except subprocess.CalledProcessError: + try: + bounds = vector_bounds(reference) + except subprocess.CalledProcessError: + raise RuntimeError(f"Reference datasource error {reference!r}") from None return bounds diff --git a/elevation/util.py b/elevation/util.py index ec0be0c..f0fa46d 100644 --- a/elevation/util.py +++ b/elevation/util.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import subprocess from collections.abc import Generator, Iterable from contextlib import contextmanager from pathlib import Path @@ -21,35 +20,6 @@ import fasteners FOLDER_LOCKFILE_NAME = ".folder_lock" -TOOLS: list[tuple[str, str]] = [ - ("gdal_translate", "gdal_translate --version"), - ("gdalbuildvrt", "gdalbuildvrt --version"), - ("gdalinfo", "gdalinfo --version"), - ("ogrinfo", "ogrinfo --version"), -] - - -def selfcheck( - tools: dict[str, str] | Iterable[tuple[str, str]] = TOOLS, - verbose: bool = False, -) -> str: - """Audit the system for issues. - - :param tools: Tools description, defaults to TOOLS. - :param verbose: Report each tool as it is tested. - """ - report = [] - issues = [] - for tool_name, check_cli in dict(tools).items(): - if verbose: - report.append(f"Checking {tool_name!r} ...") - try: - subprocess.check_output(check_cli, shell=True, stderr=subprocess.STDOUT) - except subprocess.CalledProcessError: - issues.append(f"{tool_name!r} not found or not usable.") - report.append("\n".join(issues) if issues else "Your system is ready.") - result = "\n".join(report) - return result @contextmanager diff --git a/tests/conftest.py b/tests/conftest.py index e504a82..4c44934 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,11 +9,9 @@ Makefile target does. """ -import json import os import re import shutil -import subprocess from collections.abc import Callable, Iterator from pathlib import Path from typing import Any @@ -22,6 +20,7 @@ import pytest import elevation +from elevation import spatial REFERENCE_DATA_DIR = Path(__file__).parent / "data" INTEGRATION_CACHE_DIR = Path(appdirs.user_cache_dir("elevation-integration", "bopen")) @@ -49,13 +48,6 @@ def pytest_report_header(config: pytest.Config) -> str | None: return None -def gdalinfo_json(path: Path) -> dict[str, Any]: - """Run ``gdalinfo`` with checksums and return the parsed JSON report.""" - cmd = ["gdalinfo", "-json", "-checksum", str(path)] - info: dict[str, Any] = json.loads(subprocess.check_output(cmd)) - return info - - def raster_fingerprint(info: dict[str, Any]) -> dict[str, Any]: """Reduce a ``gdalinfo`` report to the fields that identify the raster content. @@ -77,8 +69,8 @@ def raster_fingerprint(info: dict[str, Any]) -> dict[str, Any]: def assert_same_raster(produced: Path, reference: Path) -> None: - produced_info = gdalinfo_json(produced) - reference_info = gdalinfo_json(reference) + produced_info = spatial.gdal_json("gdalinfo -json -checksum", str(produced)) + reference_info = spatial.gdal_json("gdalinfo -json -checksum", str(reference)) message = f"{produced} differs from the reference data {reference}" fingerprint = raster_fingerprint(produced_info) assert fingerprint == raster_fingerprint(reference_info), message diff --git a/tests/test_10_spatial.py b/tests/test_10_spatial.py index f530951..2ccf377 100644 --- a/tests/test_10_spatial.py +++ b/tests/test_10_spatial.py @@ -2,9 +2,11 @@ # Copyright (c) 2016-2026 B-Open Solutions srl - https://bopen.eu # +import shutil from pathlib import Path import pytest +from pytest_mock import MockerFixture from elevation import spatial @@ -13,6 +15,11 @@ VECTOR = DATA_DIR / "reference.geojson" BOUNDS = (10.0, 40.0, 11.0, 41.0) +requires_gdal = pytest.mark.skipif( + shutil.which("gdal_translate") is None, + reason="the GDAL command line tools are not installed", +) + def test_import_bounds_raster() -> None: assert spatial.import_bounds(RASTER) == BOUNDS @@ -23,5 +30,73 @@ def test_import_bounds_vector() -> None: def test_import_bounds_invalid() -> None: - with pytest.raises(RuntimeError, match="could not be opened"): + with pytest.raises(RuntimeError): spatial.import_bounds(RASTER.with_suffix(".bad")) + + +def test_selfcheck() -> None: + assert "NAME" not in spatial.selfcheck({"NAME": "true"}) + assert "NAME" in spatial.selfcheck({"NAME": "false"}) + + +def test_selfcheck_verbose() -> None: + assert spatial.selfcheck({"NAME": "true"}, verbose=True) == ( + "Checking 'NAME' ...\nYour system is ready." + ) + assert spatial.selfcheck({"NAME": "false"}, verbose=True) == ( + "Checking 'NAME' ...\n'NAME' not found or not usable." + ) + + +def test_call_gdal_translate_command(tmp_path: Path, mocker: MockerFixture) -> None: + check_call = mocker.patch("subprocess.check_call") + destination = tmp_path / "cache" / "destination.tif" + source = f"-srcwin 0 0 1 1 {RASTER}" + + cmd = spatial.call_gdal_translate(source, destination) + + assert cmd == [ + "gdal_translate", + *spatial.DEFAULT_GDAL_OPTIONS.split(), + *source.split(), + str(destination), + ] + check_call.assert_called_once_with(cmd) + assert destination.parent.is_dir() + + +def test_call_gdalbuildvrt_command(tmp_path: Path, mocker: MockerFixture) -> None: + check_call = mocker.patch("subprocess.check_call") + destination = tmp_path / "cache" / "destination.vrt" + sources = [ + str(tmp_path / "cache" / "N41E012.tif"), + str(tmp_path / "cache" / "N42E012.tif"), + ] + + cmd = spatial.call_gdalbuildvrt(sources, destination) + + assert cmd == [ + "gdalbuildvrt", + *spatial.DEFAULT_GDAL_OPTIONS.split(), + str(destination), + *sources, + ] + check_call.assert_called_once_with(cmd) + assert destination.parent.is_dir() + + +@requires_gdal +def test_call_gdal_translate(tmp_path: Path) -> None: + destination = tmp_path / "cache" / "destination.tif" + + spatial.call_gdal_translate(str(RASTER), destination) + + source = spatial.gdal_json("gdalinfo -json -checksum", str(RASTER)) + tile = spatial.gdal_json("gdalinfo -json -checksum", str(destination)) + assert tile["size"] == source["size"] + assert tile["geoTransform"] == pytest.approx(source["geoTransform"]) + assert tile["coordinateSystem"] == source["coordinateSystem"] + tile_band, source_band = tile["bands"][0], source["bands"][0] + assert tile_band["type"] == source_band["type"] + assert tile_band.get("noDataValue") == source_band.get("noDataValue") + assert tile_band["checksum"] == source_band["checksum"] diff --git a/tests/test_10_util.py b/tests/test_10_util.py index 3c846e9..6213fff 100644 --- a/tests/test_10_util.py +++ b/tests/test_10_util.py @@ -7,20 +7,6 @@ from elevation import util -def test_selfcheck() -> None: - assert "NAME" not in util.selfcheck({"NAME": "true"}) - assert "NAME" in util.selfcheck({"NAME": "false"}) - - -def test_selfcheck_verbose() -> None: - assert util.selfcheck({"NAME": "true"}, verbose=True) == ( - "Checking 'NAME' ...\nYour system is ready." - ) - assert util.selfcheck({"NAME": "false"}, verbose=True) == ( - "Checking 'NAME' ...\n'NAME' not found or not usable." - ) - - def test_lock_tiles(tmp_path: Path) -> None: root = tmp_path / "root" with util.lock_tiles(root, ["a.tiff"]): diff --git a/tests/test_20_datasource.py b/tests/test_20_datasource.py index 371d04c..80686d1 100644 --- a/tests/test_20_datasource.py +++ b/tests/test_20_datasource.py @@ -2,9 +2,6 @@ # Copyright (c) 2016-2026 B-Open Solutions srl - https://bopen.eu # -import json -import shutil -import subprocess from pathlib import Path from typing import Any @@ -12,23 +9,11 @@ from pytest_mock import MockerFixture import elevation -from elevation import datasource +from elevation import datasource, spatial DATA_DIR = Path(__file__).parent / "data" REFERENCE = DATA_DIR / "reference.tif" -requires_gdal = pytest.mark.skipif( - shutil.which("gdal_translate") is None, - reason="the GDAL command line tools are not installed", -) - - -def gdalinfo_json(path: Path) -> dict[str, Any]: - """Return the ``gdalinfo`` report of *path*, with the band checksums.""" - report = subprocess.check_output(["gdalinfo", "-json", "-checksum", str(path)]) - info: dict[str, Any] = json.loads(report) - return info - def write_ready_tile(source: str | Path, ready: Path, **kwargs: Any) -> None: """Stand in for the mocked cache write: leave a tile for the cache move.""" @@ -190,7 +175,7 @@ def test_ensure_tiles(mocker: MockerFixture, tmp_path: Path) -> None: spec = datasource.SRTM1_GEOID_SPEC mock_fetch = mocker.patch("elevation.datasource.fetch_tile") mock_write = mocker.patch( - "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + "elevation.spatial.call_gdal_translate", side_effect=write_ready_tile ) datasource.ensure_tiles( @@ -206,9 +191,9 @@ def test_ensure_tiles(mocker: MockerFixture, tmp_path: Path) -> None: member=None, ) mock_write.assert_called_once_with( - [str(tmp_path / "spool" / "N41E012.tif")], + str(tmp_path / "spool" / "N41E012.tif"), tmp_path / "spool" / "ready" / "N41E012.tif", - gdal_options=datasource.TILE_GDAL_OPTIONS, + options=spatial.INT_TILE_GDAL_OPTIONS, ) # the tile reaches the cache only once it has been written in the spool assert (tmp_path / "cache" / "N41E012.tif").read_bytes() == b"tile" @@ -220,7 +205,7 @@ def test_ensure_tiles_skips_cached(mocker: MockerFixture, tmp_path: Path) -> Non cached.parent.mkdir(parents=True) cached.write_bytes(b"cached") mock_fetch = mocker.patch("elevation.datasource.fetch_tile") - mock_write = mocker.patch("elevation.datasource.write_cache_tile") + mock_write = mocker.patch("elevation.spatial.call_gdal_translate") datasource.ensure_tiles( tmp_path, @@ -238,7 +223,7 @@ def test_ensure_tiles_remote(mocker: MockerFixture, tmp_path: Path) -> None: spec = datasource.GLO_90_SPEC mock_fetch = mocker.patch("elevation.datasource.fetch_tile") mock_write = mocker.patch( - "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + "elevation.spatial.call_gdal_translate", side_effect=write_ready_tile ) # the chunk the Rome bounds of the integration tests fall in tiles = list( @@ -258,16 +243,11 @@ def test_ensure_tiles_remote(mocker: MockerFixture, tmp_path: Path) -> None: # connection string keeps the CRS in the store, so no ``:/dsm`` suffix mock_fetch.assert_not_called() mock_write.assert_called_once_with( - [ - "-srcwin", - "230400", - "57600", - "2400", - "2400", - 'ZARR:"/vsicurl/https://data.earthdatahub.destine.eu/copernicus-dem/GLO-90-v1.zarr":/dsm', - ], + "-srcwin 230400 57600 2400 2400 " + 'ZARR:"/vsicurl/https://data.earthdatahub.destine.eu' + '/copernicus-dem/GLO-90-v1.zarr":/dsm', tmp_path / "spool" / "ready" / "24/96.tif", - gdal_options=datasource.FLOAT_TILE_GDAL_OPTIONS, + options=spatial.FLOAT_TILE_GDAL_OPTIONS, ) assert (tmp_path / "cache" / "24/96.tif").read_bytes() == b"tile" @@ -298,42 +278,6 @@ def test_fetch_tile_zip(tmp_path: Path) -> None: assert destination.read_bytes() == REFERENCE.read_bytes() -def test_write_cache_tile_command(tmp_path: Path, mocker: MockerFixture) -> None: - check_call = mocker.patch("subprocess.check_call") - destination = tmp_path / "cache" / "destination.tif" - gdal_source = ["-srcwin", "0", "0", "1", "1", str(REFERENCE)] - - cmd = datasource.write_cache_tile(gdal_source, destination) - - assert cmd == [ - "gdal_translate", - "-q", - *datasource.TILE_GDAL_OPTIONS.split(), - *gdal_source, - str(destination), - ] - check_call.assert_called_once_with(cmd) - assert destination.parent.is_dir() - - -@requires_gdal -def test_write_cache_tile(tmp_path: Path) -> None: - destination = tmp_path / "cache" / "destination.tif" - - datasource.write_cache_tile([str(REFERENCE)], destination) - - source = gdalinfo_json(REFERENCE) - tile = gdalinfo_json(destination) - assert tile["size"] == source["size"] - assert tile["geoTransform"] == pytest.approx(source["geoTransform"]) - assert tile["coordinateSystem"] == source["coordinateSystem"] - assert tile["metadata"]["IMAGE_STRUCTURE"]["COMPRESSION"] == "DEFLATE" - tile_band, source_band = tile["bands"][0], source["bands"][0] - assert tile_band["type"] == source_band["type"] - assert tile_band.get("noDataValue") == source_band.get("noDataValue") - assert tile_band["checksum"] == source_band["checksum"] - - def test_do_clip(mocker: MockerFixture, tmp_path: Path) -> None: bounds = (13.1, 43.1, 14.9, 44.9) mock_check_call = mocker.patch("subprocess.check_call") @@ -393,7 +337,7 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: mock_check_call = mocker.patch("subprocess.check_call") mock_fetch = mocker.patch("elevation.datasource.fetch_tile") mock_write = mocker.patch( - "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + "elevation.spatial.call_gdal_translate", side_effect=write_ready_tile ) datasource_root = datasource.seed( @@ -407,9 +351,9 @@ def test_seed(mocker: MockerFixture, tmp_path: Path) -> None: member=None, ) mock_write.assert_called_once_with( - [str(datasource_root / "spool" / "N43E013.tif")], + str(datasource_root / "spool" / "N43E013.tif"), datasource_root / "spool" / "ready" / "N43E013.tif", - gdal_options=datasource.TILE_GDAL_OPTIONS, + options=spatial.INT_TILE_GDAL_OPTIONS, ) assert mock_check_call.call_args[0][0][0] == "gdalbuildvrt" @@ -425,7 +369,7 @@ def test_seed_remote(mocker: MockerFixture, tmp_path: Path) -> None: mock_check_call = mocker.patch("subprocess.check_call") mock_fetch = mocker.patch("elevation.datasource.fetch_tile") mock_write = mocker.patch( - "elevation.datasource.write_cache_tile", side_effect=write_ready_tile + "elevation.spatial.call_gdal_translate", side_effect=write_ready_tile ) datasource_root = datasource.seed( @@ -439,16 +383,11 @@ def test_seed_remote(mocker: MockerFixture, tmp_path: Path) -> None: # grid, and the tile name mirrors the chunk layout mock_fetch.assert_not_called() mock_write.assert_called_once_with( - [ - "-srcwin", - "691200", - "172800", - "3600", - "1800", - 'ZARR:"/vsicurl/https://data.earthdatahub.destine.eu/copernicus-dem/GLO-30-v1.zarr":/dsm', - ], + "-srcwin 691200 172800 3600 1800 " + 'ZARR:"/vsicurl/https://data.earthdatahub.destine.eu' + '/copernicus-dem/GLO-30-v1.zarr":/dsm', datasource_root / "spool" / "ready" / "96/192.tif", - gdal_options=datasource.FLOAT_TILE_GDAL_OPTIONS, + options=spatial.FLOAT_TILE_GDAL_OPTIONS, ) assert (datasource_root / "cache" / "96/192.tif").read_bytes() == b"tile" assert mock_check_call.call_args[0][0][0] == "gdalbuildvrt" @@ -480,26 +419,23 @@ def test_build_bounds() -> None: def test_clip(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" bounds = (13.1, 43.1, 14.9, 44.9) - mock_check_call = mocker.patch("subprocess.check_call") + output = tmp_path / "out.tif" + mocker.patch("subprocess.check_call") mocker.patch("elevation.datasource.fetch_tile") - mocker.patch("elevation.datasource.write_cache_tile", side_effect=write_ready_tile) + mock_translate = mocker.patch( + "elevation.spatial.call_gdal_translate", side_effect=write_ready_tile + ) - datasource.clip(cache_dir=root, bounds=bounds, output="out.tif") + datasource.clip(cache_dir=root, bounds=bounds, output=output) datasource_root = root / "MAPZEN" - expected_cmd = [ - "gdal_translate", - "-q", - *datasource.DEFAULT_GDAL_OPTIONS.split(), - "-projwin", - "13.1", - "44.9", - "14.9", - "43.1", - str(datasource_root / "MAPZEN.vrt"), - str(Path("out.tif").resolve()), - ] - assert mock_check_call.call_args[0][0] == expected_cmd + # the clip is the last call, after the one call per cached tile + source, destination = mock_translate.call_args.args + assert source == str(datasource_root / "MAPZEN.vrt") + assert destination == output + assert mock_translate.call_args.kwargs["options"] == ( + f"-q {datasource.DEFAULT_GDAL_OPTIONS} -projwin 13.1 44.9 14.9 43.1" + ) def test_clean(tmp_path: Path) -> None: diff --git a/tests/test_40_main.py b/tests/test_40_main.py index 2833c6d..5b3b71e 100644 --- a/tests/test_40_main.py +++ b/tests/test_40_main.py @@ -99,7 +99,7 @@ def test_eio_seed(mocker: MockerFixture, tmp_path: Path) -> None: options = f"--cache_dir {root!s} seed --bounds 12.5 42 12.5 42" mock_check_call = mocker.patch("subprocess.check_call") mocker.patch("elevation.datasource.fetch_tile") - mocker.patch("elevation.datasource.write_cache_tile") + mocker.patch("elevation.spatial.call_gdal_translate") result = runner.invoke(__main__.app, options.split()) assert not result.exception assert mock_check_call.call_count == 1 @@ -112,10 +112,11 @@ def test_eio_clip(mocker: MockerFixture, tmp_path: Path) -> None: options = f"--cache_dir {root!s} clip --bounds 12.5 42 12.5 42" mock_check_call = mocker.patch("subprocess.check_call") mocker.patch("elevation.datasource.fetch_tile") - mocker.patch("elevation.datasource.write_cache_tile") + mock_translate = mocker.patch("elevation.spatial.call_gdal_translate") result = runner.invoke(__main__.app, options.split()) assert not result.exception - assert mock_check_call.call_count == 2 + assert mock_check_call.call_count == 1 + assert mock_translate.call_count == 1 mock_check_call = mocker.patch("subprocess.check_call") result = runner.invoke(__main__.app, ["clip"]) @@ -131,9 +132,9 @@ def test_eio_clip(mocker: MockerFixture, tmp_path: Path) -> None: def test_eio_clip_gdal_options(mocker: MockerFixture, tmp_path: Path) -> None: root = tmp_path / "root" runner = typer.testing.CliRunner() - mock_check_call = mocker.patch("subprocess.check_call") + mocker.patch("subprocess.check_call") mocker.patch("elevation.datasource.fetch_tile") - mocker.patch("elevation.datasource.write_cache_tile") + mock_translate = mocker.patch("elevation.spatial.call_gdal_translate") result = runner.invoke( __main__.app, @@ -152,7 +153,7 @@ def test_eio_clip_gdal_options(mocker: MockerFixture, tmp_path: Path) -> None: ) assert not result.exception - assert "COMPRESS=LZW" in mock_check_call.call_args[0][0] + assert "COMPRESS=LZW" in mock_translate.call_args.kwargs["options"] def test_eio_clean(tmp_path: Path) -> None: @@ -185,7 +186,7 @@ def test_eio(mocker: MockerFixture, tmp_path: Path) -> None: options = f"--cache_dir {root!s} seed --bounds 12.5 42 12.5 42" mock_check_call = mocker.patch("subprocess.check_call") mocker.patch("elevation.datasource.fetch_tile") - mocker.patch("elevation.datasource.write_cache_tile") + mocker.patch("elevation.spatial.call_gdal_translate") result = runner.invoke(__main__.app, options.split()) assert not result.exception assert mock_check_call.call_count == 1