diff --git a/pixi.lock b/pixi.lock index d6fc8c7..c8b5156 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1,8 +1,20 @@ version: 7 platforms: - name: linux-64 + virtual-packages: + - __unix=0=0 + - __linux=4.18 + - __glibc=2.28 + - __archspec=0=x86_64 - name: osx-arm64 + virtual-packages: + - __unix=0=0 + - __osx=13.0 + - __archspec=0=m1 - name: win-64 + virtual-packages: + - __win=10.0 + - __archspec=0=x86_64 environments: default: channels: @@ -17,7 +29,7 @@ environments: - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-report-plugins-1.3.0-pyhd4c3c12_0.conda - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-scheduler-plugins-2.0.2-pyhd4c3c12_0.conda - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-storage-plugins-4.4.1-pyh84498cf_0.conda - - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.19.0-pyhdfd78af_0.conda + - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.27.0-pyhdfd78af_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-atk-2.38.0-h0630a04_3.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/at-spi2-core-2.40.3-h0630a04_0.tar.bz2 @@ -176,7 +188,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/amply-0.1.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-types-0.7.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/archspec-0.2.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argparse-dataclass-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.1-pyhd8ed1ab_0.conda @@ -294,12 +305,11 @@ environments: - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-report-plugins-1.3.0-pyhd4c3c12_0.conda - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-scheduler-plugins-2.0.2-pyhd4c3c12_0.conda - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-storage-plugins-4.4.1-pyh84498cf_0.conda - - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.19.0-pyhdfd78af_0.conda + - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.27.0-pyhdfd78af_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-49.0-unix_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/amply-0.1.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-types-0.7.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/archspec-0.2.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argparse-dataclass-2.0.0-pyhd8ed1ab_1.conda @@ -533,11 +543,10 @@ environments: - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-report-plugins-1.3.0-pyhd4c3c12_0.conda - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-scheduler-plugins-2.0.2-pyhd4c3c12_0.conda - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-interface-storage-plugins-4.4.1-pyh84498cf_0.conda - - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.19.0-pyhdfd78af_0.conda + - conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.27.0-pyhdfd78af_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/amply-0.1.6-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-types-0.7.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/archspec-0.2.5-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argparse-dataclass-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/asttokens-3.0.1-pyhd8ed1ab_0.conda @@ -1659,15 +1668,14 @@ packages: license_family: MIT size: 22783 timestamp: 1773699846635 -- conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.19.0-pyhdfd78af_0.conda - sha256: 043b09633a430437d10d32949eebb104c12fc28ec6057de4253b5f45736c9a84 - md5: ff434de1e63f54864df031454a35b1ae +- conda: https://conda.anaconda.org/bioconda/noarch/snakemake-minimal-9.27.0-pyhdfd78af_0.conda + sha256: 2c80dd4dce15bf6eb1806ccab28f12a47d81e623d50e8543f58f6eac9090498b + md5: ef7be2b45f827b930836cd91db64e769 depends: - - appdirs - conda-inject >=1.3.1,<2.0 - configargparse - connection_pool >=0.0.3 - - docutils + - docutils >=0.20,<0.23 - dpath >=2.1.6,<3.0.0 - gitpython - humanfriendly @@ -1675,11 +1683,13 @@ packages: - jinja2 >=3.0,<4.0 - jsonschema - nbformat - - packaging >=24.0 + - packaging >=24.0,<26 + - platformdirs - psutil - pulp >=2.3.1,<3.4 - - python >=3.11,<3.14 + - python >=3.11 - pyyaml + - referencing - requests >=2.8.1,<3.0 - smart_open >=4.0,<8.0 - snakemake-interface-common >=1.20.1,<2.0 @@ -1687,7 +1697,7 @@ packages: - snakemake-interface-logger-plugins >=1.1.0,<3.0.0 - snakemake-interface-report-plugins >=1.2.0,<2.0.0 - snakemake-interface-scheduler-plugins >=2.0.0,<3.0.0 - - snakemake-interface-storage-plugins >=4.3.2,<5.0 + - snakemake-interface-storage-plugins >=4.4.1,<5.0 - sqlmodel >=0.0.37,<0.0.38 - tabulate - tenacity >=9.1.4,<10.0 @@ -1696,8 +1706,8 @@ packages: - yte >=1.5.5,<2.0 license: MIT license_family: MIT - size: 883944 - timestamp: 1774700572519 + size: 891186 + timestamp: 1789337861924 - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda build_number: 20 sha256: 1dd3fffd892081df9726d7eb7e0dea6198962ba775bd88842135a4ddb4deb3c9 @@ -6343,15 +6353,6 @@ packages: run_exports: {} size: 18074 timestamp: 1733247158254 -- conda: https://conda.anaconda.org/conda-forge/noarch/appdirs-1.4.4-pyhd8ed1ab_1.conda - sha256: 5b9ef6d338525b332e17c3ed089ca2f53a5d74b7a7b432747d29c6466e39346d - md5: f4e90937bbfc3a4a92539545a37bb448 - depends: - - python >=3.9 - license: MIT - license_family: MIT - size: 14835 - timestamp: 1733754069532 - conda: https://conda.anaconda.org/conda-forge/noarch/appnope-0.1.4-pyhd8ed1ab_1.conda sha256: 8f032b140ea4159806e4969a68b4a3c0a7cab1ad936eb958a2b5ffe5335e19bf md5: 54898d0f524c9dee622d44bbb081a8ab diff --git a/tests/conftest.py b/tests/conftest.py index 70c3365..060ce52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ """Shared test fixtures.""" +import os import shutil import zipfile from pathlib import Path @@ -10,15 +11,34 @@ TEST_FILES = "https://zenodo.org/records/19401947/files/test_suite.zip?download=1" -@pytest.fixture(scope="module") -def module_path(): - """Parent directory of the project.""" - # If your module needs files in resources/user/, place automated downloads here. - return Path(__file__).parent.parent +def dir_to_dir_copy(source_dir: Path, target_dir: Path) -> None: + """Copy all files from one directory into another.""" + target_dir.mkdir(parents=True, exist_ok=True) + for file in source_dir.iterdir(): + if file.is_file(): + shutil.copy(file, target_dir / file.name) + + +@pytest.fixture(scope="session") +def cds_api_file() -> Path: + """Ensure the CDS API file is present in the system.""" + cds_file = Path.home().joinpath(".cdsapirc") + + if not cds_file.exists(): + cdsapi_key = os.getenv("CDSAPI_KEY") + + if cdsapi_key: + cds_file.write_text( + f"url: https://cds.climate.copernicus.eu/api\nkey: {cdsapi_key}\n" + ) + else: + raise RuntimeError("Neither CDSAPI_KEY or the CDS API file is present.") + + return cds_file @pytest.fixture(scope="session") -def user_path() -> Path: +def user_path(cds_api_file) -> Path: """Download and unzip test files.""" user_dir = Path("resources/user/") # If test suite has been downloaded, assume everything is OK. @@ -31,3 +51,26 @@ def user_path() -> Path: with zipfile.ZipFile(test_zip, "r") as zfile: zfile.extractall(user_dir) return user_dir + + +@pytest.fixture(scope="module") +def module_path(user_path: Path): + """Parent directory of the project.""" + module_path = Path(__file__).parent.parent + + integration_dir = Path(module_path / "tests/integration") + + # clean files produced by the integration test + if integration_dir.exists(): + shutil.rmtree(integration_dir / "resources/", ignore_errors=True) + shutil.rmtree(integration_dir / "results/", ignore_errors=True) + + # copy user inputs needed by the module + dir_to_dir_copy(user_path / "MNE", integration_dir / "resources/inputs/MNE") + # copy downloads with unstable Github CI access + dir_to_dir_copy( + module_path / "tests/files/hydrobasins", + integration_dir / "resources/module/resources/automatic/hydrobasins", + ) + + return module_path diff --git a/tests/files/README.md b/tests/files/README.md new file mode 100644 index 0000000..db2ae64 --- /dev/null +++ b/tests/files/README.md @@ -0,0 +1,5 @@ +# Synthetic tests files + +Used during integration CI tests. + +- `hydrobasins`: synthetic alternative to the `.zip` files downloaded from the HydroBASINS website. diff --git a/tests/files/assorted_europe/powerplants.parquet b/tests/files/assorted_europe/powerplants.parquet deleted file mode 100644 index b6a8a13..0000000 Binary files a/tests/files/assorted_europe/powerplants.parquet and /dev/null differ diff --git a/tests/files/assorted_europe/shapes.parquet b/tests/files/assorted_europe/shapes.parquet deleted file mode 100644 index 46ad07c..0000000 Binary files a/tests/files/assorted_europe/shapes.parquet and /dev/null differ diff --git a/tests/files/hydrobasins/af.zip b/tests/files/hydrobasins/af.zip new file mode 100644 index 0000000..9148bb3 Binary files /dev/null and b/tests/files/hydrobasins/af.zip differ diff --git a/tests/files/hydrobasins/ar.zip b/tests/files/hydrobasins/ar.zip new file mode 100644 index 0000000..98d940d Binary files /dev/null and b/tests/files/hydrobasins/ar.zip differ diff --git a/tests/files/hydrobasins/as.zip b/tests/files/hydrobasins/as.zip new file mode 100644 index 0000000..03a6328 Binary files /dev/null and b/tests/files/hydrobasins/as.zip differ diff --git a/tests/files/hydrobasins/au.zip b/tests/files/hydrobasins/au.zip new file mode 100644 index 0000000..65b6f5a Binary files /dev/null and b/tests/files/hydrobasins/au.zip differ diff --git a/tests/files/hydrobasins/check_vs_synthetic.png b/tests/files/hydrobasins/check_vs_synthetic.png new file mode 100644 index 0000000..7af9cbc Binary files /dev/null and b/tests/files/hydrobasins/check_vs_synthetic.png differ diff --git a/tests/files/hydrobasins/eu.zip b/tests/files/hydrobasins/eu.zip new file mode 100644 index 0000000..d95a1ac Binary files /dev/null and b/tests/files/hydrobasins/eu.zip differ diff --git a/tests/files/hydrobasins/gr.zip b/tests/files/hydrobasins/gr.zip new file mode 100644 index 0000000..582a3ea Binary files /dev/null and b/tests/files/hydrobasins/gr.zip differ diff --git a/tests/files/hydrobasins/na.zip b/tests/files/hydrobasins/na.zip new file mode 100644 index 0000000..895e42c Binary files /dev/null and b/tests/files/hydrobasins/na.zip differ diff --git a/tests/files/hydrobasins/sa.zip b/tests/files/hydrobasins/sa.zip new file mode 100644 index 0000000..9536835 Binary files /dev/null and b/tests/files/hydrobasins/sa.zip differ diff --git a/tests/files/hydrobasins/si.zip b/tests/files/hydrobasins/si.zip new file mode 100644 index 0000000..d26eced Binary files /dev/null and b/tests/files/hydrobasins/si.zip differ diff --git a/tests/files/national_generation.parquet b/tests/files/national_generation.parquet deleted file mode 100644 index 9f4e104..0000000 Binary files a/tests/files/national_generation.parquet and /dev/null differ diff --git a/tests/integration_test.py b/tests/integration_test.py index 8c4080c..c56b086 100644 --- a/tests/integration_test.py +++ b/tests/integration_test.py @@ -4,8 +4,6 @@ Contents may be updated in future template updates. """ -import os -import shutil import subprocess import tomllib from pathlib import Path @@ -13,25 +11,6 @@ import pytest from clio_tools.data_module import ModuleInterface -CDSAPI_KEY = os.getenv("CDSAPI_KEY") -CDS_FILE = Path.home().joinpath(".cdsapirc") - - -@pytest.fixture(scope="module") -def integration_path(user_path: Path, module_path: Path): - """Ensures the minimal integration test is ready.""" - integration_dir = Path(module_path / "tests/integration") - if integration_dir.exists(): # clean everything - shutil.rmtree(integration_dir / "resources/", ignore_errors=True) - shutil.rmtree(integration_dir / "results/", ignore_errors=True) - user_integ_dir = integration_dir / "resources/inputs/" - files_to_copy = ["MNE/powerplants.parquet", "MNE/shapes.parquet"] - for file in files_to_copy: - destination_file = Path(user_integ_dir / file) - destination_file.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(user_path / file, destination_file) - return integration_dir - @pytest.fixture(scope="module") def pixi_platforms(module_path) -> list[str]: @@ -95,17 +74,11 @@ def test_snakemake_all_failure(module_path): assert "INVALID" in str(process.stderr) -@pytest.mark.skipif( - not (CDSAPI_KEY or CDS_FILE.exists()), - reason="Neither CDSAPI_KEY env var nor ~/.cdsapirc file available.", -) -def test_snakemake_integration_testing(integration_path): +def test_snakemake_integration_testing(module_path): """Run a light-weight test simulating someone using this module.""" - if CDSAPI_KEY and not CDS_FILE.exists(): - CDS_FILE.write_text( - f"url: https://cds.climate.copernicus.eu/api\nkey: {CDSAPI_KEY}\n" - ) - assert subprocess.run( - "snakemake --use-conda --cores 1", shell=True, check=True, cwd=integration_path + "snakemake --use-conda --cores 1", + shell=True, + check=True, + cwd=module_path / "tests/integration", ) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3211735..0acb1e8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -32,7 +32,7 @@ with open(workflow.source_path("internal/settings.yaml"), "r") as f: # Add all your includes here. include: "rules/_utils.smk" -include: "rules/automatic.smk" +include: "rules/downloads.smk" include: "rules/statistics.smk" include: "rules/basins.smk" include: "rules/powerplants.smk" diff --git a/workflow/rules/_utils.smk b/workflow/rules/_utils.smk index c3b42db..2d7fb27 100644 --- a/workflow/rules/_utils.smk +++ b/workflow/rules/_utils.smk @@ -1,5 +1,20 @@ """Utility functions for the module.""" +from snakemake.io import apply_wildcards + + +def prefer_existing(file_path): + """Prevents re-downloads even in forced cases. + + Useful for protecting invariable files. + """ + + def resolve_output(wildcards): + path = apply_wildcards(str(file_path), wildcards) + return before_update(path) if exists(path) else path + + return resolve_output + def validate_year_config(): start = config["years"]["start"] diff --git a/workflow/rules/basins.smk b/workflow/rules/basins.smk index 2c7040b..0453241 100644 --- a/workflow/rules/basins.smk +++ b/workflow/rules/basins.smk @@ -3,7 +3,7 @@ rule basins_extract_pfafstetter_level: input: - zip_file=rules.download_basin.output.path, + zip_file=prefer_existing(rules.download_basin.output.path), output: parquet_file=temp( "/automatic/hydrobasins/{continent}_{level}.parquet" diff --git a/workflow/rules/automatic.smk b/workflow/rules/downloads.smk similarity index 92% rename from workflow/rules/automatic.smk rename to workflow/rules/downloads.smk index 8f64c6a..0b72beb 100644 --- a/workflow/rules/automatic.smk +++ b/workflow/rules/downloads.smk @@ -14,7 +14,7 @@ rule download_eia: message: "Download the EIA International energy statistics in bulk." shell: - r'curl -fsSLo {output.zipfile:q} "{params.url}"' + 'curl -fsSLo {output.zipfile:q} "{params.url}" 2>{log:q}' rule download_basin: @@ -34,7 +34,7 @@ rule download_basin: message: "Downloading HydroBASINS file for '{wildcards.continent}'." shell: - r'curl -fsSLo {output.path:q} "{params.url}"' + 'curl -fsSLo {output.path:q} "{params.url}" 2>{log:q}' rule download_cutout: diff --git a/workflow/scripts/basins_extract_pfafstetter_level.py b/workflow/scripts/basins_extract_pfafstetter_level.py index eb4b678..23191ff 100644 --- a/workflow/scripts/basins_extract_pfafstetter_level.py +++ b/workflow/scripts/basins_extract_pfafstetter_level.py @@ -7,7 +7,6 @@ if TYPE_CHECKING: snakemake: Any -sys.stderr = open(snakemake.log[0], "w", buffering=1) SHP_ZIP_PATH = "hybas_{continent}_lev{level}_v1c.shp" @@ -36,6 +35,7 @@ def basins_extract_pfafstetter_level(zip_file, continent, level, parquet_file): if __name__ == "__main__": + sys.stderr = open(snakemake.log[0], "w", buffering=1) basins_extract_pfafstetter_level( zip_file=snakemake.input.zip_file, continent=snakemake.params.continent, diff --git a/workflow/scripts/powerplants_get_inflow_m3.py b/workflow/scripts/powerplants_get_inflow_m3.py index 61de7e7..a4ca3a4 100644 --- a/workflow/scripts/powerplants_get_inflow_m3.py +++ b/workflow/scripts/powerplants_get_inflow_m3.py @@ -10,7 +10,6 @@ if TYPE_CHECKING: snakemake: Any -sys.stderr = open(snakemake.log[0], "w", buffering=1) def powerplants_get_inflow_m3( @@ -66,6 +65,7 @@ def powerplants_get_inflow_m3( if __name__ == "__main__": + sys.stderr = open(snakemake.log[0], "w", buffering=1) powerplants_get_inflow_m3( shapes_file=snakemake.input.shapes, powerplants_file=snakemake.input.adjusted_powerplants, diff --git a/workflow/scripts/powerplants_get_inflow_mwh.py b/workflow/scripts/powerplants_get_inflow_mwh.py index c793f1e..3b7807f 100644 --- a/workflow/scripts/powerplants_get_inflow_mwh.py +++ b/workflow/scripts/powerplants_get_inflow_mwh.py @@ -12,7 +12,6 @@ if TYPE_CHECKING: snakemake: Any -sys.stderr = open(snakemake.log[0], "w", buffering=1) # --- @@ -162,6 +161,7 @@ def powerplants_get_inflow_mwh( if __name__ == "__main__": + sys.stderr = open(snakemake.log[0], "w", buffering=1) powerplants_get_inflow_mwh( inflow_m3_file=snakemake.input.inflow_m3, powerplants_file=snakemake.input.adjusted_powerplants,