diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml index 68a741680..92e23cfc1 100644 --- a/.github/workflows/io-test.yml +++ b/.github/workflows/io-test.yml @@ -42,10 +42,11 @@ jobs: # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git id: ephy_testing_data run: | + # if gin cannot be reached the key matches no cache and the restore-keys below restore the most recent one set -o pipefail if ! hash="$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" || [ -z "$hash" ]; then - echo "::error::could not resolve ephy_testing_data HEAD on gin; refusing to build a degenerate cache key" - exit 1 + echo "::warning::could not resolve ephy_testing_data HEAD on gin; restoring the most recent dataset cache" + hash="" fi echo "dataset_hash=$hash" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/plexon2-testing.yml b/.github/workflows/plexon2-testing.yml index c3b2f314a..8fdab0b5c 100644 --- a/.github/workflows/plexon2-testing.yml +++ b/.github/workflows/plexon2-testing.yml @@ -33,10 +33,11 @@ jobs: # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git id: ephy_testing_data run: | + # if gin cannot be reached the key matches no cache and the restore-keys below restore the most recent one set -o pipefail if ! hash="$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" || [ -z "$hash" ]; then - echo "::error::could not resolve ephy_testing_data HEAD on gin; refusing to build a degenerate cache key" - exit 1 + echo "::warning::could not resolve ephy_testing_data HEAD on gin; restoring the most recent dataset cache" + hash="" fi echo "dataset_hash=$hash" >> "$GITHUB_OUTPUT" diff --git a/neo/utils/datasets.py b/neo/utils/datasets.py index 5d9527551..7e70ddad8 100644 --- a/neo/utils/datasets.py +++ b/neo/utils/datasets.py @@ -3,6 +3,7 @@ """ import os +import warnings from pathlib import Path default_testing_repo = "https://gin.g-node.org/NeuralEnsemble/ephy_testing_data" @@ -14,6 +15,9 @@ # set in home local_testing_data_folder = Path.home() / "ephy_testing_data" +# local copies already updated in this process, so each one asks its remote at most once +_updated_folders = set() + def get_local_testing_data_folder(): global local_testing_data_folder @@ -64,7 +68,12 @@ def download_dataset(repo=default_testing_repo, remote_path=None, local_folder=N # make sure git repo is in clean state repo = dataset.repo repo.call_git(["checkout", "--force", "master"]) - dataset.update(merge=True) + if local_folder not in _updated_folders: + _updated_folders.add(local_folder) + # the remote can be unreachable (gin answers 403 to CI runners at times), so keep the local copy then + results = dataset.update(merge=True, on_failure="ignore") + if any(result["status"] in ("impossible", "error") for result in results): + warnings.warn(f"Could not update {local_folder} from its remote, using the local copy as it is") else: dataset = datalad.api.install(path=local_folder, source=repo)