From c596cda7dc2421caf7f50f0a80dbf64b84357be7 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 21:10:37 +0000
Subject: [PATCH 01/11] Introduced `pip-install-cached.sh` and
 `conda-install-cached.sh` to speed up Python package installation by caching
 environments in GCS.

-   **GCS Caching:** Packages are installed once, and the resulting environment is tarred and uploaded to GCS for reuse.
-   **Deterministic Cache Keys:** Cache keys are generated based on requested packages, OS, Dataproc version, and architecture.
-   **Concurrency Lock:** Uses a `.building` sentinel file in GCS to prevent multiple nodes from simultaneously building the same environment.
-   **Robust Fallback:** If GCS is unreachable or errors occur during caching, the scripts fall back to standard non-cached installation.
-   **Customization Options:**
    -   `CACHE_BUCKET`: Specify custom GCS bucket (defaults to `dataproc-temp-bucket`).
    -   `CACHE_KEY_OVERRIDE`: Manually set cache key.
    -   `CACHE_TIMEOUT`: Set cache expiration (defaults to 10 years/infinite).
    -   `TARGET_ENV_PATH`: Specify target environment directory.

-   Updated `python/README.md` with usage instructions and examples for both cached installers.
---
 python/README.md               |  53 ++++++++
 python/conda-install-cached.sh | 210 +++++++++++++++++++++++++++++
 python/pip-install-cached.sh   | 233 +++++++++++++++++++++++++++++++++
 3 files changed, 496 insertions(+)
 create mode 100644 python/conda-install-cached.sh
 create mode 100644 python/pip-install-cached.sh

diff --git a/python/README.md b/python/README.md
index 682c2db1d..133aca636 100644
--- a/python/README.md
+++ b/python/README.md
@@ -94,3 +94,56 @@ gcloud dataproc clusters create ${CLUSTER_NAME} \
     --metadata 'CONDA_PACKAGES=recentrifuge=1.0.0' \
     --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/conda-install.sh
 ```
+
+## Install PyPI packages with Caching
+
+Install [PyPI](https://pypi.org) packages into user Python environment using `pip` command, with support for caching the resulting environment in GCS to speed up subsequent cluster creations.
+
+### Options
+
+-   `PIP_PACKAGES` - a space separated list of packages to install. Packages can contain version selectors.
+-   `CACHE_BUCKET` - (Optional) The GCS bucket to store/fetch caches. Defaults to the cluster's temporary bucket (`dataproc-temp-bucket`).
+-   `CACHE_KEY_OVERRIDE` - (Optional) Explicit name for the cache file. If omitted, a hash of the packages and environment is used.
+-   `CACHE_TIMEOUT` - (Optional) Cache validity period in seconds. Defaults to 10 years (effectively infinite).
+-   `TARGET_ENV_PATH` - (Optional) Explicit path to the target Python environment. If omitted, defaults to the active environment.
+
+### Examples
+
+Installing packages with default caching (uses temp bucket):
+
+```bash
+REGION=<region>
+CLUSTER_NAME=<cluster_name>
+gcloud dataproc clusters create ${CLUSTER_NAME} \
+    --region ${REGION} \
+    --metadata 'PIP_PACKAGES=pandas scipy' \
+    --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/pip-install-cached.sh
+```
+
+## Install Conda packages with Caching
+
+Install Conda packages into user Python environment using `conda` command, with support for caching the resulting environment in GCS to speed up subsequent cluster creations.
+
+### Options
+
+-   `CONDA_CHANNELS` - a space separated list of new channels to configure in addition to default channels.
+-   `CONDA_PACKAGES` - a space separated list of packages to install. Packages can contain version selectors.
+-   `CACHE_BUCKET` - (Optional) The GCS bucket to store/fetch caches. Defaults to the cluster's temporary bucket (`dataproc-temp-bucket`).
+-   `CACHE_KEY_OVERRIDE` - (Optional) Explicit name for the cache file. If omitted, a hash of the channels, packages, and environment is used.
+-   `CACHE_TIMEOUT` - (Optional) Cache validity period in seconds. Defaults to 10 years (effectively infinite).
+-   `TARGET_ENV_PATH` - (Optional) Explicit path to the target Conda environment. If omitted, defaults to the active environment.
+
+### Examples
+
+Installing packages with explicit cache bucket and override:
+
+```bash
+REGION=<region>
+CLUSTER_NAME=<cluster_name>
+gcloud dataproc clusters create ${CLUSTER_NAME} \
+    --region ${REGION} \
+    --metadata 'CONDA_PACKAGES=scipy' \
+    --metadata 'CACHE_BUCKET=my-custom-cache-bucket' \
+    --metadata 'CACHE_KEY_OVERRIDE=my-scipy-env' \
+    --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/python/conda-install-cached.sh
+```
diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
new file mode 100644
index 000000000..a49839ec0
--- /dev/null
+++ b/python/conda-install-cached.sh
@@ -0,0 +1,210 @@
+#!/bin/bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs Conda packages, using GCS to cache the resulting environment.
+
+set -exo pipefail
+
+readonly CHANNELS=$(/usr/share/google/get_metadata_value attributes/CONDA_CHANNELS || true)
+readonly PACKAGES=$(/usr/share/google/get_metadata_value attributes/CONDA_PACKAGES || true)
+CACHE_BUCKET=$(/usr/share/google/get_metadata_value attributes/CACHE_BUCKET || true)
+if [[ -z "${CACHE_BUCKET}" ]]; then
+  CACHE_BUCKET=$(/usr/share/google/get_metadata_value attributes/dataproc-temp-bucket || true)
+  if [[ -n "${CACHE_BUCKET}" ]]; then
+    echo "CACHE_BUCKET not specified, defaulting to dataproc-temp-bucket: ${CACHE_BUCKET}"
+  fi
+fi
+readonly CACHE_BUCKET
+readonly CACHE_KEY_OVERRIDE=$(/usr/share/google/get_metadata_value attributes/CACHE_KEY_OVERRIDE || true)
+CACHE_TIMEOUT=$(/usr/share/google/get_metadata_value attributes/CACHE_TIMEOUT || true)
+if [[ -z "${CACHE_TIMEOUT}" ]]; then
+  CACHE_TIMEOUT=315360000 # 10 years (effectively infinite)
+fi
+readonly CACHE_TIMEOUT
+readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
+readonly OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+
+# Detect dataproc image version from its various names
+if (! test -v DATAPROC_IMAGE_VERSION) && test -v DATAPROC_VERSION; then
+  DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
+fi
+
+function version_ge(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|tail -n1)" ]]; }
+function version_le(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|head -n1)" ]]; }
+
+function err() {
+  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
+  exit 1
+}
+
+function get_cache_key() {
+  local sorted_packages=$(echo "${PACKAGES}" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs)
+  local sorted_channels=$(echo "${CHANNELS}" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs)
+  # Only hash the sorted packages and channels, environment details are in the path
+  echo "${sorted_packages}_${sorted_channels}" | sha256sum | awk '{print $1}'
+}
+
+function main() {
+  if [[ -z "${PACKAGES}" ]]; then
+    echo "ERROR: Must specify CONDA_PACKAGES metadata key"
+    exit 1
+  fi
+
+  if [[ -z "${CACHE_BUCKET}" ]]; then
+    echo "ERROR: Must specify CACHE_BUCKET metadata key for cached installation"
+    exit 1
+  fi
+
+  # Configure channels first
+  if [[ -n "${CHANNELS}" ]]; then
+    for channel in ${CHANNELS}; do
+      conda config --add channels "${channel}"
+    done
+  fi
+
+  local conda_path=$(which conda)
+  local env_path
+
+  if [[ -n "${TARGET_ENV_PATH}" ]]; then
+    env_path="${TARGET_ENV_PATH}"
+    echo "Using target environment path from metadata: ${env_path}"
+  else
+    env_path=$(dirname $(dirname "${conda_path}"))
+    echo "Inferred target environment path: ${env_path}"
+  fi
+
+  echo "Target environment path: ${env_path}"
+
+  local cache_key
+  if [[ -n "${CACHE_KEY_OVERRIDE}" ]]; then
+    cache_key="${CACHE_KEY_OVERRIDE}"
+    echo "Using manual cache key override: ${cache_key}"
+  else
+    cache_key=$(get_cache_key)
+  fi
+
+  local arch=$(uname -m)
+  # Follow design doc storage layout: gs://<CACHE_BUCKET>/dataproc-caching/conda/<OS_DATAPROC_VER>_<ARCH>/<HASH_OR_KEY>.tar.gz
+  local gcs_tarball="gs://${CACHE_BUCKET}/dataproc-caching/conda/${OS_NAME}_${DATAPROC_IMAGE_VERSION}_${arch}/${cache_key}.tar.gz"
+  local local_tarball="/tmp/${cache_key}.tar.gz"
+
+  set +e
+  gsutil -q stat "${gcs_tarball}"
+  local cache_exists_code=$?
+  set -e
+
+  if [[ ${cache_exists_code} -eq 0 ]]; then
+    local file_time=$(gsutil stat "${gcs_tarball}" | grep -oP 'Creation time:\s*\K.*' || echo "")
+    if [[ -n "${file_time}" ]]; then
+      local file_epoch=$(date -u -d "${file_time}" +%s)
+      local now_epoch=$(date -u +%s)
+      local age=$((now_epoch - file_epoch))
+
+      if (( age <= CACHE_TIMEOUT )); then
+        echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
+        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+        echo "Cache unpacked successfully."
+        return 0
+      else
+        echo "Cache expired for key ${cache_key} (age: ${age}s > ${CACHE_TIMEOUT}s). Rebuilding."
+      fi
+    else
+      echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
+      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+      return 0
+    fi
+  fi
+
+  echo "Cache miss for key ${cache_key}. Checking for concurrent builds."
+
+  set +e
+  local building_output=$(gsutil -q stat "${gcs_tarball}.building" 2>/dev/null)
+  local sentinel_exists_code=$?
+  set -e
+
+  if [[ ${sentinel_exists_code} -eq 0 ]]; then
+    echo "Another node is building this environment. Waiting..."
+    local wait_start=$(date +%s)
+    local timeout="${CACHE_TIMEOUT}"
+    while gsutil -q stat "${gcs_tarball}.building" > /dev/null 2>&1; do
+      if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
+        echo "Cache file appeared while waiting. Skipping build."
+        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+        return 0
+      fi
+      local now=$(date +%s)
+      if (( now - wait_start > timeout )); then
+        echo "Timeout waiting for concurrent build. Proceeding to build myself."
+        break
+      fi
+      echo "Waiting 30 seconds..."
+      sleep 30
+    done
+
+    # Double check if cache appeared just after sentinel removal
+    if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
+      echo "Cache file appeared. Skipping build."
+      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+      return 0
+    fi
+  fi
+
+  echo "Proceeding to build and cache environment."
+
+  touch "${local_tarball}.building"
+  
+  set +e
+  gsutil cp "${local_tarball}.building" "${gcs_tarball}.building"
+  local sentinel_upload_code=$?
+  set -e
+
+  if [[ ${sentinel_upload_code} -ne 0 ]]; then
+    echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
+    echo "Falling back to standard non-cached installation."
+    conda install -y -p "${env_path}" ${PACKAGES}
+    rm -f "${local_tarball}.building"
+    return 0
+  fi
+
+  # Ensure we clean up sentinel on exit if we uploaded it
+  trap 'gsutil rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
+
+  # Perform standard installation
+  conda install -y -p "${env_path}" ${PACKAGES}
+
+  echo "Packaging environment."
+  # We package the contents of the environment path
+  pushd "${env_path}"
+  tar czf "${local_tarball}" .
+  popd
+
+  echo "Uploading to GCS."
+  set +e
+  gsutil cp "${local_tarball}" "${gcs_tarball}"
+  local cache_upload_code=$?
+  set -e
+
+  if [[ ${cache_upload_code} -ne 0 ]]; then
+    echo "WARNING: Failed to upload cache to GCS."
+  fi
+
+  echo "Cleaning up."
+  trap - EXIT
+  gsutil rm "${gcs_tarball}.building" || true
+  rm -f "${local_tarball}" "${local_tarball}.building"
+
+  echo "Done."
+}
+
+main
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
new file mode 100644
index 000000000..b040dd670
--- /dev/null
+++ b/python/pip-install-cached.sh
@@ -0,0 +1,233 @@
+#!/bin/bash
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS-IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script installs Python packages via pip, using GCS to cache the resulting environment.
+
+set -exo pipefail
+
+readonly PACKAGES=$(/usr/share/google/get_metadata_value attributes/PIP_PACKAGES || true)
+CACHE_BUCKET=$(/usr/share/google/get_metadata_value attributes/CACHE_BUCKET || true)
+if [[ -z "${CACHE_BUCKET}" ]]; then
+  CACHE_BUCKET=$(/usr/share/google/get_metadata_value attributes/dataproc-temp-bucket || true)
+  if [[ -n "${CACHE_BUCKET}" ]]; then
+    echo "CACHE_BUCKET not specified, defaulting to dataproc-temp-bucket: ${CACHE_BUCKET}"
+  fi
+fi
+readonly CACHE_BUCKET
+readonly CACHE_KEY_OVERRIDE=$(/usr/share/google/get_metadata_value attributes/CACHE_KEY_OVERRIDE || true)
+CACHE_TIMEOUT=$(/usr/share/google/get_metadata_value attributes/CACHE_TIMEOUT || true)
+if [[ -z "${CACHE_TIMEOUT}" ]]; then
+  CACHE_TIMEOUT=315360000 # 10 years (effectively infinite)
+fi
+readonly CACHE_TIMEOUT
+readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
+readonly OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+
+# Detect dataproc image version from its various names
+if (! test -v DATAPROC_IMAGE_VERSION) && test -v DATAPROC_VERSION; then
+  DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
+fi
+
+function version_ge(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|tail -n1)" ]]; }
+function version_le(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|head -n1)" ]]; }
+
+function err() {
+  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
+  exit 1
+}
+
+function run_with_retry() {
+  local -r cmd=("$@")
+  for ((i = 0; i < 10; i++)); do
+    if "${cmd[@]}"; then
+      return 0
+    fi
+    sleep 5
+  done
+  err "Failed to run command: ${cmd[*]}"
+}
+
+function install_pip() {
+  if command -v pip >/dev/null; then
+    echo "pip is already installed."
+    return 0
+  fi
+
+  if command -v easy_install >/dev/null; then
+    echo "Installing pip with easy_install..."
+    run_with_retry easy_install pip
+    return 0
+  fi
+
+  echo "Installing python-pip..."
+  run_with_retry apt update
+  run_with_retry apt install python-pip -y
+}
+
+function get_cache_key() {
+  local sorted_packages=$(echo "${PACKAGES}" | tr ' ' '\n' | sort | tr '\n' ' ' | xargs)
+  # Only hash the sorted packages, environment details are in the path
+  echo "${sorted_packages}" | sha256sum | awk '{print $1}'
+}
+
+function main() {
+  if [[ -z "${PACKAGES}" ]]; then
+    echo "ERROR: Must specify PIP_PACKAGES metadata key"
+    exit 1
+  fi
+
+  if [[ -z "${CACHE_BUCKET}" ]]; then
+    echo "ERROR: Must specify CACHE_BUCKET metadata key for cached installation"
+    exit 1
+  fi
+
+  local env_path
+  local pip_bin
+
+  if [[ -n "${TARGET_ENV_PATH}" ]]; then
+    env_path="${TARGET_ENV_PATH}"
+    pip_bin="${env_path}/bin/pip"
+    echo "Using target environment path from metadata: ${env_path}"
+  else
+    install_pip
+    local pip_path=$(which pip)
+    env_path=$(dirname $(dirname "${pip_path}"))
+    pip_bin="pip"
+    echo "Inferred target environment path: ${env_path}"
+  fi
+
+  echo "Target environment path: ${env_path}"
+
+  local cache_key
+  if [[ -n "${CACHE_KEY_OVERRIDE}" ]]; then
+    cache_key="${CACHE_KEY_OVERRIDE}"
+    echo "Using manual cache key override: ${cache_key}"
+  else
+    cache_key=$(get_cache_key)
+  fi
+
+  local arch=$(uname -m)
+  # Follow design doc storage layout: gs://<CACHE_BUCKET>/dataproc-caching/pip/<OS_DATAPROC_VER>_<ARCH>/<HASH_OR_KEY>.tar.gz
+  local gcs_tarball="gs://${CACHE_BUCKET}/dataproc-caching/pip/${OS_NAME}_${DATAPROC_IMAGE_VERSION}_${arch}/${cache_key}.tar.gz"
+  local local_tarball="/tmp/${cache_key}.tar.gz"
+
+  set +e
+  gsutil -q stat "${gcs_tarball}"
+  local cache_exists_code=$?
+  set -e
+
+  if [[ ${cache_exists_code} -eq 0 ]]; then
+    local file_time=$(gsutil stat "${gcs_tarball}" | grep -oP 'Creation time:\s*\K.*' || echo "")
+    if [[ -n "${file_time}" ]]; then
+      local file_epoch=$(date -u -d "${file_time}" +%s)
+      local now_epoch=$(date -u +%s)
+      local age=$((now_epoch - file_epoch))
+
+      if (( age <= CACHE_TIMEOUT )); then
+        echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
+        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+        echo "Cache unpacked successfully."
+        return 0
+      else
+        echo "Cache expired for key ${cache_key} (age: ${age}s > ${CACHE_TIMEOUT}s). Rebuilding."
+      fi
+    else
+      echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
+      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+      return 0
+    fi
+  fi
+
+  echo "Cache miss for key ${cache_key}. Checking for concurrent builds."
+
+  set +e
+  local building_output=$(gsutil -q stat "${gcs_tarball}.building" 2>/dev/null)
+  local sentinel_exists_code=$?
+  set -e
+
+  if [[ ${sentinel_exists_code} -eq 0 ]]; then
+    echo "Another node is building this environment. Waiting..."
+    local wait_start=$(date +%s)
+    local timeout="${CACHE_TIMEOUT}"
+    while gsutil -q stat "${gcs_tarball}.building" > /dev/null 2>&1; do
+      if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
+        echo "Cache file appeared while waiting. Skipping build."
+        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+        return 0
+      fi
+      local now=$(date +%s)
+      if (( now - wait_start > timeout )); then
+        echo "Timeout waiting for concurrent build. Proceeding to build myself."
+        break
+      fi
+      echo "Waiting 30 seconds..."
+      sleep 30
+    done
+    
+    # Double check if cache appeared just after sentinel removal
+    if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
+      echo "Cache file appeared. Skipping build."
+      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
+      return 0
+    fi
+  fi
+
+  echo "Proceeding to build and cache environment."
+  
+  touch "${local_tarball}.building"
+  
+  set +e
+  gsutil cp "${local_tarball}.building" "${gcs_tarball}.building"
+  local sentinel_upload_code=$?
+  set -e
+
+  if [[ ${sentinel_upload_code} -ne 0 ]]; then
+    echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
+    echo "Falling back to standard non-cached installation."
+    run_with_retry "${pip_bin}" install --upgrade ${PACKAGES}
+    rm -f "${local_tarball}.building"
+    return 0
+  fi
+
+  # Ensure we clean up sentinel on exit if we uploaded it
+  trap 'gsutil rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
+
+  # Perform standard installation
+  run_with_retry "${pip_bin}" install --upgrade ${PACKAGES}
+
+  echo "Packaging environment."
+  # We package the contents of the environment path
+  pushd "${env_path}"
+  tar czf "${local_tarball}" .
+  popd
+
+  echo "Uploading to GCS."
+  set +e
+  gsutil cp "${local_tarball}" "${gcs_tarball}"
+  local cache_upload_code=$?
+  set -e
+
+  if [[ ${cache_upload_code} -ne 0 ]]; then
+    echo "WARNING: Failed to upload cache to GCS."
+  fi
+
+  echo "Cleaning up."
+  trap - EXIT
+  gsutil rm "${gcs_tarball}.building" || true
+  rm -f "${local_tarball}" "${local_tarball}.building"
+
+  echo "Done."
+}
+
+main

From 2fad29385bc336fde0baf955ce6506d90abd7c40 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 21:37:55 +0000
Subject: [PATCH 02/11] Fix safety, error handling, and globbing in cached
 installers

- Refuse to cache system-wide directories (/usr, /usr/local, /) to prevent system corruption and excessive cache sizes.
- Handle date parsing failures gracefully during cache age verification.
- Wrap tar unpacking in conditional blocks to allow graceful fallback to rebuilding if cache is corrupted.
- Use Bash arrays to pass package lists, preventing pathname expansion (globbing) bugs with wildcards.
- Remove unused version comparison helper functions.
---
 python/conda-install-cached.sh | 49 +++++++++++++++++++++++----------
 python/pip-install-cached.sh   | 50 ++++++++++++++++++++++++----------
 2 files changed, 71 insertions(+), 28 deletions(-)

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index a49839ec0..d2613ab00 100644
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -40,8 +40,6 @@ if (! test -v DATAPROC_IMAGE_VERSION) && test -v DATAPROC_VERSION; then
   DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
 fi
 
-function version_ge(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|tail -n1)" ]]; }
-function version_le(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|head -n1)" ]]; }
 
 function err() {
   echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
@@ -81,6 +79,11 @@ function main() {
     echo "Using target environment path from metadata: ${env_path}"
   else
     env_path=$(dirname $(dirname "${conda_path}"))
+    if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
+      echo "ERROR: Refusing to cache system-wide environment path: ${env_path}."
+      echo "Caching the entire system directory is unsafe. Please specify a target environment path using TARGET_ENV_PATH."
+      exit 1
+    fi
     echo "Inferred target environment path: ${env_path}"
   fi
 
@@ -106,23 +109,33 @@ function main() {
 
   if [[ ${cache_exists_code} -eq 0 ]]; then
     local file_time=$(gsutil stat "${gcs_tarball}" | grep -oP 'Creation time:\s*\K.*' || echo "")
+    local file_epoch=""
     if [[ -n "${file_time}" ]]; then
-      local file_epoch=$(date -u -d "${file_time}" +%s)
+      file_epoch=$(date -u -d "${file_time}" +%s 2>/dev/null || echo "")
+    fi
+
+    if [[ -n "${file_epoch}" ]]; then
       local now_epoch=$(date -u +%s)
       local age=$((now_epoch - file_epoch))
 
       if (( age <= CACHE_TIMEOUT )); then
         echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
-        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-        echo "Cache unpacked successfully."
-        return 0
+        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+          echo "Cache unpacked successfully."
+          return 0
+        else
+          echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
+        fi
       else
         echo "Cache expired for key ${cache_key} (age: ${age}s > ${CACHE_TIMEOUT}s). Rebuilding."
       fi
     else
       echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
-      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-      return 0
+      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        return 0
+      else
+        echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
+      fi
     fi
   fi
 
@@ -140,8 +153,11 @@ function main() {
     while gsutil -q stat "${gcs_tarball}.building" > /dev/null 2>&1; do
       if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
         echo "Cache file appeared while waiting. Skipping build."
-        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-        return 0
+        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+          return 0
+        fi
+        echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+        break
       fi
       local now=$(date +%s)
       if (( now - wait_start > timeout )); then
@@ -155,13 +171,18 @@ function main() {
     # Double check if cache appeared just after sentinel removal
     if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
       echo "Cache file appeared. Skipping build."
-      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-      return 0
+      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        return 0
+      fi
+      echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
     fi
   fi
 
   echo "Proceeding to build and cache environment."
 
+  local -a pack_arr
+  read -r -a pack_arr <<< "${PACKAGES}"
+
   touch "${local_tarball}.building"
   
   set +e
@@ -172,7 +193,7 @@ function main() {
   if [[ ${sentinel_upload_code} -ne 0 ]]; then
     echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
     echo "Falling back to standard non-cached installation."
-    conda install -y -p "${env_path}" ${PACKAGES}
+    conda install -y -p "${env_path}" "${pack_arr[@]}"
     rm -f "${local_tarball}.building"
     return 0
   fi
@@ -181,7 +202,7 @@ function main() {
   trap 'gsutil rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
 
   # Perform standard installation
-  conda install -y -p "${env_path}" ${PACKAGES}
+  conda install -y -p "${env_path}" "${pack_arr[@]}"
 
   echo "Packaging environment."
   # We package the contents of the environment path
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index b040dd670..c02be1978 100644
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -39,8 +39,6 @@ if (! test -v DATAPROC_IMAGE_VERSION) && test -v DATAPROC_VERSION; then
   DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
 fi
 
-function version_ge(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|tail -n1)" ]]; }
-function version_le(){ [[ "$1" = "$(echo -e "$1\n$2"|sort -V|head -n1)" ]]; }
 
 function err() {
   echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
@@ -103,6 +101,11 @@ function main() {
     install_pip
     local pip_path=$(which pip)
     env_path=$(dirname $(dirname "${pip_path}"))
+    if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
+      echo "ERROR: Refusing to cache system-wide environment path: ${env_path}."
+      echo "Caching the entire system directory is unsafe. Please specify a virtual environment path using TARGET_ENV_PATH."
+      exit 1
+    fi
     pip_bin="pip"
     echo "Inferred target environment path: ${env_path}"
   fi
@@ -129,23 +132,33 @@ function main() {
 
   if [[ ${cache_exists_code} -eq 0 ]]; then
     local file_time=$(gsutil stat "${gcs_tarball}" | grep -oP 'Creation time:\s*\K.*' || echo "")
+    local file_epoch=""
     if [[ -n "${file_time}" ]]; then
-      local file_epoch=$(date -u -d "${file_time}" +%s)
+      file_epoch=$(date -u -d "${file_time}" +%s 2>/dev/null || echo "")
+    fi
+
+    if [[ -n "${file_epoch}" ]]; then
       local now_epoch=$(date -u +%s)
       local age=$((now_epoch - file_epoch))
 
       if (( age <= CACHE_TIMEOUT )); then
         echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
-        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-        echo "Cache unpacked successfully."
-        return 0
+        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+          echo "Cache unpacked successfully."
+          return 0
+        else
+          echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
+        fi
       else
         echo "Cache expired for key ${cache_key} (age: ${age}s > ${CACHE_TIMEOUT}s). Rebuilding."
       fi
     else
       echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
-      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-      return 0
+      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        return 0
+      else
+        echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
+      fi
     fi
   fi
 
@@ -163,8 +176,11 @@ function main() {
     while gsutil -q stat "${gcs_tarball}.building" > /dev/null 2>&1; do
       if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
         echo "Cache file appeared while waiting. Skipping build."
-        gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-        return 0
+        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+          return 0
+        fi
+        echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+        break
       fi
       local now=$(date +%s)
       if (( now - wait_start > timeout )); then
@@ -178,8 +194,10 @@ function main() {
     # Double check if cache appeared just after sentinel removal
     if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
       echo "Cache file appeared. Skipping build."
-      gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz
-      return 0
+      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        return 0
+      fi
+      echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
     fi
   fi
 
@@ -195,7 +213,9 @@ function main() {
   if [[ ${sentinel_upload_code} -ne 0 ]]; then
     echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
     echo "Falling back to standard non-cached installation."
-    run_with_retry "${pip_bin}" install --upgrade ${PACKAGES}
+    local -a pack_arr
+    read -r -a pack_arr <<< "${PACKAGES}"
+    run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
     rm -f "${local_tarball}.building"
     return 0
   fi
@@ -204,7 +224,9 @@ function main() {
   trap 'gsutil rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
 
   # Perform standard installation
-  run_with_retry "${pip_bin}" install --upgrade ${PACKAGES}
+  local -a pack_arr
+  read -r -a pack_arr <<< "${PACKAGES}"
+  run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
 
   echo "Packaging environment."
   # We package the contents of the environment path

From a04ba43525e121c3fc5b069b58f51401c891b59c Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 22:06:21 +0000
Subject: [PATCH 03/11] Prefer gcloud storage over gsutil in cached installers

- Introduce GCS helper functions to abstract the underlying tool (gcloud storage or gsutil).
- Prefer `gcloud storage` for better performance if available.
- Fall back to `gsutil` on older environments transparently.
- Maintain atomic uploads and generation matching across both tools.
- Also includes previous fixes for safety warnings, error handling, and atomic locks.
---
 python/conda-install-cached.sh | 192 ++++++++++++++++++++++----------
 python/pip-install-cached.sh   | 198 ++++++++++++++++++++++-----------
 2 files changed, 267 insertions(+), 123 deletions(-)

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index d2613ab00..7a15bf0b4 100644
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -35,6 +35,74 @@ readonly CACHE_TIMEOUT
 readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
 readonly OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
 
+GCS_CMD="gsutil"
+if gcloud --help >/dev/null 2>&1 && gcloud storage --help >/dev/null 2>&1; then
+  GCS_CMD="gcloud storage"
+fi
+readonly GCS_CMD
+
+function gcs_exists() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage objects describe "${uri}" >/dev/null 2>&1
+  else
+    gsutil -q stat "${uri}"
+  fi
+}
+
+function gcs_file_epoch() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    local file_time=$(gcloud storage objects describe "${uri}" --format="value(timeCreated)" 2>/dev/null || echo "")
+    if [[ -n "${file_time}" ]]; then
+      date -u -d "${file_time}" +%s 2>/dev/null || echo ""
+    fi
+  else
+    local file_time=$(gsutil stat "${uri}" | grep -oP 'Creation time:\s*\K.*' || echo "")
+    if [[ -n "${file_time}" ]]; then
+      date -u -d "${file_time}" +%s 2>/dev/null || echo ""
+    fi
+  fi
+}
+
+function gcs_cat() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage cat "${uri}"
+  else
+    gsutil cat "${uri}"
+  fi
+}
+
+function gcs_atomic_upload() {
+  local local_path=$1
+  local gcs_uri=$2
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage cp --if-generation-match=0 "${local_path}" "${gcs_uri}" 2>/dev/null
+  else
+    gsutil -h "x-goog-if-generation-match:0" cp "${local_path}" "${gcs_uri}" 2>/dev/null
+  fi
+}
+
+function gcs_cp() {
+  local local_path=$1
+  local gcs_uri=$2
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage cp "${local_path}" "${gcs_uri}"
+  else
+    gsutil cp "${local_path}" "${gcs_uri}"
+  fi
+}
+
+function gcs_rm() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage rm "${uri}"
+  else
+    gsutil rm "${uri}"
+  fi
+}
+
 # Detect dataproc image version from its various names
 if (! test -v DATAPROC_IMAGE_VERSION) && test -v DATAPROC_VERSION; then
   DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
@@ -80,9 +148,8 @@ function main() {
   else
     env_path=$(dirname $(dirname "${conda_path}"))
     if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
-      echo "ERROR: Refusing to cache system-wide environment path: ${env_path}."
-      echo "Caching the entire system directory is unsafe. Please specify a target environment path using TARGET_ENV_PATH."
-      exit 1
+      echo "WARNING: Inferred system-wide environment path: ${env_path}."
+      echo "Caching the entire system directory is unsafe and may result in large cache files. It is strongly recommended to specify a target environment path using TARGET_ENV_PATH."
     fi
     echo "Inferred target environment path: ${env_path}"
   fi
@@ -103,16 +170,12 @@ function main() {
   local local_tarball="/tmp/${cache_key}.tar.gz"
 
   set +e
-  gsutil -q stat "${gcs_tarball}"
+  gcs_exists "${gcs_tarball}"
   local cache_exists_code=$?
   set -e
 
   if [[ ${cache_exists_code} -eq 0 ]]; then
-    local file_time=$(gsutil stat "${gcs_tarball}" | grep -oP 'Creation time:\s*\K.*' || echo "")
-    local file_epoch=""
-    if [[ -n "${file_time}" ]]; then
-      file_epoch=$(date -u -d "${file_time}" +%s 2>/dev/null || echo "")
-    fi
+    local file_epoch=$(gcs_file_epoch "${gcs_tarball}")
 
     if [[ -n "${file_epoch}" ]]; then
       local now_epoch=$(date -u +%s)
@@ -120,7 +183,7 @@ function main() {
 
       if (( age <= CACHE_TIMEOUT )); then
         echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
-        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
           echo "Cache unpacked successfully."
           return 0
         else
@@ -131,7 +194,7 @@ function main() {
       fi
     else
       echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
-      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+      if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
         return 0
       else
         echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
@@ -141,65 +204,74 @@ function main() {
 
   echo "Cache miss for key ${cache_key}. Checking for concurrent builds."
 
-  set +e
-  local building_output=$(gsutil -q stat "${gcs_tarball}.building" 2>/dev/null)
-  local sentinel_exists_code=$?
-  set -e
+  local -a pack_arr
+  read -r -a pack_arr <<< "${PACKAGES}"
 
-  if [[ ${sentinel_exists_code} -eq 0 ]]; then
-    echo "Another node is building this environment. Waiting..."
-    local wait_start=$(date +%s)
-    local timeout="${CACHE_TIMEOUT}"
-    while gsutil -q stat "${gcs_tarball}.building" > /dev/null 2>&1; do
-      if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
-        echo "Cache file appeared while waiting. Skipping build."
-        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+  while true; do
+    set +e
+    gcs_exists "${gcs_tarball}.building"
+    local sentinel_exists_code=$?
+    set -e
+
+    if [[ ${sentinel_exists_code} -eq 0 ]]; then
+      echo "Another node is building this environment. Waiting..."
+      local wait_start=$(date +%s)
+      local timeout="${CACHE_TIMEOUT}"
+      while gcs_exists "${gcs_tarball}.building"; do
+        if gcs_exists "${gcs_tarball}"; then
+          echo "Cache file appeared while waiting. Skipping build."
+          if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+            return 0
+          fi
+          echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+          break
+        fi
+        local now=$(date +%s)
+        if (( now - wait_start > timeout )); then
+          echo "Timeout waiting for concurrent build. Proceeding to build myself."
+          break
+        fi
+        echo "Waiting 30 seconds..."
+        sleep 30
+      done
+
+      # Double check if cache appeared just after sentinel removal
+      if gcs_exists "${gcs_tarball}"; then
+        echo "Cache file appeared. Skipping build."
+        if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
           return 0
         fi
         echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
-        break
-      fi
-      local now=$(date +%s)
-      if (( now - wait_start > timeout )); then
-        echo "Timeout waiting for concurrent build. Proceeding to build myself."
-        break
-      fi
-      echo "Waiting 30 seconds..."
-      sleep 30
-    done
-
-    # Double check if cache appeared just after sentinel removal
-    if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
-      echo "Cache file appeared. Skipping build."
-      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
-        return 0
       fi
-      echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
     fi
-  fi
 
-  echo "Proceeding to build and cache environment."
+    echo "Proceeding to build and cache environment."
 
-  local -a pack_arr
-  read -r -a pack_arr <<< "${PACKAGES}"
+    touch "${local_tarball}.building"
 
-  touch "${local_tarball}.building"
-  
-  set +e
-  gsutil cp "${local_tarball}.building" "${gcs_tarball}.building"
-  local sentinel_upload_code=$?
-  set -e
+    set +e
+    gcs_atomic_upload "${local_tarball}.building" "${gcs_tarball}.building"
+    local sentinel_upload_code=$?
+    set -e
 
-  if [[ ${sentinel_upload_code} -ne 0 ]]; then
-    echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
-    echo "Falling back to standard non-cached installation."
-    conda install -y -p "${env_path}" "${pack_arr[@]}"
-    rm -f "${local_tarball}.building"
-    return 0
-  fi
+    if [[ ${sentinel_upload_code} -eq 0 ]]; then
+      break
+    fi
+
+    # Check if the failure was due to a race condition (sentinel exists) or GCS being unreachable
+    if ! gcs_exists "${gcs_tarball}.building"; then
+      echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
+      echo "Falling back to standard non-cached installation."
+      conda install -y -p "${env_path}" "${pack_arr[@]}"
+      rm -f "${local_tarball}.building"
+      return 0
+    fi
+
+    echo "Lost the lock race to another node. Retrying..."
+  done
 
   # Ensure we clean up sentinel on exit if we uploaded it
-  trap 'gsutil rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
+  trap 'gcs_rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
 
   # Perform standard installation
   conda install -y -p "${env_path}" "${pack_arr[@]}"
@@ -212,7 +284,7 @@ function main() {
 
   echo "Uploading to GCS."
   set +e
-  gsutil cp "${local_tarball}" "${gcs_tarball}"
+  gcs_cp "${local_tarball}" "${gcs_tarball}"
   local cache_upload_code=$?
   set -e
 
@@ -222,7 +294,7 @@ function main() {
 
   echo "Cleaning up."
   trap - EXIT
-  gsutil rm "${gcs_tarball}.building" || true
+  gcs_rm "${gcs_tarball}.building" || true
   rm -f "${local_tarball}" "${local_tarball}.building"
 
   echo "Done."
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index c02be1978..8e69855ca 100644
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -34,6 +34,74 @@ readonly CACHE_TIMEOUT
 readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
 readonly OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
 
+GCS_CMD="gsutil"
+if gcloud --help >/dev/null 2>&1 && gcloud storage --help >/dev/null 2>&1; then
+  GCS_CMD="gcloud storage"
+fi
+readonly GCS_CMD
+
+function gcs_exists() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage objects describe "${uri}" >/dev/null 2>&1
+  else
+    gsutil -q stat "${uri}"
+  fi
+}
+
+function gcs_file_epoch() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    local file_time=$(gcloud storage objects describe "${uri}" --format="value(timeCreated)" 2>/dev/null || echo "")
+    if [[ -n "${file_time}" ]]; then
+      date -u -d "${file_time}" +%s 2>/dev/null || echo ""
+    fi
+  else
+    local file_time=$(gsutil stat "${uri}" | grep -oP 'Creation time:\s*\K.*' || echo "")
+    if [[ -n "${file_time}" ]]; then
+      date -u -d "${file_time}" +%s 2>/dev/null || echo ""
+    fi
+  fi
+}
+
+function gcs_cat() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage cat "${uri}"
+  else
+    gsutil cat "${uri}"
+  fi
+}
+
+function gcs_atomic_upload() {
+  local local_path=$1
+  local gcs_uri=$2
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage cp --if-generation-match=0 "${local_path}" "${gcs_uri}" 2>/dev/null
+  else
+    gsutil -h "x-goog-if-generation-match:0" cp "${local_path}" "${gcs_uri}" 2>/dev/null
+  fi
+}
+
+function gcs_cp() {
+  local local_path=$1
+  local gcs_uri=$2
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage cp "${local_path}" "${gcs_uri}"
+  else
+    gsutil cp "${local_path}" "${gcs_uri}"
+  fi
+}
+
+function gcs_rm() {
+  local uri=$1
+  if [[ "${GCS_CMD}" == "gcloud storage" ]]; then
+    gcloud storage rm "${uri}"
+  else
+    gsutil rm "${uri}"
+  fi
+}
+
 # Detect dataproc image version from its various names
 if (! test -v DATAPROC_IMAGE_VERSION) && test -v DATAPROC_VERSION; then
   DATAPROC_IMAGE_VERSION="${DATAPROC_VERSION}"
@@ -68,9 +136,9 @@ function install_pip() {
     return 0
   fi
 
-  echo "Installing python-pip..."
+  echo "Installing python3-pip..."
   run_with_retry apt update
-  run_with_retry apt install python-pip -y
+  run_with_retry apt install python3-pip -y
 }
 
 function get_cache_key() {
@@ -102,9 +170,8 @@ function main() {
     local pip_path=$(which pip)
     env_path=$(dirname $(dirname "${pip_path}"))
     if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
-      echo "ERROR: Refusing to cache system-wide environment path: ${env_path}."
-      echo "Caching the entire system directory is unsafe. Please specify a virtual environment path using TARGET_ENV_PATH."
-      exit 1
+      echo "WARNING: Inferred system-wide environment path: ${env_path}."
+      echo "Caching the entire system directory is unsafe and may result in large cache files. It is strongly recommended to specify a virtual environment path using TARGET_ENV_PATH."
     fi
     pip_bin="pip"
     echo "Inferred target environment path: ${env_path}"
@@ -126,16 +193,12 @@ function main() {
   local local_tarball="/tmp/${cache_key}.tar.gz"
 
   set +e
-  gsutil -q stat "${gcs_tarball}"
+  gcs_exists "${gcs_tarball}"
   local cache_exists_code=$?
   set -e
 
   if [[ ${cache_exists_code} -eq 0 ]]; then
-    local file_time=$(gsutil stat "${gcs_tarball}" | grep -oP 'Creation time:\s*\K.*' || echo "")
-    local file_epoch=""
-    if [[ -n "${file_time}" ]]; then
-      file_epoch=$(date -u -d "${file_time}" +%s 2>/dev/null || echo "")
-    fi
+    local file_epoch=$(gcs_file_epoch "${gcs_tarball}")
 
     if [[ -n "${file_epoch}" ]]; then
       local now_epoch=$(date -u +%s)
@@ -143,7 +206,7 @@ function main() {
 
       if (( age <= CACHE_TIMEOUT )); then
         echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
-        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
           echo "Cache unpacked successfully."
           return 0
         else
@@ -154,7 +217,7 @@ function main() {
       fi
     else
       echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
-      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+      if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
         return 0
       else
         echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
@@ -164,64 +227,73 @@ function main() {
 
   echo "Cache miss for key ${cache_key}. Checking for concurrent builds."
 
-  set +e
-  local building_output=$(gsutil -q stat "${gcs_tarball}.building" 2>/dev/null)
-  local sentinel_exists_code=$?
-  set -e
-
-  if [[ ${sentinel_exists_code} -eq 0 ]]; then
-    echo "Another node is building this environment. Waiting..."
-    local wait_start=$(date +%s)
-    local timeout="${CACHE_TIMEOUT}"
-    while gsutil -q stat "${gcs_tarball}.building" > /dev/null 2>&1; do
-      if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
-        echo "Cache file appeared while waiting. Skipping build."
-        if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+  while true; do
+    set +e
+    gcs_exists "${gcs_tarball}.building"
+    local sentinel_exists_code=$?
+    set -e
+
+    if [[ ${sentinel_exists_code} -eq 0 ]]; then
+      echo "Another node is building this environment. Waiting..."
+      local wait_start=$(date +%s)
+      local timeout="${CACHE_TIMEOUT}"
+      while gcs_exists "${gcs_tarball}.building"; do
+        if gcs_exists "${gcs_tarball}"; then
+          echo "Cache file appeared while waiting. Skipping build."
+          if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+            return 0
+          fi
+          echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+          break
+        fi
+        local now=$(date +%s)
+        if (( now - wait_start > timeout )); then
+          echo "Timeout waiting for concurrent build. Proceeding to build myself."
+          break
+        fi
+        echo "Waiting 30 seconds..."
+        sleep 30
+      done
+      
+      # Double check if cache appeared just after sentinel removal
+      if gcs_exists "${gcs_tarball}"; then
+        echo "Cache file appeared. Skipping build."
+        if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
           return 0
         fi
         echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
-        break
-      fi
-      local now=$(date +%s)
-      if (( now - wait_start > timeout )); then
-        echo "Timeout waiting for concurrent build. Proceeding to build myself."
-        break
       fi
-      echo "Waiting 30 seconds..."
-      sleep 30
-    done
+    fi
+
+    echo "Proceeding to build and cache environment."
     
-    # Double check if cache appeared just after sentinel removal
-    if gsutil -q stat "${gcs_tarball}" > /dev/null 2>&1; then
-      echo "Cache file appeared. Skipping build."
-      if gsutil cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
-        return 0
-      fi
-      echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+    touch "${local_tarball}.building"
+    
+    set +e
+    gcs_atomic_upload "${local_tarball}.building" "${gcs_tarball}.building"
+    local sentinel_upload_code=$?
+    set -e
+
+    if [[ ${sentinel_upload_code} -eq 0 ]]; then
+      break
     fi
-  fi
 
-  echo "Proceeding to build and cache environment."
-  
-  touch "${local_tarball}.building"
-  
-  set +e
-  gsutil cp "${local_tarball}.building" "${gcs_tarball}.building"
-  local sentinel_upload_code=$?
-  set -e
+    # Check if the failure was due to a race condition (sentinel exists) or GCS being unreachable
+    if ! gcs_exists "${gcs_tarball}.building"; then
+      echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
+      echo "Falling back to standard non-cached installation."
+      local -a pack_arr
+      read -r -a pack_arr <<< "${PACKAGES}"
+      run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
+      rm -f "${local_tarball}.building"
+      return 0
+    fi
 
-  if [[ ${sentinel_upload_code} -ne 0 ]]; then
-    echo "WARNING: Failed to upload sentinel to GCS. GCS might be unreachable."
-    echo "Falling back to standard non-cached installation."
-    local -a pack_arr
-    read -r -a pack_arr <<< "${PACKAGES}"
-    run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
-    rm -f "${local_tarball}.building"
-    return 0
-  fi
+    echo "Lost the lock race to another node. Retrying..."
+  done
 
   # Ensure we clean up sentinel on exit if we uploaded it
-  trap 'gsutil rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
+  trap 'gcs_rm "${gcs_tarball}.building" || true; rm -f "${local_tarball}.building"' EXIT
 
   # Perform standard installation
   local -a pack_arr
@@ -236,7 +308,7 @@ function main() {
 
   echo "Uploading to GCS."
   set +e
-  gsutil cp "${local_tarball}" "${gcs_tarball}"
+  gcs_cp "${local_tarball}" "${gcs_tarball}"
   local cache_upload_code=$?
   set -e
 
@@ -246,7 +318,7 @@ function main() {
 
   echo "Cleaning up."
   trap - EXIT
-  gsutil rm "${gcs_tarball}.building" || true
+  gcs_rm "${gcs_tarball}.building" || true
   rm -f "${local_tarball}" "${local_tarball}.building"
 
   echo "Done."

From 8f2b78f084cab327d8b06ec372edaaaa84b28cfa Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 22:21:31 +0000
Subject: [PATCH 04/11] Use isolated venv for system Python in
 pip-install-cached.sh

- Prevent caching system-wide directories like `/usr` by switching to a temporary isolated virtual environment (`/tmp/pip-isolated-env`) when system Python is detected.
- Add `ensure_venv` helper to install `python3-venv` or `python3-virtualenv` if missing.
- Add `unpack_and_merge` helper to handle unpacking and merging isolated environments back to the original system path.
- Merge isolated environment contents back to the system path after installation or unpacking to maintain backward compatibility.
---
 python/pip-install-cached.sh | 64 ++++++++++++++++++++++++++++++++----
 1 file changed, 58 insertions(+), 6 deletions(-)

diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index 8e69855ca..910d8152d 100644
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -147,6 +147,40 @@ function get_cache_key() {
   echo "${sorted_packages}" | sha256sum | awk '{print $1}'
 }
 
+function ensure_venv() {
+  if python3 -m venv --help >/dev/null 2>&1; then
+    echo "venv is available."
+    return 0
+  fi
+
+  echo "Installing venv..."
+  if [[ "${OS_NAME}" == "debian" || "${OS_NAME}" == "ubuntu" ]]; then
+    run_with_retry apt update
+    run_with_retry apt install python3-venv -y
+  elif [[ "${OS_NAME}" == "rocky" ]]; then
+    run_with_retry yum install -y python3-virtualenv || run_with_retry dnf install -y python3-virtualenv
+  fi
+}
+
+function unpack_and_merge() {
+  local tarball=$1
+  local target_path=$2
+  local original_path=$3
+  local is_isolated=$4
+
+  mkdir -p "${target_path}"
+  if gcs_cat "${tarball}" | tar -C "${target_path}" -xz; then
+    if [[ "${is_isolated}" == "true" ]]; then
+      echo "Merging isolated environment back to ${original_path}"
+      # Ensure target directories exist in original path if needed?
+      # cp -a should handle it if original_path is directory.
+      cp -a "${target_path}/." "${original_path}/"
+    fi
+    return 0
+  fi
+  return 1
+}
+
 function main() {
   if [[ -z "${PACKAGES}" ]]; then
     echo "ERROR: Must specify PIP_PACKAGES metadata key"
@@ -160,6 +194,9 @@ function main() {
 
   local env_path
   local pip_bin
+  local original_env_path=""
+  local isolated_env_path="/tmp/pip-isolated-env"
+  local isolated="false"
 
   if [[ -n "${TARGET_ENV_PATH}" ]]; then
     env_path="${TARGET_ENV_PATH}"
@@ -171,9 +208,15 @@ function main() {
     env_path=$(dirname $(dirname "${pip_path}"))
     if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
       echo "WARNING: Inferred system-wide environment path: ${env_path}."
-      echo "Caching the entire system directory is unsafe and may result in large cache files. It is strongly recommended to specify a virtual environment path using TARGET_ENV_PATH."
+      echo "Switching to isolated venv for caching to avoid archiving system directories."
+      ensure_venv
+      original_env_path="${env_path}"
+      env_path="${isolated_env_path}"
+      pip_bin="${env_path}/bin/pip"
+      isolated="true"
+    else
+      pip_bin="pip"
     fi
-    pip_bin="pip"
     echo "Inferred target environment path: ${env_path}"
   fi
 
@@ -206,7 +249,7 @@ function main() {
 
       if (( age <= CACHE_TIMEOUT )); then
         echo "Cache hit for key ${cache_key} (age: ${age}s). Unpacking from ${gcs_tarball}"
-        if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        if unpack_and_merge "${gcs_tarball}" "${env_path}" "${original_env_path}" "${isolated}"; then
           echo "Cache unpacked successfully."
           return 0
         else
@@ -217,7 +260,7 @@ function main() {
       fi
     else
       echo "Cache hit for key ${cache_key} (could not verify age). Unpacking."
-      if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+      if unpack_and_merge "${gcs_tarball}" "${env_path}" "${original_env_path}" "${isolated}"; then
         return 0
       else
         echo "WARNING: Failed to unpack cache. Falling back to rebuilding."
@@ -240,7 +283,7 @@ function main() {
       while gcs_exists "${gcs_tarball}.building"; do
         if gcs_exists "${gcs_tarball}"; then
           echo "Cache file appeared while waiting. Skipping build."
-          if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+          if unpack_and_merge "${gcs_tarball}" "${env_path}" "${original_env_path}" "${isolated}"; then
             return 0
           fi
           echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
@@ -258,7 +301,7 @@ function main() {
       # Double check if cache appeared just after sentinel removal
       if gcs_exists "${gcs_tarball}"; then
         echo "Cache file appeared. Skipping build."
-        if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
+        if unpack_and_merge "${gcs_tarball}" "${env_path}" "${original_env_path}" "${isolated}"; then
           return 0
         fi
         echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
@@ -285,6 +328,10 @@ function main() {
       local -a pack_arr
       read -r -a pack_arr <<< "${PACKAGES}"
       run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
+      if [[ "${isolated}" == "true" ]]; then
+        echo "Merging isolated environment back to ${original_env_path}"
+        cp -a "${env_path}/." "${original_env_path}/"
+      fi
       rm -f "${local_tarball}.building"
       return 0
     fi
@@ -300,6 +347,11 @@ function main() {
   read -r -a pack_arr <<< "${PACKAGES}"
   run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
 
+  if [[ "${isolated}" == "true" ]]; then
+    echo "Merging isolated environment back to ${original_env_path}"
+    cp -a "${env_path}/." "${original_env_path}/"
+  fi
+
   echo "Packaging environment."
   # We package the contents of the environment path
   pushd "${env_path}"

From dc082263ae96109999b7b851b1590461cc436a8f Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 22:32:24 +0000
Subject: [PATCH 05/11] Fix system corruption, stale sentinel hangs, and older
 image compatibility in cached installers

- **Fix System Python Corruption (pip):** Replaced dangerous `cp -a` of entire isolated venv to `/usr` with `merge_isolated_env`, which only copies `site-packages`. This avoids creating circular symlinks and corrupting system binaries.
- **Fix Stale Sentinel Hangs (pip/conda):** Added checks to detect and remove stale `.building` sentinel files (older than 20 minutes) in GCS. Capped wait timeouts to 20 minutes and ensured stale sentinels are removed on timeout.
- **Fix Older Image Compatibility (pip):** Re-introduced `remove_old_backports` to fix `apt update` failures on Debian <= 2.1 images.
- **Improve Persistence (pip):** Moved isolated venv path from `/tmp` to `/opt/pip-isolated-env` to prevent loss of python files on node reboot.
- **Clarify Warnings:** Improved phrasing of warnings related to failed unpacking of newly available caches.
---
 python/conda-install-cached.sh | 22 +++++++++--
 python/pip-install-cached.sh   | 69 +++++++++++++++++++++++++++-------
 2 files changed, 74 insertions(+), 17 deletions(-)

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index 7a15bf0b4..57aa14e89 100644
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -213,22 +213,36 @@ function main() {
     local sentinel_exists_code=$?
     set -e
 
+    if [[ ${sentinel_exists_code} -eq 0 ]]; then
+      local sentinel_epoch=$(gcs_file_epoch "${gcs_tarball}.building")
+      if [[ -n "${sentinel_epoch}" ]]; then
+        local now_epoch=$(date -u +%s)
+        local sentinel_age=$((now_epoch - sentinel_epoch))
+        if (( sentinel_age > 1200 )); then
+          echo "Found stale sentinel (age: ${sentinel_age}s > 1200s). Removing it."
+          gcs_rm "${gcs_tarball}.building" || true
+          sentinel_exists_code=1
+        fi
+      fi
+    fi
+
     if [[ ${sentinel_exists_code} -eq 0 ]]; then
       echo "Another node is building this environment. Waiting..."
       local wait_start=$(date +%s)
-      local timeout="${CACHE_TIMEOUT}"
+      local timeout=1200
       while gcs_exists "${gcs_tarball}.building"; do
         if gcs_exists "${gcs_tarball}"; then
           echo "Cache file appeared while waiting. Skipping build."
           if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
             return 0
           fi
-          echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+          echo "WARNING: Failed to unpack the newly available cache. Proceeding to build."
           break
         fi
         local now=$(date +%s)
         if (( now - wait_start > timeout )); then
-          echo "Timeout waiting for concurrent build. Proceeding to build myself."
+          echo "Timeout waiting for concurrent build. Removing stale sentinel and proceeding to build myself."
+          gcs_rm "${gcs_tarball}.building" || true
           break
         fi
         echo "Waiting 30 seconds..."
@@ -241,7 +255,7 @@ function main() {
         if gcs_cat "${gcs_tarball}" | tar -C "${env_path}" -xz; then
           return 0
         fi
-        echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+        echo "WARNING: Failed to unpack the newly available cache. Proceeding to build."
       fi
     fi
 
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index 910d8152d..a4732b675 100644
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -113,6 +113,24 @@ function err() {
   exit 1
 }
 
+function remove_old_backports() {
+  # This script uses 'apt-get update' and is therefore potentially dependent on
+  # backports repositories which have been archived.  In order to mitigate this
+  # problem, we will remove any reference to backports repos older than oldstable
+
+  # https://github.com/GoogleCloudDataproc/initialization-actions/issues/1157
+  local oldstable=$(curl -s https://deb.debian.org/debian/dists/oldstable/Release | awk '/^Codename/ {print $2}');
+  local stable=$(curl -s https://deb.debian.org/debian/dists/stable/Release | awk '/^Codename/ {print $2}');
+
+  local matched_files=( $(grep -rsil '\-backports' /etc/apt/sources.list*||:) )
+  if [[ -n "$matched_files" ]]; then
+    for filename in "${matched_files[@]}"; do
+      grep -e "$oldstable-backports" -e "$stable-backports" "$filename" || \
+        sed -i -e 's/^.*-backports.*$//' "$filename"
+    done
+  fi
+}
+
 function run_with_retry() {
   local -r cmd=("$@")
   for ((i = 0; i < 10; i++)); do
@@ -162,6 +180,19 @@ function ensure_venv() {
   fi
 }
 
+function merge_isolated_env() {
+  local venv_path=$1
+  echo "Merging isolated environment back to system Python"
+  local system_site_packages=$(/usr/bin/python3 -c "import site; print(site.getsitepackages()[0])" 2>/dev/null)
+  local venv_site_packages=$(find "${venv_path}/lib" -maxdepth 2 -type d -name "site-packages" | head -n 1)
+  if [[ -n "${system_site_packages}" && -n "${venv_site_packages}" ]]; then
+    cp -a "${venv_site_packages}/." "${system_site_packages}/"
+  else
+    echo "ERROR: Could not find system or venv site-packages directory"
+    return 1
+  fi
+}
+
 function unpack_and_merge() {
   local tarball=$1
   local target_path=$2
@@ -171,10 +202,7 @@ function unpack_and_merge() {
   mkdir -p "${target_path}"
   if gcs_cat "${tarball}" | tar -C "${target_path}" -xz; then
     if [[ "${is_isolated}" == "true" ]]; then
-      echo "Merging isolated environment back to ${original_path}"
-      # Ensure target directories exist in original path if needed?
-      # cp -a should handle it if original_path is directory.
-      cp -a "${target_path}/." "${original_path}/"
+      merge_isolated_env "${target_path}"
     fi
     return 0
   fi
@@ -195,7 +223,7 @@ function main() {
   local env_path
   local pip_bin
   local original_env_path=""
-  local isolated_env_path="/tmp/pip-isolated-env"
+  local isolated_env_path="/opt/pip-isolated-env"
   local isolated="false"
 
   if [[ -n "${TARGET_ENV_PATH}" ]]; then
@@ -203,6 +231,9 @@ function main() {
     pip_bin="${env_path}/bin/pip"
     echo "Using target environment path from metadata: ${env_path}"
   else
+    if [[ "${OS_NAME}" == "debian" ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} <= 2.1" | bc -l) == 1 ]]; then
+      remove_old_backports
+    fi
     install_pip
     local pip_path=$(which pip)
     env_path=$(dirname $(dirname "${pip_path}"))
@@ -276,22 +307,36 @@ function main() {
     local sentinel_exists_code=$?
     set -e
 
+    if [[ ${sentinel_exists_code} -eq 0 ]]; then
+      local sentinel_epoch=$(gcs_file_epoch "${gcs_tarball}.building")
+      if [[ -n "${sentinel_epoch}" ]]; then
+        local now_epoch=$(date -u +%s)
+        local sentinel_age=$((now_epoch - sentinel_epoch))
+        if (( sentinel_age > 1200 )); then
+          echo "Found stale sentinel (age: ${sentinel_age}s > 1200s). Removing it."
+          gcs_rm "${gcs_tarball}.building" || true
+          sentinel_exists_code=1
+        fi
+      fi
+    fi
+
     if [[ ${sentinel_exists_code} -eq 0 ]]; then
       echo "Another node is building this environment. Waiting..."
       local wait_start=$(date +%s)
-      local timeout="${CACHE_TIMEOUT}"
+      local timeout=1200
       while gcs_exists "${gcs_tarball}.building"; do
         if gcs_exists "${gcs_tarball}"; then
           echo "Cache file appeared while waiting. Skipping build."
           if unpack_and_merge "${gcs_tarball}" "${env_path}" "${original_env_path}" "${isolated}"; then
             return 0
           fi
-          echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+          echo "WARNING: Failed to unpack the newly available cache. Proceeding to build."
           break
         fi
         local now=$(date +%s)
         if (( now - wait_start > timeout )); then
-          echo "Timeout waiting for concurrent build. Proceeding to build myself."
+          echo "Timeout waiting for concurrent build. Removing stale sentinel and proceeding to build myself."
+          gcs_rm "${gcs_tarball}.building" || true
           break
         fi
         echo "Waiting 30 seconds..."
@@ -304,7 +349,7 @@ function main() {
         if unpack_and_merge "${gcs_tarball}" "${env_path}" "${original_env_path}" "${isolated}"; then
           return 0
         fi
-        echo "WARNING: Failed to unpack appeared cache. Proceeding to build."
+        echo "WARNING: Failed to unpack the newly available cache. Proceeding to build."
       fi
     fi
 
@@ -329,8 +374,7 @@ function main() {
       read -r -a pack_arr <<< "${PACKAGES}"
       run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
       if [[ "${isolated}" == "true" ]]; then
-        echo "Merging isolated environment back to ${original_env_path}"
-        cp -a "${env_path}/." "${original_env_path}/"
+        merge_isolated_env "${env_path}"
       fi
       rm -f "${local_tarball}.building"
       return 0
@@ -348,8 +392,7 @@ function main() {
   run_with_retry "${pip_bin}" install --upgrade "${pack_arr[@]}"
 
   if [[ "${isolated}" == "true" ]]; then
-    echo "Merging isolated environment back to ${original_env_path}"
-    cp -a "${env_path}/." "${original_env_path}/"
+    merge_isolated_env "${env_path}"
   fi
 
   echo "Packaging environment."

From 02c966ddaf87d0bb519cf7f6f4450d67a649b386 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 22:35:06 +0000
Subject: [PATCH 06/11] Prevent pathname expansion (globbing) when configuring
 Conda channels

- Use a Bash array (`chan_arr`) to safely iterate over space-separated `CONDA_CHANNELS`, avoiding potential globbing bugs if a channel name or specifier contains wildcards.
---
 python/conda-install-cached.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index 57aa14e89..53578c206 100644
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -134,7 +134,9 @@ function main() {
 
   # Configure channels first
   if [[ -n "${CHANNELS}" ]]; then
-    for channel in ${CHANNELS}; do
+    local -a chan_arr
+    read -r -a chan_arr <<< "${CHANNELS}"
+    for channel in "${chan_arr[@]}"; do
       conda config --add channels "${channel}"
     done
   fi

From 1fe7a3daf0dc361d85c7d011e077952f7c4d8839 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Mon, 5 Oct 2026 22:41:42 +0000
Subject: [PATCH 07/11] Address additional Gemini reviewer findings in
 pip-install-cached.sh

- **Add Curl Timeouts:** Added `--connect-timeout 5 --max-time 10` to `curl` commands in `remove_old_backports` to prevent hanging in air-gapped environments.
- **Merge Binaries:** Updated `merge_isolated_env` to copy executable binaries from isolated venv's `bin/` to `/usr/local/bin/`, making command-line tools available.
- **Robust Version Check:** Verified `DATAPROC_IMAGE_VERSION` is non-empty before running `bc` comparison to avoid script crashes.
---
 python/pip-install-cached.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index a4732b675..70a3b925d 100644
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -119,8 +119,8 @@ function remove_old_backports() {
   # problem, we will remove any reference to backports repos older than oldstable
 
   # https://github.com/GoogleCloudDataproc/initialization-actions/issues/1157
-  local oldstable=$(curl -s https://deb.debian.org/debian/dists/oldstable/Release | awk '/^Codename/ {print $2}');
-  local stable=$(curl -s https://deb.debian.org/debian/dists/stable/Release | awk '/^Codename/ {print $2}');
+  local oldstable=$(curl -s --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/dists/oldstable/Release | awk '/^Codename/ {print $2}');
+  local stable=$(curl -s --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/dists/stable/Release | awk '/^Codename/ {print $2}');
 
   local matched_files=( $(grep -rsil '\-backports' /etc/apt/sources.list*||:) )
   if [[ -n "$matched_files" ]]; then
@@ -184,9 +184,12 @@ function merge_isolated_env() {
   local venv_path=$1
   echo "Merging isolated environment back to system Python"
   local system_site_packages=$(/usr/bin/python3 -c "import site; print(site.getsitepackages()[0])" 2>/dev/null)
-  local venv_site_packages=$(find "${venv_path}/lib" -maxdepth 2 -type d -name "site-packages" | head -n 1)
+  local venv_site_packages=$(find "${venv_path}/lib" -maxdepth 2 -type d -name "site-packages" 2>/dev/null | head -n 1)
   if [[ -n "${system_site_packages}" && -n "${venv_site_packages}" ]]; then
     cp -a "${venv_site_packages}/." "${system_site_packages}/"
+    if [[ -d "${venv_path}/bin" ]]; then
+      find "${venv_path}/bin" -type f ! -name "python*" ! -name "pip*" ! -name "easy_install*" ! -name "activate*" -exec cp -a {} /usr/local/bin/ \;
+    fi
   else
     echo "ERROR: Could not find system or venv site-packages directory"
     return 1
@@ -231,7 +234,7 @@ function main() {
     pip_bin="${env_path}/bin/pip"
     echo "Using target environment path from metadata: ${env_path}"
   else
-    if [[ "${OS_NAME}" == "debian" ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} <= 2.1" | bc -l) == 1 ]]; then
+    if [[ "${OS_NAME}" == "debian" ]] && [[ -n "${DATAPROC_IMAGE_VERSION}" ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} <= 2.1" | bc -l) == 1 ]]; then
       remove_old_backports
     fi
     install_pip

From 792096328a94b1cf5b017323018517c29de012c9 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Tue, 6 Oct 2026 01:54:30 +0000
Subject: [PATCH 08/11] Address latest Gemini reviewer findings in cached
 installers

- **Fix Isolated Venv Initialization (pip):** Explicitly create the isolated virtual environment using `python3 -m venv` or `virtualenv` after ensuring its capabilities are installed.
- **Improve Error Handling (pip):** Propagate failures from `merge_isolated_env` in `unpack_and_merge` instead of silently ignoring them.
- **Robust Shebangs (pip):** Rewrite shebangs of executable scripts copied from isolated venv to `/usr/local/bin/` to use system python (`#!/usr/bin/env python3`), making them independent of the venv path.
- **Robust OS Detection (pip/conda):** Added fallback to `/etc/os-release` when `lsb_release` is missing on minimal images.
---
 python/conda-install-cached.sh |  2 +-
 python/pip-install-cached.sh   | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 python/conda-install-cached.sh
 mode change 100644 => 100755 python/pip-install-cached.sh

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
old mode 100644
new mode 100755
index 53578c206..933479bcd
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -33,7 +33,7 @@ if [[ -z "${CACHE_TIMEOUT}" ]]; then
 fi
 readonly CACHE_TIMEOUT
 readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
-readonly OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+readonly OS_NAME=$(if command -v lsb_release >/dev/null 2>&1; then lsb_release -is | tr '[:upper:]' '[:lower:]'; else . /etc/os-release; echo "${ID}" | tr '[:upper:]' '[:lower:]'; fi)
 
 GCS_CMD="gsutil"
 if gcloud --help >/dev/null 2>&1 && gcloud storage --help >/dev/null 2>&1; then
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
old mode 100644
new mode 100755
index 70a3b925d..c0ffe7767
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -32,7 +32,7 @@ if [[ -z "${CACHE_TIMEOUT}" ]]; then
 fi
 readonly CACHE_TIMEOUT
 readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
-readonly OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]')
+readonly OS_NAME=$(if command -v lsb_release >/dev/null 2>&1; then lsb_release -is | tr '[:upper:]' '[:lower:]'; else . /etc/os-release; echo "${ID}" | tr '[:upper:]' '[:lower:]'; fi)
 
 GCS_CMD="gsutil"
 if gcloud --help >/dev/null 2>&1 && gcloud storage --help >/dev/null 2>&1; then
@@ -189,6 +189,12 @@ function merge_isolated_env() {
     cp -a "${venv_site_packages}/." "${system_site_packages}/"
     if [[ -d "${venv_path}/bin" ]]; then
       find "${venv_path}/bin" -type f ! -name "python*" ! -name "pip*" ! -name "easy_install*" ! -name "activate*" -exec cp -a {} /usr/local/bin/ \;
+      # Rewrite shebangs of copied scripts to use system python
+      for f in $(find "${venv_path}/bin" -type f ! -name "python*" ! -name "pip*" ! -name "easy_install*" ! -name "activate*" -exec basename {} \;); do
+        if head -n 1 "/usr/local/bin/${f}" | grep -q "python"; then
+          sed -i "1s|^#!.*python.*|#!/usr/bin/env python3|" "/usr/local/bin/${f}"
+        fi
+      done
     fi
   else
     echo "ERROR: Could not find system or venv site-packages directory"
@@ -205,7 +211,7 @@ function unpack_and_merge() {
   mkdir -p "${target_path}"
   if gcs_cat "${tarball}" | tar -C "${target_path}" -xz; then
     if [[ "${is_isolated}" == "true" ]]; then
-      merge_isolated_env "${target_path}"
+      merge_isolated_env "${target_path}" || return 1
     fi
     return 0
   fi
@@ -244,6 +250,7 @@ function main() {
       echo "WARNING: Inferred system-wide environment path: ${env_path}."
       echo "Switching to isolated venv for caching to avoid archiving system directories."
       ensure_venv
+      python3 -m venv "${isolated_env_path}" || virtualenv "${isolated_env_path}"
       original_env_path="${env_path}"
       env_path="${isolated_env_path}"
       pip_bin="${env_path}/bin/pip"

From 2b27757850d7b97ac6bb30d0b7ec1a060ebeb40a Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Tue, 6 Oct 2026 02:03:37 +0000
Subject: [PATCH 09/11] feat(python): add integration tests for cached
 installers and improve OS detection

## Summary
Introduces integration tests for `pip-install-cached.sh` and `conda-install-cached.sh` to verify GCS-based caching behavior. Also improves the robustness of OS detection in both scripts by directly parsing `/etc/os-release`.

## Details
- **Tests**: Added `test_pip_cached.py` and `test_conda_cached.py` in the `python/` directory. These tests verify that packages are installed correctly and that cache tarballs are generated in GCS.
- **Build System**: Created `python/BUILD` to define test targets and a `test_python` suite. Integrated this suite into the root `DataprocInitActionsTestSuite`.
- **Robustness**: Replaced `lsb_release` dependency in installer scripts with direct file parsing of `/etc/os-release` via `grep` and `cut` for better compatibility across minimal images.

## Changes
- `BUILD`: Added `//python:test_python` to `DataprocInitActionsTestSuite`.
- `python/BUILD`: New file defining test targets.
- `python/conda-install-cached.sh`: Updated `OS_NAME` detection.
- `python/pip-install-cached.sh`: Updated `OS_NAME` detection.
- `python/test_conda_cached.py`: New integration test for cached conda installs.
- `python/test_pip_cached.py`: New integration test for cached pip installs.
---
 BUILD                          |  1 +
 python/BUILD                   | 38 +++++++++++++++++++
 python/conda-install-cached.sh |  2 +-
 python/pip-install-cached.sh   |  2 +-
 python/test_conda_cached.py    | 64 +++++++++++++++++++++++++++++++
 python/test_pip_cached.py      | 69 ++++++++++++++++++++++++++++++++++
 6 files changed, 174 insertions(+), 2 deletions(-)
 create mode 100644 python/BUILD
 create mode 100644 python/test_conda_cached.py
 create mode 100644 python/test_pip_cached.py

diff --git a/BUILD b/BUILD
index 3dd5093ab..f5f8dab8b 100644
--- a/BUILD
+++ b/BUILD
@@ -28,6 +28,7 @@ test_suite(
         "//mlvm:test_mlvm",
         "//oozie:test_oozie",
         "//presto:test_presto",
+        "//python:test_python",
         "//ranger:test_ranger",
         "//rapids:test_rapids",
         "//rstudio:test_rstudio",
diff --git a/python/BUILD b/python/BUILD
new file mode 100644
index 000000000..aff430881
--- /dev/null
+++ b/python/BUILD
@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+
+py_test(
+    name = "test_pip_cached",
+    size = "enormous",
+    srcs = ["test_pip_cached.py"],
+    data = [
+        "pip-install-cached.sh",
+    ],
+    local = True,
+    deps = [
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+py_test(
+    name = "test_conda_cached",
+    size = "enormous",
+    srcs = ["test_conda_cached.py"],
+    data = [
+        "conda-install-cached.sh",
+    ],
+    local = True,
+    deps = [
+        "//integration_tests:dataproc_test_case",
+        "@io_abseil_py//absl/testing:parameterized",
+    ],
+)
+
+test_suite(
+    name = "test_python",
+    tests = [
+        ":test_pip_cached",
+        ":test_conda_cached",
+    ],
+)
diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index 933479bcd..8fc8ddc18 100755
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -33,7 +33,7 @@ if [[ -z "${CACHE_TIMEOUT}" ]]; then
 fi
 readonly CACHE_TIMEOUT
 readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
-readonly OS_NAME=$(if command -v lsb_release >/dev/null 2>&1; then lsb_release -is | tr '[:upper:]' '[:lower:]'; else . /etc/os-release; echo "${ID}" | tr '[:upper:]' '[:lower:]'; fi)
+readonly OS_NAME=$(grep '^ID=' /etc/os-release | cut -d= -f2 | xargs | tr '[:upper:]' '[:lower:]')
 
 GCS_CMD="gsutil"
 if gcloud --help >/dev/null 2>&1 && gcloud storage --help >/dev/null 2>&1; then
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index c0ffe7767..3e769fc7e 100755
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -32,7 +32,7 @@ if [[ -z "${CACHE_TIMEOUT}" ]]; then
 fi
 readonly CACHE_TIMEOUT
 readonly TARGET_ENV_PATH=$(/usr/share/google/get_metadata_value attributes/TARGET_ENV_PATH || true)
-readonly OS_NAME=$(if command -v lsb_release >/dev/null 2>&1; then lsb_release -is | tr '[:upper:]' '[:lower:]'; else . /etc/os-release; echo "${ID}" | tr '[:upper:]' '[:lower:]'; fi)
+readonly OS_NAME=$(grep '^ID=' /etc/os-release | cut -d= -f2 | xargs | tr '[:upper:]' '[:lower:]')
 
 GCS_CMD="gsutil"
 if gcloud --help >/dev/null 2>&1 && gcloud storage --help >/dev/null 2>&1; then
diff --git a/python/test_conda_cached.py b/python/test_conda_cached.py
new file mode 100644
index 000000000..900bb312e
--- /dev/null
+++ b/python/test_conda_cached.py
@@ -0,0 +1,64 @@
+import json
+import packaging.version
+from absl.testing import absltest
+from absl.testing import parameterized
+
+from integration_tests.dataproc_test_case import DataprocTestCase
+
+CONDA_BINARY = "/opt/conda/bin/conda"
+
+class CondaCachedTestCase(DataprocTestCase):
+    COMPONENT = "python"
+    INIT_ACTIONS = ["python/conda-install-cached.sh"]
+
+    # Test packages
+    CONDA_PKGS = ["numpy"]
+    GCS_BUCKET = None
+
+    def setUp(self):
+        super().setUp()
+        self.GCS_BUCKET = "test-conda-cached-{}-{}".format(self.datetime_str(),
+                                                           self.random_str())
+        self.assert_command('gsutil mb -c regional -l {} gs://{}'.format(
+            self.REGION, self.GCS_BUCKET))
+
+    def tearDown(self):
+        self.assert_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET))
+        super().tearDown()
+
+    def _verify_conda_packages(self, instance, conda_packages):
+        _, stdout, _ = self.assert_instance_command(instance, CONDA_BINARY + " list")
+        installed_packages = self._parse_packages(stdout)
+        for package in conda_packages:
+            self.assertIn(
+                package, installed_packages,
+                "Expected package {} to be installed, but wasn't."
+                " Packages installed: {}".format(package, installed_packages))
+
+    def _verify_cache_created(self):
+        ret_code, stdout, _ = self.run_command("gsutil ls gs://{}".format(self.GCS_BUCKET))
+        self.assertEqual(ret_code, 0, "Failed to list bucket")
+        self.assertTrue(len(stdout.strip()) > 0, "Cache was not created in GCS")
+
+    @staticmethod
+    def _parse_packages(stdout):
+        return set(
+            l.split()[0] for l in stdout.splitlines() if not l.startswith("#"))
+
+    @parameterized.parameters(
+        ("STANDARD", CONDA_PKGS),
+    )
+    def test_conda_cached(self, configuration, conda_packages):
+        metadata = "'CONDA_PACKAGES={},CACHE_BUCKET={}'".format(
+            " ".join(conda_packages), self.GCS_BUCKET)
+        self.createCluster(
+            configuration,
+            self.INIT_ACTIONS,
+            metadata=metadata)
+
+        instance_name = self.getClusterName() + "-m"
+        self._verify_conda_packages(instance_name, conda_packages)
+        self._verify_cache_created()
+
+if __name__ == "__main__":
+    absltest.main()
diff --git a/python/test_pip_cached.py b/python/test_pip_cached.py
new file mode 100644
index 000000000..1c940ea07
--- /dev/null
+++ b/python/test_pip_cached.py
@@ -0,0 +1,69 @@
+import json
+import packaging.version
+from absl.testing import absltest
+from absl.testing import parameterized
+
+from integration_tests.dataproc_test_case import DataprocTestCase
+
+class PipCachedTestCase(DataprocTestCase):
+    COMPONENT = "python"
+    INIT_ACTIONS = ["python/pip-install-cached.sh"]
+
+    # Test packages
+    PIP_PKGS = ["pandas-gbq"]
+    GCS_BUCKET = None
+
+    def setUp(self):
+        super().setUp()
+        self.GCS_BUCKET = "test-pip-cached-{}-{}".format(self.datetime_str(),
+                                                         self.random_str())
+        self.assert_command('gsutil mb -c regional -l {} gs://{}'.format(
+            self.REGION, self.GCS_BUCKET))
+
+    def tearDown(self):
+        self.assert_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET))
+        super().tearDown()
+
+    def _verify_pip_packages(self, instance, pip_packages):
+        # Try both pip and python3 -m pip to be robust
+        _, stdout, _ = self.assert_instance_command(instance, "python3 -m pip list")
+        installed_packages = self._parse_packages(stdout)
+        for package in pip_packages:
+            if package not in installed_packages:
+                # Try pip list as fallback
+                _, stdout_pip, _ = self.assert_instance_command(instance, "pip list")
+                installed_packages_pip = self._parse_packages(stdout_pip)
+                self.assertIn(
+                    package, installed_packages_pip,
+                    "Expected package {} to be installed, but wasn't."
+                    " Packages installed (python3 -m pip): {}"
+                    " Packages installed (pip): {}".format(
+                        package, installed_packages, installed_packages_pip))
+
+    def _verify_cache_created(self):
+        ret_code, stdout, _ = self.run_command("gsutil ls gs://{}".format(self.GCS_BUCKET))
+        self.assertEqual(ret_code, 0, "Failed to list bucket")
+        self.assertTrue(len(stdout.strip()) > 0, "Cache was not created in GCS")
+
+    @staticmethod
+    def _parse_packages(stdout):
+        return set(
+            l.split()[0] for l in stdout.splitlines() if not l.startswith("#"))
+
+    @parameterized.parameters(
+        ("STANDARD", PIP_PKGS),
+    )
+    def test_pip_cached(self, configuration, pip_packages):
+        metadata = "'PIP_PACKAGES={},CACHE_BUCKET={}'".format(
+            " ".join(pip_packages), self.GCS_BUCKET)
+        self.createCluster(
+            configuration,
+            self.INIT_ACTIONS,
+            metadata=metadata)
+
+        instance_name = self.getClusterName() + "-m"
+        self._verify_pip_packages(instance_name, pip_packages)
+        self._verify_cache_created()
+
+if __name__ == "__main__":
+    absltest.main()

From 60ce9ff4fa3f034432f597c2f3ba1e1b3602adca Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Tue, 6 Oct 2026 02:13:27 +0000
Subject: [PATCH 10/11] fix(python): improve robustness of cached installers in
 restricted environments

## Summary
Addressed critical feedback to improve the reliability of `conda-install-cached.sh` and `pip-install-cached.sh`, particularly in private clusters or when custom target environments are used.

## Details
- **Conda Installer**:
  - Ensured target environment path is created (`mkdir -p`) before unpacking, preventing failure when `TARGET_ENV_PATH` points to a non-existent directory.
  - Moved channel configuration to after `conda` binary detection to avoid crashes if `conda` is not yet available in `PATH`.
  - Added graceful handling and warnings if `conda` is missing but `TARGET_ENV_PATH` is specified.
- **Pip Installer**:
  - Added fallbacks and error silencing (`2>/dev/null`) to `curl` commands in `remove_old_backports`. This prevents script failure in air-gapped or private clusters where `deb.debian.org` is unreachable.

## Changes
- `python/conda-install-cached.sh`: Fixed channel config order, added directory creation, improved `conda` binary detection.
- `python/pip-install-cached.sh`: Added fallbacks for `curl` calls.
---
 python/conda-install-cached.sh | 30 ++++++++++++++++++++----------
 python/pip-install-cached.sh   |  7 +++++--
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index 8fc8ddc18..c99d8417e 100755
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -132,22 +132,17 @@ function main() {
     exit 1
   fi
 
-  # Configure channels first
-  if [[ -n "${CHANNELS}" ]]; then
-    local -a chan_arr
-    read -r -a chan_arr <<< "${CHANNELS}"
-    for channel in "${chan_arr[@]}"; do
-      conda config --add channels "${channel}"
-    done
-  fi
-
-  local conda_path=$(which conda)
+  local conda_path
+  conda_path=$(which conda 2>/dev/null) || true
   local env_path
 
   if [[ -n "${TARGET_ENV_PATH}" ]]; then
     env_path="${TARGET_ENV_PATH}"
     echo "Using target environment path from metadata: ${env_path}"
   else
+    if [[ -z "${conda_path}" ]]; then
+      err "conda command not found and TARGET_ENV_PATH not specified."
+    fi
     env_path=$(dirname $(dirname "${conda_path}"))
     if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
       echo "WARNING: Inferred system-wide environment path: ${env_path}."
@@ -157,6 +152,21 @@ function main() {
   fi
 
   echo "Target environment path: ${env_path}"
+  mkdir -p "${env_path}"
+
+  # Configure channels
+  if [[ -n "${CHANNELS}" ]]; then
+    if [[ -z "${conda_path}" ]]; then
+       echo "WARNING: Cannot configure channels, conda binary not found in PATH"
+    else
+      local -a chan_arr
+      read -r -a chan_arr <<< "${CHANNELS}"
+      for channel in "${chan_arr[@]}"; do
+        conda config --add channels "${channel}"
+      done
+    fi
+  fi
+
 
   local cache_key
   if [[ -n "${CACHE_KEY_OVERRIDE}" ]]; then
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index 3e769fc7e..bf1b63236 100755
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -119,8 +119,11 @@ function remove_old_backports() {
   # problem, we will remove any reference to backports repos older than oldstable
 
   # https://github.com/GoogleCloudDataproc/initialization-actions/issues/1157
-  local oldstable=$(curl -s --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/dists/oldstable/Release | awk '/^Codename/ {print $2}');
-  local stable=$(curl -s --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/dists/stable/Release | awk '/^Codename/ {print $2}');
+  local oldstable
+  oldstable=$(curl -s --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/dists/oldstable/Release 2>/dev/null | awk '/^Codename/ {print $2}' || echo "oldstable")
+  local stable
+  stable=$(curl -s --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/dists/stable/Release 2>/dev/null | awk '/^Codename/ {print $2}' || echo "stable")
+
 
   local matched_files=( $(grep -rsil '\-backports' /etc/apt/sources.list*||:) )
   if [[ -n "$matched_files" ]]; then

From bf9d440ca580e54154693ae3e0cb9ff3090115a4 Mon Sep 17 00:00:00 2001
From: "C.J. Collier" <cjac@google.com>
Date: Tue, 6 Oct 2026 02:23:20 +0000
Subject: [PATCH 11/11] fix(python): optimize venv creation, sanitize version
 strings, and improve test cleanup

## Summary
Addressed further review feedback to optimize performance, prevent script crashes, and improve test stability.

## Details
- **Pip Installer Optimization**: Added `--system-site-packages` to isolated `venv` creation. This avoids redundant downloads of packages already present in the base image, speeding up installations and reducing cache size.
- **Version Sanitization**: Added logic to sanitize `DATAPROC_IMAGE_VERSION` to `major.minor` format before passing to `bc`, preventing syntax errors on versions with non-numeric suffixes.
- **Safety Warnings**: Added `/opt/conda` to the list of unsafe directories to cache in `conda-install-cached.sh`, warning users against overwriting the base environment.
- **Test Stability**: Refactored `tearDown` in integration tests to use `run_command` instead of `assert_command` and added existence checks for buckets. This prevents cleanup failures from masking actual test errors.

## Changes
- `python/conda-install-cached.sh`: Added `/opt/conda` to unsafe directories list.
- `python/pip-install-cached.sh`: Added `--system-site-packages` to venv creation, sanitized version for `bc`.
- `python/test_conda_cached.py`: Robust `tearDown` implementation.
- `python/test_pip_cached.py`: Robust `tearDown` implementation.
---
 python/conda-install-cached.sh |  2 +-
 python/pip-install-cached.sh   | 10 +++++++---
 python/test_conda_cached.py    |  3 ++-
 python/test_pip_cached.py      |  3 ++-
 4 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/python/conda-install-cached.sh b/python/conda-install-cached.sh
index c99d8417e..a16a4b80a 100755
--- a/python/conda-install-cached.sh
+++ b/python/conda-install-cached.sh
@@ -144,7 +144,7 @@ function main() {
       err "conda command not found and TARGET_ENV_PATH not specified."
     fi
     env_path=$(dirname $(dirname "${conda_path}"))
-    if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" ]]; then
+    if [[ "${env_path}" == "/usr" || "${env_path}" == "/usr/local" || "${env_path}" == "/" || "${env_path}" == "/opt/conda" ]]; then
       echo "WARNING: Inferred system-wide environment path: ${env_path}."
       echo "Caching the entire system directory is unsafe and may result in large cache files. It is strongly recommended to specify a target environment path using TARGET_ENV_PATH."
     fi
diff --git a/python/pip-install-cached.sh b/python/pip-install-cached.sh
index bf1b63236..c9af9c057 100755
--- a/python/pip-install-cached.sh
+++ b/python/pip-install-cached.sh
@@ -243,8 +243,11 @@ function main() {
     pip_bin="${env_path}/bin/pip"
     echo "Using target environment path from metadata: ${env_path}"
   else
-    if [[ "${OS_NAME}" == "debian" ]] && [[ -n "${DATAPROC_IMAGE_VERSION}" ]] && [[ $(echo "${DATAPROC_IMAGE_VERSION} <= 2.1" | bc -l) == 1 ]]; then
-      remove_old_backports
+    if [[ "${OS_NAME}" == "debian" ]] && [[ -n "${DATAPROC_IMAGE_VERSION}" ]]; then
+      local dp_ver=$(echo "${DATAPROC_IMAGE_VERSION}" | sed -E 's/^([0-9]+\.[0-9]+).*$/\1/')
+      if [[ $(echo "${dp_ver} <= 2.1" | bc -l 2>/dev/null) == 1 ]]; then
+        remove_old_backports
+      fi
     fi
     install_pip
     local pip_path=$(which pip)
@@ -253,7 +256,8 @@ function main() {
       echo "WARNING: Inferred system-wide environment path: ${env_path}."
       echo "Switching to isolated venv for caching to avoid archiving system directories."
       ensure_venv
-      python3 -m venv "${isolated_env_path}" || virtualenv "${isolated_env_path}"
+      python3 -m venv --system-site-packages "${isolated_env_path}" || virtualenv --system-site-packages "${isolated_env_path}"
+
       original_env_path="${env_path}"
       env_path="${isolated_env_path}"
       pip_bin="${env_path}/bin/pip"
diff --git a/python/test_conda_cached.py b/python/test_conda_cached.py
index 900bb312e..039be51b0 100644
--- a/python/test_conda_cached.py
+++ b/python/test_conda_cached.py
@@ -23,7 +23,8 @@ def setUp(self):
             self.REGION, self.GCS_BUCKET))
 
     def tearDown(self):
-        self.assert_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET))
+        if self.GCS_BUCKET:
+            self.run_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET))
         super().tearDown()
 
     def _verify_conda_packages(self, instance, conda_packages):
diff --git a/python/test_pip_cached.py b/python/test_pip_cached.py
index 1c940ea07..5ec5ba97a 100644
--- a/python/test_pip_cached.py
+++ b/python/test_pip_cached.py
@@ -21,7 +21,8 @@ def setUp(self):
             self.REGION, self.GCS_BUCKET))
 
     def tearDown(self):
-        self.assert_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET))
+        if self.GCS_BUCKET:
+            self.run_command('gsutil -m rm -rf gs://{}'.format(self.GCS_BUCKET))
         super().tearDown()
 
     def _verify_pip_packages(self, instance, pip_packages):