Upgrade MongoDB in place when an app package depends on it

.github/workflows/verify-split-files.yml

@@ -25,6 +25,7 @@ on:
       - "tests/vm-iso-storage.sh"
       - "tests/vm-consoles.sh"
       - "tests/vm-helpers.sh"
+      - "tests/mongodb-upgrade.sh"
       - "tests/uv-required-version.sh"
       - ".github/workflows/verify-split-files.yml"
   push:
@@ -42,6 +43,7 @@ on:
       - "tests/vm-iso-storage.sh"
       - "tests/vm-consoles.sh"
       - "tests/vm-helpers.sh"
+      - "tests/mongodb-upgrade.sh"
       - "tests/uv-required-version.sh"
       - ".github/workflows/verify-split-files.yml"
   workflow_dispatch:
@@ -73,6 +75,9 @@ jobs:
       - name: VM helper regression tests
         run: bash tests/vm-helpers.sh
 
+      - name: MongoDB upgrade regression tests
+        run: bash tests/mongodb-upgrade.sh
+
       - name: uv pin regression tests
         run: bash tests/uv-required-version.sh
 

lib/API.txt

@@ -30,6 +30,8 @@ _gh_scan_older_releases
 _github_release_noapi
 _gl_scan_older_releases
 _install_uvx_wrapper
+_mongodb_in_place_precheck
+_mongodb_in_place_simulate
 _purge_is_safe
 _resolve_igc_tag
 _setup_amd_apu
@@ -113,6 +115,8 @@ is_service_running
 is_tool_installed
 is_ubuntu
 manage_tool_repository
+mongodb_get_fcv
+mongodb_set_fcv
 nginx_enable_site
 prepare_repository_setup
 prompt_for_github_token

lib/db.func

@@ -754,6 +754,70 @@ EOF
 #   MONGO_VERSION  - MongoDB version to install (e.g. 7.0, 8.2)
 # ------------------------------------------------------------------------------
 
+mongodb_get_fcv() {
+  local port="${1:-27017}"
+  command -v mongosh >/dev/null 2>&1 || return 1
+  mongosh --quiet --port "$port" \
+    --eval 'db.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}).featureCompatibilityVersion.version' 2>/dev/null |
+    tr -d '[:space:]"' | grep -E '^[0-9]+\.[0-9]+$'
+}
+
+# mongodb_set_fcv <version> [port]
+# Data left at an older featureCompatibilityVersion keeps the next major's
+# binary from starting. Apps with their own mongod (Omada: 27217) call this
+# for that instance once it runs on the new binary.
+mongodb_set_fcv() {
+  local version="$1" port="${2:-27017}" cmd
+  command -v mongosh >/dev/null 2>&1 || {
+    msg_warn "mongosh not found; set featureCompatibilityVersion ${version} by hand"
+    return 1
+  }
+  cmd="db.adminCommand({setFeatureCompatibilityVersion: '${version}'"
+  ((${version%%.*} >= 7)) && cmd+=", confirm: true"
+  cmd+="})"
+  $STD mongosh --quiet --port "$port" --eval "$cmd" || {
+    msg_warn "Could not set featureCompatibilityVersion ${version} on port ${port}"
+    return 1
+  }
+  return 0
+}
+
+_mongodb_in_place_precheck() {
+  local from="$1" to="$2" from_major="${1%%.*}" to_major="${2%%.*}" fcv
+  if ((to_major < from_major)); then
+    msg_error "MongoDB ${from} cannot be downgraded to ${to} in place"
+    return 1
+  fi
+  if ((to_major - from_major > 1)); then
+    msg_error "MongoDB upgrades one major release at a time: ${from} -> ${to} needs $((from_major + 1)).0 first (MONGO_VERSION=$((from_major + 1)).0)"
+    return 1
+  fi
+  fcv="$(mongodb_get_fcv 2>/dev/null)" || fcv=""
+  if [[ -n "$fcv" && "$fcv" != "$from" ]]; then
+    msg_info "Raising featureCompatibilityVersion from ${fcv} to ${from}"
+    mongodb_set_fcv "$from" || return 1
+    msg_ok "Raised featureCompatibilityVersion to ${from}"
+  fi
+  return 0
+}
+
+# apt must be able to replace the mongodb-org packages without removing
+# anything else; a dependency ceiling below the target shows up here.
+_mongodb_in_place_simulate() {
+  local sim collateral
+  sim=$(apt-get -s install -y mongodb-org 2>&1) || {
+    msg_error "apt cannot upgrade mongodb-org in place:"
+    printf '%s\n' "$sim" | grep -E '^(E:|W:| )' | head -n 5
+    return 1
+  }
+  collateral=$(awk '/^Remv /{print $2}' <<<"$sim" | grep -v '^mongodb' || true)
+  if [[ -n "$collateral" ]]; then
+    msg_error "Upgrading mongodb-org in place would remove: ${collateral//$'\n'/ }"
+    return 1
+  fi
+  return 0
+}
+
 setup_mongodb() {
   local MONGO_VERSION="${MONGO_VERSION:-8.0}"
   local DISTRO_ID DISTRO_CODENAME
@@ -818,12 +882,18 @@ setup_mongodb() {
   fi
 
   # Scenario 2: Different version installed - clean upgrade
+  local in_place=0
   if [[ -n "$INSTALLED_VERSION" && "$INSTALLED_VERSION" != "$MONGO_VERSION" ]]; then
     msg_info "Upgrade MongoDB from $INSTALLED_VERSION to $MONGO_VERSION"
-    remove_old_tool_version "mongodb" || {
-      msg_error "Aborting MongoDB upgrade: another package depends on it (e.g. an app using it)"
-      return 1
-    }
+    if _purge_is_safe 'mongodb*'; then
+      remove_old_tool_version "mongodb" || return 1
+    else
+      # An app package depends on mongodb-org-server (Omada), so the purge
+      # would take it along. MongoDB swaps binaries one major release at a
+      # time, which is what apt does once the new repository is in place.
+      _mongodb_in_place_precheck "$INSTALLED_VERSION" "$MONGO_VERSION" || return 1
+      in_place=1
+    fi
   else
     msg_info "Setup MongoDB $MONGO_VERSION"
   fi
@@ -852,6 +922,15 @@ setup_mongodb() {
     return 100
   }
 
+  if ((in_place)) && ! _mongodb_in_place_simulate; then
+    local old_key="$INSTALLED_VERSION"
+    [[ "$old_key" == 8.[1-9]* ]] && old_key="8.0"
+    manage_tool_repository "mongodb" "$INSTALLED_VERSION" "$MONGO_BASE_URL" \
+      "https://www.mongodb.org/static/pgp/server-${old_key}.asc" >/dev/null 2>&1 || true
+    $STD apt update || true
+    return 1
+  fi
+
   # Install MongoDB with retry logic
   install_packages_with_retry "mongodb-org" || {
     msg_error "Failed to install MongoDB packages"
@@ -883,6 +962,11 @@ EOF
   }
   safe_service_restart mongod
 
+  if ((in_place)); then
+    mongodb_set_fcv "$MONGO_VERSION" || true
+    msg_custom "💡" "${YW:-}" "Apps that run their own mongod (Omada: port 27217) need mongodb_set_fcv ${MONGO_VERSION} <port> once they are back up"
+  fi
+
   # Verify MongoDB version
   local INSTALLED_VERSION
   INSTALLED_VERSION=$(mongod --version 2>/dev/null | grep -oP 'db version v\K[0-9]+\.[0-9]+' | head -n1 || echo "0.0")

tests/mongodb-upgrade.sh

@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
+# The MongoDB helpers and the purge guard; the rest of the engine is stubbed.
+eval "$(awk '/^(setup_mongodb|_mongodb_in_place_precheck|_mongodb_in_place_simulate|mongodb_get_fcv|mongodb_set_fcv)\(\) \{/,/^}/' "$CORE_DIR/lib/db.func")"
+eval "$(awk '/^_purge_is_safe\(\) \{/,/^}/' "$CORE_DIR/lib/system.func")"
+
+TEST_DIR="$(mktemp -d)"
+trap 'rm -rf -- "$TEST_DIR"' EXIT
+LOG="$TEST_DIR/calls"
+# shellcheck disable=SC2034  # read by the eval'd helpers
+STD="" YW=""
+passed=0 failed=0 case_failed=0 case_name=""
+
+log() { printf '%s\n' "$*" >>"$LOG"; }
+msg_info() { log "info: $*"; }
+msg_ok() { log "ok: $*"; }
+msg_warn() { log "warn: $*"; }
+msg_error() { log "error: $*"; }
+msg_custom() { log "custom: $3"; }
+get_os_info() { case "$1" in id) echo debian ;; codename) echo bookworm ;; esac; }
+is_tool_installed() { [[ -n "${MOCK_INSTALLED:-}" ]] && echo "$MOCK_INSTALLED"; }
+ensure_apt_working() { :; }
+upgrade_packages_with_retry() { log "upgrade $*"; }
+install_packages_with_retry() { log "install $*"; }
+remove_old_tool_version() { log "purge $*"; }
+cleanup_orphaned_sources() { :; }
+prepare_repository_setup() { :; }
+manage_tool_repository() { log "repo $2"; }
+cache_installed_version() { log "cache $2"; }
+verify_tool_version() { :; }
+safe_service_restart() { log "restart $1"; }
+systemctl() { :; }
+mkdir() { :; }
+chown() { :; }
+apt() { :; }
+uname() { case "$1" in -m) echo x86_64 ;; -r) echo 6.1.0 ;; esac; }
+mongod() { echo "db version v${MOCK_MONGOD:-7.0.22}"; }
+mongosh() {
+  log "mongosh $*"
+  case "$*" in
+  *getParameter*) [[ -n "${MOCK_FCV:-}" ]] && echo "$MOCK_FCV" || return 1 ;;
+  *) return 0 ;;
+  esac
+}
+apt-get() {
+  case "$*" in
+  "-s purge"*) printf '%s\n' "${MOCK_PURGE:-}" ;;
+  "-s install"*)
+    printf '%s\n' "${MOCK_SIM:-Inst mongodb-org [7.0.22] (8.0.3)}"
+    return "${MOCK_SIM_RC:-0}"
+    ;;
+  esac
+}
+
+begin() {
+  case_name="$1" case_failed=0
+  : >"$LOG"
+  local rc=0
+  MONGO_VERSION="${TARGET:-8.0}" setup_mongodb >/dev/null 2>&1 || rc=$?
+  [[ "$rc" == "$2" ]] || { echo "  rc=$rc, expected $2"; case_failed=1; }
+}
+expect() { grep -qF -- "$1" "$LOG" || { echo "  missing: $1"; case_failed=1; }; }
+reject() { ! grep -qF -- "$1" "$LOG" || { echo "  unexpected: $1"; case_failed=1; }; }
+before() {
+  local a b
+  a=$(grep -nF -- "$1" "$LOG" | head -n1 | cut -d: -f1)
+  b=$(grep -nF -- "$2" "$LOG" | head -n1 | cut -d: -f1)
+  [[ -n "$a" && -n "$b" && "$a" -lt "$b" ]] || { echo "  order: '$1' must come before '$2'"; case_failed=1; }
+}
+end() {
+  if ((case_failed)); then
+    echo "FAIL $case_name"
+    sed 's/^/    /' "$LOG"
+    failed=$((failed + 1))
+  else
+    echo "PASS $case_name"
+    passed=$((passed + 1))
+  fi
+}
+
+MOCK_INSTALLED="" begin "fresh install" 0
+expect "repo 8.0"; expect "install mongodb-org"; reject "purge"; reject "mongosh"
+end
+
+MOCK_INSTALLED="8.0" begin "same series only takes patch updates" 0
+expect "upgrade mongodb-org"; reject "repo"; reject "install mongodb-org"
+end
+
+MOCK_INSTALLED="7.0" MOCK_PURGE=$'Purg mongodb-org [7.0.22]\nPurg mongodb-org-server [7.0.22]' begin "purge when nothing depends on it" 0
+expect "purge mongodb"; expect "repo 8.0"; expect "install mongodb-org"; reject "setFeatureCompatibilityVersion"
+end
+
+MOCK_INSTALLED="7.0" MOCK_PURGE=$'Purg mongodb-org-server [7.0.22]\nRemv omadac [6.2.10.17]' MOCK_FCV="7.0" begin "in place when a package depends on it" 0
+reject "purge mongodb"; expect "repo 8.0"; expect "install mongodb-org"
+expect "setFeatureCompatibilityVersion: '8.0', confirm: true"; expect "custom: Apps that run their own mongod"
+end
+
+MOCK_INSTALLED="7.0" MOCK_PURGE="Remv omadac [6.2.10.17]" MOCK_FCV="6.0" begin "data still at the older fcv is raised first" 0
+expect "setFeatureCompatibilityVersion: '7.0', confirm: true"
+before "setFeatureCompatibilityVersion: '7.0'" "install mongodb-org"
+before "install mongodb-org" "setFeatureCompatibilityVersion: '8.0'"
+end
+
+MOCK_INSTALLED="6.0" MOCK_PURGE="Remv omadac [6.2.10.17]" begin "refuse a two-major jump" 1
+expect "needs 7.0 first"; reject "install mongodb-org"; reject "repo"
+end
+
+MOCK_INSTALLED="8.0" TARGET="7.0" MOCK_PURGE="Remv omadac [6.2.10.17]" begin "refuse a downgrade" 1
+expect "cannot be downgraded"; reject "install mongodb-org"
+end
+
+MOCK_INSTALLED="7.0" MOCK_PURGE="Remv omadac [6.1.0]" MOCK_FCV="7.0" MOCK_SIM=$'Inst mongodb-org-server [7.0.22] (8.0.3)\nRemv omadac [6.1.0]' begin "refuse when apt would remove the app" 1
+expect "would remove: omadac"; reject "install mongodb-org"
+before "repo 8.0" "repo 7.0"
+end
+
+MOCK_INSTALLED="7.0" MOCK_PURGE="Remv omadac [6.1.0]" MOCK_FCV="7.0" MOCK_SIM="E: Unable to correct problems, you have held broken packages." MOCK_SIM_RC=100 begin "abort when apt refuses the upgrade" 1
+expect "cannot upgrade mongodb-org in place"; expect "repo 7.0"; reject "install mongodb-org"
+end
+
+printf '%s passed, %s failed\n' "$passed" "$failed"
+((failed == 0))