diff --git a/.github/workflows/verify-split-files.yml b/.github/workflows/verify-split-files.yml index 3069935..d6dfae7 100644 --- a/.github/workflows/verify-split-files.yml +++ b/.github/workflows/verify-split-files.yml @@ -27,6 +27,7 @@ on: - "tests/vm-helpers.sh" - "tests/mongodb-upgrade.sh" - "tests/uv-required-version.sh" + - "tests/docker-lxcfs.sh" - ".github/workflows/verify-split-files.yml" push: branches: [main] @@ -45,6 +46,7 @@ on: - "tests/vm-helpers.sh" - "tests/mongodb-upgrade.sh" - "tests/uv-required-version.sh" + - "tests/docker-lxcfs.sh" - ".github/workflows/verify-split-files.yml" workflow_dispatch: @@ -81,6 +83,9 @@ jobs: - name: uv pin regression tests run: bash tests/uv-required-version.sh + - name: Docker lxcfs visibility regression tests + run: bash tests/docker-lxcfs.sh + - name: Prefetch lists cover every engine file run: | set -euo pipefail diff --git a/docs/lib.md b/docs/lib.md index 5bfb900..de79f7e 100644 --- a/docs/lib.md +++ b/docs/lib.md @@ -52,6 +52,47 @@ Language runtimes and the application-level installers: `setup_nodejs`, `setup_java`, `setup_dotnet`, `setup_composer`, `setup_ffmpeg`, `setup_imagemagick`, `setup_yq`, `setup_adminer`. +### Docker inside LXC + +Inside an LXC container, Docker gives each container a fresh procfs, so the +lxcfs mounts on `/proc/meminfo` and friends never reach it: nested containers +see the physical host's memory instead of the CT's limit. Limits are still +enforced, but anything that sizes itself from those numbers (JVM heap, Node, Go, +OpenMP thread pools) gets OOM-killed instead of staying within them. + +`setup_docker` can fix this. It is off by default; enable it with +`DOCKER_LXCFS_VISIBILITY=yes` (or `true`), or the app variable +`var_docker_lxcfs=yes`. When enabled, it registers `/usr/local/bin/runc-lxcfs` as +Docker's `default-runtime`. On each container create, the wrapper binds the lxcfs +files into the container. If the container sets no memory limit, the wrapper +also applies the CT's limit to its cgroup, which musl and Node's +`uv_get_constrained_memory()` read. CPU needs nothing extra: `cpuset` is +inherited, and `/sys/devices/system/cpu/online` is bound for glibc's +`_SC_NPROCESSORS_ONLN`. + +- It is a no-op, with a warning, outside LXC or where lxcfs is not mounted. +- It never replaces an existing `default-runtime` (e.g. `nvidia`). It never + overrides an explicit `--memory` / `mem_limit`. +- On an update it only runs when explicitly requested. Without the variable, + `daemon.json` is left as it is. With it, a running Docker is reloaded + (`SIGHUP`) so the new default runtime applies without a restart. +- Setup leaves Docker untouched when `runc` or `jq` is missing or `daemon.json` + cannot be parsed. If the wrapper cannot patch a container's config, it hands + that config to runc unmodified. +- Use `docker run --runtime=runc` (compose: `runtime: runc`) to bypass it for one + container. +- The limit is written at container creation. After `pct set` changes the CT's + memory, existing containers keep the old value until they are recreated, or + until `docker update --memory= --memory-swap= `. +- **Monitoring containers are affected.** lxcfs reports *usage* from the + reading process's own cgroup. A container that graphs "system memory used" + from `/proc/meminfo` (gopsutil-based tools such as netdata, glances, Arcane's + live stats) therefore shows only its own footprint. Total is correct, but + free, available and cached memory are not. Run those containers with + `runtime: runc`. Tools that read the Docker API instead (Portainer's + dashboard, Arcane's Docker info panel) are unaffected, and `/proc/stat` (CPU) + is not cgroup-scoped. + ## [`db.func`](../lib/db.func) `setup_postgresql`, `setup_mysql`, `setup_mariadb`, `setup_mongodb`, diff --git a/lib/API.txt b/lib/API.txt index 40df175..fabe0f7 100644 --- a/lib/API.txt +++ b/lib/API.txt @@ -11,6 +11,7 @@ _deploy_source_tarball _deploy_unpacked_archive _diagnose_deb_failure _docker_is_noninteractive +_docker_setup_lxcfs_visibility _download_source_tarball _forge_asset_urls _forge_curl diff --git a/lib/runtime.func b/lib/runtime.func index b6b3e57..20c389e 100644 --- a/lib/runtime.func +++ b/lib/runtime.func @@ -142,6 +142,8 @@ setup_composer() { # DOCKER_PORTAINER - Install Portainer CE (optional, "true" to enable) # DOCKER_LOG_DRIVER - Log driver (optional, default: "journald") # DOCKER_SKIP_UPDATES - Skip container update check (optional, "true" to skip) +# DOCKER_LXCFS_VISIBILITY - "yes"/"true" to show the CT's resource limits inside +# nested containers (default: no; also var_docker_lxcfs) # # Features: # - Uses official Docker repository by default @@ -219,6 +221,131 @@ ensure_docker() { fi } +# ------------------------------------------------------------------------------ +# Opt-in runc wrapper that shows the LXC container's resource limits to nested +# Docker containers. Caveats and per-container bypass: docs/lib.md#docker-inside-lxc. +# ------------------------------------------------------------------------------ +_docker_setup_lxcfs_visibility() { + local enabled="${DOCKER_LXCFS_VISIBILITY:-${var_docker_lxcfs:-no}}" + case "${enabled,,}" in + yes | true) ;; + *) return 0 ;; + esac + + # Only meaningful inside an LXC container whose /proc is served by lxcfs + if ! grep -qs 'fuse\.lxcfs' /proc/mounts; then + msg_warn "lxcfs not mounted - skipping LXC resource visibility for Docker" + return 0 + fi + + if ! command -v runc >/dev/null 2>&1; then + msg_warn "runc not found - skipping LXC resource visibility for Docker" + return 0 + fi + + if ! command -v jq >/dev/null 2>&1; then + $STD apt-get install -y jq || { + msg_warn "jq unavailable - skipping LXC resource visibility for Docker" + return 0 + } + fi + + local current_runtime + current_runtime=$(jq -r '."default-runtime" // empty' /etc/docker/daemon.json 2>/dev/null) || current_runtime="" + if [[ -n "$current_runtime" && "$current_runtime" != "lxcfs" ]]; then + msg_warn "Docker default-runtime is already '${current_runtime}' - skipping LXC resource visibility" + return 0 + fi + + msg_info "Configuring LXC resource visibility for Docker" + + cat <<'RUNC_LXCFS' >/usr/local/bin/runc-lxcfs +#!/bin/sh +# Managed by community-scripts - see setup_docker() in lib/runtime.func +# +# runc wrapper: let Docker containers running inside this LXC container observe +# the LXC container's resource limits instead of the physical host's. +# +# Docker mounts a fresh procfs per container, so the lxcfs binds LXC placed on +# /proc/meminfo & friends never reach it. Re-bind them into the OCI bundle, and +# apply the CT's memory limit to the container cgroup for runtimes that consult +# cgroups rather than /proc. +# +# Bypass for a single container with: docker run --runtime=runc ... +set -eu + +REAL_RUNC= +for candidate in /usr/bin/runc /usr/sbin/runc /usr/local/bin/runc; do + if [ -x "$candidate" ]; then + REAL_RUNC=$candidate + break + fi +done +[ -n "$REAL_RUNC" ] || exit 127 + +bundle="" +prev="" +for arg in "$@"; do + case "$prev" in + -b | --bundle) bundle="$arg" ;; + esac + case "$arg" in + --bundle=*) bundle="${arg#--bundle=}" ;; + esac + prev="$arg" +done + +# Only "create" carries a bundle; everything else passes straight through. +[ -n "$bundle" ] && [ -f "$bundle/config.json" ] || exec "$REAL_RUNC" "$@" + +mem=$(awk '/^MemTotal:/ { print $2 * 1024 }' /proc/meminfo) +[ -n "$mem" ] || exec "$REAL_RUNC" "$@" + +config="$bundle/config.json" +patched="$config.lxcfs" +if jq --argjson mem "$mem" ' + ["/proc/meminfo", "/proc/cpuinfo", "/proc/stat", "/proc/uptime", + "/proc/swaps", "/proc/loadavg", "/proc/diskstats", + "/sys/devices/system/cpu/online"] as $lxcfs + | [.mounts[].destination] as $taken + | .mounts += [ + $lxcfs[] + | select(. as $f | $taken | index($f) | not) + | {destination: ., type: "bind", source: ., options: ["rbind", "rprivate", "ro"]} + ] + | if (.linux.resources.memory.limit // 0) <= 0 + then .linux.resources.memory.limit = $mem + else . end +' "$config" >"$patched" 2>/dev/null && [ -s "$patched" ]; then + cat "$patched" >"$config" +fi +rm -f "$patched" + +exec "$REAL_RUNC" "$@" +RUNC_LXCFS + + chmod 755 /usr/local/bin/runc-lxcfs + + local tmpfile + tmpfile=$(mktemp) + if jq '. + { + "default-runtime": "lxcfs", + "runtimes": ((.runtimes // {}) + {"lxcfs": {"path": "/usr/local/bin/runc-lxcfs"}}) + }' /etc/docker/daemon.json >"$tmpfile" && [ -s "$tmpfile" ]; then + mv "$tmpfile" /etc/docker/daemon.json + # An already-running dockerd (update path) picks up default-runtime on SIGHUP. + if systemctl is-active -q docker 2>/dev/null; then + $STD systemctl reload docker || msg_warn "Could not reload Docker - restart it to apply LXC resource visibility" + fi + msg_ok "Configured LXC resource visibility for Docker" + else + rm -f "$tmpfile" + # A previous run's daemon.json still points at the wrapper; keep it. + [[ -n "$current_runtime" ]] || rm -f /usr/local/bin/runc-lxcfs + msg_warn "Could not update /etc/docker/daemon.json - LXC resource visibility not enabled" + fi +} + setup_docker() { local docker_installed=false local portainer_installed=false @@ -275,6 +402,8 @@ setup_docker() { EOF fi + _docker_setup_lxcfs_visibility + # Enable and start Docker systemctl enable -q --now docker @@ -349,6 +478,8 @@ EOF EOF fi + _docker_setup_lxcfs_visibility + # Enable and start Docker systemctl enable -q --now docker fi diff --git a/tests/docker-lxcfs.sh b/tests/docker-lxcfs.sh new file mode 100755 index 0000000..6de23d4 --- /dev/null +++ b/tests/docker-lxcfs.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_DIR="$(mktemp -d)" +trap 'rm -rf -- "$TEST_DIR"' EXIT +passed=0 failed=0 + +# Only the lxcfs helper, with its fixed system paths redirected into TEST_DIR. +eval "$(awk '/^_docker_setup_lxcfs_visibility\(\) \{/,/^}/' "$CORE_DIR/lib/runtime.func" | + sed -e "s#/proc/mounts#$TEST_DIR/mounts#g" \ + -e "s#/etc/docker/daemon.json#$TEST_DIR/daemon.json#g" \ + -e "s#/usr/local/bin/runc-lxcfs#$TEST_DIR/runc-lxcfs#g")" + +export STD="" +msg_info() { :; } +msg_ok() { :; } +msg_warn() { :; } +runc() { :; } +# Logs calls; Docker counts as running only when docker_active is set. +systemctl() { echo "$*" >>"$TEST_DIR/systemctl.log"; [[ "$1" != is-active ]] || [[ -n "${docker_active:-}" ]]; } +lxcfs_mounts='lxcfs /proc/meminfo fuse.lxcfs rw 0 0' + +check() { + if [[ "$3" == "$2" ]]; then + echo "PASS $1" + passed=$((passed + 1)) + else + echo "FAIL $1: expected <$2>, got <$3>" >&2 + failed=$((failed + 1)) + fi +} +# run [VAR=value...]: fresh state, run the helper once. +run() { + echo "$1" >"$TEST_DIR/daemon.json" + echo "$lxcfs_mounts" >"$TEST_DIR/mounts" + rm -f "$TEST_DIR/runc-lxcfs" + shift + ( + unset DOCKER_LXCFS_VISIBILITY var_docker_lxcfs + # shellcheck disable=SC2163 # exports each VAR=value argument + (($#)) && export "$@" + _docker_setup_lxcfs_visibility + ) +} +field() { jq -r "$1 // \"none\"" "$TEST_DIR/daemon.json"; } +plain='{"log-driver": "journald"}' + +run "$plain" +check "off by default" "none" "$(field '."default-runtime"')" +for v in no false 0; do + run "$plain" DOCKER_LXCFS_VISIBILITY=$v + check "DOCKER_LXCFS_VISIBILITY=$v stays off" "none" "$(field '."default-runtime"')" +done +run "$plain" DOCKER_LXCFS_VISIBILITY=no var_docker_lxcfs=yes +check "DOCKER_LXCFS_VISIBILITY wins over var_docker_lxcfs" "none" "$(field '."default-runtime"')" +run "$plain" var_docker_lxcfs=yes +check "var_docker_lxcfs=yes enables" "lxcfs" "$(field '."default-runtime"')" + +run "$plain" DOCKER_LXCFS_VISIBILITY=TRUE +check "enabled: default-runtime" "lxcfs" "$(field '."default-runtime"')" +check "enabled: runtime registered" "$TEST_DIR/runc-lxcfs" "$(field '.runtimes.lxcfs.path')" +check "enabled: other keys kept" "journald" "$(field '."log-driver"')" +check "enabled: wrapper executable" "yes" "$([[ -x "$TEST_DIR/runc-lxcfs" ]] && echo yes)" +check "enabled: wrapper is valid sh" "yes" "$(sh -n "$TEST_DIR/runc-lxcfs" && echo yes)" +first="$(cat "$TEST_DIR/daemon.json")" +(export DOCKER_LXCFS_VISIBILITY=yes && _docker_setup_lxcfs_visibility) +check "second run leaves daemon.json identical" "$first" "$(cat "$TEST_DIR/daemon.json")" + +# Update path: a running dockerd only sees the new default runtime after a reload. +(docker_active=1 && run "$plain" DOCKER_LXCFS_VISIBILITY=yes) +check "running Docker is reloaded" "yes" "$(grep -qx 'reload docker' "$TEST_DIR/systemctl.log" && echo yes)" +rm -f "$TEST_DIR/systemctl.log" +run "$plain" DOCKER_LXCFS_VISIBILITY=yes +check "stopped Docker is not reloaded" "no" "$(grep -q reload "$TEST_DIR/systemctl.log" && echo yes || echo no)" + +run '{"default-runtime": "nvidia", "runtimes": {"nvidia": {"path": "nvidia-container-runtime"}}}' DOCKER_LXCFS_VISIBILITY=yes +check "existing default-runtime kept" "nvidia" "$(field '."default-runtime"')" +check "existing default-runtime: no wrapper" "no" "$([[ -e "$TEST_DIR/runc-lxcfs" ]] && echo yes || echo no)" + +(lxcfs_mounts='proc /proc proc rw 0 0' && run "$plain" DOCKER_LXCFS_VISIBILITY=yes) +check "no lxcfs mounted: untouched" "none" "$(field '."default-runtime"')" + +run '{"log-driver": ' DOCKER_LXCFS_VISIBILITY=yes +check "malformed daemon.json untouched" '{"log-driver": ' "$(cat "$TEST_DIR/daemon.json")" +check "malformed daemon.json: wrapper removed" "no" "$([[ -e "$TEST_DIR/runc-lxcfs" ]] && echo yes || echo no)" + +printf '%s passed, %s failed\n' "$passed" "$failed" +((failed == 0))