diff --git a/baseline/experiments/mnist_mlp3_tangent_rg/README.md b/baseline/experiments/mnist_mlp3_tangent_rg/README.md index df2521e..545b306 100644 --- a/baseline/experiments/mnist_mlp3_tangent_rg/README.md +++ b/baseline/experiments/mnist_mlp3_tangent_rg/README.md @@ -981,3 +981,29 @@ The principal outputs under `quotient_flow.log`: live progress/ETA and full terminal-equivalent logs; - `report/index.html`: shareable documentation, separate MuonClip-RMS and AdamW plots, tables, method coverage, and saved-data links. + + +## Detached command-line lifecycle manager + +Use the lifecycle manager instead of pasting shell strict-mode settings or attaching +`tail -f` to the analysis. The manager launches the complete four-stage analysis +in a new OS session, redirects standard output and standard error to +`/private/tmp`, and returns the terminal prompt immediately. + +```bash +bash baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh start +``` + +The remaining commands are one-shot operations; none attaches a foreground job: + +```bash +bash baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh status +bash baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh log +bash baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh stop +bash baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh open +``` + +`stop` signals only the detached analysis session. It also recognizes the +previous `nohup` launcher and terminates only its recorded process tree, never +the interactive shell or its process group. Completed checkpoint units are +resumable by the next `start`. diff --git a/baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh b/baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh new file mode 100644 index 0000000..58bfee8 --- /dev/null +++ b/baseline/experiments/mnist_mlp3_tangent_rg/scripts/manage_short100_complete_rg_analysis.sh @@ -0,0 +1,299 @@ +#!/usr/bin/env bash +# Detached lifecycle manager for the complete short-100 RG analysis. +# +# This file intentionally avoids `set -e` because callers commonly invoke it +# from an interactive terminal. Every failure is handled inside this process; +# no shell options are changed in the caller. + +set -u + +COMMAND="${1:-start}" +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +DEFAULT_REPO="$(CDPATH= cd -- "$SCRIPT_DIR/../../../.." && pwd)" + +REPO="${REPO:-$DEFAULT_REPO}" +RUNS="${RUNS:-/private/tmp/rg-mnist-mlp3-short100-runs}" +CACHE="${CACHE:-/private/tmp/rg-mnist-mlp3-short100-checkpoints}" +OUT="${OUT:-/private/tmp/rg-mnist-mlp3-short100-jacobians-reduced}" +MPLCONFIGDIR="${MPLCONFIGDIR:-/private/tmp/matplotlib-cache}" +PYTHON_BIN="${PYTHON_BIN:-python}" + +WORKER="$REPO/baseline/experiments/mnist_mlp3_tangent_rg/scripts/run_short100_complete_rg_analysis.sh" +LOG="$OUT/complete_rg_analysis.log" +PID_FILE="$OUT/run.pid" +LAUNCH_FILE="$OUT/launch.json" + +export REPO RUNS CACHE OUT MPLCONFIGDIR +export RG_MNIST_TANGENT_ROOT="$RUNS" +export RG_MNIST_TANGENT_CHECKPOINT_CACHE_ROOT="$CACHE" +export RG_MNIST_REDUCED_JACOBIAN_OUTPUT_ROOT="$OUT" +export PYTHONUNBUFFERED=1 + +usage() { + printf '%s\n' \ + "Usage: bash $0 {start|status|log|stop|open|help}" \ + "" \ + " start launch in a new OS session and return immediately" \ + " status print process state, stage JSON, and recent log lines" \ + " log print the most recent log lines and return immediately" \ + " stop send SIGTERM only to the recorded analysis process group" \ + " open open the completed static HTML report" \ + "" \ + "Data, logs, PID state, figures, CSVs, and HTML all remain under:" \ + " $OUT" +} + +read_pid() { + if [ ! -f "$PID_FILE" ]; then + return 1 + fi + local value + value="$(tr -d '[:space:]' < "$PID_FILE" 2>/dev/null)" + case "$value" in + ''|*[!0-9]*) return 1 ;; + *) printf '%s\n' "$value" ;; + esac +} + +pid_is_live() { + local pid="$1" + kill -0 "$pid" 2>/dev/null +} + +print_json_if_present() { + local label="$1" + local path="$2" + if [ -f "$path" ]; then + printf '\n%s\n' "$label" + "$PYTHON_BIN" -m json.tool "$path" 2>/dev/null || sed -n '1,120p' "$path" + fi +} + +start_analysis() { + mkdir -p "$OUT" "$MPLCONFIGDIR" || { + printf 'ERROR: cannot create output directories beneath %s\n' "$OUT" >&2 + return 1 + } + if [ ! -f "$WORKER" ]; then + printf 'ERROR: worker script not found: %s\n' "$WORKER" >&2 + return 1 + fi + + local old_pid + old_pid="$(read_pid 2>/dev/null || true)" + if [ -n "$old_pid" ] && pid_is_live "$old_pid"; then + printf 'Analysis is already running as PID %s.\n' "$old_pid" + printf 'Use `%s status` for a one-shot progress report.\n' "$0" + return 0 + fi + + if [ "${RG_SKIP_ENV_PREFLIGHT:-0}" != "1" ]; then + "$PYTHON_BIN" -c 'import torch, weightwatcher, powerlaw' >/dev/null 2>&1 || { + printf '%s\n' \ + "ERROR: the active Python environment lacks torch, weightwatcher, or powerlaw." \ + "Activate rg-muonclip-run, then run this start command again." >&2 + return 1 + } + fi + + local launched_pid + launched_pid="$("$PYTHON_BIN" - "$REPO" "$WORKER" "$LOG" "$LAUNCH_FILE" <<'PY' +import json +import os +from pathlib import Path +import subprocess +import sys +from datetime import datetime, timezone + +repo, worker, log_path, launch_path = sys.argv[1:] +Path(log_path).parent.mkdir(parents=True, exist_ok=True) +with open(log_path, "ab", buffering=0) as log_handle: + process = subprocess.Popen( + ["bash", worker], + cwd=repo, + env=os.environ.copy(), + stdin=subprocess.DEVNULL, + stdout=log_handle, + stderr=subprocess.STDOUT, + start_new_session=True, + close_fds=True, + ) +payload = { + "pid": process.pid, + "session_id": process.pid, + "started_at_utc": datetime.now(timezone.utc).isoformat(), + "repository": repo, + "worker": worker, + "log": log_path, + "output_root": os.environ["OUT"], +} +temporary = Path(launch_path + ".tmp") +temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") +temporary.replace(launch_path) +print(process.pid) +PY +)" || { + printf 'ERROR: failed to launch the detached analysis.\n' >&2 + return 1 + } + + printf '%s\n' "$launched_pid" > "$PID_FILE" + printf '%s\n' \ + "Analysis started in a detached OS session." \ + "PID: $launched_pid" \ + "Log: $LOG" \ + "Output: $OUT" \ + "Your prompt is now safe to use. Do not press Ctrl-Z; no foreground job is attached." \ + "Run this for a one-shot update:" \ + " bash $0 status" +} + +show_status() { + local pid + pid="$(read_pid 2>/dev/null || true)" + if [ -n "$pid" ] && pid_is_live "$pid"; then + printf 'PROCESS: RUNNING\n' + ps -p "$pid" -o pid,ppid,%cpu,%mem,etime,state,command 2>/dev/null || \ + printf 'PID %s is live; detailed process statistics are unavailable.\n' "$pid" + elif [ -n "$pid" ]; then + printf 'PROCESS: NOT RUNNING (recorded PID %s)\n' "$pid" + else + printf 'PROCESS: NOT STARTED (no valid PID file)\n' + fi + print_json_if_present "STAGE 1 STATUS" "$OUT/status.json" + print_json_if_present "STAGES 2-3 STATUS" "$OUT/quotient_flow_status.json" + if [ -f "$OUT/report/index.html" ]; then + printf '\nREPORT: %s\n' "$OUT/report/index.html" + fi + printf '\nLATEST LOG LINES\n' + if [ -f "$LOG" ]; then + tail -n "${LOG_LINES:-40}" "$LOG" + else + printf 'No log exists yet: %s\n' "$LOG" + fi +} + +show_log() { + if [ ! -f "$LOG" ]; then + printf 'No log exists yet: %s\n' "$LOG" >&2 + return 1 + fi + tail -n "${LOG_LINES:-100}" "$LOG" +} + +stop_analysis() { + local pid + pid="$(read_pid 2>/dev/null || true)" + if [ -z "$pid" ]; then + printf 'No valid analysis PID is recorded in %s\n' "$PID_FILE" + return 0 + fi + if ! pid_is_live "$pid"; then + printf 'Analysis PID %s is already stopped.\n' "$pid" + return 0 + fi + "$PYTHON_BIN" - "$pid" "$LAUNCH_FILE" "$WORKER" <<'PY' +import json +import os +from pathlib import Path +import signal +import subprocess +import sys + +pid = int(sys.argv[1]) +launch_path = Path(sys.argv[2]) +expected_worker = str(Path(sys.argv[3]).resolve()) +managed = False +if launch_path.is_file(): + launch = json.loads(launch_path.read_text()) + managed = ( + int(launch.get("pid", -1)) == pid + and str(Path(launch.get("worker", "")).resolve()) == expected_worker + ) + +if managed: + try: + session_id = os.getsid(pid) + except ProcessLookupError: + raise SystemExit(f"Analysis PID {pid} is already stopped") + if session_id != pid: + raise SystemExit( + f"Refusing to signal PID {pid}: it is not the detached session leader" + ) + os.killpg(pid, signal.SIGTERM) +else: + # Compatibility path for the old nohup launcher. Signal only the recorded + # job and its descendants--never its process group, which may contain the + # interactive shell. Refuse unless the process tree contains a known RG + # analysis command. + listing = subprocess.run( + ["ps", "-axo", "pid=,ppid=,command="], + check=True, + capture_output=True, + text=True, + ).stdout + processes = {} + children = {} + for line in listing.splitlines(): + fields = line.strip().split(None, 2) + if len(fields) < 2: + continue + observed_pid, parent_pid = map(int, fields[:2]) + command = fields[2] if len(fields) == 3 else "" + processes[observed_pid] = command + children.setdefault(parent_pid, []).append(observed_pid) + if pid not in processes: + raise SystemExit(f"Analysis PID {pid} is already stopped") + descendants = [] + stack = list(children.get(pid, ())) + while stack: + child = stack.pop() + descendants.append(child) + stack.extend(children.get(child, ())) + known = ( + "run_short100_complete_rg_analysis.sh", + "run_short100_jacobians_cli.py", + "run_short100_quotient_flow_cli.py", + "build_short100_jacobian_report.py", + ) + tree = [pid, *descendants] + if not any(any(name in processes.get(item, "") for name in known) for item in tree): + raise SystemExit( + f"Refusing to signal PID {pid}: its process tree is not the RG analysis" + ) + for item in [*reversed(descendants), pid]: + try: + os.kill(item, signal.SIGTERM) + except ProcessLookupError: + pass +PY + if [ "$?" -ne 0 ]; then + return 1 + fi + printf 'Sent SIGTERM to detached analysis process group %s.\n' "$pid" + printf 'Completed checkpoint units remain on disk and the next start resumes them.\n' +} + +open_report() { + local report="$OUT/report/index.html" + if [ ! -f "$report" ]; then + printf 'Report is not ready: %s\n' "$report" >&2 + printf 'Use `%s status` to inspect the current stage.\n' "$0" >&2 + return 1 + fi + /usr/bin/open "$report" +} + +case "$COMMAND" in + start) start_analysis ;; + status) show_status ;; + log) show_log ;; + stop) stop_analysis ;; + open) open_report ;; + help|-h|--help) usage ;; + *) + printf 'Unknown command: %s\n\n' "$COMMAND" >&2 + usage >&2 + exit 2 + ;; +esac diff --git a/baseline/tests/tangent_rg/test_detached_analysis_manager.py b/baseline/tests/tangent_rg/test_detached_analysis_manager.py new file mode 100644 index 0000000..0bd9665 --- /dev/null +++ b/baseline/tests/tangent_rg/test_detached_analysis_manager.py @@ -0,0 +1,43 @@ +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MANAGER = ( + REPOSITORY_ROOT + / "experiments" + / "mnist_mlp3_tangent_rg" + / "scripts" + / "manage_short100_complete_rg_analysis.sh" +) + + +def test_manager_never_changes_the_callers_errexit_or_attaches_a_log_tail(): + source = MANAGER.read_text(encoding="utf-8") + executable_lines = [ + line.strip() for line in source.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + + assert not any(line.startswith("set -e") for line in executable_lines) + assert "tail -f" not in source + assert "tail -F" not in source + + +def test_manager_launches_a_new_session_and_exposes_safe_lifecycle_commands(): + source = MANAGER.read_text(encoding="utf-8") + + assert "start_new_session=True" in source + assert "stdin=subprocess.DEVNULL" in source + assert "stdout=log_handle" in source + assert "stderr=subprocess.STDOUT" in source + for command in ("start", "status", "log", "stop", "open"): + assert f"{command})" in source + + +def test_legacy_stop_never_signals_the_interactive_process_group(): + source = MANAGER.read_text(encoding="utf-8") + + assert "old nohup launcher" in source + assert "os.kill(item, signal.SIGTERM)" in source + assert "os.killpg(pid, signal.SIGTERM)" in source + assert "if managed:" in source