diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43b1c870..916fd3af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,11 +157,12 @@ jobs: shard: [0, 1, 2, 3] steps: - uses: actions/checkout@v7 - - name: Install Ansible + - name: Install Ansible and zsh + # zsh is one of the two shells the host tools role writes startup lines for, and its tests read those lines back with a real zsh. run: | set -euo pipefail sudo apt-get update -qq - sudo apt-get install -y -qq ansible pkg-config + sudo apt-get install -y -qq ansible pkg-config zsh - name: Unit-test the collection # Ubuntu's ansible package puts ansible-core on the system Python the filter plugins import from. The runner image can carry another Ansible earlier on PATH, so the tests run with /usr/bin first and ANSIBLE_PLAYBOOK pointing at Ubuntu's, which makes every playbook a test runs use the same ansible-core as the filters it imports; both versions are printed for the record. A module is a shard's when its position in the list of modules, largest first, leaves this shard's index modulo the number of shards. env: diff --git a/plugins/filter/host_tools.py b/plugins/filter/host_tools.py index e0121484..a3de4ec5 100644 --- a/plugins/filter/host_tools.py +++ b/plugins/filter/host_tools.py @@ -1,4 +1,4 @@ -"""Filters the github_runner_host_tools role uses: the Homebrew prefix a host has, the shell startup lines that bring Homebrew and mise into a shell or set environment variables in it, mise's configuration rendered to TOML, the managed lines of an npmrc file and a YAML file merged with settings.""" +"""Filters the github_runner_host_tools role uses: the Homebrew prefix a host has, the shell startup lines that bring Homebrew and mise into a shell or set environment variables in it, whether a managed block leads its startup file, mise's configuration rendered to TOML, the managed lines of an npmrc file and a YAML file merged with settings.""" from __future__ import annotations @@ -52,12 +52,12 @@ def host_tools_homebrew_prefix(system: Any, machine: Any) -> str: raise AnsibleFilterError(f"Homebrew supports Darwin and Linux, not {system!r}.") -def host_tools_shell_integrations(prefix: Any, shell: Any, homebrew: Any, mise: Any) -> list[dict[str, str]]: +def host_tools_shell_integrations(prefix: Any, shell: Any, homebrew: Any, mise: Any) -> list[dict[str, Any]]: """Return the startup lines that bring Homebrew and mise into a shell. - Each line is guarded by a test that the binary is executable, so a startup file that outlives the tool does no harm. ``pattern`` is an extended regular expression that matches a line of a startup file that already does the same, outside a comment, and ``name`` tells the integrations apart. + Each line is guarded by a test that the binary is executable, so a startup file that outlives the tool does no harm. ``pattern`` is an extended regular expression that matches a line of a startup file that already does the same, outside a comment, ``name`` tells the integrations apart, and ``non_interactive`` is false because Homebrew and mise are wanted in an interactive shell only (the line goes where the file's own interactive setup is, never ahead of an early return for non-interactive shells). - :param prefix: the Homebrew prefix, whose ``bin`` holds ``brew`` and ``mise``. :param shell: the shell's name, ``zsh`` or ``bash``, which mise's activation is generated for. :param homebrew: whether Homebrew is wanted in the shell. :param mise: whether mise is wanted in the shell. :returns: ``[{"name", "pattern", "line"}, ...]``, Homebrew first, with a dictionary for each tool that is wanted. :raises AnsibleFilterError: the prefix is not an absolute path or the shell is not one the role supports. + :param prefix: the Homebrew prefix, whose ``bin`` holds ``brew`` and ``mise``. :param shell: the shell's name, ``zsh`` or ``bash``, which mise's activation is generated for. :param homebrew: whether Homebrew is wanted in the shell. :param mise: whether mise is wanted in the shell. :returns: ``[{"name", "pattern", "line", "non_interactive"}, ...]``, Homebrew first, with a dictionary for each tool that is wanted. :raises AnsibleFilterError: the prefix is not an absolute path or the shell is not one the role supports. """ if not isinstance(prefix, str) or not prefix.startswith("/"): raise AnsibleFilterError(f"The Homebrew prefix must be an absolute path, not {prefix!r}.") @@ -65,11 +65,11 @@ def host_tools_shell_integrations(prefix: Any, shell: Any, homebrew: Any, mise: raise AnsibleFilterError(f"The shell must be zsh or bash, not {shell!r}.") brew = shlex.quote(f"{prefix}/bin/brew") mise_binary = shlex.quote(f"{prefix}/bin/mise") - integrations: list[dict[str, str]] = [] + integrations: list[dict[str, Any]] = [] if homebrew: - integrations.append({"name": "homebrew", "pattern": "^[^#]*brew shellenv", "line": f'[ -x {brew} ] && eval "$({brew} shellenv)"'}) + integrations.append({"name": "homebrew", "pattern": "^[^#]*brew shellenv", "line": f'[ -x {brew} ] && eval "$({brew} shellenv)"', "non_interactive": False}) if mise: - integrations.append({"name": "mise", "pattern": "^[^#]*mise activate", "line": f'[ -x {mise_binary} ] && eval "$({mise_binary} activate {shell})"'}) + integrations.append({"name": "mise", "pattern": "^[^#]*mise activate", "line": f'[ -x {mise_binary} ] && eval "$({mise_binary} activate {shell})"', "non_interactive": False}) return integrations @@ -90,24 +90,87 @@ def _env_value(name: str, value: Any) -> str: raise AnsibleFilterError(f"The value of {name} must be a string, a number or a boolean, not {type(value).__name__}.") -def host_tools_shell_env(env: Any) -> list[dict[str, str]]: - """Return the startup lines that set environment variables in a shell, in the shape host_tools_shell_integrations returns. +def _env_name(name: Any) -> str: + """Return the name of an environment variable. - Each value is quoted for the shell with ``shlex.quote``, so it reaches the variable exactly as given whatever it holds (spaces, quotes, ``$``, backticks), in zsh and in bash alike. ``pattern`` is an extended regular expression that matches a line of a startup file that already assigns the variable, outside a comment, whatever its value, so a value the user chose is never overridden, and ``name`` tells the lines apart. + :raises AnsibleFilterError: the name is not one the shell accepts in an assignment. + """ + if not isinstance(name, str) or not ENV_NAME.match(name): + raise AnsibleFilterError(f"{name!r} is not a name a shell accepts for an environment variable.") + return name + + +def _export_line(name: str, word: str) -> dict[str, Any]: + """Return the startup line that exports a variable set to a shell word, with the pattern that finds an assignment of it already in a startup file.""" + return {"name": f"env-{name}", "pattern": f"^[[:space:]]*(export[[:space:]]+)?{name}=", "line": f"export {name}={word}", "non_interactive": True} + + +def _home_relative_value(name: str, value: Any) -> str: + """Return the shell word that expands to the user's home directory followed by a relative path: ``"$HOME/"`` with the path escaped for double quotes. + + Inside double quotes the shell expands ``$`` and the backtick and gives the backslash and the double quote a meaning, and nothing else, in sh, bash and zsh alike, so a backslash before each of those four makes the path reach the variable exactly as given. (``!`` is special only to an interactive shell reading a typed line, never to one reading a startup file.) + + :raises AnsibleFilterError: the value is not a string, is empty, is absolute, has a ``..`` component, or has a newline or a NUL, which a startup file line cannot hold. + """ + if not isinstance(value, str): + raise AnsibleFilterError(f"The home-relative path of {name} must be a string, not {type(value).__name__}.") + if any(char in value for char in "\n\r\x00"): + raise AnsibleFilterError(f"The home-relative path of {name} must be one line without a NUL.") + if not value: + raise AnsibleFilterError(f"The home-relative path of {name} must not be empty.") + if value.startswith("/"): + raise AnsibleFilterError(f"The path of {name} must be relative to the user's home directory, not absolute ({value!r}); a fixed absolute path belongs in github_runner_host_tools_shell_env.") + if ".." in value.split("/"): + raise AnsibleFilterError(f"The path of {name} must stay inside the user's home directory, so it must not have a '..' component ({value!r}).") + return '"$HOME/' + re.sub(r'([\\$`"])', r"\\\1", value) + '"' - :param env: a mapping of variable names to string, number or boolean values. :returns: ``[{"name", "pattern", "line"}, ...]`` in the mapping's order, with a dictionary for each variable. :raises AnsibleFilterError: ``env`` is not a mapping, a name is not one the shell accepts, or a value is not a single-line string, a number or a boolean. + +def host_tools_shell_env(env: Any, home_relative: Any) -> list[dict[str, Any]]: + """Return the startup lines that set environment variables in a shell, in the shape host_tools_shell_integrations returns, with ``non_interactive`` true: a variable is for every shell the user's commands run in, and a non-interactive bash returns from ``~/.bashrc`` at the file's interactive guard, so the caller writes these lines ahead of everything else in the file. + + A value in ``env`` is quoted for the shell with ``shlex.quote``, so it reaches the variable exactly as given whatever it holds (spaces, quotes, ``$``, backticks), in zsh and in bash alike. A value in ``home_relative`` is a path below the user's home directory and is written as ``"$HOME/"``, so the shell works out the home directory when the file is read, which the role cannot know when it runs without gathering facts. ``pattern`` is an extended regular expression that matches a line of a startup file that already assigns the variable, outside a comment, whatever its value, so a value the user chose is never overridden, and ``name`` tells the lines apart. + + :param env: a mapping of variable names to string, number or boolean values. :param home_relative: a mapping of variable names to paths relative to the user's home directory. :returns: ``[{"name", "pattern", "line", "non_interactive"}, ...]``, the variables of ``env`` in its order and then those of ``home_relative`` in its. :raises AnsibleFilterError: an argument is not a mapping, a name is not one the shell accepts or is in both mappings, a value of ``env`` is not a single-line string, a number or a boolean, or a path of ``home_relative`` is not a non-empty relative path without a ``..`` component. """ if not isinstance(env, dict): raise AnsibleFilterError(f"The shell environment must be a mapping, not {type(env).__name__}.") - lines: list[dict[str, str]] = [] + if not isinstance(home_relative, dict): + raise AnsibleFilterError(f"The home-relative shell environment must be a mapping, not {type(home_relative).__name__}.") + both = [name for name in env if name in home_relative] + if both: + raise AnsibleFilterError(f"{', '.join(map(str, both))} is set in both the shell environment and the home-relative one; give each variable once.") + lines: list[dict[str, Any]] = [] for name, value in env.items(): - if not isinstance(name, str) or not ENV_NAME.match(name): - raise AnsibleFilterError(f"{name!r} is not a name a shell accepts for an environment variable.") - text = _env_value(name, value) - lines.append({"name": f"env-{name}", "pattern": f"^[[:space:]]*(export[[:space:]]+)?{name}=", "line": f"export {name}={shlex.quote(text)}"}) + lines.append(_export_line(name, shlex.quote(_env_value(_env_name(name), value)))) + for name, value in home_relative.items(): + lines.append(_export_line(name, _home_relative_value(_env_name(name), value))) return lines +def host_tools_block_leads(existing: Any, marker: Any, stem: Any) -> bool: + """Return whether a managed block comes before everything in a startup file that could return early. + + The role writes the blocks a non-interactive shell must read at the top of the file, because a non-interactive bash reads ``~/.bashrc`` only as far as the interactive guard (``case $- in *i*) ;; *) return;; esac``). A block leads the file when every line above its BEGIN marker is blank, a comment, or inside a block of the role's own, which never returns. The test is the file's own order, not a guess at what the guard looks like, so a guard written any other way is handled alike: any code above the block counts as the guard. + + :param existing: the text of the startup file. :param marker: the block's marker line with ``{mark}`` where BEGIN and END go. :param stem: the start of the marker line of every block the role writes in this file, with ``{mark}`` where BEGIN and END go (the marker without the block's name). :returns: whether nothing but blank lines, comments and the role's blocks come above the block. :raises AnsibleFilterError: an argument is not a string, or the file has no block with the marker. + """ + if not (isinstance(existing, str) and isinstance(marker, str) and isinstance(stem, str)): + raise AnsibleFilterError("The startup file's text, the block's marker and the marker stem must be strings.") + begin, own_begin, own_end = marker.replace("{mark}", "BEGIN"), stem.replace("{mark}", "BEGIN"), stem.replace("{mark}", "END") + inside = False + for line in existing.splitlines(): + text = line.rstrip() + if text == begin: + return True + if inside: + inside = not text.startswith(own_end) + elif text.startswith(own_begin): + inside = True + elif text.strip() and not text.lstrip().startswith("#"): + return False + raise AnsibleFilterError(f"The startup file has no block that starts with {begin!r}.") + + def _key(name: Any) -> str: """Return a TOML key: bare when it can be, a quoted string when it has characters a bare key cannot.""" if not isinstance(name, str) or not name: @@ -356,6 +419,7 @@ def filters(self) -> dict[str, Any]: "host_tools_homebrew_prefix": host_tools_homebrew_prefix, "host_tools_shell_integrations": host_tools_shell_integrations, "host_tools_shell_env": host_tools_shell_env, + "host_tools_block_leads": host_tools_block_leads, "host_tools_mise_toml": host_tools_mise_toml, "host_tools_npmrc": host_tools_npmrc, "host_tools_yaml_merge": host_tools_yaml_merge, diff --git a/roles/github_runner_host_tools/README.md b/roles/github_runner_host_tools/README.md index b8bb2733..eacf6001 100644 --- a/roles/github_runner_host_tools/README.md +++ b/roles/github_runner_host_tools/README.md @@ -33,7 +33,7 @@ For each tool the role looks for an equivalent line (`brew shellenv` for Homebre | zsh | `.zshenv`, `.zshrc`, `.zprofile` | `.zshenv` | | bash | `.bashrc`, `.bash_profile`, `.profile` | `.bashrc` | -`.zshenv` is read by every zsh, including the non-interactive one an SSH command starts, and `.bashrc` by the non-interactive bash `sshd` starts, so a command run over SSH finds both tools. Each block is a `blockinfile` block between `ANSIBLE MANAGED BLOCK` marker lines, one per tool, holding one line guarded by `[ -x ... ]` that uses the absolute path of the binary: +`.zshenv` is read by every zsh, including the non-interactive one an SSH command starts ([zsh, Startup Files](https://zsh.sourceforge.io/Doc/Release/Files.html); [Wayback Machine snapshot](https://web.archive.org/web/20261024165218/https://zsh.sourceforge.io/Doc/Release/Files.html)), so a command run over SSH finds both tools on a zsh host. A non-interactive bash reads `.bashrc` only as far as the file's interactive guard, which Debian's and Ubuntu's default `.bashrc` puts near its top, so on a bash host the two lines, which are appended at the end of the file where the interactive setup is, are for interactive shells; the environment variables below are written at the top instead. Each block is a `blockinfile` block between `ANSIBLE MANAGED BLOCK` marker lines, one per tool, holding one line guarded by `[ -x ... ]` that uses the absolute path of the binary: ```sh [ -x /opt/homebrew/bin/brew ] && eval "$(/opt/homebrew/bin/brew shellenv)" @@ -125,14 +125,35 @@ The role sets each key it is given and edits nothing else: every other key, comm ## Shell environment -`github_runner_host_tools_shell_env` is a mapping of environment variable names to string, number or boolean values, for example `UV_PYTHON_DOWNLOADS: never`. Each variable gets its own managed block with one `export` line in the shell's startup file (`.zshenv` for zsh, `.bashrc` for bash, as under Shell integration), so a command run over SSH sees it: +`github_runner_host_tools_shell_env` is a mapping of environment variable names to string, number or boolean values, for example `UV_PYTHON_DOWNLOADS: never`. Each variable gets its own managed block with one `export` line in the shell's startup file (`~/.zshenv` for zsh, `~/.bashrc` for bash, as under Shell integration), written at the top of the file, ahead of anything the user or the distribution has in it, so a command run over SSH sees it: ```sh export UV_PYTHON_DOWNLOADS=never ``` +The top is where a non-interactive shell reads it. `sshd` starts the user's shell non-interactively for an SSH command, and zsh reads `~/.zshenv` in full for every instance. Debian's bash is built to read `~/.bashrc` for a non-interactive shell whose standard input is a network connection, as `sshd` starts ([bash(1) in Debian](https://manpages.debian.org/unstable/bash/bash.1.en.html); [Wayback Machine snapshot](https://web.archive.org/web/20261018084059/https://manpages.debian.org/unstable/bash/bash.1.en.html); upstream bash does so only when built with `SSH_SOURCE_BASHRC`), but the `~/.bashrc` that Debian and Ubuntu give every user starts with `case $- in *i*) ;; *) return;; esac`, which ends the file for that shell, so a block appended below it is never read. The role does not look for the guard, because a guard can be written many ways: a block belongs at the top when everything above it is blank, a comment or another block of the role's, and any other code above it is treated as the guard. A block an earlier version of the role appended lower down is taken out and written again at the top on the next run, with the value it now has, and nothing else in the file moves, so a host converges without anyone touching it. A file that already has the block at the top is not rewritten, and `--check` reports a move as a change and makes none. The startup file is read for this with `no_log`, because it can hold secrets; only the position of the block is worked out from it. A `~/.zshenv` whose variable blocks follow the role's Homebrew and mise blocks is already in order and is left as it is. + The same discipline as the Homebrew and mise lines applies: a variable is added only when none of that shell's startup files assigns it already (`NAME=` or `export NAME=` outside a comment, whatever the value), so a value the user chose is never overridden, and a block the role wrote earlier is kept current when the value changes. The value is quoted for the shell (`shlex.quote`: single quotes, with an embedded single quote written as `'"'"'`), so spaces, quotes, `$`, backticks and `;` reach the variable exactly as given in zsh and bash alike. A name the shell does not accept, and a value of more than one line, fail the run with a message. A boolean is written as `true` or `false`. +### Paths below the home directory + +`github_runner_host_tools_shell_env_home_relative` is a mapping of environment variable names to paths relative to the connecting user's home directory, for a value that has to name that directory and cannot be written as a literal path: the play runs with `gather_facts: false`, so a fleet's variables cannot say "the user's home plus `.cache/turbo`", and `github_runner_host_tools_shell_env` quotes `$HOME` so that it stays literal. + +```yaml +github_runner_host_tools_shell_env_home_relative: + TURBO_CACHE_DIR: .cache/turbo +``` + +writes + +```sh +export TURBO_CACHE_DIR="$HOME/.cache/turbo" +``` + +so the shell works out the home directory when it reads the file, whichever user and home the host has. Inside double quotes the shell expands `$` and the backtick and gives the backslash and the double quote a meaning, in sh, bash and zsh alike, so the role puts a backslash before each of those four characters in the path and it reaches the variable as given (`!` matters only to an interactive shell reading a typed line, not one reading a startup file). The block, its place at the top of the file, the rule that a variable any startup file of the shell assigns already is never added and the `--check` behaviour are those of `github_runner_host_tools_shell_env`, and a variable that is in both mappings fails the run. + +A path that is empty, absolute (it starts with `/`; a fixed path belongs in `github_runner_host_tools_shell_env`), has a `..` component (so the variable cannot point outside the home directory; a name such as `a..b` is fine), is not a string, or holds a newline or a NUL fails the run with a message before any file is changed. + The variables are written whether or not `github_runner_host_tools_shell_integration` is on, and without Homebrew: that variable covers only the Homebrew and mise lines, and a mapping the host gives is the host asking for the variables. A user whose login shell is neither zsh nor bash is left alone with a message. A variable dropped from the mapping keeps its line. ## Variables @@ -146,7 +167,8 @@ The variables are written whether or not `github_runner_host_tools_shell_integra - `github_runner_host_tools_npm_min_release_age_exclude` (default an empty list): package names or patterns written as `min-release-age-exclude[]=NAME` lines in the same block. - `github_runner_host_tools_pnpm_settings` (default an empty mapping): top-level keys and values merged into pnpm's global `config.yaml`. - `github_runner_host_tools_pnpm_minimum_release_age_exclude` (default an empty list): package names or patterns written under `minimumReleaseAgeExclude` in that file. -- `github_runner_host_tools_shell_env` (default an empty mapping): environment variables written as `export` lines in the shell's startup file; see Shell environment above. +- `github_runner_host_tools_shell_env` (default an empty mapping): environment variables written as `export` lines at the top of the shell's startup file; see Shell environment above. +- `github_runner_host_tools_shell_env_home_relative` (default an empty mapping): environment variables whose value is a path relative to the user's home directory, written as `export NAME="$HOME/"`; see Paths below the home directory above. - `github_runner_host_tools_homebrew_prefix` (default empty): the Homebrew prefix. Empty takes the prefix described under Homebrew. Not a setting; a variable only so the tests can point it at a stand-in. - `github_runner_host_tools_homebrew_install_base_url`: where the installer is downloaded from, as `//install.sh`. Not a setting; a variable only so the tests can point it at a stand-in. @@ -170,7 +192,7 @@ github_runner_host_tools_mise_config: ``` ```yaml -# host_vars/example.yml: refuse packages younger than a week and skip install scripts, and keep uv from downloading Pythons +# host_vars/example.yml: refuse packages younger than a week, skip install scripts, keep uv from downloading Pythons and give every repository on a host one turbo cache github_runner_host_tools_npm_settings: min-release-age: 7 ignore-scripts: true @@ -178,8 +200,10 @@ github_runner_host_tools_pnpm_settings: minimumReleaseAge: 10080 github_runner_host_tools_shell_env: UV_PYTHON_DOWNLOADS: never +github_runner_host_tools_shell_env_home_relative: + TURBO_CACHE_DIR: .cache/turbo ``` ## Testing -`tests/unit/test_host_tools.py` covers the filters in `plugins/filter/host_tools.py` (the prefix of each system and architecture, the shell startup lines, and the TOML rendering against golden strings), the defaults against the argument specs and this file, the shape of the pinned commit and checksum, that the only formula the role ever names to Homebrew is mise, the wiring in `playbooks/site.yml`, and the role's tasks (`tests/host_tools/run.yml`) against stand-ins for `brew`, `sudo`, `apt-get`, `dpkg-query`, `getent`, `dscl` and Homebrew's installer on `PATH`, with the installer served from a local directory and every shell startup file in a temporary home. They check that the installer runs non-interactively as the user and only when its checksum matches, that a second run and `--check` change nothing, that root is refused, that a startup file keeps its mode, its content and its symlink, and that `config.toml` is never written. They also check the package-install settings and the shell variables: that nothing is read or written when the variables are empty, that the `.npmrc` block holds only the role's lines, keeps the user's lines and credentials (made-up values) out of the output, is idempotent and shrinks as entries are dropped, that a key the user sets outside the block is kept and reported, that a new `.npmrc` is `0600` and no existing file or symlink changes mode, that the pnpm merge keeps unrelated keys and comments, and that a shell variable's value, quotes and dollar signs included, round-trips through `sh` and `bash` reading the written file. Nothing is downloaded and no real package manager, privilege or startup file is touched. +`tests/unit/test_host_tools.py` covers the filters in `plugins/filter/host_tools.py` (the prefix of each system and architecture, the shell startup lines, and the TOML rendering against golden strings), the defaults against the argument specs and this file, the shape of the pinned commit and checksum, that the only formula the role ever names to Homebrew is mise, the wiring in `playbooks/site.yml`, and the role's tasks (`tests/host_tools/run.yml`) against stand-ins for `brew`, `sudo`, `apt-get`, `dpkg-query`, `getent`, `dscl` and Homebrew's installer on `PATH`, with the installer served from a local directory and every shell startup file in a temporary home. They check that the installer runs non-interactively as the user and only when its checksum matches, that a second run and `--check` change nothing, that root is refused, that a startup file keeps its mode, its content and its symlink, and that `config.toml` is never written. They also check the package-install settings and the shell variables: that nothing is read or written when the variables are empty, that the `.npmrc` block holds only the role's lines, keeps the user's lines and credentials (made-up values) out of the output, is idempotent and shrinks as entries are dropped, that a key the user sets outside the block is kept and reported, that a new `.npmrc` is `0600` and no existing file or symlink changes mode, that the pnpm merge keeps unrelated keys and comments, and that a shell variable's value, quotes and dollar signs included, round-trips through `sh` and `bash` reading the written file. The bash cases run against `tests/host_tools/ubuntu-26.04-skel.bashrc`, a byte-for-byte copy of Ubuntu's default `/etc/skel/.bashrc` (its checksum is asserted), and read the result with a real non-interactive `bash`, which sees a variable only when the block is above the interactive guard; they cover a block an earlier version appended below the guard being moved to the top without a duplicate, a second run and `--check` changing nothing, and a variable the file assigns already being kept. The home-relative variables are read back through real `sh`, `bash` and `zsh` with a home directory whose name has a space, a quote and a dollar sign, and a path with every character special in double quotes, to show that the home directory is expanded when the file is read and nothing else is. Nothing is downloaded and no real package manager, privilege or startup file is touched. diff --git a/roles/github_runner_host_tools/defaults/main.yml b/roles/github_runner_host_tools/defaults/main.yml index 2aa67b8e..381c9466 100644 --- a/roles/github_runner_host_tools/defaults/main.yml +++ b/roles/github_runner_host_tools/defaults/main.yml @@ -25,8 +25,10 @@ github_runner_host_tools_pnpm_settings: {} # Package names or patterns exempt from minimumReleaseAge. A list for minimumReleaseAgeExclude, which replaces whatever list the file holds under that key when this is not empty; do not set the key in github_runner_host_tools_pnpm_settings as well. github_runner_host_tools_pnpm_minimum_release_age_exclude: [] -# Environment variables for the connecting user's shell, as a mapping of names to string, number or boolean values, for example {UV_PYTHON_DOWNLOADS: never}. Each becomes an export line in a managed block in the shell's startup file (.zshenv for zsh, .bashrc for bash), quoted so the value reaches the variable as given, and only when none of that shell's startup files assigns the variable already. Written whether or not github_runner_host_tools_shell_integration is on, since that variable covers only the Homebrew and mise lines. A variable dropped from the mapping keeps its line. +# Environment variables for the connecting user's shell, as a mapping of names to string, number or boolean values, for example {UV_PYTHON_DOWNLOADS: never}. Each becomes an export line in a managed block at the top of the shell's startup file (.zshenv for zsh, .bashrc for bash), above anything else in it, because a non-interactive bash, which is what sshd starts for an SSH command, reads .bashrc only as far as the interactive guard near its top; a block an earlier version appended lower down is moved. Each value is quoted so it reaches the variable as given, and a variable is added only when none of that shell's startup files assigns it already. Written whether or not github_runner_host_tools_shell_integration is on, since that variable covers only the Homebrew and mise lines. A variable dropped from the mapping keeps its line. github_runner_host_tools_shell_env: {} +# Environment variables whose value is a path below the connecting user's home directory, as a mapping of names to paths relative to it, for example {TURBO_CACHE_DIR: .cache/turbo}. The play runs without gathering facts, so a fleet cannot name the home directory itself; each becomes export TURBO_CACHE_DIR="$HOME/.cache/turbo", so the shell works the home directory out when it reads the file. A path that is absolute, empty or has a .. component fails the run; the characters that are special inside double quotes (a backslash, $, a backtick and a double quote) are escaped, so the path reaches the variable as given. Placed, written and skipped exactly as github_runner_host_tools_shell_env is, and a name may be in only one of the two. +github_runner_host_tools_shell_env_home_relative: {} # Not a setting: the Homebrew prefix. Empty takes /opt/homebrew on Apple silicon, /usr/local on Intel and /home/linuxbrew/.linuxbrew on Linux, which are the only places Homebrew supports. It is a variable only so the tests can point it at a stand-in. github_runner_host_tools_homebrew_prefix: "" diff --git a/roles/github_runner_host_tools/meta/argument_specs.yml b/roles/github_runner_host_tools/meta/argument_specs.yml index 7eb63e59..cea146e2 100644 --- a/roles/github_runner_host_tools/meta/argument_specs.yml +++ b/roles/github_runner_host_tools/meta/argument_specs.yml @@ -51,7 +51,11 @@ argument_specs: github_runner_host_tools_shell_env: type: dict default: {} - description: Environment variables for the connecting user's shell, as a mapping of names to string, number or boolean values, each written as a quoted export line in a managed block of the shell's startup file unless a startup file of the shell assigns it already. + description: Environment variables for the connecting user's shell, as a mapping of names to string, number or boolean values, each written as a quoted export line in a managed block at the top of the shell's startup file (above any interactive guard, so an SSH command sees it) unless a startup file of the shell assigns it already. + github_runner_host_tools_shell_env_home_relative: + type: dict + default: {} + description: Environment variables whose value is a path below the connecting user's home directory, as a mapping of names to paths relative to it, each written as export NAME="$HOME/" so the shell expands the home directory when it reads the startup file. An absolute path, an empty one and one with a .. component are refused. Placed and skipped like github_runner_host_tools_shell_env, and a name may be in only one of the two. github_runner_host_tools_homebrew_prefix: type: str default: "" diff --git a/roles/github_runner_host_tools/tasks/main.yml b/roles/github_runner_host_tools/tasks/main.yml index e1357e44..465069e6 100644 --- a/roles/github_runner_host_tools/tasks/main.yml +++ b/roles/github_runner_host_tools/tasks/main.yml @@ -44,11 +44,11 @@ - github_runner_host_tools_homebrew | bool - name: Set up the connecting user's shell - # Homebrew's and mise's lines when the integration is on, and the environment variables of github_runner_host_tools_shell_env, which are written whether or not it is: the mapping is the consumer asking for them. Nothing is read or written when there is neither. + # Homebrew's and mise's lines when the integration is on, and the environment variables of github_runner_host_tools_shell_env and github_runner_host_tools_shell_env_home_relative, which are written whether or not it is: the mappings are the consumer asking for them. Nothing is read or written when there is neither. ansible.builtin.include_tasks: shell_integration.yml when: - github_runner_host_tools_supported | bool - - (github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool) or github_runner_host_tools_shell_env | length > 0 + - (github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool) or github_runner_host_tools_shell_env | length > 0 or github_runner_host_tools_shell_env_home_relative | length > 0 - name: Write the npm package-install security settings ansible.builtin.include_tasks: npm_settings.yml diff --git a/roles/github_runner_host_tools/tasks/shell_integration.yml b/roles/github_runner_host_tools/tasks/shell_integration.yml index 303dc30c..c3bb658d 100644 --- a/roles/github_runner_host_tools/tasks/shell_integration.yml +++ b/roles/github_runner_host_tools/tasks/shell_integration.yml @@ -1,5 +1,5 @@ --- -# Adds Homebrew's shellenv, the mise activation and the environment variables of github_runner_host_tools_shell_env to the connecting user's shell, and only what is missing: a line goes into a startup file only when none of the shell's startup files has an equivalent already, so a host that was set up by hand, or by a dotfiles manager, is left exactly as it is. The user's login shell is read from the account database, never assumed. +# Adds Homebrew's shellenv, the mise activation and the environment variables of github_runner_host_tools_shell_env and github_runner_host_tools_shell_env_home_relative to the connecting user's shell, and only what is missing: a line goes into a startup file only when none of the shell's startup files has an equivalent already, so a host that was set up by hand, or by a dotfiles manager, is left exactly as it is. The user's login shell is read from the account database, never assumed. - name: Read the connecting user's account entry on Linux ansible.builtin.command: @@ -30,6 +30,6 @@ - name: Add what each shell startup file is missing # The list is empty for a shell the role does not set up, because a loop is evaluated before its when and the filter refuses a shell it does not know. Homebrew's and mise's lines come first, only when the integration is on and Homebrew is managed (the prefix is worked out only then), and the environment variables follow. ansible.builtin.include_tasks: shell_integration_line.yml - loop: "{{ (((github_runner_host_tools_effective_prefix | exadev.github_runner.host_tools_shell_integrations(github_runner_host_tools_shell, github_runner_host_tools_homebrew | bool, github_runner_host_tools_mise | bool)) if github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool else []) + (github_runner_host_tools_shell_env | exadev.github_runner.host_tools_shell_env)) if github_runner_host_tools_shell in github_runner_host_tools_shell_files else [] }}" + loop: "{{ (((github_runner_host_tools_effective_prefix | exadev.github_runner.host_tools_shell_integrations(github_runner_host_tools_shell, github_runner_host_tools_homebrew | bool, github_runner_host_tools_mise | bool)) if github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool else []) + (github_runner_host_tools_shell_env | exadev.github_runner.host_tools_shell_env(github_runner_host_tools_shell_env_home_relative))) if github_runner_host_tools_shell in github_runner_host_tools_shell_files else [] }}" loop_control: loop_var: github_runner_host_tools_integration diff --git a/roles/github_runner_host_tools/tasks/shell_integration_line.yml b/roles/github_runner_host_tools/tasks/shell_integration_line.yml index 399b0ff1..6b3ad528 100644 --- a/roles/github_runner_host_tools/tasks/shell_integration_line.yml +++ b/roles/github_runner_host_tools/tasks/shell_integration_line.yml @@ -1,11 +1,11 @@ --- -# One line of shell integration (github_runner_host_tools_integration: its name, the pattern that finds an equivalent line and the line itself), added to the shell's startup file unless a startup file of the shell has an equivalent already. +# One line of shell integration (github_runner_host_tools_integration: its name, the pattern that finds an equivalent line, the line itself and whether a non-interactive shell must read it), added to the shell's startup file unless a startup file of the shell has an equivalent already. A line a non-interactive shell must read goes at the top of the file: a non-interactive bash, which is what sshd starts for a command, reads ~/.bashrc only as far as the interactive guard that distributions put near its top (Debian's and Ubuntu's skeleton .bashrc returns there), so a block appended at the end of the file is never read by it. - name: Work out where the shell's startup files are ansible.builtin.set_fact: github_runner_host_tools_startup_files: "{{ github_runner_host_tools_shell_files[github_runner_host_tools_shell]['read'] | map('regex_replace', '^', ansible_facts['user_dir'] ~ '/') | list }}" github_runner_host_tools_startup_target: "{{ ansible_facts['user_dir'] }}/{{ github_runner_host_tools_shell_files[github_runner_host_tools_shell]['write'] }}" - github_runner_host_tools_marker: "# {mark} ANSIBLE MANAGED BLOCK github_runner_host_tools {{ github_runner_host_tools_integration['name'] }}" + github_runner_host_tools_marker: "{{ github_runner_host_tools_shell_marker_stem }}{{ github_runner_host_tools_integration['name'] }}" - name: Look for an equivalent line in the startup files # grep exits 0 for a match, 1 for none and 2 when it could not read a file, which a startup file that does not exist is; with -q a match is 0 whatever else went wrong. Read-only, so it runs under --check too. @@ -25,11 +25,32 @@ failed_when: github_runner_host_tools_own_block.rc not in [0, 1, 2] check_mode: false +- name: Read the startup file to see where the role's block is + # Only for a line a non-interactive shell must read and a block the role wrote earlier: it may sit below the guard, where an earlier version of the role appended it. no_log keeps the file out of the output and the log, because a startup file can hold secrets, and the filter below returns a boolean only. + ansible.builtin.slurp: + src: "{{ github_runner_host_tools_startup_target }}" + register: github_runner_host_tools_startup_content + no_log: true + check_mode: false + when: github_runner_host_tools_integration['non_interactive'] and github_runner_host_tools_own_block.rc == 0 + +- name: Take the role's block out of where a non-interactive shell does not read + # blockinfile rewrites an existing block where it stands and inserts at the top only a block that is missing, so a block below the guard is removed here and written again at the top by the next task. Once it is at the top this task is skipped, which keeps a second run unchanged. + ansible.builtin.blockinfile: + path: "{{ github_runner_host_tools_startup_target }}" + marker: "{{ github_runner_host_tools_marker }}" + state: absent + when: + - github_runner_host_tools_integration['non_interactive'] + - github_runner_host_tools_own_block.rc == 0 + - not (github_runner_host_tools_startup_content.content | b64decode | exadev.github_runner.host_tools_block_leads(github_runner_host_tools_marker, github_runner_host_tools_shell_marker_stem)) + - name: Add the line to the startup file # noqa: risky-file-permissions - # No mode is passed on purpose, so an existing file keeps the mode it has (blockinfile always edits the target of a symlink, so a startup file that is a link stays one), and a file that does not exist yet gets the user's umask. + # No mode is passed on purpose, so an existing file keeps the mode it has (blockinfile always edits the target of a symlink, so a startup file that is a link stays one), and a file that does not exist yet gets the user's umask. A line a non-interactive shell must read is inserted before the file's first line, and a block already there is updated where it stands. ansible.builtin.blockinfile: path: "{{ github_runner_host_tools_startup_target }}" marker: "{{ github_runner_host_tools_marker }}" block: "{{ github_runner_host_tools_integration['line'] }}" + insertbefore: "{{ 'BOF' if github_runner_host_tools_integration['non_interactive'] else omit }}" create: true when: github_runner_host_tools_equivalent.rc != 0 or github_runner_host_tools_own_block.rc == 0 diff --git a/roles/github_runner_host_tools/vars/main.yml b/roles/github_runner_host_tools/vars/main.yml index 18901937..ca4693ed 100644 --- a/roles/github_runner_host_tools/vars/main.yml +++ b/roles/github_runner_host_tools/vars/main.yml @@ -1,5 +1,5 @@ --- -# Not settings, so not defaults: what the role knows about the shells it supports. Each shell's startup files, in the order the role looks in them for a line it would add, and the one it writes to when none has the line. zsh's .zshenv is read by every zsh, including the non-interactive one an SSH command starts, and bash's .bashrc is read by the non-interactive shell sshd starts, so a command run over SSH finds the tools too. +# Not settings, so not defaults: what the role knows about the shells it supports. Each shell's startup files, in the order the role looks in them for a line it would add, and the one it writes to when none has the line. zsh's .zshenv is read by every zsh, including the non-interactive one an SSH command starts. Debian's bash reads .bashrc for the non-interactive shell sshd starts, but only down to the file's interactive guard, so what a non-interactive shell must see is written at the top of the file. github_runner_host_tools_shell_files: zsh: read: [.zshenv, .zshrc, .zprofile] @@ -8,6 +8,9 @@ github_runner_host_tools_shell_files: read: [.bashrc, .bash_profile, .profile] write: .bashrc +# The start of the marker line of every block the role writes in a shell startup file, with {mark} where BEGIN and END go; the block's own name follows it. A block with such a marker is the role's, and everything else in the file is the user's. +github_runner_host_tools_shell_marker_stem: "# {mark} ANSIBLE MANAGED BLOCK github_runner_host_tools " + # The packages Homebrew's documented requirements name for Debian and Ubuntu (https://docs.brew.sh/Homebrew-on-Linux#requirements). github_runner_host_tools_apt_prerequisites: - build-essential diff --git a/tests/host_tools/ubuntu-26.04-skel.bashrc b/tests/host_tools/ubuntu-26.04-skel.bashrc new file mode 100644 index 00000000..b488fcc4 --- /dev/null +++ b/tests/host_tools/ubuntu-26.04-skel.bashrc @@ -0,0 +1,117 @@ +# ~/.bashrc: executed by bash(1) for non-login shells. +# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc) +# for examples + +# If not running interactively, don't do anything +case $- in + *i*) ;; + *) return;; +esac + +# don't put duplicate lines or lines starting with space in the history. +# See bash(1) for more options +HISTCONTROL=ignoreboth + +# append to the history file, don't overwrite it +shopt -s histappend + +# for setting history length see HISTSIZE and HISTFILESIZE in bash(1) +HISTSIZE=1000 +HISTFILESIZE=2000 + +# check the window size after each command and, if necessary, +# update the values of LINES and COLUMNS. +shopt -s checkwinsize + +# If set, the pattern "**" used in a pathname expansion context will +# match all files and zero or more directories and subdirectories. +#shopt -s globstar + +# make less more friendly for non-text input files, see lesspipe(1) +[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)" + +# set variable identifying the chroot you work in (used in the prompt below) +if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then + debian_chroot=$(cat /etc/debian_chroot) +fi + +# set a fancy prompt (non-color, unless we know we "want" color) +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +# uncomment for a colored prompt, if the terminal has the capability; turned +# off by default to not distract the user: the focus in a terminal window +# should be on the output of commands, not on the prompt +#force_color_prompt=yes + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi + +if [ "$color_prompt" = yes ]; then + PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +else + PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ ' +fi +unset color_prompt force_color_prompt + +# If this is an xterm set the title to user@host:dir +case "$TERM" in +xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; +*) + ;; +esac + +# enable color support of ls and also add handy aliases +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='ls --color=auto' + #alias dir='dir --color=auto' + #alias vdir='vdir --color=auto' + + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi + +# colored GCC warnings and errors +#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01' + +# some more ls aliases +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' + +# Add an "alert" alias for long running commands. Use like so: +# sleep 10; alert +alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' + +# Alias definitions. +# You may want to put all your additions into a separate file like +# ~/.bash_aliases, instead of adding them here directly. +# See /usr/share/doc/bash-doc/examples in the bash-doc package. + +if [ -f ~/.bash_aliases ]; then + . ~/.bash_aliases +fi + +# enable programmable completion features (you don't need to enable +# this, if it's already enabled in /etc/bash.bashrc and /etc/profile +# sources /etc/bash.bashrc). +if ! shopt -oq posix; then + if [ -f /usr/share/bash-completion/bash_completion ]; then + . /usr/share/bash-completion/bash_completion + elif [ -f /etc/bash_completion ]; then + . /etc/bash_completion + fi +fi diff --git a/tests/unit/test_host_tools.py b/tests/unit/test_host_tools.py index aa8b9718..eb3f474e 100644 --- a/tests/unit/test_host_tools.py +++ b/tests/unit/test_host_tools.py @@ -145,7 +145,11 @@ def need_root(): PNPM_SETTINGS = {"minimumReleaseAge": 10080, "saveExact": True} PNPM_USER = "# Mine.\nstoreDir: /somewhere/store # the store\n\n# The age, in minutes.\nminimumReleaseAge: 1440\n\n# Exclusions.\nminimumReleaseAgeExclude:\n- old-one\n\nregistry: https://registry.example.test/\n" PNPM_MERGED = "# Mine.\nstoreDir: /somewhere/store # the store\n\n# The age, in minutes.\nminimumReleaseAge: 10080\n\n# Exclusions.\nminimumReleaseAgeExclude:\n - example-a\n\nregistry: https://registry.example.test/\nsaveExact: true\n" +# The ~/.bashrc every Ubuntu 26.04 user starts from, exactly as /etc/skel/.bashrc has it (package bash 5.3-2ubuntu1, read from two hosts, which agree), so the role is driven against the real structure: its comment header and the interactive guard near the top that returns for a non-interactive shell. SKEL_SHA256 is the checksum of that file, so a copy changed by an editor or a hook (which rejoins hard-wrapped comments) fails loudly. +UBUNTU_BASHRC = ROOT / "tests" / "host_tools" / "ubuntu-26.04-skel.bashrc" +SKEL_SHA256 = "342099da4dd28c394d3f8782d90d7465cb2eaa611193f8f378d6918261cb9bb8" ENV_BLOCK_UV = "# BEGIN ANSIBLE MANAGED BLOCK github_runner_host_tools env-UV_PYTHON_DOWNLOADS\nexport UV_PYTHON_DOWNLOADS=never\n# END ANSIBLE MANAGED BLOCK github_runner_host_tools env-UV_PYTHON_DOWNLOADS\n" +TURBO_BLOCK = "# BEGIN ANSIBLE MANAGED BLOCK github_runner_host_tools env-TURBO_CACHE_DIR\nexport TURBO_CACHE_DIR=\"$HOME/.cache/turbo\"\n# END ANSIBLE MANAGED BLOCK github_runner_host_tools env-TURBO_CACHE_DIR\n" class QuietHandler(http.server.SimpleHTTPRequestHandler): @@ -165,7 +169,7 @@ def mise_block(prefix: Path, shell: str) -> str: class FilterRegistrationTest(unittest.TestCase): def test_the_filters_are_registered_under_the_names_the_role_uses(self) -> None: - self.assertEqual(set(host_tools.FilterModule().filters()), {"host_tools_homebrew_prefix", "host_tools_shell_integrations", "host_tools_shell_env", "host_tools_mise_toml", "host_tools_npmrc", "host_tools_yaml_merge"}) + self.assertEqual(set(host_tools.FilterModule().filters()), {"host_tools_homebrew_prefix", "host_tools_shell_integrations", "host_tools_shell_env", "host_tools_block_leads", "host_tools_mise_toml", "host_tools_npmrc", "host_tools_yaml_merge"}) used = set(re.findall(r"exadev\.github_runner\.(host_tools_\w+)", "".join(path.read_text() for path in (ROLE / "tasks").glob("*.yml")))) self.assertEqual(used, set(host_tools.FilterModule().filters())) @@ -265,9 +269,17 @@ def test_the_installer_is_never_run_as_root(self) -> None: self.assertNotIn("become", task) def test_the_shell_integration_never_sets_a_mode_on_a_startup_file(self) -> None: - [block] = [task["ansible.builtin.blockinfile"] for name, task in self.tasks() if "ansible.builtin.blockinfile" in task and name == "shell_integration_line.yml"] - self.assertNotIn("mode", block) - self.assertIs(block["create"], True) + blocks = [task["ansible.builtin.blockinfile"] for name, task in self.tasks() if "ansible.builtin.blockinfile" in task and name == "shell_integration_line.yml"] + self.assertEqual(len(blocks), 2) + for block in blocks: + self.assertNotIn("mode", block) + [write] = [block for block in blocks if block.get("state", "present") == "present"] + self.assertIs(write["create"], True) + + def test_the_startup_file_is_read_only_to_find_the_block_and_never_shown_or_logged(self) -> None: + [read] = [task for name, task in self.tasks() if name == "shell_integration_line.yml" and "ansible.builtin.slurp" in task] + self.assertIs(read["no_log"], True) + self.assertIs(read["check_mode"], False) def test_the_npm_settings_block_never_sets_a_mode_and_a_missing_npmrc_is_created_private(self) -> None: [block] = [task["ansible.builtin.blockinfile"] for name, task in self.tasks() if "ansible.builtin.blockinfile" in task and name == "npm_settings.yml"] @@ -290,9 +302,9 @@ def test_the_pnpm_config_paths_are_the_ones_pnpm_documents(self) -> None: self.assertEqual(VARS["github_runner_host_tools_pnpm_config_files"], {"Darwin": "Library/Preferences/pnpm/config.yaml", "Linux": ".config/pnpm/config.yaml"}) def test_the_package_install_settings_and_the_shell_environment_write_nothing_by_default(self) -> None: - for name in ("npm_settings", "npm_min_release_age_exclude", "pnpm_settings", "pnpm_minimum_release_age_exclude", "shell_env"): + for name in ("npm_settings", "npm_min_release_age_exclude", "pnpm_settings", "pnpm_minimum_release_age_exclude", "shell_env", "shell_env_home_relative"): with self.subTest(name=name): - self.assertEqual(DEFAULTS[f"github_runner_host_tools_{name}"], {} if name.endswith(("_settings", "_env")) else []) + self.assertEqual(DEFAULTS[f"github_runner_host_tools_{name}"], {} if name.endswith(("_settings", "_env", "_home_relative")) else []) def test_the_mise_configuration_goes_to_conf_d_and_never_to_config_toml(self) -> None: paths = [str(module.get("dest") or module.get("path")) for _, task in self.tasks() for key, module in task.items() if key.startswith("ansible.builtin.") and isinstance(module, dict) and (module.get("dest") or module.get("path"))] @@ -1012,11 +1024,11 @@ def test_the_variables_are_written_whether_or_not_the_shell_integration_is_on(se self.ok(self.run_settings(github_runner_host_tools_shell_integration=integration, github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) self.assertEqual((self.home / ".zshenv").read_text(), ENV_BLOCK_UV) - def test_the_variables_come_with_homebrew_and_mise_lines_in_the_same_file_when_both_are_wanted(self) -> None: + def test_the_variables_come_ahead_of_the_homebrew_and_mise_lines_in_the_same_file_when_both_are_wanted(self) -> None: self.have_homebrew() (self.stub / "mise.state").write_text("current") self.ok(self.run_role(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) - self.assertEqual((self.home / ".zshenv").read_text(), homebrew_block(self.prefix) + mise_block(self.prefix, "zsh") + ENV_BLOCK_UV) + self.assertEqual((self.home / ".zshenv").read_text(), ENV_BLOCK_UV + homebrew_block(self.prefix) + mise_block(self.prefix, "zsh")) def test_a_login_shell_the_role_does_not_set_up_gets_no_variables(self) -> None: (self.stub / "shell").write_text("/usr/bin/fish") @@ -1031,7 +1043,7 @@ def test_an_existing_startup_file_keeps_its_mode_and_its_other_lines(self) -> No startup.chmod(0o600) self.ok(self.run_settings(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) self.assertEqual(self.mode(startup), 0o600) - self.assertEqual(startup.read_text(), "export EXAMPLE=1\n" + ENV_BLOCK_UV) + self.assertEqual(startup.read_text(), ENV_BLOCK_UV + "export EXAMPLE=1\n") def test_a_name_a_shell_would_not_accept_is_refused_before_any_file_is_changed(self) -> None: result = self.run_settings(github_runner_host_tools_shell_env={"NOT-A-NAME": "x"}) @@ -1039,6 +1051,265 @@ def test_a_name_a_shell_would_not_accept_is_refused_before_any_file_is_changed(s self.assertIn("not a name a shell accepts", self.squashed(result)) self.assertEqual(list(self.home.iterdir()), []) + # Where the variables go in a bash startup file: a non-interactive bash, which is what sshd starts for a command, reads ~/.bashrc only as far as the interactive guard. + + def bash_sees(self, name: str, *, interactive: bool = False) -> str: + """The value of a variable in a real bash that sources ~/.bashrc, as sshd's non-interactive bash does, or an interactive one.""" + script = f'. "$1"\nprintf %s "${{{name}-unset}}"' + result = subprocess.run(["bash", *(["-i"] if interactive else []), "-c", script, "bash", str(self.home / ".bashrc")], capture_output=True, text=True, check=True, env={"PATH": "/usr/bin:/bin", "HOME": str(self.home)}) + return result.stdout + + def ubuntu_home(self) -> str: + """Puts the Ubuntu skeleton ~/.bashrc in the home and returns its text.""" + text = UBUNTU_BASHRC.read_text() + (self.home / ".bashrc").write_text(text) + return text + + def run_ubuntu(self, *extra: str, **variables: Any) -> subprocess.CompletedProcess[str]: + return self.run_settings(*extra, system="Linux", os_family="Debian", machine="x86_64", **variables) + + def test_the_ubuntu_skeleton_fixture_is_the_real_file(self) -> None: + self.assertEqual(hashlib.sha256(UBUNTU_BASHRC.read_bytes()).hexdigest(), SKEL_SHA256) + self.assertRegex(UBUNTU_BASHRC.read_text(), r"(?m)^case \$- in\n \*i\*\) ;;\n \*\) return;;\nesac$") + + def test_the_variable_goes_above_the_interactive_guard_of_the_ubuntu_bashrc_and_the_rest_of_the_file_is_untouched(self) -> None: + skeleton = self.ubuntu_home() + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual((self.home / ".bashrc").read_text(), ENV_BLOCK_UV + skeleton) + + def test_a_non_interactive_bash_sees_the_variable_and_an_interactive_one_does_too(self) -> None: + self.ubuntu_home() + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS"), "unset") + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS"), "never") + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS", interactive=True), "never") + + def test_the_homebrew_and_mise_lines_stay_at_the_end_below_the_guard_and_the_variable_is_above_it(self) -> None: + self.have_homebrew() + (self.stub / "mise.state").write_text("current") + skeleton = self.ubuntu_home() + self.ok(self.run_role(system="Linux", os_family="Debian", machine="x86_64", github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual((self.home / ".bashrc").read_text(), ENV_BLOCK_UV + skeleton + homebrew_block(self.prefix) + mise_block(self.prefix, "bash")) + + def test_a_second_run_on_the_ubuntu_bashrc_changes_nothing(self) -> None: + self.ubuntu_home() + variables = {"github_runner_host_tools_shell_env": {"UV_PYTHON_DOWNLOADS": "never", "EXAMPLE_OTHER": "x"}} + self.ok(self.run_ubuntu(**variables)) + after_first = (self.home / ".bashrc").read_text() + result = self.run_ubuntu(**variables) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), after_first) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_block_an_earlier_version_appended_below_the_guard_is_moved_to_the_top_without_a_duplicate(self) -> None: + skeleton = self.ubuntu_home() + (self.home / ".bashrc").write_text(skeleton + ENV_BLOCK_UV) + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS"), "unset") + variables = {"github_runner_host_tools_shell_env": {"UV_PYTHON_DOWNLOADS": "never"}} + result = self.run_ubuntu(**variables) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), ENV_BLOCK_UV + skeleton) + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS"), "never") + result = self.run_ubuntu(**variables) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), ENV_BLOCK_UV + skeleton) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_moved_block_takes_the_new_value_and_the_other_blocks_below_the_guard_stay_where_they_are(self) -> None: + skeleton = self.ubuntu_home() + (self.home / ".bashrc").write_text(skeleton + homebrew_block(self.prefix) + ENV_BLOCK_UV.replace("never", "automatic") + mise_block(self.prefix, "bash")) + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual((self.home / ".bashrc").read_text(), ENV_BLOCK_UV + skeleton + homebrew_block(self.prefix) + mise_block(self.prefix, "bash")) + + def test_check_mode_reports_the_move_and_leaves_the_file_as_it_is(self) -> None: + skeleton = self.ubuntu_home() + (self.home / ".bashrc").write_text(skeleton + ENV_BLOCK_UV) + result = self.run_ubuntu("--check", github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), skeleton + ENV_BLOCK_UV) + self.assertGreaterEqual(self.recap(result)["changed"], 1, result.stdout) + result = self.run_ubuntu("--check", github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) + self.assertEqual((self.home / ".bashrc").read_text(), skeleton + ENV_BLOCK_UV) + + def test_check_mode_on_a_file_with_the_block_at_the_top_changes_nothing(self) -> None: + skeleton = self.ubuntu_home() + (self.home / ".bashrc").write_text(ENV_BLOCK_UV + skeleton) + result = self.run_ubuntu("--check", github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) + self.ok(result) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_variable_the_ubuntu_bashrc_assigns_already_is_not_added_or_overridden(self) -> None: + skeleton = self.ubuntu_home() + for assignment in ("export UV_PYTHON_DOWNLOADS=automatic\n", "UV_PYTHON_DOWNLOADS=automatic\n"): + with self.subTest(assignment=assignment): + (self.home / ".bashrc").write_text(skeleton + assignment) + result = self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), skeleton + assignment) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_guard_written_another_way_is_still_stepped_over(self) -> None: + for guard in ('[ -z "$PS1" ] && return\n', '[[ $- != *i* ]] && return\n', 'case "$-" in *i*) ;; *) return ;; esac\n'): + with self.subTest(guard=guard): + (self.home / ".bashrc").write_text("# mine\n" + guard + "export AFTER=1\n") + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS"), "never") + self.assertEqual(self.bash_sees("AFTER"), "unset") + + def test_a_bashrc_is_not_printed_when_the_block_is_looked_for(self) -> None: + skeleton = self.ubuntu_home() + (self.home / ".bashrc").write_text(skeleton + ENV_BLOCK_UV + f"export EXAMPLE_TOKEN={NPMRC_TOKEN}\n") + result = self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) + self.ok(result) + self.assertNotIn(NPMRC_TOKEN, result.stdout + result.stderr) + self.assertIn(NPMRC_TOKEN, (self.home / ".bashrc").read_text()) + + def test_a_bashrc_that_is_a_symlink_stays_one_and_its_target_is_edited_and_keeps_its_mode(self) -> None: + target = self.work / "dotfiles-bashrc" + target.write_text(UBUNTU_BASHRC.read_text() + ENV_BLOCK_UV) + target.chmod(0o640) + (self.home / ".bashrc").symlink_to(target) + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertTrue((self.home / ".bashrc").is_symlink()) + self.assertEqual(target.read_text(), ENV_BLOCK_UV + UBUNTU_BASHRC.read_text()) + self.assertEqual(self.mode(target), 0o640) + + def test_a_new_bashrc_gets_the_block_alone(self) -> None: + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual((self.home / ".bashrc").read_text(), ENV_BLOCK_UV) + self.assertEqual(self.bash_sees("UV_PYTHON_DOWNLOADS"), "never") + + def test_several_variables_are_all_above_the_guard(self) -> None: + skeleton = self.ubuntu_home() + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env={"EXAMPLE_ONE": "1", "EXAMPLE_TWO": "2", "EXAMPLE_THREE": "3"})) + text = (self.home / ".bashrc").read_text() + self.assertTrue(text.endswith(skeleton)) + self.assertEqual([self.bash_sees(name) for name in ("EXAMPLE_ONE", "EXAMPLE_TWO", "EXAMPLE_THREE")], ["1", "2", "3"]) + + def test_a_zshenv_the_earlier_version_wrote_with_the_variable_after_the_roles_other_blocks_is_left_as_it_is(self) -> None: + self.have_homebrew() + (self.stub / "mise.state").write_text("current") + before = homebrew_block(self.prefix) + mise_block(self.prefix, "zsh") + ENV_BLOCK_UV + (self.home / ".zshenv").write_text(before) + result = self.run_role(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) + self.ok(result) + self.assertEqual((self.home / ".zshenv").read_text(), before) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_zshenv_with_the_block_below_code_of_the_users_gets_it_moved_above_that_code(self) -> None: + (self.home / ".zshenv").write_text("export EXAMPLE=1\n" + ENV_BLOCK_UV) + self.ok(self.run_settings(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"})) + self.assertEqual((self.home / ".zshenv").read_text(), ENV_BLOCK_UV + "export EXAMPLE=1\n") + + # Variables whose value is a path below the user's home directory. + + def shell_sees(self, shell: str, startup: Path, name: str, home: Path) -> str: + """The value of a variable in a real shell that reads a startup file with the given home directory.""" + script = f'. "$1"\nprintf %s "${{{name}-unset}}"' if shell != "zsh" else f'source "$1"\nprintf %s "${{{name}-unset}}"' + result = subprocess.run([shell, "-c", script, shell, str(startup)], capture_output=True, text=True, check=True, env={"PATH": "/usr/bin:/bin:/opt/homebrew/bin", "HOME": str(home)}) + return result.stdout + + def test_a_home_relative_variable_goes_into_zshenv_as_dollar_home_and_a_second_run_changes_nothing(self) -> None: + variables = {"github_runner_host_tools_shell_env_home_relative": {"TURBO_CACHE_DIR": ".cache/turbo"}} + self.ok(self.run_settings(**variables)) + self.assertEqual((self.home / ".zshenv").read_text(), TURBO_BLOCK) + result = self.run_settings(**variables) + self.ok(result) + self.assertEqual((self.home / ".zshenv").read_text(), TURBO_BLOCK) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_the_home_directory_is_worked_out_by_the_shell_that_reads_the_file_not_by_the_role(self) -> None: + elsewhere = self.work / "another user's home" + elsewhere.mkdir() + self.ok(self.run_settings(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"})) + for shell in ("sh", "bash", "zsh"): + with self.subTest(shell=shell): + self.assertEqual(self.shell_sees(shell, self.home / ".zshenv", "TURBO_CACHE_DIR", elsewhere), f"{elsewhere}/.cache/turbo") + self.assertNotIn(str(self.home), (self.home / ".zshenv").read_text()) + + def test_a_path_with_characters_special_in_double_quotes_reaches_the_variable_as_given_through_the_role(self) -> None: + path = 'it\'s "$HOME" `id` $(id) ${HOME} \\n; echo injected' + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env_home_relative={"EXAMPLE_PATH": path})) + for shell in ("sh", "bash", "zsh"): + with self.subTest(shell=shell): + self.assertEqual(self.shell_sees(shell, self.home / ".bashrc", "EXAMPLE_PATH", self.work), f"{self.work}/{path}") + + def test_a_non_interactive_bash_on_the_ubuntu_bashrc_sees_a_home_relative_variable_expanded(self) -> None: + skeleton = self.ubuntu_home() + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"})) + self.assertEqual((self.home / ".bashrc").read_text(), TURBO_BLOCK + skeleton) + self.assertEqual(self.bash_sees("TURBO_CACHE_DIR"), f"{self.home}/.cache/turbo") + self.assertEqual(self.bash_sees("TURBO_CACHE_DIR", interactive=True), f"{self.home}/.cache/turbo") + + def test_both_mappings_are_written_together_above_the_guard_and_a_second_run_changes_nothing(self) -> None: + skeleton = self.ubuntu_home() + variables = {"github_runner_host_tools_shell_env": {"UV_PYTHON_DOWNLOADS": "never"}, "github_runner_host_tools_shell_env_home_relative": {"TURBO_CACHE_DIR": ".cache/turbo"}} + self.ok(self.run_ubuntu(**variables)) + text = (self.home / ".bashrc").read_text() + self.assertTrue(text.endswith(skeleton)) + self.assertEqual(sorted([text[: -len(skeleton)].count(ENV_BLOCK_UV), text[: -len(skeleton)].count(TURBO_BLOCK)]), [1, 1]) + self.assertEqual((self.bash_sees("UV_PYTHON_DOWNLOADS"), self.bash_sees("TURBO_CACHE_DIR")), ("never", f"{self.home}/.cache/turbo")) + result = self.run_ubuntu(**variables) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), text) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_home_relative_variable_the_bashrc_assigns_already_is_kept_as_it_is(self) -> None: + skeleton = self.ubuntu_home() + own = "# One turbo cache for every repository on this machine.\nexport TURBO_CACHE_DIR=\"$HOME/.cache/turbo\"\n" + (self.home / ".bashrc").write_text(own + skeleton) + result = self.run_ubuntu(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/elsewhere"}) + self.ok(result) + self.assertEqual((self.home / ".bashrc").read_text(), own + skeleton) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_changed_home_relative_path_updates_the_roles_block_in_place(self) -> None: + skeleton = self.ubuntu_home() + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/old"})) + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"})) + self.assertEqual((self.home / ".bashrc").read_text(), TURBO_BLOCK + skeleton) + + def test_a_home_relative_block_below_the_guard_is_moved_to_the_top(self) -> None: + skeleton = self.ubuntu_home() + (self.home / ".bashrc").write_text(skeleton + TURBO_BLOCK) + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"})) + self.assertEqual((self.home / ".bashrc").read_text(), TURBO_BLOCK + skeleton) + + def test_the_home_relative_variables_alone_are_enough_to_write_the_startup_file(self) -> None: + result = self.run_settings("--check", github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"}) + self.ok(result) + self.assertEqual(list(self.home.iterdir()), []) + self.assertGreaterEqual(self.recap(result)["changed"], 1, result.stdout) + + def test_a_host_the_role_does_not_support_gets_no_home_relative_variable(self) -> None: + self.ok(self.run_settings(system="FreeBSD", os_family="FreeBSD", github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"})) + self.assertEqual(list(self.home.iterdir()), []) + + def test_a_login_shell_the_role_does_not_set_up_gets_no_home_relative_variable(self) -> None: + (self.stub / "shell").write_text("/usr/bin/fish") + self.ok(self.run_settings(github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"})) + self.assertEqual(list(self.home.iterdir()), []) + + def test_a_path_that_is_not_below_the_home_directory_is_refused_before_any_file_is_changed(self) -> None: + cases = {"absolute": ("/var/cache/turbo", "relative to the user's home"), "parent": ("../turbo", "'..' component"), "empty": ("", "must not be empty"), "number": (3, "must be a string"), "newline": ("a\nb", "one line")} + for label, (path, message) in cases.items(): + with self.subTest(label): + result = self.run_settings(github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}, github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": path}) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, self.squashed(result)) + self.assertEqual(list(self.home.iterdir()), []) + + def test_a_variable_in_both_mappings_is_refused_before_any_file_is_changed(self) -> None: + result = self.run_settings(github_runner_host_tools_shell_env={"TURBO_CACHE_DIR": "/x"}, github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("set in both", self.squashed(result)) + self.assertEqual(list(self.home.iterdir()), []) + + def test_a_home_relative_mapping_of_the_wrong_type_is_refused_by_the_argument_specs(self) -> None: + result = self.run_settings(github_runner_host_tools_shell_env_home_relative=[".cache/turbo"]) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(list(self.home.iterdir()), []) + def test_check_mode_reports_a_variable_and_writes_nothing(self) -> None: result = self.run_settings("--check", github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) self.ok(result) diff --git a/tests/unit/test_host_tools_filters.py b/tests/unit/test_host_tools_filters.py index 2f58de7f..673a43b4 100644 --- a/tests/unit/test_host_tools_filters.py +++ b/tests/unit/test_host_tools_filters.py @@ -1,4 +1,4 @@ -"""Tests the filters of the github_runner_host_tools role in plugins/filter/host_tools.py: the Homebrew prefix a host has, the shell startup lines that bring Homebrew and mise into a shell or set environment variables in it, mise's configuration rendered to TOML, the lines of the managed block of an npmrc file and a YAML file merged with settings, against golden strings.""" +"""Tests the filters of the github_runner_host_tools role in plugins/filter/host_tools.py: the Homebrew prefix a host has, the shell startup lines that bring Homebrew and mise into a shell or set environment variables in it, whether a managed block leads its startup file, mise's configuration rendered to TOML, the lines of the managed block of an npmrc file and a YAML file merged with settings, against golden strings.""" from __future__ import annotations @@ -43,8 +43,8 @@ def test_each_tool_gets_a_guarded_line_and_a_pattern_for_an_equivalent(self) -> self.assertEqual( host_tools.host_tools_shell_integrations("/opt/homebrew", "zsh", True, True), [ - {"name": "homebrew", "pattern": "^[^#]*brew shellenv", "line": '[ -x /opt/homebrew/bin/brew ] && eval "$(/opt/homebrew/bin/brew shellenv)"'}, - {"name": "mise", "pattern": "^[^#]*mise activate", "line": '[ -x /opt/homebrew/bin/mise ] && eval "$(/opt/homebrew/bin/mise activate zsh)"'}, + {"name": "homebrew", "pattern": "^[^#]*brew shellenv", "line": '[ -x /opt/homebrew/bin/brew ] && eval "$(/opt/homebrew/bin/brew shellenv)"', "non_interactive": False}, + {"name": "mise", "pattern": "^[^#]*mise activate", "line": '[ -x /opt/homebrew/bin/mise ] && eval "$(/opt/homebrew/bin/mise activate zsh)"', "non_interactive": False}, ], ) @@ -124,35 +124,35 @@ def grep_finds(pattern: str, text: str) -> bool: class ShellEnvFilterTest(unittest.TestCase): def test_a_variable_becomes_a_quoted_export_line_and_a_pattern_that_finds_an_assignment_of_it(self) -> None: - [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}) - self.assertEqual(item, {"name": "env-UV_PYTHON_DOWNLOADS", "pattern": "^[[:space:]]*(export[[:space:]]+)?UV_PYTHON_DOWNLOADS=", "line": "export UV_PYTHON_DOWNLOADS=never"}) + [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}, {}) + self.assertEqual(item, {"name": "env-UV_PYTHON_DOWNLOADS", "pattern": "^[[:space:]]*(export[[:space:]]+)?UV_PYTHON_DOWNLOADS=", "line": "export UV_PYTHON_DOWNLOADS=never", "non_interactive": True}) def test_the_variables_come_out_in_the_order_of_the_mapping(self) -> None: - self.assertEqual([item["name"] for item in host_tools.host_tools_shell_env({"B": "1", "A": "2"})], ["env-B", "env-A"]) + self.assertEqual([item["name"] for item in host_tools.host_tools_shell_env({"B": "1", "A": "2"}, {})], ["env-B", "env-A"]) def test_an_empty_mapping_has_no_lines(self) -> None: - self.assertEqual(host_tools.host_tools_shell_env({}), []) + self.assertEqual(host_tools.host_tools_shell_env({}, {}), []) def test_a_value_reaches_the_variable_exactly_as_given_whatever_it_holds(self) -> None: values = ["it's", 'say "hi"', "$HOME", "${HOME}", "`id`", "$(id)", "a b c", "back\\slash", "tab\there", "*", "; echo injected", "", "-n", "'", "''", "\"'\"", "é"] for value in values: with self.subTest(value=value): - [item] = host_tools.host_tools_shell_env({"EXAMPLE": value}) + [item] = host_tools.host_tools_shell_env({"EXAMPLE": value}, {}) for shell in ("sh", "bash"): result = subprocess.run([shell, "-c", item["line"] + '\nprintf %s "$EXAMPLE"'], capture_output=True, text=True, check=True, env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent-home"}) self.assertEqual(result.stdout, value) self.assertEqual(result.stderr, "") def test_a_dollar_sign_and_a_single_quote_are_quoted_and_not_expanded(self) -> None: - self.assertEqual(host_tools.host_tools_shell_env({"A": "$HOME"})[0]["line"], "export A='$HOME'") - self.assertEqual(host_tools.host_tools_shell_env({"A": "it's"})[0]["line"], "export A='it'\"'\"'s'") + self.assertEqual(host_tools.host_tools_shell_env({"A": "$HOME"}, {})[0]["line"], "export A='$HOME'") + self.assertEqual(host_tools.host_tools_shell_env({"A": "it's"}, {})[0]["line"], "export A='it'\"'\"'s'") def test_a_boolean_is_true_or_false_and_a_number_is_its_digits(self) -> None: - lines = {item["name"]: item["line"] for item in host_tools.host_tools_shell_env({"A": True, "B": False, "C": 3, "D": 0.5})} + lines = {item["name"]: item["line"] for item in host_tools.host_tools_shell_env({"A": True, "B": False, "C": 3, "D": 0.5}, {})} self.assertEqual(lines, {"env-A": "export A=true", "env-B": "export B=false", "env-C": "export C=3", "env-D": "export D=0.5"}) def test_the_pattern_finds_an_assignment_with_or_without_export_and_not_a_comment_or_another_variable(self) -> None: - [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}) + [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}, {}) for text in ("UV_PYTHON_DOWNLOADS=automatic", "export UV_PYTHON_DOWNLOADS=automatic", " export UV_PYTHON_DOWNLOADS='x'", "\texport UV_PYTHON_DOWNLOADS="): with self.subTest(text=text): self.assertTrue(grep_finds(item["pattern"], text)) @@ -175,7 +175,125 @@ def test_what_a_shell_would_not_take_is_refused(self) -> None: } for label, (env, message) in cases.items(): with self.subTest(label), self.assertRaisesRegex(Exception, message): - host_tools.host_tools_shell_env(env) + host_tools.host_tools_shell_env(env, {}) + + +def shell_reads(shell: str, line: str, home: str) -> str: + """The value of EXAMPLE after a real shell reads a startup line, with the given home directory.""" + result = subprocess.run([shell, "-c", line + '\nprintf %s "$EXAMPLE"'], capture_output=True, text=True, check=True, env={"PATH": "/usr/bin:/bin:/opt/homebrew/bin", "HOME": home}) + assert result.stderr == "", result.stderr + return result.stdout + + +class HomeRelativeFilterTest(unittest.TestCase): + def test_a_path_becomes_an_export_of_the_home_directory_and_the_path_in_double_quotes(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {"TURBO_CACHE_DIR": ".cache/turbo"}) + self.assertEqual(item, {"name": "env-TURBO_CACHE_DIR", "pattern": "^[[:space:]]*(export[[:space:]]+)?TURBO_CACHE_DIR=", "line": 'export TURBO_CACHE_DIR="$HOME/.cache/turbo"', "non_interactive": True}) + + def test_the_variables_of_both_mappings_come_out_the_plain_ones_first_each_in_its_own_order(self) -> None: + items = host_tools.host_tools_shell_env({"B": "1", "A": "2"}, {"D": "d", "C": "c"}) + self.assertEqual([item["name"] for item in items], ["env-B", "env-A", "env-D", "env-C"]) + + def test_a_path_reaches_the_variable_as_given_in_every_shell_whatever_it_holds_and_the_home_is_expanded_once(self) -> None: + paths = [".cache/turbo", "a b c", "it's", 'say "hi"', "$HOME", "${HOME}", "`id`", "$(id)", "back\\slash", "\\$", "tab\there", "*", "?", "[a]", "; echo injected", "-n", "~", "~/x", "!x", "a..b", "..x", "x..", ".hidden/.also", "dir/", "./x", "é", "'", "\""] + homes = ["/home/example", "/home/a b", "/home/it's", '/home/"q"', "/home/$d"] + for path in paths: + [item] = host_tools.host_tools_shell_env({}, {"EXAMPLE": path}) + for home in homes: + for shell in ("sh", "bash", "zsh"): + with self.subTest(path=path, home=home, shell=shell): + self.assertEqual(shell_reads(shell, item["line"], home), f"{home}/{path}") + + def test_the_special_characters_are_escaped_and_nothing_else_is(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {"A": 'x\\y$z`w"v !*~'}) + self.assertEqual(item["line"], 'export A="$HOME/x\\\\y\\$z\\`w\\"v !*~"') + + def test_the_pattern_finds_an_assignment_of_the_variable_already_in_a_startup_file(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {"TURBO_CACHE_DIR": ".cache/turbo"}) + self.assertTrue(grep_finds(item["pattern"], 'export TURBO_CACHE_DIR="$HOME/.cache/turbo"')) + self.assertTrue(grep_finds(item["pattern"], "TURBO_CACHE_DIR=/elsewhere")) + self.assertFalse(grep_finds(item["pattern"], "# export TURBO_CACHE_DIR=/elsewhere")) + + def test_an_empty_mapping_has_no_lines(self) -> None: + self.assertEqual(host_tools.host_tools_shell_env({}, {}), []) + + def test_what_is_not_a_path_below_the_home_directory_is_refused(self) -> None: + cases = { + "absolute": ({"A": "/var/cache"}, "relative to the user's home"), + "the root": ({"A": "/"}, "relative to the user's home"), + "empty": ({"A": ""}, "must not be empty"), + "a parent component": ({"A": "../x"}, "'..' component"), + "a parent component in the middle": ({"A": "a/../b"}, "'..' component"), + "a parent component at the end": ({"A": "a/.."}, "'..' component"), + "only a parent component": ({"A": ".."}, "'..' component"), + "a newline": ({"A": "a\nb"}, "one line"), + "a carriage return": ({"A": "a\rb"}, "one line"), + "a NUL": ({"A": "a\x00b"}, "one line"), + "a number": ({"A": 3}, "must be a string"), + "a boolean": ({"A": True}, "must be a string"), + "a list": ({"A": ["a"]}, "must be a string"), + "a null": ({"A": None}, "must be a string"), + "a name with a hyphen": ({"A-B": "x"}, "not a name a shell accepts"), + "a name that is not a string": ({1: "x"}, "not a name a shell accepts"), + } + for label, (home_relative, message) in cases.items(): + with self.subTest(label), self.assertRaisesRegex(Exception, message): + host_tools.host_tools_shell_env({}, home_relative) + + def test_a_mapping_that_is_not_one_is_refused(self) -> None: + with self.assertRaisesRegex(Exception, "home-relative shell environment must be a mapping"): + host_tools.host_tools_shell_env({}, [".cache"]) + + def test_a_variable_in_both_mappings_is_refused_and_named(self) -> None: + with self.assertRaisesRegex(Exception, "A is set in both"): + host_tools.host_tools_shell_env({"A": "x"}, {"A": "y"}) + + +STEM = "# {mark} ANSIBLE MANAGED BLOCK example " +OWN = "# BEGIN ANSIBLE MANAGED BLOCK example env-A\nexport A=1\n# END ANSIBLE MANAGED BLOCK example env-A\n" +OTHER = OWN.replace("env-A", "env-B").replace("A=1", "B=2") + + +class BlockLeadsFilterTest(unittest.TestCase): + def leads(self, text: str) -> bool: + return bool(host_tools.host_tools_block_leads(text, STEM + "env-A", STEM)) + + def test_a_block_at_the_start_of_the_file_leads(self) -> None: + self.assertTrue(self.leads(OWN + "export X=1\n")) + + def test_blank_lines_and_comments_above_the_block_do_not_matter(self) -> None: + self.assertTrue(self.leads("#!/bin/bash\n\n # a note\n" + OWN)) + + def test_other_blocks_of_the_role_above_it_do_not_matter_whatever_they_hold(self) -> None: + self.assertTrue(self.leads(OTHER + OWN)) + self.assertTrue(self.leads(OTHER.replace("export B=2", "[ -z \"$PS1\" ] && return") + OWN)) + + def test_any_code_above_the_block_means_it_does_not_lead(self) -> None: + for code in ("export X=1\n", "case $- in\n *i*) ;;\n *) return;;\nesac\n", "[ -z \"$PS1\" ] && return\n", " return\n"): + with self.subTest(code=code): + self.assertFalse(self.leads("# note\n" + code + OWN)) + self.assertFalse(self.leads(OTHER + code + OWN)) + + def test_the_real_ubuntu_skeleton_with_the_block_appended_does_not_lead_and_with_it_prepended_does(self) -> None: + skeleton = (ROOT / "tests" / "host_tools" / "ubuntu-26.04-skel.bashrc").read_text() + self.assertFalse(self.leads(skeleton + OWN)) + self.assertTrue(self.leads(OWN + skeleton)) + + def test_blocks_below_it_do_not_matter(self) -> None: + self.assertTrue(self.leads(OTHER + OWN + OTHER.replace("B", "C"))) + + def test_the_block_is_matched_by_its_whole_marker_line(self) -> None: + text = OWN.replace("env-A", "env-AB") + with self.assertRaisesRegex(Exception, "no block"): + host_tools.host_tools_block_leads(text, STEM + "env-A", STEM) + + def test_a_file_without_the_block_is_refused(self) -> None: + with self.assertRaisesRegex(Exception, "no block"): + host_tools.host_tools_block_leads("# note\n", STEM + "env-A", STEM) + + def test_what_is_not_text_is_refused(self) -> None: + with self.assertRaisesRegex(Exception, "must be strings"): + host_tools.host_tools_block_leads(None, STEM + "env-A", STEM) MARKER = "# {mark} ANSIBLE MANAGED BLOCK example"