From 50ba88b137161e1970b7a8a00da0e40bb20537ee Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 3 Nov 2026 13:40:53 +0000 Subject: [PATCH] feat(host-tools): add variables that are a share of the host's CPUs, for the shell and for mise A worker count has to follow the size of the host, and tools such as vitest read their variable as an integer, so a percentage would be read as a count. The fleet cannot state the count either: it has hosts of several sizes and the play gathers no facts. github_runner_host_tools_shell_env_cpu_share and github_runner_host_tools_mise_env_cpu_share map a variable name to an integer divisor of at least 1. The value is the logical CPU count divided by it, rounded down and never below 1, worked out when the shell or mise reads the file. The shell runs getconf _NPROCESSORS_ONLN in a subshell that leaves no variable behind, and mise evaluates a template over its num_cpus(). The template divides the count less its remainder, because Tera's int filter refuses a float that is not whole, and floors with an if, because Tera has no max filter. Both forms share the divisor check and the check that a variable is given once across the plain, home-relative and CPU-share mappings. The tests read the shell line back through real sh, bash and zsh with a stand-in getconf, and the template through a real mise with a stand-in count, for counts and divisors including a single CPU, and check that the two agree. --- plugins/filter/host_tools.py | 71 ++++-- roles/github_runner_host_tools/README.md | 47 +++- .../defaults/main.yml | 4 + .../meta/argument_specs.yml | 8 + roles/github_runner_host_tools/tasks/main.yml | 4 +- .../tasks/mise_config.yml | 4 +- .../tasks/shell_integration.yml | 4 +- .../github_runner_host_tools/tasks/tools.yml | 2 +- tests/unit/test_host_tools.py | 109 ++++++++- tests/unit/test_host_tools_filters.py | 209 +++++++++++++++--- 10 files changed, 399 insertions(+), 63 deletions(-) diff --git a/plugins/filter/host_tools.py b/plugins/filter/host_tools.py index e67908f5..ca9a0934 100644 --- a/plugins/filter/host_tools.py +++ b/plugins/filter/host_tools.py @@ -146,14 +146,46 @@ def _home_relative_path(name: str, value: Any) -> str: return value -def _reject_names_in_both(env: dict[Any, Any], home_relative: dict[Any, Any]) -> None: - """Raise unless every variable is given once, as a plain value or as a path below the home directory. +def _reject_repeated_names(*mappings: dict[Any, Any]) -> None: + """Raise unless every variable is given once across the mappings: as a plain value, as a path below the home directory or as a share of the CPUs. - :raises AnsibleFilterError: a name is in both mappings. + :raises AnsibleFilterError: a name is in more than one mapping. """ - both = [name for name in env if name in home_relative] - if both: - raise AnsibleFilterError(f"{', '.join(map(str, both))} is set both as a plain value and as a home-relative path; give each variable once.") + seen: set[Any] = set() + repeated: list[Any] = [] + for mapping in mappings: + repeated += [name for name in mapping if name in seen] + seen.update(mapping) + if repeated: + raise AnsibleFilterError(f"{', '.join(map(str, repeated))} is given more than once, as a plain value, a home-relative path or a CPU share; give each variable once.") + + +def _cpu_divisor(name: str, value: Any) -> int: + """Return the divisor of a variable that is a share of the host's logical CPUs. + + :raises AnsibleFilterError: the value is not an integer of at least 1. A boolean is not one, though Python counts it as an integer, and neither is a float or a string. + """ + if isinstance(value, bool) or not isinstance(value, int): + raise AnsibleFilterError(f"The CPU divisor of {name} must be an integer, not {value!r}.") + if value < 1: + raise AnsibleFilterError(f"The CPU divisor of {name} must be at least 1, not {value}.") + return value + + +def _cpu_share_shell(divisor: int) -> str: + """Return the shell word that is the number of logical CPUs the host has online divided by ``divisor``, rounded down and never below 1. + + The count is ``getconf _NPROCESSORS_ONLN``, which POSIX hosts, macOS and Linux alike, have, and the arithmetic runs in a command substitution so that no variable is left behind in the user's shell. It is the same in sh, bash and zsh. + """ + return f'"$(n=$(getconf _NPROCESSORS_ONLN); n=$((n / {divisor})); echo $((n > 0 ? n : 1)))"' + + +def _cpu_share_tera(divisor: int) -> str: + """Return the mise template that is the number of logical CPUs divided by ``divisor``, rounded down and never below 1. + + ``num_cpus()`` returns a string, so it needs ``int`` before any arithmetic. Tera's ``/`` gives a float and its ``int`` filter refuses a float that is not whole (mise reports "received an unexpected type" for 12 / 8), so the division is of a multiple of the divisor: the count less its remainder, which is a whole number whatever the count. Tera has no ``max`` filter, so the floor of 1 is an ``if``. The template is the role's own and holds no ``$``, so it needs none of the escaping of a user's value. + """ + return f"{{% set c = num_cpus() | int %}}{{% set n = ((c - c % {divisor}) / {divisor}) | int %}}{{% if n < 1 %}}1{{% else %}}{{{{ n }}}}{{% endif %}}" def _double_quoted(text: str) -> str: @@ -171,23 +203,27 @@ def _home_relative_value(name: str, value: Any) -> str: return '"$HOME/' + _double_quoted(_home_relative_path(name, value)) + '"' -def host_tools_shell_env(env: Any, home_relative: Any) -> list[dict[str, Any]]: +def host_tools_shell_env(env: Any, home_relative: Any, cpu_share: Any) -> list[dict[str, Any]]: """Return the startup lines that set environment variables in a shell, in the shape host_tools_shell_integrations returns, with ``non_interactive`` true: a variable is for every shell the user's commands run in, and a non-interactive bash returns from ``~/.bashrc`` at the file's interactive guard, so the caller writes these lines ahead of everything else in the file. A value in ``env`` is quoted for the shell with ``shlex.quote``, so it reaches the variable exactly as given whatever it holds (spaces, quotes, ``$``, backticks), in zsh and in bash alike. A value in ``home_relative`` is a path below the user's home directory and is written as ``"$HOME/"``, so the shell works out the home directory when the file is read, which the role cannot know when it runs without gathering facts. ``pattern`` is an extended regular expression that matches a line of a startup file that already assigns the variable, outside a comment, whatever its value, so a value the user chose is never overridden, and ``name`` tells the lines apart. - :param env: a mapping of variable names to string, number or boolean values. :param home_relative: a mapping of variable names to paths relative to the user's home directory. :returns: ``[{"name", "pattern", "line", "non_interactive"}, ...]``, the variables of ``env`` in its order and then those of ``home_relative`` in its. :raises AnsibleFilterError: an argument is not a mapping, a name is not one the shell accepts or is in both mappings (a name in both is refused by the same check as in ``host_tools_mise_toml``), a value of ``env`` is not a single-line string, a number or a boolean, or a path of ``home_relative`` is not a non-empty relative path without a ``..`` component. + :param env: a mapping of variable names to string, number or boolean values. :param home_relative: a mapping of variable names to paths relative to the user's home directory. :param cpu_share: a mapping of variable names to an integer divisor of at least 1, each variable being the host's logical CPU count divided by it, rounded down and never below 1, worked out by the shell each time it reads the line. :returns: ``[{"name", "pattern", "line", "non_interactive"}, ...]``, the variables of ``env`` in its order, then those of ``home_relative`` and then those of ``cpu_share``, each in its own. :raises AnsibleFilterError: an argument is not a mapping, a name is not one the shell accepts or is in more than one mapping (refused by the same check as in ``host_tools_mise_toml``), a divisor is not an integer of at least 1, a value of ``env`` is not a single-line string, a number or a boolean, or a path of ``home_relative`` is not a non-empty relative path without a ``..`` component. """ if not isinstance(env, dict): raise AnsibleFilterError(f"The shell environment must be a mapping, not {type(env).__name__}.") if not isinstance(home_relative, dict): raise AnsibleFilterError(f"The home-relative shell environment must be a mapping, not {type(home_relative).__name__}.") - _reject_names_in_both(env, home_relative) + if not isinstance(cpu_share, dict): + raise AnsibleFilterError(f"The CPU-share shell environment must be a mapping, not {type(cpu_share).__name__}.") + _reject_repeated_names(env, home_relative, cpu_share) lines: list[dict[str, Any]] = [] for name, value in env.items(): lines.append(_export_line(name, shlex.quote(_env_value(_env_name(name), value)))) for name, value in home_relative.items(): lines.append(_export_line(name, _home_relative_value(_env_name(name), value))) + for name, value in cpu_share.items(): + lines.append(_export_line(name, _cpu_share_shell(_cpu_divisor(_env_name(name), value)))) return lines @@ -276,27 +312,30 @@ def _mise_literal(text: str) -> str: return re.sub(r"\{(?=[{%#])", "{{ '{' }}", text.replace("$", "$$")) -def _env_table(env: Any, home_relative: Any) -> list[str]: +def _env_table(env: Any, home_relative: Any, cpu_share: Any) -> list[str]: """Return the lines of mise's ``[env]`` table, empty when neither mapping has an entry. - A plain value is read back as given. A home-relative path is written as ``{{ env.HOME }}/``, so mise works the home directory out when it reads the file, which the role cannot know when it runs without gathering facts. + A plain value is read back as given. A home-relative path is written as ``{{ env.HOME }}/``, so mise works the home directory out when it reads the file, which the role cannot know when it runs without gathering facts. A CPU share is a template that works the host's logical CPU count out when mise reads the file, divided by the divisor, rounded down and never below 1. - :raises AnsibleFilterError: a name is not one a shell accepts (a mise variable is exported to commands run by shells), is in both mappings, a plain value is not a single-line string, a number or a boolean, or a path is not a non-empty relative path without a ``..`` component. + :raises AnsibleFilterError: a name is not one a shell accepts (a mise variable is exported to commands run by shells), is in more than one mapping, a divisor is not an integer of at least 1, a plain value is not a single-line string, a number or a boolean, or a path is not a non-empty relative path without a ``..`` component. """ if not isinstance(env, dict): raise AnsibleFilterError(f"env must be a mapping, not {type(env).__name__}.") if not isinstance(home_relative, dict): raise AnsibleFilterError(f"The home-relative env must be a mapping, not {type(home_relative).__name__}.") - _reject_names_in_both(env, home_relative) + if not isinstance(cpu_share, dict): + raise AnsibleFilterError(f"The CPU-share env must be a mapping, not {type(cpu_share).__name__}.") + _reject_repeated_names(env, home_relative, cpu_share) lines = [f"{_env_name(name)} = {_string(_mise_literal(_env_value(name, value)))}" for name, value in env.items()] lines += [f"{_env_name(name)} = {_string('{{ env.HOME }}/' + _mise_literal(_home_relative_path(name, value)))}" for name, value in home_relative.items()] + lines += [f"{_env_name(name)} = {_string(_cpu_share_tera(_cpu_divisor(name, value)))}" for name, value in cpu_share.items()] return ["[env]", *lines] if lines else [] -def host_tools_mise_toml(config: Any, env_home_relative: Any) -> str: +def host_tools_mise_toml(config: Any, env_home_relative: Any, env_cpu_share: Any) -> str: """Return a mise configuration file for a mapping of its tables. - :param config: a mapping with the tables ``tools`` (a tool name to a version string or a list of them), ``settings`` (a setting name to a string, number or boolean, or a list of those) and ``env`` (a variable name to a string, number or boolean, read back by mise as given). A table that is empty is left out. :param env_home_relative: a mapping of variable names to paths relative to the user's home directory, written into the ``env`` table as mise's ``{{ env.HOME }}`` template followed by the path. :returns: the TOML text, each table after a blank line and the whole ending in a newline, in the order tools, settings and env whatever the order of the mapping. :raises AnsibleFilterError: an argument is not a mapping, ``config`` has a table the role does not render or a value of a kind its table does not take, a variable is in both ``config['env']`` and ``env_home_relative``, a path is absolute, empty or has a ``..`` component, or there is nothing to render. + :param config: a mapping with the tables ``tools`` (a tool name to a version string or a list of them), ``settings`` (a setting name to a string, number or boolean, or a list of those) and ``env`` (a variable name to a string, number or boolean, read back by mise as given). A table that is empty is left out. :param env_home_relative: a mapping of variable names to paths relative to the user's home directory, written into the ``env`` table as mise's ``{{ env.HOME }}`` template followed by the path. :param env_cpu_share: a mapping of variable names to an integer divisor of at least 1, written into the ``env`` table as a template for the host's logical CPU count divided by it, rounded down and never below 1. :returns: the TOML text, each table after a blank line and the whole ending in a newline, in the order tools, settings and env whatever the order of the mapping. :raises AnsibleFilterError: an argument is not a mapping, ``config`` has a table the role does not render or a value of a kind its table does not take, a variable is in more than one of ``config['env']``, ``env_home_relative`` and ``env_cpu_share``, a path is absolute, empty or has a ``..`` component, a divisor is not an integer of at least 1, or there is nothing to render. """ if not isinstance(config, dict): raise AnsibleFilterError(f"The mise configuration must be a mapping, not {type(config).__name__}.") @@ -304,7 +343,7 @@ def host_tools_mise_toml(config: Any, env_home_relative: Any) -> str: if unknown: raise AnsibleFilterError(f"The mise configuration has the tables {', '.join(map(repr, unknown))}; the role renders only {', '.join(MISE_TABLES)}.") tables = {name: _table(name, config[name]) for name in MISE_TABLES if name in config and name != "env"} - tables["env"] = _env_table(config.get("env", {}), env_home_relative) + tables["env"] = _env_table(config.get("env", {}), env_home_relative, env_cpu_share) blocks = [tables[name] for name in MISE_TABLES if tables.get(name)] if not blocks: raise AnsibleFilterError("The mise configuration has no entry in any table, so there is nothing to render.") diff --git a/roles/github_runner_host_tools/README.md b/roles/github_runner_host_tools/README.md index cf817bf2..e7833e5f 100644 --- a/roles/github_runner_host_tools/README.md +++ b/roles/github_runner_host_tools/README.md @@ -105,7 +105,25 @@ renders to TURBO_CACHE_DIR = "{{ env.HOME }}/.cache/turbo" ``` -so mise works the home directory out when it reads the file, whichever user and home the host has. The path reaches the variable exactly as given (the same escaping as for a plain value, applied to the path). The rules are those of `github_runner_host_tools_shell_env_home_relative`, whose validation the role shares: a path that is empty, absolute, has a `..` component, is not a string or holds a newline or a NUL fails the run before anything is written, and a variable that is both a plain `env` value and here fails it too. With `github_runner_host_tools_mise_config` empty, this mapping alone makes the role write the file. A fleet that wants one value in the shell's startup file and in mise gives it to both `github_runner_host_tools_shell_env_home_relative` and this mapping: the shell file serves commands run by a shell, and mise's table serves commands run by `mise exec`, `mise run` and shims. +so mise works the home directory out when it reads the file, whichever user and home the host has. The path reaches the variable exactly as given (the same escaping as for a plain value, applied to the path). The rules are those of `github_runner_host_tools_shell_env_home_relative`, whose validation the role shares: a path that is empty, absolute, has a `..` component, is not a string or holds a newline or a NUL fails the run before anything is written, and a variable that is also a plain `env` value or a CPU share fails it too. With `github_runner_host_tools_mise_config` empty, this mapping alone makes the role write the file. A fleet that wants one value in the shell's startup file and in mise gives it to both `github_runner_host_tools_shell_env_home_relative` and this mapping: the shell file serves commands run by a shell, and mise's table serves commands run by `mise exec`, `mise run` and shims. + +### A share of the CPUs + +`github_runner_host_tools_mise_env_cpu_share` is a mapping of variable names to an integer divisor, for a value that depends on the size of the host, such as a worker count. Each variable is the host's logical CPU count divided by its divisor, rounded down and never below 1, worked out when mise reads the file: + +```yaml +github_runner_host_tools_mise_env_cpu_share: + VITEST_MAX_WORKERS: 2 +``` + +renders to + +```toml +[env] +VITEST_MAX_WORKERS = "{% set c = num_cpus() | int %}{% set n = ((c - c % 2) / 2) | int %}{% if n < 1 %}1{% else %}{{ n }}{% endif %}" +``` + +which is 5 on a 10-CPU host and 1 on a 1-CPU host. It is a number and not a percentage because tools read these variables as integers: vitest 5 reads `VITEST_MAX_WORKERS` with `Number.parseInt`, so `50%` would be 50 workers. mise's `num_cpus()` returns a string, so the template converts it with `int` before dividing (Tera refuses arithmetic on a string). Tera's `int` filter also refuses a float that is not whole, so the template divides the count less its remainder (a whole number) rather than the count itself, and Tera has no `max` filter, so the floor of 1 is an `if`. The divisor must be an integer of at least 1 (a boolean, a float, a string, zero and a negative number fail the run), and a variable may be in only one of the three mappings that write the `env` table. The template is the role's own text, so none of the escaping of a plain value applies to it. A fleet that wants the value in the shell's startup file and in mise gives it to both this mapping and `github_runner_host_tools_shell_env_cpu_share`. Any other table, or a value of a kind its table does not take, fails the run with a message rather than being written in a form mise might read differently. @@ -188,10 +206,27 @@ writes export TURBO_CACHE_DIR="$HOME/.cache/turbo" ``` -so the shell works out the home directory when it reads the file, whichever user and home the host has. Inside double quotes the shell expands `$` and the backtick and gives the backslash and the double quote a meaning, in sh, bash and zsh alike, so the role puts a backslash before each of those four characters in the path and it reaches the variable as given (`!` matters only to an interactive shell reading a typed line, not one reading a startup file). The block, its place at the top of the file, the rule that a variable any startup file of the shell assigns already is never added and the `--check` behaviour are those of `github_runner_host_tools_shell_env`, and a variable that is in both mappings fails the run. +so the shell works out the home directory when it reads the file, whichever user and home the host has. Inside double quotes the shell expands `$` and the backtick and gives the backslash and the double quote a meaning, in sh, bash and zsh alike, so the role puts a backslash before each of those four characters in the path and it reaches the variable as given (`!` matters only to an interactive shell reading a typed line, not one reading a startup file). The block, its place at the top of the file, the rule that a variable any startup file of the shell assigns already is never added and the `--check` behaviour are those of `github_runner_host_tools_shell_env`, and a variable that is in more than one of the shell mappings fails the run. A path that is empty, absolute (it starts with `/`; a fixed path belongs in `github_runner_host_tools_shell_env`), has a `..` component (so the variable cannot point outside the home directory; a name such as `a..b` is fine), is not a string, or holds a newline or a NUL fails the run with a message before any file is changed. +### A share of the CPUs + +`github_runner_host_tools_shell_env_cpu_share` is a mapping of environment variable names to an integer divisor, for a value that depends on the size of the host. Each variable is the host's logical CPU count divided by its divisor, rounded down and never below 1, worked out by the shell each time it reads the file: + +```yaml +github_runner_host_tools_shell_env_cpu_share: + VITEST_MAX_WORKERS: 2 +``` + +writes + +```sh +export VITEST_MAX_WORKERS="$(n=$(getconf _NPROCESSORS_ONLN); n=$((n / 2)); echo $((n > 0 ? n : 1)))" +``` + +`getconf _NPROCESSORS_ONLN` is the count of logical CPUs online, and macOS and Linux both have it (POSIX). The arithmetic runs in a command substitution, so no variable is left behind in the user's shell, and the line is the same in sh, bash and zsh. The value is a number and not a percentage because tools read these variables as integers (see the mise section above). The block is placed at the top of the file and skipped when a startup file assigns the variable already, as for the other two mappings. The line starts one `getconf` when a shell reads the file, which for bash on a Debian host is every SSH command. The divisor must be an integer of at least 1, and a variable may be in only one of the three shell mappings. + The variables are written whether or not `github_runner_host_tools_shell_integration` is on, and without Homebrew: that variable covers only the Homebrew and mise lines, and a mapping the host gives is the host asking for the variables. A user whose login shell is neither zsh nor bash is left alone with a message. A variable dropped from the mapping keeps its line. ## Variables @@ -208,6 +243,8 @@ The variables are written whether or not `github_runner_host_tools_shell_integra - `github_runner_host_tools_pnpm_minimum_release_age_exclude` (default an empty list): package names or patterns written under `minimumReleaseAgeExclude` in that file. - `github_runner_host_tools_shell_env` (default an empty mapping): environment variables written as `export` lines at the top of the shell's startup file; see Shell environment above. - `github_runner_host_tools_shell_env_home_relative` (default an empty mapping): environment variables whose value is a path relative to the user's home directory, written as `export NAME="$HOME/"`; see Paths below the home directory above. +- `github_runner_host_tools_shell_env_cpu_share` (default an empty mapping): environment variables whose value is a share of the host's logical CPUs, written as an `export` line whose value the shell works out; see A share of the CPUs under Shell environment above. +- `github_runner_host_tools_mise_env_cpu_share` (default an empty mapping): the same for mise's `env` table; see A share of the CPUs under Global mise defaults above. - `github_runner_host_tools_homebrew_prefix` (default empty): the Homebrew prefix. Empty takes the prefix described under Homebrew. Not a setting; a variable only so the tests can point it at a stand-in. - `github_runner_host_tools_homebrew_install_base_url`: where the installer is downloaded from, as `//install.sh`. Not a setting; a variable only so the tests can point it at a stand-in. @@ -243,8 +280,12 @@ github_runner_host_tools_shell_env_home_relative: TURBO_CACHE_DIR: .cache/turbo github_runner_host_tools_mise_env_home_relative: TURBO_CACHE_DIR: .cache/turbo +github_runner_host_tools_shell_env_cpu_share: + VITEST_MAX_WORKERS: 2 +github_runner_host_tools_mise_env_cpu_share: + VITEST_MAX_WORKERS: 2 ``` ## Testing -`tests/unit/test_host_tools.py` covers the filters in `plugins/filter/host_tools.py` (the prefix of each system and architecture, the shell startup lines, and the TOML rendering against golden strings), the defaults against the argument specs and this file, the shape of the pinned commit and checksum, that the only formula the role ever names to Homebrew is mise, the wiring in `playbooks/site.yml`, and the role's tasks (`tests/host_tools/run.yml`) against stand-ins for `brew`, `sudo`, `apt-get`, `dpkg-query`, `getent`, `dscl` and Homebrew's installer on `PATH`, with the installer served from a local directory and every shell startup file in a temporary home. They check that the installer runs non-interactively as the user and only when its checksum matches, that a second run and `--check` change nothing, that root is refused, that a startup file keeps its mode, its content and its symlink, and that `config.toml` is never written. The mise `env` table is read back with a real `mise env`, which has to be on `PATH` (the CI job installs it), for values with `$`, template braces, quotes and a home directory whose name has a space, a quote and a dollar sign. They also check the package-install settings and the shell variables: that nothing is read or written when the variables are empty, that the `.npmrc` block holds only the role's lines, keeps the user's lines and credentials (made-up values) out of the output, is idempotent and shrinks as entries are dropped, that a key the user sets outside the block is kept and reported, that a new `.npmrc` is `0600` and no existing file or symlink changes mode, that the pnpm merge keeps unrelated keys and comments, and that a shell variable's value, quotes and dollar signs included, round-trips through `sh` and `bash` reading the written file. A non-interactive `bash` sourcing the Ubuntu `.bashrc` fixture finds `brew`, `mise` and a shim after a run and finds none before it, and the `PATH` line is read back by `sh` and `bash` under prefixes and home directories holding a space, quotes and dollar signs. The bash cases run against `tests/host_tools/ubuntu-26.04-skel.bashrc`, a byte-for-byte copy of Ubuntu's default `/etc/skel/.bashrc` (its checksum is asserted), and read the result with a real non-interactive `bash`, which sees a variable only when the block is above the interactive guard; they cover a block an earlier version appended below the guard being moved to the top without a duplicate, a second run and `--check` changing nothing, and a variable the file assigns already being kept. The home-relative variables are read back through real `sh`, `bash` and `zsh` with a home directory whose name has a space, a quote and a dollar sign, and a path with every character special in double quotes, to show that the home directory is expanded when the file is read and nothing else is. Nothing is downloaded and no real package manager, privilege or startup file is touched. +`tests/unit/test_host_tools.py` covers the filters in `plugins/filter/host_tools.py` (the prefix of each system and architecture, the shell startup lines, and the TOML rendering against golden strings), the defaults against the argument specs and this file, the shape of the pinned commit and checksum, that the only formula the role ever names to Homebrew is mise, the wiring in `playbooks/site.yml`, and the role's tasks (`tests/host_tools/run.yml`) against stand-ins for `brew`, `sudo`, `apt-get`, `dpkg-query`, `getent`, `dscl` and Homebrew's installer on `PATH`, with the installer served from a local directory and every shell startup file in a temporary home. They check that the installer runs non-interactively as the user and only when its checksum matches, that a second run and `--check` change nothing, that root is refused, that a startup file keeps its mode, its content and its symlink, and that `config.toml` is never written. The mise `env` table is read back with a real `mise env`, which has to be on `PATH` (the CI job installs it), for values with `$`, template braces, quotes and a home directory whose name has a space, a quote and a dollar sign. They also check the package-install settings and the shell variables: that nothing is read or written when the variables are empty, that the `.npmrc` block holds only the role's lines, keeps the user's lines and credentials (made-up values) out of the output, is idempotent and shrinks as entries are dropped, that a key the user sets outside the block is kept and reported, that a new `.npmrc` is `0600` and no existing file or symlink changes mode, that the pnpm merge keeps unrelated keys and comments, and that a shell variable's value, quotes and dollar signs included, round-trips through `sh` and `bash` reading the written file. A non-interactive `bash` sourcing the Ubuntu `.bashrc` fixture finds `brew`, `mise` and a shim after a run and finds none before it, and the `PATH` line is read back by `sh` and `bash` under prefixes and home directories holding a space, quotes and dollar signs. The bash cases run against `tests/host_tools/ubuntu-26.04-skel.bashrc`, a byte-for-byte copy of Ubuntu's default `/etc/skel/.bashrc` (its checksum is asserted), and read the result with a real non-interactive `bash`, which sees a variable only when the block is above the interactive guard; they cover a block an earlier version appended below the guard being moved to the top without a duplicate, a second run and `--check` changing nothing, and a variable the file assigns already being kept. The home-relative variables are read back through real `sh`, `bash` and `zsh` with a home directory whose name has a space, a quote and a dollar sign, and a path with every character special in double quotes, to show that the home directory is expanded when the file is read and nothing else is. The CPU-share lines are read back through real `sh`, `bash` and `zsh` with a stand-in `getconf` on `PATH` that reports 1, 2, 3, 10 and 64 CPUs, and the mise template through a real `mise env` with `num_cpus()` replaced by the same counts as strings, so the floor of 1 and the rounding are checked at sizes the test host does not have; the real `num_cpus()` is checked against the host's own count. Nothing is downloaded and no real package manager, privilege or startup file is touched. diff --git a/roles/github_runner_host_tools/defaults/main.yml b/roles/github_runner_host_tools/defaults/main.yml index 36474b5b..6c1a03b0 100644 --- a/roles/github_runner_host_tools/defaults/main.yml +++ b/roles/github_runner_host_tools/defaults/main.yml @@ -16,6 +16,8 @@ github_runner_host_tools_homebrew_install_sha256: 5f333bbe53bc490e51e7ccb1df8779 github_runner_host_tools_mise_config: {} # Variables for mise's env table whose value is a path below the connecting user's home directory, as a mapping of names to paths relative to it, for example {TURBO_CACHE_DIR: .cache/turbo}. The play runs without gathering facts, so a fleet cannot name the home directory itself; each is written as mise's template for the home directory followed by the path, so mise works it out when it reads the file. The same rules as github_runner_host_tools_shell_env_home_relative: a path that is absolute, empty or has a .. component fails the run, and a name may not also be in the env table of github_runner_host_tools_mise_config. Empty writes nothing by itself. github_runner_host_tools_mise_env_home_relative: {} +# Variables for mise's env table whose value is a share of the host's logical CPUs, as a mapping of names to an integer divisor of at least 1, for example {VITEST_MAX_WORKERS: 2} for half. Written as a mise template that divides the CPU count (mise's num_cpus()) by the divisor, rounds down and never goes below 1, worked out when mise reads the file. The same rules as github_runner_host_tools_shell_env_cpu_share, and a name may not also be in the env table of github_runner_host_tools_mise_config or in github_runner_host_tools_mise_env_home_relative. Empty writes nothing by itself. +github_runner_host_tools_mise_env_cpu_share: {} # Package-install security settings for npm and pnpm. All empty by default, and empty they write nothing, so a host that sets none of them sees no change. # npm settings, as a mapping of npmrc keys to string, number or boolean values, written to the connecting user's ~/.npmrc inside one block between ANSIBLE MANAGED BLOCK marker lines, for example {min-release-age: 7, ignore-scripts: true} (min-release-age is in days). A key the file already sets outside the block is the user's own: it is kept, the role's value is not written, and the run says so. Dropping a key here, or all of them, removes its line, and the block with the last one. The file can hold credentials, so the role never prints it, never changes the mode of an existing one, and creates a missing one with mode 0600. @@ -31,6 +33,8 @@ github_runner_host_tools_pnpm_minimum_release_age_exclude: [] github_runner_host_tools_shell_env: {} # Environment variables whose value is a path below the connecting user's home directory, as a mapping of names to paths relative to it, for example {TURBO_CACHE_DIR: .cache/turbo}. The play runs without gathering facts, so a fleet cannot name the home directory itself; each becomes export TURBO_CACHE_DIR="$HOME/.cache/turbo", so the shell works the home directory out when it reads the file. A path that is absolute, empty or has a .. component fails the run; the characters that are special inside double quotes (a backslash, $, a backtick and a double quote) are escaped, so the path reaches the variable as given. Placed, written and skipped exactly as github_runner_host_tools_shell_env is, and a name may be in only one of the two. github_runner_host_tools_shell_env_home_relative: {} +# Environment variables whose value is a share of the host's logical CPUs, as a mapping of names to an integer divisor of at least 1, for example {VITEST_MAX_WORKERS: 2} for half. The value is the CPU count (getconf _NPROCESSORS_ONLN, on macOS and Linux alike) divided by the divisor, rounded down and never below 1, worked out by the shell each time it reads the startup file, so one fleet value fits hosts of any size. It is a number, not a percentage, because tools such as vitest read their worker-count variable as an integer only. A divisor that is not an integer of at least 1 fails the run. Placed, written and skipped exactly as github_runner_host_tools_shell_env is, and a name may be in only one of the shell mappings. +github_runner_host_tools_shell_env_cpu_share: {} # Not a setting: the Homebrew prefix. Empty takes /opt/homebrew on Apple silicon, /usr/local on Intel and /home/linuxbrew/.linuxbrew on Linux, which are the only places Homebrew supports. It is a variable only so the tests can point it at a stand-in. github_runner_host_tools_homebrew_prefix: "" diff --git a/roles/github_runner_host_tools/meta/argument_specs.yml b/roles/github_runner_host_tools/meta/argument_specs.yml index 7d069837..9c5b251f 100644 --- a/roles/github_runner_host_tools/meta/argument_specs.yml +++ b/roles/github_runner_host_tools/meta/argument_specs.yml @@ -34,6 +34,10 @@ argument_specs: type: dict default: {} description: Variables for mise's env table whose value is a path below the connecting user's home directory, as a mapping of names to paths relative to it, written as mise's template for the home directory followed by the path. An absolute path, an empty one and one with a .. component are refused, and a name may not also be in the env table of github_runner_host_tools_mise_config. + github_runner_host_tools_mise_env_cpu_share: + type: dict + default: {} + description: Variables for mise's env table whose value is a share of the host's logical CPUs, as a mapping of names to an integer divisor of at least 1, written as a template for the CPU count divided by it, rounded down and never below 1. A name may not also be in the env table of github_runner_host_tools_mise_config or in github_runner_host_tools_mise_env_home_relative. github_runner_host_tools_npm_settings: type: dict default: {} @@ -60,6 +64,10 @@ argument_specs: type: dict default: {} description: Environment variables whose value is a path below the connecting user's home directory, as a mapping of names to paths relative to it, each written as export NAME="$HOME/" so the shell expands the home directory when it reads the startup file. An absolute path, an empty one and one with a .. component are refused. Placed and skipped like github_runner_host_tools_shell_env, and a name may be in only one of the two. + github_runner_host_tools_shell_env_cpu_share: + type: dict + default: {} + description: Environment variables whose value is a share of the host's logical CPUs, as a mapping of names to an integer divisor of at least 1, each written as an export line whose value the shell works out as the CPU count divided by it, rounded down and never below 1. Placed and skipped like github_runner_host_tools_shell_env, and a name may be in only one of the shell mappings. github_runner_host_tools_homebrew_prefix: type: str default: "" diff --git a/roles/github_runner_host_tools/tasks/main.yml b/roles/github_runner_host_tools/tasks/main.yml index 465069e6..b0f2f1b9 100644 --- a/roles/github_runner_host_tools/tasks/main.yml +++ b/roles/github_runner_host_tools/tasks/main.yml @@ -44,11 +44,11 @@ - github_runner_host_tools_homebrew | bool - name: Set up the connecting user's shell - # Homebrew's and mise's lines when the integration is on, and the environment variables of github_runner_host_tools_shell_env and github_runner_host_tools_shell_env_home_relative, which are written whether or not it is: the mappings are the consumer asking for them. Nothing is read or written when there is neither. + # Homebrew's and mise's lines when the integration is on, and the environment variables of github_runner_host_tools_shell_env, github_runner_host_tools_shell_env_home_relative and github_runner_host_tools_shell_env_cpu_share, which are written whether or not it is: the mappings are the consumer asking for them. Nothing is read or written when there is neither. ansible.builtin.include_tasks: shell_integration.yml when: - github_runner_host_tools_supported | bool - - (github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool) or github_runner_host_tools_shell_env | length > 0 or github_runner_host_tools_shell_env_home_relative | length > 0 + - (github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool) or github_runner_host_tools_shell_env | length > 0 or github_runner_host_tools_shell_env_home_relative | length > 0 or github_runner_host_tools_shell_env_cpu_share | length > 0 - name: Write the npm package-install security settings ansible.builtin.include_tasks: npm_settings.yml diff --git a/roles/github_runner_host_tools/tasks/mise_config.yml b/roles/github_runner_host_tools/tasks/mise_config.yml index f4d91bbc..509851f1 100644 --- a/roles/github_runner_host_tools/tasks/mise_config.yml +++ b/roles/github_runner_host_tools/tasks/mise_config.yml @@ -1,5 +1,5 @@ --- -# Writes the global mise defaults to conf.d, never to config.toml: that file is the host's own and takes precedence over conf.d, so the role neither writes nor deletes it. Only reached when there is something to write; with github_runner_host_tools_mise_config and github_runner_host_tools_mise_env_home_relative both empty, any existing file is left alone. The content is a filter's output, which Ansible does not template again, so the braces of mise's own templates in it reach the file as they are. +# Writes the global mise defaults to conf.d, never to config.toml: that file is the host's own and takes precedence over conf.d, so the role neither writes nor deletes it. Only reached when there is something to write; with github_runner_host_tools_mise_config, github_runner_host_tools_mise_env_home_relative and github_runner_host_tools_mise_env_cpu_share all empty, any existing file is left alone. The content is a filter's output, which Ansible does not template again, so the braces of mise's own templates in it reach the file as they are. - name: Look for the conf.d directory ansible.builtin.stat: @@ -16,7 +16,7 @@ - name: Write the defaults # Under --check the directory above was not made if it was missing, and copy refuses a destination whose directory is not there, so that case is reported below. ansible.builtin.copy: - content: "# Managed by Ansible (exadev.github_runner.github_runner_host_tools). Edits to this file are overwritten.\n{{ github_runner_host_tools_mise_config | exadev.github_runner.host_tools_mise_toml(github_runner_host_tools_mise_env_home_relative) }}" + content: "# Managed by Ansible (exadev.github_runner.github_runner_host_tools). Edits to this file are overwritten.\n{{ github_runner_host_tools_mise_config | exadev.github_runner.host_tools_mise_toml(github_runner_host_tools_mise_env_home_relative, github_runner_host_tools_mise_env_cpu_share) }}" dest: "{{ ansible_facts['user_dir'] }}/.config/mise/conf.d/00-fleet.toml" mode: "0644" when: github_runner_host_tools_conf_dir.stat.exists or not ansible_check_mode diff --git a/roles/github_runner_host_tools/tasks/shell_integration.yml b/roles/github_runner_host_tools/tasks/shell_integration.yml index bbcf550d..84457d4b 100644 --- a/roles/github_runner_host_tools/tasks/shell_integration.yml +++ b/roles/github_runner_host_tools/tasks/shell_integration.yml @@ -1,5 +1,5 @@ --- -# Adds Homebrew's shellenv, the mise activation and the environment variables of github_runner_host_tools_shell_env and github_runner_host_tools_shell_env_home_relative to the connecting user's shell, and only what is missing: a line goes into a startup file only when none of the shell's startup files has an equivalent already, so a host that was set up by hand, or by a dotfiles manager, is left exactly as it is. The user's login shell is read from the account database, never assumed. +# Adds Homebrew's shellenv, the mise activation and the environment variables of github_runner_host_tools_shell_env, github_runner_host_tools_shell_env_home_relative and github_runner_host_tools_shell_env_cpu_share to the connecting user's shell, and only what is missing: a line goes into a startup file only when none of the shell's startup files has an equivalent already, so a host that was set up by hand, or by a dotfiles manager, is left exactly as it is. The user's login shell is read from the account database, never assumed. - name: Read the connecting user's account entry on Linux ansible.builtin.command: @@ -30,6 +30,6 @@ - name: Add what each shell startup file is missing # The list is empty for a shell the role does not set up, because a loop is evaluated before its when and the filter refuses a shell it does not know. Homebrew's and mise's lines come first (on bash, with the PATH block for a non-interactive shell, which is written at the top of the file), only when the integration is on and Homebrew is managed (the prefix is worked out only then), and the environment variables follow. ansible.builtin.include_tasks: shell_integration_line.yml - loop: "{{ (((github_runner_host_tools_effective_prefix | exadev.github_runner.host_tools_shell_integrations(github_runner_host_tools_shell, github_runner_host_tools_homebrew | bool, github_runner_host_tools_mise | bool)) if github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool else []) + (github_runner_host_tools_shell_env | exadev.github_runner.host_tools_shell_env(github_runner_host_tools_shell_env_home_relative))) if github_runner_host_tools_shell in github_runner_host_tools_shell_files else [] }}" + loop: "{{ (((github_runner_host_tools_effective_prefix | exadev.github_runner.host_tools_shell_integrations(github_runner_host_tools_shell, github_runner_host_tools_homebrew | bool, github_runner_host_tools_mise | bool)) if github_runner_host_tools_shell_integration | bool and github_runner_host_tools_homebrew | bool else []) + (github_runner_host_tools_shell_env | exadev.github_runner.host_tools_shell_env(github_runner_host_tools_shell_env_home_relative, github_runner_host_tools_shell_env_cpu_share))) if github_runner_host_tools_shell in github_runner_host_tools_shell_files else [] }}" loop_control: loop_var: github_runner_host_tools_integration diff --git a/roles/github_runner_host_tools/tasks/tools.yml b/roles/github_runner_host_tools/tasks/tools.yml index 3b1d92c0..0166ca5e 100644 --- a/roles/github_runner_host_tools/tasks/tools.yml +++ b/roles/github_runner_host_tools/tasks/tools.yml @@ -37,4 +37,4 @@ ansible.builtin.include_tasks: mise_config.yml when: - github_runner_host_tools_mise | bool - - github_runner_host_tools_mise_config | length > 0 or github_runner_host_tools_mise_env_home_relative | length > 0 + - github_runner_host_tools_mise_config | length > 0 or github_runner_host_tools_mise_env_home_relative | length > 0 or github_runner_host_tools_mise_env_cpu_share | length > 0 diff --git a/tests/unit/test_host_tools.py b/tests/unit/test_host_tools.py index 972bc544..dd70ea7a 100644 --- a/tests/unit/test_host_tools.py +++ b/tests/unit/test_host_tools.py @@ -820,7 +820,7 @@ def test_a_variable_in_both_the_env_table_and_the_home_relative_mapping_fails_th self.have_homebrew() result = self.run_role(github_runner_host_tools_shell_integration=False, github_runner_host_tools_mise_config={"env": {"TURBO_CACHE_DIR": "/x"}}, github_runner_host_tools_mise_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"}) self.assertNotEqual(result.returncode, 0) - self.assertIn("TURBO_CACHE_DIR is set both", self.squashed(result)) + self.assertIn("TURBO_CACHE_DIR is given more than once", self.squashed(result)) self.assertFalse(self.fleet_file().exists()) def test_the_home_relative_paths_are_not_written_when_mise_is_left_out(self) -> None: @@ -1396,7 +1396,7 @@ def test_a_path_that_is_not_below_the_home_directory_is_refused_before_any_file_ def test_a_variable_in_both_mappings_is_refused_before_any_file_is_changed(self) -> None: result = self.run_settings(github_runner_host_tools_shell_env={"TURBO_CACHE_DIR": "/x"}, github_runner_host_tools_shell_env_home_relative={"TURBO_CACHE_DIR": ".cache/turbo"}) self.assertNotEqual(result.returncode, 0) - self.assertIn("set both as a plain value", self.squashed(result)) + self.assertIn("is given more than once", self.squashed(result)) self.assertEqual(list(self.home.iterdir()), []) def test_a_home_relative_mapping_of_the_wrong_type_is_refused_by_the_argument_specs(self) -> None: @@ -1547,6 +1547,111 @@ def test_zsh_gets_no_path_block_because_zshenv_is_read_by_every_zsh(self) -> Non self.ok(self.run_role()) self.assertNotIn("github_runner_host_tools path", (self.home / ".zshenv").read_text()) + # Variables that are a share of the host's logical CPUs. + + def stand_in_cpus(self, cpus: int) -> str: + """A PATH, with a getconf that reports ``cpus`` first, for a real shell to read a startup file with.""" + directory = self.work / "cpu-stand-in" + directory.mkdir(exist_ok=True) + (directory / "getconf").write_text(f'#!/bin/sh\n[ "$1" = _NPROCESSORS_ONLN ] || exit 2\necho {cpus}\n') + (directory / "getconf").chmod(0o755) + return f"{directory}:/usr/bin:/bin" + + def shell_sees_with_cpus(self, shell: str, startup: Path, name: str, cpus: int) -> str: + script = f'. "$1"\nprintf %s "${{{name}-unset}}"' if shell != "zsh" else f'source "$1"\nprintf %s "${{{name}-unset}}"' + return subprocess.run([shell, "-c", script, shell, str(startup)], capture_output=True, text=True, check=True, env={"PATH": self.stand_in_cpus(cpus), "HOME": str(self.home)}).stdout + + def test_a_cpu_share_goes_into_zshenv_and_a_second_run_changes_nothing(self) -> None: + variables = {"github_runner_host_tools_shell_env_cpu_share": {"VITEST_MAX_WORKERS": 2}} + self.ok(self.run_settings(**variables)) + written = (self.home / ".zshenv").read_text() + self.assertEqual(written, "# BEGIN ANSIBLE MANAGED BLOCK github_runner_host_tools env-VITEST_MAX_WORKERS\nexport VITEST_MAX_WORKERS=\"$(n=$(getconf _NPROCESSORS_ONLN); n=$((n / 2)); echo $((n > 0 ? n : 1)))\"\n# END ANSIBLE MANAGED BLOCK github_runner_host_tools env-VITEST_MAX_WORKERS\n") + result = self.run_settings(**variables) + self.ok(result) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + self.assertEqual((self.home / ".zshenv").read_text(), written) + + def test_the_written_startup_file_gives_half_the_cpus_in_a_real_shell_whatever_the_host_has(self) -> None: + self.ok(self.run_settings(github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 2})) + for cpus, expected in ((1, "1"), (2, "1"), (3, "1"), (10, "5"), (11, "5"), (64, "32")): + for shell in ("sh", "bash", "zsh"): + with self.subTest(cpus=cpus, shell=shell): + self.assertEqual(self.shell_sees_with_cpus(shell, self.home / ".zshenv", "VITEST_MAX_WORKERS", cpus), expected) + + def test_a_non_interactive_bash_on_the_ubuntu_bashrc_sees_the_share_and_did_not_before(self) -> None: + skeleton = self.ubuntu_home() + self.ok(self.run_ubuntu(github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 2})) + self.assertTrue((self.home / ".bashrc").read_text().endswith(skeleton)) + self.assertEqual(self.shell_sees_with_cpus("bash", self.home / ".bashrc", "VITEST_MAX_WORKERS", 10), "5") + result = subprocess.run(["bash", "-c", '. "$1"\nprintf %s "${VITEST_MAX_WORKERS-unset}"', "bash", str(UBUNTU_BASHRC)], capture_output=True, text=True, check=True, env={"PATH": "/usr/bin:/bin", "HOME": str(self.home)}) + self.assertEqual(result.stdout, "unset") + + def test_a_cpu_share_the_startup_files_assign_already_is_not_added_or_overridden(self) -> None: + (self.home / ".zshrc").write_text("export VITEST_MAX_WORKERS=3\n") + result = self.run_settings(github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 2}) + self.ok(result) + self.assertEqual((self.home / ".zshrc").read_text(), "export VITEST_MAX_WORKERS=3\n") + self.assertFalse((self.home / ".zshenv").exists()) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + + def test_a_changed_cpu_divisor_updates_the_roles_block_in_place(self) -> None: + self.ok(self.run_settings(github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 4})) + self.ok(self.run_settings(github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 2})) + text = (self.home / ".zshenv").read_text() + self.assertEqual(text.count("BEGIN ANSIBLE MANAGED BLOCK"), 1) + self.assertEqual(self.shell_sees_with_cpus("zsh", self.home / ".zshenv", "VITEST_MAX_WORKERS", 10), "5") + + def test_a_cpu_divisor_that_is_not_an_integer_of_at_least_one_fails_the_run_before_any_file_is_changed(self) -> None: + for label, divisor, message in (("zero", 0, "at least 1"), ("percentage", "50%", "must be an integer"), ("float", 2.5, "must be an integer")): + with self.subTest(label): + result = self.run_settings(github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": divisor}) + self.assertNotEqual(result.returncode, 0) + self.assertIn(message, self.squashed(result)) + self.assertEqual(list(self.home.iterdir()), []) + + def test_a_variable_in_the_share_and_another_shell_mapping_fails_the_run_before_any_file_is_changed(self) -> None: + result = self.run_settings(github_runner_host_tools_shell_env={"VITEST_MAX_WORKERS": "4"}, github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 2}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("VITEST_MAX_WORKERS is given more than once", self.squashed(result)) + self.assertEqual(list(self.home.iterdir()), []) + + def test_a_cpu_share_alone_is_enough_to_make_the_role_write_the_startup_file_and_check_mode_writes_nothing(self) -> None: + result = self.run_settings("--check", github_runner_host_tools_shell_env_cpu_share={"VITEST_MAX_WORKERS": 2}) + self.ok(result) + self.assertEqual(list(self.home.iterdir()), []) + self.assertGreaterEqual(self.recap(result)["changed"], 1, result.stdout) + + def test_a_mise_cpu_share_is_written_to_the_fleet_file_untouched_by_ansibles_templating_and_a_second_run_changes_nothing(self) -> None: + self.have_homebrew() + (self.stub / "mise.state").write_text("current") + variables = {"github_runner_host_tools_shell_integration": False, "github_runner_host_tools_mise_env_cpu_share": {"VITEST_MAX_WORKERS": 2}} + self.ok(self.run_role(**variables)) + written = self.fleet_file().read_text() + self.assertEqual(written.splitlines()[1:], ["[env]", 'VITEST_MAX_WORKERS = "{% set c = num_cpus() | int %}{% set n = ((c - c % 2) / 2) | int %}{% if n < 1 %}1{% else %}{{ n }}{% endif %}"']) + result = self.run_role(**variables) + self.ok(result) + self.assertEqual(self.recap(result)["changed"], 0, result.stdout) + self.assertEqual(self.fleet_file().read_text(), written) + + def test_mise_reads_the_cpu_share_the_role_wrote_back_as_half_the_cpus_of_the_host(self) -> None: + self.have_homebrew() + (self.stub / "mise.state").write_text("current") + self.ok(self.run_role(github_runner_host_tools_shell_integration=False, github_runner_host_tools_mise_env_cpu_share={"VITEST_MAX_WORKERS": 2})) + elsewhere = self.work / "reading-home" + (elsewhere / ".config" / "mise").mkdir(parents=True) + shutil.copytree(self.fleet_file().parent, elsewhere / ".config" / "mise" / "conf.d") + exported, _ = mise_reads(elsewhere) + cpus = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() or 1 + self.assertEqual(exported["VITEST_MAX_WORKERS"], str(max(1, cpus // 2))) + + def test_a_mise_cpu_divisor_that_is_not_an_integer_of_at_least_one_fails_the_run_and_writes_nothing(self) -> None: + self.have_homebrew() + for divisor in (0, "50%"): + with self.subTest(divisor=divisor): + result = self.run_role(github_runner_host_tools_shell_integration=False, github_runner_host_tools_mise_env_cpu_share={"VITEST_MAX_WORKERS": divisor}) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.fleet_file().exists()) + def test_check_mode_reports_a_variable_and_writes_nothing(self) -> None: result = self.run_settings("--check", github_runner_host_tools_shell_env={"UV_PYTHON_DOWNLOADS": "never"}) self.ok(result) diff --git a/tests/unit/test_host_tools_filters.py b/tests/unit/test_host_tools_filters.py index 5432ee35..85ce030c 100644 --- a/tests/unit/test_host_tools_filters.py +++ b/tests/unit/test_host_tools_filters.py @@ -3,6 +3,7 @@ from __future__ import annotations import importlib.util +import os import re import shutil import subprocess @@ -135,27 +136,27 @@ class MiseTomlFilterTest(unittest.TestCase): def test_tools_and_settings_render_as_two_tables(self) -> None: config = {"tools": {"node": "22", "python": ["3.13", "3.12"]}, "settings": {"experimental": True, "jobs": 4, "ratio": 0.5, "http_timeout": "30s"}} self.assertEqual( - host_tools.host_tools_mise_toml(config, {}), + host_tools.host_tools_mise_toml(config, {}, {}), '[tools]\nnode = "22"\npython = ["3.13", "3.12"]\n\n[settings]\nexperimental = true\njobs = 4\nratio = 0.5\nhttp_timeout = "30s"\n', ) def test_the_tables_come_out_in_one_order_whatever_the_mapping_has(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"experimental": False}, "tools": {"node": "lts"}}, {}), '[tools]\nnode = "lts"\n\n[settings]\nexperimental = false\n') + self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"experimental": False}, "tools": {"node": "lts"}}, {}, {}), '[tools]\nnode = "lts"\n\n[settings]\nexperimental = false\n') def test_a_key_that_is_not_bare_is_quoted(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"tools": {"npm:typescript": "latest", "cargo:ripgrep": ["14"]}}, {}), '[tools]\n"npm:typescript" = "latest"\n"cargo:ripgrep" = ["14"]\n') + self.assertEqual(host_tools.host_tools_mise_toml({"tools": {"npm:typescript": "latest", "cargo:ripgrep": ["14"]}}, {}, {}), '[tools]\n"npm:typescript" = "latest"\n"cargo:ripgrep" = ["14"]\n') def test_strings_are_escaped(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"note": 'say "hi"\\\n\t\x01'}}, {}), '[settings]\nnote = "say \\"hi\\"\\\\\\n\\t\\u0001"\n') + self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"note": 'say "hi"\\\n\t\x01'}}, {}, {}), '[settings]\nnote = "say \\"hi\\"\\\\\\n\\t\\u0001"\n') def test_a_setting_can_be_a_list_of_scalars(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"disable_tools": ["node", "python"], "levels": [1, 2]}}, {}), '[settings]\ndisable_tools = ["node", "python"]\nlevels = [1, 2]\n') + self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"disable_tools": ["node", "python"], "levels": [1, 2]}}, {}, {}), '[settings]\ndisable_tools = ["node", "python"]\nlevels = [1, 2]\n') def test_an_empty_table_is_left_out(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"tools": {}, "settings": {"experimental": True}}, {}), "[settings]\nexperimental = true\n") + self.assertEqual(host_tools.host_tools_mise_toml({"tools": {}, "settings": {"experimental": True}}, {}, {}), "[settings]\nexperimental = true\n") def test_a_boolean_is_not_written_as_a_number(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"on": True, "one": 1}}, {}), "[settings]\non = true\none = 1\n") + self.assertEqual(host_tools.host_tools_mise_toml({"settings": {"on": True, "one": 1}}, {}, {}), "[settings]\non = true\none = 1\n") def test_what_it_cannot_render_is_refused(self) -> None: cases = { @@ -171,29 +172,29 @@ def test_what_it_cannot_render_is_refused(self) -> None: } for label, (config, message) in cases.items(): with self.subTest(label), self.assertRaisesRegex(Exception, message): - host_tools.host_tools_mise_toml(config, {}) + host_tools.host_tools_mise_toml(config, {}, {}) class MiseEnvFilterTest(unittest.TestCase): def read_back(self, env: dict[str, object], home_relative: dict[str, str], home: str = "home") -> tuple[dict[str, str], str]: root = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, root) - return mise_env(host_tools.host_tools_mise_toml({"env": env}, home_relative), root / home) + return mise_env(host_tools.host_tools_mise_toml({"env": env}, home_relative, {}), root / home) def test_the_env_table_is_written_last_with_plain_values_then_home_relative_paths(self) -> None: self.assertEqual( - host_tools.host_tools_mise_toml({"env": {"B": "x", "A": True, "N": 3, "F": 0.5}, "tools": {"node": "22"}}, {"TURBO_CACHE_DIR": ".cache/turbo"}), + host_tools.host_tools_mise_toml({"env": {"B": "x", "A": True, "N": 3, "F": 0.5}, "tools": {"node": "22"}}, {"TURBO_CACHE_DIR": ".cache/turbo"}, {}), '[tools]\nnode = "22"\n\n[env]\nB = "x"\nA = "true"\nN = "3"\nF = "0.5"\nTURBO_CACHE_DIR = "{{ env.HOME }}/.cache/turbo"\n', ) def test_home_relative_paths_alone_are_enough_to_render(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({}, {"TURBO_CACHE_DIR": ".cache/turbo"}), '[env]\nTURBO_CACHE_DIR = "{{ env.HOME }}/.cache/turbo"\n') + self.assertEqual(host_tools.host_tools_mise_toml({}, {"TURBO_CACHE_DIR": ".cache/turbo"}, {}), '[env]\nTURBO_CACHE_DIR = "{{ env.HOME }}/.cache/turbo"\n') def test_an_empty_env_table_is_left_out(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"env": {}, "tools": {"node": "22"}}, {}), '[tools]\nnode = "22"\n') + self.assertEqual(host_tools.host_tools_mise_toml({"env": {}, "tools": {"node": "22"}}, {}, {}), '[tools]\nnode = "22"\n') def test_a_dollar_sign_and_a_template_brace_are_escaped_for_mise_and_the_string_for_toml(self) -> None: - self.assertEqual(host_tools.host_tools_mise_toml({"env": {"A": 'a$b "c" {{d}} {%e%} {#f#} {g}'}}, {}), "[env]\nA = \"a$$b \\\"c\\\" {{ '{' }}{d}} {{ '{' }}%e%} {{ '{' }}#f#} {g}\"\n") + self.assertEqual(host_tools.host_tools_mise_toml({"env": {"A": 'a$b "c" {{d}} {%e%} {#f#} {g}'}}, {}, {}), "[env]\nA = \"a$$b \\\"c\\\" {{ '{' }}{d}} {{ '{' }}%e%} {{ '{' }}#f#} {g}\"\n") def test_a_home_relative_variable_is_expanded_by_mise_to_the_home_directory_it_reads_the_file_in(self) -> None: for home in ("home", "a home", "it's", 'say "hi"', "$d"): @@ -240,15 +241,15 @@ def test_what_mise_would_not_take_as_a_variable_is_refused(self) -> None: "only a parent component": ({}, {"A": ".."}, "'..' component"), "a number as a path": ({}, {"A": 3}, "must be a string"), "a newline in a path": ({}, {"A": "a\nb"}, "one line"), - "a name in both": ({"env": {"A": "x"}}, {"A": ".cache"}, "A is set both"), + "a name in both": ({"env": {"A": "x"}}, {"A": ".cache"}, "A is given more than once"), } for label, (config, home_relative, message) in cases.items(): with self.subTest(label), self.assertRaisesRegex(Exception, message): - host_tools.host_tools_mise_toml(config, home_relative) + host_tools.host_tools_mise_toml(config, home_relative, {}) def test_nothing_in_either_mapping_is_nothing_to_render(self) -> None: with self.assertRaisesRegex(Exception, "nothing to render"): - host_tools.host_tools_mise_toml({"env": {}}, {}) + host_tools.host_tools_mise_toml({"env": {}}, {}, {}) def grep_finds(pattern: str, text: str) -> bool: @@ -258,35 +259,35 @@ def grep_finds(pattern: str, text: str) -> bool: class ShellEnvFilterTest(unittest.TestCase): def test_a_variable_becomes_a_quoted_export_line_and_a_pattern_that_finds_an_assignment_of_it(self) -> None: - [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}, {}) + [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}, {}, {}) self.assertEqual(item, {"name": "env-UV_PYTHON_DOWNLOADS", "pattern": "^[[:space:]]*(export[[:space:]]+)?UV_PYTHON_DOWNLOADS=", "line": "export UV_PYTHON_DOWNLOADS=never", "non_interactive": True}) def test_the_variables_come_out_in_the_order_of_the_mapping(self) -> None: - self.assertEqual([item["name"] for item in host_tools.host_tools_shell_env({"B": "1", "A": "2"}, {})], ["env-B", "env-A"]) + self.assertEqual([item["name"] for item in host_tools.host_tools_shell_env({"B": "1", "A": "2"}, {}, {})], ["env-B", "env-A"]) def test_an_empty_mapping_has_no_lines(self) -> None: - self.assertEqual(host_tools.host_tools_shell_env({}, {}), []) + self.assertEqual(host_tools.host_tools_shell_env({}, {}, {}), []) def test_a_value_reaches_the_variable_exactly_as_given_whatever_it_holds(self) -> None: values = ["it's", 'say "hi"', "$HOME", "${HOME}", "`id`", "$(id)", "a b c", "back\\slash", "tab\there", "*", "; echo injected", "", "-n", "'", "''", "\"'\"", "é"] for value in values: with self.subTest(value=value): - [item] = host_tools.host_tools_shell_env({"EXAMPLE": value}, {}) + [item] = host_tools.host_tools_shell_env({"EXAMPLE": value}, {}, {}) for shell in ("sh", "bash"): result = subprocess.run([shell, "-c", item["line"] + '\nprintf %s "$EXAMPLE"'], capture_output=True, text=True, check=True, env={"PATH": "/usr/bin:/bin", "HOME": "/nonexistent-home"}) self.assertEqual(result.stdout, value) self.assertEqual(result.stderr, "") def test_a_dollar_sign_and_a_single_quote_are_quoted_and_not_expanded(self) -> None: - self.assertEqual(host_tools.host_tools_shell_env({"A": "$HOME"}, {})[0]["line"], "export A='$HOME'") - self.assertEqual(host_tools.host_tools_shell_env({"A": "it's"}, {})[0]["line"], "export A='it'\"'\"'s'") + self.assertEqual(host_tools.host_tools_shell_env({"A": "$HOME"}, {}, {})[0]["line"], "export A='$HOME'") + self.assertEqual(host_tools.host_tools_shell_env({"A": "it's"}, {}, {})[0]["line"], "export A='it'\"'\"'s'") def test_a_boolean_is_true_or_false_and_a_number_is_its_digits(self) -> None: - lines = {item["name"]: item["line"] for item in host_tools.host_tools_shell_env({"A": True, "B": False, "C": 3, "D": 0.5}, {})} + lines = {item["name"]: item["line"] for item in host_tools.host_tools_shell_env({"A": True, "B": False, "C": 3, "D": 0.5}, {}, {})} self.assertEqual(lines, {"env-A": "export A=true", "env-B": "export B=false", "env-C": "export C=3", "env-D": "export D=0.5"}) def test_the_pattern_finds_an_assignment_with_or_without_export_and_not_a_comment_or_another_variable(self) -> None: - [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}, {}) + [item] = host_tools.host_tools_shell_env({"UV_PYTHON_DOWNLOADS": "never"}, {}, {}) for text in ("UV_PYTHON_DOWNLOADS=automatic", "export UV_PYTHON_DOWNLOADS=automatic", " export UV_PYTHON_DOWNLOADS='x'", "\texport UV_PYTHON_DOWNLOADS="): with self.subTest(text=text): self.assertTrue(grep_finds(item["pattern"], text)) @@ -309,7 +310,7 @@ def test_what_a_shell_would_not_take_is_refused(self) -> None: } for label, (env, message) in cases.items(): with self.subTest(label), self.assertRaisesRegex(Exception, message): - host_tools.host_tools_shell_env(env, {}) + host_tools.host_tools_shell_env(env, {}, {}) def shell_reads(shell: str, line: str, home: str) -> str: @@ -321,35 +322,35 @@ def shell_reads(shell: str, line: str, home: str) -> str: class HomeRelativeFilterTest(unittest.TestCase): def test_a_path_becomes_an_export_of_the_home_directory_and_the_path_in_double_quotes(self) -> None: - [item] = host_tools.host_tools_shell_env({}, {"TURBO_CACHE_DIR": ".cache/turbo"}) + [item] = host_tools.host_tools_shell_env({}, {"TURBO_CACHE_DIR": ".cache/turbo"}, {}) self.assertEqual(item, {"name": "env-TURBO_CACHE_DIR", "pattern": "^[[:space:]]*(export[[:space:]]+)?TURBO_CACHE_DIR=", "line": 'export TURBO_CACHE_DIR="$HOME/.cache/turbo"', "non_interactive": True}) def test_the_variables_of_both_mappings_come_out_the_plain_ones_first_each_in_its_own_order(self) -> None: - items = host_tools.host_tools_shell_env({"B": "1", "A": "2"}, {"D": "d", "C": "c"}) + items = host_tools.host_tools_shell_env({"B": "1", "A": "2"}, {"D": "d", "C": "c"}, {}) self.assertEqual([item["name"] for item in items], ["env-B", "env-A", "env-D", "env-C"]) def test_a_path_reaches_the_variable_as_given_in_every_shell_whatever_it_holds_and_the_home_is_expanded_once(self) -> None: paths = [".cache/turbo", "a b c", "it's", 'say "hi"', "$HOME", "${HOME}", "`id`", "$(id)", "back\\slash", "\\$", "tab\there", "*", "?", "[a]", "; echo injected", "-n", "~", "~/x", "!x", "a..b", "..x", "x..", ".hidden/.also", "dir/", "./x", "é", "'", "\""] homes = ["/home/example", "/home/a b", "/home/it's", '/home/"q"', "/home/$d"] for path in paths: - [item] = host_tools.host_tools_shell_env({}, {"EXAMPLE": path}) + [item] = host_tools.host_tools_shell_env({}, {"EXAMPLE": path}, {}) for home in homes: for shell in ("sh", "bash", "zsh"): with self.subTest(path=path, home=home, shell=shell): self.assertEqual(shell_reads(shell, item["line"], home), f"{home}/{path}") def test_the_special_characters_are_escaped_and_nothing_else_is(self) -> None: - [item] = host_tools.host_tools_shell_env({}, {"A": 'x\\y$z`w"v !*~'}) + [item] = host_tools.host_tools_shell_env({}, {"A": 'x\\y$z`w"v !*~'}, {}) self.assertEqual(item["line"], 'export A="$HOME/x\\\\y\\$z\\`w\\"v !*~"') def test_the_pattern_finds_an_assignment_of_the_variable_already_in_a_startup_file(self) -> None: - [item] = host_tools.host_tools_shell_env({}, {"TURBO_CACHE_DIR": ".cache/turbo"}) + [item] = host_tools.host_tools_shell_env({}, {"TURBO_CACHE_DIR": ".cache/turbo"}, {}) self.assertTrue(grep_finds(item["pattern"], 'export TURBO_CACHE_DIR="$HOME/.cache/turbo"')) self.assertTrue(grep_finds(item["pattern"], "TURBO_CACHE_DIR=/elsewhere")) self.assertFalse(grep_finds(item["pattern"], "# export TURBO_CACHE_DIR=/elsewhere")) def test_an_empty_mapping_has_no_lines(self) -> None: - self.assertEqual(host_tools.host_tools_shell_env({}, {}), []) + self.assertEqual(host_tools.host_tools_shell_env({}, {}, {}), []) def test_what_is_not_a_path_below_the_home_directory_is_refused(self) -> None: cases = { @@ -372,15 +373,153 @@ def test_what_is_not_a_path_below_the_home_directory_is_refused(self) -> None: } for label, (home_relative, message) in cases.items(): with self.subTest(label), self.assertRaisesRegex(Exception, message): - host_tools.host_tools_shell_env({}, home_relative) + host_tools.host_tools_shell_env({}, home_relative, {}) def test_a_mapping_that_is_not_one_is_refused(self) -> None: with self.assertRaisesRegex(Exception, "home-relative shell environment must be a mapping"): - host_tools.host_tools_shell_env({}, [".cache"]) + host_tools.host_tools_shell_env({}, [".cache"], {}) def test_a_variable_in_both_mappings_is_refused_and_named(self) -> None: - with self.assertRaisesRegex(Exception, "A is set both"): - host_tools.host_tools_shell_env({"A": "x"}, {"A": "y"}) + with self.assertRaisesRegex(Exception, "A is given more than once"): + host_tools.host_tools_shell_env({"A": "x"}, {"A": "y"}, {}) + + +# The CPU counts the share tests pretend a host has, and the divisors they divide by. The counts include 1, which a divisor above 1 would round down to 0, the exact multiples and the counts one below them. +CPU_COUNTS = (1, 2, 3, 4, 5, 10, 11, 12, 63, 64) +CPU_DIVISORS = (1, 2, 3, 4, 8, 100) + + +def expected_share(cpus: int, divisor: int) -> int: + return max(1, cpus // divisor) + + +def host_cpus() -> int: + """The count mise's num_cpus() and getconf report on this host: the CPUs the process may run on where the OS says, else all of them.""" + return len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() or 1 + + +class CpuShareFilterTest(unittest.TestCase): + def stand_in_getconf(self, cpus: int) -> Path: + """A directory holding a getconf that reports ``cpus`` for _NPROCESSORS_ONLN and refuses anything else, so the count is the test's and not the host's.""" + directory = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, directory) + stand_in = directory / "getconf" + stand_in.write_text(f'#!/bin/sh\n[ "$1" = _NPROCESSORS_ONLN ] || {{ echo "unexpected getconf $*" >&2; exit 2; }}\necho {cpus}\n') + stand_in.chmod(0o755) + return directory + + def read(self, shell: str, line: str, path: str, extra: str = "") -> subprocess.CompletedProcess[str]: + return subprocess.run([shell, "-c", f'{extra}{line}\nprintf %s "$V"'], capture_output=True, text=True, check=True, env={"PATH": path, "HOME": "/nonexistent-home"}) + + def test_a_share_is_one_export_line_that_counts_the_cpus_and_divides_in_a_subshell(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {}, {"V": 2}) + self.assertEqual(item, {"name": "env-V", "pattern": "^[[:space:]]*(export[[:space:]]+)?V=", "line": 'export V="$(n=$(getconf _NPROCESSORS_ONLN); n=$((n / 2)); echo $((n > 0 ? n : 1)))"', "non_interactive": True}) + + def test_the_value_is_the_cpu_count_divided_rounded_down_and_never_below_one_in_every_shell(self) -> None: + for cpus in CPU_COUNTS: + path = f"{self.stand_in_getconf(cpus)}:/usr/bin:/bin" + for divisor in CPU_DIVISORS: + [item] = host_tools.host_tools_shell_env({}, {}, {"V": divisor}) + for shell in ("sh", "bash", "zsh"): + with self.subTest(cpus=cpus, divisor=divisor, shell=shell): + result = self.read(shell, str(item["line"]), path) + self.assertEqual(result.stdout, str(expected_share(cpus, divisor))) + self.assertEqual(result.stderr, "") + + def test_the_real_getconf_gives_the_hosts_own_count(self) -> None: + for divisor in CPU_DIVISORS: + [item] = host_tools.host_tools_shell_env({}, {}, {"V": divisor}) + for shell in ("sh", "bash", "zsh"): + with self.subTest(divisor=divisor, shell=shell): + self.assertEqual(self.read(shell, str(item["line"]), "/usr/bin:/bin").stdout, str(max(1, host_cpus() // divisor))) + + def test_no_variable_of_the_arithmetic_is_left_in_the_users_shell_and_one_they_have_is_kept(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {}, {"V": 2}) + path = f"{self.stand_in_getconf(10)}:/usr/bin:/bin" + for shell in ("sh", "bash", "zsh"): + with self.subTest(shell=shell): + result = subprocess.run([shell, "-c", f'n=keep\n{item["line"]}\nprintf "%s|%s" "$V" "$n"; unset n\n{item["line"]}\nprintf "|%s" "${{n-unset}}"'], capture_output=True, text=True, check=True, env={"PATH": path, "HOME": "/nonexistent-home"}) + self.assertEqual(result.stdout, "5|keep|unset") + + def test_the_value_is_exported_to_the_commands_the_shell_runs(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {}, {"VITEST_MAX_WORKERS": 2}) + path = f"{self.stand_in_getconf(10)}:/usr/bin:/bin" + for shell in ("bash", "zsh"): + with self.subTest(shell=shell): + result = subprocess.run([shell, "-c", f'{item["line"]}\nenv'], capture_output=True, text=True, check=True, env={"PATH": path, "HOME": "/nonexistent-home"}) + self.assertIn("VITEST_MAX_WORKERS=5\n", result.stdout) + + def test_the_pattern_finds_an_assignment_of_the_variable_already_in_a_startup_file(self) -> None: + [item] = host_tools.host_tools_shell_env({}, {}, {"VITEST_MAX_WORKERS": 2}) + self.assertTrue(grep_finds(str(item["pattern"]), "export VITEST_MAX_WORKERS=4")) + self.assertFalse(grep_finds(str(item["pattern"]), "# export VITEST_MAX_WORKERS=4")) + + def test_the_variables_come_out_plain_then_home_relative_then_shares(self) -> None: + items = host_tools.host_tools_shell_env({"B": "x"}, {"D": "d"}, {"F": 2, "E": 3}) + self.assertEqual([item["name"] for item in items], ["env-B", "env-D", "env-F", "env-E"]) + + def test_the_mise_share_is_a_template_in_the_env_table(self) -> None: + self.assertEqual(host_tools.host_tools_mise_toml({}, {}, {"V": 2}), '[env]\nV = "{% set c = num_cpus() | int %}{% set n = ((c - c % 2) / 2) | int %}{% if n < 1 %}1{% else %}{{ n }}{% endif %}"\n') + + def test_the_mise_share_is_the_cpu_count_divided_rounded_down_and_never_below_one_for_counts_the_host_does_not_have(self) -> None: + names = {divisor: f"D{divisor}" for divisor in CPU_DIVISORS} + rendered = host_tools.host_tools_mise_toml({}, {}, {name: divisor for divisor, name in names.items()}) + self.assertEqual(rendered.count("num_cpus()"), len(names)) + for cpus in CPU_COUNTS: + root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, root) + # num_cpus() returns a string, so a single-quoted Tera string (the template sits in a double-quoted TOML string) stands in for the host's count. + exported, stderr = mise_env(rendered.replace("num_cpus()", f"'{cpus}'"), root / "home") + with self.subTest(cpus=cpus): + self.assertEqual({divisor: exported[name] for divisor, name in names.items()}, {divisor: str(expected_share(cpus, divisor)) for divisor in names}) + self.assertEqual(stderr, "") + + def test_the_mise_share_with_the_real_num_cpus_gives_the_hosts_own_count(self) -> None: + names = {divisor: f"D{divisor}" for divisor in CPU_DIVISORS} + root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, root) + exported, _ = mise_env(host_tools.host_tools_mise_toml({}, {}, {name: divisor for divisor, name in names.items()}), root / "home") + self.assertEqual({divisor: exported[name] for divisor, name in names.items()}, {divisor: str(max(1, host_cpus() // divisor)) for divisor in names}) + + def test_the_shell_and_mise_agree_for_every_count_and_divisor(self) -> None: + for cpus in (1, 3, 10, 64): + path = f"{self.stand_in_getconf(cpus)}:/usr/bin:/bin" + root = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, root) + rendered = host_tools.host_tools_mise_toml({}, {}, {f"D{divisor}": divisor for divisor in CPU_DIVISORS}) + exported, _ = mise_env(rendered.replace("num_cpus()", f"'{cpus}'"), root / "home") + for divisor in CPU_DIVISORS: + [item] = host_tools.host_tools_shell_env({}, {}, {"V": divisor}) + with self.subTest(cpus=cpus, divisor=divisor): + self.assertEqual(self.read("bash", str(item["line"]), path).stdout, exported[f"D{divisor}"]) + + def test_a_divisor_that_is_not_an_integer_of_at_least_one_is_refused(self) -> None: + cases = {"zero": (0, "at least 1"), "negative": (-2, "at least 1"), "a float": (2.0, "must be an integer"), "half": (0.5, "must be an integer"), "a string": ("2", "must be an integer"), "a percentage": ("50%", "must be an integer"), "true": (True, "must be an integer"), "false": (False, "must be an integer"), "null": (None, "must be an integer"), "a list": ([2], "must be an integer")} + for label, (divisor, message) in cases.items(): + with self.subTest(label, target="shell"), self.assertRaisesRegex(Exception, message): + host_tools.host_tools_shell_env({}, {}, {"V": divisor}) + with self.subTest(label, target="mise"), self.assertRaisesRegex(Exception, message): + host_tools.host_tools_mise_toml({}, {}, {"V": divisor}) + + def test_a_name_a_shell_would_not_accept_is_refused(self) -> None: + with self.assertRaisesRegex(Exception, "not a name a shell accepts"): + host_tools.host_tools_shell_env({}, {}, {"A-B": 2}) + with self.assertRaisesRegex(Exception, "not a name a shell accepts"): + host_tools.host_tools_mise_toml({}, {}, {"A-B": 2}) + + def test_a_mapping_that_is_not_one_is_refused(self) -> None: + with self.assertRaisesRegex(Exception, "CPU-share shell environment must be a mapping"): + host_tools.host_tools_shell_env({}, {}, [2]) + with self.assertRaisesRegex(Exception, "CPU-share env must be a mapping"): + host_tools.host_tools_mise_toml({}, {}, [2]) + + def test_a_variable_given_in_two_mappings_is_refused_and_named_in_every_pairing(self) -> None: + pairings = {"plain and share": (({"A": "x"}, {}, {"A": 2})), "path and share": (({}, {"A": "x"}, {"A": 2})), "plain and path": (({"A": "x"}, {"A": "y"}, {}))} + for label, (env, home_relative, cpu_share) in pairings.items(): + with self.subTest(label, target="shell"), self.assertRaisesRegex(Exception, "A is given more than once"): + host_tools.host_tools_shell_env(env, home_relative, cpu_share) + with self.subTest(label, target="mise"), self.assertRaisesRegex(Exception, "A is given more than once"): + host_tools.host_tools_mise_toml({"env": env}, home_relative, cpu_share) STEM = "# {mark} ANSIBLE MANAGED BLOCK example "