fix(cluster): start a Colima profile in its own home at login

plugins/filter/container_runtime.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import posixpath
 import re
 from typing import Any
 
@@ -11,6 +12,11 @@
 # Colima names the Docker context of its default profile "colima" and any other profile's "colima-<profile>".
 COLIMA_CONTEXT = "colima"
 COLIMA_DEFAULT_PROFILE = "default"
+# Colima's default home, which it falls back to when COLIMA_HOME is not set, relative to the user's home directory.
+COLIMA_DEFAULT_HOME = ".colima"
+# The Docker socket Colima creates in a profile's directory, <COLIMA_HOME>/<profile>/docker.sock, and so the endpoint of the Docker context it creates for the profile.
+COLIMA_SOCKET = "docker.sock"
+UNIX_SCHEME = "unix://"
 # Docker Desktop's own context for its Linux engine.
 DOCKER_DESKTOP_CONTEXTS = ("desktop-linux",)
 # Lima's rule for an instance name, which is what a Colima profile becomes. It also keeps the value free of every character that would need escaping in a launchd property list or on a command line.
@@ -66,9 +72,35 @@ def cluster_container_runtime(context: Any, requested: Any = "auto", colima_prof
     return {"kind": "colima", "colima_profile": profile or context_profile or COLIMA_DEFAULT_PROFILE}
 
 
+def cluster_colima_home(endpoint: Any, profile: Any, user_home: Any) -> str:
+    """Return the ``COLIMA_HOME`` a Colima profile needs, read from the endpoint of the Docker context Colima made for it.
+
+    Colima puts a profile's Docker socket at ``<COLIMA_HOME>/<profile>/docker.sock`` and points the profile's context at it, so the context says where the profile lives. A command that starts the profile has to run with that ``COLIMA_HOME``, because Colima falls back to ``~/.colima`` when the variable is missing and would then start a different profile of the same name there.
+
+    :param endpoint: The Docker host of the profile's context, ``unix://<COLIMA_HOME>/<profile>/docker.sock``.
+    :param profile: The Colima profile the context belongs to.
+    :param user_home: The home directory of the user the profile belongs to.
+    :returns: The directory, or an empty string when it is ``<user_home>/.colima``, the home Colima uses when ``COLIMA_HOME`` is not set, so a profile in the default home keeps running with no variable.
+    :raises AnsibleFilterError: a value is not a string, the profile is not a valid profile name, or the endpoint is not the socket of that profile in an absolute directory.
+    """
+    host = _text("The Docker context's endpoint", endpoint)
+    profile_name = _text("The Colima profile", profile)
+    home = _text("The user's home directory", user_home)
+    if not PROFILE_PATTERN.match(profile_name):
+        raise AnsibleFilterError(f"The Colima profile {profile_name!r} is not a valid profile name: use letters and digits, separated by single dots, hyphens or underscores.")
+    path = host.removeprefix(UNIX_SCHEME) if host.startswith(UNIX_SCHEME) else ""
+    profile_directory, socket = posixpath.split(path)
+    colima_home, directory = posixpath.split(profile_directory)
+    if socket != COLIMA_SOCKET or directory != profile_name or not posixpath.isabs(colima_home):
+        raise AnsibleFilterError(f"The Docker context's endpoint {host!r} is not the socket of the Colima profile {profile_name!r} (unix://<COLIMA_HOME>/{profile_name}/{COLIMA_SOCKET}), so the role cannot tell where that profile lives: set github_runner_cluster_colima_home.")
+    if colima_home == posixpath.join(home, COLIMA_DEFAULT_HOME):
+        return ""
+    return colima_home
+
+
 class FilterModule:
     """Registers the container runtime filter with Ansible."""
 
     def filters(self) -> dict[str, Any]:
         """Return the filters this plugin provides."""
-        return {"cluster_container_runtime": cluster_container_runtime}
+        return {"cluster_container_runtime": cluster_container_runtime, "cluster_colima_home": cluster_colima_home}

roles/github_runner_cluster/README.md

@@ -145,7 +145,7 @@ Which runtime a Mac runs is taken from the Docker context the role drives (see [
 **Colima.** With `github_runner_cluster_runtime_autostart` on (the default, which acts on Colima hosts only), the role:
 
 - starts the profile if it is stopped, with `colima start --profile <profile> --activate=false`, so the profile does not make its context the host's current one;
-- installs a launchd agent, `com.exadev.github-runner.colima.<profile>`, as `~/Library/LaunchAgents/<label>.plist` of the user Ansible connects to, which runs the same command at every login with a `PATH` that names both Homebrew prefixes (`github_runner_cluster_runtime_path`), since launchd gives an agent only the system `PATH` and Colima starts `limactl` and `docker` from the one it has; and
+- installs a launchd agent, `com.exadev.github-runner.colima.<profile>`, as `~/Library/LaunchAgents/<label>.plist` of the user Ansible connects to, which runs the same command at every login with a `PATH` that names both Homebrew prefixes (`github_runner_cluster_runtime_path`), since launchd gives an agent only the system `PATH` and Colima starts `limactl` and `docker` from the one it has, and with the profile's `COLIMA_HOME` when the profile does not live in `~/.colima` (a profile of a bridged node, such as `colima-node` in `~/.colima-node`, which Colima would otherwise look for in `~/.colima` and start as a new VM there). The `COLIMA_HOME` is `github_runner_cluster_colima_home` when that is set, and otherwise the directory the profile's Docker context points at, read with `docker context inspect`, which needs no running VM; a context whose endpoint is not `<COLIMA_HOME>/<profile>/docker.sock` fails the run with a message asking for the variable. A host whose profile lives in `~/.colima` gets an agent with no `COLIMA_HOME`, exactly as before; and
 - loads the agent into that user's login session. A user with no login session on the host (nobody has logged in at the console since the last boot) cannot have an agent loaded, so the role installs the file, says so, and leaves launchd to load it at the next login.
 
 The role reads what it needs to tell a Mac from another host (the kernel name) and to name the agent (the connecting user's id and home directory) with minimal `setup` subsets of its own when the caller has not gathered them, so a play with `gather_facts: false` works. The role never uses `become` for this: a LaunchAgent belongs to the user whose login session loads it, and Colima's VM runs as that user. Changing a definition unloads the agent before loading the new one; a rerun with nothing changed touches neither the agent nor the VM, and `--check --diff` shows the property list that would be written. With the variable off, the role installs nothing and leaves an agent from an earlier run as it is.
@@ -379,7 +379,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon
 - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host.
 - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)).
 - `github_runner_cluster_network_mode` (default `bridge`, or `host`): the network the node's container runs on, and with `host`, `github_runner_cluster_lan_cidr` (required: the LAN the nodes share) and `github_runner_cluster_network_probe_image` (default `busybox:1.37`, for reading the host's addresses) (see [Networking](#networking)).
-- `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty, Colima's own `~/.colima`; the `COLIMA_HOME` of a profile that lives in a home of its own, passed to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-at-login-macos)).
+- `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_colima_home` (default empty: the `COLIMA_HOME` is read from the endpoint of the profile's Docker context, and is unset for Colima's own `~/.colima`; set it only for a context that does not say, and the role passes it to colima and to the launchd agent), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-at-login-macos)).
 - `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)).
 - `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir`, `github_runner_cluster_compose_project` and container names; nodes of one cluster share its API port, so all but one of them on a host also need an extra file that publishes none (`ports: !reset []`, as `tests/mesh` does).
 - `github_runner_cluster_wait_for_nodes`: wait for every node to be Ready afterwards, clearing the node-password registration of a replaced node that fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`).

roles/github_runner_cluster/defaults/main.yml

@@ -180,8 +180,9 @@ github_runner_cluster_runtime_autostart: true
 github_runner_cluster_runtime: auto
 # The Colima profile to start. Empty takes it from the Docker context (colima is the default profile, colima-<profile> another) and falls back to the default profile when the context names none.
 github_runner_cluster_colima_profile: ""
-# The directory the Colima profile lives in, as COLIMA_HOME: empty is Colima's own default (~/.colima). A profile in a home of its own (a bridged profile beside another that runs in the default home, which would share its network daemon) needs it set, because Colima falls back to the default home when the variable is missing, so the agent would start a different profile there. The role passes it to colima and to the launchd agent. The directory must exist: Colima falls back silently to the default home when it does not.
+# The directory the Colima profile lives in, as COLIMA_HOME. Empty, the default, takes it from the endpoint of the profile's Docker context (Colima points the context at <COLIMA_HOME>/<profile>/docker.sock), which is empty for Colima's own default (~/.colima), so a host needs to set this only when its context does not say. A profile in a home of its own (a bridged profile beside another that runs in the default home, which would share its network daemon) needs the variable at colima and in the launchd agent, because Colima falls back to the default home when it is missing and would start a different profile there; the role passes it to both. The directory must exist: Colima falls back silently to the default home when it does not.
 github_runner_cluster_colima_home: ""
+# Derived by tasks/runtime.yml, never set directly: the Colima home the role starts the profile in, which is github_runner_cluster_colima_home when that is set and otherwise the one the profile's Docker context names.
 # The PATH the agent runs colima with, and the role runs colima and docker with on a macOS host: both Homebrew prefixes (Apple Silicon and Intel), since launchd gives an agent only the system PATH and colima starts limactl and docker from the one it is given.
 github_runner_cluster_runtime_path: "/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin"
 # Derived, never set directly: the agent's launchd label, one per Colima profile.

roles/github_runner_cluster/tasks/runtime.yml

@@ -5,7 +5,7 @@
 #
 # On a Colima host it also makes the VM load br_netfilter and set the bridge sysctls now and at every VM boot (tasks/colima_kernel.yml), whatever the autostart setting says.
 #
-# Publishes github_runner_cluster_effective_runtime ({kind, colima_profile}), github_runner_cluster_runtime_environment (what docker and colima run with), github_runner_cluster_runtime_down_message and github_runner_cluster_runtime_up, for end_checks.yml.
+# Publishes github_runner_cluster_effective_runtime ({kind, colima_profile}), github_runner_cluster_runtime_colima_home, github_runner_cluster_runtime_environment (what docker and colima run with), github_runner_cluster_runtime_down_message and github_runner_cluster_runtime_up, for end_checks.yml.
 
 - name: Read the connecting user and its environment
   # The user id the launchd domain is named after and the home directory the agent lives in. A caller that gathered facts already has both; one that did not (host_os.yml reads only the platform) gets just these two subsets.
@@ -23,11 +23,31 @@
       {{ github_runner_cluster_docker_cli_context
          | exadev.github_runner.cluster_container_runtime(github_runner_cluster_runtime, github_runner_cluster_colima_profile) }}
 
+- name: Read the Docker host of the Colima profile's context
+  # Where the profile lives: Colima points the context at <COLIMA_HOME>/<profile>/docker.sock. Needed only when github_runner_cluster_colima_home does not say. Read-only and answered by the context file alone, so it works with the VM stopped and under --check. It runs with the environment of every other docker call, so it reads the same context store.
+  ansible.builtin.command:
+    argv: [docker, context, inspect, --format, "{{ '{{' }}.Endpoints.docker.Host{{ '}}' }}", "{{ github_runner_cluster_docker_cli_context }}"]
+  environment: "{{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}) }}"
+  register: github_runner_cluster_runtime_context_host
+  changed_when: false
+  check_mode: false
+  when:
+    - github_runner_cluster_effective_runtime.kind == 'colima'
+    - github_runner_cluster_colima_home | length == 0
+
+- name: Work out the Colima home the profile lives in
+  # The variable when a host sets it, else the one its Docker context names, empty for Colima's own default. Nothing outside Colima has one.
+  ansible.builtin.set_fact:
+    github_runner_cluster_runtime_colima_home: >-
+      {{ github_runner_cluster_colima_home
+         if (github_runner_cluster_colima_home | length > 0 or github_runner_cluster_effective_runtime.kind != 'colima')
+         else (github_runner_cluster_runtime_context_host.stdout | exadev.github_runner.cluster_colima_home(github_runner_cluster_effective_runtime.colima_profile, ansible_facts['env']['HOME'])) }}
+
 - name: Publish the environment and the failure message the runtime tasks use
   ansible.builtin.set_fact:
     # The PATH is what the agent gets too, so a run that works here is the run the agent repeats at login.
     github_runner_cluster_runtime_environment: >-
-      {{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}, ({'COLIMA_HOME': github_runner_cluster_colima_home} if github_runner_cluster_colima_home | length > 0 else {})) }}
+      {{ github_runner_cluster_docker_cli_environment | combine({'PATH': github_runner_cluster_runtime_path}, ({'COLIMA_HOME': github_runner_cluster_runtime_colima_home} if github_runner_cluster_runtime_colima_home | length > 0 else {})) }}
     github_runner_cluster_runtime_down_message: >-
       {{
         {

roles/github_runner_cluster/templates/colima-launchd.plist.j2

@@ -18,9 +18,9 @@
   <dict>
     <key>PATH</key>
     <string>{{ github_runner_cluster_runtime_path | e }}</string>
-{% if github_runner_cluster_colima_home | length > 0 %}
+{% if github_runner_cluster_runtime_colima_home | length > 0 %}
     <key>COLIMA_HOME</key>
-    <string>{{ github_runner_cluster_colima_home | e }}</string>
+    <string>{{ github_runner_cluster_runtime_colima_home | e }}</string>
 {% endif %}
   </dict>
   <key>RunAtLoad</key>

tests/runtime_autostart/render_plist.yml

@@ -7,6 +7,8 @@
     - ../../roles/github_runner_cluster/defaults/main.yml
   vars:
     docker_context: colima
+    # What tasks/runtime.yml works out; a test passes its own.
+    github_runner_cluster_runtime_colima_home: ""
     ansible_facts:
       env:
         HOME: /Users/example

tests/unit/test_container_runtime.py

@@ -13,6 +13,9 @@
 _spec.loader.exec_module(container_runtime)
 
 runtime = container_runtime.cluster_container_runtime
+colima_home = container_runtime.cluster_colima_home
+AnsibleFilterError = container_runtime.AnsibleFilterError
+HOME = "/Users/example"
 
 
 class AutoDetectionTest(unittest.TestCase):
@@ -85,6 +88,47 @@ def test_a_value_that_is_not_a_string_is_an_error(self) -> None:
 
     def test_the_filter_is_registered(self) -> None:
         self.assertIs(container_runtime.FilterModule().filters()["cluster_container_runtime"], runtime)
+        self.assertIs(container_runtime.FilterModule().filters()["cluster_colima_home"], colima_home)
+
+
+class ColimaHomeTest(unittest.TestCase):
+    def test_a_profile_in_a_home_of_its_own_names_it(self) -> None:
+        self.assertEqual(colima_home("unix:///Users/example/.colima-node/node/docker.sock", "node", HOME), "/Users/example/.colima-node")
+
+    def test_the_default_home_is_no_variable_at_all(self) -> None:
+        self.assertEqual(colima_home("unix:///Users/example/.colima/default/docker.sock", "default", HOME), "")
+        self.assertEqual(colima_home("unix:///Users/example/.colima/build/docker.sock", "build", HOME), "")
+
+    def test_a_home_that_only_starts_like_the_default_is_a_home_of_its_own(self) -> None:
+        self.assertEqual(colima_home("unix:///Users/example/.colima2/build/docker.sock", "build", HOME), "/Users/example/.colima2")
+
+    def test_the_endpoint_is_trimmed_as_docker_prints_it(self) -> None:
+        self.assertEqual(colima_home("unix:///Users/example/.colima-node/node/docker.sock\n", "node", HOME), "/Users/example/.colima-node")
+
+    def test_an_endpoint_that_is_not_the_profiles_socket_is_refused(self) -> None:
+        for endpoint in (
+            "",
+            "unix:///var/run/docker.sock",
+            "unix:///Users/example/.colima-node/other/docker.sock",
+            "unix:///Users/example/.colima-node/node/other.sock",
+            "tcp://127.0.0.1:2375",
+            "ssh://example@host/Users/example/.colima-node/node/docker.sock",
+            "unix://relative/node/docker.sock",
+            "unix://node/docker.sock",
+        ):
+            with self.subTest(endpoint=endpoint):
+                with self.assertRaisesRegex(AnsibleFilterError, "set github_runner_cluster_colima_home"):
+                    colima_home(endpoint, "node", HOME)
+
+    def test_a_profile_that_is_not_a_valid_name_is_refused(self) -> None:
+        with self.assertRaisesRegex(AnsibleFilterError, "not a valid profile name"):
+            colima_home("unix:///Users/example/.colima-node/no de/docker.sock", "no de", HOME)
+
+    def test_a_value_that_is_not_a_string_is_refused(self) -> None:
+        for args in ((1, "node", HOME), ("unix:///a/node/docker.sock", ["node"], HOME), ("unix:///a/node/docker.sock", "node", 3)):
+            with self.subTest(args=args):
+                with self.assertRaises(AnsibleFilterError):
+                    colima_home(*args)
 
 
 if __name__ == "__main__":