From f02c3158743b51d4d348ac95cef3368607d7d7c4 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 23:54:03 +0100 Subject: [PATCH 01/43] feat(arc): copy capability payloads once per node into a capability store Every runner pod copied each capability image's payload, and the dind sidecar's runner externals, before its job started, so every pod start wrote the same bytes to the node's disk. A profile's capability_store copies each payload once per node into an entry named for the image's digest: the copy goes to .incoming and is published with mv -T, so concurrent pods never see a partial entry and a losing copy is deleted. The sysroot is merged per node with hard links and mounted read-only; the tool cache stays the pod's own, filled with links into the store, so setup actions still install beside them. Pod templates list the entries they use in an annotation. --- plugins/filter/arc.py | 276 ++++++++++++++++- roles/github_runner_arc/README.md | 20 +- roles/github_runner_arc/defaults/main.yml | 7 + .../tasks/render_scale_set_values.yml | 28 +- roles/github_runner_arc/tasks/validate.yml | 10 + tests/unit/test_capabilities.py | 278 +++++++++++++++++- 6 files changed, 605 insertions(+), 14 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index b78fedb..1e75d7f 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy +import hashlib import re from collections.abc import Mapping, Sequence from decimal import ROUND_FLOOR, Decimal, InvalidOperation @@ -554,6 +555,11 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er except TypeError: errors.append(f"{where}.autoscale must be a boolean") autoscale = False + try: + capability_store = boolean(profile.get("capability_store", False), strict=True) + except TypeError: + errors.append(f"{where}.capability_store must be a boolean") + capability_store = False sizing_result = None if profile.get("sizing") is not None: sizing_result = arc_max_runners(profile["sizing"]) @@ -595,6 +601,7 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er "burst_runners": burst_runners, "max_runners": max_runners, "capabilities": _capability_names(profile, where, errors), + "capability_store": capability_store, "label": f"{org_name}{suffix}", "settings": dict(profile), } @@ -1031,13 +1038,244 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca } -def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str) -> dict[str, Any]: +# --- The capability store: one copy per node of what every runner pod would otherwise copy at its start --- +# A pod's tool-cache and system-library copies (and the runner's externals for the dind sidecar) are the same bytes on every pod of a profile, taken from digest-pinned images. With a profile's capability_store on, each image's payload is copied once per node into /, where is the image's sha256 digest, and pods mount the store read-only instead of writing their own copy. The store root is a hostPath of its own, apart from the node cache, whose last component names it. +_CAPABILITY_STORE_HOST_PATH_NAME = re.compile(r"^(?:capability-store|[a-z0-9][a-z0-9-]*-capability-store)$") +# A copy is written under /.incoming/.. and published with one rename onto /: rename(2) onto a non-empty directory fails, so of two pods copying the same image at once the first rename wins and the other's copy is deleted, and a reader never sees a part-written entry, since an entry exists only once it is complete. The pod uid comes from the downward API and the container name keeps two copies of one image within a pod apart. +_CAPABILITY_STORE_INCOMING = ".incoming" +_CAPABILITY_STORE_VOLUME = "capability-store" +_CAPABILITY_STORE_PREPARE = "capability-store-prepare" +_CAPABILITY_STORE_PUBLISH = "capability-store-publish" +_CAPABILITY_STORE_SYSROOT = "capability-store-sysroot" +_CAPABILITY_STORE_TOOL_CACHE = "capability-store-tool-cache" +_CAPABILITY_STORE_EXTERNALS_PUBLISH = "capability-store-publish-externals" +# The annotation that lists the store entries a pod uses, separated by spaces, which the node cache pruner reads so that it never removes an entry a pod on its node still uses. +_CAPABILITY_STORE_KEYS_ANNOTATION = "github-runner.exadev.io/capability-store-keys" +_IMAGE_DIGEST = re.compile(r"@sha256:([0-9a-f]{64})$") +_POD_UID_ENV = {"name": "POD_UID", "valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}} +# The chart's names for the runner externals copy the dind sidecar mounts (gha-runner-scale-set templates/_helpers.tpl: dind-init-container and dind-volume), which a values file that writes the dind containers out uses as the chart does. +_DIND_EXTERNALS_INIT = "init-dind-externals" +_DIND_EXTERNALS_VOLUME = "dind-externals" + + +def arc_capability_store_host_path_errors(host_path: Any) -> list[str]: + """Return what is wrong with github_runner_arc_capability_store_host_path: the same rules as the node cache's root (an absolute canonical path at least a dedicated directory deep, not a system directory or inside one), with a last component named capability-store or ending in -capability-store, since the role creates it and the pruner removes entries from it as root.""" + if not isinstance(host_path, str) or not host_path.startswith("/") or any(character in host_path for character in ":\n\r"): + return [f"the capability store's host path must be an absolute directory with no ':' or line break (got {host_path!r})"] + if host_path != _canonical_path(host_path): + return [f"the capability store's host path must not contain '//' or end with '/' (got {host_path!r}, which is {_canonical_path(host_path)!r} written plainly)"] + components = host_path.split("/")[1:] + if len(components) < _NODE_CACHE_HOST_PATH_MIN_COMPONENTS or ".." in components or "." in components: + return [f"the capability store's host path must be a dedicated directory at least {_NODE_CACHE_HOST_PATH_MIN_COMPONENTS} levels deep with no '.' or '..' (such as /var/lib/github-runner/capability-store) (got {host_path!r})"] + if not _CAPABILITY_STORE_HOST_PATH_NAME.match(components[-1]): + return [f"the capability store's host path must end in a directory named capability-store, or a lower-case name ending in -capability-store (got {host_path!r}), since the role creates it and removes entries from it as root"] + lowered = host_path.lower() + inside = [denied for denied in _NODE_CACHE_DENIED_HOST_PATHS if lowered == denied or lowered.startswith(f"{denied}/")] + return [f"the capability store's host path {host_path} is inside {inside[0]}, which the node itself uses"] if inside else [] + + +def arc_capability_store_path_errors(store_path: Any) -> list[str]: + """Return what is wrong with github_runner_arc_capability_store_path, where the store is mounted read-only on the runner container: empty when it is an absolute directory other than / with no colon or line break.""" + if not isinstance(store_path, str) or not store_path.startswith("/") or store_path == "/" or any(character in store_path for character in ":\n\r"): + return [f"the capability store path must be an absolute directory other than / with no ':' or line break (got {store_path!r})"] + return [] + + +def arc_capability_store_image_errors(image: Any) -> list[str]: + """Return what is wrong with github_runner_arc_capability_store_image: empty when it is pinned by digest (name@sha256:<64 hex digits>), as an image whose container runs as root on a hostPath must be.""" + if isinstance(image, str) and _IMAGE_DIGEST.search(image) and image.split("@")[0]: + return [] + return [f"github_runner_arc_capability_store_image must be pinned by digest (name@sha256:<64 hex digits>), since its preparation container runs as root on a hostPath (got {image!r})"] + + +def arc_capability_store_key(image: str) -> str | None: + """Return the store entry name of an image's payload, its sha256 digest in hex, or None when the image is not pinned by digest and so has no name that changes exactly when its content does.""" + match = _IMAGE_DIGEST.search(image) + return match.group(1) if match else None + + +def _capability_store_merged_key(keys: Sequence[str]) -> str: + """Return the store entry name of the sysroot merged from keys in their order: a hash of the ordered keys, so a different set or order of system libraries is a different entry, since a later one's file overwrites an earlier one's.""" + return "sysroot-" + hashlib.sha256(" ".join(keys).encode()).hexdigest() + + +def _capability_store_prepare_container(store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: + """Return the init container that creates the store's root and its incoming directory and gives both to the runner's user, as the node cache's preparation does: as root with CHOWN, and DAC_OVERRIDE to create the incoming directory in a root already handed over.""" + root = store["path"].rstrip("/") + return { + "name": _CAPABILITY_STORE_PREPARE, + "image": store["image"], + "command": ["/bin/sh", "-c", 'owner=$1; shift; for directory in "$@"; do mkdir -p "$directory" && chown "$owner" "$directory"; done', "prepare-capability-store", f"{int(runner_user)}:{int(runner_group)}", root, f"{root}/{_CAPABILITY_STORE_INCOMING}"], + "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}}, + "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"]}], + } + + +def _runner_security_context(runner_user: int, runner_group: int) -> dict[str, Any]: + """Return the securityContext of an init container that runs as the runner's user, as the per-pod copies do.""" + return {"runAsUser": int(runner_user), "runAsGroup": int(runner_group), "runAsNonRoot": True, "allowPrivilegeEscalation": False} + + +def _capability_store_copy_container(name: str, image: str, payload: str, key: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: + """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin and cp. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed.""" + root = store["path"].rstrip("/") + return { + "name": name, + "image": image, + "command": ["/bin/sh", "-c", '[ -d "$1/$2" ] || cp -R "$3" "$1/' + _CAPABILITY_STORE_INCOMING + '/$2.$POD_UID.$4"', "copy-to-capability-store", root, key, payload, name], + "env": [dict(_POD_UID_ENV)], + "securityContext": _runner_security_context(runner_user, runner_group), + "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"]}], + } + + +# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. +_PUBLISH_SCRIPT = ( + 'root=$1; shift; for key in "$@"; do' + ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; mv -T "$incoming" "$root/$key" 2>/dev/null || rm -rf "$incoming"; done;' + ' [ -d "$root/$key" ] || { echo "capability store: $key is not in $root after publishing" >&2; exit 1; };' + ' done' +) + + +def _capability_store_publish_container(name: str, keys: Sequence[str], store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: + """Return the init container that publishes this pod's copies of keys into the store with the store image (busybox), whose mv has -T.""" + return { + "name": name, + "image": store["image"], + "command": ["/bin/sh", "-c", _PUBLISH_SCRIPT, "publish-capability-store", store["path"].rstrip("/"), *keys], + "env": [dict(_POD_UID_ENV)], + "securityContext": _runner_security_context(runner_user, runner_group), + "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"]}], + } + + +# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then publishes it like any other entry. +_SYSROOT_MERGE_SCRIPT = ( + 'root=$1; merged=$2; shift 2; [ -d "$root/$merged" ] && exit 0;' + ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; mkdir "$incoming" || exit 1;' + ' for key in "$@"; do cp -alf "$root/$key/." "$incoming/" || exit 1; done;' + ' mv -T "$incoming" "$root/$merged" 2>/dev/null || rm -rf "$incoming";' + ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' +) + +# Fills the pod's own tool cache with a link to each // directory in the store and a copy of its .complete marker, which is how the Actions tool cache finds a tool. The version directories are the pod's own, so a setup action can still install another version or architecture beside them; the store itself is read-only to the pod. +_TOOL_CACHE_LINK_SCRIPT = ( + 'root=$1; cache=$2; shift 2; for key in "$@"; do for arch in "$root/$key"/*/*/*; do' + ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"};' + ' mkdir -p "$cache/${relative%/*}" && ln -sfn "$arch" "$cache/$relative" || exit 1;' + ' [ ! -e "$arch.complete" ] || cp "$arch.complete" "$cache/$relative.complete" || exit 1;' + ' done; done' +) + + +def _capability_store_volume(store: Mapping[str, Any]) -> dict[str, Any]: + return {"name": _CAPABILITY_STORE_VOLUME, "hostPath": {"path": store["host_path"], "type": "DirectoryOrCreate"}} + + +def _annotate_store_keys(template: dict[str, Any], keys: Sequence[str]) -> None: + """Add keys to the pod template's list of the store entries its pods use, keeping any already there.""" + annotations = template.setdefault("metadata", {}).setdefault("annotations", {}) + existing = annotations.get(_CAPABILITY_STORE_KEYS_ANNOTATION, "").split() + annotations[_CAPABILITY_STORE_KEYS_ANNOTATION] = " ".join(dict.fromkeys([*existing, *keys])) + + +def _ensure_store_prepared(spec: dict[str, Any], init_containers: list[dict[str, Any]], store: Mapping[str, Any], runner_user: int, runner_group: int) -> None: + """Add the store's volume to spec, and its preparation to the front of init_containers, unless spec already has them.""" + if not any(volume.get("name") == _CAPABILITY_STORE_VOLUME for volume in spec.get("volumes") or []): + spec["volumes"] = list(spec.get("volumes") or []) + [_capability_store_volume(store)] + if not any(container.get("name") == _CAPABILITY_STORE_PREPARE for container in [*(spec.get("initContainers") or []), *init_containers]): + init_containers.insert(0, _capability_store_prepare_container(store, runner_user, runner_group)) + + +def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> tuple[list[dict[str, Any]], list[str], str | None]: + """Return the init containers that put a profile's tool-cache and system-library payloads in the store and give the pod its view of them, the store entries the pod uses, and the merged sysroot's entry (None without a system library).""" + root = store["path"].rstrip("/") + copies: list[dict[str, Any]] = [] + tool_keys: list[str] = [] + sysroot_keys: list[str] = [] + for name in names: + capability = catalogue[name] + image = capability.get("image") or capability.get("sysroot_image") + if image is None: + continue + key = arc_capability_store_key(image) + assert key is not None + payload = _TOOL_IMAGE_PAYLOAD if capability.get("image") is not None else _SYSROOT_IMAGE_PAYLOAD + copies.append(_capability_store_copy_container(f"{_CAPABILITY_CONTAINER_PREFIX}{name}", image, payload, key, store, runner_user, runner_group)) + (tool_keys if capability.get("image") is not None else sysroot_keys).append(key) + keys = list(dict.fromkeys([*tool_keys, *sysroot_keys])) + containers = copies + [_capability_store_publish_container(_CAPABILITY_STORE_PUBLISH, keys, store, runner_user, runner_group)] + merged = _capability_store_merged_key(sysroot_keys) if sysroot_keys else None + if merged is not None: + containers.append({ + "name": _CAPABILITY_STORE_SYSROOT, + "image": store["image"], + "command": ["/bin/sh", "-c", _SYSROOT_MERGE_SCRIPT, "merge-capability-store-sysroot", root, merged, *sysroot_keys], + "env": [dict(_POD_UID_ENV)], + "securityContext": _runner_security_context(runner_user, runner_group), + "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"]}], + }) + if tool_keys: + containers.append({ + "name": _CAPABILITY_STORE_TOOL_CACHE, + "image": store["image"], + "command": ["/bin/sh", "-c", _TOOL_CACHE_LINK_SCRIPT, "link-capability-store-tool-cache", root, tool_cache_path, *dict.fromkeys(tool_keys)], + "securityContext": _runner_security_context(runner_user, runner_group), + "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"], "readOnly": True}, {"name": _TOOL_CACHE_VOLUME, "mountPath": tool_cache_path}], + }) + return containers, keys + ([merged] if merged is not None else []), merged + + +def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str, Any] | None, runner_user: int, runner_group: int) -> dict[str, Any]: + """Move a scale set's copy of the runner's externals for the dind sidecar into the capability store. + + Applies to values that write the dind containers out (a values file, or the role's registry cache wiring), whose init container named init-dind-externals copies the runner image's externals into the volume named dind-externals, as the chart's own dind template does. That copy becomes a store entry named for the image's digest, published like a capability's, and the dind container mounts it read-only through a subPath of the store. Values without that init container, or a profile without the store, are returned as given. + + Args: + values: the release's rendered Helm values. store: the profile's capability store settings ({host_path, path, image}), or None when the profile does not use the store. runner_user, runner_group: the runner's user and group ids, which the copy and publish containers run as. + + Returns: + A dict with ``errors`` (empty when the values can use the store) and ``values`` (the values with the copy moved, or as given). + """ + if store is None: + return {"errors": [], "values": values} + result = copy.deepcopy(dict(values)) + template = result.setdefault("template", {}) + spec = template.setdefault("spec", {}) + init_containers = list(spec.get("initContainers") or []) + copier = next((container for container in init_containers if container.get("name") == _DIND_EXTERNALS_INIT), None) + if copier is None: + return {"errors": [], "values": values} + key = arc_capability_store_key(str(copier.get("image", ""))) + if key is None: + return {"errors": [f"the capability store needs {_DIND_EXTERNALS_INIT}'s image pinned by digest (name@sha256:<64 hex digits>), since its store entry is named for the digest (got {copier.get('image')!r})"], "values": values} + index = init_containers.index(copier) + replacement = [ + _capability_store_copy_container(_DIND_EXTERNALS_INIT, copier["image"], _RUNNER_EXTERNALS_DIR, key, store, runner_user, runner_group), + _capability_store_publish_container(_CAPABILITY_STORE_EXTERNALS_PUBLISH, [key], store, runner_user, runner_group), + ] + init_containers[index:index + 1] = replacement + spec["volumes"] = [volume for volume in spec.get("volumes") or [] if volume.get("name") != _DIND_EXTERNALS_VOLUME] + for container in spec.get("containers") or []: + container["volumeMounts"] = [ + {"name": _CAPABILITY_STORE_VOLUME, "mountPath": mount["mountPath"], "subPath": key, "readOnly": True} if mount.get("name") == _DIND_EXTERNALS_VOLUME else mount + for mount in container.get("volumeMounts") or [] + ] + prefix: list[dict[str, Any]] = [] + _ensure_store_prepared(spec, prefix, store, runner_user, runner_group) + spec["initContainers"] = prefix + init_containers + _annotate_store_keys(template, [key]) + return {"errors": [], "values": result} + + +def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str, store: Mapping[str, Any] | None) -> dict[str, Any]: """Add a profile's capabilities to its scale set's Helm values. Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay. Args: - values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. + values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it once per node instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, filled with links to the store, the sysroot is the store's merged entry mounted read-only, and the pod template is annotated with the entries it uses. Returns: A dict with ``errors`` (empty when the values can carry the capabilities), ``values`` (the values with the capabilities added, or as given when there are none or errors) and ``hooks`` (the ConfigMap's data apart from the dispatcher, which the role adds from its own file; empty when the profile needs no ConfigMap). @@ -1061,6 +1299,14 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu errors.extend(_sysroot_path_errors(sysroot_path, {"tool cache path": tool_cache_path, "job-started hooks directory": hooks_dir})) if node_cache_capabilities: errors.extend(_node_cache_path_errors(node_cache_path, node_cache_host_path, {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir})) + if store is not None and (tool_capabilities or sysroot_capabilities): + store_path_errors = arc_capability_store_path_errors(store["path"]) + errors.extend(store_path_errors or _overlap_errors("capability store path", store["path"], {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir, "node cache path": node_cache_path})) + errors.extend( + f"capability {name}'s image must be pinned by digest (name@sha256:<64 hex digits>) for the capability store, since its store entry is named for the digest" + for name, capability in [*tool_capabilities, *sysroot_capabilities] + if arc_capability_store_key(capability.get("image") or capability["sysroot_image"]) is None + ) tool_paths = [capability["path"] for _, capability in tool_capabilities if capability.get("path") is not None] hooks: dict[str, str] = {} if tool_paths: @@ -1078,7 +1324,12 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu volumes.append({"name": _TOOL_CACHE_VOLUME, "emptyDir": {}}) mounts.append({"name": _TOOL_CACHE_VOLUME, "mountPath": tool_cache_path}) env.append({"name": _TOOL_CACHE_ENV, "value": tool_cache_path}) - if sysroot_capabilities: + use_store = store is not None and bool(tool_capabilities or sysroot_capabilities) + if use_store: + assert store is not None + volumes.append(_capability_store_volume(store)) + mounts.append({"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"], "readOnly": True}) + elif sysroot_capabilities: volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": {}}) mounts.append({"name": _SYSROOT_VOLUME, "mountPath": sysroot_path}) if node_cache_capabilities: @@ -1100,6 +1351,20 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu if errors: return {"errors": errors, "values": values, "hooks": {}} assert runner is not None + if use_store: + assert store is not None + store_containers, store_keys, merged = _capability_store_init_containers(names, catalogue, tool_cache_path, store, runner_user, runner_group) + if merged is not None: + mounts.append({"name": _CAPABILITY_STORE_VOLUME, "mountPath": sysroot_path, "subPath": merged, "readOnly": True}) + init_containers = [_capability_store_prepare_container(store, runner_user, runner_group), *store_containers] + [ + _node_cache_init_container(name, catalogue[name], node_cache_path, runner_user, runner_group) for name in names if catalogue[name].get("node_cache") is not None + ] + _annotate_store_keys(result.setdefault("template", {}), store_keys) + spec["initContainers"] = list(spec.get("initContainers") or []) + init_containers + spec["volumes"] = list(spec.get("volumes") or []) + volumes + runner["env"] = list(runner.get("env") or []) + env + runner["volumeMounts"] = list(runner.get("volumeMounts") or []) + mounts + return {"errors": [], "values": result, "hooks": hooks} # In the profile's order, so a later system-library capability's files overwrite an earlier one's. init_containers = [ _copy_init_container(name, catalogue[name]["image"], _TOOL_IMAGE_PAYLOAD, _TOOL_CACHE_VOLUME, tool_cache_path, runner_user, runner_group) @@ -1466,6 +1731,11 @@ def filters(self) -> dict[str, Any]: "arc_runner_placement": arc_runner_placement, "arc_capability_errors": arc_capability_errors, "arc_capability_pod": arc_capability_pod, + "arc_capability_store_externals": arc_capability_store_externals, + "arc_capability_store_host_path_errors": arc_capability_store_host_path_errors, + "arc_capability_store_path_errors": arc_capability_store_path_errors, + "arc_capability_store_key": arc_capability_store_key, + "arc_capability_store_image_errors": arc_capability_store_image_errors, "arc_node_cache_profiles": arc_node_cache_profiles, "arc_node_cache_host_path_errors": arc_node_cache_host_path_errors, "arc_node_cache_root": arc_node_cache_root, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index ef949b8..b0c3847 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -24,10 +24,12 @@ Before touching the cluster the role then checks the secrets it is about to writ - `autoscale`: `true` on at most one profile in the whole inventory makes it the scale set the autoscaler manages. - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it bounds the ceiling the autoscaler works out from the nodes, leaving `maxRunners` as the floor. - `capabilities`: names of entries in `github_runner_arc_capabilities` that the profile's runner pods carry, in order (see [Capabilities](#capabilities)). + - `capability_store`: `true` copies the capabilities' tool-cache and system-library payloads, and the dind sidecar's copy of the runner's externals, once per node into the capability store instead of into every pod (see [The capability store](#the-capability-store)). Default `false`. - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`. - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. - `github_runner_arc_node_cache_path`, `github_runner_arc_node_cache_host_path`, `github_runner_arc_node_cache_size_limit_gib`, `github_runner_arc_node_cache_hard_limit_factor`, `github_runner_arc_node_cache_prune_seconds`: where a node-cache capability's volume is mounted on the runner container (`/node-cache`), the dedicated directory on each node under which each profile's cache lives (`/var/lib/github-runner/node-cache`), the most gibibytes a profile's cache may hold before the pruner empties it (20; 0 for no pruner, a negative value is an error), the hard limit as a multiple of that (2) past which a busy cache is reported on every round, and the seconds between the pruner's checks (300, at least 1). See [Capabilities](#capabilities). +- `github_runner_arc_capability_store_host_path`, `github_runner_arc_capability_store_path`, `github_runner_arc_capability_store_image`: the dedicated directory on each node the capability store lives in (`/var/lib/github-runner/capability-store`, with the same rules as the node cache's, its last component `capability-store` or ending in `-capability-store`), where it is mounted read-only on the runner container (`/opt/capability-store`), and the digest-pinned busybox image its own init containers run. See [The capability store](#the-capability-store). - `github_runner_arc_registry_cache_enabled` (default `false`) and the other `github_runner_arc_registry_cache_*` variables: a pull-through registry cache every Docker-in-Docker profile pulls through, falling back to each registry when the cache is down (see [Registry cache](#registry-cache)). - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`. - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart. The controller chart's CRDs are applied (server-side) from `github_runner_arc_controller_chart_ref` at the controller's version before every controller install or upgrade, because Helm never upgrades a chart's CRDs itself; without that, a newer scale-set chart is rejected for fields the cluster's CRDs predate. Move the two pins together, and the CRDs follow. `tests/arc_upgrade/run.sh` covers the upgrade in a kind cluster. @@ -301,7 +303,23 @@ When a profile has a hook capability, a `path` or a system-library capability, t The tool cache, the sysroot, the node cache and the hook are on the runner container, so they serve the steps that run there: every step in the default container mode and in Docker-in-Docker mode (`container_mode: dind`). A job that runs in a `container:` reaches the tool cache only through the runner's own mapping of it, which in Docker-in-Docker mode points at the Docker daemon's filesystem, where the cache is not mounted, and it does not see the sysroot at all. The `kubernetes` container modes run every step in a separate pod, so the role refuses capabilities on a profile whose values use them. It also refuses them when the values define no runner container (the role's own template does), when the runner container already sets `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, when a volume already uses the name `tool-cache`, `sysroot` or `job-started-hooks`, or, for a profile with a system-library capability, when `github_runner_arc_sysroot_path` is not an absolute directory without a `:` or overlaps the tool cache or the hooks directory. During validation it refuses a system-library capability whose `env` sets one of the search paths, `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, a name that is not a variable name or a value that is not a single-line string (a line break would add a line to `GITHUB_ENV`), and two of a profile's system-library capabilities that set one variable to different values, since they share one sysroot. A values file's own init containers, volumes, environment and mounts are kept, and the capabilities' are added after them, by the `exadev.github_runner.arc_capability_pod` filter. `exadev.github_runner.arc_capability_errors` checks the catalogue and each profile's references during validation, before anything changes. -Each tool-cache and system-library capability adds its copy to every pod's start, which is not yet measured. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage. +Each tool-cache and system-library capability adds its copy to every pod's start. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage, and every pod start writes all of it. A profile with `capability_store: true` writes it once per node instead (see [The capability store](#the-capability-store)). + +### The capability store + +Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied once per node into `github_runner_arc_capability_store_host_path` and the pods use that copy. + +An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. A pod's init containers do the work, in this order: + +- `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. +- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. +- `capability-store-publish` renames each copy onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete, two pods starting together never see each other's half-written copy, and the container fails unless every entry the pod needs is in the store. +- `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, published the same way. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. +- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, with a link to each `//` directory in the store and a copy of its `.complete` marker. The version directories are the pod's own, so a setup action can install another version or architecture beside them, and the store is mounted read-only on the runner container, so a job cannot change what other pods use. + +`arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume. Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. + +The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template in a store profile's namespace lists and no unfinished pod on its node lists, after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. The pruner's Role in such a profile's namespace also lists `autoscalingrunnersets.actions.github.com`. `tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the tool images or the test. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index bb6c3c2..ce07aec 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -25,6 +25,13 @@ github_runner_arc_node_cache_path: "/node-cache" # Where on the node the cache lives, as the directory every profile's own directory (/) sits under, so profiles never share a cache. It must be a dedicated directory the role may create and empty as root: written without a repeated or trailing slash (every path of the cache is built from this one string), at least three levels deep, with a last component ending in node-cache, and not inside a system directory such as /etc or /var/lib/kubelet. A symbolic link at or above the path defeats this validation. Each profile's directory is a hostPath volume created when a pod first needs it. Its lifetime is the node's: it survives pods, and is emptied by the pruner below when it outgrows its limit. github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" +# Where on the node the capability store lives, for profiles with capability_store on: one entry per image a profile's capabilities (and the dind sidecar's externals) copy from, named for the image's digest, written once per node and mounted read-only by the pods. A dedicated directory the role creates and the pruner removes entries from as root, with the same rules as the node cache's root, whose last component is capability-store or ends in -capability-store. +github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" +# Where the store is mounted, read-only, on the runner container: the tool cache's links point into it. +github_runner_arc_capability_store_path: "/opt/capability-store" +# The image the store's own init containers run (creating the root, publishing a copy with mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chown, mv, cp, ln and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. +github_runner_arc_capability_store_image: "docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" + # The most gibibytes one profile's cache may hold on a node before the pruner empties it. The limit is per profile, so a node's worst case is the sum of the limits of the profiles with a cache on it, times github_runner_arc_node_cache_hard_limit_factor. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) empties a cache only while no runner pod of its profile runs on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow; a negative value, a leading zero, a fraction or a value above 1073741824 is an error (the pruner's own arithmetic bounds it, and the role refuses what the pruner would). github_runner_arc_node_cache_size_limit_gib: 20 diff --git a/roles/github_runner_arc/tasks/render_scale_set_values.yml b/roles/github_runner_arc/tasks/render_scale_set_values.yml index 84fa876..bd28f50 100644 --- a/roles/github_runner_arc/tasks/render_scale_set_values.yml +++ b/roles/github_runner_arc/tasks/render_scale_set_values.yml @@ -1,5 +1,14 @@ --- -# Renders one scale-set profile's Helm values on the control node, with `profile` (the expanded profile from plugins/filter/arc.py) and `org` (its org entry) set by the includer: the profile's values file (or the role's default template), the role's own overlay merged over it, then the profile's capabilities added (plugins/filter/arc.py's arc_capability_pod), and then, with the registry cache on, its Docker daemon pointed at the cache (arc_registry_cache_pod). Sets github_runner_arc_scale_set_values, the release's values, github_runner_arc_job_started_data, the job-started ConfigMap's data (empty when the profile needs none), github_runner_arc_registry_cache_wired, whether the profile's pods use the registry cache, and github_runner_arc_registry_cache_data, the data of the registry cache ConfigMap they then mount. Included by install_scale_set_profile.yml, and by tests/capabilities/render.yml so the tests render exactly what the role installs. +# Renders one scale-set profile's Helm values on the control node, with `profile` (the expanded profile from plugins/filter/arc.py) and `org` (its org entry) set by the includer: the profile's values file (or the role's default template), the role's own overlay merged over it, then the profile's capabilities added (plugins/filter/arc.py's arc_capability_pod), then, with the registry cache on, its Docker daemon pointed at the cache (arc_registry_cache_pod), and last, with the profile's capability store on, the dind sidecar's copy of the runner's externals moved into the store (arc_capability_store_externals). Sets github_runner_arc_scale_set_values, the release's values, github_runner_arc_job_started_data, the job-started ConfigMap's data (empty when the profile needs none), github_runner_arc_registry_cache_wired, whether the profile's pods use the registry cache, and github_runner_arc_registry_cache_data, the data of the registry cache ConfigMap they then mount. Included by install_scale_set_profile.yml, and by tests/capabilities/render.yml so the tests render exactly what the role installs. + +- name: Set the capability store settings for {{ profile.label }} + ansible.builtin.set_fact: + # None when the profile copies into each pod; the filters take the store's root on the node, its mount path in the pod and the image that publishes its entries. + github_runner_arc_capability_store_settings: >- + {{ + {'host_path': github_runner_arc_capability_store_host_path, 'path': github_runner_arc_capability_store_path, 'image': github_runner_arc_capability_store_image} + if profile.capability_store else none + }} - name: Render the Helm values for {{ profile.label }} ansible.builtin.set_fact: @@ -12,7 +21,7 @@ profile.capabilities, github_runner_arc_capabilities, github_runner_arc_tool_cache_path, github_runner_arc_sysroot_path, github_runner_arc_node_cache_path, github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_dir(profile.namespace, profile.release), github_runner_arc_runner_uid, github_runner_arc_runner_gid, - github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap) + github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap, github_runner_arc_capability_store_settings) }} vars: github_runner_arc_values_source: >- @@ -58,3 +67,18 @@ github_runner_arc_registry_cache_wired: "{{ github_runner_arc_registry_cache_render.wired }}" # The registry cache ConfigMap's data for the profile's namespace; empty when its pods do not use the cache, and the ConfigMap is then removed. github_runner_arc_registry_cache_data: "{{ (github_runner_arc_registry_cache | exadev.github_runner.arc_registry_cache_configmap) if github_runner_arc_registry_cache_render.wired else {} }}" + +# After the registry cache, which may write the dind containers out with their externals copy; values that leave the dind sidecar to the chart have no copy here to move. +- name: Move the dind sidecar's externals copy into the capability store for {{ profile.label }} + ansible.builtin.set_fact: + github_runner_arc_capability_store_render: >- + {{ github_runner_arc_scale_set_values | exadev.github_runner.arc_capability_store_externals(github_runner_arc_capability_store_settings, github_runner_arc_runner_uid, github_runner_arc_runner_gid) }} + +- name: Fail when the values cannot keep the externals copy in the capability store for {{ profile.label }} + ansible.builtin.fail: + msg: "{{ github_runner_arc_capability_store_render.errors | map('regex_replace', '^', profile.label ~ ': ') | list }}" + when: github_runner_arc_capability_store_render.errors | length > 0 + +- name: Keep the values with the externals copy in the capability store for {{ profile.label }} + ansible.builtin.set_fact: + github_runner_arc_scale_set_values: "{{ github_runner_arc_capability_store_render['values'] }}" diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index 1dce2c1..f1cc00a 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -40,9 +40,19 @@ | exadev.github_runner.arc_bounded_integer_errors('github_runner_arc_node_cache_hard_limit_factor', github_runner_arc_node_cache_max_hard_limit_factor, github_runner_arc_node_cache_positive_integer_pattern)) + (github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_host_path_errors) + (github_runner_arc_node_cache_path | exadev.github_runner.arc_node_cache_path_errors) + + (github_runner_arc_capability_store_host_path | exadev.github_runner.arc_capability_store_host_path_errors) + + (github_runner_arc_capability_store_path | exadev.github_runner.arc_capability_store_path_errors) + + (github_runner_arc_capability_store_image | exadev.github_runner.arc_capability_store_image_errors) }} when: github_runner_arc_node_cache_problems | length > 0 +# The pruner is what removes the store's unused entries, and a size limit of zero installs no pruner, so the store would only grow. +- name: Fail when a profile uses the capability store without the pruner + ansible.builtin.fail: + msg: >- + {{ github_runner_arc_local.profiles | selectattr('capability_store') | map(attribute='label') | join(', ') }} use the capability store, which needs the node cache pruner to remove the entries no scale set uses, but github_runner_arc_node_cache_size_limit_gib is 0, which installs no pruner + when: github_runner_arc_node_cache_size_limit_gib | int == 0 and github_runner_arc_local.profiles | selectattr('capability_store') | list | length > 0 + - name: Fail on invalid registry cache settings ansible.builtin.fail: msg: "{{ github_runner_arc_registry_cache_problems }}" diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 673483c..0c09e0d 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -54,7 +54,7 @@ def expanded(profile: dict[str, Any]) -> dict[str, Any]: def pod(values: dict[str, Any], names: list[str]) -> dict[str, Any]: """Add names' capabilities from CATALOGUE to values with the role's default settings.""" - result: dict[str, Any] = arc.arc_capability_pod(values, names, CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result: dict[str, Any] = arc.arc_capability_pod(values, names, CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) return result @@ -306,7 +306,7 @@ def test_every_kind_together_keeps_the_profiles_order_and_shares_one_sysroot(sel self.assertTrue(result["hooks"]["sysroot-env"].endswith("CMAKE_PREFIX_PATH+=/opt/sysroot\nOPAL_PREFIX=/opt/sysroot\n")) def test_the_sysroot_follows_its_configured_path(self) -> None: - result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], {"mpi": {"sysroot_image": "ghcr.io/example/m:1"}}, TOOL_CACHE, "/srv/libs/", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], {"mpi": {"sysroot_image": "ghcr.io/example/m:1"}}, TOOL_CACHE, "/srv/libs/", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) self.assertEqual(result["errors"], []) self.assertEqual(result["values"]["template"]["spec"]["initContainers"][0]["command"][-1], "/srv/libs/") self.assertEqual(result["hooks"]["sysroot-env"].splitlines()[:2], ["PATH+=/srv/libs/bin", "LD_LIBRARY_PATH+=/srv/libs/lib"]) @@ -322,12 +322,12 @@ def test_a_sysroot_path_that_cannot_carry_a_sysroot_is_an_error(self) -> None: } for case, (path, message) in cases.items(): with self.subTest(case): - result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], CATALOGUE, TOOL_CACHE, path, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], CATALOGUE, TOOL_CACHE, path, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) self.assertIs(result["values"], DEFAULT_VALUES) def test_the_sysroot_path_is_not_checked_for_a_profile_without_a_system_library(self) -> None: - self.assertEqual(arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, "relative", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP)["errors"], []) + self.assertEqual(arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, "relative", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None)["errors"], []) def test_a_volume_already_named_sysroot_is_an_error(self) -> None: values = {"template": {"spec": {"volumes": [{"name": "sysroot", "emptyDir": {}}], "containers": [{"name": "runner"}]}}} @@ -371,7 +371,7 @@ def test_the_init_container_command_makes_and_gives_away_every_directory_it_name def test_a_node_cache_capability_shares_one_volume_with_every_capability_of_the_profile(self) -> None: catalogue = {**CATALOGUE, "yarn-cache": {"node_cache": {"init_image": BUSYBOX, "dirs": {"YARN_CACHE_FOLDER": "yarn"}}}} - result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store", "yarn-cache", "node"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store", "yarn-cache", "node"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) self.assertEqual(result["errors"], []) spec = result["values"]["template"]["spec"] self.assertEqual([volume["name"] for volume in spec["volumes"]], ["tool-cache", "node-cache"]) @@ -379,7 +379,7 @@ def test_a_node_cache_capability_shares_one_volume_with_every_capability_of_the_ self.assertEqual({variable["name"] for variable in runner(result["values"])["env"]}, {"RUNNER_TOOL_CACHE", "pnpm_config_store_dir", "pnpm_config_enable_global_virtual_store", "YARN_CACHE_FOLDER"}) def test_a_profile_without_a_node_cache_capability_gets_no_node_volume_and_its_path_is_not_checked(self) -> None: - result = arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, SYSROOT, "relative", "also/relative", 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, SYSROOT, "relative", "also/relative", 1001, 1001, HOOKS_DIR, CONFIGMAP, None) self.assertEqual(result["errors"], []) self.assertNotIn("node-cache", [volume["name"] for volume in result["values"]["template"]["spec"]["volumes"]]) @@ -397,7 +397,7 @@ def test_node_cache_paths_that_cannot_carry_a_cache_are_errors(self) -> None: } for case, (mount, host, message) in cases.items(): with self.subTest(case): - result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store"], CATALOGUE, TOOL_CACHE, SYSROOT, mount, host, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store"], CATALOGUE, TOOL_CACHE, SYSROOT, mount, host, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) def test_two_node_cache_capabilities_setting_one_variable_differently_is_an_error(self) -> None: @@ -405,7 +405,7 @@ def test_two_node_cache_capabilities_setting_one_variable_differently_is_an_erro "a": {"node_cache": {"init_image": BUSYBOX, "dirs": {"STORE": "one"}}}, "b": {"node_cache": {"init_image": BUSYBOX, "dirs": {"STORE": "two"}}}, } - result = arc.arc_capability_pod(DEFAULT_VALUES, ["a", "b"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["a", "b"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) self.assertEqual(result["errors"], ["node-cache capabilities a and b set STORE to different values"]) def test_a_volume_already_named_node_cache_is_an_error(self) -> None: @@ -592,5 +592,267 @@ def test_a_missing_subdirectory_fails_the_job(self) -> None: self.assertIn("does not exist", completed.stderr) + +DIGEST_A = "a" * 64 +DIGEST_B = "b" * 64 +DIGEST_C = "c" * 64 +DIGEST_RUNNER = "d" * 64 +STORE = {"host_path": "/var/lib/example/capability-store", "path": "/opt/capability-store", "image": BUSYBOX} +STORE_CATALOGUE: dict[str, Any] = { + "node": {"image": f"ghcr.io/example/toolcache-node:22@sha256:{DIGEST_A}"}, + "mpi": {"sysroot_image": f"ghcr.io/example/sysroot-openmpi:5@sha256:{DIGEST_B}", "env": {"OPAL_PREFIX": SYSROOT}}, + "libpq": {"sysroot_image": f"ghcr.io/example/sysroot-libpq:16@sha256:{DIGEST_C}"}, + "pnpm-store": CATALOGUE["pnpm-store"], + "untagged": {"image": "ghcr.io/example/toolcache-go:1.23"}, +} + + +def store_pod(values: dict[str, Any], names: list[str], catalogue: dict[str, Any] = STORE_CATALOGUE) -> dict[str, Any]: + """Add names' capabilities with the capability store on.""" + result: dict[str, Any] = arc.arc_capability_pod(values, names, catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, STORE) + return result + + +class CapabilityStoreProfileTest(unittest.TestCase): + def test_a_profile_copies_into_each_pod_unless_it_turns_the_store_on(self) -> None: + self.assertFalse(expanded({"max_runners": 1})["capability_store"]) + self.assertTrue(expanded({"max_runners": 1, "capability_store": True})["capability_store"]) + + def test_a_capability_store_that_is_not_a_boolean_is_an_error(self) -> None: + result = arc.arc_profiles([{"name": "Example", "image": "i:1", "scale_set_profiles": [{"max_runners": 1, "capability_store": "sometimes"}]}], require_app_id=False) + self.assertTrue(any("capability_store must be a boolean" in error for error in result["errors"]), result["errors"]) + + +class CapabilityStoreSettingsTest(unittest.TestCase): + def test_a_dedicated_store_directory_is_accepted(self) -> None: + self.assertEqual(arc.arc_capability_store_host_path_errors("/var/lib/github-runner/capability-store"), []) + self.assertEqual(arc.arc_capability_store_host_path_errors("/srv/ci/runner-capability-store"), []) + + def test_a_store_directory_that_is_not_dedicated_is_refused(self) -> None: + for path, message in ( + ("var/lib/x/capability-store", "absolute directory"), + ("/var/lib//capability-store", "must not contain '//'"), + ("/srv/capability-store", "at least 3 levels deep"), + ("/var/lib/github-runner/store", "named capability-store"), + ("/var/lib/kubelet/x/capability-store", "inside /var/lib/kubelet"), + ): + with self.subTest(path=path): + errors = arc.arc_capability_store_host_path_errors(path) + self.assertEqual(len(errors), 1, errors) + self.assertIn(message, errors[0]) + + def test_the_store_mount_path_must_be_an_absolute_directory(self) -> None: + self.assertEqual(arc.arc_capability_store_path_errors("/opt/capability-store"), []) + for path in ("relative", "/", "/a:b", 3): + with self.subTest(path=path): + self.assertEqual(len(arc.arc_capability_store_path_errors(path)), 1) + + def test_an_entry_is_named_for_its_images_digest_and_an_unpinned_image_has_no_name(self) -> None: + self.assertEqual(arc.arc_capability_store_key(f"ghcr.io/example/x:1@sha256:{DIGEST_A}"), DIGEST_A) + self.assertIsNone(arc.arc_capability_store_key("ghcr.io/example/x:1")) + + +class CapabilityStorePodTest(unittest.TestCase): + def test_without_the_store_the_copies_stay_in_each_pod(self) -> None: + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], STORE_CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) + spec = result["values"]["template"]["spec"] + self.assertEqual([volume["name"] for volume in spec["volumes"]], ["sysroot", "job-started-hooks"]) + self.assertNotIn("metadata", result["values"]["template"]) + + def test_with_the_store_each_payload_is_copied_once_per_node_and_published_before_the_pod_uses_it(self) -> None: + result = store_pod(DEFAULT_VALUES, ["node", "mpi", "libpq", "pnpm-store"]) + self.assertEqual(result["errors"], []) + template = result["values"]["template"] + spec = template["spec"] + names = [container["name"] for container in spec["initContainers"]] + self.assertEqual(names, ["capability-store-prepare", "capability-node", "capability-mpi", "capability-libpq", "capability-store-publish", "capability-store-sysroot", "capability-store-tool-cache", "capability-pnpm-store"]) + containers = {container["name"]: container for container in spec["initContainers"]} + # The copy runs the capability's own image as the runner user, only when the store lacks the entry, into a name of its own under .incoming. + copier = containers["capability-node"] + self.assertEqual(copier["image"], STORE_CATALOGUE["node"]["image"]) + self.assertEqual(copier["command"][3:], ["copy-to-capability-store", "/opt/capability-store", DIGEST_A, "/toolcache", "capability-node"]) + self.assertEqual(copier["securityContext"], {"runAsUser": 1001, "runAsGroup": 1001, "runAsNonRoot": True, "allowPrivilegeEscalation": False}) + self.assertEqual(copier["env"], [{"name": "POD_UID", "valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}}]) + self.assertEqual(containers["capability-mpi"]["command"][6], "/sysroot") + self.assertEqual(containers["capability-store-publish"]["command"][4:], ["/opt/capability-store", DIGEST_A, DIGEST_B, DIGEST_C]) + self.assertEqual(containers["capability-store-publish"]["image"], BUSYBOX) + # Only the preparation runs as root, with the two capabilities it needs. + self.assertEqual(containers["capability-store-prepare"]["securityContext"]["capabilities"], {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}) + merged = containers["capability-store-sysroot"]["command"][5] + self.assertRegex(merged, r"^sysroot-[0-9a-f]{64}$") + self.assertEqual(containers["capability-store-sysroot"]["command"][6:], [DIGEST_B, DIGEST_C]) + self.assertEqual(containers["capability-store-tool-cache"]["command"][4:], ["/opt/capability-store", TOOL_CACHE, DIGEST_A]) + self.assertEqual(containers["capability-store-tool-cache"]["volumeMounts"], [{"name": "capability-store", "mountPath": "/opt/capability-store", "readOnly": True}, {"name": "tool-cache", "mountPath": TOOL_CACHE}]) + volumes = {volume["name"]: volume for volume in spec["volumes"]} + self.assertEqual(volumes["capability-store"], {"name": "capability-store", "hostPath": {"path": STORE["host_path"], "type": "DirectoryOrCreate"}}) + self.assertEqual(volumes["tool-cache"], {"name": "tool-cache", "emptyDir": {}}) + self.assertNotIn("sysroot", volumes) + mounts = runner(result["values"])["volumeMounts"] + self.assertIn({"name": "capability-store", "mountPath": "/opt/capability-store", "readOnly": True}, mounts) + self.assertIn({"name": "capability-store", "mountPath": SYSROOT, "subPath": merged, "readOnly": True}, mounts) + self.assertIn({"name": "tool-cache", "mountPath": TOOL_CACHE}, mounts) + self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{DIGEST_A} {DIGEST_B} {DIGEST_C} {merged}") + + def test_a_different_order_of_system_libraries_is_a_different_merged_sysroot(self) -> None: + first = store_pod(DEFAULT_VALUES, ["mpi", "libpq"])["values"]["template"]["spec"]["initContainers"] + second = store_pod(DEFAULT_VALUES, ["libpq", "mpi"])["values"]["template"]["spec"]["initContainers"] + merged = [next(container for container in containers if container["name"] == "capability-store-sysroot")["command"][5] for containers in (first, second)] + self.assertNotEqual(merged[0], merged[1]) + + def test_with_the_store_an_unpinned_capability_image_is_an_error(self) -> None: + result = store_pod(DEFAULT_VALUES, ["untagged"]) + self.assertEqual(result["errors"], ["capability untagged's image must be pinned by digest (name@sha256:<64 hex digits>) for the capability store, since its store entry is named for the digest"]) + + def test_a_store_path_that_overlaps_a_capability_mount_is_an_error(self) -> None: + result = arc.arc_capability_pod(DEFAULT_VALUES, ["node"], STORE_CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, {**STORE, "path": f"{TOOL_CACHE}/store"}) + self.assertEqual(len(result["errors"]), 1, result["errors"]) + self.assertIn("capability store path", result["errors"][0]) + + def test_a_profile_with_only_hooks_or_a_node_cache_uses_no_store(self) -> None: + result = store_pod(DEFAULT_VALUES, ["pnpm-store"]) + self.assertNotIn("capability-store", [volume["name"] for volume in result["values"]["template"]["spec"]["volumes"]]) + self.assertNotIn("metadata", result["values"]["template"]) + + +DIND_VALUES: dict[str, Any] = { + "template": { + "spec": { + "initContainers": [ + {"name": "init-dind-externals", "image": f"ghcr.io/example/runner:2@sha256:{DIGEST_RUNNER}", "command": ["cp", "-r", "/home/runner/externals/.", "/home/runner/tmpDir/"], "volumeMounts": [{"name": "dind-externals", "mountPath": "/home/runner/tmpDir"}]}, + {"name": "init-dind-mtu", "image": BUSYBOX}, + ], + "containers": [ + {"name": "runner", "volumeMounts": [{"name": "work", "mountPath": "/home/runner/_work"}]}, + {"name": "dind", "volumeMounts": [{"name": "work", "mountPath": "/home/runner/_work"}, {"name": "dind-externals", "mountPath": "/home/runner/externals"}]}, + ], + "volumes": [{"name": "work", "emptyDir": {}}, {"name": "dind-externals", "emptyDir": {}}], + } + } +} + + +class CapabilityStoreExternalsTest(unittest.TestCase): + def test_the_dind_externals_copy_moves_into_the_store_and_dind_mounts_its_entry_read_only(self) -> None: + result = arc.arc_capability_store_externals(DIND_VALUES, STORE, 1001, 1001) + self.assertEqual(result["errors"], []) + template = result["values"]["template"] + spec = template["spec"] + self.assertEqual([container["name"] for container in spec["initContainers"]], ["capability-store-prepare", "init-dind-externals", "capability-store-publish-externals", "init-dind-mtu"]) + copier = spec["initContainers"][1] + self.assertEqual(copier["command"][3:], ["copy-to-capability-store", "/opt/capability-store", DIGEST_RUNNER, "/home/runner/externals", "init-dind-externals"]) + self.assertEqual([volume["name"] for volume in spec["volumes"]], ["work", "capability-store"]) + dind = next(container for container in spec["containers"] if container["name"] == "dind") + self.assertEqual(dind["volumeMounts"][1], {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": DIGEST_RUNNER, "readOnly": True}) + self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], DIGEST_RUNNER) + self.assertEqual(DIND_VALUES["template"]["spec"]["volumes"][1]["name"], "dind-externals") + + def test_after_the_capabilities_the_store_is_prepared_once_and_the_annotation_keeps_both_lists(self) -> None: + values = store_pod(DIND_VALUES, ["node"])["values"] + result = arc.arc_capability_store_externals(values, STORE, 1001, 1001) + spec = result["values"]["template"]["spec"] + self.assertEqual([container["name"] for container in spec["initContainers"]].count("capability-store-prepare"), 1) + self.assertEqual([volume["name"] for volume in spec["volumes"]].count("capability-store"), 1) + self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{DIGEST_A} {DIGEST_RUNNER}") + + def test_without_the_store_or_without_an_externals_copy_the_values_are_left_alone(self) -> None: + self.assertIs(arc.arc_capability_store_externals(DIND_VALUES, None, 1001, 1001)["values"], DIND_VALUES) + self.assertIs(arc.arc_capability_store_externals(DEFAULT_VALUES, STORE, 1001, 1001)["values"], DEFAULT_VALUES) + + def test_an_unpinned_runner_image_cannot_use_the_store(self) -> None: + values = json.loads(json.dumps(DIND_VALUES)) + values["template"]["spec"]["initContainers"][0]["image"] = "ghcr.io/example/runner:2" + self.assertIn("pinned by digest", arc.arc_capability_store_externals(values, STORE, 1001, 1001)["errors"][0]) + + +class CapabilityStoreScriptTest(unittest.TestCase): + """Runs the store's init container commands with sh against a temporary store, as the pods would: the copy, the publishing (including two pods racing to publish one image), the sysroot merge and the tool cache links. Needs a mv with -T and a cp with -l, as busybox and GNU coreutils have.""" + + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.root = self.work / "store" + (self.root / ".incoming").mkdir(parents=True) + + def run_container(self, container: dict[str, Any], uid: str, rewrite: dict[str, str]) -> subprocess.CompletedProcess[str]: + """Run an init container's command with sh, as pod uid, with its paths rewritten to the temporary ones.""" + command = [rewrite.get(argument, argument) for argument in container["command"]] + return subprocess.run(command, env={**os.environ, "POD_UID": uid}, capture_output=True, text=True, check=False) + + def payload(self, name: str, files: dict[str, str]) -> Path: + directory = self.work / name + for relative, content in files.items(): + (directory / relative).parent.mkdir(parents=True, exist_ok=True) + (directory / relative).write_text(content) + return directory + + def containers(self, names: list[str]) -> dict[str, dict[str, Any]]: + spec = store_pod(DEFAULT_VALUES, names)["values"]["template"]["spec"] + return {container["name"]: container for container in spec["initContainers"]} + + def test_a_payload_is_copied_once_and_published_and_a_later_pod_copies_nothing(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary", "node/22.0.0/x64.complete": ""}) + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + for uid in ("pod-1", "pod-2"): + self.assertEqual(self.run_container(containers["capability-node"], uid, rewrite).returncode, 0) + published = self.run_container(containers["capability-store-publish"], uid, rewrite) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual(sorted(path.name for path in (self.root / ".incoming").iterdir()), []) + self.assertEqual((self.root / DIGEST_A / "node/22.0.0/x64/bin/node").read_text(), "binary") + + def test_of_two_pods_copying_one_image_at_once_the_first_to_publish_wins_and_the_other_copy_is_deleted(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + # Both copy before either publishes, as two pods starting together on a node without the entry would. + for uid in ("pod-1", "pod-2"): + self.assertEqual(self.run_container(containers["capability-node"], uid, rewrite).returncode, 0) + (self.root / ".incoming" / f"{DIGEST_A}.pod-1.capability-node" / "marker").write_text("pod-1") + for uid in ("pod-1", "pod-2"): + published = self.run_container(containers["capability-store-publish"], uid, rewrite) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual((self.root / DIGEST_A / "marker").read_text(), "pod-1") + self.assertFalse((self.root / DIGEST_A / f"{DIGEST_A}.pod-2.capability-node").exists()) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def test_publishing_fails_when_the_entry_is_neither_copied_nor_in_the_store(self) -> None: + containers = self.containers(["node"]) + published = self.run_container(containers["capability-store-publish"], "pod-1", {"/opt/capability-store": str(self.root)}) + self.assertNotEqual(published.returncode, 0) + self.assertIn(f"{DIGEST_A} is not in", published.stderr) + + def test_the_merged_sysroot_links_every_library_and_a_later_one_wins_a_shared_file(self) -> None: + for digest, files in ((DIGEST_B, {"lib/libmpi.so": "mpi", "include/shared.h": "from mpi"}), (DIGEST_C, {"lib/libpq.so": "pq", "include/shared.h": "from libpq"})): + for relative, content in files.items(): + (self.root / digest / relative).parent.mkdir(parents=True, exist_ok=True) + (self.root / digest / relative).write_text(content) + merge = self.containers(["mpi", "libpq"])["capability-store-sysroot"] + merged = merge["command"][5] + completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual((self.root / merged / "include/shared.h").read_text(), "from libpq") + self.assertEqual((self.root / merged / "lib/libmpi.so").stat().st_ino, (self.root / DIGEST_B / "lib/libmpi.so").stat().st_ino) + # A second pod finds it published and does nothing. + self.assertEqual(self.run_container(merge, "pod-2", {"/opt/capability-store": str(self.root)}).returncode, 0) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def test_the_tool_cache_links_each_version_into_the_pods_own_directory_with_its_marker(self) -> None: + (self.root / DIGEST_A / "node/22.0.0/x64/bin").mkdir(parents=True) + (self.root / DIGEST_A / "node/22.0.0/x64/bin/node").write_text("binary") + (self.root / DIGEST_A / "node/22.0.0/x64.complete").write_text("") + cache = self.work / "toolcache-pod" + cache.mkdir() + link = self.containers(["node"])["capability-store-tool-cache"] + completed = self.run_container(link, "pod-1", {"/opt/capability-store": str(self.root), TOOL_CACHE: str(cache)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertTrue((cache / "node/22.0.0/x64").is_symlink()) + self.assertEqual((cache / "node/22.0.0/x64/bin/node").read_text(), "binary") + self.assertTrue((cache / "node/22.0.0/x64.complete").is_file()) + self.assertFalse((cache / "node/22.0.0/x64.complete").is_symlink()) + # The version directory is the pod's own, so a setup action can add an architecture beside the link without touching the store. + (cache / "node/22.0.0/arm64").mkdir() + self.assertFalse((self.root / DIGEST_A / "node/22.0.0/arm64").exists()) + + if __name__ == "__main__": unittest.main() From 6ba3438d0146c7c0ad5c763a7cd338b43cfdca42 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 23:54:10 +0100 Subject: [PATCH 02/43] feat(arc): prune capability store entries no scale set or pod uses The node cache pruner removes a store entry once no scale set's pod template in a store profile's namespace and no unfinished pod on its node lists it, listing both again directly before the rename, and removes a copy left under .incoming once its pod has gone. Its Role in a store profile's namespace may list autoscalingrunnersets, and it is installed for a fleet with a store profile even without a node cache. --- .../tasks/decide_node_cache_pruner.yml | 7 +- .../templates/node-cache-pruner.yaml.j2 | 26 ++++++- scripts/node-cache-pruner.sh | 58 ++++++++++++++ tests/unit/test_node_cache_pruner.py | 78 ++++++++++++++++++- tests/unit/test_node_cache_pruner_role.py | 69 ++++++++++++++-- 5 files changed, 227 insertions(+), 11 deletions(-) diff --git a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml index fb55e47..94a9d72 100644 --- a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml +++ b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml @@ -5,6 +5,11 @@ ansible.builtin.set_fact: github_runner_arc_node_cache_profiles: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_node_cache_profiles(github_runner_arc_capabilities) }}" +# The pruner also removes the capability store's entries that no scale set or pod uses, so it runs for a fleet with a store profile too. +- name: "Node cache: find the namespaces of the profiles that use the capability store" + ansible.builtin.set_fact: + github_runner_arc_capability_store_namespaces: "{{ github_runner_arc_fleet.profiles | selectattr('capability_store') | map(attribute='namespace') | unique | list }}" + - name: "Node cache: decide whether the pruner is wanted" ansible.builtin.set_fact: - github_runner_arc_node_cache_pruner_wanted: "{{ github_runner_arc_node_cache_size_limit_gib | int > 0 and github_runner_arc_node_cache_profiles | length > 0 }}" + github_runner_arc_node_cache_pruner_wanted: "{{ github_runner_arc_node_cache_size_limit_gib | int > 0 and (github_runner_arc_node_cache_profiles | length > 0 or github_runner_arc_capability_store_namespaces | length > 0) }}" diff --git a/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 b/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 index d8cb810..d1fd42d 100644 --- a/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 +++ b/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 @@ -1,4 +1,4 @@ -{# The node cache pruner (see tasks/install_node_cache_pruner.yml): a ServiceAccount, a Role and RoleBinding in each fleet profile's namespace (so the pruner can list the pods it must not prune under, including those of a profile whose cache capability was removed while its directory remains, and nothing wider), and the DaemonSet. `github_runner_arc_fleet` is the fleet's expanded profiles and `github_runner_arc_node_cache_profiles` those with a node cache. #} +{# The node cache pruner (see tasks/install_node_cache_pruner.yml): a ServiceAccount, a Role and RoleBinding in each fleet profile's namespace (so the pruner can list the pods it must not prune under, including those of a profile whose cache capability was removed while its directory remains, and nothing wider), and the DaemonSet. `github_runner_arc_fleet` is the fleet's expanded profiles, `github_runner_arc_node_cache_profiles` those with a node cache and `github_runner_arc_capability_store_namespaces` the namespaces of those that use the capability store, whose root the DaemonSet then mounts as well. #} {% set name = 'node-cache-pruner' %} {% set labels = github_runner_arc_node_cache_pruner_labels %} {% set platform = github_runner_arc_platform_namespace %} @@ -17,10 +17,16 @@ metadata: namespace: {{ profile.namespace | to_json }} labels: {{ labels | to_json }} rules: - # list, to see whether a runner pod of this profile runs on the pruner's node. Nothing else. + # list, to see whether a runner pod of this profile runs on the pruner's node, and which capability store entries the pods there use. Nothing else. - apiGroups: [""] resources: ["pods"] verbs: ["list"] +{% if profile.namespace in github_runner_arc_capability_store_namespaces %} + # list, to see which capability store entries the scale sets' pod templates in a namespace that uses the store list. + - apiGroups: ["actions.github.com"] + resources: ["autoscalingrunnersets"] + verbs: ["list"] +{% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -79,6 +85,12 @@ spec: value: {{ github_runner_arc_node_cache_prune_seconds | string | to_json }} - name: WANTED_CACHES value: {{ (github_runner_arc_node_cache_profiles | map(attribute='target') | list | join(' ')) | to_json }} +{% if github_runner_arc_capability_store_namespaces | length > 0 %} + - name: STORE_ROOT + value: /host-capability-store + - name: STORE_NAMESPACES + value: {{ github_runner_arc_capability_store_namespaces | join(' ') | to_json }} +{% endif %} # kubectl writes its cache under HOME, and the root filesystem is read-only. - name: HOME value: /tmp @@ -94,6 +106,10 @@ spec: volumeMounts: - name: node-cache mountPath: /host-node-cache +{% if github_runner_arc_capability_store_namespaces | length > 0 %} + - name: capability-store + mountPath: /host-capability-store +{% endif %} - name: tmp mountPath: /tmp resources: @@ -110,5 +126,11 @@ spec: hostPath: path: {{ github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_root | to_json }} type: DirectoryOrCreate +{% if github_runner_arc_capability_store_namespaces | length > 0 %} + - name: capability-store + hostPath: + path: {{ github_runner_arc_capability_store_host_path | to_json }} + type: DirectoryOrCreate +{% endif %} - name: tmp emptyDir: {} diff --git a/scripts/node-cache-pruner.sh b/scripts/node-cache-pruner.sh index c662741..cd9e727 100755 --- a/scripts/node-cache-pruner.sh +++ b/scripts/node-cache-pruner.sh @@ -11,6 +11,8 @@ # # Every step of one profile's check may fail without stopping the others: a failed or partial size reading skips that profile for the round, and a failed pod list means no pruning, since an unknown set of pods may be using the cache: NODE_CACHE_UNPRUNABLE when the server refused the list as Forbidden (there is no Role for the pruner in that namespace), NODE_CACHE_LIST_FAILED for any other failure (the API server was unreachable, the answer was not a pod list), which the next round retries. # +# The capability store (the role README's "Where the copies live"), when STORE_ROOT is set, holds one entry per image payload, named for the image's digest, plus merged sysroots, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. +# # Which pods count: those in the profile's namespace on this node, not finished (Succeeded or Failed), that have a hostPath volume at exactly the profile's directory on the host (HOST_CACHE_ROOT//). Another profile's pods, finished pods and pods that do not mount the cache are ignored. # # Env: @@ -22,6 +24,8 @@ # - HARD_LIMIT_FACTOR: the hard limit as a multiple of the limit, a positive integer of at most MAX_HARD_LIMIT_FACTOR (default when unset: 2; set and empty is refused). # - WANTED_CACHES: the profiles that have a cache, as / separated by spaces. When it is set, any other directory under the cache root is removed. Unset leaves every directory alone. # - CHECK_SECONDS: seconds between checks, a positive integer of at most MAX_CHECK_SECONDS (default when unset: 300). +# - STORE_ROOT: where the node's capability store is mounted in this pod. Unset leaves the store alone, as on a fleet with no profile using it. +# - STORE_NAMESPACES: the namespaces of the profiles that use the store, separated by spaces, whose scale sets' templates and pods on this node are what keep an entry. # - PRUNE_ONCE: when "true", check once and exit, for tests. # # Start-up configuration errors, every one of them (a required variable missing, a number that is not a positive integer or is above its maximum), end the pruner with status 2 and a message on stderr that names the variable, before it does anything. The DaemonSet's pods then crash-loop with that message in their log. The role validates the numeric settings it templates with the same pattern and maxima, but a value changed some other way (an edited DaemonSet) reaches only this check. @@ -206,6 +210,59 @@ check_profile() { esac } +readonly STORE_KEYS_ANNOTATION=github-runner.exadev.io/capability-store-keys + +# Every store entry a scale set's pod template or an unfinished pod on this node lists, one per line, and each such pod's uid as uid:, or a failure when anything could not be listed, in which case the errors are in LIST_ERRORS. +store_references() { + local namespace + : > "$LIST_ERRORS" + for namespace in ${STORE_NAMESPACES-}; do + kubectl get autoscalingrunnersets.actions.github.com -n "$namespace" -o json 2>>"$LIST_ERRORS" \ + | jq -r --arg key "$STORE_KEYS_ANNOTATION" '.items[] | (.spec.template.metadata.annotations[$key] // "") | split(" ")[] | select(length > 0)' 2>>"$LIST_ERRORS" || return 1 + kubectl get pods -n "$namespace" --field-selector "spec.nodeName=${NODE_NAME},status.phase!=Succeeded,status.phase!=Failed" -o json 2>>"$LIST_ERRORS" \ + | jq -r --arg key "$STORE_KEYS_ANNOTATION" '.items[] | ("uid:" + .metadata.uid), ((.metadata.annotations[$key] // "") | split(" ")[] | select(length > 0))' 2>>"$LIST_ERRORS" || return 1 + done +} + +# Whether a store entry, or a copy under .incoming, is still used according to references: an entry by name, a copy by the uid in its name, .. (neither a key nor a uid nor a container name has a dot). +store_entry_used() { + local entry=$1 references=$2 name + name="${entry##*/}" + if [ "${entry%/*}" = "${STORE_ROOT}/.incoming" ]; then + name="uid:$(echo "$name" | cut -d. -f2)" + fi + grep -qxF -- "$name" <<<"$references" +} + +check_store() { + local entry references candidates=() + [ -n "${STORE_ROOT-}" ] && [ -d "$STORE_ROOT" ] || return 0 + rm -rf "${STORE_ROOT:?}/.trash" 2>/dev/null + if ! references="$(store_references)"; then + log "$(list_failure_prefix) capability store: the scale sets or the pods on this node could not be listed; not pruned" + return 0 + fi + shopt -s nullglob + for entry in "$STORE_ROOT"/* "$STORE_ROOT"/.incoming/*; do + store_entry_used "$entry" "$references" || candidates+=("$entry") + done + shopt -u nullglob + for entry in ${candidates[@]+"${candidates[@]}"}; do + # Listed again directly before the rename: a template or a pod may have come to use the entry since. + if ! references="$(store_references)"; then + log "$(list_failure_prefix) capability store: the scale sets or the pods on this node could not be listed part way through; the rest is left in place" + return 0 + fi + store_entry_used "$entry" "$references" && continue + if mkdir -p "${STORE_ROOT}/.trash" && mv "$entry" "${STORE_ROOT}/.trash/"; then + rm -rf "${STORE_ROOT:?}/.trash/${entry##*/}" || log "capability store: ${entry##*/} was set aside but could not be deleted; the next round retries" + log "capability store: removed ${entry##*/}, which no scale set or pod on this node uses" + else + log "capability store: ${entry##*/} could not be set aside; the next round retries" + fi + done +} + check() { local directory # Whatever an earlier round left in the trash was set aside from a cache already: it is garbage. @@ -218,6 +275,7 @@ check() { while true; do check + check_store [ "$PRUNE_ONCE" = "true" ] && exit 0 sleep "$CHECK_SECONDS" done diff --git a/tests/unit/test_node_cache_pruner.py b/tests/unit/test_node_cache_pruner.py index ec0d03b..454ef10 100644 --- a/tests/unit/test_node_cache_pruner.py +++ b/tests/unit/test_node_cache_pruner.py @@ -15,7 +15,7 @@ SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "node-cache-pruner.sh" HOST_ROOT = "/var/lib/example/node-cache" -# `get pods` answers JSON for the pods in FAKE_PODS (a JSON list of {name, namespace, node, phase, volumes: [hostPath paths], from_call?}) that match its -n namespace and its --field-selector terms (spec.nodeName= and status.phase!=, as kubectl evaluates them), counting that namespace's calls from 0 and leaving out a pod whose from_call is later. It fails for a namespace in FAKE_FAIL_PODS, answers a Forbidden error for one in FAKE_FORBIDDEN, and text that is not JSON for one in FAKE_BAD_JSON. Every call is recorded, with a snapshot of the cache root at that moment. +# `get autoscalingrunnersets.actions.github.com` answers JSON for the scale sets in FAKE_SCALE_SETS (a JSON list of {namespace, store_keys}) of its -n namespace, and fails for a namespace in FAKE_FAIL_SCALE_SETS. `get pods` answers JSON for the pods in FAKE_PODS (a JSON list of {name, namespace, node, phase, volumes: [hostPath paths], from_call?, uid?, store_keys?}) that match its -n namespace and its --field-selector terms (spec.nodeName= and status.phase!=, as kubectl evaluates them), counting that namespace's calls from 0 and leaving out a pod whose from_call is later. It fails for a namespace in FAKE_FAIL_PODS, answers a Forbidden error for one in FAKE_FORBIDDEN, and text that is not JSON for one in FAKE_BAD_JSON. Every call is recorded, with a snapshot of the cache root at that moment. FAKE_KUBECTL = textwrap.dedent( """\ #!/usr/bin/env python3 @@ -23,6 +23,16 @@ args = sys.argv[1:] state = os.environ["FAKE_STATE_DIR"] open(os.path.join(state, "calls.txt"), "a").write(" ".join(args) + "\\n") + listed = lambda name: [item for item in os.environ.get(name, "").split(",") if item] + annotations = lambda keys: {"github-runner.exadev.io/capability-store-keys": " ".join(keys)} if keys else {} + if args[:2] == ["get", "autoscalingrunnersets.actions.github.com"]: + namespace = args[args.index("-n") + 1] + if namespace in listed("FAKE_FAIL_SCALE_SETS"): + sys.stderr.write("the server is unavailable\\n") + sys.exit(1) + scale_sets = [item for item in json.loads(os.environ.get("FAKE_SCALE_SETS", "[]")) if item["namespace"] == namespace] + print(json.dumps({"items": [{"spec": {"template": {"metadata": {"annotations": annotations(item["store_keys"])}}}} for item in scale_sets]})) + sys.exit(0) if args[:2] != ["get", "pods"]: sys.stderr.write("unexpected kubectl call: %s\\n" % args) sys.exit(2) @@ -64,7 +74,7 @@ def matches(pod): items = [] for pod in json.loads(os.environ.get("FAKE_PODS", "[]")): if pod["namespace"] == namespace and matches(pod) and count >= pod.get("from_call", 0): - metadata = {"name": pod["name"]} + metadata = {"name": pod["name"], "uid": pod.get("uid", pod["name"]), "annotations": annotations(pod.get("store_keys", []))} if pod.get("deleting"): metadata["deletionTimestamp"] = "2026-01-01T00:00:00Z" items.append({"metadata": metadata, "spec": {"volumes": [{"name": "v%d" % index, "hostPath": {"path": path}} for index, path in enumerate(pod["volumes"])]}}) @@ -565,5 +575,69 @@ def test_valid_settings_start_normally_and_a_gibibyte_limit_is_converted(self) - self.assertFalse(self.calls.exists()) + +KEY_A = "a" * 64 +KEY_B = "b" * 64 +KEY_C = "c" * 64 + + +class CapabilityStorePrunerTest(unittest.TestCase): + """The capability store's pruning: an entry stays while a scale set's template or an unfinished pod on this node lists it, and a copy under .incoming stays while its pod is unfinished on this node.""" + + # PrunerTest's helpers, borrowed rather than inherited so its cases do not run twice. + setup_work = PrunerTest.setup_work + environment = PrunerTest.environment + run_pruner = PrunerTest.run_pruner + + def run_store(self, entries: list[str], incoming: list[str] | None = None, scale_sets: list[dict[str, Any]] | None = None, pods: list[dict[str, Any]] | None = None, **env: str) -> tuple[Run, Path]: + root, bin_dir, state = self.setup_work() + store = root.parent / "store" + for name in entries: + (store / name).mkdir(parents=True) + (store / name / "payload").write_text(name) + for name in incoming or []: + (store / ".incoming" / name).mkdir(parents=True) + environment = {**self.environment(root, bin_dir, state, pods, [], {"STORE_ROOT": str(store), "STORE_NAMESPACES": "arc-a", "FAKE_SCALE_SETS": json.dumps(scale_sets or []), **env})} + completed = subprocess.run(["bash", str(SCRIPT)], check=True, env=environment, capture_output=True, text=True) + return Run(root, state, completed.stdout), store + + def store_pod(self, keys: list[str], node: str = "node-a", phase: str = "Running", uid: str = "pod-1", from_call: int = 0) -> dict[str, Any]: + return {"name": f"runner-{uid}", "uid": uid, "namespace": "arc-a", "node": node, "phase": phase, "volumes": [], "store_keys": keys, "from_call": from_call} + + def remaining(self, store: Path) -> list[str]: + return sorted(path.name for path in store.iterdir() if not path.name.startswith(".")) + + def test_an_entry_no_template_or_pod_uses_is_removed_and_one_a_template_lists_is_kept(self) -> None: + run, store = self.run_store([KEY_A, KEY_B], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_A]}]) + self.assertEqual(self.remaining(store), [KEY_A]) + self.assertIn(f"capability store: removed {KEY_B}, which no scale set or pod on this node uses", run.output) + + def test_an_entry_an_unfinished_pod_on_this_node_lists_is_kept_after_its_template_moved_on(self) -> None: + _, store = self.run_store([KEY_A, KEY_B, KEY_C], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_C]}], pods=[self.store_pod([KEY_A]), self.store_pod([KEY_B], phase="Pending", uid="pod-2")]) + self.assertEqual(self.remaining(store), [KEY_A, KEY_B, KEY_C]) + + def test_a_finished_pod_or_one_on_another_node_keeps_nothing(self) -> None: + _, store = self.run_store([KEY_A, KEY_B], pods=[self.store_pod([KEY_A], phase="Succeeded"), self.store_pod([KEY_B], node="node-b", uid="pod-2")]) + self.assertEqual(self.remaining(store), []) + + def test_a_pod_that_starts_using_an_entry_before_it_is_removed_keeps_it(self) -> None: + # The pod appears from the second list on: the first finds the entry unused, the list taken again before the rename finds the pod. + _, store = self.run_store([KEY_A], pods=[self.store_pod([KEY_A], from_call=1)]) + self.assertEqual(self.remaining(store), [KEY_A]) + + def test_a_copy_in_progress_is_kept_while_its_pod_runs_and_removed_once_it_is_gone(self) -> None: + _, store = self.run_store([], incoming=[f"{KEY_A}.pod-1.capability-node", f"{KEY_B}.pod-gone.capability-node"], pods=[self.store_pod([KEY_A])]) + self.assertEqual(sorted(path.name for path in (store / ".incoming").iterdir()), [f"{KEY_A}.pod-1.capability-node"]) + + def test_nothing_is_removed_when_the_scale_sets_cannot_be_listed(self) -> None: + run, store = self.run_store([KEY_A], FAKE_FAIL_SCALE_SETS="arc-a") + self.assertEqual(self.remaining(store), [KEY_A]) + self.assertIn("NODE_CACHE_LIST_FAILED capability store: the scale sets or the pods on this node could not be listed; not pruned", run.output) + + def test_without_a_store_root_the_store_is_not_looked_at(self) -> None: + run = self.run_pruner({}) + self.assertFalse([call for call in run.calls() if "autoscalingrunnersets" in call]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index aa31905..3675d35 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -49,7 +49,7 @@ def render(**role_vars: Any) -> list[dict[str, Any]]: """Render the pruner's manifests, in the order the template writes them.""" with tempfile.TemporaryDirectory() as work: output = Path(work) / "pruner.yaml" - variables = {"github_runner_arc_fleet": FLEET, "github_runner_arc_node_cache_profiles": PROFILES, "github_runner_arc_node_label_key": "example/ci", "github_runner_arc_node_label_value": "true", **role_vars} + variables = {"github_runner_arc_fleet": FLEET, "github_runner_arc_node_cache_profiles": PROFILES, "github_runner_arc_capability_store_namespaces": [], "github_runner_arc_node_label_key": "example/ci", "github_runner_arc_node_label_value": "true", **role_vars} completed = ansible(RENDER, pruner_output=str(output), **variables) assert completed.returncode == 0, completed.stdout + completed.stderr documents: list[dict[str, Any]] = [document for document in yaml.safe_load_all(output.read_text()) if document] @@ -57,8 +57,10 @@ def render(**role_vars: Any) -> list[dict[str, Any]]: def decide(fleet_targets: list[str], **role_vars: Any) -> dict[str, Any]: - """Run the decision for a fleet of the given profile targets, each with the pnpm-store capability when its target is listed in with_cache.""" - profiles = [{"namespace": target.split("/")[0], "release": target.split("/")[1], "target": target, "capabilities": ["pnpm-store" if target in role_vars.pop("with_cache", fleet_targets) else "node"]} for target in fleet_targets] + """Run the decision for a fleet of the given profile targets, each with the pnpm-store capability when its target is listed in with_cache, and using the capability store when it is listed in with_store.""" + with_cache = role_vars.pop("with_cache", fleet_targets) + with_store = role_vars.pop("with_store", []) + profiles = [{"namespace": target.split("/")[0], "release": target.split("/")[1], "target": target, "capabilities": ["pnpm-store" if target in with_cache else "node"], "capability_store": target in with_store} for target in fleet_targets] with tempfile.TemporaryDirectory() as work: output = Path(work) / "decision.json" completed = ansible(DECIDE, decide_output=str(output), github_runner_arc_fleet={"profiles": profiles}, github_runner_arc_capabilities=CATALOGUE, **role_vars) @@ -136,6 +138,37 @@ def test_with_no_node_cache_anywhere_it_is_not_wanted_so_it_follows_the_fleet(se def test_a_size_limit_of_zero_means_it_is_not_wanted(self) -> None: self.assertEqual(decide(["arc-a/main"], github_runner_arc_node_cache_size_limit_gib=0)["wanted"], False) + def test_a_profile_using_the_capability_store_wants_the_pruner_without_a_node_cache(self) -> None: + self.assertEqual(decide(["arc-a/main"], with_cache=[], with_store=["arc-a/main"]), {"wanted": True, "targets": []}) + + +class CapabilityStoreManifestTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.documents = render(github_runner_arc_capability_store_namespaces=["arc-a"]) + [daemonset] = [document for document in cls.documents if document["kind"] == "DaemonSet"] + cls.pod = daemonset["spec"]["template"]["spec"] + [cls.container] = cls.pod["containers"] + + def test_the_pruner_is_told_where_the_store_is_and_which_namespaces_use_it(self) -> None: + env = {item["name"]: item.get("value") for item in self.container["env"]} + self.assertEqual((env["STORE_ROOT"], env["STORE_NAMESPACES"]), ("/host-capability-store", "arc-a")) + self.assertIn({"name": "capability-store", "mountPath": "/host-capability-store"}, self.container["volumeMounts"]) + volumes = {volume["name"]: volume for volume in self.pod["volumes"]} + self.assertEqual(volumes["capability-store"]["hostPath"], {"path": "/var/lib/github-runner/capability-store", "type": "DirectoryOrCreate"}) + + def test_only_a_namespace_that_uses_the_store_may_have_its_scale_sets_listed(self) -> None: + rules = {role["metadata"]["namespace"]: role["rules"] for role in self.documents if role["kind"] == "Role"} + self.assertIn({"apiGroups": ["actions.github.com"], "resources": ["autoscalingrunnersets"], "verbs": ["list"]}, rules["arc-a"]) + self.assertEqual(rules["arc-b"], [{"apiGroups": [""], "resources": ["pods"], "verbs": ["list"]}]) + + def test_without_a_store_profile_the_pruner_neither_mounts_nor_prunes_the_store(self) -> None: + documents = render() + [daemonset] = [document for document in documents if document["kind"] == "DaemonSet"] + container = daemonset["spec"]["template"]["spec"]["containers"][0] + self.assertNotIn("STORE_ROOT", [item["name"] for item in container["env"]]) + self.assertNotIn("capability-store", [volume["name"] for volume in daemonset["spec"]["template"]["spec"]["volumes"]]) + # A stand-in for the kubernetes.core modules the install tasks call: it appends the module name and its arguments to the file named by STUB_LOG, reports no change and finds no resources, so the tasks' control flow runs and what they ask the cluster for can be read. STUB_MODULE = """from ansible.module_utils.basic import AnsibleModule @@ -188,7 +221,7 @@ def legacy_binding(subjects: list[dict[str, Any]] | None = None) -> dict[str, An def install_with_output(fleet_targets: list[str], with_cache: list[str] | None = None, existing: list[dict[str, Any]] | None = None, **role_vars: Any) -> tuple[list[dict[str, Any]], str]: """Run the role's pruner install tasks against the stand-in kubernetes.core, and return the module calls it recorded and the playbook's output. Each fleet profile carries the pnpm-store capability when its target is in with_cache (all of them when it is None).""" - profiles = [{"namespace": t.split("/")[0], "release": t.split("/")[1], "target": t, "capabilities": ["pnpm-store"] if with_cache is None or t in with_cache else ["node"]} for t in fleet_targets] + profiles = [{"namespace": t.split("/")[0], "release": t.split("/")[1], "target": t, "capabilities": ["pnpm-store"] if with_cache is None or t in with_cache else ["node"], "capability_store": False} for t in fleet_targets] with tempfile.TemporaryDirectory() as work: modules = Path(work) / "stubs" / "ansible_collections" / "kubernetes" / "core" / "plugins" / "modules" modules.mkdir(parents=True) @@ -372,6 +405,30 @@ def problems(self, **role_vars: Any) -> list[str]: found: list[str] = evaluate(self.expression, {**defaults, **constants, **role_vars}) return found + def test_the_capability_store_settings_are_checked(self) -> None: + self.assertEqual(self.problems(), []) + for role_vars, message in ( + ({"github_runner_arc_capability_store_host_path": "/var/lib/github-runner/store"}, "named capability-store"), + ({"github_runner_arc_capability_store_path": "relative"}, "capability store path must be an absolute directory"), + ({"github_runner_arc_capability_store_image": "docker.io/library/busybox:1.37.0"}, "github_runner_arc_capability_store_image must be pinned by digest"), + ): + with self.subTest(role_vars): + problems = self.problems(**role_vars) + self.assertEqual(len(problems), 1, problems) + self.assertIn(message, problems[0]) + + def test_a_profile_using_the_capability_store_is_refused_without_the_pruner(self) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "validate.yml").read_text()) + [task] = [task for task in tasks if task["name"] == "Fail when a profile uses the capability store without the pruner"] + local = {"profiles": [{"label": "Example-store", "capability_store": True}, {"label": "Example", "capability_store": False}]} + refused = lambda limit: evaluate("{{ " + task["when"] + " }}", {"github_runner_arc_node_cache_size_limit_gib": limit, "github_runner_arc_local": local}) + self.assertTrue(refused(0)) + self.assertFalse(refused(20)) + import jinja2 + + message = jinja2.Environment(undefined=jinja2.StrictUndefined).from_string(task["ansible.builtin.fail"]["msg"]).render(github_runner_arc_local=local) + self.assertTrue(message.startswith("Example-store use the capability store"), message) + def test_the_task_expression_gives_the_same_answer_under_ansible(self) -> None: cases = ({}, {"github_runner_arc_node_cache_host_path": "/var/lib"}, {"github_runner_arc_node_cache_hard_limit_factor": "08"}, {"github_runner_arc_node_cache_size_limit_gib": 1073741825}) for role_vars in cases: @@ -540,12 +597,12 @@ def test_the_readmes_pnpm_example_validates_and_renders_into_a_pod(self) -> None profile = arc.arc_profiles([{"name": "Example", "image": "ghcr.io/example/runner:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["pnpm-store"]}]}], require_app_id=False)["profiles"][0] self.assertEqual(arc.arc_capability_errors([profile], catalogue), []) values = {"template": {"spec": {"containers": [{"name": "runner", "command": ["/home/runner/run.sh"]}]}}} - result = arc.arc_capability_pod(values, ["pnpm-store"], catalogue, "/opt/hostedtoolcache", "/opt/sysroot", "/node-cache", "/var/lib/github-runner/node-cache/arc/main", 1001, 1001, "/etc/github-runner/hooks", "job-started") + result = arc.arc_capability_pod(values, ["pnpm-store"], catalogue, "/opt/hostedtoolcache", "/opt/sysroot", "/node-cache", "/var/lib/github-runner/node-cache/arc/main", 1001, 1001, "/etc/github-runner/hooks", "job-started", None) self.assertEqual(result["errors"], []) def test_the_example_names_the_store_with_a_variable_pnpm_does_not_read(self) -> None: # A pod-wide pnpm_config_store_dir reaches pnpm/action-setup's version switch, which corrupts a shared global virtual store (README, Capabilities), so the example names the directory for a workflow to pass to its install step. - env = {item["name"]: item["value"] for item in arc.arc_capability_pod({"template": {"spec": {"containers": [{"name": "runner"}]}}}, ["pnpm-store"], self.example()["github_runner_arc_capabilities"], "/a", "/b", "/node-cache", "/var/lib/x/node-cache/a/b", 1001, 1001, "/h", "c")["values"]["template"]["spec"]["containers"][0]["env"]} + env = {item["name"]: item["value"] for item in arc.arc_capability_pod({"template": {"spec": {"containers": [{"name": "runner"}]}}}, ["pnpm-store"], self.example()["github_runner_arc_capabilities"], "/a", "/b", "/node-cache", "/var/lib/x/node-cache/a/b", 1001, 1001, "/h", "c", None)["values"]["template"]["spec"]["containers"][0]["env"]} self.assertEqual(env["NODE_CACHE_PNPM_STORE_DIR"], "/node-cache/pnpm") self.assertEqual([name for name in env if name.lower().startswith(("pnpm_config_", "npm_config_"))], []) readme = (ROLE / "README.md").read_text() From b80fb047749c8981d6f57b53f079f1db5667294b Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Wed, 21 Oct 2026 00:00:14 +0100 Subject: [PATCH 03/43] fix(arc): prepare the capability store before the dind externals copy A values file's own init-dind-externals copy comes before the capabilities' containers, so the store's preparation, which the capabilities added after it, ran too late for the externals copy on a node without the store. The externals step now makes the preparation the first init container. --- plugins/filter/arc.py | 13 ++++++------- tests/unit/test_capabilities.py | 5 ++++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 1e75d7f..c4d5997 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1180,12 +1180,12 @@ def _annotate_store_keys(template: dict[str, Any], keys: Sequence[str]) -> None: annotations[_CAPABILITY_STORE_KEYS_ANNOTATION] = " ".join(dict.fromkeys([*existing, *keys])) -def _ensure_store_prepared(spec: dict[str, Any], init_containers: list[dict[str, Any]], store: Mapping[str, Any], runner_user: int, runner_group: int) -> None: - """Add the store's volume to spec, and its preparation to the front of init_containers, unless spec already has them.""" +def _ensure_store_prepared(spec: dict[str, Any], store: Mapping[str, Any], runner_user: int, runner_group: int) -> None: + """Add the store's volume to spec unless it has it, and make the store's preparation spec's first init container, moving it there when the capabilities put it after the values' own init containers, since every copy into the store needs the directories it creates.""" if not any(volume.get("name") == _CAPABILITY_STORE_VOLUME for volume in spec.get("volumes") or []): spec["volumes"] = list(spec.get("volumes") or []) + [_capability_store_volume(store)] - if not any(container.get("name") == _CAPABILITY_STORE_PREPARE for container in [*(spec.get("initContainers") or []), *init_containers]): - init_containers.insert(0, _capability_store_prepare_container(store, runner_user, runner_group)) + others = [container for container in spec.get("initContainers") or [] if container.get("name") != _CAPABILITY_STORE_PREPARE] + spec["initContainers"] = [_capability_store_prepare_container(store, runner_user, runner_group), *others] def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> tuple[list[dict[str, Any]], list[str], str | None]: @@ -1262,9 +1262,8 @@ def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str {"name": _CAPABILITY_STORE_VOLUME, "mountPath": mount["mountPath"], "subPath": key, "readOnly": True} if mount.get("name") == _DIND_EXTERNALS_VOLUME else mount for mount in container.get("volumeMounts") or [] ] - prefix: list[dict[str, Any]] = [] - _ensure_store_prepared(spec, prefix, store, runner_user, runner_group) - spec["initContainers"] = prefix + init_containers + spec["initContainers"] = init_containers + _ensure_store_prepared(spec, store, runner_user, runner_group) _annotate_store_keys(template, [key]) return {"errors": [], "values": result} diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 0c09e0d..86f4b9b 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -750,7 +750,10 @@ def test_after_the_capabilities_the_store_is_prepared_once_and_the_annotation_ke values = store_pod(DIND_VALUES, ["node"])["values"] result = arc.arc_capability_store_externals(values, STORE, 1001, 1001) spec = result["values"]["template"]["spec"] - self.assertEqual([container["name"] for container in spec["initContainers"]].count("capability-store-prepare"), 1) + names = [container["name"] for container in spec["initContainers"]] + # The values' own externals copy comes before the capabilities' containers, so the preparation moves ahead of it. + self.assertEqual(names[:3], ["capability-store-prepare", "init-dind-externals", "capability-store-publish-externals"]) + self.assertEqual(names.count("capability-store-prepare"), 1) self.assertEqual([volume["name"] for volume in spec["volumes"]].count("capability-store"), 1) self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{DIGEST_A} {DIGEST_RUNNER}") From b13be7658a4220471be00860b044c96c825f109b Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Wed, 21 Oct 2026 00:53:04 +0100 Subject: [PATCH 04/43] fix(arc): choose the capability store settings inside each filter call A set_fact whose value is none comes back as the string None on ansible-core 2.16, so the filters were handed a string for a profile without the store. The settings fact is now always the store's settings, and each call passes them or none in its own expression. --- .../tasks/render_scale_set_values.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/roles/github_runner_arc/tasks/render_scale_set_values.yml b/roles/github_runner_arc/tasks/render_scale_set_values.yml index bd28f50..23754f0 100644 --- a/roles/github_runner_arc/tasks/render_scale_set_values.yml +++ b/roles/github_runner_arc/tasks/render_scale_set_values.yml @@ -3,12 +3,11 @@ - name: Set the capability store settings for {{ profile.label }} ansible.builtin.set_fact: - # None when the profile copies into each pod; the filters take the store's root on the node, its mount path in the pod and the image that publishes its entries. - github_runner_arc_capability_store_settings: >- - {{ - {'host_path': github_runner_arc_capability_store_host_path, 'path': github_runner_arc_capability_store_path, 'image': github_runner_arc_capability_store_image} - if profile.capability_store else none - }} + # The store's root on the node, its mount path in the pod and the image that publishes its entries. The filters below are given these, or none for a profile that copies into each pod, inside their own expressions: a fact set to none comes back as the string "None" on some ansible-core releases (2.16 among them). + github_runner_arc_capability_store_settings: + host_path: "{{ github_runner_arc_capability_store_host_path }}" + path: "{{ github_runner_arc_capability_store_path }}" + image: "{{ github_runner_arc_capability_store_image }}" - name: Render the Helm values for {{ profile.label }} ansible.builtin.set_fact: @@ -21,7 +20,8 @@ profile.capabilities, github_runner_arc_capabilities, github_runner_arc_tool_cache_path, github_runner_arc_sysroot_path, github_runner_arc_node_cache_path, github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_dir(profile.namespace, profile.release), github_runner_arc_runner_uid, github_runner_arc_runner_gid, - github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap, github_runner_arc_capability_store_settings) + github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap, + github_runner_arc_capability_store_settings if profile.capability_store else none) }} vars: github_runner_arc_values_source: >- @@ -72,7 +72,10 @@ - name: Move the dind sidecar's externals copy into the capability store for {{ profile.label }} ansible.builtin.set_fact: github_runner_arc_capability_store_render: >- - {{ github_runner_arc_scale_set_values | exadev.github_runner.arc_capability_store_externals(github_runner_arc_capability_store_settings, github_runner_arc_runner_uid, github_runner_arc_runner_gid) }} + {{ + github_runner_arc_scale_set_values | exadev.github_runner.arc_capability_store_externals( + github_runner_arc_capability_store_settings if profile.capability_store else none, github_runner_arc_runner_uid, github_runner_arc_runner_gid) + }} - name: Fail when the values cannot keep the externals copy in the capability store for {{ profile.label }} ansible.builtin.fail: From 80a003c15ccfa5ba133dbc532eb8d500c4733512 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 09:19:24 +0100 Subject: [PATCH 05/43] fix(arc): keep a pod's cached tool trees writable when the capability store is on The tool cache linked each // directory into the read-only store, so a job writing inside a tool's own tree failed with a read-only filesystem error, where each pod's own copy had been writable (`npm install -g`, `corepack enable`). The pod's tree is now real directories holding a symlink to each store file, so a job can create, replace and remove files and directories while the store stays untouched. A symlink in the store is copied as a symlink so its relative target resolves inside the pod, and a regular file directly in /bin is copied, since a launcher derives its own tree (npm's global prefix) from its resolved executable path, which a link would put in the store. The script rerun over a filled cache converges. The README now states what a job can and cannot change, and that the sysroot and the dind externals are read-only consumers. --- plugins/filter/arc.py | 11 ++- roles/github_runner_arc/README.md | 4 +- tests/unit/test_capabilities.py | 126 +++++++++++++++++++++++++++--- 3 files changed, 124 insertions(+), 17 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index c4d5997..873d691 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1159,11 +1159,14 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' ) -# Fills the pod's own tool cache with a link to each // directory in the store and a copy of its .complete marker, which is how the Actions tool cache finds a tool. The version directories are the pod's own, so a setup action can still install another version or architecture beside them; the store itself is read-only to the pod. +# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well. cp -asf, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. +_TOOL_CACHE_COPY_SCRIPT = 'source=$1; destination=$2; shift 2; for entry; do target=$destination/${entry#"$source"/}; rm -f "$target" && cp -P "$entry" "$target" || exit 1; done' _TOOL_CACHE_LINK_SCRIPT = ( 'root=$1; cache=$2; shift 2; for key in "$@"; do for arch in "$root/$key"/*/*/*; do' - ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"};' - ' mkdir -p "$cache/${relative%/*}" && ln -sfn "$arch" "$cache/$relative" || exit 1;' + ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"}; parent="$cache/${relative%/*}"; tree="$cache/$relative";' + ' mkdir -p "$parent" && cp -asf "$arch" "$parent/" || exit 1;' + " find \"$arch\" -type l -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" + " [ ! -d \"$arch/bin\" ] || find \"$arch/bin\" -maxdepth 1 -type f -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" ' [ ! -e "$arch.complete" ] || cp "$arch.complete" "$cache/$relative.complete" || exit 1;' ' done; done' ) @@ -1274,7 +1277,7 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay. Args: - values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it once per node instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, filled with links to the store, the sysroot is the store's merged entry mounted read-only, and the pod template is annotated with the entries it uses. + values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it once per node instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, rebuilt as real directories holding links to the store's files (see _TOOL_CACHE_LINK_SCRIPT), the sysroot is the store's merged entry mounted read-only, and the pod template is annotated with the entries it uses. Returns: A dict with ``errors`` (empty when the values can carry the capabilities), ``values`` (the values with the capabilities added, or as given when there are none or errors) and ``hooks`` (the ConfigMap's data apart from the dispatcher, which the role adds from its own file; empty when the profile needs no ConfigMap). diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index b0c3847..49258b7 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -315,7 +315,9 @@ An entry is named for its image's digest, so every image a profile's capabilitie - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. - `capability-store-publish` renames each copy onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete, two pods starting together never see each other's half-written copy, and the container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, published the same way. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. -- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, with a link to each `//` directory in the store and a copy of its `.complete` marker. The version directories are the pod's own, so a setup action can install another version or architecture beside them, and the store is mounted read-only on the runner container, so a job cannot change what other pods use. +- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. + +What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and it cannot change anything in the store, so no job changes what another pod uses. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a job cannot write into them, where the per-pod copies were writable. `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume. Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 86f4b9b..2a42970 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -839,22 +839,124 @@ def test_the_merged_sysroot_links_every_library_and_a_later_one_wins_a_shared_fi self.assertEqual(self.run_container(merge, "pod-2", {"/opt/capability-store": str(self.root)}).returncode, 0) self.assertEqual(list((self.root / ".incoming").iterdir()), []) - def test_the_tool_cache_links_each_version_into_the_pods_own_directory_with_its_marker(self) -> None: - (self.root / DIGEST_A / "node/22.0.0/x64/bin").mkdir(parents=True) - (self.root / DIGEST_A / "node/22.0.0/x64/bin/node").write_text("binary") - (self.root / DIGEST_A / "node/22.0.0/x64.complete").write_text("") - cache = self.work / "toolcache-pod" - cache.mkdir() + def tool_tree(self, files: dict[str, str] | None = None) -> Path: + """Fill the store's entry for the node capability with a tool tree shaped like the Actions tool cache's Node.js, as a store entry is after publishing, and return its directory.""" + arch = self.root / DIGEST_A / "node/22.0.0/x64" + for relative, content in (files or {"bin/node": "binary", "lib/node_modules/npm/bin/npm-cli.js": "npm", "lib/node_modules/npm/package.json": "{}", "include/node/node.h": "header", "dir with space/a file": "spaced", ".hidden/file": "hidden"}).items(): + (arch / relative).parent.mkdir(parents=True, exist_ok=True) + (arch / relative).write_text(content) + (arch / "bin/node").chmod(0o755) + (arch / "empty").mkdir() + (arch / "bin/npm").symlink_to("../lib/node_modules/npm/bin/npm-cli.js") + (arch / "bin/absolute").symlink_to("/absolute/target") + (arch / "libdir").symlink_to("lib") + (arch.parent / "x64.complete").write_text("") + return arch + + def link_tool_cache(self, cache: Path, uid: str = "pod-1") -> None: link = self.containers(["node"])["capability-store-tool-cache"] - completed = self.run_container(link, "pod-1", {"/opt/capability-store": str(self.root), TOOL_CACHE: str(cache)}) + completed = self.run_container(link, uid, {"/opt/capability-store": str(self.root), TOOL_CACHE: str(cache)}) self.assertEqual(completed.returncode, 0, completed.stderr) - self.assertTrue((cache / "node/22.0.0/x64").is_symlink()) - self.assertEqual((cache / "node/22.0.0/x64/bin/node").read_text(), "binary") + + def snapshot(self, directory: Path) -> dict[str, str]: + """Return every path under directory with what it holds (a file's content, a link's target, a directory's name), so a change anywhere shows.""" + return {str(path.relative_to(directory)): path.readlink().as_posix() if path.is_symlink() else path.read_text() if path.is_file() else "" for path in sorted(directory.rglob("*"))} + + def test_the_tool_cache_rebuilds_each_tool_tree_as_directories_holding_a_link_to_each_store_file_and_the_marker(self) -> None: + arch = self.tool_tree() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0/x64" + self.assertTrue(tree.is_dir() and not tree.is_symlink()) + for directory in ("bin", "lib/node_modules/npm", "include/node", "dir with space", ".hidden", "empty"): + self.assertTrue((tree / directory).is_dir() and not (tree / directory).is_symlink(), directory) + for file, content in (("lib/node_modules/npm/package.json", "{}"), ("include/node/node.h", "header"), ("dir with space/a file", "spaced"), (".hidden/file", "hidden")): + self.assertEqual((tree / file).readlink(), arch / file, file) + self.assertEqual((tree / file).read_text(), content) self.assertTrue((cache / "node/22.0.0/x64.complete").is_file()) self.assertFalse((cache / "node/22.0.0/x64.complete").is_symlink()) - # The version directory is the pod's own, so a setup action can add an architecture beside the link without touching the store. - (cache / "node/22.0.0/arm64").mkdir() - self.assertFalse((self.root / DIGEST_A / "node/22.0.0/arm64").exists()) + + def test_a_job_can_create_replace_and_remove_files_and_directories_in_a_tool_tree_and_the_store_is_unchanged(self) -> None: + arch = self.tool_tree() + before = self.snapshot(self.root) + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0/x64" + # What npm install -g and corepack enable do: create a package directory and a shim beside the launcher, replace an installed package's file, remove another. + (tree / "lib/node_modules/added").mkdir() + (tree / "lib/node_modules/added/index.js").write_text("added") + (tree / "bin/shim").symlink_to("../lib/node_modules/added/index.js") + (tree / "lib/node_modules/npm/package.json").unlink() + (tree / "lib/node_modules/npm/package.json").write_text('{"version": "2"}') + (tree / "include/node/node.h").unlink() + (tree / "empty/inside").write_text("created") + self.assertEqual((tree / "lib/node_modules/npm/package.json").read_text(), '{"version": "2"}') + self.assertFalse((tree / "include/node/node.h").exists()) + self.assertEqual(self.snapshot(self.root), before) + self.assertEqual((arch / "include/node/node.h").read_text(), "header") + + def test_a_launcher_is_a_copy_and_not_a_link_so_the_program_finds_its_own_tree_in_the_pod(self) -> None: + self.tool_tree() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + launcher = cache / "node/22.0.0/x64/bin/node" + self.assertTrue(launcher.is_file() and not launcher.is_symlink()) + self.assertEqual(launcher.read_text(), "binary") + self.assertTrue(os.access(launcher, os.X_OK)) + # A program that locates its tree from its resolved executable path sees the pod's tree. + self.assertEqual(launcher.resolve(), launcher.parent.resolve() / launcher.name) + # A file elsewhere in the tree stays a link. + self.assertTrue((cache / "node/22.0.0/x64/lib/node_modules/npm/package.json").is_symlink()) + + def test_a_link_in_the_store_is_copied_so_its_relative_target_resolves_inside_the_pods_tree(self) -> None: + self.tool_tree() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0/x64" + self.assertEqual((tree / "bin/npm").readlink(), Path("../lib/node_modules/npm/bin/npm-cli.js")) + self.assertEqual((tree / "bin/absolute").readlink(), Path("/absolute/target")) + self.assertEqual((tree / "libdir").readlink(), Path("lib")) + self.assertEqual((tree / "libdir").resolve(), (tree / "lib").resolve()) + self.assertEqual((tree / "libdir/node_modules/npm/package.json").read_text(), "{}") + + def test_linking_the_tool_cache_again_leaves_the_same_tree(self) -> None: + self.tool_tree() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + first = self.snapshot(cache) + self.link_tool_cache(cache) + self.assertEqual(self.snapshot(cache), first) + + def test_every_tool_version_and_architecture_of_every_entry_is_linked_beside_the_others(self) -> None: + self.tool_tree() + other = self.root / DIGEST_B / "python/3.12.0/x64" + (other / "bin").mkdir(parents=True) + (other / "bin/python3").write_text("python") + (other.parent / "x64.complete").write_text("") + (self.root / DIGEST_A / "node/22.0.0/arm64/bin").mkdir(parents=True) + (self.root / DIGEST_A / "node/22.0.0/arm64/bin/node").write_text("arm") + cache = self.work / "toolcache-pod" + cache.mkdir() + link = dict(self.containers(["node"])["capability-store-tool-cache"]) + link["command"] = [*link["command"], DIGEST_B] + completed = self.run_container(link, "pod-1", {"/opt/capability-store": str(self.root), TOOL_CACHE: str(cache)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual((cache / "python/3.12.0/x64/bin/python3").read_text(), "python") + self.assertEqual((cache / "node/22.0.0/arm64/bin/node").read_text(), "arm") + self.assertTrue((cache / "node/22.0.0/x64/bin/node").is_file()) + self.assertFalse((cache / "node/22.0.0/arm64.complete").exists()) + + def test_linking_fails_when_an_entry_cannot_be_linked(self) -> None: + self.tool_tree() + cache = self.work / "toolcache-pod" + cache.write_text("a file where the tool cache directory should be") + link = self.containers(["node"])["capability-store-tool-cache"] + self.assertNotEqual(self.run_container(link, "pod-1", {"/opt/capability-store": str(self.root), TOOL_CACHE: str(cache)}).returncode, 0) if __name__ == "__main__": From 8a48f4041b284f4a6dbd39bce1e4b5bac6f00152 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 09:31:48 +0100 Subject: [PATCH 06/43] fix(arc): keep capability store entries a departed profile's pods still use The pruner read the store's references only from the namespaces of the profiles that use the store in the current fleet. A profile that switched the store off, or left the fleet, dropped out of that scan while pods made from its old template kept running, so the next round removed their entries under them. The namespaces are now every fleet profile's and every namespace holding one of the pruner's own Roles, which the role keeps for a profile that has left the fleet, as the node cache already relies on. A per-namespace Role and RoleBinding grant listing pods and scale sets in each of them, so a departed profile's namespace is re-granted on every run. --- plugins/filter/arc.py | 15 +++++ roles/github_runner_arc/README.md | 2 +- .../tasks/decide_node_cache_pruner.yml | 6 +- .../tasks/install_node_cache_pruner.yml | 22 +++---- .../templates/node-cache-pruner.yaml.j2 | 44 ++++++++++--- scripts/node-cache-pruner.sh | 4 +- tests/node_cache_pruner/decide.yml | 6 +- tests/unit/test_node_cache_pruner.py | 14 +++- tests/unit/test_node_cache_pruner_role.py | 64 ++++++++++++++++--- 9 files changed, 141 insertions(+), 36 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 873d691..db33070 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1008,6 +1008,20 @@ def arc_node_cache_access_to_remove(found: Sequence[Mapping[str, Any]], labels: return sorted(stale, key=lambda item: (item["kind"] != "RoleBinding", item["namespace"], item["name"])) +def arc_capability_store_namespaces(profiles: Sequence[Mapping[str, Any]], found: Sequence[Mapping[str, Any]], labels: Mapping[str, str]) -> list[str]: + """Return the namespaces the node cache pruner reads the capability store's references from, sorted, or none while no profile uses the store (the pruner then leaves the store alone). + + A pod of a profile that has since switched the store off, or left the fleet, keeps running from the template it was made from and keeps using the entries that template named, so its namespace stays among them: the namespaces of every profile in the fleet, and of every Role the pruner has installed, which the role keeps for a profile that has left the fleet (see arc_node_cache_access_to_remove). An entry is then never removed under a pod because its profile stopped appearing in the fleet. Only a Role carrying every one of labels counts, so another software's objects are ignored. + + Args: + profiles: expanded profiles from arc_profiles, each with its namespace and capability_store. found: the Roles and RoleBindings carrying the pruner's labels, as the cluster returned them. labels: the labels the role puts on every object it creates for the pruner. + """ + if not any(profile["capability_store"] for profile in profiles): + return [] + roles = [resource["metadata"]["namespace"] for resource in found if resource["kind"] == "Role" and all((resource["metadata"].get("labels") or {}).get(key) == value for key, value in labels.items())] + return sorted({profile["namespace"] for profile in profiles} | set(roles)) + + def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapping[str, Any]]], errors: list[str]) -> list[dict[str, str]]: """Return the runner container's variables for a profile's node-cache capabilities: each dirs variable naming its subdirectory of the node cache, and each env variable as given. A variable that two capabilities set to different values is recorded in errors.""" root = node_cache_path.rstrip("/") @@ -1738,6 +1752,7 @@ def filters(self) -> dict[str, Any]: "arc_capability_store_path_errors": arc_capability_store_path_errors, "arc_capability_store_key": arc_capability_store_key, "arc_capability_store_image_errors": arc_capability_store_image_errors, + "arc_capability_store_namespaces": arc_capability_store_namespaces, "arc_node_cache_profiles": arc_node_cache_profiles, "arc_node_cache_host_path_errors": arc_node_cache_host_path_errors, "arc_node_cache_root": arc_node_cache_root, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 49258b7..5e47874 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -321,7 +321,7 @@ What a job can change, with the store mounted read-only on the runner container: `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume. Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. -The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template in a store profile's namespace lists and no unfinished pod on its node lists, after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. The pruner's Role in such a profile's namespace also lists `autoscalingrunnersets.actions.github.com`. +The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's own Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; it stays with the other grants for as long as the pruner is installed, which is what keeps a departed profile's namespace among the namespaces read. `tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the tool images or the test. diff --git a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml index 94a9d72..f41be1d 100644 --- a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml +++ b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml @@ -5,10 +5,10 @@ ansible.builtin.set_fact: github_runner_arc_node_cache_profiles: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_node_cache_profiles(github_runner_arc_capabilities) }}" -# The pruner also removes the capability store's entries that no scale set or pod uses, so it runs for a fleet with a store profile too. -- name: "Node cache: find the namespaces of the profiles that use the capability store" +# The pruner also removes the capability store's entries that no scale set or pod uses, so it runs for a fleet with a store profile too. Which entries are used is read from the scale sets and pods of every namespace a pod made from an earlier template may still run in, not only those of the profiles that use the store now (see arc_capability_store_namespaces); the list is empty, and the store left alone, while no profile uses it. It needs the pruner's Roles as the cluster has them (github_runner_arc_node_cache_pruner_access). +- name: "Node cache: find the namespaces the capability store's references are read from" ansible.builtin.set_fact: - github_runner_arc_capability_store_namespaces: "{{ github_runner_arc_fleet.profiles | selectattr('capability_store') | map(attribute='namespace') | unique | list }}" + github_runner_arc_capability_store_namespaces: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_capability_store_namespaces(github_runner_arc_node_cache_pruner_access.results | map(attribute='resources') | flatten, github_runner_arc_node_cache_pruner_labels) }}" - name: "Node cache: decide whether the pruner is wanted" ansible.builtin.set_fact: diff --git a/roles/github_runner_arc/tasks/install_node_cache_pruner.yml b/roles/github_runner_arc/tasks/install_node_cache_pruner.yml index a8b83e1..fc6f6ce 100644 --- a/roles/github_runner_arc/tasks/install_node_cache_pruner.yml +++ b/roles/github_runner_arc/tasks/install_node_cache_pruner.yml @@ -31,6 +31,17 @@ msg: "{{ github_runner_arc_node_cache_legacy_access.left | join(' ') }}" when: github_runner_arc_node_cache_legacy_access.left | length > 0 +# The pruner's Roles and RoleBindings are found by the labels the role puts on them (all of them, so another software's objects are left alone) in every namespace. They are removed only when the pruner is, and found before the decision because the namespaces of those that exist are among the namespaces the capability store's references are read from (see decide_node_cache_pruner.yml). While it is installed a profile that has left the fleet keeps its grant, which is read-only (it lists pods and nothing else) and confined to that profile's own namespace, so staying is harmless, and it lets the pruner still check, and so clear, a directory the profile left. Bindings go before the roles they bind. +- name: "Node cache: find the pruner's Roles and RoleBindings" + kubernetes.core.k8s_info: + api_version: rbac.authorization.k8s.io/v1 + kind: "{{ item }}" + label_selectors: "{{ github_runner_arc_node_cache_pruner_labels | exadev.github_runner.arc_label_selectors }}" + loop: + - Role + - RoleBinding + register: github_runner_arc_node_cache_pruner_access + - name: "Node cache: decide what the pruner needs" ansible.builtin.include_tasks: decide_node_cache_pruner.yml @@ -82,17 +93,6 @@ loop_control: label: "{{ item.kind }}/node-cache-pruner" -# The pruner's Roles and RoleBindings are found by the labels the role puts on them (all of them, so another software's objects are left alone) in every namespace. They are removed only when the pruner is. While it is installed a profile that has left the fleet keeps its grant, which is read-only (it lists pods and nothing else) and confined to that profile's own namespace, so staying is harmless, and it lets the pruner still check, and so clear, a directory the profile left. Bindings go before the roles they bind. -- name: "Node cache: find the pruner's Roles and RoleBindings" - kubernetes.core.k8s_info: - api_version: rbac.authorization.k8s.io/v1 - kind: "{{ item }}" - label_selectors: "{{ github_runner_arc_node_cache_pruner_labels | exadev.github_runner.arc_label_selectors }}" - loop: - - Role - - RoleBinding - register: github_runner_arc_node_cache_pruner_access - - name: "Node cache: remove the pruner's access when the pruner is not wanted" kubernetes.core.k8s: state: absent diff --git a/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 b/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 index d1fd42d..d8e7fda 100644 --- a/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 +++ b/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 @@ -1,4 +1,4 @@ -{# The node cache pruner (see tasks/install_node_cache_pruner.yml): a ServiceAccount, a Role and RoleBinding in each fleet profile's namespace (so the pruner can list the pods it must not prune under, including those of a profile whose cache capability was removed while its directory remains, and nothing wider), and the DaemonSet. `github_runner_arc_fleet` is the fleet's expanded profiles, `github_runner_arc_node_cache_profiles` those with a node cache and `github_runner_arc_capability_store_namespaces` the namespaces of those that use the capability store, whose root the DaemonSet then mounts as well. #} +{# The node cache pruner (see tasks/install_node_cache_pruner.yml): a ServiceAccount, a Role and RoleBinding in each fleet profile's namespace (so the pruner can list the pods it must not prune under, including those of a profile whose cache capability was removed while its directory remains, and nothing wider), and the DaemonSet. `github_runner_arc_fleet` is the fleet's expanded profiles, `github_runner_arc_node_cache_profiles` those with a node cache and `github_runner_arc_capability_store_namespaces` the namespaces the capability store's references are read from (empty while no profile uses the store), whose root the DaemonSet then mounts as well. #} {% set name = 'node-cache-pruner' %} {% set labels = github_runner_arc_node_cache_pruner_labels %} {% set platform = github_runner_arc_platform_namespace %} @@ -17,16 +17,10 @@ metadata: namespace: {{ profile.namespace | to_json }} labels: {{ labels | to_json }} rules: - # list, to see whether a runner pod of this profile runs on the pruner's node, and which capability store entries the pods there use. Nothing else. + # list, to see whether a runner pod of this profile runs on the pruner's node. Nothing else. - apiGroups: [""] resources: ["pods"] verbs: ["list"] -{% if profile.namespace in github_runner_arc_capability_store_namespaces %} - # list, to see which capability store entries the scale sets' pod templates in a namespace that uses the store list. - - apiGroups: ["actions.github.com"] - resources: ["autoscalingrunnersets"] - verbs: ["list"] -{% endif %} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -43,6 +37,40 @@ roleRef: kind: Role name: {{ (name ~ '-' ~ profile.release) | to_json }} {% endfor %} +{# The capability store's references are read in every namespace a pod made from an earlier template may still run in, which includes a profile that has switched the store off or left the fleet, so this Role is per namespace, not per release. Its name has a colon, which a release name (a Helm release name) cannot have, so it never collides with a profile's Role. #} +{% set store_name = name ~ ':capability-store' %} +{% for namespace in github_runner_arc_capability_store_namespaces %} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ store_name | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ labels | to_json }} +rules: + # list, to see which capability store entries the unfinished pods on the pruner's node use, and which the scale sets' pod templates list. Nothing else. + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiGroups: ["actions.github.com"] + resources: ["autoscalingrunnersets"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ store_name | to_json }} + namespace: {{ namespace | to_json }} + labels: {{ labels | to_json }} +subjects: + - kind: ServiceAccount + name: {{ name | to_json }} + namespace: {{ platform | to_json }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ store_name | to_json }} +{% endfor %} --- apiVersion: apps/v1 kind: DaemonSet diff --git a/scripts/node-cache-pruner.sh b/scripts/node-cache-pruner.sh index cd9e727..8c8f65d 100755 --- a/scripts/node-cache-pruner.sh +++ b/scripts/node-cache-pruner.sh @@ -11,7 +11,7 @@ # # Every step of one profile's check may fail without stopping the others: a failed or partial size reading skips that profile for the round, and a failed pod list means no pruning, since an unknown set of pods may be using the cache: NODE_CACHE_UNPRUNABLE when the server refused the list as Forbidden (there is no Role for the pruner in that namespace), NODE_CACHE_LIST_FAILED for any other failure (the API server was unreachable, the answer was not a pod list), which the next round retries. # -# The capability store (the role README's "Where the copies live"), when STORE_ROOT is set, holds one entry per image payload, named for the image's digest, plus merged sysroots, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. +# The capability store (the role README's "Where the copies live"), when STORE_ROOT is set, holds one entry per image payload, named for the image's digest, plus merged sysroots, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it (STORE_NAMESPACES holds every namespace a pod may still run in from an earlier template, not only those of the profiles using the store now, so an entry is not removed under a pod whose profile has switched the store off or left the fleet) in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. # # Which pods count: those in the profile's namespace on this node, not finished (Succeeded or Failed), that have a hostPath volume at exactly the profile's directory on the host (HOST_CACHE_ROOT//). Another profile's pods, finished pods and pods that do not mount the cache are ignored. # @@ -25,7 +25,7 @@ # - WANTED_CACHES: the profiles that have a cache, as / separated by spaces. When it is set, any other directory under the cache root is removed. Unset leaves every directory alone. # - CHECK_SECONDS: seconds between checks, a positive integer of at most MAX_CHECK_SECONDS (default when unset: 300). # - STORE_ROOT: where the node's capability store is mounted in this pod. Unset leaves the store alone, as on a fleet with no profile using it. -# - STORE_NAMESPACES: the namespaces of the profiles that use the store, separated by spaces, whose scale sets' templates and pods on this node are what keep an entry. +# - STORE_NAMESPACES: the namespaces whose scale sets' templates and pods on this node are what keep an entry, separated by spaces: every namespace a pod that uses the store may run in, which includes a profile that has switched the store off or left the fleet while pods made from its old template run. An entry kept only by a pod in a namespace not listed here is removed. # - PRUNE_ONCE: when "true", check once and exit, for tests. # # Start-up configuration errors, every one of them (a required variable missing, a number that is not a positive integer or is above its maximum), end the pruner with status 2 and a message on stderr that names the variable, before it does anything. The DaemonSet's pods then crash-loop with that message in their log. The role validates the numeric settings it templates with the same pattern and maxima, but a value changed some other way (an edited DaemonSet) reaches only this check. diff --git a/tests/node_cache_pruner/decide.yml b/tests/node_cache_pruner/decide.yml index 079a44a..7f51f1f 100644 --- a/tests/node_cache_pruner/decide.yml +++ b/tests/node_cache_pruner/decide.yml @@ -5,11 +5,15 @@ gather_facts: false vars_files: - ../../roles/github_runner_arc/defaults/main.yml + - ../../roles/github_runner_arc/vars/main.yml + vars: + # The pruner's Roles and RoleBindings as the install tasks found them; a test gives the ones its cluster has. + github_runner_arc_node_cache_pruner_access: {results: [{resources: []}, {resources: []}]} tasks: - name: Decide ansible.builtin.include_tasks: ../../roles/github_runner_arc/tasks/decide_node_cache_pruner.yml - name: Write the decision ansible.builtin.copy: - content: "{{ {'wanted': github_runner_arc_node_cache_pruner_wanted, 'targets': github_runner_arc_node_cache_profiles | map(attribute='target') | list} | to_json }}" + content: "{{ {'wanted': github_runner_arc_node_cache_pruner_wanted, 'targets': github_runner_arc_node_cache_profiles | map(attribute='target') | list, 'store_namespaces': github_runner_arc_capability_store_namespaces} | to_json }}" dest: "{{ decide_output }}" mode: "0644" diff --git a/tests/unit/test_node_cache_pruner.py b/tests/unit/test_node_cache_pruner.py index 454ef10..7b3dda6 100644 --- a/tests/unit/test_node_cache_pruner.py +++ b/tests/unit/test_node_cache_pruner.py @@ -601,8 +601,8 @@ def run_store(self, entries: list[str], incoming: list[str] | None = None, scale completed = subprocess.run(["bash", str(SCRIPT)], check=True, env=environment, capture_output=True, text=True) return Run(root, state, completed.stdout), store - def store_pod(self, keys: list[str], node: str = "node-a", phase: str = "Running", uid: str = "pod-1", from_call: int = 0) -> dict[str, Any]: - return {"name": f"runner-{uid}", "uid": uid, "namespace": "arc-a", "node": node, "phase": phase, "volumes": [], "store_keys": keys, "from_call": from_call} + def store_pod(self, keys: list[str], node: str = "node-a", phase: str = "Running", uid: str = "pod-1", from_call: int = 0, namespace: str = "arc-a") -> dict[str, Any]: + return {"name": f"runner-{uid}", "uid": uid, "namespace": namespace, "node": node, "phase": phase, "volumes": [], "store_keys": keys, "from_call": from_call} def remaining(self, store: Path) -> list[str]: return sorted(path.name for path in store.iterdir() if not path.name.startswith(".")) @@ -616,6 +616,16 @@ def test_an_entry_an_unfinished_pod_on_this_node_lists_is_kept_after_its_templat _, store = self.run_store([KEY_A, KEY_B, KEY_C], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_C]}], pods=[self.store_pod([KEY_A]), self.store_pod([KEY_B], phase="Pending", uid="pod-2")]) self.assertEqual(self.remaining(store), [KEY_A, KEY_B, KEY_C]) + def test_a_pod_in_a_namespace_that_left_the_store_profiles_keeps_its_entries_while_the_namespace_is_listed(self) -> None: + # arc-gone has no scale set any more but a pod made from its old template runs on the node: the role lists the namespace for as long as the pruner holds a Role there. + _, store = self.run_store([KEY_A, KEY_B], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_B]}], pods=[self.store_pod([KEY_A], namespace="arc-gone")], STORE_NAMESPACES="arc-a arc-gone") + self.assertEqual(self.remaining(store), [KEY_A, KEY_B]) + + def test_a_pod_in_a_namespace_that_is_not_listed_keeps_nothing(self) -> None: + # What the old scan did to arc-gone: the namespace dropped out of STORE_NAMESPACES and the entry went under the running pod. + _, store = self.run_store([KEY_A, KEY_B], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_B]}], pods=[self.store_pod([KEY_A], namespace="arc-gone")]) + self.assertEqual(self.remaining(store), [KEY_B]) + def test_a_finished_pod_or_one_on_another_node_keeps_nothing(self) -> None: _, store = self.run_store([KEY_A, KEY_B], pods=[self.store_pod([KEY_A], phase="Succeeded"), self.store_pod([KEY_B], node="node-b", uid="pod-2")]) self.assertEqual(self.remaining(store), []) diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index 3675d35..561777e 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -56,19 +56,28 @@ def render(**role_vars: Any) -> list[dict[str, Any]]: return documents -def decide(fleet_targets: list[str], **role_vars: Any) -> dict[str, Any]: - """Run the decision for a fleet of the given profile targets, each with the pnpm-store capability when its target is listed in with_cache, and using the capability store when it is listed in with_store.""" +def decide_everything(fleet_targets: list[str], **role_vars: Any) -> dict[str, Any]: + """Run the decision for a fleet of the given profile targets, each with the pnpm-store capability when its target is listed in with_cache, and using the capability store when it is listed in with_store. Roles are the pruner's Roles the cluster has, as (namespace, release) pairs.""" with_cache = role_vars.pop("with_cache", fleet_targets) with_store = role_vars.pop("with_store", []) + roles = role_vars.pop("roles", []) profiles = [{"namespace": target.split("/")[0], "release": target.split("/")[1], "target": target, "capabilities": ["pnpm-store" if target in with_cache else "node"], "capability_store": target in with_store} for target in fleet_targets] + found = {"results": [{"resources": [access("Role", namespace, release) for namespace, release in roles]}, {"resources": [access("RoleBinding", namespace, release) for namespace, release in roles]}]} with tempfile.TemporaryDirectory() as work: output = Path(work) / "decision.json" - completed = ansible(DECIDE, decide_output=str(output), github_runner_arc_fleet={"profiles": profiles}, github_runner_arc_capabilities=CATALOGUE, **role_vars) + completed = ansible(DECIDE, decide_output=str(output), github_runner_arc_fleet={"profiles": profiles}, github_runner_arc_capabilities=CATALOGUE, github_runner_arc_node_cache_pruner_access=found, **role_vars) assert completed.returncode == 0, completed.stdout + completed.stderr decision: dict[str, Any] = json.loads(output.read_text()) return decision +def decide(fleet_targets: list[str], **role_vars: Any) -> dict[str, Any]: + """The decision whether the pruner is wanted and for which node caches, see decide_everything.""" + decision = decide_everything(fleet_targets, **role_vars) + del decision["store_namespaces"] + return decision + + class ManifestTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -142,6 +151,35 @@ def test_a_profile_using_the_capability_store_wants_the_pruner_without_a_node_ca self.assertEqual(decide(["arc-a/main"], with_cache=[], with_store=["arc-a/main"]), {"wanted": True, "targets": []}) +class CapabilityStoreNamespacesTest(unittest.TestCase): + """The namespaces the pruner reads the store's references from: those a pod made from an earlier template may still run in, not only those of the profiles using the store now.""" + + def test_a_namespace_whose_profile_switched_the_store_off_is_still_read(self) -> None: + decision = decide_everything(["arc-a/main", "arc-b/builders"], with_store=["arc-a/main"]) + self.assertEqual(decision["store_namespaces"], ["arc-a", "arc-b"]) + + def test_a_namespace_whose_profile_left_the_fleet_is_still_read_while_the_pruner_keeps_its_role(self) -> None: + decision = decide_everything(["arc-a/main"], with_store=["arc-a/main"], roles=[("arc-a", "main"), ("arc-gone", "old")]) + self.assertEqual(decision["store_namespaces"], ["arc-a", "arc-gone"]) + + def test_no_namespace_is_read_while_no_profile_uses_the_store_so_the_store_is_left_alone(self) -> None: + decision = decide_everything(["arc-a/main", "arc-b/builders"], roles=[("arc-gone", "old")]) + self.assertEqual(decision["store_namespaces"], []) + + def test_only_roles_carrying_every_label_of_the_pruner_count(self) -> None: + profiles = [{"namespace": "arc-a", "capability_store": True}] + found = [access("Role", "arc-gone", "old"), access("Role", "other", "app", labels={"app.kubernetes.io/name": "node-cache-pruner"}), access("RoleBinding", "bound-only", "old")] + self.assertEqual(arc.arc_capability_store_namespaces(profiles, found, OWN_LABELS), ["arc-a", "arc-gone"]) + + def test_the_install_reads_the_pruners_roles_before_it_decides(self) -> None: + calls = install(["arc-a/main", "arc-b/builders"], with_store=["arc-a/main"], existing=[access("Role", "arc-gone", "old"), access("RoleBinding", "arc-gone", "old")]) + daemonset = next(call["definition"] for call in calls if (call.get("definition") or {}).get("kind") == "DaemonSet") + env = {item["name"]: item.get("value") for item in daemonset["spec"]["template"]["spec"]["containers"][0]["env"]} + self.assertEqual(env["STORE_NAMESPACES"], "arc-a arc-b arc-gone") + store_roles = sorted(call["definition"]["metadata"]["namespace"] for call in calls if (call.get("definition") or {}).get("metadata", {}).get("name") == "node-cache-pruner:capability-store" and call["definition"]["kind"] == "Role") + self.assertEqual(store_roles, ["arc-a", "arc-b", "arc-gone"]) + + class CapabilityStoreManifestTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: @@ -157,10 +195,19 @@ def test_the_pruner_is_told_where_the_store_is_and_which_namespaces_use_it(self) volumes = {volume["name"]: volume for volume in self.pod["volumes"]} self.assertEqual(volumes["capability-store"]["hostPath"], {"path": "/var/lib/github-runner/capability-store", "type": "DirectoryOrCreate"}) - def test_only_a_namespace_that_uses_the_store_may_have_its_scale_sets_listed(self) -> None: - rules = {role["metadata"]["namespace"]: role["rules"] for role in self.documents if role["kind"] == "Role"} - self.assertIn({"apiGroups": ["actions.github.com"], "resources": ["autoscalingrunnersets"], "verbs": ["list"]}, rules["arc-a"]) - self.assertEqual(rules["arc-b"], [{"apiGroups": [""], "resources": ["pods"], "verbs": ["list"]}]) + def test_every_namespace_the_store_is_read_from_has_a_role_that_lists_pods_and_scale_sets_and_nothing_else(self) -> None: + store_roles = {role["metadata"]["namespace"]: role["rules"] for role in self.documents if role["kind"] == "Role" and role["metadata"]["name"] == "node-cache-pruner:capability-store"} + self.assertEqual(sorted(store_roles), ["arc-a"]) + self.assertEqual(store_roles["arc-a"], [{"apiGroups": [""], "resources": ["pods"], "verbs": ["list"]}, {"apiGroups": ["actions.github.com"], "resources": ["autoscalingrunnersets"], "verbs": ["list"]}]) + bindings = [binding for binding in self.documents if binding["kind"] == "RoleBinding" and binding["metadata"]["name"] == "node-cache-pruner:capability-store"] + self.assertEqual([binding["metadata"]["namespace"] for binding in bindings], ["arc-a"]) + self.assertEqual(bindings[0]["roleRef"]["name"], "node-cache-pruner:capability-store") + self.assertEqual({key: bindings[0]["metadata"]["labels"].get(key) for key in OWN_LABELS}, OWN_LABELS) + + def test_a_profiles_own_role_never_lists_scale_sets(self) -> None: + for role in self.documents: + if role["kind"] == "Role" and role["metadata"]["name"] != "node-cache-pruner:capability-store": + self.assertEqual(role["rules"], [{"apiGroups": [""], "resources": ["pods"], "verbs": ["list"]}], role["metadata"]) def test_without_a_store_profile_the_pruner_neither_mounts_nor_prunes_the_store(self) -> None: documents = render() @@ -221,7 +268,8 @@ def legacy_binding(subjects: list[dict[str, Any]] | None = None) -> dict[str, An def install_with_output(fleet_targets: list[str], with_cache: list[str] | None = None, existing: list[dict[str, Any]] | None = None, **role_vars: Any) -> tuple[list[dict[str, Any]], str]: """Run the role's pruner install tasks against the stand-in kubernetes.core, and return the module calls it recorded and the playbook's output. Each fleet profile carries the pnpm-store capability when its target is in with_cache (all of them when it is None).""" - profiles = [{"namespace": t.split("/")[0], "release": t.split("/")[1], "target": t, "capabilities": ["pnpm-store"] if with_cache is None or t in with_cache else ["node"], "capability_store": False} for t in fleet_targets] + with_store = role_vars.pop("with_store", []) + profiles = [{"namespace": t.split("/")[0], "release": t.split("/")[1], "target": t, "capabilities": ["pnpm-store"] if with_cache is None or t in with_cache else ["node"], "capability_store": t in with_store} for t in fleet_targets] with tempfile.TemporaryDirectory() as work: modules = Path(work) / "stubs" / "ansible_collections" / "kubernetes" / "core" / "plugins" / "modules" modules.mkdir(parents=True) From 6260f35c8c9270de6c713b8a709bae030b9a8f51 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 09:36:50 +0100 Subject: [PATCH 07/43] fix(arc): refuse a node cache and capability store that share or contain a directory Each validator accepted a path inside the other's root. The pruner removes whatever it finds directly under each root as root, so a node cache inside the store was an unreferenced store entry and was deleted while pods used it, and a store inside the node cache was a profile directory the fleet did not want. Validation now refuses equal or nested paths in either direction, with a message naming both settings. --- plugins/filter/arc.py | 15 ++++++++++ roles/github_runner_arc/README.md | 2 +- roles/github_runner_arc/tasks/validate.yml | 1 + tests/unit/test_node_cache_pruner_role.py | 32 ++++++++++++++++++++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index db33070..9d98d66 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1088,6 +1088,20 @@ def arc_capability_store_host_path_errors(host_path: Any) -> list[str]: return [f"the capability store's host path {host_path} is inside {inside[0]}, which the node itself uses"] if inside else [] +def arc_host_path_overlap_errors(node_cache_host_path: Any, capability_store_host_path: Any) -> list[str]: + """Return an error when github_runner_arc_node_cache_host_path and github_runner_arc_capability_store_host_path are the same directory or one is inside the other, and none otherwise. + + The pruner empties each as root and treats whatever it finds directly under a root as that root's own entries: a node cache inside the store is an unreferenced store entry and is deleted while pods use it, and a store inside the node cache is a profile directory the fleet no longer wants. A value that is not a string is reported by the setting's own check, so it is not compared here. + """ + if not isinstance(node_cache_host_path, str) or not isinstance(capability_store_host_path, str): + return [] + cache = _canonical_path(node_cache_host_path) + store = _canonical_path(capability_store_host_path) + if cache == store or cache.startswith(f"{store}/") or store.startswith(f"{cache}/"): + return [f"github_runner_arc_node_cache_host_path ({node_cache_host_path}) and github_runner_arc_capability_store_host_path ({capability_store_host_path}) must be separate directories, neither inside the other, since the pruner removes what it finds directly under each as root"] + return [] + + def arc_capability_store_path_errors(store_path: Any) -> list[str]: """Return what is wrong with github_runner_arc_capability_store_path, where the store is mounted read-only on the runner container: empty when it is an absolute directory other than / with no colon or line break.""" if not isinstance(store_path, str) or not store_path.startswith("/") or store_path == "/" or any(character in store_path for character in ":\n\r"): @@ -1753,6 +1767,7 @@ def filters(self) -> dict[str, Any]: "arc_capability_store_key": arc_capability_store_key, "arc_capability_store_image_errors": arc_capability_store_image_errors, "arc_capability_store_namespaces": arc_capability_store_namespaces, + "arc_host_path_overlap_errors": arc_host_path_overlap_errors, "arc_node_cache_profiles": arc_node_cache_profiles, "arc_node_cache_host_path_errors": arc_node_cache_host_path_errors, "arc_node_cache_root": arc_node_cache_root, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 5e47874..4487e2a 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -29,7 +29,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. - `github_runner_arc_node_cache_path`, `github_runner_arc_node_cache_host_path`, `github_runner_arc_node_cache_size_limit_gib`, `github_runner_arc_node_cache_hard_limit_factor`, `github_runner_arc_node_cache_prune_seconds`: where a node-cache capability's volume is mounted on the runner container (`/node-cache`), the dedicated directory on each node under which each profile's cache lives (`/var/lib/github-runner/node-cache`), the most gibibytes a profile's cache may hold before the pruner empties it (20; 0 for no pruner, a negative value is an error), the hard limit as a multiple of that (2) past which a busy cache is reported on every round, and the seconds between the pruner's checks (300, at least 1). See [Capabilities](#capabilities). -- `github_runner_arc_capability_store_host_path`, `github_runner_arc_capability_store_path`, `github_runner_arc_capability_store_image`: the dedicated directory on each node the capability store lives in (`/var/lib/github-runner/capability-store`, with the same rules as the node cache's, its last component `capability-store` or ending in `-capability-store`), where it is mounted read-only on the runner container (`/opt/capability-store`), and the digest-pinned busybox image its own init containers run. See [The capability store](#the-capability-store). +- `github_runner_arc_capability_store_host_path`, `github_runner_arc_capability_store_path`, `github_runner_arc_capability_store_image`: the dedicated directory on each node the capability store lives in (`/var/lib/github-runner/capability-store`, with the same rules as the node cache's, its last component `capability-store` or ending in `-capability-store`, and neither the same directory as the node cache's nor inside it or containing it, since the pruner removes what it finds directly under each as root), where it is mounted read-only on the runner container (`/opt/capability-store`), and the digest-pinned busybox image its own init containers run. See [The capability store](#the-capability-store). - `github_runner_arc_registry_cache_enabled` (default `false`) and the other `github_runner_arc_registry_cache_*` variables: a pull-through registry cache every Docker-in-Docker profile pulls through, falling back to each registry when the cache is down (see [Registry cache](#registry-cache)). - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`. - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart. The controller chart's CRDs are applied (server-side) from `github_runner_arc_controller_chart_ref` at the controller's version before every controller install or upgrade, because Helm never upgrades a chart's CRDs itself; without that, a newer scale-set chart is rejected for fields the cluster's CRDs predate. Move the two pins together, and the CRDs follow. `tests/arc_upgrade/run.sh` covers the upgrade in a kind cluster. diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index f1cc00a..75a8d4b 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -41,6 +41,7 @@ + (github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_host_path_errors) + (github_runner_arc_node_cache_path | exadev.github_runner.arc_node_cache_path_errors) + (github_runner_arc_capability_store_host_path | exadev.github_runner.arc_capability_store_host_path_errors) + + (github_runner_arc_node_cache_host_path | exadev.github_runner.arc_host_path_overlap_errors(github_runner_arc_capability_store_host_path)) + (github_runner_arc_capability_store_path | exadev.github_runner.arc_capability_store_path_errors) + (github_runner_arc_capability_store_image | exadev.github_runner.arc_capability_store_image_errors) }} diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index 561777e..1f2ee55 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -465,6 +465,38 @@ def test_the_capability_store_settings_are_checked(self) -> None: self.assertEqual(len(problems), 1, problems) self.assertIn(message, problems[0]) + def test_the_node_cache_and_the_capability_store_may_not_share_or_contain_a_directory(self) -> None: + store = "/var/lib/github-runner/capability-store" + cache = "/var/lib/github-runner/node-cache" + self.assertEqual(self.problems(github_runner_arc_node_cache_host_path=cache, github_runner_arc_capability_store_host_path=store), []) + for role_vars, shape in ( + ({"github_runner_arc_node_cache_host_path": f"{store}/node-cache"}, "node cache inside the store"), + ({"github_runner_arc_capability_store_host_path": f"{cache}/capability-store"}, "store inside the node cache"), + ): + with self.subTest(shape): + problems = self.problems(**role_vars) + self.assertEqual(len(problems), 1, problems) + self.assertIn("github_runner_arc_node_cache_host_path", problems[0]) + self.assertIn("github_runner_arc_capability_store_host_path", problems[0]) + self.assertIn("neither inside the other", problems[0]) + + def test_the_same_directory_for_both_is_an_overlap(self) -> None: + # Each setting's own name rule rules out a shared path in the role, so the check is exercised directly as well as through the role. + for first, second in (("/srv/ci/shared", "/srv/ci/shared"), ("/srv/ci/shared/", "/srv/ci//shared")): + with self.subTest(first=first, second=second): + errors = arc.arc_host_path_overlap_errors(first, second) + self.assertEqual(len(errors), 1, errors) + self.assertIn("github_runner_arc_node_cache_host_path (" + first + ")", errors[0]) + self.assertIn("github_runner_arc_capability_store_host_path (" + second + ")", errors[0]) + + def test_a_sibling_with_the_same_prefix_is_not_an_overlap(self) -> None: + self.assertEqual(arc.arc_host_path_overlap_errors("/srv/ci/node-cache", "/srv/ci/node-cache-2/capability-store"), []) + self.assertEqual(arc.arc_host_path_overlap_errors("/srv/ci/capability-store-2", "/srv/ci/capability-store"), []) + + def test_a_value_that_is_not_a_path_is_left_to_its_own_check(self) -> None: + self.assertEqual(arc.arc_host_path_overlap_errors(None, "/srv/ci/capability-store"), []) + self.assertNotEqual(self.problems(github_runner_arc_node_cache_host_path=None), []) + def test_a_profile_using_the_capability_store_is_refused_without_the_pruner(self) -> None: tasks = yaml.safe_load((ROLE / "tasks" / "validate.yml").read_text()) [task] = [task for task in tasks if task["name"] == "Fail when a profile uses the capability store without the pruner"] From c9553effd6880df5861676aca33eab5a7bf6f893 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 10:07:10 +0100 Subject: [PATCH 08/43] fix(arc): flush capability store entries before publishing and rebuild a pod's partial copy on rerun Publishing renamed a copy onto its digest-named entry with nothing written to disk first. A power loss or crash could persist the rename ahead of the data, leaving an entry of empty files that the existence check then served to every later pod, because entries are never revalidated. The publish container and the sysroot merge now run sync before the rename, so an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: the order of writes already guarantees what it would state, and every pod would pay a check at its start. The kubelet reruns a pod's init containers after a sandbox change. A copy killed part-way left its destination behind, and the rerun's cp -R then nested the payload under it and kept the partial first copy, which publishing renamed into the store with the wrong layout for good. The copy container, for capability payloads and the dind externals alike, now removes the pod's own destination first, and so does the sysroot merge, whose mkdir failed outright on a leftover. A capability image needs rm beside sh and cp when its profile uses the store. --- plugins/filter/arc.py | 21 ++++-- roles/github_runner_arc/README.md | 10 +-- roles/github_runner_arc/defaults/main.yml | 2 +- tests/unit/test_capabilities.py | 89 +++++++++++++++++++++++ 4 files changed, 109 insertions(+), 13 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 9d98d66..c758eaf 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1144,23 +1144,30 @@ def _runner_security_context(runner_user: int, runner_group: int) -> dict[str, A return {"runAsUser": int(runner_user), "runAsGroup": int(runner_group), "runAsNonRoot": True, "allowPrivilegeEscalation": False} +# Copies an image's payload to this pod's own incoming directory, unless the store already holds the entry. The destination is removed first so that a rerun after a copy killed part-way starts from nothing: cp -R onto an existing directory would nest the payload under it and keep the partial copy, and publishing would then rename that mixed tree. +_COPY_SCRIPT = ( + '[ -d "$1/$2" ] && exit 0; incoming="$1/' + _CAPABILITY_STORE_INCOMING + '/$2.$POD_UID.$4";' + ' rm -rf "$incoming" && cp -R "$3" "$incoming"' +) + + def _capability_store_copy_container(name: str, image: str, payload: str, key: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: - """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin and cp. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed.""" + """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin, rm and cp. It removes this pod's own destination first, since the kubelet reruns a pod's init containers after a sandbox change and a copy killed part-way leaves the destination behind, and cp -R onto a directory that exists nests the payload inside it and keeps the partial first copy. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed.""" root = store["path"].rstrip("/") return { "name": name, "image": image, - "command": ["/bin/sh", "-c", '[ -d "$1/$2" ] || cp -R "$3" "$1/' + _CAPABILITY_STORE_INCOMING + '/$2.$POD_UID.$4"', "copy-to-capability-store", root, key, payload, name], + "command": ["/bin/sh", "-c", _COPY_SCRIPT, "copy-to-capability-store", root, key, payload, name], "env": [dict(_POD_UID_ENV)], "securityContext": _runner_security_context(runner_user, runner_group), "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"]}], } -# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. +# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. Every copy is flushed to disk (sync, which writes out every file's data and every directory's entries) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. _PUBLISH_SCRIPT = ( 'root=$1; shift; for key in "$@"; do' - ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; mv -T "$incoming" "$root/$key" 2>/dev/null || rm -rf "$incoming"; done;' + ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; mv -T "$incoming" "$root/$key" 2>/dev/null || rm -rf "$incoming"; done;' ' [ -d "$root/$key" ] || { echo "capability store: $key is not in $root after publishing" >&2; exit 1; };' ' done' ) @@ -1178,12 +1185,12 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M } -# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then publishes it like any other entry. +# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then flushes and publishes it like any other entry, rebuilding from nothing when this pod's own earlier partial merge is found. _SYSROOT_MERGE_SCRIPT = ( 'root=$1; merged=$2; shift 2; [ -d "$root/$merged" ] && exit 0;' - ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; mkdir "$incoming" || exit 1;' + ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; rm -rf "$incoming" && mkdir "$incoming" || exit 1;' ' for key in "$@"; do cp -alf "$root/$key/." "$incoming/" || exit 1; done;' - ' mv -T "$incoming" "$root/$merged" 2>/dev/null || rm -rf "$incoming";' + ' sync || exit 1; mv -T "$incoming" "$root/$merged" 2>/dev/null || rm -rf "$incoming";' ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' ) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 4487e2a..5f9acdb 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -220,7 +220,7 @@ A profile is the unit a workflow selects, so a set of capabilities becomes a nam A tool image is any image that has: -- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /toolcache/. "$1"/'`, with the tool-cache mount as `$1`; +- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /toolcache/. "$1"/'`, with the tool-cache mount as `$1` (a profile with `capability_store: true` copies with `rm -rf` and `cp -R` instead, so the image needs `rm` as well); - the tool under `/toolcache`, already in the Actions tool-cache layout: `/toolcache////` holding the tool, and an empty `/toolcache///.complete` file beside that directory, which `@actions/tool-cache` requires before it treats an entry as present; - files readable, and directories searchable, by any user, since the copy runs as the runner's user. @@ -248,7 +248,7 @@ and is declared as `{image: ghcr.io/example/toolcache-example-cli:1.2.3, path: e A system-library image is any image that has: -- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /sysroot/. "$1"/'`, with the sysroot mount as `$1`; `cp -R` keeps symbolic links as links, which a library's versioned names (`libfoo.so` to `libfoo.so.1`) rely on; +- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /sysroot/. "$1"/'`, with the sysroot mount as `$1` (a profile with `capability_store: true` also needs `rm`, as for a tool-cache image); `cp -R` keeps symbolic links as links, which a library's versioned names (`libfoo.so` to `libfoo.so.1`) rely on; - the payload under `/sysroot`, laid out as an install prefix: programs in `bin/`, shared libraries in `lib/` (not `lib64/` or a multiarch subdirectory, which the search paths do not include), headers in `include/`, pkg-config files in `lib/pkgconfig/` or `share/pkgconfig/`, and CMake packages anywhere CMake's search under a prefix finds them, such as `lib/cmake//`; - a payload that works from whatever directory it is copied to, given the search paths and the capability's `env`: it must not need its build prefix to exist. In practice that means pkg-config files that name their prefix relative to their own location (`prefix=${pcfiledir}/../..`, which pkg-config and pkgconf both expand), CMake packages that compute their paths relative to themselves (what `install(EXPORT)` generates), no libtool `.la` archives (they record absolute paths), and, for a library that records its install prefix in its binaries, an environment variable it reads to find its files elsewhere, set through `env`; - programs and libraries built for the runner image's own C library, on the same distribution release as the runner image or an older one, linking only libraries the runner image already has or the payload carries itself. The payload should not carry its own copy of a library the runner image has, such as the C library, OpenSSL or the C++ runtime: `LD_LIBRARY_PATH` puts the sysroot first for every step, so its copy would replace the image's for everything those steps run; @@ -312,9 +312,9 @@ Every runner pod of a profile copies the same bytes at its start: each capabilit An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. -- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. -- `capability-store-publish` renames each copy onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete, two pods starting together never see each other's half-written copy, and the container fails unless every entry the pod needs is in the store. -- `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, published the same way. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. +- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. +- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. The container fails unless every entry the pod needs is in the store. +- `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and it cannot change anything in the store, so no job changes what another pod uses. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a job cannot write into them, where the per-pod copies were writable. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index ce07aec..e0b7900 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -29,7 +29,7 @@ github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" # Where the store is mounted, read-only, on the runner container: the tool cache's links point into it. github_runner_arc_capability_store_path: "/opt/capability-store" -# The image the store's own init containers run (creating the root, publishing a copy with mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chown, mv, cp, ln and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. +# The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chown, sync, mv, cp, ln and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. github_runner_arc_capability_store_image: "docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" # The most gibibytes one profile's cache may hold on a node before the pruner empties it. The limit is per profile, so a node's worst case is the sum of the limits of the profiles with a cache on it, times github_runner_arc_node_cache_hard_limit_factor. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) empties a cache only while no runner pod of its profile runs on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow; a negative value, a leading zero, a fraction or a value above 1073741824 is an error (the pruner's own arithmetic bounds it, and the role refuses what the pruner would). diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 2a42970..a0a241a 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -818,6 +818,95 @@ def test_of_two_pods_copying_one_image_at_once_the_first_to_publish_wins_and_the self.assertFalse((self.root / DIGEST_A / f"{DIGEST_A}.pod-2.capability-node").exists()) self.assertEqual(list((self.root / ".incoming").iterdir()), []) + def layout(self, directory: Path) -> dict[str, str]: + """Return every path under directory with what it holds (a file's content, a link's target, a directory's marker), so two trees compare by layout and content.""" + return {str(path.relative_to(directory)): path.readlink().as_posix() if path.is_symlink() else path.read_text() if path.is_file() else "" for path in sorted(directory.rglob("*"))} + + def fake_sync(self, exit_status: int) -> dict[str, str]: + """Return an environment whose sync records whether the copy was still unpublished when it ran, in sync.log, and exits with exit_status. It takes the copy's and the entry's paths from the environment, since the test sets them for it.""" + bin_dir = self.work / "fake-bin" + bin_dir.mkdir(exist_ok=True) + script = bin_dir / "sync" + script.write_text('#!/bin/sh\n{ [ -d "$SYNC_INCOMING" ] && printf "incoming " || printf "no-incoming "; [ -d "$SYNC_ENTRY" ] && echo published || echo unpublished; } >> "$SYNC_LOG"\nexit ' + str(exit_status) + "\n") + script.chmod(0o755) + return {"PATH": f"{bin_dir}:{os.environ['PATH']}", "SYNC_LOG": str(self.work / "sync.log")} + + def run_with_env(self, container: dict[str, Any], uid: str, rewrite: dict[str, str], env: dict[str, str]) -> subprocess.CompletedProcess[str]: + command = [rewrite.get(argument, argument) for argument in container["command"]] + return subprocess.run(command, env={**os.environ, "POD_UID": uid, **env}, capture_output=True, text=True, check=False) + + def test_a_copy_is_flushed_to_disk_before_it_is_renamed_into_the_store(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + self.assertEqual(self.run_container(containers["capability-node"], "pod-1", rewrite).returncode, 0) + env = {**self.fake_sync(0), "SYNC_INCOMING": str(self.root / ".incoming" / f"{DIGEST_A}.pod-1.capability-node"), "SYNC_ENTRY": str(self.root / DIGEST_A)} + published = self.run_with_env(containers["capability-store-publish"], "pod-1", rewrite, env) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual((self.work / "sync.log").read_text(), "incoming unpublished\n") + self.assertEqual((self.root / DIGEST_A / "node/22.0.0/x64/bin/node").read_text(), "binary") + + def test_a_copy_that_cannot_be_flushed_is_not_published(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + self.assertEqual(self.run_container(containers["capability-node"], "pod-1", rewrite).returncode, 0) + env = {**self.fake_sync(1), "SYNC_INCOMING": "unused", "SYNC_ENTRY": "unused"} + published = self.run_with_env(containers["capability-store-publish"], "pod-1", rewrite, env) + self.assertNotEqual(published.returncode, 0) + self.assertFalse((self.root / DIGEST_A).exists()) + + def test_a_merged_sysroot_is_flushed_to_disk_before_it_is_renamed_into_the_store(self) -> None: + for digest, files in ((DIGEST_B, {"lib/libmpi.so": "mpi"}), (DIGEST_C, {"lib/libpq.so": "pq"})): + for relative, content in files.items(): + (self.root / digest / relative).parent.mkdir(parents=True, exist_ok=True) + (self.root / digest / relative).write_text(content) + merge = self.containers(["mpi", "libpq"])["capability-store-sysroot"] + merged = merge["command"][5] + env = {**self.fake_sync(0), "SYNC_INCOMING": str(self.root / ".incoming" / f"{merged}.pod-1.merge"), "SYNC_ENTRY": str(self.root / merged)} + completed = self.run_with_env(merge, "pod-1", {"/opt/capability-store": str(self.root)}, env) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual((self.work / "sync.log").read_text(), "incoming unpublished\n") + self.assertTrue((self.root / merged / "lib/libpq.so").is_file()) + + def assert_rerun_after_a_partial_copy_publishes_the_source_layout(self, copy: dict[str, Any], publish: dict[str, Any], key: str, source: Path, rewrite: dict[str, str]) -> None: + """Leave what a copy killed part-way leaves in the pod's own destination, run the copy twice (the kubelet reruns a pod's init containers after a sandbox change), publish, and require the entry to hold exactly the source's layout and content.""" + debris = self.root / ".incoming" / f"{key}.pod-1.{copy['name']}" + (debris / "bin").mkdir(parents=True) + (debris / "bin/partial").write_text("half a file") + (debris / source.name).mkdir() + (debris / source.name / "stale").write_text("nested by a copy onto an existing destination") + for _ in range(2): + completed = self.run_container(copy, "pod-1", rewrite) + self.assertEqual(completed.returncode, 0, completed.stderr) + published = self.run_container(publish, "pod-1", rewrite) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual(self.layout(self.root / key), self.layout(source)) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def test_a_capability_copy_rerun_after_a_partial_copy_publishes_exactly_the_payloads_layout(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary", "node/22.0.0/x64.complete": ""}) + containers = self.containers(["node"]) + self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["capability-node"], containers["capability-store-publish"], DIGEST_A, toolcache, {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)}) + + def test_the_dind_externals_copy_rerun_after_a_partial_copy_publishes_exactly_the_externals_layout(self) -> None: + externals = self.payload("externals", {"node20/bin/node": "binary", "node20/lib/libnode.so": "library"}) + spec = arc.arc_capability_store_externals(DIND_VALUES, STORE, 1001, 1001)["values"]["template"]["spec"] + containers = {container["name"]: container for container in spec["initContainers"]} + self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["init-dind-externals"], containers["capability-store-publish-externals"], DIGEST_RUNNER, externals, {"/opt/capability-store": str(self.root), "/home/runner/externals": str(externals)}) + + def test_a_sysroot_merge_rerun_after_a_partial_merge_publishes_only_the_libraries(self) -> None: + (self.root / DIGEST_B / "lib").mkdir(parents=True) + (self.root / DIGEST_B / "lib/libmpi.so").write_text("mpi") + merge = self.containers(["mpi"])["capability-store-sysroot"] + merged = merge["command"][5] + debris = self.root / ".incoming" / f"{merged}.pod-1.merge" + (debris / "lib").mkdir(parents=True) + (debris / "lib/partial").write_text("half a file") + completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.layout(self.root / merged), {"lib": "", "lib/libmpi.so": "mpi"}) + def test_publishing_fails_when_the_entry_is_neither_copied_nor_in_the_store(self) -> None: containers = self.containers(["node"]) published = self.run_container(containers["capability-store-publish"], "pod-1", {"/opt/capability-store": str(self.root)}) From 0e080c76739fc276d7fea054e628bcb57121d9c0 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:12:28 +0100 Subject: [PATCH 09/43] test(arc): run two capability store pods and the pruner against real mounts in kind The kind capability test only exercised copies into each pod. It now pushes the tool and library images to a registry container in the cluster so they have the digest a store entry is named for, renders a profile with the store on, and starts two pods against the node's store. The first pod meets a store root the runner user already owns with no incoming directory, which only works because the preparation keeps DAC_OVERRIDE. The checks cover that the entries are exactly the ones the pod lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's own tree but is refused a write through a link and any write into the store or the sysroot, and that @actions/tool-cache finds the tool. The node cache pruner runs once from the role's rendered manifests in a pod of its own. It must keep every entry a running pod lists, remove an entry and a copy that nothing lists, and remove the pods' entries once they have finished. --- .github/workflows/capabilities.yml | 5 +- tests/capabilities/run.sh | 298 ++++++++++++++++++++++++++--- 2 files changed, 273 insertions(+), 30 deletions(-) diff --git a/.github/workflows/capabilities.yml b/.github/workflows/capabilities.yml index 91adef8..687629e 100644 --- a/.github/workflows/capabilities.yml +++ b/.github/workflows/capabilities.yml @@ -1,6 +1,6 @@ name: Capabilities -# Runs the ARC role's scale-set profile capabilities in a kind cluster (tests/capabilities/run.sh): the reference Node.js tool image and libpq system-library image built from this checkout, values rendered through the role's own tasks, the scale-set chart rendered with them and checked against the ARC CRDs, and a pod from the chart's runner pod template whose init containers fill the tool cache and the sysroot, whose job-started hook puts the tool on PATH and the library on its search paths, and whose step finds the tool through @actions/tool-cache and runs the library's programs. Only on changes that can affect them. Builds nothing it publishes. +# Runs the ARC role's scale-set profile capabilities in a kind cluster (tests/capabilities/run.sh): the reference Node.js tool image and libpq system-library image built from this checkout, values rendered through the role's own tasks, the scale-set chart rendered with them and checked against the ARC CRDs, and a pod from the chart's runner pod template whose init containers fill the tool cache and the sysroot, whose job-started hook puts the tool on PATH and the library on its search paths, and whose step finds the tool through @actions/tool-cache and runs the library's programs; then the same images pushed to a registry container in the test, two pods of a profile with the capability store on against the node's store, and the node cache pruner run once in a pod from the role's rendered manifests. Only on changes that can affect them. Builds nothing it publishes. # # GitHub-hosted ubuntu-latest has Docker, kind, kubectl and helm already. @@ -9,8 +9,11 @@ on: paths: - roles/github_runner_arc/** - plugins/filter/arc.py + - scripts/node-cache-pruner.sh + - autoscaler/Dockerfile - tool-images/** - tests/capabilities/** + - tests/node_cache_pruner/render.yml - .github/workflows/capabilities.yml workflow_dispatch: diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index cd8a516..16ee9de 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -1,15 +1,21 @@ #!/usr/bin/env bash # Integration test for scale-set profile capabilities: builds the reference Node.js tool image from tool-images/node and the reference libpq system-library image from tool-images/libpq, renders a profile carrying them (the tool with a path, so the job-started hook puts it on PATH; the library with an extra variable) and a hook capability through the role's own tasks, renders the gha-runner-scale-set chart with those values, and checks the AutoscalingRunnerSet against the ARC CRDs with a server-side dry run in a kind cluster. It then starts a pod from the chart's own runner pod template in that cluster, with the runner's command replaced by what the runner does at job start (run the ACTIONS_RUNNER_HOOK_JOB_STARTED hook with GITHUB_PATH and GITHUB_ENV) followed by a step that asserts the cached Node.js is on PATH and that @actions/tool-cache's find, the lookup actions/setup-node makes before downloading, returns the copy the init container put in the tool cache, and that libpq's programs run from the sysroot its init container filled, with the search paths and the extra variable the hook set, and that the runner user can write to the node-cache directory its init container prepared, under a pod-level non-root securityContext. # -# Usage: tests/capabilities/run.sh. Needs Docker, kind, kubectl, helm, python3 with PyYAML, and ansible-playbook (ANSIBLE_PLAYBOOK overrides which); reaches the network for the chart, the runner image and the npm package. GRTEST_SCALESET_CHART_VERSION pins the scale-set chart and the controller chart its CRDs come from (default: the latest). Creates a kind cluster named grtest-capabilities and removes it on exit unless GRTEST_KEEP=1. +# It then does the same for a profile with capability_store on, which only a real pod shows: the images are pushed to a registry in the test (the store names an entry for its image's digest, which an image loaded into kind has none of), and two pods run against the node's capability store (prepared as root with CHOWN and DAC_OVERRIDE over a store root the runner user already owns), checking that the entries are published once and the second pod copies nothing, that both pods see the same content through a read-only mount and a read-only sysroot subPath, that a job can change the tool cache's own tree but not the store, and that the node cache pruner, run from the rendered manifests in a pod of its own, keeps every entry a running pod lists, removes an entry nothing lists and removes the pods' entries once they have finished. +# +# Usage: tests/capabilities/run.sh. Needs Docker, kind, kubectl, helm, python3 with PyYAML, and ansible-playbook (ANSIBLE_PLAYBOOK overrides which); reaches the network for the chart, the runner image, the registry image and the npm package. GRTEST_SCALESET_CHART_VERSION pins the scale-set chart and the controller chart its CRDs come from (default: the latest). Creates a kind cluster named grtest-capabilities and a registry container named grtest-capabilities-registry, and removes both on exit unless GRTEST_KEEP=1. set -euo pipefail repo_root="$(cd "$(dirname "$0")/../.." && pwd)" ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" cluster=grtest-capabilities +registry_container=grtest-capabilities-registry namespace=arc-runners-example tool_image=grtest/toolcache-node:test library_image=grtest/sysroot-libpq:test +# The node cache pruner's own image, built from the autoscaler image's Dockerfile with only the files it copies, and the registry the store test's images are pushed to (Docker Hub's, a version tag: the registry's own releases are not pinned by digest anywhere else here either). +pruner_image=grtest/autoscaler:test +registry_image=docker.io/library/registry:2.8.3 # A node-cache capability's init image, pinned by digest as the role requires. busybox_image=docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e sysroot=/opt/sysroot @@ -32,6 +38,7 @@ fail() { cleanup() { if [ "${GRTEST_KEEP:-0}" != 1 ]; then kind delete cluster --name "$cluster" >/dev/null 2>&1 || true + docker rm -f "$registry_container" >/dev/null 2>&1 || true rm -rf "$work" fi } @@ -46,19 +53,57 @@ docker build -q -t "$tool_image" "$repo_root/tool-images/node" >/dev/null log "Building the reference libpq system-library image" docker build -q -t "$library_image" "$repo_root/tool-images/libpq" >/dev/null -log "Creating kind cluster ${cluster}" +log "Building the node cache pruner's image" +# The autoscaler image's Dockerfile copies several scripts, of which the pruner needs only its own; a context holding that one keeps the build small, and the Dockerfile's other COPY lines are satisfied by empty files that nothing here runs. +pruner_context="$work/pruner-context" +mkdir -p "$pruner_context/autoscaler" "$pruner_context/scripts" +cp "$repo_root/autoscaler/Dockerfile" "$pruner_context/autoscaler/Dockerfile" +for path in autoscaler/loop.sh scripts/autoscaler.sh scripts/autoscaler-ceiling.jq scripts/psi-reporter.sh scripts/node-cache-pruner.sh; do + cp "$repo_root/$path" "$pruner_context/$path" +done +docker build -q -f "$pruner_context/autoscaler/Dockerfile" -t "$pruner_image" "$pruner_context" >/dev/null + +log "Creating kind cluster ${cluster} and its registry" +# A registry on the host's loopback that the nodes reach by its container name over the kind network, as kind's local registry guide has it (https://kind.sigs.k8s.io/docs/user/local-registry/): a pod's image reference names localhost:, and each node's containerd maps that host to the container. +docker rm -f "$registry_container" >/dev/null 2>&1 || true +docker pull -q "$registry_image" >/dev/null +docker run -d --name "$registry_container" -p 127.0.0.1::5000 "$registry_image" >/dev/null kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null -kind load docker-image "$tool_image" "$library_image" --name "$cluster" >/dev/null +docker network connect kind "$registry_container" +# The published port is read after the container joins the kind network: joining it republishes the port, which can change the host port. +registry_port="$(docker port "$registry_container" 5000/tcp | sed -n 's/.*:\([0-9]*\)$/\1/p' | head -n 1)" +[ -n "$registry_port" ] || fail "could not read the registry's published port" +for node in $(kind get nodes --name "$cluster"); do + docker exec "$node" mkdir -p "/etc/containerd/certs.d/localhost:${registry_port}" + printf '[host."http://%s:5000"]\n' "$registry_container" | docker exec -i "$node" tee "/etc/containerd/certs.d/localhost:${registry_port}/hosts.toml" >/dev/null +done +kind load docker-image "$tool_image" "$library_image" "$pruner_image" --name "$cluster" >/dev/null + +# What the store names its entries for: each image's digest in the registry, which the push returns. +push_image() { + local reference="localhost:${registry_port}/$1" digest output + docker tag "$1" "$reference" + output="$(docker push "$reference" 2>&1)" || { echo "$output" >&2; docker ps -a >&2; docker logs "$registry_container" >&2 || true; fail "the push of $1 failed"; } + digest="$(sed -n 's/.*digest: \(sha256:[0-9a-f]\{64\}\).*/\1/p' <<<"$output" | tail -n 1)" + [ -n "$digest" ] || { echo "$output" >&2; fail "the push of $1 returned no digest"; } + echo "${reference%:*}@${digest}" +} +log "Pushing the tool and library images to the registry" +tool_reference="$(push_image "$tool_image")" +library_reference="$(push_image "$library_image")" -log "Rendering the profile's values through the role" -# The profile as arc_profiles expands it, and a catalogue with the tool image (put on PATH through its bin directory), the library image (with PGSYSCONFDIR, where libpq looks for its service file, as its extra variable) and a hook capability that records it ran. -python3 - "$repo_root/plugins/filter/arc.py" "$work/extra.json" "$tool_image" "$node_version" "$work/rendered.json" "$library_image" "$sysroot" "$busybox_image" <<'EOF' +# What a profile's capabilities render to, for a release named $1 (the files are $work/$1.*): $2 is "true" for a profile with capability_store on, $3 the capabilities it names and $4 and $5 the tool and library images they pull from (a store entry is named for an image's digest, so a store profile takes the registry's references). +render_release() { + local name=$1 store=$2 capabilities=$3 tool=$4 library=$5 + log "Rendering the ${name} profile's values through the role" + # The profile as arc_profiles expands it, and a catalogue with the tool image (put on PATH through its bin directory), the library image (with PGSYSCONFDIR, where libpq looks for its service file, as its extra variable) and a hook capability that records it ran. + python3 - "$repo_root/plugins/filter/arc.py" "$work/$name.extra.json" "$tool" "$node_version" "$work/$name.rendered.json" "$library" "$sysroot" "$busybox_image" "$capabilities" "$store" <<'EOF' import importlib.util, json, sys -plugin, extra, image, version, output, library_image, sysroot, busybox_image = sys.argv[1:] +plugin, extra, image, version, output, library_image, sysroot, busybox_image, capabilities, store = sys.argv[1:] spec = importlib.util.spec_from_file_location("arc", plugin) arc = importlib.util.module_from_spec(spec) spec.loader.exec_module(arc) -result = arc.arc_profiles([{"name": "Example", "image": "unused:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["node", "libpq", "marker", "pnpm-store"]}]}], require_app_id=False) +result = arc.arc_profiles([{"name": "Example", "image": "unused:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": capabilities.split(","), "capability_store": store == "true"}]}], require_app_id=False) assert result["errors"] == [], result["errors"] catalogue = { "node": {"image": image, "path": f"node/{version}/bin"}, @@ -69,19 +114,19 @@ catalogue = { with open(extra, "w") as out: json.dump({"profile": result["profiles"][0], "github_runner_arc_capabilities": catalogue, "render_output": output}, out) EOF -ANSIBLE_COLLECTIONS_PATH="$repo_root/playbooks/collections" ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false \ - "$ansible_playbook" "$repo_root/tests/capabilities/render.yml" -e "@$work/extra.json" >/dev/null -python3 -c 'import json, sys, yaml; rendered = json.load(open(sys.argv[1])); yaml.safe_dump(rendered["values"], open(sys.argv[2], "w")); yaml.safe_dump({"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "github-runner-job-started"}, "data": rendered["job_started"]}, open(sys.argv[3], "w"))' \ - "$work/rendered.json" "$work/values.yaml" "$work/configmap.yaml" - -log "Rendering the gha-runner-scale-set chart with those values" -# The same --set values the role passes to every release (tasks/install_scale_set_profile.yml). The chart looks the controller's service account up in a live cluster, which helm template has none of, so it is named here. -helm template example-runners "$charts/gha-runner-scale-set" "${version_flag[@]}" --namespace "$namespace" -f "$work/values.yaml" \ - --set githubConfigUrl=https://github.com/example --set githubConfigSecret=example-github-app \ - --set "template.spec.containers[0].image=$runner_image" \ - --set controllerServiceAccount.name=arc-gha-rs-controller --set controllerServiceAccount.namespace=actions-runner-controller \ - >"$work/chart.yaml" 2>"$work/helm.log" || { cat "$work/helm.log" >&2; fail "helm template failed"; } -python3 - "$work/chart.yaml" "$work/autoscalingrunnerset.yaml" <<'EOF' + ANSIBLE_COLLECTIONS_PATH="$repo_root/playbooks/collections" ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false \ + "$ansible_playbook" "$repo_root/tests/capabilities/render.yml" -e "@$work/$name.extra.json" >/dev/null + python3 -c 'import json, sys, yaml; rendered = json.load(open(sys.argv[1])); yaml.safe_dump(rendered["values"], open(sys.argv[2], "w")); yaml.safe_dump({"apiVersion": "v1", "kind": "ConfigMap", "metadata": {"name": "github-runner-job-started"}, "data": rendered["job_started"]}, open(sys.argv[3], "w"))' \ + "$work/$name.rendered.json" "$work/$name.values.yaml" "$work/$name.configmap.yaml" + + log "Rendering the gha-runner-scale-set chart with the ${name} profile's values" + # The same --set values the role passes to every release (tasks/install_scale_set_profile.yml). The chart looks the controller's service account up in a live cluster, which helm template has none of, so it is named here. + helm template example-runners "$charts/gha-runner-scale-set" "${version_flag[@]}" --namespace "$namespace" -f "$work/$name.values.yaml" \ + --set githubConfigUrl=https://github.com/example --set githubConfigSecret=example-github-app \ + --set "template.spec.containers[0].image=$runner_image" \ + --set controllerServiceAccount.name=arc-gha-rs-controller --set controllerServiceAccount.namespace=actions-runner-controller \ + >"$work/$name.chart.yaml" 2>"$work/helm.log" || { cat "$work/helm.log" >&2; fail "helm template failed"; } + python3 - "$work/$name.chart.yaml" "$work/$name.autoscalingrunnerset.yaml" <<'EOF' import sys, yaml # Helm 4 prints its "Pulled: ... Digest: ..." notice for an OCI chart on standard output too, which parses as a mapping with no kind. documents = [document for document in yaml.safe_load_all(open(sys.argv[1])) if isinstance(document, dict) and "kind" in document] @@ -89,15 +134,21 @@ sets = [document for document in documents if document["kind"] == "AutoscalingRu assert len(sets) == 1, [document["kind"] for document in documents] yaml.safe_dump(sets[0], open(sys.argv[2], "w")) EOF +} + +render_release plain false node,libpq,marker,pnpm-store "$tool_image" "$library_image" +render_release store true node,libpq,marker "$tool_reference" "$library_reference" -log "Checking the AutoscalingRunnerSet against the ARC CRDs" +log "Checking the AutoscalingRunnerSets against the ARC CRDs" helm pull "$charts/gha-runner-scale-set-controller" "${version_flag[@]}" --untar --untardir "$work/controller" >/dev/null 2>&1 kubectl apply --server-side -f "$work/controller/gha-runner-scale-set-controller/crds/" >/dev/null kubectl wait --for condition=established --timeout=60s crd/autoscalingrunnersets.actions.github.com >/dev/null kubectl create namespace "$namespace" >/dev/null -kubectl apply --dry-run=server -f "$work/autoscalingrunnerset.yaml" >/dev/null || fail "the API server rejected the rendered AutoscalingRunnerSet" +for name in plain store; do + kubectl apply --dry-run=server -f "$work/$name.autoscalingrunnerset.yaml" >/dev/null || fail "the API server rejected the rendered ${name} AutoscalingRunnerSet" +done -kubectl -n "$namespace" apply -f "$work/configmap.yaml" >/dev/null +kubectl -n "$namespace" apply -f "$work/plain.configmap.yaml" >/dev/null # What the runner user must find in the node cache once the init container has run: the directory the profile names, writable, and the cache's root writable too, since a job recreates a subdirectory the pruner has set aside directly under it. cat >"$work/cache-step.sh" <<'EOF' set -euo pipefail @@ -149,7 +200,9 @@ spec["restartPolicy"] = "Never" spec["securityContext"] = {"runAsNonRoot": True, "runAsUser": 1001, "runAsGroup": 1001} runner = next(container for container in spec["containers"] if container["name"] == "runner") runner["command"] = ["bash", "-c", open(sys.argv[2]).read()] -yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": sys.argv[3], "labels": template.get("metadata", {}).get("labels", {})}, "spec": spec}, open(sys.argv[4], "w")) +# The annotations too: the capability store's pruner reads the entries a pod uses from the pod's own. +metadata = {"name": sys.argv[3], "labels": template.get("metadata", {}).get("labels", {}), "annotations": template.get("metadata", {}).get("annotations", {})} +yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": metadata, "spec": spec}, open(sys.argv[4], "w")) EOF # The runner image is large, so the first pull dominates; a pod that has not finished within the deadline is reported with its events. wait_for_pod() { @@ -163,13 +216,24 @@ wait_for_pod() { kubectl -n "$namespace" logs "$1" -c runner || fail "the pod $1 failed before the step ran" [ "$phase" = Succeeded ] || fail "the step of pod $1 failed" } +# Waits until a pod that is still running has printed the line $2 on its runner container, for a pod that holds on after its checks so that the pruner can see it running. +wait_for_log() { + local deadline phase + deadline=$(($(date +%s) + 900)) + until kubectl -n "$namespace" logs "$1" -c runner 2>/dev/null | grep -qx "$2"; do + phase="$(kubectl -n "$namespace" get pod "$1" -o jsonpath='{.status.phase}')" + [ "$phase" != Failed ] && [ "$phase" != Succeeded ] || fail "the pod $1 ended before it printed $2" + [ "$(date +%s)" -lt "$deadline" ] || fail "the pod $1 did not print $2" + sleep 5 + done +} # The state a node is really in: the directory already given to the runner user, with no subdirectory in it, which is what the pruner leaves after it empties the cache and what an earlier pod leaves behind. The init container must still create the subdirectory there, as root over a directory it no longer owns. -cache_host_path="$(python3 -c 'import sys, yaml; spec = yaml.safe_load(open(sys.argv[1]))["spec"]["template"]["spec"]; print(next(volume["hostPath"]["path"] for volume in spec["volumes"] if volume["name"] == "node-cache"))' "$work/autoscalingrunnerset.yaml")" +cache_host_path="$(python3 -c 'import sys, yaml; spec = yaml.safe_load(open(sys.argv[1]))["spec"]["template"]["spec"]; print(next(volume["hostPath"]["path"] for volume in spec["volumes"] if volume["name"] == "node-cache"))' "$work/plain.autoscalingrunnerset.yaml")" log "Starting a pod on a node cache already owned by the runner user, with no subdirectory" docker exec "${cluster}-control-plane" sh -c 'mkdir -p "$1" && chown 1001:1001 "$1"' sh "$cache_host_path" { cat "$work/cache-step.sh"; echo "echo CACHE-OK"; } >"$work/seeded-step.sh" -python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/seeded-step.sh" seeded "$work/seeded-pod.yaml" +python3 "$work/make-pod.py" "$work/plain.autoscalingrunnerset.yaml" "$work/seeded-step.sh" seeded "$work/seeded-pod.yaml" kubectl -n "$namespace" apply -f "$work/seeded-pod.yaml" >/dev/null wait_for_pod seeded kubectl -n "$namespace" logs seeded -c runner | grep -qx CACHE-OK || fail "the step of pod seeded did not reach its end" @@ -177,7 +241,7 @@ kubectl -n "$namespace" delete pod seeded --wait >/dev/null docker exec "${cluster}-control-plane" rm -rf "$cache_host_path" log "Starting a pod from the chart's runner pod template" -python3 "$work/make-pod.py" "$work/autoscalingrunnerset.yaml" "$work/step.sh" runner "$work/pod.yaml" +python3 "$work/make-pod.py" "$work/plain.autoscalingrunnerset.yaml" "$work/step.sh" runner "$work/pod.yaml" kubectl -n "$namespace" apply -f "$work/pod.yaml" >/dev/null wait_for_pod runner @@ -188,4 +252,180 @@ init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initC [ "$init_state" = 0 ] || fail "the capability-libpq init container did not succeed (exit code '${init_state}')" init_state="$(kubectl -n "$namespace" get pod runner -o jsonpath='{.status.initContainerStatuses[?(@.name=="capability-pnpm-store")].state.terminated.exitCode}')" [ "$init_state" = 0 ] || fail "the capability-pnpm-store init container did not succeed (exit code '${init_state}')" -log "PASS: the init containers populated the tool cache and the sysroot, the hook ran, put the tool on PATH and the library on its search paths, and @actions/tool-cache found the tool, and the runner user could write to the node cache" + +# The capability store, with what only a real pod shows. The step every store pod runs: the hook puts the tool and the library on their paths, then the checks of what a job may and may not change. The values it needs come first, as assignments; the body is quoted so that nothing in it is expanded here. +store_host_path="$(python3 -c 'import sys, yaml; spec = yaml.safe_load(open(sys.argv[1]))["spec"]["template"]["spec"]; print(next(volume["hostPath"]["path"] for volume in spec["volumes"] if volume["name"] == "capability-store"))' "$work/store.autoscalingrunnerset.yaml")" +store_mount=/opt/capability-store +node="${cluster}-control-plane" +# $1 is "true" for the pod that also asks @actions/tool-cache for the tool, which needs npm and the network. +write_store_step() { + { + printf 'node_version=%s\nsysroot=%s\nstore=%s\nuse_tool_cache=%s\ntool_cache_package=%s\n' "$node_version" "$sysroot" "$store_mount" "$1" "$tool_cache_package" + cat <<'EOF' +set -euo pipefail +export GITHUB_PATH="$(mktemp)" GITHUB_ENV="$(mktemp)" +bash -e "$ACTIONS_RUNNER_HOOK_JOB_STARTED" +while IFS= read -r directory; do PATH="$directory:$PATH"; done <"$GITHUB_PATH" +export PATH +while IFS= read -r line; do export "${line?}"; done <"$GITHUB_ENV" +grep -qx CAPABILITY_HOOK_RAN=yes "$GITHUB_ENV" +[ "$(node --version)" = "v${node_version}" ] +[ "$(command -v psql)" = "${sysroot}/bin/psql" ] +[ "$(pg_config --includedir)" = "${sysroot}/include" ] +[ "${LD_LIBRARY_PATH%%:*}" = "${sysroot}/lib" ] +[ "$PGSYSCONFDIR" = "${sysroot}/etc" ] +echo "the tool and the library resolve from the store" + +# The tool's tree in the pod's own tool cache: real directories holding a link to each store file. +arch_dirs="$(find "$RUNNER_TOOL_CACHE/node/$node_version" -mindepth 1 -maxdepth 1 -type d)" +[ "$(printf '%s\n' "$arch_dirs" | wc -l)" = 1 ] +arch_dir="$arch_dirs" +[ -e "$arch_dir.complete" ] +target="$(readlink "$arch_dir/README.md")" +case "$target" in "$store"/*) ;; *) echo "README.md links to '$target', not into $store"; exit 1 ;; esac +key="${target#"$store"/}" +key="${key%%/*}" +[ -d "$store/$key" ] +[ "$(stat -c %u "$store/$key")" = 1001 ] +listing() { (cd "$store/$key" && find . -printf '%p %s %l\n' | LC_ALL=C sort | sha256sum | cut -d' ' -f1); } +before="$(listing)" +echo "STORE-LISTING-BEFORE $before" + +# What a job may do in the tool's tree: add a file, a directory and a package, and replace a file (a setup action installing beside or over the cached tool, npm install -g). +touch "$arch_dir/job-file" +mkdir "$arch_dir/job-dir" +touch "$arch_dir/job-dir/inner" +mkdir -p "$arch_dir/lib/node_modules/job-package" +rm "$arch_dir/LICENSE" +echo replaced >"$arch_dir/LICENSE" +[ "$(cat "$arch_dir/LICENSE")" = replaced ] + +# What it may not: change a store file in place through its link, or anything in the store or the sysroot. +errors="$(mktemp)" +refused() { + if "$@" 2>"$errors"; then echo "this was allowed, and must not be: $*"; exit 1; fi + grep -q "Read-only file system" "$errors" || { echo "refused for another reason: $*"; cat "$errors"; exit 1; } +} +refused sh -c 'echo changed >>"$1"' sh "$arch_dir/README.md" +refused rm "$target" +refused touch "$store/$key/job-file" +refused mkdir "$store/job-entry" +refused rm -rf "$store/$key" +refused touch "$sysroot/job-file" +refused rm "$sysroot/bin/psql" +after="$(listing)" +echo "STORE-LISTING-AFTER $after" +[ "$before" = "$after" ] + +if [ "$use_tool_cache" = true ]; then + cd "$(mktemp -d)" + npm install --silent --no-audit --no-fund "${tool_cache_package}" >/dev/null + node --input-type=module -e "import * as tc from '@actions/tool-cache'; const found = tc.find('node', '${node_version%%.*}', process.arch); if (!found) { console.error('not found in ' + process.env.RUNNER_TOOL_CACHE); process.exit(1); } console.log('tool-cache find: ' + found);" +fi + +# Hold on until the test has run the pruner against this running pod. +echo STORE-STEP-DONE +until [ -e /tmp/release ]; do sleep 1; done +echo STORE-OK +EOF + } >"$work/store-step-$2.sh" + python3 "$work/make-pod.py" "$work/store.autoscalingrunnerset.yaml" "$work/store-step-$2.sh" "$2" "$work/store-pod-$2.yaml" +} + +# The state of the store's root, its incoming directory and every entry's directory: name, inode and modification time. A pod that copies or publishes anything changes it; a pod that finds every entry already there leaves it as it was. +store_state() { + docker exec "$node" sh -c 'stat -c "%n %i %y" "$1" "$1/.incoming" "$1"/*' sh "$store_host_path" +} +store_entries() { + docker exec "$node" ls -A "$store_host_path" | grep -vxE '\.incoming|\.trash' | LC_ALL=C sort || true +} + +log "Starting a pod on a capability store already owned by the runner user, with no incoming directory" +# The state a node is really in after an earlier pod or the pruner: the store's root given to the runner user, with no incoming directory in it. The preparation must still create it, as root over a directory it no longer owns, which needs DAC_OVERRIDE. +docker exec "$node" sh -c 'mkdir -p "$1" && chown 1001:1001 "$1"' sh "$store_host_path" +write_store_step true store-a +kubectl -n "$namespace" apply -f "$work/store-pod-store-a.yaml" >/dev/null +wait_for_log store-a STORE-STEP-DONE + +expected_entries="$(kubectl -n "$namespace" get pod store-a -o jsonpath='{.metadata.annotations.github-runner\.exadev\.io/capability-store-keys}' | tr ' ' '\n' | LC_ALL=C sort)" +[ "$(printf '%s\n' "$expected_entries" | wc -l)" = 3 ] || fail "the pod's annotation does not list the tool, the library and the merged sysroot: '$expected_entries'" +[ "$(store_entries)" = "$expected_entries" ] || fail "the store holds '$(store_entries)', not exactly the entries the pod's annotation lists: '$expected_entries'" +[ -z "$(docker exec "$node" ls -A "$store_host_path/.incoming")" ] || fail "a copy was left in the incoming directory after the pod published it" +[ "$(docker exec "$node" stat -c %u "$store_host_path/.incoming")" = 1001 ] || fail "the incoming directory is not the runner user's" + +log "Starting a second pod against the filled store" +state_before="$(store_state)" +write_store_step false store-b +kubectl -n "$namespace" apply -f "$work/store-pod-store-b.yaml" >/dev/null +wait_for_log store-b STORE-STEP-DONE +[ "$(store_state)" = "$state_before" ] || fail "the second pod changed the store, which it should only have read: $(diff <(echo "$state_before") <(store_state))" +listings="$(for pod in store-a store-b; do kubectl -n "$namespace" logs "$pod" -c runner | sed -n 's/^STORE-LISTING-[A-Z]* //p'; done | sort -u)" +[ "$(printf '%s\n' "$listings" | wc -l)" = 1 ] || fail "the two pods saw different content in the entry: $listings" +[ "$(kubectl -n "$namespace" get pod store-b -o jsonpath='{.metadata.annotations.github-runner\.exadev\.io/capability-store-keys}' | tr ' ' '\n' | LC_ALL=C sort)" = "$expected_entries" ] || fail "the two pods list different store entries" +kubectl -n "$namespace" logs store-a -c runner | grep -q '^tool-cache find: ' || fail "@actions/tool-cache did not find the tool in the pod's tool cache" + +log "Rendering the node cache pruner's manifests from the role's template" +python3 - "$work/plain.extra.json" "$work/pruner-extra.json" "$work/pruner.yaml" <<'EOF' +import json, sys +# The fleet is this test's one profile, with the capability store on, and its namespace is where the store's references are read from. +profile = json.load(open(sys.argv[1]))["profile"] +profile = {"namespace": profile["namespace"], "release": profile["release"], "target": f"{profile['namespace']}/{profile['release']}"} +json.dump({"github_runner_arc_fleet": {"profiles": [profile]}, "github_runner_arc_node_cache_profiles": [profile], "github_runner_arc_capability_store_namespaces": [profile["namespace"]], "github_runner_arc_node_label_key": "example/ci", "github_runner_arc_node_label_value": "true", "pruner_output": sys.argv[3]}, open(sys.argv[2], "w")) +EOF +ANSIBLE_COLLECTIONS_PATH="$repo_root/playbooks/collections" ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false \ + "$ansible_playbook" "$repo_root/tests/node_cache_pruner/render.yml" -e "@$work/pruner-extra.json" >/dev/null +# The rendered Roles, bindings and service account are applied as they are. The DaemonSet becomes a pod that checks once and ends (PRUNE_ONCE), on the image built here, without the placement that picks the fleet's nodes (the kind node carries no such label). +python3 - "$work/pruner.yaml" "$pruner_image" "$work/pruner-rbac.yaml" "$work/pruner-pod.yaml" <<'EOF' +import sys, yaml +documents = [document for document in yaml.safe_load_all(open(sys.argv[1])) if document] +daemon_set = next(document for document in documents if document["kind"] == "DaemonSet") +yaml.safe_dump_all([document for document in documents if document["kind"] != "DaemonSet"], open(sys.argv[3], "w")) +spec = daemon_set["spec"]["template"]["spec"] +for field in ("nodeSelector", "tolerations", "priorityClassName", "imagePullSecrets"): + spec.pop(field, None) +container = spec["containers"][0] +container["image"] = sys.argv[2] +container["imagePullPolicy"] = "IfNotPresent" +container["env"].append({"name": "PRUNE_ONCE", "value": "true"}) +spec["restartPolicy"] = "Never" +yaml.safe_dump({"apiVersion": "v1", "kind": "Pod", "metadata": {"name": "pruner", "namespace": daemon_set["metadata"]["namespace"], "labels": daemon_set["spec"]["template"]["metadata"]["labels"]}, "spec": spec}, open(sys.argv[4], "w")) +EOF +platform_namespace="$(python3 -c 'import sys, yaml; print(next(document["metadata"]["namespace"] for document in yaml.safe_load_all(open(sys.argv[1])) if document and document["kind"] == "DaemonSet"))' "$work/pruner.yaml")" +kubectl create namespace "$platform_namespace" >/dev/null +kubectl apply -f "$work/pruner-rbac.yaml" >/dev/null +# One round of the pruner, from the rendered pod; prints its log and leaves it in $work/pruner.log. +prune_once() { + local deadline phase + kubectl -n "$platform_namespace" delete pod pruner --ignore-not-found --wait >/dev/null + kubectl apply -f "$work/pruner-pod.yaml" >/dev/null + deadline=$(($(date +%s) + 300)) + until phase="$(kubectl -n "$platform_namespace" get pod pruner -o jsonpath='{.status.phase}')" && [ "$phase" = Succeeded ] || [ "$phase" = Failed ]; do + [ "$(date +%s)" -lt "$deadline" ] || fail "the pruner did not finish" + sleep 3 + done + kubectl -n "$platform_namespace" logs pruner | tee "$work/pruner.log" + [ "$phase" = Succeeded ] || fail "the pruner failed" +} + +log "Pruning the store while both pods run, with an entry and a copy that nothing lists beside them" +orphan=$(printf 'f%.0s' $(seq 64)) +docker exec "$node" sh -c 'mkdir -p "$1/$2" "$1/.incoming/$2.00000000-0000-0000-0000-000000000000.capability-node" && touch "$1/$2/file"' sh "$store_host_path" "$orphan" +prune_once +grep -q "removed ${orphan}," "$work/pruner.log" || fail "the pruner did not remove the entry nothing lists" +[ "$(store_entries)" = "$expected_entries" ] || fail "the pruner left '$(store_entries)', not the entries the running pods list: '$expected_entries'" +[ -z "$(docker exec "$node" ls -A "$store_host_path/.incoming")" ] || fail "the pruner left a copy no pod owns" +if grep -E "removed ($(printf '%s\n' "$expected_entries" | paste -sd'|' -))," "$work/pruner.log"; then fail "the pruner removed an entry a running pod lists"; fi +kubectl -n "$namespace" get pod store-a store-b -o jsonpath='{range .items[*]}{.metadata.name} {.status.phase}{"\n"}{end}' | grep -vq Running && fail "a pod stopped running during the pruner's round" + +log "Letting both pods finish, then pruning again" +for pod in store-a store-b; do + kubectl -n "$namespace" exec "$pod" -c runner -- touch /tmp/release + wait_for_pod "$pod" + kubectl -n "$namespace" logs "$pod" -c runner | grep -qx STORE-OK || fail "the step of pod $pod did not reach its end" +done +prune_once +[ -z "$(store_entries)" ] || fail "the pruner left '$(store_entries)' after every pod that listed it had finished" + +init_state="$(kubectl -n "$namespace" get pod store-a -o jsonpath='{.status.initContainerStatuses[*].state.terminated.exitCode}')" +[ "$(printf '%s\n' "$init_state" | tr ' ' '\n' | sort -u)" = 0 ] || fail "an init container of the store pod did not succeed (exit codes '${init_state}')" +log "PASS: the init containers populated the tool cache and the sysroot, the hook ran, put the tool on PATH and the library on its search paths, and @actions/tool-cache found the tool, and the runner user could write to the node cache; with the store on, the entries were published once and read by a second pod that copied nothing, a job changed the tool cache's tree but nothing in the store or the sysroot, and the pruner kept the entries running pods list and removed the rest" From 688ded769a9cd5faf52d418268fdbad07f03e147 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:12:47 +0100 Subject: [PATCH 10/43] docs(arc): state the store's pruner, grants and dind limit as the code has them The pruner script cited a README section that does not exist; it is the capability store section. The README said the pruner only lists pods through per-namespace Roles, but in the namespaces the store is read from it also lists autoscalingrunnersets, and it said the pruner is installed only while some profile has a node cache, when a profile using the store installs it too. The claim that no job changes what another pod uses held only for a profile whose steps all run on the runner container. The chart's dind sidecar runs privileged, so a job in a dind profile can write to the node's store as root and the shared store carries it to later pods; the README now says so and repeats the one-trust-level rule. A capability image must supply rm as well as sh and cp when its profile uses the store. Every reference image ends in busybox, which has all three, and the Dockerfile comments and the README now name rm. --- roles/github_runner_arc/README.md | 14 ++++++++------ .../tasks/decide_node_cache_pruner.yml | 2 +- scripts/node-cache-pruner.sh | 2 +- tool-images/awscli/Dockerfile | 2 +- tool-images/glpk/Dockerfile | 2 +- tool-images/highs/Dockerfile | 2 +- tool-images/libmariadb/Dockerfile | 2 +- tool-images/libpq/Dockerfile | 2 +- tool-images/node/Dockerfile | 2 +- tool-images/openmpi/Dockerfile | 2 +- tool-images/terraform/Dockerfile | 2 +- tool-images/tflint/Dockerfile | 2 +- 12 files changed, 19 insertions(+), 17 deletions(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 5f9acdb..2e31e1a 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -128,11 +128,11 @@ A node-cache capability, `{node_cache: {init_image, dirs}, env?}`, gives the pro The cache outlives the pod, so the next job on the node starts with it filled, and a job on a node whose cache is empty fills it as it goes. It is the workflow's own cache step that remains the fallback: a workflow checks that the variable's path exists before relying on it, so it still runs on a runner without the capability, such as a GitHub-hosted one. -A pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image, on the nodes carrying `github_runner_arc_node_label_key`, which must therefore be set) keeps each cache under its limit. A pod counts as using a cache when it is in the profile's namespace on that node, has not finished, and has a `hostPath` volume at exactly the profile's directory; other profiles' pods, finished pods and pods without the volume are ignored. +A pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image, on the nodes carrying `github_runner_arc_node_label_key`, which must therefore be set) keeps each cache under its limit, and removes the capability store's entries that nothing uses (see [The capability store](#the-capability-store)). A pod counts as using a cache when it is in the profile's namespace on that node, has not finished, and has a `hostPath` volume at exactly the profile's directory; other profiles' pods, finished pods and pods without the volume are ignored. - **Over `github_runner_arc_node_cache_size_limit_gib`, no pod using the cache on the node:** it moves the cache's entries, one rename each, into a unique trash directory beside the profiles (same filesystem, outside every pod's mount), listing the pods again before every rename, and deletes the trash only when the moving is done. A store is therefore seen whole or not at all by a pod that starts at any point; a pod that starts part way finds the stores not yet moved whole and the rest empty, and a pod appearing ends the moving and leaves what has not moved in place. It never trims a store, because a running job may be linked into any part of one. A deletion that fails is reported and retried at the start of the next round. What remains is the instant between a list and the rename after it, shorter than a runner pod needs to start a job; a pod that does start in it meets an empty cache it can refill, because the cache's root is owned by the runner's user. - **Over the hard limit (`github_runner_arc_node_cache_hard_limit_factor` times the size limit), a pod using the cache:** it cannot be emptied, and the pruner changes nothing about the node to force it. It logs a line starting `NODE_CACHE_OVER_HARD_LIMIT` on every round, with the profile, the size and the node, for whatever collects the pruner's logs to alert on. Draining the node, or lowering its runner count, lets the next round empty the cache, and kubelet's disk-pressure eviction still protects the node itself. An automatic cordon was considered and left out: a controller that cordons nodes needs a floor of schedulable nodes, a record of which cordons are its own, and a grant to patch nodes, and a transient failure part way through a drain would lift it. -- **A profile directory the fleet no longer wants** (the capability was removed from a profile, or the profile has left the fleet) is emptied and removed the same way, while the pruner is installed. The role keeps a read-only grant to list pods (a Role and RoleBinding, named for the release, in the profile's namespace) for every profile it has installed while the pruner is installed, including one that has since left the fleet, so such a directory can always be checked (a grant only lists pods in its own profile's namespace, so one that stays is harmless); the grants are found by the role's own labels (`app.kubernetes.io/name`, `part-of` and `managed-by`, all three, so another software's objects are never touched) and removed only with the pruner. The pruner follows the fleet: it is installed while some profile has a node cache and removed when none does, so a directory left when the last cache goes is for the operator to delete from `` on each node. A list the server refuses all the same, which only happens when the namespace was deleted together with its Role, is reported on every round with a line starting `NODE_CACHE_UNPRUNABLE` and the directory is left for the operator (with the namespace gone there are no pods, so deleting it is safe). A list that fails for any other reason (the API server unreachable, an answer that is not a pod list) is reported with `NODE_CACHE_LIST_FAILED` and retried on the next round; both lines carry the cache's size, and a cache past the hard limit whose pods could not be listed is also reported as over it. +- **A profile directory the fleet no longer wants** (the capability was removed from a profile, or the profile has left the fleet) is emptied and removed the same way, while the pruner is installed. The role keeps a read-only grant to list pods (a Role and RoleBinding, named for the release, in the profile's namespace) for every profile it has installed while the pruner is installed, including one that has since left the fleet, so such a directory can always be checked (a grant only lists pods in its own profile's namespace, so one that stays is harmless); the grants are found by the role's own labels (`app.kubernetes.io/name`, `part-of` and `managed-by`, all three, so another software's objects are never touched) and removed only with the pruner. The pruner follows the fleet: it is installed while the size limit is above zero and some profile has a node cache or uses the capability store, and removed when none does, so a directory left when the pruner goes (a cache directory, or the store's entries) is for the operator to delete from `` and `` on each node. A list the server refuses all the same, which only happens when the namespace was deleted together with its Role, is reported on every round with a line starting `NODE_CACHE_UNPRUNABLE` and the directory is left for the operator (with the namespace gone there are no pods, so deleting it is safe). A list that fails for any other reason (the API server unreachable, an answer that is not a pod list) is reported with `NODE_CACHE_LIST_FAILED` and retried on the next round; both lines carry the cache's size, and a cache past the hard limit whose pods could not be listed is also reported as over it. - **The credential must be able to read cluster roles.** Release 1.31.0 installed the pruner with a cluster-wide `ClusterRole` and `ClusterRoleBinding` named `node-cache-pruner`; every run now looks for them by name and removes them, binding first, only if they are what 1.31.0 created (a role that lists pods and nothing else, not aggregated, and a binding of it to the pruner's account in the platform namespace and nothing else; 1.31.0 set no labels, so the content is the identity). The lookup needs the credential the role reaches the cluster with to be allowed to `get` ClusterRoles and ClusterRoleBindings (cluster-scoped, read-only), also on a fleet that never ran 1.31.0, and `delete` on them only when it finds the 1.31.0 objects. A role of that name whose binding is not the one 1.31.0 created (the platform namespace changed, another subject was added) is left in place with the binding and reported once per run with a line starting `NODE_CACHE_LEGACY_ACCESS_LEFT`, so a dangling binding can never regain cluster-wide pod listing; delete them by hand if they are not wanted. The cleanup goes in a later release (ExaDev/github-runner#92). A flag that skipped the lookup on a fleet that never ran 1.31.0 would be one more setting to maintain, so there is none. - **Upgrading from 1.31.0:** the host path root is now validated whether or not any profile has a node cache, so a configuration whose `github_runner_arc_node_cache_host_path` does not end in `node-cache` (the default does) fails validation even if it uses no node-cache capability; set the path to a conforming one (a new path starts an empty cache, and the old directory is yours to delete). - **A cluster that ran release 1.31.0** has a cluster-wide grant to list pods named `node-cache-pruner`; the role removes the 1.31.0 objects where it finds them (binding, then role), subject to the paragraph above: only what matches what 1.31.0 created, and a role whose binding is not the one 1.31.0 created is left with it and reported. @@ -140,7 +140,7 @@ A pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image, o - **A bad setting stops the pruner before it does anything.** The pruner's start-up checks end it with status 2 and a message naming the variable, and the DaemonSet's pods then crash-loop with that message in their log. The role refuses the numeric settings it templates by the same pattern and maxima (the size limit, the hard limit factor and the check interval: positive integers without sign, leading zero or fraction, up to 1073741824 GiB, 1024 and 86400), so only a value changed some other way, such as by editing the DaemonSet, reaches that check. - **A failure never stops the others:** a size that cannot be read skips that profile for the round, and a pod list that fails or is not a pod list means no pruning (`NODE_CACHE_UNPRUNABLE` when it is refused as Forbidden, `NODE_CACHE_LIST_FAILED` otherwise), since an unknown set of pods may be using the cache. -The limits are per profile and the node's disk is shared by all of them, so a node's worst case is the sum of the limits of the profiles with a cache on it, times the hard limit factor. Size the limit so that sum fits the node's free disk. The pruner can list pods through a Role and RoleBinding per profile namespace (named for the release, not a cluster-wide grant), has no access to nodes, and runs as root with only `DAC_OVERRIDE` and `FOWNER` (to delete files and sticky-bit directories the runner's user made), a read-only root filesystem and no privilege escalation. +The limits are per profile and the node's disk is shared by all of them, so a node's worst case is the sum of the limits of the profiles with a cache on it, times the hard limit factor. Size the limit so that sum fits the node's free disk. The pruner can list pods through a Role and RoleBinding per profile namespace (named for the release, not a cluster-wide grant) and, in each namespace the capability store's references are read from, list pods and `autoscalingrunnersets` through a second Role and RoleBinding (`node-cache-pruner:capability-store`); it has no access to nodes, and runs as root with only `DAC_OVERRIDE` and `FOWNER` (to delete files and sticky-bit directories the runner's user made), a read-only root filesystem and no privilege escalation. A read-only mount for sensitive profiles was considered and is not offered: a profile's cache is its own and only its own jobs fill it, so a read-only copy of it would never be filled. A profile that must not share a cache with other jobs of its kind should not have the capability. @@ -226,7 +226,7 @@ A tool image is any image that has: `` and `` are what the consuming setup action looks up: `node` and the plain version (`24.21.0`, no leading `v`) for `actions/setup-node`, `tflint` and the version without its `v` for `setup-tflint`. `` is named the way that action names the architecture, and actions differ: `actions/setup-node` uses Node's `os.arch()` (`x64`, `arm64`), while `setup-tflint` maps `x64` to `amd64` (`amd64`, `arm64`). For a tool used only through `path` the name does not matter, since `path` finds whichever architecture is there; the reference images use `os.arch()` names, the tool cache's own default. A multi-architecture image holds, for each platform, only that platform's `` directory, so the tool cache in any one pod has one architecture per tool, which is what `path` relies on. `path` adds `//[/]` for the one `` with a `.complete` marker, and the job fails at its start if there is none or more than one. -`tool-images/` holds Dockerfiles for reference images following this contract: Node.js, Terraform, TFLint and the AWS CLI. Each downloads the official release for the target architecture in a first stage, verifies it (against the release's checksums, and for Terraform and the AWS CLI against the vendor's signing key, pinned by fingerprint), lays it out under `/toolcache`, and copies that into `busybox`, which supplies `sh` and `cp`. To build one yourself, for both architectures: +`tool-images/` holds Dockerfiles for reference images following this contract: Node.js, Terraform, TFLint and the AWS CLI. Each downloads the official release for the target architecture in a first stage, verifies it (against the release's checksums, and for Terraform and the AWS CLI against the vendor's signing key, pinned by fingerprint), lays it out under `/toolcache`, and copies that into `busybox`, which supplies `sh`, `cp` and `rm`. To build one yourself, for both architectures: ```sh docker buildx build --platform linux/amd64,linux/arm64 --build-arg NODE_VERSION=24.21.0 \ @@ -317,13 +317,15 @@ An entry is named for its image's digest, so every image a profile's capabilitie - `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. -What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and it cannot change anything in the store, so no job changes what another pod uses. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a job cannot write into them, where the per-pod copies were writable. +What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. + +**A profile in Docker-in-Docker mode (`container_mode: dind`, or values that write the chart's dind sidecar out) does not have that limit.** The chart's dind sidecar runs privileged (`securityContext.privileged: true` in the `gha-runner-scale-set` chart's dind container), so a job that reaches its Docker daemon can start a container that mounts the node's filesystem and write to the store as root, replacing an entry. Entries are named for an image's digest and never revalidated, so every later pod on that node that uses the entry, in any profile with the store on, then runs the replaced bytes, until the pruner removes an entry no scale set or pod lists. The shared store is therefore a persistence channel from a dind job to other pods on its node, as the node cache is between the jobs of its profile. Give `capability_store: true` only to profiles whose workflows are all of one trust level, and never to one that runs pull requests from forks. `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume. Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's own Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; it stays with the other grants for as long as the pruner is installed, which is what keeps a departed profile's namespace among the namespaces read. -`tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the tool images or the test. +`tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. It then pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the pruner, the tool images or the test. ## Registry cache diff --git a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml index f41be1d..d4398a6 100644 --- a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml +++ b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml @@ -1,5 +1,5 @@ --- -# Decides which profiles have a node cache and whether the pruner runs, from the fleet and the capability catalogue. The pruner runs while the size limit is above zero and some profile has a node cache, and is removed otherwise, so it follows the fleet. A directory a removed capability leaves is emptied only while another profile keeps the pruner installed; once the pruner is gone, delete on each node by hand. +# Decides which profiles have a node cache and whether the pruner runs, from the fleet and the capability catalogue. The pruner runs while the size limit is above zero and some profile has a node cache or uses the capability store, and is removed otherwise, so it follows the fleet. A directory a removed capability leaves is emptied only while another profile keeps the pruner installed; once the pruner is gone, delete on each node by hand. - name: "Node cache: find the profiles that carry a node-cache capability" ansible.builtin.set_fact: diff --git a/scripts/node-cache-pruner.sh b/scripts/node-cache-pruner.sh index 8c8f65d..de9cb45 100755 --- a/scripts/node-cache-pruner.sh +++ b/scripts/node-cache-pruner.sh @@ -11,7 +11,7 @@ # # Every step of one profile's check may fail without stopping the others: a failed or partial size reading skips that profile for the round, and a failed pod list means no pruning, since an unknown set of pods may be using the cache: NODE_CACHE_UNPRUNABLE when the server refused the list as Forbidden (there is no Role for the pruner in that namespace), NODE_CACHE_LIST_FAILED for any other failure (the API server was unreachable, the answer was not a pod list), which the next round retries. # -# The capability store (the role README's "Where the copies live"), when STORE_ROOT is set, holds one entry per image payload, named for the image's digest, plus merged sysroots, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it (STORE_NAMESPACES holds every namespace a pod may still run in from an earlier template, not only those of the profiles using the store now, so an entry is not removed under a pod whose profile has switched the store off or left the fleet) in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. +# The capability store (the role README's "The capability store"), when STORE_ROOT is set, holds one entry per image payload, named for the image's digest, plus merged sysroots, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it (STORE_NAMESPACES holds every namespace a pod may still run in from an earlier template, not only those of the profiles using the store now, so an entry is not removed under a pod whose profile has switched the store off or left the fleet) in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. # # Which pods count: those in the profile's namespace on this node, not finished (Succeeded or Failed), that have a hostPath volume at exactly the profile's directory on the host (HOST_CACHE_ROOT//). Another profile's pods, finished pods and pods that do not mount the cache are ignored. # diff --git a/tool-images/awscli/Dockerfile b/tool-images/awscli/Dockerfile index 20c4e70..980fda1 100644 --- a/tool-images/awscli/Dockerfile +++ b/tool-images/awscli/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for the AWS CLI (version 2), following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/aws-cli/// holds the CLI's self-contained distribution (the aws executable and the files it loads beside it), with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. No setup action looks the AWS CLI up in the tool cache, so this capability is meant to be declared with path: aws-cli/, which puts aws on PATH at job start. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# Tool-cache capability image for the AWS CLI (version 2), following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/aws-cli/// holds the CLI's self-contained distribution (the aws executable and the files it loads beside it), with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. No setup action looks the AWS CLI up in the tool cache, so this capability is meant to be declared with path: aws-cli/, which puts aws on PATH at job start. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg AWSCLI_VERSION= -t /: tool-images/awscli FROM alpine:3.22 AS fetch diff --git a/tool-images/glpk/Dockerfile b/tool-images/glpk/Dockerfile index 5df28f2..5ff52e8 100644 --- a/tool-images/glpk/Dockerfile +++ b/tool-images/glpk/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for GLPK, the GNU Linear Programming Kit, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libglpk.so), its header (include/glpk.h) and the glpsol command-line solver (bin/glpsol), built on the runner image's own distribution release with GLPK's own arithmetic in place of GMP, so it needs nothing beyond what the runner image already has. GLPK ships no pkg-config file, as the distribution's own development package does not either, so a build finds it through the search paths the job-started hook sets (CPATH and LIBRARY_PATH: -lglpk alone compiles and links), exactly as it would find the distribution's copy in the system directories. GLPK reads no path at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# System-library capability image for GLPK, the GNU Linear Programming Kit, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libglpk.so), its header (include/glpk.h) and the glpsol command-line solver (bin/glpsol), built on the runner image's own distribution release with GLPK's own arithmetic in place of GMP, so it needs nothing beyond what the runner image already has. GLPK ships no pkg-config file, as the distribution's own development package does not either, so a build finds it through the search paths the job-started hook sets (CPATH and LIBRARY_PATH: -lglpk alone compiles and links), exactly as it would find the distribution's copy in the system directories. GLPK reads no path at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg GLPK_VERSION= --build-arg GLPK_SHA256= -t /: tool-images/glpk FROM ubuntu:24.04 AS build diff --git a/tool-images/highs/Dockerfile b/tool-images/highs/Dockerfile index 87e942c..c6d7544 100644 --- a/tool-images/highs/Dockerfile +++ b/tool-images/highs/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for HiGHS, the linear and mixed-integer programming solver, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libhighs.so), its C and C++ headers (include/highs/), a pkg-config file (lib/pkgconfig/highs.pc), a CMake package (lib/cmake/highs/) and the highs command-line solver (bin/highs), built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. HiGHS records no install path that it reads at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# System-library capability image for HiGHS, the linear and mixed-integer programming solver, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libhighs.so), its C and C++ headers (include/highs/), a pkg-config file (lib/pkgconfig/highs.pc), a CMake package (lib/cmake/highs/) and the highs command-line solver (bin/highs), built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. HiGHS records no install path that it reads at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg HIGHS_VERSION= --build-arg HIGHS_SHA256= -t /: tool-images/highs FROM ubuntu:24.04 AS build diff --git a/tool-images/libmariadb/Dockerfile b/tool-images/libmariadb/Dockerfile index 1a043cd..04683a2 100644 --- a/tool-images/libmariadb/Dockerfile +++ b/tool-images/libmariadb/Dockerfile @@ -1,6 +1,6 @@ # System-library capability image for MariaDB Connector/C, the LGPL client library for MySQL and MariaDB servers, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libmariadb.so, with the libmysqlclient.so names linked to it), its headers (include/mariadb/, with the include/mariadb/mysql/ subdirectory MySQL's client headers sit in), a pkg-config file (lib/pkgconfig/libmariadb.pc) and mariadb_config with its mysql_config name, built on the runner image's own distribution release against its OpenSSL and zlib, so it needs nothing beyond what the runner image already has. It is what the distribution's MySQL client development package is for a build: the mysqlclient Python package, for one, finds it through pkg-config as libmariadb, and a build that asks mysql_config or links -lmysqlclient finds it under those names. # -# The library loads its authentication plugins from a directory compiled in as an absolute path, which a relocated payload cannot honour, so every plugin a client on Linux uses is built into the library instead (caching_sha2_password and sha256_password, which MySQL 8's default accounts need, and MariaDB's ed25519, PARSEC, dialog and clear-text ones), leaving no plugin to load and no variable to set; GSSAPI, which would need the Kerberos libraries, and the remote-file plugin, which would need curl, are left out. mariadb_config finds its installation from its own location, so it reports the relocated prefix. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# The library loads its authentication plugins from a directory compiled in as an absolute path, which a relocated payload cannot honour, so every plugin a client on Linux uses is built into the library instead (caching_sha2_password and sha256_password, which MySQL 8's default accounts need, and MariaDB's ed25519, PARSEC, dialog and clear-text ones), leaving no plugin to load and no variable to set; GSSAPI, which would need the Kerberos libraries, and the remote-file plugin, which would need curl, are left out. mariadb_config finds its installation from its own location, so it reports the relocated prefix. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg MARIADB_CONNECTOR_C_VERSION= --build-arg MARIADB_CONNECTOR_C_SHA256= -t /: tool-images/libmariadb FROM ubuntu:24.04 AS build diff --git a/tool-images/libpq/Dockerfile b/tool-images/libpq/Dockerfile index d042c5a..d87f04a 100644 --- a/tool-images/libpq/Dockerfile +++ b/tool-images/libpq/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for libpq, PostgreSQL's client library, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libpq.so), its headers (include/libpq-fe.h and the headers it includes), a pkg-config file (lib/pkgconfig/libpq.pc), and the psql and pg_config programs (bin/), built on the runner image's own distribution release against its OpenSSL, so it needs nothing beyond what the runner image already has. PostgreSQL's programs find their installation relative to their own location, so it needs no extra environment. Only the client side is built: no server, no readline, no ICU. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# System-library capability image for libpq, PostgreSQL's client library, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libpq.so), its headers (include/libpq-fe.h and the headers it includes), a pkg-config file (lib/pkgconfig/libpq.pc), and the psql and pg_config programs (bin/), built on the runner image's own distribution release against its OpenSSL, so it needs nothing beyond what the runner image already has. PostgreSQL's programs find their installation relative to their own location, so it needs no extra environment. Only the client side is built: no server, no readline, no ICU. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg POSTGRESQL_VERSION= -t /: tool-images/libpq FROM ubuntu:24.04 AS build diff --git a/tool-images/node/Dockerfile b/tool-images/node/Dockerfile index fc35452..347a93d 100644 --- a/tool-images/node/Dockerfile +++ b/tool-images/node/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for Node.js, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/node/// holds the official Linux build, with .complete beside it, and is named the way actions/setup-node looks it up (Node's os.arch(): x64 or arm64). A job whose setup-node node-version matches the version finds it there and skips the download. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# Tool-cache capability image for Node.js, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/node/// holds the official Linux build, with .complete beside it, and is named the way actions/setup-node looks it up (Node's os.arch(): x64 or arm64). A job whose setup-node node-version matches the version finds it there and skips the download. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg NODE_VERSION= -t /: tool-images/node FROM alpine:3.22 AS fetch diff --git a/tool-images/openmpi/Dockerfile b/tool-images/openmpi/Dockerfile index 8e502cc..d8491e3 100644 --- a/tool-images/openmpi/Dockerfile +++ b/tool-images/openmpi/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for Open MPI, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix (bin/, lib/, include/, lib/pkgconfig/) built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. Open MPI records its install prefix in its binaries, so the capability is declared with env OPAL_PREFIX set to the prefix it is extracted into; with that, mpicc, mpirun and the libraries find their help files, plugins and wrapper data there. The bundled hwloc, libevent, PMIx and PRRTE are built in, so nothing else needs installing; PMIx is built with zlib, which the runner image has, so it can compress large messages. Fortran bindings are left out, since the runner image has no Fortran runtime. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# System-library capability image for Open MPI, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix (bin/, lib/, include/, lib/pkgconfig/) built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. Open MPI records its install prefix in its binaries, so the capability is declared with env OPAL_PREFIX set to the prefix it is extracted into; with that, mpicc, mpirun and the libraries find their help files, plugins and wrapper data there. The bundled hwloc, libevent, PMIx and PRRTE are built in, so nothing else needs installing; PMIx is built with zlib, which the runner image has, so it can compress large messages. Fortran bindings are left out, since the runner image has no Fortran runtime. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg OPENMPI_VERSION= --build-arg OPENMPI_SHA256= -t /: tool-images/openmpi FROM ubuntu:24.04 AS build diff --git a/tool-images/terraform/Dockerfile b/tool-images/terraform/Dockerfile index 1dc84b1..3ff88e4 100644 --- a/tool-images/terraform/Dockerfile +++ b/tool-images/terraform/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for Terraform, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/terraform///terraform is the release binary, with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. hashicorp/setup-terraform always downloads and never consults the tool cache, so this capability is meant to be declared with path: terraform/, which puts the binary on PATH at job start. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# Tool-cache capability image for Terraform, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/terraform///terraform is the release binary, with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. hashicorp/setup-terraform always downloads and never consults the tool cache, so this capability is meant to be declared with path: terraform/, which puts the binary on PATH at job start. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg TERRAFORM_VERSION= -t /: tool-images/terraform FROM alpine:3.22 AS fetch diff --git a/tool-images/tflint/Dockerfile b/tool-images/tflint/Dockerfile index 206e9ac..e364378 100644 --- a/tool-images/tflint/Dockerfile +++ b/tool-images/tflint/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for TFLint, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/tflint///tflint is the release binary, with .complete beside it, and is named the way terraform-linters/setup-tflint looks it up (amd64 or arm64, its own mapping of Node's os.arch()). A job whose setup-tflint tflint_version is this version (with or without the leading v) finds it there and skips the download. The final stage is busybox, which supplies the sh and cp the role's init container runs. +# Tool-cache capability image for TFLint, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/tflint///tflint is the release binary, with .complete beside it, and is named the way terraform-linters/setup-tflint looks it up (amd64 or arm64, its own mapping of Node's os.arch()). A job whose setup-tflint tflint_version is this version (with or without the leading v) finds it there and skips the download. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg TFLINT_VERSION= -t /: tool-images/tflint FROM alpine:3.22 AS fetch From 5837c493c949dfee224164c95c8fcbf0b8aef2eb Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:24:51 +0100 Subject: [PATCH 11/43] fix(arc): remove a pod's capability store copy even when it holds read-only directories The copy, the sysroot merge and the publish loser path remove a pod's own incoming copy with rm -rf as the non-root runner user. A payload can hold directories without write permission (a Go module cache's are mode 0555), which rm -rf cannot empty, so a rerun after a killed copy failed in the init container on every kubelet retry and the pod never started, and a publish that lost the race left its copy in .incoming until the pruner cleared it. Each of the three now makes the copy owner-writable with chmod -R u+rwx first. chmod -R does not follow symbolic links on busybox or GNU, so it changes nothing outside the copy. --- plugins/filter/arc.py | 11 ++++-- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 61 +++++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index c758eaf..f2b218b 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1144,10 +1144,13 @@ def _runner_security_context(runner_user: int, runner_group: int) -> dict[str, A return {"runAsUser": int(runner_user), "runAsGroup": int(runner_group), "runAsNonRoot": True, "allowPrivilegeEscalation": False} +# Makes this pod's own incoming copy owner-writable, ahead of removing it. A copy can hold directories without write permission (a Go module cache's are mode 0555), and rm -rf cannot empty such a directory as the non-root runner user, so the init container would fail on every kubelet retry and the pod never start. chmod -R does not follow symbolic links, on busybox or GNU, so it changes nothing outside the copy; its failure on a copy that does not exist is silenced, and a copy it could not make removable fails the rm -rf after it. +_MAKE_INCOMING_WRITABLE = 'chmod -R u+rwx "$incoming" 2>/dev/null;' + # Copies an image's payload to this pod's own incoming directory, unless the store already holds the entry. The destination is removed first so that a rerun after a copy killed part-way starts from nothing: cp -R onto an existing directory would nest the payload under it and keep the partial copy, and publishing would then rename that mixed tree. _COPY_SCRIPT = ( '[ -d "$1/$2" ] && exit 0; incoming="$1/' + _CAPABILITY_STORE_INCOMING + '/$2.$POD_UID.$4";' - ' rm -rf "$incoming" && cp -R "$3" "$incoming"' + " " + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming" && cp -R "$3" "$incoming"' ) @@ -1167,7 +1170,7 @@ def _capability_store_copy_container(name: str, image: str, payload: str, key: s # Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. Every copy is flushed to disk (sync, which writes out every file's data and every directory's entries) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. _PUBLISH_SCRIPT = ( 'root=$1; shift; for key in "$@"; do' - ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; mv -T "$incoming" "$root/$key" 2>/dev/null || rm -rf "$incoming"; done;' + ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; mv -T "$incoming" "$root/$key" 2>/dev/null || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; }; done;' ' [ -d "$root/$key" ] || { echo "capability store: $key is not in $root after publishing" >&2; exit 1; };' ' done' ) @@ -1188,9 +1191,9 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M # Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then flushes and publishes it like any other entry, rebuilding from nothing when this pod's own earlier partial merge is found. _SYSROOT_MERGE_SCRIPT = ( 'root=$1; merged=$2; shift 2; [ -d "$root/$merged" ] && exit 0;' - ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; rm -rf "$incoming" && mkdir "$incoming" || exit 1;' + ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming" && mkdir "$incoming" || exit 1;' ' for key in "$@"; do cp -alf "$root/$key/." "$incoming/" || exit 1; done;' - ' sync || exit 1; mv -T "$incoming" "$root/$merged" 2>/dev/null || rm -rf "$incoming";' + ' sync || exit 1; mv -T "$incoming" "$root/$merged" 2>/dev/null || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; };' ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' ) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 2e31e1a..a74db1d 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -312,7 +312,7 @@ Every runner pod of a profile copies the same bytes at its start: each capabilit An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. -- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. +- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. - `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index a0a241a..c3c9ce1 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -773,9 +773,25 @@ class CapabilityStoreScriptTest(unittest.TestCase): def setUp(self) -> None: self.work = Path(tempfile.mkdtemp()) self.addCleanup(shutil.rmtree, self.work) + # Registered second so that it runs first: a tree with directories lacking owner write permission cannot be removed until it is writable again. + self.addCleanup(self.make_writable, self.work) self.root = self.work / "store" (self.root / ".incoming").mkdir(parents=True) + @staticmethod + def make_writable(directory: Path) -> None: + """Give every directory under directory (and itself) owner read, write and search permission, so that it can be removed.""" + for path in [directory, *directory.rglob("*")]: + if path.is_dir() and not path.is_symlink(): + path.chmod(0o755) + + @staticmethod + def make_read_only(directory: Path) -> None: + """Remove owner write permission from every directory under directory (and itself), deepest first, as a Go module cache's directories are (mode 0555).""" + for path in sorted([directory, *directory.rglob("*")], key=lambda candidate: len(candidate.parts), reverse=True): + if path.is_dir() and not path.is_symlink(): + path.chmod(0o555) + def run_container(self, container: dict[str, Any], uid: str, rewrite: dict[str, str]) -> subprocess.CompletedProcess[str]: """Run an init container's command with sh, as pod uid, with its paths rewritten to the temporary ones.""" command = [rewrite.get(argument, argument) for argument in container["command"]] @@ -870,12 +886,15 @@ def test_a_merged_sysroot_is_flushed_to_disk_before_it_is_renamed_into_the_store self.assertTrue((self.root / merged / "lib/libpq.so").is_file()) def assert_rerun_after_a_partial_copy_publishes_the_source_layout(self, copy: dict[str, Any], publish: dict[str, Any], key: str, source: Path, rewrite: dict[str, str]) -> None: - """Leave what a copy killed part-way leaves in the pod's own destination, run the copy twice (the kubelet reruns a pod's init containers after a sandbox change), publish, and require the entry to hold exactly the source's layout and content.""" + """Leave what a copy killed part-way leaves in the pod's own destination (including a directory without owner write permission and content below it), run the copy twice (the kubelet reruns a pod's init containers after a sandbox change, and the second run finds the first's complete copy, whose own source may hold such directories), publish, and require the entry to hold exactly the source's layout and content.""" debris = self.root / ".incoming" / f"{key}.pod-1.{copy['name']}" (debris / "bin").mkdir(parents=True) (debris / "bin/partial").write_text("half a file") (debris / source.name).mkdir() (debris / source.name / "stale").write_text("nested by a copy onto an existing destination") + (debris / "pkg/mod/cache").mkdir(parents=True) + (debris / "pkg/mod/cache/download").write_text("a Go module cache's read-only directory") + self.make_read_only(debris / "pkg") for _ in range(2): completed = self.run_container(copy, "pod-1", rewrite) self.assertEqual(completed.returncode, 0, completed.stderr) @@ -885,12 +904,14 @@ def assert_rerun_after_a_partial_copy_publishes_the_source_layout(self, copy: di self.assertEqual(list((self.root / ".incoming").iterdir()), []) def test_a_capability_copy_rerun_after_a_partial_copy_publishes_exactly_the_payloads_layout(self) -> None: - toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary", "node/22.0.0/x64.complete": ""}) + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary", "node/22.0.0/x64.complete": "", "go/pkg/mod/example@v1/file": "module"}) + self.make_read_only(toolcache / "go") containers = self.containers(["node"]) self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["capability-node"], containers["capability-store-publish"], DIGEST_A, toolcache, {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)}) def test_the_dind_externals_copy_rerun_after_a_partial_copy_publishes_exactly_the_externals_layout(self) -> None: - externals = self.payload("externals", {"node20/bin/node": "binary", "node20/lib/libnode.so": "library"}) + externals = self.payload("externals", {"node20/bin/node": "binary", "node20/lib/libnode.so": "library", "node20/read-only/file": "content"}) + self.make_read_only(externals / "node20/read-only") spec = arc.arc_capability_store_externals(DIND_VALUES, STORE, 1001, 1001)["values"]["template"]["spec"] containers = {container["name"]: container for container in spec["initContainers"]} self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["init-dind-externals"], containers["capability-store-publish-externals"], DIGEST_RUNNER, externals, {"/opt/capability-store": str(self.root), "/home/runner/externals": str(externals)}) @@ -903,10 +924,44 @@ def test_a_sysroot_merge_rerun_after_a_partial_merge_publishes_only_the_librarie debris = self.root / ".incoming" / f"{merged}.pod-1.merge" (debris / "lib").mkdir(parents=True) (debris / "lib/partial").write_text("half a file") + (debris / "share/doc").mkdir(parents=True) + (debris / "share/doc/readme").write_text("a read-only directory's content") + self.make_read_only(debris / "share") completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual(self.layout(self.root / merged), {"lib": "", "lib/libmpi.so": "mpi"}) + def test_a_sysroot_merge_that_loses_the_race_deletes_its_copy_even_when_it_holds_read_only_directories(self) -> None: + (self.root / DIGEST_B / "lib/private").mkdir(parents=True) + (self.root / DIGEST_B / "lib/private/libmpi.so").write_text("mpi") + self.make_read_only(self.root / DIGEST_B) + merge = self.containers(["mpi"])["capability-store-sysroot"] + merged = merge["command"][5] + # Another pod publishes the merged entry while this pod's sync runs, as two pods starting together on a node without it would. + bin_dir = self.work / "race-bin" + bin_dir.mkdir() + sync = bin_dir / "sync" + sync.write_text('#!/bin/sh\nmkdir -p "$RACE_ENTRY" && printf winner > "$RACE_ENTRY/marker"\n') + sync.chmod(0o755) + env = {"PATH": f"{bin_dir}:{os.environ['PATH']}", "RACE_ENTRY": str(self.root / merged)} + completed = self.run_with_env(merge, "pod-1", {"/opt/capability-store": str(self.root)}, env) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.layout(self.root / merged), {"marker": "winner"}) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def test_of_two_pods_publishing_one_image_the_loser_deletes_its_copy_even_when_it_holds_read_only_directories(self) -> None: + toolcache = self.payload("toolcache", {"go/pkg/mod/example@v1/file": "module", "go/bin/go": "binary"}) + self.make_read_only(toolcache / "go") + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + for uid in ("pod-1", "pod-2"): + self.assertEqual(self.run_container(containers["capability-node"], uid, rewrite).returncode, 0) + for uid in ("pod-1", "pod-2"): + published = self.run_container(containers["capability-store-publish"], uid, rewrite) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual(self.layout(self.root / DIGEST_A), self.layout(toolcache)) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + def test_publishing_fails_when_the_entry_is_neither_copied_nor_in_the_store(self) -> None: containers = self.containers(["node"]) published = self.run_container(containers["capability-store-publish"], "pod-1", {"/opt/capability-store": str(self.root)}) From 1b16ccccb508b9911a24ea0078fec6c4d49cc185 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:26:13 +0100 Subject: [PATCH 12/43] fix(arc): make a pod's linked tool trees owner-writable whatever mode the store's directories have The tool cache is built with cp -as, which gives each directory the mode of the store's. A store tree can hold directories without write permission (a Go module cache's are mode 0555), so the pod's directories came out read-only: the rm -f of a copied entry in one failed with EACCES and the init container exited 1 on every retry, and a job could create nothing in them, where the per-pod cp -R this replaced tolerated such a tree. Every directory of the pod's tree is now made owner-writable with chmod -R u+rwx after it is built, and before it is rebuilt, so a rerun after a run killed between the two converges instead of failing on its own read-only directories. chmod -R does not follow symbolic links, so it changes only the pod's directories. Directories the store holds with writable modes keep them. --- plugins/filter/arc.py | 5 ++- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 62 +++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 3 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index f2b218b..f565ca6 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1197,12 +1197,13 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' ) -# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well. cp -asf, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. +# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. _TOOL_CACHE_COPY_SCRIPT = 'source=$1; destination=$2; shift 2; for entry; do target=$destination/${entry#"$source"/}; rm -f "$target" && cp -P "$entry" "$target" || exit 1; done' _TOOL_CACHE_LINK_SCRIPT = ( 'root=$1; cache=$2; shift 2; for key in "$@"; do for arch in "$root/$key"/*/*/*; do' ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"}; parent="$cache/${relative%/*}"; tree="$cache/$relative";' - ' mkdir -p "$parent" && cp -asf "$arch" "$parent/" || exit 1;' + ' mkdir -p "$parent" || exit 1; chmod -R u+rwx "$tree" 2>/dev/null;' + ' cp -asf "$arch" "$parent/" && chmod -R u+rwx "$tree" || exit 1;' " find \"$arch\" -type l -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" " [ ! -d \"$arch/bin\" ] || find \"$arch/bin\" -maxdepth 1 -type f -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" ' [ ! -e "$arch.complete" ] || cp "$arch.complete" "$cache/$relative.complete" || exit 1;' diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index a74db1d..bc48fad 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -315,7 +315,7 @@ An entry is named for its image's digest, so every image a profile's capabilitie - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. - `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. -- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. +- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index c3c9ce1..acbb2ee 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -6,6 +6,7 @@ import json import os import shutil +import stat import subprocess import tempfile import unittest @@ -1095,6 +1096,67 @@ def test_every_tool_version_and_architecture_of_every_entry_is_linked_beside_the self.assertTrue((cache / "node/22.0.0/x64/bin/node").is_file()) self.assertFalse((cache / "node/22.0.0/arm64.complete").exists()) + def read_only_tool_tree(self) -> Path: + """Return the store's directory of a tool tree whose directories, at several depths and including itself, lack owner write permission, as a Go module cache's do (mode 0555), holding a file and a link at the deepest level.""" + arch = self.tool_tree() + (arch / "lib/deep/er").mkdir(parents=True) + (arch / "lib/deep/er/file").write_text("deep") + (arch / "lib/deep/er/link").symlink_to("file") + self.make_read_only(arch) + return arch + + def test_a_tool_tree_the_store_holds_read_only_is_linked_and_every_directory_in_the_pod_is_writable(self) -> None: + self.read_only_tool_tree() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0/x64" + directories = [tree, *(path for path in tree.rglob("*") if path.is_dir() and not path.is_symlink())] + self.assertGreater(len(directories), 6) + for directory in directories: + self.assertTrue(directory.stat().st_mode & stat.S_IWUSR, directory) + self.assertEqual((tree / "lib/deep/er/link").readlink(), Path("file")) + self.assertEqual((tree / "lib/deep/er/file").read_text(), "deep") + + def test_a_job_can_change_a_directory_the_store_holds_read_only_and_the_store_is_unchanged(self) -> None: + arch = self.read_only_tool_tree() + before = self.snapshot(self.root) + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + deepest = cache / "node/22.0.0/x64/lib/deep/er" + (deepest / "created").write_text("created") + (deepest / "file").unlink() + (deepest / "file").write_text("replaced") + (deepest / "link").unlink() + (cache / "node/22.0.0/x64/lib/new").mkdir() + self.assertEqual((deepest / "file").read_text(), "replaced") + self.assertEqual(self.snapshot(self.root), before) + self.assertEqual((arch / "lib/deep/er/file").read_text(), "deep") + self.assertFalse((arch / "lib/deep/er/created").exists()) + + def test_linking_again_over_a_pod_tree_whose_directories_were_left_read_only_converges(self) -> None: + self.read_only_tool_tree() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + first = self.snapshot(cache) + # What a run killed after it built the directories and before it made them writable leaves. + self.make_read_only(cache / "node") + self.link_tool_cache(cache) + self.assertEqual(self.snapshot(cache), first) + self.assertTrue((cache / "node/22.0.0/x64/lib/deep/er").stat().st_mode & stat.S_IWUSR) + + def test_the_pods_directories_keep_the_stores_modes_when_those_are_writable(self) -> None: + arch = self.tool_tree() + (arch / "bin").chmod(0o750) + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0/x64" + for relative in (".", "bin", "lib", "lib/node_modules/npm", "empty"): + self.assertEqual(stat.S_IMODE((tree / relative).stat().st_mode), stat.S_IMODE((arch / relative).stat().st_mode), relative) + def test_linking_fails_when_an_entry_cannot_be_linked(self) -> None: self.tool_tree() cache = self.work / "toolcache-pod" From d520f676041f9a4ff9fc9dc78561751fd5920afc Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:26:58 +0100 Subject: [PATCH 13/43] docs(arc): state chmod in the capability image and store image contracts The store's copy container now runs chmod -R before rm -rf, in the capability's own image, and the publish, merge and tool cache containers run it in the store image, so both contracts list it beside rm. --- plugins/filter/arc.py | 2 +- roles/github_runner_arc/README.md | 4 ++-- roles/github_runner_arc/defaults/main.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index f565ca6..9628f2d 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1155,7 +1155,7 @@ def _runner_security_context(runner_user: int, runner_group: int) -> dict[str, A def _capability_store_copy_container(name: str, image: str, payload: str, key: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: - """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin, rm and cp. It removes this pod's own destination first, since the kubelet reruns a pod's init containers after a sandbox change and a copy killed part-way leaves the destination behind, and cp -R onto a directory that exists nests the payload inside it and keeps the partial first copy. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed.""" + """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin, chmod, rm and cp. It makes this pod's own destination owner-writable and removes it first, since the kubelet reruns a pod's init containers after a sandbox change and a copy killed part-way leaves the destination behind, and cp -R onto a directory that exists nests the payload inside it and keeps the partial first copy. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed.""" root = store["path"].rstrip("/") return { "name": name, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index bc48fad..16f1134 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -220,7 +220,7 @@ A profile is the unit a workflow selects, so a set of capabilities becomes a nam A tool image is any image that has: -- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /toolcache/. "$1"/'`, with the tool-cache mount as `$1` (a profile with `capability_store: true` copies with `rm -rf` and `cp -R` instead, so the image needs `rm` as well); +- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /toolcache/. "$1"/'`, with the tool-cache mount as `$1` (a profile with `capability_store: true` copies with `chmod -R`, `rm -rf` and `cp -R` instead, so the image needs `rm` and `chmod` as well); - the tool under `/toolcache`, already in the Actions tool-cache layout: `/toolcache////` holding the tool, and an empty `/toolcache///.complete` file beside that directory, which `@actions/tool-cache` requires before it treats an entry as present; - files readable, and directories searchable, by any user, since the copy runs as the runner's user. @@ -248,7 +248,7 @@ and is declared as `{image: ghcr.io/example/toolcache-example-cli:1.2.3, path: e A system-library image is any image that has: -- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /sysroot/. "$1"/'`, with the sysroot mount as `$1` (a profile with `capability_store: true` also needs `rm`, as for a tool-cache image); `cp -R` keeps symbolic links as links, which a library's versioned names (`libfoo.so` to `libfoo.so.1`) rely on; +- a POSIX `sh` and `cp` at `/bin/sh` and on its `PATH`, which the init container runs as `/bin/sh -c 'cp -R /sysroot/. "$1"/'`, with the sysroot mount as `$1` (a profile with `capability_store: true` also needs `rm` and `chmod`, as for a tool-cache image); `cp -R` keeps symbolic links as links, which a library's versioned names (`libfoo.so` to `libfoo.so.1`) rely on; - the payload under `/sysroot`, laid out as an install prefix: programs in `bin/`, shared libraries in `lib/` (not `lib64/` or a multiarch subdirectory, which the search paths do not include), headers in `include/`, pkg-config files in `lib/pkgconfig/` or `share/pkgconfig/`, and CMake packages anywhere CMake's search under a prefix finds them, such as `lib/cmake//`; - a payload that works from whatever directory it is copied to, given the search paths and the capability's `env`: it must not need its build prefix to exist. In practice that means pkg-config files that name their prefix relative to their own location (`prefix=${pcfiledir}/../..`, which pkg-config and pkgconf both expand), CMake packages that compute their paths relative to themselves (what `install(EXPORT)` generates), no libtool `.la` archives (they record absolute paths), and, for a library that records its install prefix in its binaries, an environment variable it reads to find its files elsewhere, set through `env`; - programs and libraries built for the runner image's own C library, on the same distribution release as the runner image or an older one, linking only libraries the runner image already has or the payload carries itself. The payload should not carry its own copy of a library the runner image has, such as the C library, OpenSSL or the C++ runtime: `LD_LIBRARY_PATH` puts the sysroot first for every step, so its copy would replace the image's for everything those steps run; diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index e0b7900..e181d69 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -29,7 +29,7 @@ github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" # Where the store is mounted, read-only, on the runner container: the tool cache's links point into it. github_runner_arc_capability_store_path: "/opt/capability-store" -# The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chown, sync, mv, cp, ln and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. +# The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chmod, chown, sync, mv, cp, ln, find and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. github_runner_arc_capability_store_image: "docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" # The most gibibytes one profile's cache may hold on a node before the pruner empties it. The limit is per profile, so a node's worst case is the sum of the limits of the profiles with a cache on it, times github_runner_arc_node_cache_hard_limit_factor. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) empties a cache only while no runner pod of its profile runs on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow; a negative value, a leading zero, a fraction or a value above 1073741824 is an error (the pruner's own arithmetic bounds it, and the role refuses what the pruner would). From ade2c3bfe278a9d093bb873fd4603adfdc2fc96f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:27:09 +0100 Subject: [PATCH 14/43] docs(arc): state that the store's flush before publishing is host-wide and why that is accepted The publish and sysroot merge containers flush a copy with sync before renaming it, and the comments and the README described that as writing out the copy's data. It is sync(2), which writes out every dirty page on the host and waits for it, so the first pod to copy an entry on a busy node waits for more than its own copy. It cannot be scoped with the store image: busybox's sync takes no argument (it warns and ignores one, so there is no sync -f on a filesystem), and its fsync applet syncs one file or directory per call, which for a large tree of small files is a journal commit for each where sync is one pass over the dirty pages. The host-wide flush is kept and accepted because only a pod that copied the entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The script comments and the README now say so, with the condition for revisiting it: a store image whose sync can be scoped to a path. --- plugins/filter/arc.py | 4 ++-- roles/github_runner_arc/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 9628f2d..f16f31b 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1167,7 +1167,7 @@ def _capability_store_copy_container(name: str, image: str, payload: str, key: s } -# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. Every copy is flushed to disk (sync, which writes out every file's data and every directory's entries) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. +# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. Every copy is flushed to disk (sync) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. That sync is sync(2), which writes out every dirty page on the host and not only this copy's, and waits for it: busybox's sync takes no argument (given one it prints a warning and ignores it, so it cannot be scoped to the store's filesystem), and its fsync applet syncs one file or directory per call, which for a tree of many small files is one journal commit each where sync is one pass over the dirty pages, so scoping the flush would cost more work than it saves. The host-wide flush is accepted because it runs only in a pod that copied the entry, once per entry per node and not once per pod (a pod that finds the entry in the store copies and flushes nothing). If the store image's busybox ever gains a scoped sync (sync -f on a path), use it here and in the sysroot merge. _PUBLISH_SCRIPT = ( 'root=$1; shift; for key in "$@"; do' ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; mv -T "$incoming" "$root/$key" 2>/dev/null || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; }; done;' @@ -1188,7 +1188,7 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M } -# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then flushes and publishes it like any other entry, rebuilding from nothing when this pod's own earlier partial merge is found. +# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then flushes (with the host-wide sync described at _PUBLISH_SCRIPT) and publishes it like any other entry, rebuilding from nothing when this pod's own earlier partial merge is found. _SYSROOT_MERGE_SCRIPT = ( 'root=$1; merged=$2; shift 2; [ -d "$root/$merged" ] && exit 0;' ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming" && mkdir "$incoming" || exit 1;' diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 16f1134..7836248 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -313,7 +313,7 @@ An entry is named for its image's digest, so every image a profile's capabilitie - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. -- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. The container fails unless every entry the pod needs is in the store. +- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. From 4895217798b90fcc656edb54cc7284279c5560ef Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:51:39 +0100 Subject: [PATCH 15/43] fix(arc): replace a read-only completeness marker when the tool cache is linked again The tool cache link script copied .complete with a plain cp, which keeps the store file's mode. A read-only marker in the store, or in the pod's own copy from an earlier run, then made GNU cp refuse to replace it and failed the init container on a rerun. cp -f removes the destination and retries. --- plugins/filter/arc.py | 4 ++-- tests/unit/test_capabilities.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index f16f31b..7802338 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1197,7 +1197,7 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' ) -# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. +# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well, with cp -f since it keeps the store file's mode and a rerun over a read-only copy must still replace it. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, cp -f, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. _TOOL_CACHE_COPY_SCRIPT = 'source=$1; destination=$2; shift 2; for entry; do target=$destination/${entry#"$source"/}; rm -f "$target" && cp -P "$entry" "$target" || exit 1; done' _TOOL_CACHE_LINK_SCRIPT = ( 'root=$1; cache=$2; shift 2; for key in "$@"; do for arch in "$root/$key"/*/*/*; do' @@ -1206,7 +1206,7 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M ' cp -asf "$arch" "$parent/" && chmod -R u+rwx "$tree" || exit 1;' " find \"$arch\" -type l -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" " [ ! -d \"$arch/bin\" ] || find \"$arch/bin\" -maxdepth 1 -type f -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" - ' [ ! -e "$arch.complete" ] || cp "$arch.complete" "$cache/$relative.complete" || exit 1;' + ' [ ! -e "$arch.complete" ] || cp -f "$arch.complete" "$cache/$relative.complete" || exit 1;' ' done; done' ) diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index acbb2ee..ba056dd 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -1147,6 +1147,27 @@ def test_linking_again_over_a_pod_tree_whose_directories_were_left_read_only_con self.assertEqual(self.snapshot(cache), first) self.assertTrue((cache / "node/22.0.0/x64/lib/deep/er").stat().st_mode & stat.S_IWUSR) + def test_linking_again_replaces_a_completeness_marker_that_is_read_only(self) -> None: + # The marker is copied as the store holds it, so a read-only one in the store is read-only in the pod, and a rerun must still replace it; a marker the pod's copy alone lost write permission on is the same case. + for store_mode, pod_mode in ((0o444, None), (0o644, 0o444)): + with self.subTest(store_mode=oct(store_mode), pod_mode=pod_mode and oct(pod_mode)): + self.work = Path(tempfile.mkdtemp(dir=self.work)) + self.root = self.work / "store" + (self.root / ".incoming").mkdir(parents=True) + self.tool_tree() + (self.root / DIGEST_A / "node/22.0.0/x64.complete").chmod(store_mode) + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + marker = cache / "node/22.0.0/x64.complete" + if pod_mode is not None: + marker.chmod(pod_mode) + self.assertFalse(marker.stat().st_mode & stat.S_IWUSR) + first = self.snapshot(cache) + self.link_tool_cache(cache) + self.assertEqual(self.snapshot(cache), first) + self.assertTrue(marker.is_file() and not marker.is_symlink()) + def test_the_pods_directories_keep_the_stores_modes_when_those_are_writable(self) -> None: arch = self.tool_tree() (arch / "bin").chmod(0o750) From 7fc1525c81cbe48c89e9de414b0465cf3f29ca68 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:51:57 +0100 Subject: [PATCH 16/43] docs(arc): list exactly the applets the capability store image's scripts use The store image contract named ln, which no script uses, and omitted find, which the tool cache link script runs with -exec. The README never listed the applets. A test now compares the list in the defaults and in the README with the utilities the store image's init containers run. --- roles/github_runner_arc/README.md | 2 +- roles/github_runner_arc/defaults/main.yml | 2 +- tests/unit/test_capabilities.py | 33 +++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 7836248..23a3092 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -309,7 +309,7 @@ Each tool-cache and system-library capability adds its copy to every pod's start Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied once per node into `github_runner_arc_capability_store_host_path` and the pods use that copy. -An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. A pod's init containers do the work, in this order: +An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store (the preparation, the publishing, the sysroot merge and the tool cache link) run `github_runner_arc_capability_store_image`. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index e181d69..5f669b2 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -29,7 +29,7 @@ github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" # Where the store is mounted, read-only, on the runner container: the tool cache's links point into it. github_runner_arc_capability_store_path: "/opt/capability-store" -# The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chmod, chown, sync, mv, cp, ln, find and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. +# The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chmod, chown, sync, mv, cp, find and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. github_runner_arc_capability_store_image: "docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" # The most gibibytes one profile's cache may hold on a node before the pruner empties it. The limit is per profile, so a node's worst case is the sum of the limits of the profiles with a cache on it, times github_runner_arc_node_cache_hard_limit_factor. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) empties a cache only while no runner pod of its profile runs on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow; a negative value, a leading zero, a fraction or a value above 1073741824 is an error (the pruner's own arithmetic bounds it, and the role refuses what the pruner would). diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index ba056dd..02de895 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -5,6 +5,7 @@ import importlib.util import json import os +import re import shutil import stat import subprocess @@ -768,6 +769,38 @@ def test_an_unpinned_runner_image_cannot_use_the_store(self) -> None: self.assertIn("pinned by digest", arc.arc_capability_store_externals(values, STORE, 1001, 1001)["errors"][0]) +class CapabilityStoreImageContractTest(unittest.TestCase): + """The applets the store image must provide, as the defaults and the README state them, are exactly the ones the init containers that run it use.""" + + # Utilities a script could call; the shell's own builtins (echo, printf, test, [) are not among them, since the image's sh provides those. + UTILITIES = ("awk", "basename", "cat", "chgrp", "chmod", "chown", "cp", "cut", "date", "dd", "df", "dirname", "du", "find", "grep", "head", "ln", "ls", "mkdir", "mktemp", "mv", "readlink", "rm", "rmdir", "sed", "sort", "stat", "sync", "tail", "tar", "touch", "tr", "wc", "xargs") + STATEMENT = re.compile(r"Its ((?:\w+, )+\w+ and \w+) must be busybox's or GNU's") + + @classmethod + def used(cls) -> set[str]: + """The utilities the store image's init containers run: sh itself, and each utility named as a word in their commands.""" + spec = store_pod(DEFAULT_VALUES, ["node", "mpi", "libpq"])["values"]["template"]["spec"] + scripts = [container["command"][2] for container in spec["initContainers"] if container["image"] == STORE["image"]] + assert len(scripts) == 4, scripts + return {"sh", *(utility for utility in cls.UTILITIES if any(re.search(rf"(? set[str]: + match = cls.STATEMENT.search(text.replace("`", "")) + assert match is not None + return set(re.split(r", | and ", match.group(1))) + + def test_the_defaults_name_exactly_the_applets_the_containers_use(self) -> None: + self.assertEqual(self.stated((ROOT / "roles" / "github_runner_arc" / "defaults" / "main.yml").read_text()), self.used()) + + def test_the_readme_names_exactly_the_applets_the_containers_use(self) -> None: + self.assertEqual(self.stated((ROOT / "roles" / "github_runner_arc" / "README.md").read_text()), self.used()) + + def test_the_applets_include_find_for_the_tool_cache_links_and_not_ln_which_no_script_uses(self) -> None: + self.assertIn("find", self.used()) + self.assertNotIn("ln", self.used()) + + class CapabilityStoreScriptTest(unittest.TestCase): """Runs the store's init container commands with sh against a temporary store, as the pods would: the copy, the publishing (including two pods racing to publish one image), the sysroot merge and the tool cache links. Needs a mv with -T and a cp with -l, as busybox and GNU coreutils have.""" From f1130f8f7576ba2b2e55a017b025ff03aa48aaf7 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:52:09 +0100 Subject: [PATCH 17/43] docs(arc): describe the kind test in its own section and its store case in the store section The paragraph on tests/capabilities/run.sh followed the capability store section, so it read as part of it, though it covers capabilities in general. It now has its own section, and the store section states what the kind store case checks and that the unit tests run the same scripts with host tools, not the store image's busybox. --- roles/github_runner_arc/README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 23a3092..bb2f138 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -325,7 +325,11 @@ What a job can change, with the store mounted read-only on the runner container: The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's own Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; it stays with the other grants for as long as the pruner is installed, which is what keeps a departed profile's namespace among the namespaces read. -`tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. It then pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the pruner, the tool images or the test. +The store case of `tests/capabilities/run.sh` runs real pods in kind, with the store image's own busybox and the runner user's real identity, which the unit tests cannot show: it pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. The unit tests run the same init container commands with the host's `sh` and coreutils (`tests/unit/test_capabilities.py`, which needs GNU's for `mv -T` and `cp -l` where the host's are not), the pruner's script with the host's `bash` (`tests/unit/test_node_cache_pruner.py`), and the role's tasks against a stand-in for the cluster, not busybox or a real one. + +### The capabilities test + +`tests/capabilities/run.sh` exercises a profile's capabilities in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. With a profile that has `capability_store: true` it also runs a store case (see [The capability store](#the-capability-store)). `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the pruner, the tool images or the test. ## Registry cache From fe758469871b8bd365f8b99f8f9ea18f8d78a3d5 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:53:13 +0100 Subject: [PATCH 18/43] fix(arc): keep the pruner's capability store grants to the namespaces it reads The store's Role and RoleBinding were applied to every namespace holding one of the pruner's Roles, with no check of its phase. A departed profile's namespace that is Terminating still listed its Role, and creating one there is refused, so the whole role run failed until the namespace was gone. The namespaces are now looked up by name and a Terminating one is left out. Nothing removed the store's grants once they were not wanted: with no profile using the store every labelled namespace kept list on scale sets and pods for nothing. The role now removes the store's Role and binding from every namespace the references are no longer read from, after the DaemonSet stops reading it. The store's own Role no longer counts when finding those namespaces, since it would keep its namespace in the list for ever. --- plugins/filter/arc.py | 26 ++++--- roles/github_runner_arc/README.md | 2 +- .../tasks/decide_node_cache_pruner.yml | 4 +- .../tasks/install_node_cache_pruner.yml | 20 +++++- .../templates/node-cache-pruner.yaml.j2 | 4 +- roles/github_runner_arc/vars/main.yml | 3 + tests/node_cache_pruner/decide.yml | 2 + tests/unit/test_node_cache_pruner_role.py | 68 +++++++++++++++++-- 8 files changed, 105 insertions(+), 24 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 7802338..2065b58 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -993,33 +993,38 @@ def arc_label_selectors(labels: Mapping[str, str]) -> list[str]: return [f"{key}={value}" for key, value in labels.items()] -def arc_node_cache_access_to_remove(found: Sequence[Mapping[str, Any]], labels: Mapping[str, str], pruner_wanted: bool) -> list[dict[str, str]]: +def arc_node_cache_access_to_remove(found: Sequence[Mapping[str, Any]], labels: Mapping[str, str], pruner_wanted: bool, store_role: str, store_namespaces: Sequence[str]) -> list[dict[str, str]]: """Return the Roles and RoleBindings of the node cache pruner to remove, as kind, namespace and name, bindings before the roles they bind. - While the pruner is wanted nothing is removed: a profile that has left the fleet keeps its grant, which only lists pods in that profile's own namespace, for as long as the pruner is installed, so staying is harmless and the pruner can still check, and so clear, a directory the profile left. When the pruner is not wanted every object found goes. Only an object carrying every one of labels is the role's own, so another software's object that shares one label is never removed. + While the pruner is wanted a profile that has left the fleet keeps its grant, which only lists pods in that profile's own namespace, for as long as the pruner is installed, so staying is harmless and the pruner can still check, and so clear, a directory the profile left. The capability store's grant (the object named store_role) is not kept the same way: it lists scale sets as well, and only the namespaces the pruner reads the store's references from (store_namespaces, empty while no profile uses the store) need it, so one anywhere else goes. When the pruner is not wanted every object found goes. Only an object carrying every one of labels is the role's own, so another software's object that shares one label is never removed. Args: - found: Roles and RoleBindings as the cluster returned them. labels: the labels the role puts on every object it creates for the pruner. pruner_wanted: whether the pruner is installed. + found: Roles and RoleBindings as the cluster returned them. labels: the labels the role puts on every object it creates for the pruner. pruner_wanted: whether the pruner is installed. store_role: the name of the Role and RoleBinding that grant the capability store's references. store_namespaces: the namespaces from arc_capability_store_namespaces. """ - if pruner_wanted: - return [] ours = [resource for resource in found if all((resource["metadata"].get("labels") or {}).get(key) == value for key, value in labels.items())] + if pruner_wanted: + ours = [resource for resource in ours if resource["metadata"]["name"] == store_role and resource["metadata"]["namespace"] not in store_namespaces] stale = [{"kind": resource["kind"], "namespace": resource["metadata"]["namespace"], "name": resource["metadata"]["name"]} for resource in ours] return sorted(stale, key=lambda item: (item["kind"] != "RoleBinding", item["namespace"], item["name"])) -def arc_capability_store_namespaces(profiles: Sequence[Mapping[str, Any]], found: Sequence[Mapping[str, Any]], labels: Mapping[str, str]) -> list[str]: +def arc_terminating_namespaces(namespaces: Sequence[Mapping[str, Any]]) -> list[str]: + """Return the names, sorted, of the Namespaces as the cluster returned them whose phase is Terminating, in which the API server refuses to create anything.""" + return sorted(namespace["metadata"]["name"] for namespace in namespaces if namespace["status"]["phase"] == "Terminating") + + +def arc_capability_store_namespaces(profiles: Sequence[Mapping[str, Any]], found: Sequence[Mapping[str, Any]], labels: Mapping[str, str], store_role: str, terminating: Sequence[str]) -> list[str]: """Return the namespaces the node cache pruner reads the capability store's references from, sorted, or none while no profile uses the store (the pruner then leaves the store alone). - A pod of a profile that has since switched the store off, or left the fleet, keeps running from the template it was made from and keeps using the entries that template named, so its namespace stays among them: the namespaces of every profile in the fleet, and of every Role the pruner has installed, which the role keeps for a profile that has left the fleet (see arc_node_cache_access_to_remove). An entry is then never removed under a pod because its profile stopped appearing in the fleet. Only a Role carrying every one of labels counts, so another software's objects are ignored. + A pod of a profile that has since switched the store off, or left the fleet, keeps running from the template it was made from and keeps using the entries that template named, so its namespace stays among them: the namespaces of every profile in the fleet, and of every per-profile Role the pruner has installed, which the role keeps for a profile that has left the fleet (see arc_node_cache_access_to_remove). An entry is then never removed under a pod because its profile stopped appearing in the fleet. The store's own Role (store_role) does not count, since it is what this list produces and a namespace would otherwise keep itself in it for ever. Only a Role carrying every one of labels counts, so another software's objects are ignored. A Terminating namespace is left out, since the API server refuses to create a Role in one and its pods are being deleted. Args: - profiles: expanded profiles from arc_profiles, each with its namespace and capability_store. found: the Roles and RoleBindings carrying the pruner's labels, as the cluster returned them. labels: the labels the role puts on every object it creates for the pruner. + profiles: expanded profiles from arc_profiles, each with its namespace and capability_store. found: the Roles and RoleBindings carrying the pruner's labels, as the cluster returned them. labels: the labels the role puts on every object it creates for the pruner. store_role: the name of the Role that grants the capability store's references. terminating: the names of the namespaces in phase Terminating, from arc_terminating_namespaces. """ if not any(profile["capability_store"] for profile in profiles): return [] - roles = [resource["metadata"]["namespace"] for resource in found if resource["kind"] == "Role" and all((resource["metadata"].get("labels") or {}).get(key) == value for key, value in labels.items())] - return sorted({profile["namespace"] for profile in profiles} | set(roles)) + roles = [resource["metadata"]["namespace"] for resource in found if resource["kind"] == "Role" and resource["metadata"]["name"] != store_role and all((resource["metadata"].get("labels") or {}).get(key) == value for key, value in labels.items())] + return sorted(({profile["namespace"] for profile in profiles} | set(roles)) - set(terminating)) def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapping[str, Any]]], errors: list[str]) -> list[dict[str, str]]: @@ -1783,6 +1788,7 @@ def filters(self) -> dict[str, Any]: "arc_node_cache_host_path_errors": arc_node_cache_host_path_errors, "arc_node_cache_root": arc_node_cache_root, "arc_node_cache_access_to_remove": arc_node_cache_access_to_remove, + "arc_terminating_namespaces": arc_terminating_namespaces, "arc_node_cache_legacy_access": arc_node_cache_legacy_access, "arc_label_selectors": arc_label_selectors, "arc_bounded_integer_errors": arc_bounded_integer_errors, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index bb2f138..9abb9ec 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -323,7 +323,7 @@ What a job can change, with the store mounted read-only on the runner container: `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume. Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. -The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's own Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; it stays with the other grants for as long as the pruner is installed, which is what keeps a departed profile's namespace among the namespaces read. +The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. The store case of `tests/capabilities/run.sh` runs real pods in kind, with the store image's own busybox and the runner user's real identity, which the unit tests cannot show: it pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. The unit tests run the same init container commands with the host's `sh` and coreutils (`tests/unit/test_capabilities.py`, which needs GNU's for `mv -T` and `cp -l` where the host's are not), the pruner's script with the host's `bash` (`tests/unit/test_node_cache_pruner.py`), and the role's tasks against a stand-in for the cluster, not busybox or a real one. diff --git a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml index d4398a6..fa6d765 100644 --- a/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml +++ b/roles/github_runner_arc/tasks/decide_node_cache_pruner.yml @@ -5,10 +5,10 @@ ansible.builtin.set_fact: github_runner_arc_node_cache_profiles: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_node_cache_profiles(github_runner_arc_capabilities) }}" -# The pruner also removes the capability store's entries that no scale set or pod uses, so it runs for a fleet with a store profile too. Which entries are used is read from the scale sets and pods of every namespace a pod made from an earlier template may still run in, not only those of the profiles that use the store now (see arc_capability_store_namespaces); the list is empty, and the store left alone, while no profile uses it. It needs the pruner's Roles as the cluster has them (github_runner_arc_node_cache_pruner_access). +# The pruner also removes the capability store's entries that no scale set or pod uses, so it runs for a fleet with a store profile too. Which entries are used is read from the scale sets and pods of every namespace a pod made from an earlier template may still run in, not only those of the profiles that use the store now (see arc_capability_store_namespaces); the list is empty, and the store left alone, while no profile uses it. It needs the pruner's Roles as the cluster has them (github_runner_arc_node_cache_pruner_access) and the namespaces that are Terminating (github_runner_arc_terminating_namespaces), which it leaves out. - name: "Node cache: find the namespaces the capability store's references are read from" ansible.builtin.set_fact: - github_runner_arc_capability_store_namespaces: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_capability_store_namespaces(github_runner_arc_node_cache_pruner_access.results | map(attribute='resources') | flatten, github_runner_arc_node_cache_pruner_labels) }}" + github_runner_arc_capability_store_namespaces: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_capability_store_namespaces(github_runner_arc_node_cache_pruner_access.results | map(attribute='resources') | flatten, github_runner_arc_node_cache_pruner_labels, github_runner_arc_capability_store_pruner_role, github_runner_arc_terminating_namespaces) }}" - name: "Node cache: decide whether the pruner is wanted" ansible.builtin.set_fact: diff --git a/roles/github_runner_arc/tasks/install_node_cache_pruner.yml b/roles/github_runner_arc/tasks/install_node_cache_pruner.yml index fc6f6ce..e389d0e 100644 --- a/roles/github_runner_arc/tasks/install_node_cache_pruner.yml +++ b/roles/github_runner_arc/tasks/install_node_cache_pruner.yml @@ -31,7 +31,7 @@ msg: "{{ github_runner_arc_node_cache_legacy_access.left | join(' ') }}" when: github_runner_arc_node_cache_legacy_access.left | length > 0 -# The pruner's Roles and RoleBindings are found by the labels the role puts on them (all of them, so another software's objects are left alone) in every namespace. They are removed only when the pruner is, and found before the decision because the namespaces of those that exist are among the namespaces the capability store's references are read from (see decide_node_cache_pruner.yml). While it is installed a profile that has left the fleet keeps its grant, which is read-only (it lists pods and nothing else) and confined to that profile's own namespace, so staying is harmless, and it lets the pruner still check, and so clear, a directory the profile left. Bindings go before the roles they bind. +# The pruner's Roles and RoleBindings are found by the labels the role puts on them (all of them, so another software's objects are left alone) in every namespace. A profile's own are removed only when the pruner is, and the capability store's also when the namespace they are in is no longer one the store's references are read from (see the removal below); all are found before the decision because the namespaces of those that exist are among the namespaces the capability store's references are read from (see decide_node_cache_pruner.yml). While it is installed a profile that has left the fleet keeps its grant, which is read-only (it lists pods and nothing else) and confined to that profile's own namespace, so staying is harmless, and it lets the pruner still check, and so clear, a directory the profile left. Bindings go before the roles they bind. - name: "Node cache: find the pruner's Roles and RoleBindings" kubernetes.core.k8s_info: api_version: rbac.authorization.k8s.io/v1 @@ -42,6 +42,19 @@ - RoleBinding register: github_runner_arc_node_cache_pruner_access +# The API server refuses to create a Role in a Terminating namespace, and a profile that has left the fleet leaves its namespace to be deleted while its pruner Role is still listed above, so the capability store's Roles are not made there (see arc_capability_store_namespaces). Only the namespaces that could hold one are asked about, by name, so the credential needs get on Namespaces, which applying them below needs as well, and not list. +- name: "Node cache: find the namespaces being deleted" + kubernetes.core.k8s_info: + api_version: v1 + kind: Namespace + name: "{{ item }}" + loop: "{{ ((github_runner_arc_fleet.profiles | map(attribute='namespace') | list) + (github_runner_arc_node_cache_pruner_access.results | map(attribute='resources') | flatten | map(attribute='metadata.namespace') | list)) | unique | sort }}" + register: github_runner_arc_terminating_lookup + +- name: "Node cache: note the namespaces being deleted" + ansible.builtin.set_fact: + github_runner_arc_terminating_namespaces: "{{ github_runner_arc_terminating_lookup.results | map(attribute='resources') | flatten | exadev.github_runner.arc_terminating_namespaces }}" + - name: "Node cache: decide what the pruner needs" ansible.builtin.include_tasks: decide_node_cache_pruner.yml @@ -93,13 +106,14 @@ loop_control: label: "{{ item.kind }}/node-cache-pruner" -- name: "Node cache: remove the pruner's access when the pruner is not wanted" +# Without the pruner, all of its access goes. With it, a profile's own Role stays, as above, but the capability store's Role and binding go from every namespace the store's references are no longer read from: all of them once no profile uses the store, and one in a namespace being deleted. Bindings go before the roles they bind. +- name: "Node cache: remove the access the pruner no longer needs" kubernetes.core.k8s: state: absent api_version: rbac.authorization.k8s.io/v1 kind: "{{ item.kind }}" namespace: "{{ item.namespace }}" name: "{{ item.name }}" - loop: "{{ github_runner_arc_node_cache_pruner_access.results | map(attribute='resources') | flatten | exadev.github_runner.arc_node_cache_access_to_remove(github_runner_arc_node_cache_pruner_labels, github_runner_arc_node_cache_pruner_wanted | bool) }}" + loop: "{{ github_runner_arc_node_cache_pruner_access.results | map(attribute='resources') | flatten | exadev.github_runner.arc_node_cache_access_to_remove(github_runner_arc_node_cache_pruner_labels, github_runner_arc_node_cache_pruner_wanted | bool, github_runner_arc_capability_store_pruner_role, github_runner_arc_capability_store_namespaces) }}" loop_control: label: "{{ item.kind }}/{{ item.name }} in {{ item.namespace }}" diff --git a/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 b/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 index d8e7fda..adf74ba 100644 --- a/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 +++ b/roles/github_runner_arc/templates/node-cache-pruner.yaml.j2 @@ -37,8 +37,8 @@ roleRef: kind: Role name: {{ (name ~ '-' ~ profile.release) | to_json }} {% endfor %} -{# The capability store's references are read in every namespace a pod made from an earlier template may still run in, which includes a profile that has switched the store off or left the fleet, so this Role is per namespace, not per release. Its name has a colon, which a release name (a Helm release name) cannot have, so it never collides with a profile's Role. #} -{% set store_name = name ~ ':capability-store' %} +{# The capability store's references are read in every namespace a pod made from an earlier template may still run in, which includes a profile that has switched the store off or left the fleet, so this Role is per namespace, not per release. Its name is github_runner_arc_capability_store_pruner_role, which no profile's Role can have. #} +{% set store_name = github_runner_arc_capability_store_pruner_role %} {% for namespace in github_runner_arc_capability_store_namespaces %} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/roles/github_runner_arc/vars/main.yml b/roles/github_runner_arc/vars/main.yml index 3957d2e..845e229 100644 --- a/roles/github_runner_arc/vars/main.yml +++ b/roles/github_runner_arc/vars/main.yml @@ -50,6 +50,9 @@ github_runner_arc_node_cache_pruner_labels: app.kubernetes.io/part-of: github-runner app.kubernetes.io/managed-by: exadev.github_runner +# The name of the Role and RoleBinding that let the node cache pruner list pods and scale sets in a namespace it reads the capability store's references from. The colon keeps it from ever colliding with a profile's Role, named for a Helm release name, which cannot hold one. +github_runner_arc_capability_store_pruner_role: node-cache-pruner:capability-store + # The pattern a numeric node cache setting must match and the largest value each may have, the same constants as scripts/node-cache-pruner.sh (which derives them from its arithmetic, in the comment there) so that the role refuses what the pruner would refuse and a value the pruner would crash on never reaches the DaemonSet. tests/unit/test_node_cache_pruner_role.py checks that the two agree. github_runner_arc_node_cache_positive_integer_pattern: '^[1-9][0-9]*$' github_runner_arc_node_cache_max_limit_gib: 1073741824 diff --git a/tests/node_cache_pruner/decide.yml b/tests/node_cache_pruner/decide.yml index 7f51f1f..a64b6cb 100644 --- a/tests/node_cache_pruner/decide.yml +++ b/tests/node_cache_pruner/decide.yml @@ -9,6 +9,8 @@ vars: # The pruner's Roles and RoleBindings as the install tasks found them; a test gives the ones its cluster has. github_runner_arc_node_cache_pruner_access: {results: [{resources: []}, {resources: []}]} + # The namespaces in phase Terminating, as the install tasks found them. + github_runner_arc_terminating_namespaces: [] tasks: - name: Decide ansible.builtin.include_tasks: ../../roles/github_runner_arc/tasks/decide_node_cache_pruner.yml diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index 1f2ee55..2657e9e 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -57,15 +57,17 @@ def render(**role_vars: Any) -> list[dict[str, Any]]: def decide_everything(fleet_targets: list[str], **role_vars: Any) -> dict[str, Any]: - """Run the decision for a fleet of the given profile targets, each with the pnpm-store capability when its target is listed in with_cache, and using the capability store when it is listed in with_store. Roles are the pruner's Roles the cluster has, as (namespace, release) pairs.""" + """Run the decision for a fleet of the given profile targets, each with the pnpm-store capability when its target is listed in with_cache, and using the capability store when it is listed in with_store. Roles are the pruner's Roles the cluster has, as (namespace, release) pairs, store_roles the namespaces holding the capability store's Role, and terminating the namespaces in phase Terminating.""" with_cache = role_vars.pop("with_cache", fleet_targets) with_store = role_vars.pop("with_store", []) roles = role_vars.pop("roles", []) + store_roles = role_vars.pop("store_roles", []) + terminating = role_vars.pop("terminating", []) profiles = [{"namespace": target.split("/")[0], "release": target.split("/")[1], "target": target, "capabilities": ["pnpm-store" if target in with_cache else "node"], "capability_store": target in with_store} for target in fleet_targets] - found = {"results": [{"resources": [access("Role", namespace, release) for namespace, release in roles]}, {"resources": [access("RoleBinding", namespace, release) for namespace, release in roles]}]} + found = {"results": [{"resources": [*(access("Role", namespace, release) for namespace, release in roles), *(store_access("Role", namespace) for namespace in store_roles)]}, {"resources": [*(access("RoleBinding", namespace, release) for namespace, release in roles), *(store_access("RoleBinding", namespace) for namespace in store_roles)]}]} with tempfile.TemporaryDirectory() as work: output = Path(work) / "decision.json" - completed = ansible(DECIDE, decide_output=str(output), github_runner_arc_fleet={"profiles": profiles}, github_runner_arc_capabilities=CATALOGUE, github_runner_arc_node_cache_pruner_access=found, **role_vars) + completed = ansible(DECIDE, decide_output=str(output), github_runner_arc_fleet={"profiles": profiles}, github_runner_arc_capabilities=CATALOGUE, github_runner_arc_node_cache_pruner_access=found, github_runner_arc_terminating_namespaces=terminating, **role_vars) assert completed.returncode == 0, completed.stdout + completed.stderr decision: dict[str, Any] = json.loads(output.read_text()) return decision @@ -169,7 +171,33 @@ def test_no_namespace_is_read_while_no_profile_uses_the_store_so_the_store_is_le def test_only_roles_carrying_every_label_of_the_pruner_count(self) -> None: profiles = [{"namespace": "arc-a", "capability_store": True}] found = [access("Role", "arc-gone", "old"), access("Role", "other", "app", labels={"app.kubernetes.io/name": "node-cache-pruner"}), access("RoleBinding", "bound-only", "old")] - self.assertEqual(arc.arc_capability_store_namespaces(profiles, found, OWN_LABELS), ["arc-a", "arc-gone"]) + self.assertEqual(arc.arc_capability_store_namespaces(profiles, found, OWN_LABELS, STORE_ROLE, []), ["arc-a", "arc-gone"]) + + def test_a_terminating_namespace_is_left_out_whether_it_is_a_profiles_or_one_a_departed_profile_left(self) -> None: + decision = decide_everything(["arc-a/main", "arc-b/builders"], with_store=["arc-a/main"], roles=[("arc-gone", "old")], terminating=["arc-b", "arc-gone"]) + self.assertEqual(decision["store_namespaces"], ["arc-a"]) + + def test_the_stores_own_role_does_not_keep_its_namespace_in_the_list(self) -> None: + decision = decide_everything(["arc-a/main"], with_store=["arc-a/main"], roles=[("arc-a", "main")], store_roles=["arc-a", "arc-old"]) + self.assertEqual(decision["store_namespaces"], ["arc-a"]) + + def test_only_namespaces_in_phase_terminating_are_reported_as_terminating(self) -> None: + namespaces = [namespace_object("arc-b", "Terminating"), namespace_object("arc-a", "Active"), namespace_object("arc-c", "Terminating")] + self.assertEqual(arc.arc_terminating_namespaces(namespaces), ["arc-b", "arc-c"]) + self.assertEqual(arc.arc_terminating_namespaces([]), []) + + def test_the_install_leaves_a_terminating_namespace_out_of_the_stores_roles_and_the_namespaces_read(self) -> None: + calls = install(["arc-a/main", "arc-b/builders"], with_store=["arc-a/main"], existing=[access("Role", "arc-gone", "old"), namespace_object("arc-b", "Terminating"), namespace_object("arc-gone", "Terminating"), namespace_object("arc-a", "Active")]) + daemonset = next(call["definition"] for call in calls if (call.get("definition") or {}).get("kind") == "DaemonSet") + env = {item["name"]: item.get("value") for item in daemonset["spec"]["template"]["spec"]["containers"][0]["env"]} + self.assertEqual(env["STORE_NAMESPACES"], "arc-a") + store_roles = sorted(call["definition"]["metadata"]["namespace"] for call in calls if (call.get("definition") or {}).get("metadata", {}).get("name") == STORE_ROLE and call["definition"]["kind"] == "Role") + self.assertEqual(store_roles, ["arc-a"]) + + def test_the_install_asks_about_each_namespace_that_could_hold_a_store_role_by_name(self) -> None: + calls = install(["arc-a/main", "arc-b/builders"], with_store=["arc-a/main"], existing=[access("Role", "arc-gone", "old")]) + asked = sorted(call["name"] for call in calls if call["module"] == "k8s_info" and call["kind"] == "Namespace") + self.assertEqual(asked, ["arc-a", "arc-b", "arc-gone"]) def test_the_install_reads_the_pruners_roles_before_it_decides(self) -> None: calls = install(["arc-a/main", "arc-b/builders"], with_store=["arc-a/main"], existing=[access("Role", "arc-gone", "old"), access("RoleBinding", "arc-gone", "old")]) @@ -247,6 +275,7 @@ def main(): } +STORE_ROLE = "node-cache-pruner:capability-store" OWN_LABELS = {"app.kubernetes.io/name": "node-cache-pruner", "app.kubernetes.io/part-of": "github-runner", "app.kubernetes.io/managed-by": "exadev.github_runner"} @@ -255,6 +284,16 @@ def access(kind: str, namespace: str, release: str, labels: dict[str, str] | Non return {"kind": kind, "metadata": {"namespace": namespace, "name": f"node-cache-pruner-{release}", "labels": OWN_LABELS if labels is None else labels}} +def store_access(kind: str, namespace: str) -> dict[str, Any]: + """The capability store's Role or RoleBinding of the pruner in a namespace, as the cluster returns it.""" + return {"kind": kind, "metadata": {"namespace": namespace, "name": STORE_ROLE, "labels": OWN_LABELS}} + + +def namespace_object(name: str, phase: str) -> dict[str, Any]: + """A Namespace as the cluster returns it.""" + return {"kind": "Namespace", "metadata": {"name": name}, "status": {"phase": phase}} + + def legacy_role(rules: list[dict[str, Any]] | None = None) -> dict[str, Any]: """The ClusterRole release 1.31.0 created: no labels, one rule that lists pods.""" return {"kind": "ClusterRole", "metadata": {"name": "node-cache-pruner"}, "rules": rules if rules is not None else [{"apiGroups": [""], "resources": ["pods"], "verbs": ["list"]}]} @@ -278,7 +317,7 @@ def install_with_output(fleet_targets: list[str], with_cache: list[str] | None = log = Path(work) / "calls.jsonl" extra = Path(work) / "extra.json" extra.write_text(json.dumps({**INSTALL_VARIABLES, "github_runner_arc_fleet": {"profiles": profiles}, **role_vars})) - env = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": os.pathsep.join([str(Path(work) / "stubs"), str(ROOT / "playbooks" / "collections")]), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_HOME": str(Path(work) / "home"), "STUB_LOG": str(log), "STUB_INFO": json.dumps({kind: [item for item in existing or [] if item["kind"] == kind] for kind in ("Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding")})} + env = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": os.pathsep.join([str(Path(work) / "stubs"), str(ROOT / "playbooks" / "collections")]), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_HOME": str(Path(work) / "home"), "STUB_LOG": str(log), "STUB_INFO": json.dumps({kind: [item for item in existing or [] if item["kind"] == kind] for kind in ("Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding", "Namespace")})} completed = subprocess.run([os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook"), str(INSTALL), "-e", f"@{extra}"], env=env, cwd=work, capture_output=True, text=True, stdin=subprocess.DEVNULL, check=False) assert completed.returncode == 0, completed.stdout + completed.stderr return ([json.loads(line) for line in log.read_text().splitlines()] if log.exists() else []), completed.stdout @@ -410,6 +449,23 @@ def test_a_wanted_pruner_keeps_the_access_of_a_profile_that_has_left_the_fleet_s calls = install(list(self.FLEET), existing=existing) self.assertEqual([removal for removal in removals(calls) if removal[1] is not None], []) + def test_the_stores_access_goes_from_every_namespace_once_no_profile_uses_the_store_while_a_profiles_own_stays(self) -> None: + existing = [*(access(kind, "arc-a", "main") for kind in ("Role", "RoleBinding")), *(store_access(kind, namespace) for kind in ("Role", "RoleBinding") for namespace in ("arc-a", "arc-gone"))] + calls = install(list(self.FLEET), existing=existing) + self.assertEqual([removal for removal in removals(calls) if removal[1] is not None], [("RoleBinding", "arc-a", STORE_ROLE), ("RoleBinding", "arc-gone", STORE_ROLE), ("Role", "arc-a", STORE_ROLE), ("Role", "arc-gone", STORE_ROLE)]) + + def test_the_stores_access_stays_in_the_namespaces_the_store_is_read_from_and_goes_from_a_terminating_one(self) -> None: + existing = [*(access(kind, "arc-a", "main") for kind in ("Role", "RoleBinding")), *(store_access(kind, namespace) for kind in ("Role", "RoleBinding") for namespace in ("arc-a", "arc-c")), namespace_object("arc-c", "Terminating")] + calls = install(list(self.FLEET), with_store=["arc-a/main"], existing=existing) + self.assertEqual([removal for removal in removals(calls) if removal[1] is not None], [("RoleBinding", "arc-c", STORE_ROLE), ("Role", "arc-c", STORE_ROLE)]) + + def test_the_stores_access_is_removed_after_the_daemonset_stops_reading_the_namespace(self) -> None: + existing = [store_access(kind, "arc-a") for kind in ("Role", "RoleBinding")] + calls = install(list(self.FLEET), existing=existing) + daemonset = next(index for index, call in enumerate(calls) if (call.get("definition") or {}).get("kind") == "DaemonSet") + removed = next(index for index, call in enumerate(calls) if call.get("state") == "absent" and call.get("name") == STORE_ROLE) + self.assertLess(daemonset, removed) + def test_only_objects_carrying_every_one_of_the_roles_labels_are_removed_with_the_pruner(self) -> None: foreign = [ access("Role", "other", "app", labels={"app.kubernetes.io/name": "node-cache-pruner"}), @@ -422,7 +478,7 @@ def test_only_objects_carrying_every_one_of_the_roles_labels_are_removed_with_th def test_the_pruners_access_is_looked_up_by_all_of_its_labels_in_every_namespace(self) -> None: calls = install(list(self.FLEET)) - lookups = [call for call in calls if call["module"] == "k8s_info" and not call["kind"].startswith("Cluster")] + lookups = [call for call in calls if call["module"] == "k8s_info" and call["kind"] in ("Role", "RoleBinding")] self.assertEqual([call["kind"] for call in lookups], ["Role", "RoleBinding"]) self.assertTrue(all(call["namespace"] is None for call in lookups)) self.assertTrue(all(sorted(call["label_selectors"]) == sorted(f"{key}={value}" for key, value in OWN_LABELS.items()) for call in lookups)) From 6f936b9f0331d35d942ad91742abe633eaea7783 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:53:57 +0100 Subject: [PATCH 19/43] test(arc): fail the log wait only when the pod has ended The wait for a pod's log line ran its failure after a && b || c, which runs c whenever either test fails and which shellcheck's SC2015 flags. An if states that the failure is for a pod that has ended. --- tests/capabilities/run.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 16ee9de..44aab9f 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -222,7 +222,9 @@ wait_for_log() { deadline=$(($(date +%s) + 900)) until kubectl -n "$namespace" logs "$1" -c runner 2>/dev/null | grep -qx "$2"; do phase="$(kubectl -n "$namespace" get pod "$1" -o jsonpath='{.status.phase}')" - [ "$phase" != Failed ] && [ "$phase" != Succeeded ] || fail "the pod $1 ended before it printed $2" + if [ "$phase" = Failed ] || [ "$phase" = Succeeded ]; then + fail "the pod $1 ended before it printed $2" + fi [ "$(date +%s)" -lt "$deadline" ] || fail "the pod $1 did not print $2" sleep 5 done From a2ced4d4ab7d3138dffbf5b29b8067befff43e29 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 11:54:03 +0100 Subject: [PATCH 20/43] test(arc): build the pruner image from the repository root The kind test built the autoscaler image from a context holding a hand-picked list of the Dockerfile's files, so any COPY line added to the Dockerfile broke the build until the list followed. The Dockerfile copies only the files it names, so the repository root as the context gives the same image and never goes stale. --- tests/capabilities/run.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 44aab9f..73b5e6d 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -54,14 +54,8 @@ log "Building the reference libpq system-library image" docker build -q -t "$library_image" "$repo_root/tool-images/libpq" >/dev/null log "Building the node cache pruner's image" -# The autoscaler image's Dockerfile copies several scripts, of which the pruner needs only its own; a context holding that one keeps the build small, and the Dockerfile's other COPY lines are satisfied by empty files that nothing here runs. -pruner_context="$work/pruner-context" -mkdir -p "$pruner_context/autoscaler" "$pruner_context/scripts" -cp "$repo_root/autoscaler/Dockerfile" "$pruner_context/autoscaler/Dockerfile" -for path in autoscaler/loop.sh scripts/autoscaler.sh scripts/autoscaler-ceiling.jq scripts/psi-reporter.sh scripts/node-cache-pruner.sh; do - cp "$repo_root/$path" "$pruner_context/$path" -done -docker build -q -f "$pruner_context/autoscaler/Dockerfile" -t "$pruner_image" "$pruner_context" >/dev/null +# The repository root is the context: the Dockerfile copies only the files it names, so the image is the same whichever other files the context holds, and a COPY line added to it needs no change here. +docker build -q -f "$repo_root/autoscaler/Dockerfile" -t "$pruner_image" "$repo_root" >/dev/null log "Creating kind cluster ${cluster} and its registry" # A registry on the host's loopback that the nodes reach by its container name over the kind network, as kind's local registry guide has it (https://kind.sigs.k8s.io/docs/user/local-registry/): a pod's image reference names localhost:, and each node's containerd maps that host to the container. From e607eb7f23c2d24e027b4c05d0b13553d09a7760 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:14:11 +0100 Subject: [PATCH 21/43] fix(arc): mount the store's externals entry in a dind native sidecar as well as a dind container The dind-externals volume is removed when the externals move into the store, but only containers were rewritten. A dind sidecar under initContainers (the chart's layout from Kubernetes 1.29 and what the registry cache writes out) kept a mount of the removed volume, which the API server rejects, so such a pod never started. --- plugins/filter/arc.py | 14 +++--- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 73 +++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 61bc53b..c02e008 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1332,7 +1332,7 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str, Any] | None, runner_user: int, runner_group: int) -> dict[str, Any]: """Move a scale set's copy of the runner's externals for the dind sidecar into the capability store. - Applies to values that write the dind containers out (a values file, or the role's registry cache wiring), whose init container named init-dind-externals copies the runner image's externals into the volume named dind-externals, as the chart's own dind template does. That copy becomes a store entry named for the image's digest, published like a capability's, and the dind container mounts it read-only through a subPath of the store. Values without that init container, or a profile without the store, are returned as given. + Applies to values that write the dind containers out (a values file, or the role's registry cache wiring), whose init container named init-dind-externals copies the runner image's externals into the volume named dind-externals, as the chart's own dind template does. That copy becomes a store entry named for the image's digest, published like a capability's, and every container that mounted the volume (the dind container, whether it is a container or a native sidecar under initContainers) mounts it read-only through a subPath of the store instead. Values without that init container, or a profile without the store, are returned as given. Args: values: the release's rendered Helm values. store: the profile's capability store settings ({host_path, path, image}), or None when the profile does not use the store. runner_user, runner_group: the runner's user and group ids, which the copy and publish containers run as. @@ -1359,11 +1359,13 @@ def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str ] init_containers[index:index + 1] = replacement spec["volumes"] = [volume for volume in spec.get("volumes") or [] if volume.get("name") != _DIND_EXTERNALS_VOLUME] - for container in spec.get("containers") or []: - container["volumeMounts"] = [ - {"name": _CAPABILITY_STORE_VOLUME, "mountPath": mount["mountPath"], "subPath": key, "readOnly": True} if mount.get("name") == _DIND_EXTERNALS_VOLUME else mount - for mount in container.get("volumeMounts") or [] - ] + # The dind container is an ordinary container in the chart's layout before Kubernetes 1.29 and an init container with restartPolicy Always (a native sidecar) in the layout after it, which is also what the registry cache writes out, so both lists are rewritten. The copy and publish containers just inserted mount the store alone, so they carry no mount of the removed volume to rewrite. + for container in [*(spec.get("containers") or []), *init_containers]: + if "volumeMounts" in container: + container["volumeMounts"] = [ + {"name": _CAPABILITY_STORE_VOLUME, "mountPath": mount["mountPath"], "subPath": key, "readOnly": True} if mount.get("name") == _DIND_EXTERNALS_VOLUME else mount + for mount in container["volumeMounts"] + ] spec["initContainers"] = init_containers _ensure_store_prepared(spec, store, runner_user, runner_group) _annotate_store_keys(template, [key]) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 0572891..1656fb1 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -323,7 +323,7 @@ What a job can change, with the store mounted read-only on the runner container: **A profile in Docker-in-Docker mode (`container_mode: dind`, or values that write the chart's dind sidecar out) does not have that limit.** The chart's dind sidecar runs privileged (`securityContext.privileged: true` in the `gha-runner-scale-set` chart's dind container), so a job that reaches its Docker daemon can start a container that mounts the node's filesystem and write to the store as root, replacing an entry. Entries are named for an image's digest and never revalidated, so every later pod on that node that uses the entry, in any profile with the store on, then runs the replaced bytes, until the pruner removes an entry no scale set or pod lists. The shared store is therefore a persistence channel from a dind job to other pods on its node, as the node cache is between the jobs of its profile. Give `capability_store: true` only to profiles whose workflows are all of one trust level, and never to one that runs pull requests from forks. -`arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume. Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. +`arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume, whether the dind container is an ordinary container or a native sidecar under `initContainers` (the layout the registry cache writes out). Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 02de895..9cfcac0 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -769,6 +769,79 @@ def test_an_unpinned_runner_image_cannot_use_the_store(self) -> None: self.assertIn("pinned by digest", arc.arc_capability_store_externals(values, STORE, 1001, 1001)["errors"][0]) +RUNNER_IMAGE = f"ghcr.io/example/runner:2@sha256:{DIGEST_RUNNER}" +# Docker-in-Docker the way the chart's Kubernetes 1.29 and later layout writes it, and the way a values file can: dind as a native sidecar, a container in initContainers whose restartPolicy is Always. +DIND_SIDECAR_VALUES: dict[str, Any] = { + "template": { + "spec": { + "initContainers": [ + {"name": "init-dind-externals", "image": RUNNER_IMAGE, "command": ["cp"], "args": ["-r", "/home/runner/externals/.", "/home/runner/tmpDir/"], "volumeMounts": [{"name": "dind-externals", "mountPath": "/home/runner/tmpDir"}]}, + {"name": "dind", "image": "docker:dind", "restartPolicy": "Always", "volumeMounts": [{"name": "work", "mountPath": "/home/runner/_work"}, {"name": "dind-externals", "mountPath": "/home/runner/externals"}]}, + ], + "containers": [{"name": "runner", "volumeMounts": [{"name": "work", "mountPath": "/home/runner/_work"}]}], + "volumes": [{"name": "work", "emptyDir": {}}, {"name": "dind-externals", "emptyDir": {}}], + } + } +} + + +def assert_every_mount_has_its_volume(test: unittest.TestCase, spec: dict[str, Any]) -> None: + """Fail unless every volumeMount of every container and init container names a volume of the pod, as the API server requires.""" + volumes = {volume["name"] for volume in spec["volumes"]} + for container in [*spec.get("containers", []), *spec.get("initContainers", [])]: + for mount in container.get("volumeMounts", []): + test.assertIn(mount["name"], volumes, f"{container['name']} mounts {mount['name']}, which the pod does not define") + + +class CapabilityStoreExternalsLayoutTest(unittest.TestCase): + """The dind sidecar's externals mount is rewritten wherever the dind container is, so no container keeps a mount of the removed volume.""" + + EXTERNALS_MOUNT = {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": DIGEST_RUNNER, "readOnly": True} + + def dind(self, spec: dict[str, Any]) -> dict[str, Any]: + return next(container for container in [*spec["containers"], *spec["initContainers"]] if container["name"] == "dind") + + def test_dind_as_an_ordinary_container_mounts_the_store_entry_and_every_mount_has_its_volume(self) -> None: + spec = arc.arc_capability_store_externals(DIND_VALUES, STORE, 1001, 1001)["values"]["template"]["spec"] + assert_every_mount_has_its_volume(self, spec) + self.assertIn(self.EXTERNALS_MOUNT, self.dind(spec)["volumeMounts"]) + + def test_dind_as_a_native_sidecar_mounts_the_store_entry_and_every_mount_has_its_volume(self) -> None: + result = arc.arc_capability_store_externals(DIND_SIDECAR_VALUES, STORE, 1001, 1001) + self.assertEqual(result["errors"], []) + spec = result["values"]["template"]["spec"] + assert_every_mount_has_its_volume(self, spec) + self.assertNotIn("dind-externals", [volume["name"] for volume in spec["volumes"]]) + dind = self.dind(spec) + self.assertIn(self.EXTERNALS_MOUNT, dind["volumeMounts"]) + self.assertEqual(dind["restartPolicy"], "Always") + # The copy and publish containers the function inserted mount the store and nothing of the removed volume, and the sidecar starts after both. + names = [container["name"] for container in spec["initContainers"]] + self.assertEqual(names, ["capability-store-prepare", "init-dind-externals", "capability-store-publish-externals", "dind"]) + + def test_the_registry_cache_wiring_then_the_store_leaves_every_mount_with_its_volume(self) -> None: + # The role's order: capabilities, then the registry cache (which writes the chart's dind sidecar out under initContainers), then the store's externals. + values = json.loads(json.dumps(DEFAULT_VALUES)) + values["containerMode"] = {"type": "dind"} + values = store_pod(values, ["node", "mpi"])["values"] + cache = arc.arc_registry_cache([{"name": "hub", "host": "docker.io"}], "registry-cache", "registry-cache", "cluster.local", 5000) + self.assertEqual(cache["errors"], []) + wired = arc.arc_registry_cache_pod(values, cache, RUNNER_IMAGE, "registry-cache", "/home/runner/.buildx") + self.assertEqual(wired["errors"], []) + result = arc.arc_capability_store_externals(wired["values"], STORE, 1001, 1001) + self.assertEqual(result["errors"], []) + spec = result["values"]["template"]["spec"] + assert_every_mount_has_its_volume(self, spec) + self.assertNotIn("dind-externals", [volume["name"] for volume in spec["volumes"]]) + self.assertIn("dind", [container["name"] for container in spec["initContainers"]]) + self.assertIn(self.EXTERNALS_MOUNT, self.dind(spec)["volumeMounts"]) + self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"].split()[-1], DIGEST_RUNNER) + + def test_the_store_functions_leave_every_mount_with_its_volume_for_a_profile_without_dind(self) -> None: + spec = store_pod(DEFAULT_VALUES, ["node", "mpi", "libpq", "pnpm-store"])["values"]["template"]["spec"] + assert_every_mount_has_its_volume(self, spec) + + class CapabilityStoreImageContractTest(unittest.TestCase): """The applets the store image must provide, as the defaults and the README state them, are exactly the ones the init containers that run it use.""" From 8b3928df3de65f7215848c1fcd78decafbf1b76e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:22:35 +0100 Subject: [PATCH 22/43] fix(arc): publish a payload whose root is read-only and say why a rename failed cp -R gives the copy the payload root's mode, and renaming a directory into another parent needs write permission on the directory itself, so a root of mode 0555 made mv -T fail as the runner user. The copy script now makes its own top directory owner-writable, and the publish and sysroot merge scripts print mv's error when the entry is still missing instead of hiding it (a lost race stays silent). The sysroot merge shares the property: its destination takes the first entry's root mode and each directory the entries' modes, so read-only directories refused the next entry's files. Only the merge's directories are made writable. chmod -R on its hard-linked files changed the modes of the store entries' own files. --- plugins/filter/arc.py | 29 +++++---- roles/github_runner_arc/README.md | 6 +- tests/unit/test_capabilities.py | 104 ++++++++++++++++++++++++++++-- 3 files changed, 118 insertions(+), 21 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index c02e008..fa6f484 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1204,18 +1204,18 @@ def _runner_security_context(runner_user: int, runner_group: int) -> dict[str, A return {"runAsUser": int(runner_user), "runAsGroup": int(runner_group), "runAsNonRoot": True, "allowPrivilegeEscalation": False} -# Makes this pod's own incoming copy owner-writable, ahead of removing it. A copy can hold directories without write permission (a Go module cache's are mode 0555), and rm -rf cannot empty such a directory as the non-root runner user, so the init container would fail on every kubelet retry and the pod never start. chmod -R does not follow symbolic links, on busybox or GNU, so it changes nothing outside the copy; its failure on a copy that does not exist is silenced, and a copy it could not make removable fails the rm -rf after it. +# Makes this pod's own incoming copy owner-writable, ahead of removing it (a copy's files are its own, unlike the sysroot merge's hard links: see _WRITABLE_MERGE_DIRECTORIES). A copy can hold directories without write permission (a Go module cache's are mode 0555), and rm -rf cannot empty such a directory as the non-root runner user, so the init container would fail on every kubelet retry and the pod never start. chmod -R does not follow symbolic links, on busybox or GNU, so it changes nothing outside the copy; its failure on a copy that does not exist is silenced, and a copy it could not make removable fails the rm -rf after it. _MAKE_INCOMING_WRITABLE = 'chmod -R u+rwx "$incoming" 2>/dev/null;' -# Copies an image's payload to this pod's own incoming directory, unless the store already holds the entry. The destination is removed first so that a rerun after a copy killed part-way starts from nothing: cp -R onto an existing directory would nest the payload under it and keep the partial copy, and publishing would then rename that mixed tree. +# Copies an image's payload to this pod's own incoming directory, unless the store already holds the entry. The destination is removed first so that a rerun after a copy killed part-way starts from nothing: cp -R onto an existing directory would nest the payload under it and keep the partial copy, and publishing would then rename that mixed tree. The copy's own top directory is then made owner-writable, since cp -R gives it the payload root's mode (0555 for a tool cache after chmod -R a-w) and renaming a directory into another parent needs write permission on the directory itself; only that directory, because the rename needs nothing below it. _COPY_SCRIPT = ( '[ -d "$1/$2" ] && exit 0; incoming="$1/' + _CAPABILITY_STORE_INCOMING + '/$2.$POD_UID.$4";' - " " + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming" && cp -R "$3" "$incoming"' + " " + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming" && cp -R "$3" "$incoming" && chmod u+rwx "$incoming"' ) def _capability_store_copy_container(name: str, image: str, payload: str, key: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: - """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin, chmod, rm and cp. It makes this pod's own destination owner-writable and removes it first, since the kubelet reruns a pod's init containers after a sandbox change and a copy killed part-way leaves the destination behind, and cp -R onto a directory that exists nests the payload inside it and keeps the partial first copy. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed.""" + """Return the init container that copies an image's payload into the store's incoming directory, unless the store already holds it. It runs the capability's own image, so it may use only what the image contract promises: a POSIX sh, its test builtin, chmod, rm and cp. It makes this pod's own destination owner-writable and removes it first, since the kubelet reruns a pod's init containers after a sandbox change and a copy killed part-way leaves the destination behind, and cp -R onto a directory that exists nests the payload inside it and keeps the partial first copy. cp -R onto a path that does not exist creates it as the copy, so no mkdir is needed. It then makes the copy's top directory owner-writable so that publishing can rename it whatever mode the payload's root has.""" root = store["path"].rstrip("/") return { "name": name, @@ -1227,11 +1227,11 @@ def _capability_store_copy_container(name: str, image: str, payload: str, key: s } -# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there. Every copy is flushed to disk (sync) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. That sync is sync(2), which writes out every dirty page on the host and not only this copy's, and waits for it: busybox's sync takes no argument (given one it prints a warning and ignores it, so it cannot be scoped to the store's filesystem), and its fsync applet syncs one file or directory per call, which for a tree of many small files is one journal commit each where sync is one pass over the dirty pages, so scoping the flush would cost more work than it saves. The host-wide flush is accepted because it runs only in a pod that copied the entry, once per entry per node and not once per pod (a pod that finds the entry in the store copies and flushes nothing). If the store image's busybox ever gains a scoped sync (sync -f on a path), use it here and in the sysroot merge. +# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there, saying why the rename failed when it did: mv's message is held back while the entry may be there (a lost race fails the rename too, and is no error) and printed only when it is not. Every copy is flushed to disk (sync) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. That sync is sync(2), which writes out every dirty page on the host and not only this copy's, and waits for it: busybox's sync takes no argument (given one it prints a warning and ignores it, so it cannot be scoped to the store's filesystem), and its fsync applet syncs one file or directory per call, which for a tree of many small files is one journal commit each where sync is one pass over the dirty pages, so scoping the flush would cost more work than it saves. The host-wide flush is accepted because it runs only in a pod that copied the entry, once per entry per node and not once per pod (a pod that finds the entry in the store copies and flushes nothing). If the store image's busybox ever gains a scoped sync (sync -f on a path), use it here and in the sysroot merge. _PUBLISH_SCRIPT = ( - 'root=$1; shift; for key in "$@"; do' - ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; mv -T "$incoming" "$root/$key" 2>/dev/null || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; }; done;' - ' [ -d "$root/$key" ] || { echo "capability store: $key is not in $root after publishing" >&2; exit 1; };' + 'root=$1; shift; for key in "$@"; do moved=;' + ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; moved=$(mv -T "$incoming" "$root/$key" 2>&1) || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; }; done;' + ' [ -d "$root/$key" ] || { echo "capability store: $key is not in $root after publishing${moved:+: $moved}" >&2; exit 1; };' ' done' ) @@ -1248,13 +1248,16 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M } -# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then flushes (with the host-wide sync described at _PUBLISH_SCRIPT) and publishes it like any other entry, rebuilding from nothing when this pod's own earlier partial merge is found. +# Makes every directory of the merged sysroot (and nothing else) owner-writable. The merge's files are hard links to the store entries' own, so chmod -R would change the entries' file modes, while the directories are created by the merge and are its own: each takes its mode from the entry's (cp -a), and one of mode 0555 would refuse the next entry's files, the rename into the store and rm -rf alike. +_WRITABLE_MERGE_DIRECTORIES = 'find "$incoming" -type d -exec chmod u+rwx {} +' + +# Builds the merged sysroot from the system-library entries in the profile's order, as hard links (cp -l) so it costs no data, later entries' files replacing earlier ones' (cp -f) as the per-pod copies did, then flushes (with the host-wide sync described at _PUBLISH_SCRIPT) and publishes it like any other entry (reporting a rename that failed for another reason than a lost race, as _PUBLISH_SCRIPT does), rebuilding from nothing when this pod's own earlier partial merge is found. _SYSROOT_MERGE_SCRIPT = ( 'root=$1; merged=$2; shift 2; [ -d "$root/$merged" ] && exit 0;' - ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming" && mkdir "$incoming" || exit 1;' - ' for key in "$@"; do cp -alf "$root/$key/." "$incoming/" || exit 1; done;' - ' sync || exit 1; mv -T "$incoming" "$root/$merged" 2>/dev/null || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; };' - ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing" >&2; exit 1; }' + ' incoming="$root/' + _CAPABILITY_STORE_INCOMING + '/$merged.$POD_UID.merge"; ' + _WRITABLE_MERGE_DIRECTORIES + ' 2>/dev/null; rm -rf "$incoming" && mkdir "$incoming" || exit 1;' + ' for key in "$@"; do cp -alf "$root/$key/." "$incoming/" && ' + _WRITABLE_MERGE_DIRECTORIES + ' || exit 1; done;' + ' sync || exit 1; moved=$(mv -T "$incoming" "$root/$merged" 2>&1) || { ' + _WRITABLE_MERGE_DIRECTORIES + ' 2>/dev/null; rm -rf "$incoming"; };' + ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing${moved:+: $moved}" >&2; exit 1; }' ) # Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well, with cp -f since it keeps the store file's mode and a rerun over a read-only copy must still replace it. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, cp -f, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 1656fb1..2d9c1de 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -314,9 +314,9 @@ Every runner pod of a profile copies the same bytes at its start: each capabilit An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store (the preparation, the publishing, the sysroot merge and the tool cache link) run `github_runner_arc_capability_store_image`. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. -- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. -- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. -- `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. +- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. +- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. +- `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 9cfcac0..583bea2 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -785,6 +785,9 @@ def test_an_unpinned_runner_image_cannot_use_the_store(self) -> None: } +EXTERNALS_STORE_MOUNT = {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": DIGEST_RUNNER, "readOnly": True} + + def assert_every_mount_has_its_volume(test: unittest.TestCase, spec: dict[str, Any]) -> None: """Fail unless every volumeMount of every container and init container names a volume of the pod, as the API server requires.""" volumes = {volume["name"] for volume in spec["volumes"]} @@ -796,15 +799,13 @@ def assert_every_mount_has_its_volume(test: unittest.TestCase, spec: dict[str, A class CapabilityStoreExternalsLayoutTest(unittest.TestCase): """The dind sidecar's externals mount is rewritten wherever the dind container is, so no container keeps a mount of the removed volume.""" - EXTERNALS_MOUNT = {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": DIGEST_RUNNER, "readOnly": True} - def dind(self, spec: dict[str, Any]) -> dict[str, Any]: return next(container for container in [*spec["containers"], *spec["initContainers"]] if container["name"] == "dind") def test_dind_as_an_ordinary_container_mounts_the_store_entry_and_every_mount_has_its_volume(self) -> None: spec = arc.arc_capability_store_externals(DIND_VALUES, STORE, 1001, 1001)["values"]["template"]["spec"] assert_every_mount_has_its_volume(self, spec) - self.assertIn(self.EXTERNALS_MOUNT, self.dind(spec)["volumeMounts"]) + self.assertIn(EXTERNALS_STORE_MOUNT, self.dind(spec)["volumeMounts"]) def test_dind_as_a_native_sidecar_mounts_the_store_entry_and_every_mount_has_its_volume(self) -> None: result = arc.arc_capability_store_externals(DIND_SIDECAR_VALUES, STORE, 1001, 1001) @@ -813,7 +814,7 @@ def test_dind_as_a_native_sidecar_mounts_the_store_entry_and_every_mount_has_its assert_every_mount_has_its_volume(self, spec) self.assertNotIn("dind-externals", [volume["name"] for volume in spec["volumes"]]) dind = self.dind(spec) - self.assertIn(self.EXTERNALS_MOUNT, dind["volumeMounts"]) + self.assertIn(EXTERNALS_STORE_MOUNT, dind["volumeMounts"]) self.assertEqual(dind["restartPolicy"], "Always") # The copy and publish containers the function inserted mount the store and nothing of the removed volume, and the sidecar starts after both. names = [container["name"] for container in spec["initContainers"]] @@ -834,7 +835,7 @@ def test_the_registry_cache_wiring_then_the_store_leaves_every_mount_with_its_vo assert_every_mount_has_its_volume(self, spec) self.assertNotIn("dind-externals", [volume["name"] for volume in spec["volumes"]]) self.assertIn("dind", [container["name"] for container in spec["initContainers"]]) - self.assertIn(self.EXTERNALS_MOUNT, self.dind(spec)["volumeMounts"]) + self.assertIn(EXTERNALS_STORE_MOUNT, self.dind(spec)["volumeMounts"]) self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"].split()[-1], DIGEST_RUNNER) def test_the_store_functions_leave_every_mount_with_its_volume_for_a_profile_without_dind(self) -> None: @@ -1069,6 +1070,99 @@ def test_of_two_pods_publishing_one_image_the_loser_deletes_its_copy_even_when_i self.assertEqual(self.layout(self.root / DIGEST_A), self.layout(toolcache)) self.assertEqual(list((self.root / ".incoming").iterdir()), []) + def test_a_payload_whose_root_directory_is_read_only_is_copied_and_published_with_the_source_layout(self) -> None: + # A tool cache after chmod -R a-w has a root of mode 0555, which the copy keeps, and renaming a directory into another parent needs write permission on the directory itself. + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary", "node/22.0.0/x64.complete": ""}) + externals = self.payload("externals", {"node20/bin/node": "binary"}) + for source in (toolcache, externals): + self.make_read_only(source) + for copy_name, publish_name, key, source, mount in ( + ("capability-node", "capability-store-publish", DIGEST_A, toolcache, "/toolcache"), + ("init-dind-externals", "capability-store-publish-externals", DIGEST_RUNNER, externals, "/home/runner/externals"), + ): + with self.subTest(copy=copy_name): + spec = arc.arc_capability_store_externals(store_pod(DIND_VALUES, ["node"])["values"], STORE, 1001, 1001)["values"]["template"]["spec"] + containers = {container["name"]: container for container in spec["initContainers"]} + rewrite = {"/opt/capability-store": str(self.root), mount: str(source)} + self.assertFalse(source.stat().st_mode & stat.S_IWUSR) + copied = self.run_container(containers[copy_name], "pod-1", rewrite) + self.assertEqual(copied.returncode, 0, copied.stderr) + published = self.run_container(containers[publish_name], "pod-1", rewrite) + self.assertEqual(published.returncode, 0, published.stderr) + self.assertEqual(self.layout(self.root / key), self.layout(source)) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def test_a_rename_that_fails_for_another_reason_than_a_lost_race_reports_the_error_of_mv(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + self.assertEqual(self.run_container(containers["capability-node"], "pod-1", rewrite).returncode, 0) + # A store root the pod cannot write to: the entry cannot be created, which is no lost race. + self.root.chmod(0o555) + published = self.run_container(containers["capability-store-publish"], "pod-1", rewrite) + self.assertNotEqual(published.returncode, 0) + self.assertIn(f"{DIGEST_A} is not in", published.stderr) + self.assertIn("Permission denied", published.stderr) + + def test_a_lost_publish_race_is_not_an_error_and_prints_nothing(self) -> None: + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) + containers = self.containers(["node"]) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} + for uid in ("pod-1", "pod-2"): + self.assertEqual(self.run_container(containers["capability-node"], uid, rewrite).returncode, 0) + for uid in ("pod-1", "pod-2"): + published = self.run_container(containers["capability-store-publish"], uid, rewrite) + self.assertEqual((published.returncode, published.stderr), (0, "")) + + def test_a_sysroot_merged_from_entries_whose_root_is_read_only_is_published(self) -> None: + # The merge's destination takes the mode of the first entry's root (cp -a), and renaming it into the store needs write permission on it. + for digest, files in ((DIGEST_B, {"lib/libmpi.so": "mpi"}), (DIGEST_C, {"lib/libpq.so": "pq"})): + for relative, content in files.items(): + (self.root / digest / relative).parent.mkdir(parents=True, exist_ok=True) + (self.root / digest / relative).write_text(content) + self.make_read_only(self.root / digest) + merge = self.containers(["mpi", "libpq"])["capability-store-sysroot"] + merged = merge["command"][5] + completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.layout(self.root / merged), {"lib": "", "lib/libmpi.so": "mpi", "lib/libpq.so": "pq"}) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def store_entry_with_read_only_directories(self) -> tuple[Path, str]: + """Publish a system library's entry whose files are mode 0444 and whose directories are read-only, and return the file and the merged sysroot's entry name for it.""" + library = self.root / DIGEST_B / "lib/libmpi.so" + library.parent.mkdir(parents=True) + library.write_text("mpi") + library.chmod(0o444) + self.make_read_only(self.root / DIGEST_B) + return library, self.containers(["mpi"])["capability-store-sysroot"]["command"][5] + + def test_a_merge_that_loses_the_race_leaves_the_modes_of_the_entries_files_alone(self) -> None: + # The merge's files are hard links to the entry's, so making them writable would change the entry's own. + library, merged = self.store_entry_with_read_only_directories() + bin_dir = self.work / "race-bin" + bin_dir.mkdir() + sync = bin_dir / "sync" + sync.write_text('#!/bin/sh\nmkdir -p "$RACE_ENTRY"\n') + sync.chmod(0o755) + env = {"PATH": f"{bin_dir}:{os.environ['PATH']}", "RACE_ENTRY": str(self.root / merged)} + completed = self.run_with_env(self.containers(["mpi"])["capability-store-sysroot"], "pod-1", {"/opt/capability-store": str(self.root)}, env) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(stat.S_IMODE(library.stat().st_mode), 0o444) + self.assertEqual(list((self.root / ".incoming").iterdir()), []) + + def test_a_merge_rerun_after_a_partial_merge_leaves_the_modes_of_the_entries_files_alone(self) -> None: + library, merged = self.store_entry_with_read_only_directories() + merge = self.containers(["mpi"])["capability-store-sysroot"] + # What a merge killed after its links and before its rename leaves: hard links into the entry under directories without owner write permission. + debris = self.root / ".incoming" / f"{merged}.pod-1.merge" + debris.mkdir() + subprocess.run(["cp", "-al", f"{self.root / DIGEST_B}/.", f"{debris}/"], check=True) + completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(stat.S_IMODE(library.stat().st_mode), 0o444) + self.assertEqual(self.layout(self.root / merged), {"lib": "", "lib/libmpi.so": "mpi"}) + def test_publishing_fails_when_the_entry_is_neither_copied_nor_in_the_store(self) -> None: containers = self.containers(["node"]) published = self.run_container(containers["capability-store-publish"], "pod-1", {"/opt/capability-store": str(self.root)}) From f391e660043c521a3ca73309bc1ae1baf03a1b5b Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:24:47 +0100 Subject: [PATCH 23/43] fix(arc): validate the node label key when a node cache or the store needs the pruner The pruner runs only on labelled nodes, but only the install tasks asserted the label key, after the controller and platform were applied. The preflight now names the key, the size limit and the feature that needs the pruner. --- roles/github_runner_arc/tasks/validate.yml | 16 +++++++ tests/unit/test_node_cache_pruner_role.py | 51 ++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index d06cafd..56227db 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -57,6 +57,22 @@ {{ github_runner_arc_local.profiles | selectattr('capability_store') | map(attribute='label') | join(', ') }} use the capability store, which needs the node cache pruner to remove the entries no scale set uses, but github_runner_arc_node_cache_size_limit_gib is 0, which installs no pruner when: github_runner_arc_node_cache_size_limit_gib | int == 0 and github_runner_arc_local.profiles | selectattr('capability_store') | list | length > 0 +# The pruner runs only on the nodes carrying github_runner_arc_node_label_key, and it is installed while the size limit is above zero and a profile has a node cache or uses the capability store (see decide_node_cache_pruner.yml), so the key is needed on exactly that condition. The install tasks assert it as well, but only after the controller and platform are applied. +- name: Fail when the node cache pruner is wanted but github_runner_arc_node_label_key is not set + ansible.builtin.fail: + msg: >- + github_runner_arc_node_label_key must be set, because the node cache pruner runs only on the nodes carrying that label and is installed while github_runner_arc_node_cache_size_limit_gib is above 0 + for {{ github_runner_arc_pruner_reasons | join(' and ') }} + vars: + github_runner_arc_pruner_node_caches: "{{ github_runner_arc_fleet.profiles | exadev.github_runner.arc_node_cache_profiles(github_runner_arc_capabilities) | map(attribute='target') | list }}" + github_runner_arc_pruner_stores: "{{ github_runner_arc_fleet.profiles | selectattr('capability_store') | map(attribute='label') | list }}" + github_runner_arc_pruner_reasons: >- + {{ + (['a node cache (' ~ (github_runner_arc_pruner_node_caches | join(', ')) ~ ')'] if github_runner_arc_pruner_node_caches | length > 0 else []) + + (['the capability store (' ~ (github_runner_arc_pruner_stores | join(', ')) ~ ')'] if github_runner_arc_pruner_stores | length > 0 else []) + }} + when: github_runner_arc_node_cache_size_limit_gib | int > 0 and github_runner_arc_node_label_key | length == 0 and github_runner_arc_pruner_reasons | length > 0 + - name: Fail on invalid registry cache settings ansible.builtin.fail: msg: "{{ github_runner_arc_registry_cache_problems }}" diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index 2657e9e..b8be746 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -565,6 +565,57 @@ def test_a_profile_using_the_capability_store_is_refused_without_the_pruner(self message = jinja2.Environment(undefined=jinja2.StrictUndefined).from_string(task["ansible.builtin.fail"]["msg"]).render(github_runner_arc_local=local) self.assertTrue(message.startswith("Example-store use the capability store"), message) + def pruner_label_key_outcome(self, **role_vars: Any) -> str | None: + """The message the pruner's label key check fails with for these variables, or None when it passes: the task's own vars, when and msg, rendered with the role's filters.""" + import jinja2 + from jinja2.nativetypes import NativeEnvironment + + tasks = yaml.safe_load((ROLE / "tasks" / "validate.yml").read_text()) + [task] = [task for task in tasks if task["name"] == "Fail when the node cache pruner is wanted but github_runner_arc_node_label_key is not set"] + defaults = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + variables: dict[str, Any] = {**defaults, **role_vars} + native = NativeEnvironment(undefined=jinja2.StrictUndefined) + text = jinja2.Environment(undefined=jinja2.StrictUndefined) + for environment in (native, text): + environment.filters.update(arc.FilterModule().filters()) + for name, template in task["vars"].items(): + variables[name] = native.from_string(template.replace("exadev.github_runner.", "")).render(**variables) + if not native.from_string("{{ " + task["when"].replace("exadev.github_runner.", "") + " }}").render(**variables): + return None + return text.from_string(task["ansible.builtin.fail"]["msg"]).render(**variables).strip() + + def test_the_pruner_needs_the_node_label_key_for_a_node_cache_or_the_store(self) -> None: + cached = {"namespace": "example-ns", "release": "example-runners", "target": "example-ns/example-runners", "capabilities": ["example-cache"], "capability_store": False, "label": "Example"} + stored = {"namespace": "store-ns", "release": "store-runners", "target": "store-ns/store-runners", "capabilities": [], "capability_store": True, "label": "Store"} + plain = {"namespace": "plain-ns", "release": "plain-runners", "target": "plain-ns/plain-runners", "capabilities": [], "capability_store": False, "label": "Plain"} + catalogue = {"example-cache": {"node_cache": {"init_image": "example.com/init@sha256:" + "0" * 64, "dirs": ["a"]}}} + + def outcome(profiles: list[dict[str, Any]], key: str = "", limit: int = 20) -> str | None: + return self.pruner_label_key_outcome( + github_runner_arc_fleet={"profiles": profiles}, github_runner_arc_capabilities=catalogue, github_runner_arc_node_label_key=key, github_runner_arc_node_cache_size_limit_gib=limit + ) + + message = outcome([cached]) + assert message is not None + self.assertIn("github_runner_arc_node_label_key must be set", message) + self.assertIn("github_runner_arc_node_cache_size_limit_gib", message) + self.assertIn("a node cache (example-ns/example-runners)", message) + self.assertNotIn("capability store", message) + message = outcome([stored]) + assert message is not None + self.assertIn("the capability store (Store)", message) + self.assertNotIn("node cache (", message) + message = outcome([cached, stored, plain]) + assert message is not None + self.assertIn("a node cache (example-ns/example-runners) and the capability store (Store)", message) + self.assertNotIn("Plain", message) + self.assertNotIn("plain-ns", message) + # No pruner is installed, so no label key is needed: a size limit of 0 (a store profile is refused separately), no profile that uses either, or the key set. + self.assertIsNone(outcome([cached], limit=0)) + self.assertIsNone(outcome([plain])) + self.assertIsNone(outcome([])) + self.assertIsNone(outcome([cached, stored], key="example.com/ci")) + def test_the_task_expression_gives_the_same_answer_under_ansible(self) -> None: cases = ({}, {"github_runner_arc_node_cache_host_path": "/var/lib"}, {"github_runner_arc_node_cache_hard_limit_factor": "08"}, {"github_runner_arc_node_cache_size_limit_gib": 1073741825}) for role_vars in cases: From 16c0703f9ee2339c1c8ca0cdc1196d40d6f2c4d6 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:25:07 +0100 Subject: [PATCH 24/43] docs(tool-images): list chmod among the applets the init containers run from busybox --- tool-images/awscli/Dockerfile | 2 +- tool-images/glpk/Dockerfile | 2 +- tool-images/highs/Dockerfile | 2 +- tool-images/libmariadb/Dockerfile | 2 +- tool-images/libpq/Dockerfile | 2 +- tool-images/node/Dockerfile | 2 +- tool-images/openmpi/Dockerfile | 2 +- tool-images/terraform/Dockerfile | 2 +- tool-images/tflint/Dockerfile | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tool-images/awscli/Dockerfile b/tool-images/awscli/Dockerfile index 980fda1..652ab6a 100644 --- a/tool-images/awscli/Dockerfile +++ b/tool-images/awscli/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for the AWS CLI (version 2), following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/aws-cli/// holds the CLI's self-contained distribution (the aws executable and the files it loads beside it), with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. No setup action looks the AWS CLI up in the tool cache, so this capability is meant to be declared with path: aws-cli/, which puts aws on PATH at job start. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# Tool-cache capability image for the AWS CLI (version 2), following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/aws-cli/// holds the CLI's self-contained distribution (the aws executable and the files it loads beside it), with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. No setup action looks the AWS CLI up in the tool cache, so this capability is meant to be declared with path: aws-cli/, which puts aws on PATH at job start. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg AWSCLI_VERSION= -t /: tool-images/awscli FROM alpine:3.22 AS fetch diff --git a/tool-images/glpk/Dockerfile b/tool-images/glpk/Dockerfile index 5ff52e8..6bda866 100644 --- a/tool-images/glpk/Dockerfile +++ b/tool-images/glpk/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for GLPK, the GNU Linear Programming Kit, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libglpk.so), its header (include/glpk.h) and the glpsol command-line solver (bin/glpsol), built on the runner image's own distribution release with GLPK's own arithmetic in place of GMP, so it needs nothing beyond what the runner image already has. GLPK ships no pkg-config file, as the distribution's own development package does not either, so a build finds it through the search paths the job-started hook sets (CPATH and LIBRARY_PATH: -lglpk alone compiles and links), exactly as it would find the distribution's copy in the system directories. GLPK reads no path at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# System-library capability image for GLPK, the GNU Linear Programming Kit, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libglpk.so), its header (include/glpk.h) and the glpsol command-line solver (bin/glpsol), built on the runner image's own distribution release with GLPK's own arithmetic in place of GMP, so it needs nothing beyond what the runner image already has. GLPK ships no pkg-config file, as the distribution's own development package does not either, so a build finds it through the search paths the job-started hook sets (CPATH and LIBRARY_PATH: -lglpk alone compiles and links), exactly as it would find the distribution's copy in the system directories. GLPK reads no path at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg GLPK_VERSION= --build-arg GLPK_SHA256= -t /: tool-images/glpk FROM ubuntu:24.04 AS build diff --git a/tool-images/highs/Dockerfile b/tool-images/highs/Dockerfile index c6d7544..259dd8f 100644 --- a/tool-images/highs/Dockerfile +++ b/tool-images/highs/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for HiGHS, the linear and mixed-integer programming solver, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libhighs.so), its C and C++ headers (include/highs/), a pkg-config file (lib/pkgconfig/highs.pc), a CMake package (lib/cmake/highs/) and the highs command-line solver (bin/highs), built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. HiGHS records no install path that it reads at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# System-library capability image for HiGHS, the linear and mixed-integer programming solver, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libhighs.so), its C and C++ headers (include/highs/), a pkg-config file (lib/pkgconfig/highs.pc), a CMake package (lib/cmake/highs/) and the highs command-line solver (bin/highs), built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. HiGHS records no install path that it reads at run time, so it needs no extra environment. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg HIGHS_VERSION= --build-arg HIGHS_SHA256= -t /: tool-images/highs FROM ubuntu:24.04 AS build diff --git a/tool-images/libmariadb/Dockerfile b/tool-images/libmariadb/Dockerfile index 04683a2..f9b7165 100644 --- a/tool-images/libmariadb/Dockerfile +++ b/tool-images/libmariadb/Dockerfile @@ -1,6 +1,6 @@ # System-library capability image for MariaDB Connector/C, the LGPL client library for MySQL and MariaDB servers, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libmariadb.so, with the libmysqlclient.so names linked to it), its headers (include/mariadb/, with the include/mariadb/mysql/ subdirectory MySQL's client headers sit in), a pkg-config file (lib/pkgconfig/libmariadb.pc) and mariadb_config with its mysql_config name, built on the runner image's own distribution release against its OpenSSL and zlib, so it needs nothing beyond what the runner image already has. It is what the distribution's MySQL client development package is for a build: the mysqlclient Python package, for one, finds it through pkg-config as libmariadb, and a build that asks mysql_config or links -lmysqlclient finds it under those names. # -# The library loads its authentication plugins from a directory compiled in as an absolute path, which a relocated payload cannot honour, so every plugin a client on Linux uses is built into the library instead (caching_sha2_password and sha256_password, which MySQL 8's default accounts need, and MariaDB's ed25519, PARSEC, dialog and clear-text ones), leaving no plugin to load and no variable to set; GSSAPI, which would need the Kerberos libraries, and the remote-file plugin, which would need curl, are left out. mariadb_config finds its installation from its own location, so it reports the relocated prefix. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# The library loads its authentication plugins from a directory compiled in as an absolute path, which a relocated payload cannot honour, so every plugin a client on Linux uses is built into the library instead (caching_sha2_password and sha256_password, which MySQL 8's default accounts need, and MariaDB's ed25519, PARSEC, dialog and clear-text ones), leaving no plugin to load and no variable to set; GSSAPI, which would need the Kerberos libraries, and the remote-file plugin, which would need curl, are left out. mariadb_config finds its installation from its own location, so it reports the relocated prefix. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg MARIADB_CONNECTOR_C_VERSION= --build-arg MARIADB_CONNECTOR_C_SHA256= -t /: tool-images/libmariadb FROM ubuntu:24.04 AS build diff --git a/tool-images/libpq/Dockerfile b/tool-images/libpq/Dockerfile index d87f04a..f1a808a 100644 --- a/tool-images/libpq/Dockerfile +++ b/tool-images/libpq/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for libpq, PostgreSQL's client library, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libpq.so), its headers (include/libpq-fe.h and the headers it includes), a pkg-config file (lib/pkgconfig/libpq.pc), and the psql and pg_config programs (bin/), built on the runner image's own distribution release against its OpenSSL, so it needs nothing beyond what the runner image already has. PostgreSQL's programs find their installation relative to their own location, so it needs no extra environment. Only the client side is built: no server, no readline, no ICU. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# System-library capability image for libpq, PostgreSQL's client library, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix holding the shared library (lib/libpq.so), its headers (include/libpq-fe.h and the headers it includes), a pkg-config file (lib/pkgconfig/libpq.pc), and the psql and pg_config programs (bin/), built on the runner image's own distribution release against its OpenSSL, so it needs nothing beyond what the runner image already has. PostgreSQL's programs find their installation relative to their own location, so it needs no extra environment. Only the client side is built: no server, no readline, no ICU. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg POSTGRESQL_VERSION= -t /: tool-images/libpq FROM ubuntu:24.04 AS build diff --git a/tool-images/node/Dockerfile b/tool-images/node/Dockerfile index 347a93d..1c138aa 100644 --- a/tool-images/node/Dockerfile +++ b/tool-images/node/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for Node.js, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/node/// holds the official Linux build, with .complete beside it, and is named the way actions/setup-node looks it up (Node's os.arch(): x64 or arm64). A job whose setup-node node-version matches the version finds it there and skips the download. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# Tool-cache capability image for Node.js, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/node/// holds the official Linux build, with .complete beside it, and is named the way actions/setup-node looks it up (Node's os.arch(): x64 or arm64). A job whose setup-node node-version matches the version finds it there and skips the download. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg NODE_VERSION= -t /: tool-images/node FROM alpine:3.22 AS fetch diff --git a/tool-images/openmpi/Dockerfile b/tool-images/openmpi/Dockerfile index d8491e3..7fef366 100644 --- a/tool-images/openmpi/Dockerfile +++ b/tool-images/openmpi/Dockerfile @@ -1,4 +1,4 @@ -# System-library capability image for Open MPI, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix (bin/, lib/, include/, lib/pkgconfig/) built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. Open MPI records its install prefix in its binaries, so the capability is declared with env OPAL_PREFIX set to the prefix it is extracted into; with that, mpicc, mpirun and the libraries find their help files, plugins and wrapper data there. The bundled hwloc, libevent, PMIx and PRRTE are built in, so nothing else needs installing; PMIx is built with zlib, which the runner image has, so it can compress large messages. Fortran bindings are left out, since the runner image has no Fortran runtime. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# System-library capability image for Open MPI, following the payload contract in roles/github_runner_arc/README.md (Capabilities): /sysroot is a relocatable prefix (bin/, lib/, include/, lib/pkgconfig/) built on the runner image's own distribution release, so it needs nothing beyond what the runner image already has. Open MPI records its install prefix in its binaries, so the capability is declared with env OPAL_PREFIX set to the prefix it is extracted into; with that, mpicc, mpirun and the libraries find their help files, plugins and wrapper data there. The bundled hwloc, libevent, PMIx and PRRTE are built in, so nothing else needs installing; PMIx is built with zlib, which the runner image has, so it can compress large messages. Fortran bindings are left out, since the runner image has no Fortran runtime. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg OPENMPI_VERSION= --build-arg OPENMPI_SHA256= -t /: tool-images/openmpi FROM ubuntu:24.04 AS build diff --git a/tool-images/terraform/Dockerfile b/tool-images/terraform/Dockerfile index 3ff88e4..b7f0f30 100644 --- a/tool-images/terraform/Dockerfile +++ b/tool-images/terraform/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for Terraform, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/terraform///terraform is the release binary, with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. hashicorp/setup-terraform always downloads and never consults the tool cache, so this capability is meant to be declared with path: terraform/, which puts the binary on PATH at job start. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# Tool-cache capability image for Terraform, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/terraform///terraform is the release binary, with .complete beside it, and is Node's os.arch() name (x64 or arm64), the tool cache's own default. hashicorp/setup-terraform always downloads and never consults the tool cache, so this capability is meant to be declared with path: terraform/, which puts the binary on PATH at job start. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg TERRAFORM_VERSION= -t /: tool-images/terraform FROM alpine:3.22 AS fetch diff --git a/tool-images/tflint/Dockerfile b/tool-images/tflint/Dockerfile index e364378..9b97ae8 100644 --- a/tool-images/tflint/Dockerfile +++ b/tool-images/tflint/Dockerfile @@ -1,4 +1,4 @@ -# Tool-cache capability image for TFLint, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/tflint///tflint is the release binary, with .complete beside it, and is named the way terraform-linters/setup-tflint looks it up (amd64 or arm64, its own mapping of Node's os.arch()). A job whose setup-tflint tflint_version is this version (with or without the leading v) finds it there and skips the download. The final stage is busybox, which supplies the sh, cp and rm the role's init containers run. +# Tool-cache capability image for TFLint, following the tool-image contract in roles/github_runner_arc/README.md (Capabilities): /toolcache/tflint///tflint is the release binary, with .complete beside it, and is named the way terraform-linters/setup-tflint looks it up (amd64 or arm64, its own mapping of Node's os.arch()). A job whose setup-tflint tflint_version is this version (with or without the leading v) finds it there and skips the download. The final stage is busybox, which supplies the sh, cp, rm and chmod the role's init containers run. # # Build for both architectures: docker buildx build --platform linux/amd64,linux/arm64 --build-arg TFLINT_VERSION= -t /: tool-images/tflint FROM alpine:3.22 AS fetch From a6347596832d9c8d235ed5aa92635f7a9d23ab2c Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:25:30 +0100 Subject: [PATCH 25/43] docs(arc): name the init containers that write to the store and the one that only reads it --- roles/github_runner_arc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 2d9c1de..d9a65c2 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -311,7 +311,7 @@ Each tool-cache and system-library capability adds its copy to every pod's start Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied once per node into `github_runner_arc_capability_store_host_path` and the pods use that copy. -An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store (the preparation, the publishing, the sysroot merge and the tool cache link) run `github_runner_arc_capability_store_image`. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: +An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. From 99b255212cdc74e5980115f5174b929418807f6f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:53:50 +0100 Subject: [PATCH 26/43] fix(arc): refuse to start the node cache pruner with a store root and no store namespaces With STORE_ROOT set and STORE_NAMESPACES empty or unset, the reference scan loops over nothing and succeeds with no references, so every store entry and in-progress copy counts as unused and is removed under running pods. The missing namespaces are now a start-up configuration error, status 2, like every other missing setting. --- scripts/node-cache-pruner.sh | 8 ++++++-- tests/unit/test_node_cache_pruner.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/node-cache-pruner.sh b/scripts/node-cache-pruner.sh index de9cb45..e263518 100755 --- a/scripts/node-cache-pruner.sh +++ b/scripts/node-cache-pruner.sh @@ -25,10 +25,10 @@ # - WANTED_CACHES: the profiles that have a cache, as / separated by spaces. When it is set, any other directory under the cache root is removed. Unset leaves every directory alone. # - CHECK_SECONDS: seconds between checks, a positive integer of at most MAX_CHECK_SECONDS (default when unset: 300). # - STORE_ROOT: where the node's capability store is mounted in this pod. Unset leaves the store alone, as on a fleet with no profile using it. -# - STORE_NAMESPACES: the namespaces whose scale sets' templates and pods on this node are what keep an entry, separated by spaces: every namespace a pod that uses the store may run in, which includes a profile that has switched the store off or left the fleet while pods made from its old template run. An entry kept only by a pod in a namespace not listed here is removed. +# - STORE_NAMESPACES: the namespaces whose scale sets' templates and pods on this node are what keep an entry, separated by spaces: every namespace a pod that uses the store may run in, which includes a profile that has switched the store off or left the fleet while pods made from its old template run. An entry kept only by a pod in a namespace not listed here is removed. Required (at least one namespace) when STORE_ROOT is set: with none, nothing would reference any entry and the whole store would be removed under running pods. # - PRUNE_ONCE: when "true", check once and exit, for tests. # -# Start-up configuration errors, every one of them (a required variable missing, a number that is not a positive integer or is above its maximum), end the pruner with status 2 and a message on stderr that names the variable, before it does anything. The DaemonSet's pods then crash-loop with that message in their log. The role validates the numeric settings it templates with the same pattern and maxima, but a value changed some other way (an edited DaemonSet) reaches only this check. +# Start-up configuration errors, every one of them (a required variable missing, STORE_ROOT set without STORE_NAMESPACES, a number that is not a positive integer or is above its maximum), end the pruner with status 2 and a message on stderr that names the variable, before it does anything. The DaemonSet's pods then crash-loop with that message in their log. The role validates the numeric settings it templates with the same pattern and maxima, but a value changed some other way (an edited DaemonSet) reaches only this check. set -uo pipefail # The pattern a positive integer setting must match, and the largest value each numeric setting may have. The role's validation (roles/github_runner_arc/tasks/validate.yml, with the values in vars/main.yml) uses the same pattern and maxima, and a test checks that the two agree. @@ -63,6 +63,10 @@ require_positive_integer() { require_set NODE_NAME require_set HOST_CACHE_ROOT +# A store with no namespaces to read references from has no references, so every entry would count as unused. The names are counted after dropping whitespace, since a value of only spaces lists none. +if [ -n "${STORE_ROOT-}" ] && [ -z "$(printf %s "${STORE_NAMESPACES-}" | tr -d '[:space:]')" ]; then + configuration_error "STORE_NAMESPACES is required when STORE_ROOT is set" +fi CACHE_ROOT="${CACHE_ROOT:-/host-node-cache}" CHECK_SECONDS="${CHECK_SECONDS-300}" PRUNE_ONCE="${PRUNE_ONCE:-false}" diff --git a/tests/unit/test_node_cache_pruner.py b/tests/unit/test_node_cache_pruner.py index 7b3dda6..14a207c 100644 --- a/tests/unit/test_node_cache_pruner.py +++ b/tests/unit/test_node_cache_pruner.py @@ -544,6 +544,27 @@ def test_a_missing_required_variable_ends_the_pruner_with_the_same_status_and_na self.assertIn(f"node-cache-pruner: {name} is required", completed.stderr) self.assertFalse(self.calls.exists()) + def test_a_store_root_without_namespaces_is_refused_before_the_store_or_the_cache_is_touched(self) -> None: + # Unset, empty and only whitespace all list no namespace, so no reference would be read and every entry would count as unused. + for label, overrides in (("unset", {}), ("empty", {"STORE_NAMESPACES": ""}), ("blank", {"STORE_NAMESPACES": " "})): + with self.subTest(label): + store = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, store, True) + (store / KEY_A).mkdir() + (store / ".incoming" / f"{KEY_B}.pod-1.capability-node").mkdir(parents=True) + completed = self.start(STORE_ROOT=str(store), **overrides) + self.assertEqual(completed.returncode, 2, completed.stdout + completed.stderr) + self.assertIn("node-cache-pruner: STORE_NAMESPACES is required when STORE_ROOT is set", completed.stderr) + self.assertFalse(self.calls.exists(), "kubectl was called") + self.assertTrue((store / KEY_A).exists() and (store / ".incoming" / f"{KEY_B}.pod-1.capability-node").exists(), "the store was touched") + self.assertTrue((self.root / "arc-a" / "main" / "pnpm" / "blob").exists(), "the cache was touched") + + def test_namespaces_without_a_store_root_and_neither_set_start_normally(self) -> None: + for label, overrides in (("neither", {}), ("namespaces only", {"STORE_NAMESPACES": "arc-a"})): + with self.subTest(label): + completed = self.start(**overrides) + self.assertEqual(completed.returncode, 0, completed.stderr) + def test_every_numeric_setting_accepts_its_maximum_and_refuses_one_more_and_anything_that_would_wrap(self) -> None: maximum = {"LIMIT_KIB": 1125899906842624, "LIMIT_GIB": 1073741824, "HARD_LIMIT_FACTOR": 1024, "CHECK_SECONDS": 86400} for name, largest in maximum.items(): From 0900b858ba09ffd65169d546eed7ae8961c9fbf3 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:54:13 +0100 Subject: [PATCH 27/43] docs(arc): limit the store's dind externals copy to values that write the sidecar out The role's default template with container_mode dind leaves the sidecar to the chart, so there is no init-dind-externals container to move into the store and no externals entry or annotation key. --- roles/github_runner_arc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index d9a65c2..3e26781 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -26,7 +26,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `autoscale`: `true` makes the profile's scale set one of those the autoscaler manages, pooled against one memory budget; at most one autoscaled profile in a cluster may set `sizing`. - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it bounds the ceiling the autoscaler works out from the nodes, leaving `maxRunners` as the floor. - `capabilities`: names of entries in `github_runner_arc_capabilities` that the profile's runner pods carry, in order (see [Capabilities](#capabilities)). - - `capability_store`: `true` copies the capabilities' tool-cache and system-library payloads, and the dind sidecar's copy of the runner's externals, once per node into the capability store instead of into every pod (see [The capability store](#the-capability-store)). Default `false`. + - `capability_store`: `true` copies the capabilities' tool-cache and system-library payloads once per node into the capability store instead of into every pod, and does the same for the dind sidecar's copy of the runner's externals when the profile's values write that sidecar out with an `init-dind-externals` init container (a values file, or the registry cache's wiring), but not when the chart builds the sidecar itself from `containerMode: dind` (see [The capability store](#the-capability-store)). Default `false`. - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`. - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. From 11ab5d2058be91c053143430d5cf3bcae466c0d0 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:57:17 +0100 Subject: [PATCH 28/43] fix(arc): name capability store entries for their payload kind as well as the image digest An entry was named for the image digest alone, so one image used as both a tool image and a system-library image (or a runner image that is also a tool image) produced two copy containers for one name, one reading /toolcache and the other /sysroot or the externals directory. Whichever published first won and the other consumer silently got the wrong payload. Entries are now toolcache-, sysroot- and externals-, so each payload of an image is its own entry. The merged sysroot is merged-sysroot-, a prefix no image entry can have. Rejecting the dual use instead would refuse a legitimate catalogue, and the only cost of keeping both is one more copy of a payload the image really holds twice. The pruner treats keys as opaque names without dots, which the new names still are. --- plugins/filter/arc.py | 33 ++++-- roles/github_runner_arc/README.md | 10 +- roles/github_runner_arc/defaults/main.yml | 2 +- tests/unit/test_capabilities.py | 132 ++++++++++++++-------- 4 files changed, 112 insertions(+), 65 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index fa6f484..9206d4c 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1113,7 +1113,7 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca # --- The capability store: one copy per node of what every runner pod would otherwise copy at its start --- -# A pod's tool-cache and system-library copies (and the runner's externals for the dind sidecar) are the same bytes on every pod of a profile, taken from digest-pinned images. With a profile's capability_store on, each image's payload is copied once per node into /, where is the image's sha256 digest, and pods mount the store read-only instead of writing their own copy. The store root is a hostPath of its own, apart from the node cache, whose last component names it. +# A pod's tool-cache and system-library copies (and the runner's externals for the dind sidecar) are the same bytes on every pod of a profile, taken from digest-pinned images. With a profile's capability_store on, each image's payload is copied once per node into /-, where is the image's sha256 digest and the payload it holds (see arc_capability_store_key), and pods mount the store read-only instead of writing their own copy. The store root is a hostPath of its own, apart from the node cache, whose last component names it. _CAPABILITY_STORE_HOST_PATH_NAME = re.compile(r"^(?:capability-store|[a-z0-9][a-z0-9-]*-capability-store)$") # A copy is written under /.incoming/.. and published with one rename onto /: rename(2) onto a non-empty directory fails, so of two pods copying the same image at once the first rename wins and the other's copy is deleted, and a reader never sees a part-written entry, since an entry exists only once it is complete. The pod uid comes from the downward API and the container name keeps two copies of one image within a pod apart. _CAPABILITY_STORE_INCOMING = ".incoming" @@ -1123,6 +1123,11 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca _CAPABILITY_STORE_SYSROOT = "capability-store-sysroot" _CAPABILITY_STORE_TOOL_CACHE = "capability-store-tool-cache" _CAPABILITY_STORE_EXTERNALS_PUBLISH = "capability-store-publish-externals" +# The kinds of payload an entry holds, the first part of its name (see arc_capability_store_key): the tool cache of a tool image, the sysroot of a system-library image and the runner's externals for the dind sidecar. +_STORE_KIND_TOOL_CACHE = "toolcache" +_STORE_KIND_SYSROOT = "sysroot" +_STORE_KIND_EXTERNALS = "externals" +_CAPABILITY_STORE_MERGED_PREFIX = "merged-sysroot-" # The annotation that lists the store entries a pod uses, separated by spaces, which the node cache pruner reads so that it never removes an entry a pod on its node still uses. _CAPABILITY_STORE_KEYS_ANNOTATION = "github-runner.exadev.io/capability-store-keys" _IMAGE_DIGEST = re.compile(r"@sha256:([0-9a-f]{64})$") @@ -1176,15 +1181,18 @@ def arc_capability_store_image_errors(image: Any) -> list[str]: return [f"github_runner_arc_capability_store_image must be pinned by digest (name@sha256:<64 hex digits>), since its preparation container runs as root on a hostPath (got {image!r})"] -def arc_capability_store_key(image: str) -> str | None: - """Return the store entry name of an image's payload, its sha256 digest in hex, or None when the image is not pinned by digest and so has no name that changes exactly when its content does.""" +def arc_capability_store_key(image: str, kind: str) -> str | None: + """Return the store entry name of the payload of kind that an image holds, -, or None when the image is not pinned by digest and so has no name that changes exactly when its content does. + + The kind is part of the name because one image can be used for more than one payload (a runner image is both a tool image and the source of the dind sidecar's externals): the entry is a copy of one directory of the image, so two kinds of one digest are two entries, and neither can be taken for the other. + """ match = _IMAGE_DIGEST.search(image) - return match.group(1) if match else None + return f"{kind}-{match.group(1)}" if match else None def _capability_store_merged_key(keys: Sequence[str]) -> str: - """Return the store entry name of the sysroot merged from keys in their order: a hash of the ordered keys, so a different set or order of system libraries is a different entry, since a later one's file overwrites an earlier one's.""" - return "sysroot-" + hashlib.sha256(" ".join(keys).encode()).hexdigest() + """Return the store entry name of the sysroot merged from keys in their order: a hash of the ordered keys, so a different set or order of system libraries is a different entry, since a later one's file overwrites an earlier one's. Its prefix is not a payload kind's, so it cannot be the name of an image's entry.""" + return _CAPABILITY_STORE_MERGED_PREFIX + hashlib.sha256(" ".join(keys).encode()).hexdigest() def _capability_store_prepare_container(store: Mapping[str, Any], runner_user: int, runner_group: int) -> dict[str, Any]: @@ -1304,11 +1312,12 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s image = capability.get("image") or capability.get("sysroot_image") if image is None: continue - key = arc_capability_store_key(image) + is_tool = capability.get("image") is not None + key = arc_capability_store_key(image, _STORE_KIND_TOOL_CACHE if is_tool else _STORE_KIND_SYSROOT) assert key is not None - payload = _TOOL_IMAGE_PAYLOAD if capability.get("image") is not None else _SYSROOT_IMAGE_PAYLOAD + payload = _TOOL_IMAGE_PAYLOAD if is_tool else _SYSROOT_IMAGE_PAYLOAD copies.append(_capability_store_copy_container(f"{_CAPABILITY_CONTAINER_PREFIX}{name}", image, payload, key, store, runner_user, runner_group)) - (tool_keys if capability.get("image") is not None else sysroot_keys).append(key) + (tool_keys if is_tool else sysroot_keys).append(key) keys = list(dict.fromkeys([*tool_keys, *sysroot_keys])) containers = copies + [_capability_store_publish_container(_CAPABILITY_STORE_PUBLISH, keys, store, runner_user, runner_group)] merged = _capability_store_merged_key(sysroot_keys) if sysroot_keys else None @@ -1335,7 +1344,7 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str, Any] | None, runner_user: int, runner_group: int) -> dict[str, Any]: """Move a scale set's copy of the runner's externals for the dind sidecar into the capability store. - Applies to values that write the dind containers out (a values file, or the role's registry cache wiring), whose init container named init-dind-externals copies the runner image's externals into the volume named dind-externals, as the chart's own dind template does. That copy becomes a store entry named for the image's digest, published like a capability's, and every container that mounted the volume (the dind container, whether it is a container or a native sidecar under initContainers) mounts it read-only through a subPath of the store instead. Values without that init container, or a profile without the store, are returned as given. + Applies to values that write the dind containers out (a values file, or the role's registry cache wiring), whose init container named init-dind-externals copies the runner image's externals into the volume named dind-externals, as the chart's own dind template does. That copy becomes a store entry named for the image's digest and the kind externals, published like a capability's, and every container that mounted the volume (the dind container, whether it is a container or a native sidecar under initContainers) mounts it read-only through a subPath of the store instead. Values without that init container, or a profile without the store, are returned as given. Args: values: the release's rendered Helm values. store: the profile's capability store settings ({host_path, path, image}), or None when the profile does not use the store. runner_user, runner_group: the runner's user and group ids, which the copy and publish containers run as. @@ -1352,7 +1361,7 @@ def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str copier = next((container for container in init_containers if container.get("name") == _DIND_EXTERNALS_INIT), None) if copier is None: return {"errors": [], "values": values} - key = arc_capability_store_key(str(copier.get("image", ""))) + key = arc_capability_store_key(str(copier.get("image", "")), _STORE_KIND_EXTERNALS) if key is None: return {"errors": [f"the capability store needs {_DIND_EXTERNALS_INIT}'s image pinned by digest (name@sha256:<64 hex digits>), since its store entry is named for the digest (got {copier.get('image')!r})"], "values": values} index = init_containers.index(copier) @@ -1411,7 +1420,7 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu errors.extend( f"capability {name}'s image must be pinned by digest (name@sha256:<64 hex digits>) for the capability store, since its store entry is named for the digest" for name, capability in [*tool_capabilities, *sysroot_capabilities] - if arc_capability_store_key(capability.get("image") or capability["sysroot_image"]) is None + if arc_capability_store_key(capability.get("image") or capability["sysroot_image"], _STORE_KIND_TOOL_CACHE) is None ) tool_paths = [capability["path"] for _, capability in tool_capabilities if capability.get("path") is not None] hooks: dict[str, str] = {} diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 3e26781..9af0d4c 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -311,19 +311,19 @@ Each tool-cache and system-library capability adds its copy to every pod's start Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied once per node into `github_runner_arc_capability_store_host_path` and the pods use that copy. -An entry is named for its image's digest, so every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: +An entry is named for its image's digest and the kind of payload copied from it (`toolcache-`, `sysroot-`, `externals-`), so one image used for two payloads, such as a runner image that is also a tool image, is two entries that cannot be taken for each other, and every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. -- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. -- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. -- `capability-store-sysroot` builds the profile's sysroot as `sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. +- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. +- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. +- `capability-store-sysroot` builds the profile's sysroot as `merged-sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. **A profile in Docker-in-Docker mode (`container_mode: dind`, or values that write the chart's dind sidecar out) does not have that limit.** The chart's dind sidecar runs privileged (`securityContext.privileged: true` in the `gha-runner-scale-set` chart's dind container), so a job that reaches its Docker daemon can start a container that mounts the node's filesystem and write to the store as root, replacing an entry. Entries are named for an image's digest and never revalidated, so every later pod on that node that uses the entry, in any profile with the store on, then runs the replaced bytes, until the pruner removes an entry no scale set or pod lists. The shared store is therefore a persistence channel from a dind job to other pods on its node, as the node cache is between the jobs of its profile. Give `capability_store: true` only to profiles whose workflows are all of one trust level, and never to one that runs pull requests from forks. -`arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest, and mounts that entry read-only into the dind container in place of the `dind-externals` volume, whether the dind container is an ordinary container or a native sidecar under `initContainers` (the layout the registry cache writes out). Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. +`arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest (`externals-`), and mounts that entry read-only into the dind container in place of the `dind-externals` volume, whether the dind container is an ordinary container or a native sidecar under `initContainers` (the layout the registry cache writes out). Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index fbcdbb0..2ac3555 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -25,7 +25,7 @@ github_runner_arc_node_cache_path: "/node-cache" # Where on the node the cache lives, as the directory every profile's own directory (/) sits under, so profiles never share a cache. It must be a dedicated directory the role may create and empty as root: written without a repeated or trailing slash (every path of the cache is built from this one string), at least three levels deep, with a last component ending in node-cache, and not inside a system directory such as /etc or /var/lib/kubelet. A symbolic link at or above the path defeats this validation. Each profile's directory is a hostPath volume created when a pod first needs it. Its lifetime is the node's: it survives pods, and is emptied by the pruner below when it outgrows its limit. github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" -# Where on the node the capability store lives, for profiles with capability_store on: one entry per image a profile's capabilities (and the dind sidecar's externals) copy from, named for the image's digest, written once per node and mounted read-only by the pods. A dedicated directory the role creates and the pruner removes entries from as root, with the same rules as the node cache's root, whose last component is capability-store or ends in -capability-store. +# Where on the node the capability store lives, for profiles with capability_store on: one entry per image a profile's capabilities (and the dind sidecar's externals) copy from, named for the image's digest and the kind of payload copied from it, written once per node and mounted read-only by the pods. A dedicated directory the role creates and the pruner removes entries from as root, with the same rules as the node cache's root, whose last component is capability-store or ends in -capability-store. github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" # Where the store is mounted, read-only, on the runner container: the tool cache's links point into it. github_runner_arc_capability_store_path: "/opt/capability-store" diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 583bea2..428fc69 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -599,6 +599,11 @@ def test_a_missing_subdirectory_fails_the_job(self) -> None: DIGEST_B = "b" * 64 DIGEST_C = "c" * 64 DIGEST_RUNNER = "d" * 64 +# The store entries of those digests: each is named for the kind of payload it holds as well as the image's digest. +KEY_A = f"toolcache-{DIGEST_A}" +KEY_B = f"sysroot-{DIGEST_B}" +KEY_C = f"sysroot-{DIGEST_C}" +KEY_RUNNER = f"externals-{DIGEST_RUNNER}" STORE = {"host_path": "/var/lib/example/capability-store", "path": "/opt/capability-store", "image": BUSYBOX} STORE_CATALOGUE: dict[str, Any] = { "node": {"image": f"ghcr.io/example/toolcache-node:22@sha256:{DIGEST_A}"}, @@ -649,9 +654,9 @@ def test_the_store_mount_path_must_be_an_absolute_directory(self) -> None: with self.subTest(path=path): self.assertEqual(len(arc.arc_capability_store_path_errors(path)), 1) - def test_an_entry_is_named_for_its_images_digest_and_an_unpinned_image_has_no_name(self) -> None: - self.assertEqual(arc.arc_capability_store_key(f"ghcr.io/example/x:1@sha256:{DIGEST_A}"), DIGEST_A) - self.assertIsNone(arc.arc_capability_store_key("ghcr.io/example/x:1")) + def test_an_entry_is_named_for_its_images_digest_and_payload_kind_and_an_unpinned_image_has_no_name(self) -> None: + self.assertEqual(arc.arc_capability_store_key(f"ghcr.io/example/x:1@sha256:{DIGEST_A}", "toolcache"), KEY_A) + self.assertIsNone(arc.arc_capability_store_key("ghcr.io/example/x:1", "toolcache")) class CapabilityStorePodTest(unittest.TestCase): @@ -672,18 +677,18 @@ def test_with_the_store_each_payload_is_copied_once_per_node_and_published_befor # The copy runs the capability's own image as the runner user, only when the store lacks the entry, into a name of its own under .incoming. copier = containers["capability-node"] self.assertEqual(copier["image"], STORE_CATALOGUE["node"]["image"]) - self.assertEqual(copier["command"][3:], ["copy-to-capability-store", "/opt/capability-store", DIGEST_A, "/toolcache", "capability-node"]) + self.assertEqual(copier["command"][3:], ["copy-to-capability-store", "/opt/capability-store", KEY_A, "/toolcache", "capability-node"]) self.assertEqual(copier["securityContext"], {"runAsUser": 1001, "runAsGroup": 1001, "runAsNonRoot": True, "allowPrivilegeEscalation": False}) self.assertEqual(copier["env"], [{"name": "POD_UID", "valueFrom": {"fieldRef": {"fieldPath": "metadata.uid"}}}]) self.assertEqual(containers["capability-mpi"]["command"][6], "/sysroot") - self.assertEqual(containers["capability-store-publish"]["command"][4:], ["/opt/capability-store", DIGEST_A, DIGEST_B, DIGEST_C]) + self.assertEqual(containers["capability-store-publish"]["command"][4:], ["/opt/capability-store", KEY_A, KEY_B, KEY_C]) self.assertEqual(containers["capability-store-publish"]["image"], BUSYBOX) # Only the preparation runs as root, with the two capabilities it needs. self.assertEqual(containers["capability-store-prepare"]["securityContext"]["capabilities"], {"drop": ["ALL"], "add": ["CHOWN", "DAC_OVERRIDE"]}) merged = containers["capability-store-sysroot"]["command"][5] - self.assertRegex(merged, r"^sysroot-[0-9a-f]{64}$") - self.assertEqual(containers["capability-store-sysroot"]["command"][6:], [DIGEST_B, DIGEST_C]) - self.assertEqual(containers["capability-store-tool-cache"]["command"][4:], ["/opt/capability-store", TOOL_CACHE, DIGEST_A]) + self.assertRegex(merged, r"^merged-sysroot-[0-9a-f]{64}$") + self.assertEqual(containers["capability-store-sysroot"]["command"][6:], [KEY_B, KEY_C]) + self.assertEqual(containers["capability-store-tool-cache"]["command"][4:], ["/opt/capability-store", TOOL_CACHE, KEY_A]) self.assertEqual(containers["capability-store-tool-cache"]["volumeMounts"], [{"name": "capability-store", "mountPath": "/opt/capability-store", "readOnly": True}, {"name": "tool-cache", "mountPath": TOOL_CACHE}]) volumes = {volume["name"]: volume for volume in spec["volumes"]} self.assertEqual(volumes["capability-store"], {"name": "capability-store", "hostPath": {"path": STORE["host_path"], "type": "DirectoryOrCreate"}}) @@ -693,7 +698,7 @@ def test_with_the_store_each_payload_is_copied_once_per_node_and_published_befor self.assertIn({"name": "capability-store", "mountPath": "/opt/capability-store", "readOnly": True}, mounts) self.assertIn({"name": "capability-store", "mountPath": SYSROOT, "subPath": merged, "readOnly": True}, mounts) self.assertIn({"name": "tool-cache", "mountPath": TOOL_CACHE}, mounts) - self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{DIGEST_A} {DIGEST_B} {DIGEST_C} {merged}") + self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{KEY_A} {KEY_B} {KEY_C} {merged}") def test_a_different_order_of_system_libraries_is_a_different_merged_sysroot(self) -> None: first = store_pod(DEFAULT_VALUES, ["mpi", "libpq"])["values"]["template"]["spec"]["initContainers"] @@ -701,6 +706,26 @@ def test_a_different_order_of_system_libraries_is_a_different_merged_sysroot(sel merged = [next(container for container in containers if container["name"] == "capability-store-sysroot")["command"][5] for containers in (first, second)] self.assertNotEqual(merged[0], merged[1]) + def test_one_image_used_for_a_tool_cache_and_a_sysroot_is_two_entries_each_copied_from_its_own_payload(self) -> None: + image = f"ghcr.io/example/both:1@sha256:{DIGEST_A}" + result = store_pod(DEFAULT_VALUES, ["tools", "libraries"], {"tools": {"image": image}, "libraries": {"sysroot_image": image}}) + self.assertEqual(result["errors"], []) + containers = {container["name"]: container for container in result["values"]["template"]["spec"]["initContainers"]} + tools, libraries = containers["capability-tools"]["command"], containers["capability-libraries"]["command"] + self.assertEqual((tools[4:6], tools[6]), (["/opt/capability-store", f"toolcache-{DIGEST_A}"], "/toolcache")) + self.assertEqual((libraries[4:6], libraries[6]), (["/opt/capability-store", f"sysroot-{DIGEST_A}"], "/sysroot")) + self.assertEqual(containers["capability-store-publish"]["command"][4:], ["/opt/capability-store", f"toolcache-{DIGEST_A}", f"sysroot-{DIGEST_A}"]) + self.assertEqual(containers["capability-store-sysroot"]["command"][6:], [f"sysroot-{DIGEST_A}"]) + self.assertEqual(containers["capability-store-tool-cache"]["command"][6:], [f"toolcache-{DIGEST_A}"]) + self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"].split()[:2], [f"toolcache-{DIGEST_A}", f"sysroot-{DIGEST_A}"]) + + def test_a_runner_image_that_is_also_a_tool_image_has_a_separate_entry_for_the_dind_externals(self) -> None: + values = store_pod(DIND_VALUES, ["runner-tools"], {"runner-tools": {"image": RUNNER_IMAGE}})["values"] + result = arc.arc_capability_store_externals(values, STORE, 1001, 1001) + self.assertEqual(result["errors"], []) + keys = result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"].split() + self.assertEqual(keys, [f"toolcache-{DIGEST_RUNNER}", f"externals-{DIGEST_RUNNER}"]) + def test_with_the_store_an_unpinned_capability_image_is_an_error(self) -> None: result = store_pod(DEFAULT_VALUES, ["untagged"]) self.assertEqual(result["errors"], ["capability untagged's image must be pinned by digest (name@sha256:<64 hex digits>) for the capability store, since its store entry is named for the digest"]) @@ -741,11 +766,11 @@ def test_the_dind_externals_copy_moves_into_the_store_and_dind_mounts_its_entry_ spec = template["spec"] self.assertEqual([container["name"] for container in spec["initContainers"]], ["capability-store-prepare", "init-dind-externals", "capability-store-publish-externals", "init-dind-mtu"]) copier = spec["initContainers"][1] - self.assertEqual(copier["command"][3:], ["copy-to-capability-store", "/opt/capability-store", DIGEST_RUNNER, "/home/runner/externals", "init-dind-externals"]) + self.assertEqual(copier["command"][3:], ["copy-to-capability-store", "/opt/capability-store", KEY_RUNNER, "/home/runner/externals", "init-dind-externals"]) self.assertEqual([volume["name"] for volume in spec["volumes"]], ["work", "capability-store"]) dind = next(container for container in spec["containers"] if container["name"] == "dind") - self.assertEqual(dind["volumeMounts"][1], {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": DIGEST_RUNNER, "readOnly": True}) - self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], DIGEST_RUNNER) + self.assertEqual(dind["volumeMounts"][1], {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": KEY_RUNNER, "readOnly": True}) + self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], KEY_RUNNER) self.assertEqual(DIND_VALUES["template"]["spec"]["volumes"][1]["name"], "dind-externals") def test_after_the_capabilities_the_store_is_prepared_once_and_the_annotation_keeps_both_lists(self) -> None: @@ -757,7 +782,7 @@ def test_after_the_capabilities_the_store_is_prepared_once_and_the_annotation_ke self.assertEqual(names[:3], ["capability-store-prepare", "init-dind-externals", "capability-store-publish-externals"]) self.assertEqual(names.count("capability-store-prepare"), 1) self.assertEqual([volume["name"] for volume in spec["volumes"]].count("capability-store"), 1) - self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{DIGEST_A} {DIGEST_RUNNER}") + self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{KEY_A} {KEY_RUNNER}") def test_without_the_store_or_without_an_externals_copy_the_values_are_left_alone(self) -> None: self.assertIs(arc.arc_capability_store_externals(DIND_VALUES, None, 1001, 1001)["values"], DIND_VALUES) @@ -785,7 +810,7 @@ def test_an_unpinned_runner_image_cannot_use_the_store(self) -> None: } -EXTERNALS_STORE_MOUNT = {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": DIGEST_RUNNER, "readOnly": True} +EXTERNALS_STORE_MOUNT = {"name": "capability-store", "mountPath": "/home/runner/externals", "subPath": KEY_RUNNER, "readOnly": True} def assert_every_mount_has_its_volume(test: unittest.TestCase, spec: dict[str, Any]) -> None: @@ -836,7 +861,7 @@ def test_the_registry_cache_wiring_then_the_store_leaves_every_mount_with_its_vo self.assertNotIn("dind-externals", [volume["name"] for volume in spec["volumes"]]) self.assertIn("dind", [container["name"] for container in spec["initContainers"]]) self.assertIn(EXTERNALS_STORE_MOUNT, self.dind(spec)["volumeMounts"]) - self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"].split()[-1], DIGEST_RUNNER) + self.assertEqual(result["values"]["template"]["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"].split()[-1], KEY_RUNNER) def test_the_store_functions_leave_every_mount_with_its_volume_for_a_profile_without_dind(self) -> None: spec = store_pod(DEFAULT_VALUES, ["node", "mpi", "libpq", "pnpm-store"])["values"]["template"]["spec"] @@ -925,7 +950,7 @@ def test_a_payload_is_copied_once_and_published_and_a_later_pod_copies_nothing(s published = self.run_container(containers["capability-store-publish"], uid, rewrite) self.assertEqual(published.returncode, 0, published.stderr) self.assertEqual(sorted(path.name for path in (self.root / ".incoming").iterdir()), []) - self.assertEqual((self.root / DIGEST_A / "node/22.0.0/x64/bin/node").read_text(), "binary") + self.assertEqual((self.root / KEY_A / "node/22.0.0/x64/bin/node").read_text(), "binary") def test_of_two_pods_copying_one_image_at_once_the_first_to_publish_wins_and_the_other_copy_is_deleted(self) -> None: toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) @@ -934,14 +959,27 @@ def test_of_two_pods_copying_one_image_at_once_the_first_to_publish_wins_and_the # Both copy before either publishes, as two pods starting together on a node without the entry would. for uid in ("pod-1", "pod-2"): self.assertEqual(self.run_container(containers["capability-node"], uid, rewrite).returncode, 0) - (self.root / ".incoming" / f"{DIGEST_A}.pod-1.capability-node" / "marker").write_text("pod-1") + (self.root / ".incoming" / f"{KEY_A}.pod-1.capability-node" / "marker").write_text("pod-1") for uid in ("pod-1", "pod-2"): published = self.run_container(containers["capability-store-publish"], uid, rewrite) self.assertEqual(published.returncode, 0, published.stderr) - self.assertEqual((self.root / DIGEST_A / "marker").read_text(), "pod-1") - self.assertFalse((self.root / DIGEST_A / f"{DIGEST_A}.pod-2.capability-node").exists()) + self.assertEqual((self.root / KEY_A / "marker").read_text(), "pod-1") + self.assertFalse((self.root / KEY_A / f"{KEY_A}.pod-2.capability-node").exists()) self.assertEqual(list((self.root / ".incoming").iterdir()), []) + def test_one_image_used_for_a_tool_cache_and_a_sysroot_publishes_each_payload_under_its_own_entry(self) -> None: + image = f"ghcr.io/example/both:1@sha256:{DIGEST_A}" + spec = store_pod(DEFAULT_VALUES, ["tools", "libraries"], {"tools": {"image": image}, "libraries": {"sysroot_image": image}})["values"]["template"]["spec"] + containers = {container["name"]: container for container in spec["initContainers"]} + toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) + sysroot = self.payload("sysroot", {"lib/libmpi.so": "library"}) + rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache), "/sysroot": str(sysroot)} + for name in ("capability-tools", "capability-libraries", "capability-store-publish"): + completed = self.run_container(containers[name], "pod-1", rewrite) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(self.layout(self.root / f"toolcache-{DIGEST_A}"), self.layout(toolcache)) + self.assertEqual(self.layout(self.root / f"sysroot-{DIGEST_A}"), self.layout(sysroot)) + def layout(self, directory: Path) -> dict[str, str]: """Return every path under directory with what it holds (a file's content, a link's target, a directory's marker), so two trees compare by layout and content.""" return {str(path.relative_to(directory)): path.readlink().as_posix() if path.is_symlink() else path.read_text() if path.is_file() else "" for path in sorted(directory.rglob("*"))} @@ -964,11 +1002,11 @@ def test_a_copy_is_flushed_to_disk_before_it_is_renamed_into_the_store(self) -> containers = self.containers(["node"]) rewrite = {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)} self.assertEqual(self.run_container(containers["capability-node"], "pod-1", rewrite).returncode, 0) - env = {**self.fake_sync(0), "SYNC_INCOMING": str(self.root / ".incoming" / f"{DIGEST_A}.pod-1.capability-node"), "SYNC_ENTRY": str(self.root / DIGEST_A)} + env = {**self.fake_sync(0), "SYNC_INCOMING": str(self.root / ".incoming" / f"{KEY_A}.pod-1.capability-node"), "SYNC_ENTRY": str(self.root / KEY_A)} published = self.run_with_env(containers["capability-store-publish"], "pod-1", rewrite, env) self.assertEqual(published.returncode, 0, published.stderr) self.assertEqual((self.work / "sync.log").read_text(), "incoming unpublished\n") - self.assertEqual((self.root / DIGEST_A / "node/22.0.0/x64/bin/node").read_text(), "binary") + self.assertEqual((self.root / KEY_A / "node/22.0.0/x64/bin/node").read_text(), "binary") def test_a_copy_that_cannot_be_flushed_is_not_published(self) -> None: toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary"}) @@ -978,10 +1016,10 @@ def test_a_copy_that_cannot_be_flushed_is_not_published(self) -> None: env = {**self.fake_sync(1), "SYNC_INCOMING": "unused", "SYNC_ENTRY": "unused"} published = self.run_with_env(containers["capability-store-publish"], "pod-1", rewrite, env) self.assertNotEqual(published.returncode, 0) - self.assertFalse((self.root / DIGEST_A).exists()) + self.assertFalse((self.root / KEY_A).exists()) def test_a_merged_sysroot_is_flushed_to_disk_before_it_is_renamed_into_the_store(self) -> None: - for digest, files in ((DIGEST_B, {"lib/libmpi.so": "mpi"}), (DIGEST_C, {"lib/libpq.so": "pq"})): + for digest, files in ((KEY_B, {"lib/libmpi.so": "mpi"}), (KEY_C, {"lib/libpq.so": "pq"})): for relative, content in files.items(): (self.root / digest / relative).parent.mkdir(parents=True, exist_ok=True) (self.root / digest / relative).write_text(content) @@ -1015,18 +1053,18 @@ def test_a_capability_copy_rerun_after_a_partial_copy_publishes_exactly_the_payl toolcache = self.payload("toolcache", {"node/22.0.0/x64/bin/node": "binary", "node/22.0.0/x64.complete": "", "go/pkg/mod/example@v1/file": "module"}) self.make_read_only(toolcache / "go") containers = self.containers(["node"]) - self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["capability-node"], containers["capability-store-publish"], DIGEST_A, toolcache, {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)}) + self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["capability-node"], containers["capability-store-publish"], KEY_A, toolcache, {"/opt/capability-store": str(self.root), "/toolcache": str(toolcache)}) def test_the_dind_externals_copy_rerun_after_a_partial_copy_publishes_exactly_the_externals_layout(self) -> None: externals = self.payload("externals", {"node20/bin/node": "binary", "node20/lib/libnode.so": "library", "node20/read-only/file": "content"}) self.make_read_only(externals / "node20/read-only") spec = arc.arc_capability_store_externals(DIND_VALUES, STORE, 1001, 1001)["values"]["template"]["spec"] containers = {container["name"]: container for container in spec["initContainers"]} - self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["init-dind-externals"], containers["capability-store-publish-externals"], DIGEST_RUNNER, externals, {"/opt/capability-store": str(self.root), "/home/runner/externals": str(externals)}) + self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["init-dind-externals"], containers["capability-store-publish-externals"], KEY_RUNNER, externals, {"/opt/capability-store": str(self.root), "/home/runner/externals": str(externals)}) def test_a_sysroot_merge_rerun_after_a_partial_merge_publishes_only_the_libraries(self) -> None: - (self.root / DIGEST_B / "lib").mkdir(parents=True) - (self.root / DIGEST_B / "lib/libmpi.so").write_text("mpi") + (self.root / KEY_B / "lib").mkdir(parents=True) + (self.root / KEY_B / "lib/libmpi.so").write_text("mpi") merge = self.containers(["mpi"])["capability-store-sysroot"] merged = merge["command"][5] debris = self.root / ".incoming" / f"{merged}.pod-1.merge" @@ -1040,9 +1078,9 @@ def test_a_sysroot_merge_rerun_after_a_partial_merge_publishes_only_the_librarie self.assertEqual(self.layout(self.root / merged), {"lib": "", "lib/libmpi.so": "mpi"}) def test_a_sysroot_merge_that_loses_the_race_deletes_its_copy_even_when_it_holds_read_only_directories(self) -> None: - (self.root / DIGEST_B / "lib/private").mkdir(parents=True) - (self.root / DIGEST_B / "lib/private/libmpi.so").write_text("mpi") - self.make_read_only(self.root / DIGEST_B) + (self.root / KEY_B / "lib/private").mkdir(parents=True) + (self.root / KEY_B / "lib/private/libmpi.so").write_text("mpi") + self.make_read_only(self.root / KEY_B) merge = self.containers(["mpi"])["capability-store-sysroot"] merged = merge["command"][5] # Another pod publishes the merged entry while this pod's sync runs, as two pods starting together on a node without it would. @@ -1067,7 +1105,7 @@ def test_of_two_pods_publishing_one_image_the_loser_deletes_its_copy_even_when_i for uid in ("pod-1", "pod-2"): published = self.run_container(containers["capability-store-publish"], uid, rewrite) self.assertEqual(published.returncode, 0, published.stderr) - self.assertEqual(self.layout(self.root / DIGEST_A), self.layout(toolcache)) + self.assertEqual(self.layout(self.root / KEY_A), self.layout(toolcache)) self.assertEqual(list((self.root / ".incoming").iterdir()), []) def test_a_payload_whose_root_directory_is_read_only_is_copied_and_published_with_the_source_layout(self) -> None: @@ -1077,8 +1115,8 @@ def test_a_payload_whose_root_directory_is_read_only_is_copied_and_published_wit for source in (toolcache, externals): self.make_read_only(source) for copy_name, publish_name, key, source, mount in ( - ("capability-node", "capability-store-publish", DIGEST_A, toolcache, "/toolcache"), - ("init-dind-externals", "capability-store-publish-externals", DIGEST_RUNNER, externals, "/home/runner/externals"), + ("capability-node", "capability-store-publish", KEY_A, toolcache, "/toolcache"), + ("init-dind-externals", "capability-store-publish-externals", KEY_RUNNER, externals, "/home/runner/externals"), ): with self.subTest(copy=copy_name): spec = arc.arc_capability_store_externals(store_pod(DIND_VALUES, ["node"])["values"], STORE, 1001, 1001)["values"]["template"]["spec"] @@ -1101,7 +1139,7 @@ def test_a_rename_that_fails_for_another_reason_than_a_lost_race_reports_the_err self.root.chmod(0o555) published = self.run_container(containers["capability-store-publish"], "pod-1", rewrite) self.assertNotEqual(published.returncode, 0) - self.assertIn(f"{DIGEST_A} is not in", published.stderr) + self.assertIn(f"{KEY_A} is not in", published.stderr) self.assertIn("Permission denied", published.stderr) def test_a_lost_publish_race_is_not_an_error_and_prints_nothing(self) -> None: @@ -1116,7 +1154,7 @@ def test_a_lost_publish_race_is_not_an_error_and_prints_nothing(self) -> None: def test_a_sysroot_merged_from_entries_whose_root_is_read_only_is_published(self) -> None: # The merge's destination takes the mode of the first entry's root (cp -a), and renaming it into the store needs write permission on it. - for digest, files in ((DIGEST_B, {"lib/libmpi.so": "mpi"}), (DIGEST_C, {"lib/libpq.so": "pq"})): + for digest, files in ((KEY_B, {"lib/libmpi.so": "mpi"}), (KEY_C, {"lib/libpq.so": "pq"})): for relative, content in files.items(): (self.root / digest / relative).parent.mkdir(parents=True, exist_ok=True) (self.root / digest / relative).write_text(content) @@ -1130,11 +1168,11 @@ def test_a_sysroot_merged_from_entries_whose_root_is_read_only_is_published(self def store_entry_with_read_only_directories(self) -> tuple[Path, str]: """Publish a system library's entry whose files are mode 0444 and whose directories are read-only, and return the file and the merged sysroot's entry name for it.""" - library = self.root / DIGEST_B / "lib/libmpi.so" + library = self.root / KEY_B / "lib/libmpi.so" library.parent.mkdir(parents=True) library.write_text("mpi") library.chmod(0o444) - self.make_read_only(self.root / DIGEST_B) + self.make_read_only(self.root / KEY_B) return library, self.containers(["mpi"])["capability-store-sysroot"]["command"][5] def test_a_merge_that_loses_the_race_leaves_the_modes_of_the_entries_files_alone(self) -> None: @@ -1157,7 +1195,7 @@ def test_a_merge_rerun_after_a_partial_merge_leaves_the_modes_of_the_entries_fil # What a merge killed after its links and before its rename leaves: hard links into the entry under directories without owner write permission. debris = self.root / ".incoming" / f"{merged}.pod-1.merge" debris.mkdir() - subprocess.run(["cp", "-al", f"{self.root / DIGEST_B}/.", f"{debris}/"], check=True) + subprocess.run(["cp", "-al", f"{self.root / KEY_B}/.", f"{debris}/"], check=True) completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual(stat.S_IMODE(library.stat().st_mode), 0o444) @@ -1167,10 +1205,10 @@ def test_publishing_fails_when_the_entry_is_neither_copied_nor_in_the_store(self containers = self.containers(["node"]) published = self.run_container(containers["capability-store-publish"], "pod-1", {"/opt/capability-store": str(self.root)}) self.assertNotEqual(published.returncode, 0) - self.assertIn(f"{DIGEST_A} is not in", published.stderr) + self.assertIn(f"{KEY_A} is not in", published.stderr) def test_the_merged_sysroot_links_every_library_and_a_later_one_wins_a_shared_file(self) -> None: - for digest, files in ((DIGEST_B, {"lib/libmpi.so": "mpi", "include/shared.h": "from mpi"}), (DIGEST_C, {"lib/libpq.so": "pq", "include/shared.h": "from libpq"})): + for digest, files in ((KEY_B, {"lib/libmpi.so": "mpi", "include/shared.h": "from mpi"}), (KEY_C, {"lib/libpq.so": "pq", "include/shared.h": "from libpq"})): for relative, content in files.items(): (self.root / digest / relative).parent.mkdir(parents=True, exist_ok=True) (self.root / digest / relative).write_text(content) @@ -1179,14 +1217,14 @@ def test_the_merged_sysroot_links_every_library_and_a_later_one_wins_a_shared_fi completed = self.run_container(merge, "pod-1", {"/opt/capability-store": str(self.root)}) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual((self.root / merged / "include/shared.h").read_text(), "from libpq") - self.assertEqual((self.root / merged / "lib/libmpi.so").stat().st_ino, (self.root / DIGEST_B / "lib/libmpi.so").stat().st_ino) + self.assertEqual((self.root / merged / "lib/libmpi.so").stat().st_ino, (self.root / KEY_B / "lib/libmpi.so").stat().st_ino) # A second pod finds it published and does nothing. self.assertEqual(self.run_container(merge, "pod-2", {"/opt/capability-store": str(self.root)}).returncode, 0) self.assertEqual(list((self.root / ".incoming").iterdir()), []) def tool_tree(self, files: dict[str, str] | None = None) -> Path: """Fill the store's entry for the node capability with a tool tree shaped like the Actions tool cache's Node.js, as a store entry is after publishing, and return its directory.""" - arch = self.root / DIGEST_A / "node/22.0.0/x64" + arch = self.root / KEY_A / "node/22.0.0/x64" for relative, content in (files or {"bin/node": "binary", "lib/node_modules/npm/bin/npm-cli.js": "npm", "lib/node_modules/npm/package.json": "{}", "include/node/node.h": "header", "dir with space/a file": "spaced", ".hidden/file": "hidden"}).items(): (arch / relative).parent.mkdir(parents=True, exist_ok=True) (arch / relative).write_text(content) @@ -1279,16 +1317,16 @@ def test_linking_the_tool_cache_again_leaves_the_same_tree(self) -> None: def test_every_tool_version_and_architecture_of_every_entry_is_linked_beside_the_others(self) -> None: self.tool_tree() - other = self.root / DIGEST_B / "python/3.12.0/x64" + other = self.root / KEY_B / "python/3.12.0/x64" (other / "bin").mkdir(parents=True) (other / "bin/python3").write_text("python") (other.parent / "x64.complete").write_text("") - (self.root / DIGEST_A / "node/22.0.0/arm64/bin").mkdir(parents=True) - (self.root / DIGEST_A / "node/22.0.0/arm64/bin/node").write_text("arm") + (self.root / KEY_A / "node/22.0.0/arm64/bin").mkdir(parents=True) + (self.root / KEY_A / "node/22.0.0/arm64/bin/node").write_text("arm") cache = self.work / "toolcache-pod" cache.mkdir() link = dict(self.containers(["node"])["capability-store-tool-cache"]) - link["command"] = [*link["command"], DIGEST_B] + link["command"] = [*link["command"], KEY_B] completed = self.run_container(link, "pod-1", {"/opt/capability-store": str(self.root), TOOL_CACHE: str(cache)}) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertEqual((cache / "python/3.12.0/x64/bin/python3").read_text(), "python") @@ -1355,7 +1393,7 @@ def test_linking_again_replaces_a_completeness_marker_that_is_read_only(self) -> self.root = self.work / "store" (self.root / ".incoming").mkdir(parents=True) self.tool_tree() - (self.root / DIGEST_A / "node/22.0.0/x64.complete").chmod(store_mode) + (self.root / KEY_A / "node/22.0.0/x64.complete").chmod(store_mode) cache = self.work / "toolcache-pod" cache.mkdir() self.link_tool_cache(cache) From 78772a77fef699a0ec2ee031b08c98d523ba1ed0 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 12:59:35 +0100 Subject: [PATCH 29/43] fix(arc): copy an aliased tool-cache arch directory as the link instead of linking to the store's An image that aliases arch names (x64 and amd64) stores one of them as a symlink to a directory. The tool cache script ran cp -as on it, which made the pod's arch a link to the store's link, and the chmod -R after it followed that command-line link into the read-only store and failed, so capability-store-tool-cache exited 1 on every pod. The per-pod cp -R handled the layout because the relative link resolved inside the pod. An arch that is a symlink is now copied with cp -P, as links inside a tree already are, and gets none of the commands that would resolve into the store. Its target names the real directory the pod's tree holds beside it, which the same loop links, and its completeness marker is still copied. An absolute target is kept as written. --- plugins/filter/arc.py | 7 +++- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 67 +++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 3 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 9206d4c..2c9fd28 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1268,15 +1268,18 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing${moved:+: $moved}" >&2; exit 1; }' ) -# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. The .complete marker, which is how the Actions tool cache finds a tool, is copied as well, with cp -f since it keeps the store file's mode and a rerun over a read-only copy must still replace it. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, cp -f, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. +# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. An that is itself a symbolic link (an image aliasing x64 and amd64) is copied as the link, as an entry inside the tree is, and never linked to the store's link: the link's relative target then names the real directory the pod's tree holds beside it, which the same loop links, and no command ever runs on a path that resolves into the store (cp -as of a link to a link would make the pod's a link to the store's, which a chmod -R follows from the command line into the read-only store and fails on). The .complete marker, which is how the Actions tool cache finds a tool, is copied as well, for an alias too, with cp -f since it keeps the store file's mode and a rerun over a read-only copy must still replace it. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, cp -f, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. _TOOL_CACHE_COPY_SCRIPT = 'source=$1; destination=$2; shift 2; for entry; do target=$destination/${entry#"$source"/}; rm -f "$target" && cp -P "$entry" "$target" || exit 1; done' _TOOL_CACHE_LINK_SCRIPT = ( 'root=$1; cache=$2; shift 2; for key in "$@"; do for arch in "$root/$key"/*/*/*; do' ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"}; parent="$cache/${relative%/*}"; tree="$cache/$relative";' - ' mkdir -p "$parent" || exit 1; chmod -R u+rwx "$tree" 2>/dev/null;' + ' mkdir -p "$parent" || exit 1;' + ' if [ -L "$arch" ]; then rm -f "$tree" && cp -P "$arch" "$tree" || exit 1; else' + ' chmod -R u+rwx "$tree" 2>/dev/null;' ' cp -asf "$arch" "$parent/" && chmod -R u+rwx "$tree" || exit 1;' " find \"$arch\" -type l -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" " [ ! -d \"$arch/bin\" ] || find \"$arch/bin\" -maxdepth 1 -type f -exec sh -c '" + _TOOL_CACHE_COPY_SCRIPT + "' copy-from-capability-store \"$arch\" \"$tree\" {} + || exit 1;" + ' fi;' ' [ ! -e "$arch.complete" ] || cp -f "$arch.complete" "$cache/$relative.complete" || exit 1;' ' done; done' ) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 9af0d4c..8a021a2 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -317,7 +317,7 @@ An entry is named for its image's digest and the kind of payload copied from it - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. - `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `merged-sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. -- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. Everything else stays a link, so a pod still writes only the launchers' bytes. +- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. An `` directory that is itself a symlink (an image aliasing `x64` and `amd64`) is copied as the link too, so it names the real directory the pod's tree holds beside it, and its `.complete` marker is copied like any other. Everything else stays a link, so a pod still writes only the launchers' bytes. What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 428fc69..d38e3ce 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -1306,6 +1306,73 @@ def test_a_link_in_the_store_is_copied_so_its_relative_target_resolves_inside_th self.assertEqual((tree / "libdir").resolve(), (tree / "lib").resolve()) self.assertEqual((tree / "libdir/node_modules/npm/package.json").read_text(), "{}") + def aliased_tool_tree(self, alias: str) -> Path: + """Fill the store's entry for the node capability with a tool whose directories are an alias of a real one (an image aliasing x64 and amd64), with the alias pointing at target, and return the version directory.""" + version = self.root / KEY_A / "node/22.0.0" + (version / "real/bin").mkdir(parents=True) + (version / "real/bin/node").write_text("binary") + (version / "real/README").write_text("readme") + (version / "x64").symlink_to(alias) + (version / "x64.complete").write_text("") + return version + + def test_an_arch_directory_that_is_a_link_is_copied_as_the_link_and_the_store_is_never_written_through_it(self) -> None: + version = self.aliased_tool_tree("real") + before = self.snapshot(self.root) + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0" + # The alias is the pod's own link, naming the pod's own directory beside it, and the marker for it is copied. + self.assertEqual((tree / "x64").readlink(), Path("real")) + self.assertEqual((tree / "x64/README").resolve(), (tree / "real/README").resolve()) + self.assertEqual((tree / "x64/README").read_text(), "readme") + self.assertTrue((tree / "x64.complete").is_file() and not (tree / "x64.complete").is_symlink()) + self.assertTrue((tree / "real").is_dir() and not (tree / "real").is_symlink()) + self.assertEqual((tree / "real/bin/node").read_text(), "binary") + # A job can change the tool through either name, and the store is unchanged. + (tree / "x64/created").write_text("created") + self.assertEqual((tree / "real/created").read_text(), "created") + self.assertEqual(self.snapshot(self.root), before) + self.assertFalse((version / "real/created").exists()) + + def test_linking_again_over_an_aliased_tool_tree_leaves_the_same_tree(self) -> None: + self.aliased_tool_tree("real") + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + first = self.snapshot(cache) + self.link_tool_cache(cache) + self.assertEqual(self.snapshot(cache), first) + + def test_an_alias_of_an_alias_and_each_alias_marker_are_copied(self) -> None: + version = self.aliased_tool_tree("real") + (version / "amd64").symlink_to("x64") + (version / "amd64.complete").write_text("") + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + tree = cache / "node/22.0.0" + self.assertEqual((tree / "amd64").readlink(), Path("x64")) + self.assertEqual((tree / "amd64/bin/node").read_text(), "binary") + self.assertTrue((tree / "amd64.complete").is_file()) + + def test_an_absolute_link_for_an_arch_directory_is_copied_as_written_and_not_into_the_store(self) -> None: + cache = self.work / "toolcache-pod" + cache.mkdir() + # One that names a path in the pod's own tool cache, as an image built for that path writes it, and one that names the store itself. + for target in (cache / "node/22.0.0/real", self.root / KEY_A / "node/22.0.0/real"): + with self.subTest(target=str(target)): + self.aliased_tool_tree(str(target)) + before = self.snapshot(self.root) + self.link_tool_cache(cache) + self.assertEqual((cache / "node/22.0.0/x64").readlink(), target) + self.assertEqual((cache / "node/22.0.0/x64/bin/node").read_text(), "binary") + self.assertEqual(self.snapshot(self.root), before) + shutil.rmtree(self.root / KEY_A) + shutil.rmtree(cache) + cache.mkdir() + def test_linking_the_tool_cache_again_leaves_the_same_tree(self) -> None: self.tool_tree() cache = self.work / "toolcache-pod" From 73a503aa7f9002078ff0970879e5614f73e46cf2 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 13:20:13 +0100 Subject: [PATCH 30/43] fix(arc): check the capability store's settings in validation as well as at render The store's own checks ran only when a profile was rendered, which happens after the controller, the pruner, the registry cache and earlier profiles have been applied, while validate.yml is documented to fail before anything changes. A capability image without a digest, a store path inside the tool cache and an unpinned runner image for the dind externals copy all passed validation and failed part-way through an install. arc_capability_errors now requires a digest on every image and sysroot_image of a profile with capability_store, and the new arc_capability_store_errors, called from validate.yml, runs the store path overlap checks against the tool cache, sysroot, hooks and node cache paths, and the digest requirement on the runner image for a dind profile that gets its externals copy from the registry cache. The render path calls the same helpers, so each message has one implementation. A values file's own externals copy is still checked at render time, since its content is not known before it is rendered. --- plugins/filter/arc.py | 83 ++++++++++++++++++---- roles/github_runner_arc/README.md | 4 +- roles/github_runner_arc/tasks/validate.yml | 14 ++++ tests/unit/test_capabilities.py | 78 ++++++++++++++++++++ tests/unit/test_node_cache_pruner_role.py | 37 ++++++++++ 5 files changed, 199 insertions(+), 17 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 2c9fd28..603dcf7 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -538,7 +538,7 @@ def arc_runner_placement(profile: Mapping[str, Any], eligibility: Mapping[str, A return {"nodeSelector": dict(profile["node_selector"]), "affinity": {"nodeAffinity": node_affinity}, "tolerations": own_tolerations + burst["tolerations"]} -def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, errors: list[str]) -> dict[str, Any] | None: +def _expand_profile(org_name: str, runner_image: str, app_secret: str, index: int, profile: Any, errors: list[str]) -> dict[str, Any] | None: """Describe one scale-set profile of an org, or record why it cannot be described.""" where = f"{org_name} scale_set_profiles[{index}]" if not isinstance(profile, Mapping): @@ -591,6 +591,7 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er # namespace/release joined for the autoscaler's own AUTOSCALER_TARGETS list (see scripts/autoscaler.sh): both are validated DNS labels below, so neither can itself contain '/', keeping each token unambiguous. "target": f"{namespace}/{release}", "app_secret": app_secret, + "runner_image": runner_image, "scale_set_labels": _scale_set_labels(profile, f"{org_lower}-runners", where, errors), "values_file": _text(profile, "values_file"), "node_selector": dict(node_selector), @@ -623,7 +624,7 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. Returns: - A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, target (namespace/release, joined), app_secret, scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). + A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, target (namespace/release, joined), app_secret, runner_image (its org's image), scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). """ errors: list[str] = [] profiles: list[dict[str, Any]] = [] @@ -656,7 +657,7 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, continue app_secret = _optional_name(org, "app_secret_name", org_name, errors) or f"{org_name.lower()}-github-app" for profile_index, profile in enumerate(org_profiles): - expanded = _expand_profile(org_name, app_secret, profile_index, profile, errors) + expanded = _expand_profile(org_name, _text(org, "image"), app_secret, profile_index, profile, errors) if expanded is not None: profiles.append(expanded) namespaces: dict[str, str] = {} @@ -844,7 +845,7 @@ def arc_capability_errors(profiles: Sequence[Mapping[str, Any]], catalogue: Any) profiles: expanded profiles from arc_profiles. catalogue: github_runner_arc_capabilities, a mapping of capability name to one of {image, path?} (a tool-cache capability: a pinned image holding a payload in the tool-cache layout under /toolcache, and optionally a /[/] to put on PATH at job start), {sysroot_image, env?} (a system-library capability: a pinned image holding a relocatable prefix under /sysroot, and optionally extra variables its libraries need, such as a relocation variable) or {job_started_hook} (a hook capability: a script the runner runs before each job). Returns: - The problems found, empty when there are none: invalid entries, a profile naming an undefined capability, and two of a profile's system-library capabilities setting one variable to different values. + The problems found, empty when there are none: invalid entries, a profile naming an undefined capability, two of a profile's system-library capabilities setting one variable to different values, and a profile with capability_store whose image or sysroot_image is not pinned by digest (arc_capability_store_errors checks the store's settings against the role's paths). """ if not isinstance(catalogue, Mapping): return ["github_runner_arc_capabilities must be a mapping of capability name to capability"] @@ -852,6 +853,8 @@ def arc_capability_errors(profiles: Sequence[Mapping[str, Any]], catalogue: Any) for profile in profiles: errors.extend(f"{profile['label']}: capability '{name}' is not defined in github_runner_arc_capabilities" for name in profile["capabilities"] if name not in catalogue) errors.extend(_sysroot_env_conflicts(profile["label"], profile["capabilities"], catalogue)) + if profile["capability_store"]: + errors.extend(f"{profile['label']}: {message}" for message in _capability_store_digest_errors(profile["capabilities"], catalogue)) return errors @@ -1190,6 +1193,36 @@ def arc_capability_store_key(image: str, kind: str) -> str | None: return f"{kind}-{match.group(1)}" if match else None +def _capability_store_digest_errors(names: Sequence[str], catalogue: Mapping[str, Any]) -> list[str]: + """Return one message for each of names that is a tool-cache or system-library capability whose image is not pinned by digest, which the capability store needs because it names the entry for the digest. A name the catalogue lacks, or an entry that is not a mapping or has no string image, is arc_capability_errors' to report.""" + errors: list[str] = [] + for name in names: + capability = catalogue.get(name) + if not isinstance(capability, Mapping): + continue + image = capability.get("image") if capability.get("image") is not None else capability.get("sysroot_image") + if isinstance(image, str) and _IMAGE_DIGEST.search(image) is None: + errors.append(f"capability {name}'s image must be pinned by digest (name@sha256:<64 hex digits>) for the capability store, since its store entry is named for the digest") + return errors + + +def _capability_store_uses_entries(names: Sequence[str], catalogue: Mapping[str, Any]) -> bool: + """Return whether any of names is a tool-cache or system-library capability, the ones the store holds a payload for.""" + return any(isinstance(catalogue.get(name), Mapping) and (catalogue[name].get("image") is not None or catalogue[name].get("sysroot_image") is not None) for name in names) + + +def _capability_store_mount_errors(store_path: Any, others: Mapping[str, str]) -> list[str]: + """Return what is wrong with the store's mount path on the runner container: the rules of arc_capability_store_path_errors, or else that it overlaps one of the other mounted directories in others.""" + return arc_capability_store_path_errors(store_path) or _overlap_errors("capability store path", store_path, others) + + +def _capability_store_externals_image_errors(image: Any) -> list[str]: + """Return what is wrong with the image the dind sidecar's externals are copied from, which the store names its entry for: empty when it is pinned by digest.""" + if arc_capability_store_key(str(image), _STORE_KIND_EXTERNALS) is None: + return [f"the capability store needs {_DIND_EXTERNALS_INIT}'s image pinned by digest (name@sha256:<64 hex digits>), since its store entry is named for the digest (got {image!r})"] + return [] + + def _capability_store_merged_key(keys: Sequence[str]) -> str: """Return the store entry name of the sysroot merged from keys in their order: a hash of the ordered keys, so a different set or order of system libraries is a different entry, since a later one's file overwrites an earlier one's. Its prefix is not a payload kind's, so it cannot be the name of an image's entry.""" return _CAPABILITY_STORE_MERGED_PREFIX + hashlib.sha256(" ".join(keys).encode()).hexdigest() @@ -1344,6 +1377,28 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s return containers, keys + ([merged] if merged is not None else []), merged +def arc_capability_store_errors(profiles: Sequence[Mapping[str, Any]], catalogue: Mapping[str, Any], store_path: Any, tool_cache_path: str, sysroot_path: str, hooks_dir: str, node_cache_path: str, registry_cache_enabled: bool) -> list[str]: + """Check what a profile with capability_store needs of the role's settings, before anything is installed: the checks arc_capability_pod and arc_capability_store_externals make when they render, so that validation fails on them and not the install, which has by then applied the controller, the pruner and earlier profiles. + + Args: + profiles: expanded profiles from arc_profiles. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. store_path: where the store is mounted on the runner container. tool_cache_path, sysroot_path, hooks_dir, node_cache_path: where the role mounts the tool cache, the sysroot, the job-started hooks and the node cache, which the store's path may not equal, contain or sit inside. registry_cache_enabled: whether the role writes the registry cache into the values, which writes the dind sidecar out with its externals copy for a profile with container_mode dind. + + Returns: + The problems found, empty when there are none: for a profile that holds a tool-cache or system-library payload in the store, a store path that is not a directory of its own, and, for a profile in dind mode with the registry cache, a runner image not pinned by digest, since the externals copy's entry is named for it. A values file's own externals copy is checked when it is rendered, since its content is not known here. + """ + errors: list[str] = [] + for profile in profiles: + if not profile["capability_store"]: + continue + messages: list[str] = [] + if _capability_store_uses_entries(profile["capabilities"], catalogue): + messages.extend(_capability_store_mount_errors(store_path, {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir, "node cache path": node_cache_path})) + if registry_cache_enabled and not profile["values_file"] and profile["settings"].get("container_mode") == "dind": + messages.extend(_capability_store_externals_image_errors(profile["runner_image"])) + errors.extend(f"{profile['label']}: {message}" for message in messages) + return errors + + def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str, Any] | None, runner_user: int, runner_group: int) -> dict[str, Any]: """Move a scale set's copy of the runner's externals for the dind sidecar into the capability store. @@ -1364,9 +1419,11 @@ def arc_capability_store_externals(values: Mapping[str, Any], store: Mapping[str copier = next((container for container in init_containers if container.get("name") == _DIND_EXTERNALS_INIT), None) if copier is None: return {"errors": [], "values": values} - key = arc_capability_store_key(str(copier.get("image", "")), _STORE_KIND_EXTERNALS) - if key is None: - return {"errors": [f"the capability store needs {_DIND_EXTERNALS_INIT}'s image pinned by digest (name@sha256:<64 hex digits>), since its store entry is named for the digest (got {copier.get('image')!r})"], "values": values} + image_errors = _capability_store_externals_image_errors(copier.get("image", "")) + if image_errors: + return {"errors": image_errors, "values": values} + key = arc_capability_store_key(copier["image"], _STORE_KIND_EXTERNALS) + assert key is not None index = init_containers.index(copier) replacement = [ _capability_store_copy_container(_DIND_EXTERNALS_INIT, copier["image"], _RUNNER_EXTERNALS_DIR, key, store, runner_user, runner_group), @@ -1417,14 +1474,9 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu errors.extend(_sysroot_path_errors(sysroot_path, {"tool cache path": tool_cache_path, "job-started hooks directory": hooks_dir})) if node_cache_capabilities: errors.extend(_node_cache_path_errors(node_cache_path, node_cache_host_path, {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir})) - if store is not None and (tool_capabilities or sysroot_capabilities): - store_path_errors = arc_capability_store_path_errors(store["path"]) - errors.extend(store_path_errors or _overlap_errors("capability store path", store["path"], {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir, "node cache path": node_cache_path})) - errors.extend( - f"capability {name}'s image must be pinned by digest (name@sha256:<64 hex digits>) for the capability store, since its store entry is named for the digest" - for name, capability in [*tool_capabilities, *sysroot_capabilities] - if arc_capability_store_key(capability.get("image") or capability["sysroot_image"], _STORE_KIND_TOOL_CACHE) is None - ) + if store is not None and _capability_store_uses_entries(names, catalogue): + errors.extend(_capability_store_mount_errors(store["path"], {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir, "node cache path": node_cache_path})) + errors.extend(_capability_store_digest_errors(names, catalogue)) tool_paths = [capability["path"] for _, capability in tool_capabilities if capability.get("path") is not None] hooks: dict[str, str] = {} if tool_paths: @@ -1850,6 +1902,7 @@ def filters(self) -> dict[str, Any]: "arc_runner_placement": arc_runner_placement, "arc_capability_errors": arc_capability_errors, "arc_capability_pod": arc_capability_pod, + "arc_capability_store_errors": arc_capability_store_errors, "arc_capability_store_externals": arc_capability_store_externals, "arc_capability_store_host_path_errors": arc_capability_store_host_path_errors, "arc_capability_store_path_errors": arc_capability_store_path_errors, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 8a021a2..73206a1 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -6,7 +6,7 @@ It runs in two ways. Inside `playbooks/site.yml`, against a k3s server host that ## Validation -`tasks/validate.yml` checks every input that needs neither secrets nor a cluster (a measured sizing's node figures are read later, when the role installs the profile): each org's fields, profile names (derived or overridden) that would collide or make invalid Kubernetes names, two of an org's profiles sharing a release name, a `max_runners` that is not a whole number, the controller's release name and namespace, the same org configured on two hosts of one cluster, a scale set registered from two clusters (see below), more than one autoscaled profile, values files that do not exist, the autoscaler's pool settings, sizing inputs, burst settings, and the controller, probe and node label settings. Both playbooks run it in a play of its own before anything changes; the role runs it itself when included some other way. It also bootstraps the heartbeat gist when `github_runner_arc_heartbeat_bootstrap_gist` is on and no host in the inventory has one: it creates the gist with the control node's `gh` session and stops, printing the id to record. +`tasks/validate.yml` checks every input that needs neither secrets nor a cluster (a measured sizing's node figures are read later, when the role installs the profile): each org's fields, profile names (derived or overridden) that would collide or make invalid Kubernetes names, two of an org's profiles sharing a release name, a `max_runners` that is not a whole number, the controller's release name and namespace, the same org configured on two hosts of one cluster, a scale set registered from two clusters (see below), more than one autoscaled profile, values files that do not exist, the autoscaler's pool settings, sizing inputs, burst settings, and the controller, probe and node label settings, plus what a profile with the capability store needs of the role's paths and images. Both playbooks run it in a play of its own before anything changes; the role runs it itself when included some other way. It also bootstraps the heartbeat gist when `github_runner_arc_heartbeat_bootstrap_gist` is on and no host in the inventory has one: it creates the gist with the control node's `gh` session and stops, printing the id to record. The checks that span hosts compare the hosts of one cluster: the hosts in this host's cluster group (`github_runner_cluster_group`), which share a controller, its namespaces and the autoscaled pool, so an org is installed into a cluster from one host. A host of another cluster group may install the same org, for example while its scale sets move one at a time to a second cluster beside the first, but not the same scale set, since GitHub gives an org's scale set name to one controller at a time. A host outside its cluster group, such as the control node running `playbooks/arc.yml`, counts every host in the inventory as one cluster. @@ -311,7 +311,7 @@ Each tool-cache and system-library capability adds its copy to every pod's start Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied once per node into `github_runner_arc_capability_store_host_path` and the pods use that copy. -An entry is named for its image's digest and the kind of payload copied from it (`toolcache-`, `sysroot-`, `externals-`), so one image used for two payloads, such as a runner image that is also a tool image, is two entries that cannot be taken for each other, and every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: +An entry is named for its image's digest and the kind of payload copied from it (`toolcache-`, `sysroot-`, `externals-`), so one image used for two payloads, such as a runner image that is also a tool image, is two entries that cannot be taken for each other, and every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. `tasks/validate.yml` refuses a profile with `capability_store: true` whose capability image is not pinned by digest, whose store path overlaps the tool cache, sysroot, hooks or node cache directory, or, in dind mode with the registry cache and no values file, whose organisation's runner image is not pinned by digest (a values file's own externals copy is checked when it is rendered, since its content is not known earlier), so the failure comes before any install step. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index 56227db..0fa48c4 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -29,6 +29,20 @@ github_runner_arc_capability_problems: "{{ github_runner_arc_local.profiles | exadev.github_runner.arc_capability_errors(github_runner_arc_capabilities) }}" when: github_runner_arc_capability_problems | length > 0 +# What a profile with capability_store needs of this host's settings, which the same filters check at render time (install_scale_set_profile.yml), after the controller, the pruner and earlier profiles have been applied. The catalogue is valid here, which these checks read. +- name: Fail when a profile's capability store settings do not fit the role's mount paths and images + ansible.builtin.fail: + msg: "{{ github_runner_arc_capability_store_problems }}" + vars: + github_runner_arc_capability_store_problems: >- + {{ + github_runner_arc_local.profiles + | exadev.github_runner.arc_capability_store_errors( + github_runner_arc_capabilities, github_runner_arc_capability_store_path, github_runner_arc_tool_cache_path, github_runner_arc_sysroot_path, + github_runner_arc_job_started_hooks_dir, github_runner_arc_node_cache_path, github_runner_arc_registry_cache_enabled | bool) + }} + when: github_runner_arc_capability_store_problems | length > 0 + - name: Fail on invalid node cache settings ansible.builtin.fail: msg: "{{ github_runner_arc_node_cache_problems }}" diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index d38e3ce..ffe25e6 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -868,6 +868,84 @@ def test_the_store_functions_leave_every_mount_with_its_volume_for_a_profile_wit assert_every_mount_has_its_volume(self, spec) +class CapabilityStoreValidationTest(unittest.TestCase): + """What a profile with capability_store needs is reported by validation, with the messages the render reports, so that it fails before anything is installed.""" + + UNPINNED = {"tools": {"image": "ghcr.io/example/toolcache-node:22"}, "libraries": {"sysroot_image": "ghcr.io/example/sysroot-libpq:16"}} + + def profile(self, store: bool = True, names: list[str] | None = None, **settings: Any) -> dict[str, Any]: + return expanded({"max_runners": 1, "capability_store": store, "capabilities": names or [], **settings}) + + def catalogue_errors(self, profile: dict[str, Any], catalogue: dict[str, Any]) -> list[str]: + errors: list[str] = arc.arc_capability_errors([profile], catalogue) + return errors + + def store_errors(self, profile: dict[str, Any], catalogue: dict[str, Any], store_path: str = "/opt/capability-store", registry_cache: bool = False) -> list[str]: + errors: list[str] = arc.arc_capability_store_errors([profile], catalogue, store_path, TOOL_CACHE, SYSROOT, HOOKS_DIR, NODE_CACHE, registry_cache) + return errors + + def render_errors(self, names: list[str], catalogue: dict[str, Any], store: dict[str, str] = STORE) -> list[str]: + errors: list[str] = arc.arc_capability_pod(DEFAULT_VALUES, names, catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, store)["errors"] + return errors + + def test_a_capability_image_not_pinned_by_digest_is_a_validation_error_with_the_stores_profile_and_the_renders_message(self) -> None: + for name, catalogue in (("tool", {"tools": self.UNPINNED["tools"]}), ("library", {"libraries": self.UNPINNED["libraries"]})): + with self.subTest(name): + capability = next(iter(catalogue)) + errors = self.catalogue_errors(self.profile(names=[capability]), catalogue) + self.assertEqual(errors, [f"Example: {message}" for message in self.render_errors([capability], catalogue)]) + self.assertEqual(len(errors), 1, errors) + self.assertIn("pinned by digest", errors[0]) + + def test_a_profile_without_the_store_needs_no_digest_and_a_digest_passes(self) -> None: + self.assertEqual(self.catalogue_errors(self.profile(store=False, names=["tools", "libraries"]), self.UNPINNED), []) + self.assertEqual(self.catalogue_errors(self.profile(names=["node", "mpi", "libpq"]), STORE_CATALOGUE), []) + + def test_each_unpinned_capability_of_a_profile_is_reported_once_and_hooks_need_no_digest(self) -> None: + catalogue = {**self.UNPINNED, "hook": {"job_started_hook": "true"}} + errors = self.catalogue_errors(self.profile(names=["tools", "hook", "libraries"]), catalogue) + self.assertEqual(len(errors), 2, errors) + self.assertIn("capability tools's", errors[0]) + self.assertIn("capability libraries's", errors[1]) + + def test_a_store_path_inside_the_tool_cache_is_a_validation_error_with_the_renders_message(self) -> None: + inside = f"{TOOL_CACHE}/store" + expected = self.render_errors(["node"], STORE_CATALOGUE, {**STORE, "path": inside}) + self.assertEqual(len(expected), 1, expected) + self.assertEqual(self.store_errors(self.profile(names=["node"]), STORE_CATALOGUE, inside), [f"Example: {message}" for message in expected]) + + def test_the_store_path_is_checked_against_every_directory_the_capabilities_mount(self) -> None: + for what, path in (("tool cache", f"{TOOL_CACHE}/x"), ("sysroot", SYSROOT), ("job-started hooks", f"{HOOKS_DIR}/x"), ("node cache", "/"), ("node cache", f"{NODE_CACHE}/store")): + with self.subTest(what=what, path=path): + self.assertEqual(len(self.store_errors(self.profile(names=["node"]), STORE_CATALOGUE, path)), 1) + self.assertEqual(self.store_errors(self.profile(names=["node"]), STORE_CATALOGUE), []) + + def test_a_profile_that_holds_nothing_in_the_store_or_does_not_use_it_needs_no_separate_path(self) -> None: + inside = f"{TOOL_CACHE}/store" + self.assertEqual(self.store_errors(self.profile(names=["pnpm-store"]), STORE_CATALOGUE, inside), []) + self.assertEqual(self.store_errors(self.profile(store=False, names=["node"]), STORE_CATALOGUE, inside), []) + + def dind_profile(self, image: str, **settings: Any) -> dict[str, Any]: + result = arc.arc_profiles([{"name": "Example", "image": image, "scale_set_profiles": [{"max_runners": 1, "capability_store": True, **settings}]}], require_app_id=False) + self.assertEqual(result["errors"], []) + profile: dict[str, Any] = result["profiles"][0] + return profile + + def test_an_unpinned_runner_image_is_a_validation_error_for_the_dind_externals_the_registry_cache_writes_out(self) -> None: + profile = self.dind_profile("ghcr.io/example/runner:2", container_mode="dind") + errors = self.store_errors(profile, {}, registry_cache=True) + values = {"template": {"spec": {"initContainers": [{"name": "init-dind-externals", "image": "ghcr.io/example/runner:2"}]}}} + self.assertEqual(errors, [f"Example: {message}" for message in arc.arc_capability_store_externals(values, STORE, 1001, 1001)["errors"]]) + self.assertEqual(len(errors), 1, errors) + + def test_a_digest_pinned_runner_image_or_no_registry_cache_or_no_dind_leaves_the_externals_unchecked(self) -> None: + self.assertEqual(self.store_errors(self.dind_profile(RUNNER_IMAGE, container_mode="dind"), {}, registry_cache=True), []) + self.assertEqual(self.store_errors(self.dind_profile("ghcr.io/example/runner:2", container_mode="dind"), {}, registry_cache=False), []) + self.assertEqual(self.store_errors(self.dind_profile("ghcr.io/example/runner:2"), {}, registry_cache=True), []) + # A values file's own dind wiring is not known here; the render checks its externals copy. + self.assertEqual(self.store_errors(self.dind_profile("ghcr.io/example/runner:2", values_file="values.yaml", container_mode="dind"), {}, registry_cache=True), []) + + class CapabilityStoreImageContractTest(unittest.TestCase): """The applets the store image must provide, as the defaults and the README state them, are exactly the ones the init containers that run it use.""" diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index b8be746..e30dceb 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -626,6 +626,43 @@ def test_the_task_expression_gives_the_same_answer_under_ansible(self) -> None: self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) self.assertEqual(json.loads(output.read_text()), self.problems(**role_vars)) + def store_validation(self, profile: dict[str, Any], catalogue: dict[str, Any], **role_vars: Any) -> list[str]: + """The problems the store settings task of tasks/validate.yml finds for a profile, evaluated by Ansible with the role's defaults and constants, so the order of the arguments the task passes is exercised.""" + tasks = yaml.safe_load((ROLE / "tasks" / "validate.yml").read_text()) + [task] = [task for task in tasks if task["name"] == "Fail when a profile's capability store settings do not fit the role's mount paths and images"] + expression = task["vars"]["github_runner_arc_capability_store_problems"] + with tempfile.TemporaryDirectory() as work: + template, output = Path(work) / "problems.j2", Path(work) / "problems.json" + template.write_text("{{ (" + expression.strip().removeprefix("{{").removesuffix("}}") + ") | to_json }}") + completed = ansible(KEPT, kept_template=str(template), kept_output=str(output), github_runner_arc_local={"profiles": [profile]}, github_runner_arc_capabilities=catalogue, **role_vars) + self.assertEqual(completed.returncode, 0, completed.stdout + completed.stderr) + problems: list[str] = json.loads(output.read_text()) + return problems + + def test_the_capability_store_settings_are_checked_against_the_roles_paths_before_anything_is_installed(self) -> None: + digest = "ghcr.io/example/toolcache-node:22@sha256:" + "0" * 64 + + def profile(image: str, **settings: Any) -> dict[str, Any]: + result = arc.arc_profiles([{"name": "Example", "image": image, "scale_set_profiles": [{"max_runners": 1, "capability_store": True, **settings}]}], require_app_id=False) + assert result["errors"] == [], result["errors"] + found: dict[str, Any] = result["profiles"][0] + return found + + tools = {"tools": {"image": digest}} + stored = profile("ghcr.io/example/runner:1", capabilities=["tools"]) + self.assertEqual(self.store_validation(stored, tools), []) + inside = self.store_validation(stored, tools, github_runner_arc_capability_store_path="/opt/hostedtoolcache/store") + self.assertEqual(len(inside), 1, inside) + self.assertIn("overlaps the tool cache path /opt/hostedtoolcache", inside[0]) + sysroot_inside = self.store_validation(profile("ghcr.io/example/runner:1", capabilities=["tools"]), tools, github_runner_arc_capability_store_path="/opt/sysroot") + self.assertIn("overlaps the sysroot path /opt/sysroot", sysroot_inside[0]) + dind = profile("ghcr.io/example/runner:1", container_mode="dind") + self.assertEqual(self.store_validation(dind, {}), []) + unpinned = self.store_validation(dind, {}, github_runner_arc_registry_cache_enabled=True) + self.assertEqual(len(unpinned), 1, unpinned) + self.assertIn("init-dind-externals's image pinned by digest", unpinned[0]) + self.assertEqual(self.store_validation(profile("ghcr.io/example/runner:1@sha256:" + "1" * 64, container_mode="dind"), {}, github_runner_arc_registry_cache_enabled=True), []) + def test_the_defaults_are_valid(self) -> None: self.assertEqual(self.problems(), []) From fcc47e2d232317d941abc7e8a78e28aef515c3b0 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 13:50:50 +0100 Subject: [PATCH 31/43] fix(arc): mount only the store entries a pod uses on its runner container The runner container mounted the whole store root, so a job in one profile could read every other profile's payloads on the same node. It now mounts, read-only, each tool-cache entry its pod's tool tree links into at / and the merged sysroot at the sysroot path, all as subPaths of the store volume. The links the tool cache script creates point at those same paths, so they resolve unchanged, and the pruner reads the annotation, which is unchanged. The init containers still mount the root, since they copy and publish into it before any job runs. A profile in dind mode keeps the read and write exposure through the privileged sidecar, which the README states beside the existing write warning. The kind test now checks that the store path in the runner container holds only the pod's own tool entry. --- plugins/filter/arc.py | 36 +++++++++++------- roles/github_runner_arc/README.md | 8 ++-- roles/github_runner_arc/defaults/main.yml | 2 +- tests/capabilities/run.sh | 5 ++- tests/unit/test_capabilities.py | 45 ++++++++++++++++++++++- 5 files changed, 76 insertions(+), 20 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 603dcf7..03f69be 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -7,7 +7,7 @@ import re from collections.abc import Mapping, Sequence from decimal import ROUND_FLOOR, Decimal, InvalidOperation -from typing import Any +from typing import Any, NamedTuple from ansible.module_utils.parsing.convert_bool import boolean @@ -1171,7 +1171,7 @@ def arc_host_path_overlap_errors(node_cache_host_path: Any, capability_store_hos def arc_capability_store_path_errors(store_path: Any) -> list[str]: - """Return what is wrong with github_runner_arc_capability_store_path, where the store is mounted read-only on the runner container: empty when it is an absolute directory other than / with no colon or line break.""" + """Return what is wrong with github_runner_arc_capability_store_path, under which the runner container mounts each store entry it uses, read-only, and the init containers mount the store's root: empty when it is an absolute directory other than / with no colon or line break.""" if not isinstance(store_path, str) or not store_path.startswith("/") or store_path == "/" or any(character in store_path for character in ":\n\r"): return [f"the capability store path must be an absolute directory other than / with no ':' or line break (got {store_path!r})"] return [] @@ -1337,8 +1337,17 @@ def _ensure_store_prepared(spec: dict[str, Any], store: Mapping[str, Any], runne spec["initContainers"] = [_capability_store_prepare_container(store, runner_user, runner_group), *others] -def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> tuple[list[dict[str, Any]], list[str], str | None]: - """Return the init containers that put a profile's tool-cache and system-library payloads in the store and give the pod its view of them, the store entries the pod uses, and the merged sysroot's entry (None without a system library).""" +class _StoreInit(NamedTuple): + """What a pod's store init containers leave: the containers, every entry the pod uses (the annotation's list), the tool-cache entries its tool tree links into (the only entries the runner container mounts besides the merged sysroot), and the merged sysroot's entry (None without a system library).""" + + containers: list[dict[str, Any]] + keys: list[str] + tool_keys: list[str] + merged: str | None + + +def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, store: Mapping[str, Any], runner_user: int, runner_group: int) -> _StoreInit: + """Return the init containers that put a profile's tool-cache and system-library payloads in the store and give the pod its view of them, with the store entries the pod uses.""" root = store["path"].rstrip("/") copies: list[dict[str, Any]] = [] tool_keys: list[str] = [] @@ -1374,14 +1383,14 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s "securityContext": _runner_security_context(runner_user, runner_group), "volumeMounts": [{"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"], "readOnly": True}, {"name": _TOOL_CACHE_VOLUME, "mountPath": tool_cache_path}], }) - return containers, keys + ([merged] if merged is not None else []), merged + return _StoreInit(containers, keys + ([merged] if merged is not None else []), list(dict.fromkeys(tool_keys)), merged) def arc_capability_store_errors(profiles: Sequence[Mapping[str, Any]], catalogue: Mapping[str, Any], store_path: Any, tool_cache_path: str, sysroot_path: str, hooks_dir: str, node_cache_path: str, registry_cache_enabled: bool) -> list[str]: """Check what a profile with capability_store needs of the role's settings, before anything is installed: the checks arc_capability_pod and arc_capability_store_externals make when they render, so that validation fails on them and not the install, which has by then applied the controller, the pruner and earlier profiles. Args: - profiles: expanded profiles from arc_profiles. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. store_path: where the store is mounted on the runner container. tool_cache_path, sysroot_path, hooks_dir, node_cache_path: where the role mounts the tool cache, the sysroot, the job-started hooks and the node cache, which the store's path may not equal, contain or sit inside. registry_cache_enabled: whether the role writes the registry cache into the values, which writes the dind sidecar out with its externals copy for a profile with container_mode dind. + profiles: expanded profiles from arc_profiles. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. store_path: where the store's root is mounted in the init containers, and under which the runner container mounts each entry it uses. tool_cache_path, sysroot_path, hooks_dir, node_cache_path: where the role mounts the tool cache, the sysroot, the job-started hooks and the node cache, which the store's path may not equal, contain or sit inside. registry_cache_enabled: whether the role writes the registry cache into the values, which writes the dind sidecar out with its externals copy for a profile with container_mode dind. Returns: The problems found, empty when there are none: for a profile that holds a tool-cache or system-library payload in the store, a store path that is not a directory of its own, and, for a profile in dind mode with the registry cache, a runner image not pinned by digest, since the externals copy's entry is named for it. A values file's own externals copy is checked when it is rendered, since its content is not known here. @@ -1450,7 +1459,7 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay. Args: - values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it once per node instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, rebuilt as real directories holding links to the store's files (see _TOOL_CACHE_LINK_SCRIPT), the sysroot is the store's merged entry mounted read-only, and the pod template is annotated with the entries it uses. + values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it once per node instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, rebuilt as real directories holding links to the store's files (see _TOOL_CACHE_LINK_SCRIPT), the sysroot is the store's merged entry mounted read-only, the runner container mounts, read-only, only the tool-cache entries the tool tree links into (each at /) and that merged entry, never the store's root, and the pod template is annotated with the entries it uses. Returns: A dict with ``errors`` (empty when the values can carry the capabilities), ``values`` (the values with the capabilities added, or as given when there are none or errors) and ``hooks`` (the ConfigMap's data apart from the dispatcher, which the role adds from its own file; empty when the profile needs no ConfigMap). @@ -1498,7 +1507,6 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu if use_store: assert store is not None volumes.append(_capability_store_volume(store)) - mounts.append({"name": _CAPABILITY_STORE_VOLUME, "mountPath": store["path"], "readOnly": True}) elif sysroot_capabilities: volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": {}}) mounts.append({"name": _SYSROOT_VOLUME, "mountPath": sysroot_path}) @@ -1523,13 +1531,15 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu assert runner is not None if use_store: assert store is not None - store_containers, store_keys, merged = _capability_store_init_containers(names, catalogue, tool_cache_path, store, runner_user, runner_group) - if merged is not None: - mounts.append({"name": _CAPABILITY_STORE_VOLUME, "mountPath": sysroot_path, "subPath": merged, "readOnly": True}) - init_containers = [_capability_store_prepare_container(store, runner_user, runner_group), *store_containers] + [ + init = _capability_store_init_containers(names, catalogue, tool_cache_path, store, runner_user, runner_group) + # The runner container, which runs a job's steps, sees only the entries its own pod uses, each at / (where the tool tree's links point) and the merged sysroot at its own path, never the store's root: a job could otherwise read every other profile's payloads on its node. The init containers mount the root, since they copy and publish into it before any job runs. + mounts.extend({"name": _CAPABILITY_STORE_VOLUME, "mountPath": f"{store['path'].rstrip('/')}/{key}", "subPath": key, "readOnly": True} for key in init.tool_keys) + if init.merged is not None: + mounts.append({"name": _CAPABILITY_STORE_VOLUME, "mountPath": sysroot_path, "subPath": init.merged, "readOnly": True}) + init_containers = [_capability_store_prepare_container(store, runner_user, runner_group), *init.containers] + [ _node_cache_init_container(name, catalogue[name], node_cache_path, runner_user, runner_group) for name in names if catalogue[name].get("node_cache") is not None ] - _annotate_store_keys(result.setdefault("template", {}), store_keys) + _annotate_store_keys(result.setdefault("template", {}), init.keys) spec["initContainers"] = list(spec.get("initContainers") or []) + init_containers spec["volumes"] = list(spec.get("volumes") or []) + volumes runner["env"] = list(runner.get("env") or []) + env diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 73206a1..3585404 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -31,7 +31,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. - `github_runner_arc_node_cache_path`, `github_runner_arc_node_cache_host_path`, `github_runner_arc_node_cache_size_limit_gib`, `github_runner_arc_node_cache_hard_limit_factor`, `github_runner_arc_node_cache_prune_seconds`: where a node-cache capability's volume is mounted on the runner container (`/node-cache`), the dedicated directory on each node under which each profile's cache lives (`/var/lib/github-runner/node-cache`), the most gibibytes a profile's cache may hold before the pruner empties it (20; 0 for no pruner, a negative value is an error), the hard limit as a multiple of that (2) past which a busy cache is reported on every round, and the seconds between the pruner's checks (300, at least 1). See [Capabilities](#capabilities). -- `github_runner_arc_capability_store_host_path`, `github_runner_arc_capability_store_path`, `github_runner_arc_capability_store_image`: the dedicated directory on each node the capability store lives in (`/var/lib/github-runner/capability-store`, with the same rules as the node cache's, its last component `capability-store` or ending in `-capability-store`, and neither the same directory as the node cache's nor inside it or containing it, since the pruner removes what it finds directly under each as root), where it is mounted read-only on the runner container (`/opt/capability-store`), and the digest-pinned busybox image its own init containers run. See [The capability store](#the-capability-store). +- `github_runner_arc_capability_store_host_path`, `github_runner_arc_capability_store_path`, `github_runner_arc_capability_store_image`: the dedicated directory on each node the capability store lives in (`/var/lib/github-runner/capability-store`, with the same rules as the node cache's, its last component `capability-store` or ending in `-capability-store`, and neither the same directory as the node cache's nor inside it or containing it, since the pruner removes what it finds directly under each as root), where the init containers mount it (`/opt/capability-store`) and under which the runner container mounts, read-only, each entry its pod uses (`/opt/capability-store/`), and the digest-pinned busybox image its own init containers run. See [The capability store](#the-capability-store). - `github_runner_arc_registry_cache_enabled` (default `false`) and the other `github_runner_arc_registry_cache_*` variables: a pull-through registry cache every Docker-in-Docker profile pulls through, falling back to each registry when the cache is down (see [Registry cache](#registry-cache)). - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`. - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart. The controller chart's CRDs are applied (server-side) from `github_runner_arc_controller_chart_ref` at the controller's version before every controller install or upgrade, because Helm never upgrades a chart's CRDs itself; without that, a newer scale-set chart is rejected for fields the cluster's CRDs predate. Move the two pins together, and the CRDs follow. `tests/arc_upgrade/run.sh` covers the upgrade in a kind cluster. @@ -319,15 +319,15 @@ An entry is named for its image's digest and the kind of payload copied from it - `capability-store-sysroot` builds the profile's sysroot as `merged-sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. An `` directory that is itself a symlink (an image aliasing `x64` and `amd64`) is copied as the link too, so it names the real directory the pod's tree holds beside it, and its `.complete` marker is copied like any other. Everything else stays a link, so a pod still writes only the launchers' bytes. -What a job can change, with the store mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. +What a job can change, with the store's entries mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. Nor can a job read what another profile keeps in the store: the runner container mounts only the entries its own pod uses (each tool image's entry at `/`, where the tool tree's links point, and the merged sysroot at the sysroot path, a `subPath` of the store in both cases), never the store's root, so another profile's payloads, and even this pod's own system libraries' source entries, are not in its filesystem. The init containers mount the root, since they copy and publish into it before any job runs. -**A profile in Docker-in-Docker mode (`container_mode: dind`, or values that write the chart's dind sidecar out) does not have that limit.** The chart's dind sidecar runs privileged (`securityContext.privileged: true` in the `gha-runner-scale-set` chart's dind container), so a job that reaches its Docker daemon can start a container that mounts the node's filesystem and write to the store as root, replacing an entry. Entries are named for an image's digest and never revalidated, so every later pod on that node that uses the entry, in any profile with the store on, then runs the replaced bytes, until the pruner removes an entry no scale set or pod lists. The shared store is therefore a persistence channel from a dind job to other pods on its node, as the node cache is between the jobs of its profile. Give `capability_store: true` only to profiles whose workflows are all of one trust level, and never to one that runs pull requests from forks. +**A profile in Docker-in-Docker mode (`container_mode: dind`, or values that write the chart's dind sidecar out) does not have that limit.** The chart's dind sidecar runs privileged (`securityContext.privileged: true` in the `gha-runner-scale-set` chart's dind container), so a job that reaches its Docker daemon can start a container that mounts the node's filesystem, read every profile's payloads in the store, and write to it as root, replacing an entry. Entries are named for an image's digest and never revalidated, so every later pod on that node that uses the entry, in any profile with the store on, then runs the replaced bytes, until the pruner removes an entry no scale set or pod lists. The shared store is therefore a persistence channel from a dind job to other pods on its node, as the node cache is between the jobs of its profile. Give `capability_store: true` only to profiles whose workflows are all of one trust level, and never to one that runs pull requests from forks. `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest (`externals-`), and mounts that entry read-only into the dind container in place of the `dind-externals` volume, whether the dind container is an ordinary container or a native sidecar under `initContainers` (the layout the registry cache writes out). Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. -The store case of `tests/capabilities/run.sh` runs real pods in kind, with the store image's own busybox and the runner user's real identity, which the unit tests cannot show: it pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. The unit tests run the same init container commands with the host's `sh` and coreutils (`tests/unit/test_capabilities.py`, which needs GNU's for `mv -T` and `cp -l` where the host's are not), the pruner's script with the host's `bash` (`tests/unit/test_node_cache_pruner.py`), and the role's tasks against a stand-in for the cluster, not busybox or a real one. +The store case of `tests/capabilities/run.sh` runs real pods in kind, with the store image's own busybox and the runner user's real identity, which the unit tests cannot show: it pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, that the store path in the runner container holds only the pod's own tool entry, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. The unit tests run the same init container commands with the host's `sh` and coreutils (`tests/unit/test_capabilities.py`, which needs GNU's for `mv -T` and `cp -l` where the host's are not), the pruner's script with the host's `bash` (`tests/unit/test_node_cache_pruner.py`), and the role's tasks against a stand-in for the cluster, not busybox or a real one. ### The capabilities test diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index 2ac3555..33d2270 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -27,7 +27,7 @@ github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" # Where on the node the capability store lives, for profiles with capability_store on: one entry per image a profile's capabilities (and the dind sidecar's externals) copy from, named for the image's digest and the kind of payload copied from it, written once per node and mounted read-only by the pods. A dedicated directory the role creates and the pruner removes entries from as root, with the same rules as the node cache's root, whose last component is capability-store or ends in -capability-store. github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" -# Where the store is mounted, read-only, on the runner container: the tool cache's links point into it. +# Where the store's root is mounted in the init containers, and under which the runner container mounts, read-only, each entry its pod uses as / (the tool cache's links point into those): the runner container never mounts the root, so a job cannot read another profile's entries. github_runner_arc_capability_store_path: "/opt/capability-store" # The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chmod, chown, sync, mv, cp, find and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. github_runner_arc_capability_store_image: "docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 73b5e6d..6017282 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -305,8 +305,11 @@ refused() { refused sh -c 'echo changed >>"$1"' sh "$arch_dir/README.md" refused rm "$target" refused touch "$store/$key/job-file" -refused mkdir "$store/job-entry" +refused mkdir "$store/$key/job-dir" refused rm -rf "$store/$key" +# The runner container sees only the entries its own pod uses, never the store's root: the store holds the system libraries' entries and the merged sysroot as well, and none of them is under the store path here, and nothing can be added beside the tool's entry. +[ "$(ls -A "$store")" = "$key" ] || { echo "the store path holds '$(ls -A "$store")', not only the tool's entry '$key'"; exit 1; } +if mkdir "$store/job-entry" 2>"$errors"; then echo "a directory could be added beside the tool's entry, and must not be"; exit 1; fi refused touch "$sysroot/job-file" refused rm "$sysroot/bin/psql" after="$(listing)" diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index ffe25e6..bf303d4 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -695,11 +695,54 @@ def test_with_the_store_each_payload_is_copied_once_per_node_and_published_befor self.assertEqual(volumes["tool-cache"], {"name": "tool-cache", "emptyDir": {}}) self.assertNotIn("sysroot", volumes) mounts = runner(result["values"])["volumeMounts"] - self.assertIn({"name": "capability-store", "mountPath": "/opt/capability-store", "readOnly": True}, mounts) + self.assertIn({"name": "capability-store", "mountPath": f"/opt/capability-store/{KEY_A}", "subPath": KEY_A, "readOnly": True}, mounts) self.assertIn({"name": "capability-store", "mountPath": SYSROOT, "subPath": merged, "readOnly": True}, mounts) self.assertIn({"name": "tool-cache", "mountPath": TOOL_CACHE}, mounts) self.assertEqual(template["metadata"]["annotations"]["github-runner.exadev.io/capability-store-keys"], f"{KEY_A} {KEY_B} {KEY_C} {merged}") + def store_mounts(self, names: list[str], catalogue: dict[str, Any] = STORE_CATALOGUE) -> list[dict[str, Any]]: + """The runner container's mounts of the store volume.""" + result = store_pod(DEFAULT_VALUES, names, catalogue) + self.assertEqual(result["errors"], []) + return [mount for mount in runner(result["values"])["volumeMounts"] if mount["name"] == "capability-store"] + + def test_the_runner_container_mounts_only_the_entries_its_pod_uses_and_never_the_stores_root(self) -> None: + mounts = self.store_mounts(["node", "mpi", "libpq"]) + merged = next(container for container in store_pod(DEFAULT_VALUES, ["node", "mpi", "libpq"])["values"]["template"]["spec"]["initContainers"] if container["name"] == "capability-store-sysroot")["command"][5] + # The tool entry, where the tool tree's links point, and the merged sysroot; not the system libraries' own entries, which the merge hard-linked from. + self.assertEqual( + mounts, + [ + {"name": "capability-store", "mountPath": f"/opt/capability-store/{KEY_A}", "subPath": KEY_A, "readOnly": True}, + {"name": "capability-store", "mountPath": SYSROOT, "subPath": merged, "readOnly": True}, + ], + ) + for mount in mounts: + self.assertEqual(mount["readOnly"], True) + self.assertIn("subPath", mount) + self.assertNotEqual(mount["mountPath"], "/opt/capability-store") + + def test_the_tool_trees_links_resolve_through_the_runner_containers_mounts(self) -> None: + catalogue = {**STORE_CATALOGUE, "second": {"image": f"ghcr.io/example/toolcache-go:1@sha256:{DIGEST_B}"}} + spec = store_pod(DEFAULT_VALUES, ["node", "second"], catalogue)["values"]["template"]["spec"] + link = next(container for container in spec["initContainers"] if container["name"] == "capability-store-tool-cache") + # The links point at //..., so the runner container must mount exactly each of those keys' directories at that path. + root, keys = link["command"][4], link["command"][6:] + self.assertEqual(keys, [KEY_A, f"toolcache-{DIGEST_B}"]) + mounted = {mount["mountPath"]: mount["subPath"] for mount in runner({"template": {"spec": spec}})["volumeMounts"] if mount["name"] == "capability-store"} + self.assertEqual(mounted, {f"{root}/{key}": key for key in keys}) + + def test_a_profile_with_only_system_libraries_mounts_only_its_merged_sysroot(self) -> None: + mounts = self.store_mounts(["mpi", "libpq"]) + self.assertEqual([mount["mountPath"] for mount in mounts], [SYSROOT]) + + def test_the_init_containers_that_write_to_the_store_still_mount_its_root(self) -> None: + spec = store_pod(DEFAULT_VALUES, ["node", "mpi"])["values"]["template"]["spec"] + for container in spec["initContainers"]: + if container["name"].startswith("capability-store-") or container["name"].startswith("capability-"): + mounts = [mount for mount in container.get("volumeMounts", []) if mount["name"] == "capability-store"] + self.assertEqual([mount["mountPath"] for mount in mounts], ["/opt/capability-store"], container["name"]) + def test_a_different_order_of_system_libraries_is_a_different_merged_sysroot(self) -> None: first = store_pod(DEFAULT_VALUES, ["mpi", "libpq"])["values"]["template"]["spec"]["initContainers"] second = store_pod(DEFAULT_VALUES, ["libpq", "mpi"])["values"]["template"]["spec"]["initContainers"] From 3c05b2afa86d4c6dfc9e3f09a9fb6fde7cf91994 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:10:09 +0100 Subject: [PATCH 32/43] fix(arc): merge a system-library entry once when two capabilities share its image --- plugins/filter/arc.py | 2 ++ tests/unit/test_capabilities.py | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 03f69be..2afc78c 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1363,6 +1363,8 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s payload = _TOOL_IMAGE_PAYLOAD if is_tool else _SYSROOT_IMAGE_PAYLOAD copies.append(_capability_store_copy_container(f"{_CAPABILITY_CONTAINER_PREFIX}{name}", image, payload, key, store, runner_user, runner_group)) (tool_keys if is_tool else sysroot_keys).append(key) + # Two capabilities may name one image, and the merge links each entry's files into the merged directory, so an entry named twice would be linked over itself (cp reports the same file). Each entry stays at its last position, where its files win over the earlier entries' as they did when every capability was copied into the pod in order, so the merged entry's name, a hash of the order, is that of the order that merges the same files. + sysroot_keys = list(reversed(list(dict.fromkeys(reversed(sysroot_keys))))) keys = list(dict.fromkeys([*tool_keys, *sysroot_keys])) containers = copies + [_capability_store_publish_container(_CAPABILITY_STORE_PUBLISH, keys, store, runner_user, runner_group)] merged = _capability_store_merged_key(sysroot_keys) if sysroot_keys else None diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index bf303d4..aae3003 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -749,6 +749,25 @@ def test_a_different_order_of_system_libraries_is_a_different_merged_sysroot(sel merged = [next(container for container in containers if container["name"] == "capability-store-sysroot")["command"][5] for containers in (first, second)] self.assertNotEqual(merged[0], merged[1]) + def merge_of(self, names: list[str], catalogue: dict[str, Any]) -> dict[str, Any]: + """The sysroot merge's container for names.""" + result = store_pod(DEFAULT_VALUES, names, catalogue) + self.assertEqual(result["errors"], []) + return next(container for container in result["values"]["template"]["spec"]["initContainers"] if container["name"] == "capability-store-sysroot") + + def test_two_system_library_capabilities_on_one_image_merge_that_entry_once_into_the_entry_one_would(self) -> None: + image = f"ghcr.io/example/libs:1@sha256:{DIGEST_B}" + once = self.merge_of(["first"], {"first": {"sysroot_image": image}}) + twice = self.merge_of(["first", "second"], {"first": {"sysroot_image": image}, "second": {"sysroot_image": image}}) + self.assertEqual(twice["command"][5:], [once["command"][5], KEY_B]) + + def test_a_system_library_entry_named_again_keeps_its_last_position_so_its_files_still_win(self) -> None: + # a, b, a merges b's files and then a's, so a's win a shared file, as the per-pod copies in that order did: the entries are [b, a], never [a, b], which would let b's win. + catalogue = {"a": {"sysroot_image": f"ghcr.io/example/a:1@sha256:{DIGEST_B}"}, "b": {"sysroot_image": f"ghcr.io/example/b:1@sha256:{DIGEST_C}"}, "again": {"sysroot_image": f"ghcr.io/example/a:1@sha256:{DIGEST_B}"}} + merge = self.merge_of(["a", "b", "again"], catalogue) + self.assertEqual(merge["command"][6:], [KEY_C, KEY_B]) + self.assertNotEqual(merge["command"][5], self.merge_of(["a", "b"], catalogue)["command"][5]) + def test_one_image_used_for_a_tool_cache_and_a_sysroot_is_two_entries_each_copied_from_its_own_payload(self) -> None: image = f"ghcr.io/example/both:1@sha256:{DIGEST_A}" result = store_pod(DEFAULT_VALUES, ["tools", "libraries"], {"tools": {"image": image}, "libraries": {"sysroot_image": image}}) @@ -1183,6 +1202,24 @@ def test_the_dind_externals_copy_rerun_after_a_partial_copy_publishes_exactly_th containers = {container["name"]: container for container in spec["initContainers"]} self.assert_rerun_after_a_partial_copy_publishes_the_source_layout(containers["init-dind-externals"], containers["capability-store-publish-externals"], KEY_RUNNER, externals, {"/opt/capability-store": str(self.root), "/home/runner/externals": str(externals)}) + def test_two_capabilities_on_one_system_library_image_link_that_entry_into_the_merge_once(self) -> None: + # busybox's cp -alf of one entry into the merge twice fails on the second as the same file, so the pod never started; GNU cp tolerates it, so the test records each cp call and requires one for the entry. + image = f"ghcr.io/example/libs:1@sha256:{DIGEST_B}" + (self.root / KEY_B / "lib").mkdir(parents=True) + (self.root / KEY_B / "lib/libfoo.so.1").write_text("foo") + spec = store_pod(DEFAULT_VALUES, ["first", "second"], {"first": {"sysroot_image": image}, "second": {"sysroot_image": image}})["values"]["template"]["spec"] + merge = next(container for container in spec["initContainers"] if container["name"] == "capability-store-sysroot") + bin_dir = self.work / "cp-bin" + bin_dir.mkdir() + recorder = bin_dir / "cp" + recorder.write_text('#!/bin/sh\nprintf "%s\\n" "$*" >> "$CP_LOG"\nexec ' + str(shutil.which("cp")) + ' "$@"\n') + recorder.chmod(0o755) + log = self.work / "cp.log" + completed = self.run_with_env(merge, "pod-1", {"/opt/capability-store": str(self.root)}, {"PATH": f"{bin_dir}:{os.environ['PATH']}", "CP_LOG": str(log)}) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(len([line for line in log.read_text().splitlines() if f"{self.root}/{KEY_B}/." in line]), 1) + self.assertEqual(self.layout(self.root / merge["command"][5]), {"lib": "", "lib/libfoo.so.1": "foo"}) + def test_a_sysroot_merge_rerun_after_a_partial_merge_publishes_only_the_libraries(self) -> None: (self.root / KEY_B / "lib").mkdir(parents=True) (self.root / KEY_B / "lib/libmpi.so").write_text("mpi") From 780907f0b8660ce3157c694842f5cf3c81218ede Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:11:00 +0100 Subject: [PATCH 33/43] fix(arc): refuse a capability whose init container would take a store init container's name --- plugins/filter/arc.py | 8 ++++++-- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 2afc78c..34cfea5 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -764,9 +764,11 @@ def _capability_entry_errors(name: Any, capability: Any) -> list[str]: where = f"github_runner_arc_capabilities.{name}" if not isinstance(name, str) or len(name) > _CAPABILITY_NAME_MAX_LENGTH or not _DNS_LABEL.match(name): return [f"{where}: a capability name must be lower-case letters, digits and '-', at most {_CAPABILITY_NAME_MAX_LENGTH} characters, since its init container is named {_CAPABILITY_CONTAINER_PREFIX}"] + container = f"{_CAPABILITY_CONTAINER_PREFIX}{name}" + errors = [f"{where}: its init container would be named {container}, which is the name of one of the capability store's own init containers (a pod cannot have two with one name)"] if container in _CAPABILITY_STORE_CONTAINER_NAMES else [] if not isinstance(capability, Mapping): - return [f"{where} must be a mapping"] - errors = [f"{where} has an unknown key '{key}'" for key in sorted(set(capability) - _CAPABILITY_KEYS)] + return [*errors, f"{where} must be a mapping"] + errors.extend(f"{where} has an unknown key '{key}'" for key in sorted(set(capability) - _CAPABILITY_KEYS)) image = capability.get("image") sysroot_image = capability.get("sysroot_image") hook = capability.get("job_started_hook") @@ -1126,6 +1128,8 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca _CAPABILITY_STORE_SYSROOT = "capability-store-sysroot" _CAPABILITY_STORE_TOOL_CACHE = "capability-store-tool-cache" _CAPABILITY_STORE_EXTERNALS_PUBLISH = "capability-store-publish-externals" +# Every init container the role names itself with the store, in contrast to the capability- ones its capabilities get. A capability whose container would be named one of these is refused (see _capability_entry_errors), whether or not its profile uses the store, since a profile may switch it on later. +_CAPABILITY_STORE_CONTAINER_NAMES = frozenset({_CAPABILITY_STORE_PREPARE, _CAPABILITY_STORE_PUBLISH, _CAPABILITY_STORE_SYSROOT, _CAPABILITY_STORE_TOOL_CACHE, _CAPABILITY_STORE_EXTERNALS_PUBLISH}) # The kinds of payload an entry holds, the first part of its name (see arc_capability_store_key): the tool cache of a tool image, the sysroot of a system-library image and the runner's externals for the dind sidecar. _STORE_KIND_TOOL_CACHE = "toolcache" _STORE_KIND_SYSROOT = "sysroot" diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 3585404..c1e2b5d 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -118,7 +118,7 @@ The runner pods then lose the eligibility `nodeSelector`, and get instead a requ ## Capabilities -A capability adds something to a profile's runner pods when they start, so the runner image itself can stay a public, stock image such as `ghcr.io/actions/actions-runner`. `github_runner_arc_capabilities` defines them by name, and each profile lists the ones it wants in `capabilities`. There are four kinds. +A capability adds something to a profile's runner pods when they start, so the runner image itself can stay a public, stock image such as `ghcr.io/actions/actions-runner`. `github_runner_arc_capabilities` defines them by name, and each profile lists the ones it wants in `capabilities`. A capability's init container, where it has one, is named `capability-`, so validation refuses the names `store-prepare`, `store-publish`, `store-sysroot`, `store-tool-cache` and `store-publish-externals`, which would give a pod two init containers of one name with the capability store's own, whether or not any profile uses the store. There are four kinds. A tool-cache capability, `{image, path?}`, brings a tool. For each one the runner pod gets an init container, named `capability-`, that copies the tool from the image into an `emptyDir` mounted on the runner container at `github_runner_arc_tool_cache_path`, and `RUNNER_TOOL_CACHE` is set to that path. The runner hands `RUNNER_TOOL_CACHE` to every step, and setup actions that consult the Actions tool cache before downloading, such as `actions/setup-node` and `terraform-linters/setup-tflint`, then find the tool there and skip the download, with no change to the workflow. The init containers run as the runner's user, so the copies belong to it and a setup action can still add another version beside them. `path`, `/` optionally followed by `/`, also puts the tool on `PATH` before the job's first step, for a tool no setup action looks up (Terraform, whose `hashicorp/setup-terraform` always downloads, or the AWS CLI). diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index aae3003..67f94c0 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -111,6 +111,31 @@ def test_bad_entries_are_errors(self) -> None: errors = self.errors(catalogue) self.assertTrue(any(message in error for error in errors), errors) + RESERVED = ("store-prepare", "store-publish", "store-sysroot", "store-tool-cache", "store-publish-externals") + + def test_a_capability_named_like_one_of_the_stores_init_containers_is_an_error_with_the_store_on_or_off(self) -> None: + # capability- would be the name of the store's own container, and the API server rejects a pod with two of one name. + for name in self.RESERVED: + for store in (True, False): + with self.subTest(name=name, store=store): + errors: list[str] = arc.arc_capability_errors([expanded({"max_runners": 1, "capability_store": store, "capabilities": [name]})], {name: {"image": f"ghcr.io/example/x:1@sha256:{DIGEST_A}"}}) + self.assertEqual(len(errors), 1, errors) + self.assertIn(f"github_runner_arc_capabilities.{name}", errors[0]) + self.assertIn(f"capability-{name}", errors[0]) + self.assertIn("capability store's own init containers", errors[0]) + + def test_a_reserved_name_is_an_error_for_every_kind_of_capability_and_a_non_mapping_entry(self) -> None: + for entry in ({"sysroot_image": f"ghcr.io/example/x:1@sha256:{DIGEST_A}"}, {"job_started_hook": "true"}, {"node_cache": {"init_image": BUSYBOX, "dirs": {"STORE": "pnpm"}}}, "not a mapping"): + with self.subTest(entry=entry): + errors = self.errors({"store-publish": entry}) + self.assertTrue(any("capability-store-publish" in error for error in errors), errors) + + def test_every_init_container_the_role_names_beside_the_capabilities_is_reserved(self) -> None: + # Render everything the store adds (the preparation, the publishing, the merge, the tool-cache links and the dind externals' publishing): each name that is not capability- is one a capability's container could clash with. + spec = arc.arc_capability_store_externals(store_pod(DIND_VALUES, ["node", "mpi"])["values"], STORE, 1001, 1001)["values"]["template"]["spec"] + names = {container["name"] for container in spec["initContainers"] if container["name"].startswith("capability-")} - {"capability-node", "capability-mpi"} + self.assertEqual(names, {f"capability-{name}" for name in self.RESERVED}) + def test_bad_system_library_entries_are_errors(self) -> None: image = "ghcr.io/example/sysroot-openmpi:5.0.11" cases = { From 8c55d6af961a0aa13ca8cf31135ddedcd71315ab Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:11:35 +0100 Subject: [PATCH 34/43] docs(arc): state that a store switched off fleet-wide keeps its entries for the operator to delete When the last profile with capability_store switches it off while another profile keeps the pruner installed for a node cache, the namespaces the pruner reads for the store are empty, so the template omits STORE_ROOT and STORE_NAMESPACES and the pruner never mounts or prunes the store. Every node then keeps its entries, which the README attributed only to the pruner going. The README now says exactly that, with the path and the command. A prune-to-empty mode was rejected: the role cannot tell which nodes hold a store, and the pruner could only empty one safely while the grants listing scale sets and pods stayed in every namespace, which would keep a grant for a store no profile uses. --- roles/github_runner_arc/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index c1e2b5d..e221c2a 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -134,7 +134,7 @@ A pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image, o - **Over `github_runner_arc_node_cache_size_limit_gib`, no pod using the cache on the node:** it moves the cache's entries, one rename each, into a unique trash directory beside the profiles (same filesystem, outside every pod's mount), listing the pods again before every rename, and deletes the trash only when the moving is done. A store is therefore seen whole or not at all by a pod that starts at any point; a pod that starts part way finds the stores not yet moved whole and the rest empty, and a pod appearing ends the moving and leaves what has not moved in place. It never trims a store, because a running job may be linked into any part of one. A deletion that fails is reported and retried at the start of the next round. What remains is the instant between a list and the rename after it, shorter than a runner pod needs to start a job; a pod that does start in it meets an empty cache it can refill, because the cache's root is owned by the runner's user. - **Over the hard limit (`github_runner_arc_node_cache_hard_limit_factor` times the size limit), a pod using the cache:** it cannot be emptied, and the pruner changes nothing about the node to force it. It logs a line starting `NODE_CACHE_OVER_HARD_LIMIT` on every round, with the profile, the size and the node, for whatever collects the pruner's logs to alert on. Draining the node, or lowering its runner count, lets the next round empty the cache, and kubelet's disk-pressure eviction still protects the node itself. An automatic cordon was considered and left out: a controller that cordons nodes needs a floor of schedulable nodes, a record of which cordons are its own, and a grant to patch nodes, and a transient failure part way through a drain would lift it. -- **A profile directory the fleet no longer wants** (the capability was removed from a profile, or the profile has left the fleet) is emptied and removed the same way, while the pruner is installed. The role keeps a read-only grant to list pods (a Role and RoleBinding, named for the release, in the profile's namespace) for every profile it has installed while the pruner is installed, including one that has since left the fleet, so such a directory can always be checked (a grant only lists pods in its own profile's namespace, so one that stays is harmless); the grants are found by the role's own labels (`app.kubernetes.io/name`, `part-of` and `managed-by`, all three, so another software's objects are never touched) and removed only with the pruner. The pruner follows the fleet: it is installed while the size limit is above zero and some profile has a node cache or uses the capability store, and removed when none does, so a directory left when the pruner goes (a cache directory, or the store's entries) is for the operator to delete from `` and `` on each node. A list the server refuses all the same, which only happens when the namespace was deleted together with its Role, is reported on every round with a line starting `NODE_CACHE_UNPRUNABLE` and the directory is left for the operator (with the namespace gone there are no pods, so deleting it is safe). A list that fails for any other reason (the API server unreachable, an answer that is not a pod list) is reported with `NODE_CACHE_LIST_FAILED` and retried on the next round; both lines carry the cache's size, and a cache past the hard limit whose pods could not be listed is also reported as over it. +- **A profile directory the fleet no longer wants** (the capability was removed from a profile, or the profile has left the fleet) is emptied and removed the same way, while the pruner is installed. The role keeps a read-only grant to list pods (a Role and RoleBinding, named for the release, in the profile's namespace) for every profile it has installed while the pruner is installed, including one that has since left the fleet, so such a directory can always be checked (a grant only lists pods in its own profile's namespace, so one that stays is harmless); the grants are found by the role's own labels (`app.kubernetes.io/name`, `part-of` and `managed-by`, all three, so another software's objects are never touched) and removed only with the pruner. The pruner follows the fleet: it is installed while the size limit is above zero and some profile has a node cache or uses the capability store, and removed when none does, so a cache directory left when the pruner goes is for the operator to delete from `` on each node. The capability store's entries are not tied to the pruner being installed: they are pruned only while some profile has `capability_store: true`, and are left otherwise (see [The capability store](#the-capability-store)). A list the server refuses all the same, which only happens when the namespace was deleted together with its Role, is reported on every round with a line starting `NODE_CACHE_UNPRUNABLE` and the directory is left for the operator (with the namespace gone there are no pods, so deleting it is safe). A list that fails for any other reason (the API server unreachable, an answer that is not a pod list) is reported with `NODE_CACHE_LIST_FAILED` and retried on the next round; both lines carry the cache's size, and a cache past the hard limit whose pods could not be listed is also reported as over it. - **The credential must be able to read cluster roles.** Release 1.31.0 installed the pruner with a cluster-wide `ClusterRole` and `ClusterRoleBinding` named `node-cache-pruner`; every run now looks for them by name and removes them, binding first, only if they are what 1.31.0 created (a role that lists pods and nothing else, not aggregated, and a binding of it to the pruner's account in the platform namespace and nothing else; 1.31.0 set no labels, so the content is the identity). The lookup needs the credential the role reaches the cluster with to be allowed to `get` ClusterRoles and ClusterRoleBindings (cluster-scoped, read-only), also on a fleet that never ran 1.31.0, and `delete` on them only when it finds the 1.31.0 objects. A role of that name whose binding is not the one 1.31.0 created (the platform namespace changed, another subject was added) is left in place with the binding and reported once per run with a line starting `NODE_CACHE_LEGACY_ACCESS_LEFT`, so a dangling binding can never regain cluster-wide pod listing; delete them by hand if they are not wanted. The cleanup goes in a later release (ExaDev/github-runner#92). A flag that skipped the lookup on a fleet that never ran 1.31.0 would be one more setting to maintain, so there is none. - **Upgrading from 1.31.0:** the host path root is now validated whether or not any profile has a node cache, so a configuration whose `github_runner_arc_node_cache_host_path` does not end in `node-cache` (the default does) fails validation even if it uses no node-cache capability; set the path to a conforming one (a new path starts an empty cache, and the old directory is yours to delete). - **A cluster that ran release 1.31.0** has a cluster-wide grant to list pods named `node-cache-pruner`; the role removes the 1.31.0 objects where it finds them (binding, then role), subject to the paragraph above: only what matches what 1.31.0 created, and a role whose binding is not the one 1.31.0 created is left with it and reported. @@ -325,7 +325,7 @@ What a job can change, with the store's entries mounted read-only on the runner `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest (`externals-`), and mounts that entry read-only into the dind container in place of the `dind-externals` volume, whether the dind container is an ordinary container or a native sidecar under `initContainers` (the layout the registry cache writes out). Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. -The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so a profile that has switched the store off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. +The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so while some profile still uses the store, a profile that has switched it off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. The store is pruned only while some profile has `capability_store: true`: once the last profile switches it off, the pruner, installed or not, no longer reads or mounts the store, so every node keeps its entries and `.incoming` until the operator deletes the directory (`sudo rm -rf ` on each node, once no pod made from a template that used the store is still running; the next profile to switch the store on recreates it). Emptying it automatically was left out: the role cannot tell which nodes hold a store, and the pruner could only do it safely while the grants that list scale sets and pods stayed in place for every namespace, which would keep a grant for a store no profile uses. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. The store case of `tests/capabilities/run.sh` runs real pods in kind, with the store image's own busybox and the runner user's real identity, which the unit tests cannot show: it pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, that the store path in the runner container holds only the pod's own tool entry, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. The unit tests run the same init container commands with the host's `sh` and coreutils (`tests/unit/test_capabilities.py`, which needs GNU's for `mv -T` and `cp -l` where the host's are not), the pruner's script with the host's `bash` (`tests/unit/test_node_cache_pruner.py`), and the role's tasks against a stand-in for the cluster, not busybox or a real one. From cb4ace4caec159554154770fdecba8b72551c513 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:12:27 +0100 Subject: [PATCH 35/43] docs(arc): declare the capability store's profile key and settings in the argument specification --- plugins/filter/arc.py | 4 ++-- .../github_runner_arc/meta/argument_specs.yml | 17 +++++++++++++++-- tests/unit/test_capabilities.py | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 34cfea5..5a4f8fc 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -621,10 +621,10 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, Args: require_app_id: whether each org must set app_id. The role needs it only when it writes the App Secret; otherwise the App's id is in the existing App Secret, and an app_id that is set is only checked against it. - orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. + orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile may set capability_store (a boolean, default false) to copy its tool-cache and system-library payloads, and the dind sidecar's externals, once per node into the capability store instead of into every pod (see arc_capability_pod). A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. Returns: - A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, target (namespace/release, joined), app_secret, runner_image (its org's image), scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). + A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, target (namespace/release, joined), app_secret, runner_image (its org's image), scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), capability_store (whether it uses the capability store), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). """ errors: list[str] = [] profiles: list[dict[str, Any]] = [] diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index 71ee974..b279237 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -12,7 +12,8 @@ argument_specs: default: [] description: - Orgs whose runner scale sets this host installs. Each entry has name, app_id (needed only when the role writes the App Secret; otherwise the existing Secret supplies it), image, private_key or private_key_op_reference, and optionally installation_id and app_secret_name (the App Secret's name, default -github-app). - - Each entry's scale_set_profiles is a non-empty list of profiles, each with suffix and optionally namespace (default arc-runners-), release_name (default -runners), values_file, max_runners, min_runners, container_mode, resources, node_selector, runs_on_label, scale_set_labels, capabilities, tolerations, autoscale, sizing and burst. + - Each entry's scale_set_profiles is a non-empty list of profiles, each with suffix and optionally namespace (default arc-runners-), release_name (default -runners), values_file, max_runners, min_runners, container_mode, resources, node_selector, runs_on_label, scale_set_labels, capabilities, capability_store, tolerations, autoscale, sizing and burst. + - capability_store, a boolean (default false), makes the profile's runner pods copy each tool-cache and system-library capability's payload, and the runner's externals for a Docker-in-Docker sidecar the values write out, once per node into the capability store (github_runner_arc_capability_store_host_path) instead of into every pod's own volumes, and mount the result read-only. Every image those copies come from must be pinned by digest, and the store needs the node cache pruner, so a profile that sets it is refused while github_runner_arc_node_cache_size_limit_gib is 0. A job that can reach a privileged Docker daemon can write to the store, so give it only to a profile whose workflows are all of one trust level. - max_runners, when set, is the release's maxRunners and takes precedence over its values file and its sizing. - scale_set_labels replaces runs_on_label with the whole list of scaleSetLabels; an empty list sets none, so jobs target the release name. - tolerations is a list of Kubernetes tolerations for the profile's runner pods, for nodes the eligibility label selects that also carry a taint to keep other workloads off. Measured sizing counts a node whose only repelling taints these match, as the pods can be placed there; without them it leaves such a node out. With burst they come before burst.tolerations. @@ -44,10 +45,22 @@ argument_specs: type: str default: /var/lib/github-runner/node-cache description: The directory on each node under which every profile's node cache lives, in a directory of its own named /. It must be an absolute path with no colon, no repeated slash and no trailing slash, at least three levels deep, whose last component ends with node-cache, and not a system directory or inside one. A symbolic link at or above it defeats the validation. + github_runner_arc_capability_store_host_path: + type: str + default: /var/lib/github-runner/capability-store + description: The directory on each node that holds the capability store of the profiles with capability_store, one entry per image payload, written once per node. It must be an absolute path with no colon or line break, no repeated slash and no trailing slash, at least three levels deep, whose last component is capability-store or ends with -capability-store, not a system directory or inside one, and neither equal to, inside nor containing github_runner_arc_node_cache_host_path. A symbolic link at or above it defeats the validation. The node cache pruner removes the entries no scale set or pod uses, as root, only while some profile has capability_store; once none has, the entries stay until the operator deletes the directory on each node. + github_runner_arc_capability_store_path: + type: str + default: /opt/capability-store + description: Where the store's root is mounted in the profiles' init containers, and under which the runner container mounts, read-only, each entry its pod uses as /. An absolute directory other than / with no colon or line break, which for a profile whose capabilities have a payload in the store may not equal, contain or sit inside the tool cache path, the sysroot path, the job-started hooks directory or the node cache path. + github_runner_arc_capability_store_image: + type: str + default: docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e + description: The image the store's own init containers run (creating the root, flushing and publishing a copy, merging the sysroot, linking the tool cache). It must be pinned by digest (name@sha256:<64 hex digits>), since one of them runs as root on a hostPath, and its sh, mkdir, chmod, chown, sync, mv, cp, find and rm must be busybox's or GNU's, mv -T refusing to move onto a directory that is not empty. github_runner_arc_node_cache_size_limit_gib: type: int default: 20 - description: The most gibibytes one profile's node cache may hold on a node before the pruner empties it, which it does only while no runner pod of the profile runs on the node. The limit is per profile, so a node's worst case is the sum over the profiles with a cache on it, times the hard limit factor. 0 installs no pruner; a negative value, a leading zero, a fraction or more than 1073741824 is an error. + description: The most gibibytes one profile's node cache may hold on a node before the pruner empties it, which it does only while no runner pod of the profile runs on the node. The limit is per profile, so a node's worst case is the sum over the profiles with a cache on it, times the hard limit factor. 0 installs no pruner, which a profile with capability_store cannot do without, since the pruner removes the store's unused entries; a negative value, a leading zero, a fraction or more than 1073741824 is an error. github_runner_arc_node_cache_hard_limit_factor: type: int default: 2 diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 67f94c0..9aa168a 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -14,6 +14,8 @@ from pathlib import Path from typing import Any +import yaml + ROOT = Path(__file__).resolve().parents[2] PLUGIN = ROOT / "plugins" / "filter" / "arc.py" RENDER = ROOT / "tests" / "capabilities" / "render.yml" @@ -684,6 +686,23 @@ def test_an_entry_is_named_for_its_images_digest_and_payload_kind_and_an_unpinne self.assertIsNone(arc.arc_capability_store_key("ghcr.io/example/x:1", "toolcache")) +class CapabilityStoreArgumentSpecTest(unittest.TestCase): + """The role's argument specification declares the store's settings with the defaults the role has.""" + + ROLE = ROOT / "roles" / "github_runner_arc" + + def test_every_capability_store_default_is_declared_with_its_default_and_the_profile_key_is_described(self) -> None: + defaults = yaml.safe_load((self.ROLE / "defaults" / "main.yml").read_text()) + specification = yaml.safe_load((self.ROLE / "meta" / "argument_specs.yml").read_text())["argument_specs"]["main"]["options"] + names = sorted(name for name in defaults if name.startswith("github_runner_arc_capability_store_")) + self.assertEqual(names, ["github_runner_arc_capability_store_host_path", "github_runner_arc_capability_store_image", "github_runner_arc_capability_store_path"]) + for name in names: + with self.subTest(name): + self.assertEqual(specification[name]["default"], defaults[name]) + self.assertEqual(specification[name]["type"], "str") + self.assertIn("capability_store", " ".join(specification["github_runner_arc_orgs"]["description"])) + + class CapabilityStorePodTest(unittest.TestCase): def test_without_the_store_the_copies_stay_in_each_pod(self) -> None: result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], STORE_CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, None) From 81243ebc39ad87539a514e66f44a18da98601420 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:14:01 +0100 Subject: [PATCH 36/43] fix(arc): copy a tool or version directory that is a link as the link in the pod's tool cache --- plugins/filter/arc.py | 13 ++++-- roles/github_runner_arc/README.md | 2 +- tests/unit/test_capabilities.py | 71 +++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 5a4f8fc..69b0392 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1305,11 +1305,18 @@ def _capability_store_publish_container(name: str, keys: Sequence[str], store: M ' [ -d "$root/$merged" ] || { echo "capability store: $merged is not in $root after publishing${moved:+: $moved}" >&2; exit 1; }' ) -# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. An that is itself a symbolic link (an image aliasing x64 and amd64) is copied as the link, as an entry inside the tree is, and never linked to the store's link: the link's relative target then names the real directory the pod's tree holds beside it, which the same loop links, and no command ever runs on a path that resolves into the store (cp -as of a link to a link would make the pod's a link to the store's, which a chmod -R follows from the command line into the read-only store and fails on). The .complete marker, which is how the Actions tool cache finds a tool, is copied as well, for an alias too, with cp -f since it keeps the store file's mode and a rerun over a read-only copy must still replace it. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, cp -f, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. +# Fills the pod's own tool cache from the store so that a job can still change its tools: every tool tree is rebuilt in the pod as real directories holding a symlink to each store file, so a job can create, replace and remove files and directories in a tool's own tree while the store stays untouched (a store file cannot be written through its link, since the store is mounted read-only). Two kinds of store entry are copied into the pod instead of linked: a symlink, so that its relative target resolves inside the pod's tree and not the store's, and each regular file directly in /bin, since a launcher such as node, python or go finds its own tree (npm's global prefix, a Python's site-packages) from its resolved executable path, which a symlink would put in the read-only store. A tool or version directory that is itself a symbolic link (an image aliasing node and nodejs, or 1.0 and 1.0.0) is copied as the link too, and nothing below it is walked, since the glob would otherwise reach the real tree through it and rebuild it as a second real tree of its own, in which a job's write through one name would not appear through the other. An that is itself a symbolic link (an image aliasing x64 and amd64) is copied as the link, as an entry inside the tree is, and never linked to the store's link: the link's relative target then names the real directory the pod's tree holds beside it, which the same loop links, and no command ever runs on a path that resolves into the store (cp -as of a link to a link would make the pod's a link to the store's, which a chmod -R follows from the command line into the read-only store and fails on). The .complete marker, which is how the Actions tool cache finds a tool, is copied as well, for an alias too, with cp -f since it keeps the store file's mode and a rerun over a read-only copy must still replace it. cp -as gives each directory the store directory's mode, and a store tree can hold directories without write permission (a Go module cache's are mode 0555), in which a job could create nothing and the rm -f of a copied entry fails as the non-root runner user, so every directory of the pod's tree is made owner-writable (chmod -R u+rwx, which does not follow symbolic links, so it changes only the pod's directories) after it is built and, for a rerun after a run killed between building and chmod, before the rebuild as well. cp -asf, cp -f, chmod -R, find -exec ... + and cp -P are busybox's and GNU's alike, and rerunning the script over a filled cache converges on the same tree. _TOOL_CACHE_COPY_SCRIPT = 'source=$1; destination=$2; shift 2; for entry; do target=$destination/${entry#"$source"/}; rm -f "$target" && cp -P "$entry" "$target" || exit 1; done' _TOOL_CACHE_LINK_SCRIPT = ( - 'root=$1; cache=$2; shift 2; for key in "$@"; do for arch in "$root/$key"/*/*/*; do' - ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"}; parent="$cache/${relative%/*}"; tree="$cache/$relative";' + 'root=$1; cache=$2; shift 2; for key in "$@"; do' + ' for link in "$root/$key"/* "$root/$key"/*/*; do' + ' [ -L "$link" ] || continue; relative=${link#"$root/$key/"}; parent=$cache;' + ' case $relative in */*) [ ! -L "$root/$key/${relative%%/*}" ] || continue; parent="$cache/${relative%/*}";; esac;' + ' mkdir -p "$parent" && rm -f "$cache/$relative" && cp -P "$link" "$cache/$relative" || exit 1;' + ' done;' + ' for arch in "$root/$key"/*/*/*; do' + ' [ -d "$arch" ] || continue; relative=${arch#"$root/$key/"}; parent="$cache/${relative%/*}"; tree="$cache/$relative"; tool=${relative%%/*}; version=${relative#*/}; version=${version%%/*};' + ' [ ! -L "$root/$key/$tool" ] && [ ! -L "$root/$key/$tool/$version" ] || continue;' ' mkdir -p "$parent" || exit 1;' ' if [ -L "$arch" ]; then rm -f "$tree" && cp -P "$arch" "$tree" || exit 1; else' ' chmod -R u+rwx "$tree" 2>/dev/null;' diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index e221c2a..ee1c354 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -317,7 +317,7 @@ An entry is named for its image's digest and the kind of payload copied from it - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. - `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `merged-sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. -- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. An `` directory that is itself a symlink (an image aliasing `x64` and `amd64`) is copied as the link too, so it names the real directory the pod's tree holds beside it, and its `.complete` marker is copied like any other. Everything else stays a link, so a pod still writes only the launchers' bytes. +- `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. A tool or version directory that is itself a symlink (an image aliasing `node` and `nodejs`, or `22` and `22.0.0`) is copied as the link too, and nothing below it is walked, so the pod holds one real tree and a job's write through one name shows through the other. An `` directory that is itself a symlink (an image aliasing `x64` and `amd64`) is copied as the link too, so it names the real directory the pod's tree holds beside it, and its `.complete` marker is copied like any other. Everything else stays a link, so a pod still writes only the launchers' bytes. What a job can change, with the store's entries mounted read-only on the runner container: in the tool cache it can create, replace and remove files, links and directories anywhere in a tool's tree (`npm install -g`, `corepack enable`, a setup action installing another version or architecture beside the cached one), because every directory is the pod's own and a replaced file's link is the pod's own. It cannot change the bytes of an existing store file in place (an in-place write through a file's link fails with a read-only filesystem error, whereas replacing the file works), and from the runner container it cannot change anything in the store, which is mounted read-only there. The sysroot and the dind sidecar's externals are not tool trees but read-only consumers (the runner mounts the externals into job containers read-only, `ContainerOperationProvider.cs` in `actions/runner`, and the sysroot is reached through search paths), so a step on the runner container cannot write into them, where the per-pod copies were writable. In a profile without a Docker daemon that is the whole story: no job changes what another pod uses. Nor can a job read what another profile keeps in the store: the runner container mounts only the entries its own pod uses (each tool image's entry at `/`, where the tool tree's links point, and the merged sysroot at the sysroot path, a `subPath` of the store in both cases), never the store's root, so another profile's payloads, and even this pod's own system libraries' source entries, are not in its filesystem. The init containers mount the root, since they copy and publish into it before any job runs. diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 9aa168a..7dd5eff 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -1518,6 +1518,77 @@ def aliased_tool_tree(self, alias: str) -> Path: (version / "x64.complete").write_text("") return version + def linked_tool_names(self) -> Path: + """Fill the store's entry for the node capability with one tool whose version and tool are also reached through links, as an image aliasing node and nodejs, and 22 and 22.0.0, does: node/22.0.0 holds the real tree, node/22 is a link to it, and nodejs is a link to node. Returns the entry.""" + entry = self.root / KEY_A + arch = entry / "node/22.0.0/x64" + (arch / "bin").mkdir(parents=True) + (arch / "bin/node").write_text("binary") + (arch / "README").write_text("readme") + (entry / "node/22.0.0/x64.complete").write_text("") + (entry / "node/22").symlink_to("22.0.0") + (entry / "nodejs").symlink_to("node") + return entry + + def test_a_tool_or_version_directory_that_is_a_link_is_copied_as_the_link_and_not_rebuilt_as_a_second_tree(self) -> None: + entry = self.linked_tool_names() + before = self.snapshot(self.root) + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + # Both names are the pod's own links, and the only real tree is the one the store holds as real. + self.assertEqual((cache / "node/22").readlink(), Path("22.0.0")) + self.assertEqual((cache / "nodejs").readlink(), Path("node")) + self.assertTrue((cache / "node/22.0.0/x64").is_dir() and not (cache / "node/22.0.0").is_symlink()) + self.assertEqual(sorted(path.name for path in cache.iterdir()), ["node", "nodejs"]) + self.assertEqual(sorted(path.name for path in (cache / "node").iterdir()), ["22", "22.0.0"]) + # A job's write through one name is seen through every other, and the store is unchanged. + (cache / "node/22.0.0/x64/created").write_text("created") + for name in ("node/22/x64", "nodejs/22/x64", "nodejs/22.0.0/x64"): + self.assertEqual((cache / name / "created").read_text(), "created", name) + self.assertEqual((cache / "nodejs/22/x64/bin/node").read_text(), "binary") + self.assertEqual(self.snapshot(self.root), before) + self.assertFalse((entry / "node/22.0.0/x64/created").exists()) + + def test_a_tool_directory_that_is_a_link_alone_and_a_version_directory_that_is_a_link_alone_are_each_copied_as_the_link(self) -> None: + for case, tool_link, version_link in (("tool", True, False), ("version", False, True)): + with self.subTest(case): + entry = self.linked_tool_names() + if not tool_link: + (entry / "nodejs").unlink() + if not version_link: + (entry / "node/22").unlink() + cache = self.work / f"toolcache-{case}" + cache.mkdir() + self.link_tool_cache(cache) + self.assertEqual((cache / "nodejs").is_symlink(), tool_link) + self.assertEqual((cache / "node/22").is_symlink(), version_link) + self.assertTrue((cache / "node/22.0.0").is_dir() and not (cache / "node/22.0.0").is_symlink()) + shutil.rmtree(entry) + shutil.rmtree(cache) + + def test_a_tool_or_version_link_that_is_absolute_is_copied_as_written(self) -> None: + entry = self.linked_tool_names() + (entry / "node/22").unlink() + (entry / "nodejs").unlink() + cache = self.work / "toolcache-pod" + cache.mkdir() + (entry / "node/22").symlink_to(cache / "node/22.0.0") + (entry / "nodejs").symlink_to(cache / "node") + self.link_tool_cache(cache) + self.assertEqual((cache / "node/22").readlink(), cache / "node/22.0.0") + self.assertEqual((cache / "nodejs").readlink(), cache / "node") + self.assertEqual((cache / "nodejs/22/x64/bin/node").read_text(), "binary") + + def test_linking_again_over_a_tool_cache_with_tool_and_version_links_leaves_the_same_tree(self) -> None: + self.linked_tool_names() + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + first = self.snapshot(cache) + self.link_tool_cache(cache) + self.assertEqual(self.snapshot(cache), first) + def test_an_arch_directory_that_is_a_link_is_copied_as_the_link_and_the_store_is_never_written_through_it(self) -> None: version = self.aliased_tool_tree("real") before = self.snapshot(self.root) From 84aa90ab9579b5aaaeffe8392aa3b1b24cb3c3b8 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:14:54 +0100 Subject: [PATCH 37/43] test(arc): name the pruner test's store entries as the role does --- scripts/node-cache-pruner.sh | 2 +- tests/unit/test_node_cache_pruner.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/node-cache-pruner.sh b/scripts/node-cache-pruner.sh index e263518..a785d3f 100755 --- a/scripts/node-cache-pruner.sh +++ b/scripts/node-cache-pruner.sh @@ -11,7 +11,7 @@ # # Every step of one profile's check may fail without stopping the others: a failed or partial size reading skips that profile for the round, and a failed pod list means no pruning, since an unknown set of pods may be using the cache: NODE_CACHE_UNPRUNABLE when the server refused the list as Forbidden (there is no Role for the pruner in that namespace), NODE_CACHE_LIST_FAILED for any other failure (the API server was unreachable, the answer was not a pod list), which the next round retries. # -# The capability store (the role README's "The capability store"), when STORE_ROOT is set, holds one entry per image payload, named for the image's digest, plus merged sysroots, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it (STORE_NAMESPACES holds every namespace a pod may still run in from an earlier template, not only those of the profiles using the store now, so an entry is not removed under a pod whose profile has switched the store off or left the fleet) in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. +# The capability store (the role README's "The capability store"), when STORE_ROOT is set, holds one entry per image payload, named - (toolcache, sysroot or externals), plus merged sysroots named merged-sysroot-, and the copies pods are writing under .incoming/... An entry stays while a scale set's pod template in one of STORE_NAMESPACES, or an unfinished pod of those namespaces on this node, lists it (STORE_NAMESPACES holds every namespace a pod may still run in from an earlier template, not only those of the profiles using the store now, so an entry is not removed under a pod whose profile has switched the store off or left the fleet) in its github-runner.exadev.io/capability-store-keys annotation; any other entry is renamed into .trash and deleted, after the lists are taken again directly before the rename, as for a cache. A copy under .incoming stays while the pod whose uid it carries is unfinished on this node. A failed list means no store pruning that round, since an unknown template or pod may use any entry. # # Which pods count: those in the profile's namespace on this node, not finished (Succeeded or Failed), that have a hostPath volume at exactly the profile's directory on the host (HOST_CACHE_ROOT//). Another profile's pods, finished pods and pods that do not mount the cache are ignored. # diff --git a/tests/unit/test_node_cache_pruner.py b/tests/unit/test_node_cache_pruner.py index 14a207c..502eba7 100644 --- a/tests/unit/test_node_cache_pruner.py +++ b/tests/unit/test_node_cache_pruner.py @@ -597,9 +597,10 @@ def test_valid_settings_start_normally_and_a_gibibyte_limit_is_converted(self) - -KEY_A = "a" * 64 -KEY_B = "b" * 64 -KEY_C = "c" * 64 +# The shapes of the entry names the role produces: a tool cache, a merged sysroot and the dind sidecar's externals (see arc_capability_store_key and _capability_store_merged_key). A name sorts by its prefix, so the expected lists below are sorted rather than written in the order of the keys. +KEY_A = "toolcache-" + "a" * 64 +KEY_B = "merged-sysroot-" + "b" * 64 +KEY_C = "externals-" + "c" * 64 class CapabilityStorePrunerTest(unittest.TestCase): @@ -635,12 +636,12 @@ def test_an_entry_no_template_or_pod_uses_is_removed_and_one_a_template_lists_is def test_an_entry_an_unfinished_pod_on_this_node_lists_is_kept_after_its_template_moved_on(self) -> None: _, store = self.run_store([KEY_A, KEY_B, KEY_C], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_C]}], pods=[self.store_pod([KEY_A]), self.store_pod([KEY_B], phase="Pending", uid="pod-2")]) - self.assertEqual(self.remaining(store), [KEY_A, KEY_B, KEY_C]) + self.assertEqual(self.remaining(store), sorted([KEY_A, KEY_B, KEY_C])) def test_a_pod_in_a_namespace_that_left_the_store_profiles_keeps_its_entries_while_the_namespace_is_listed(self) -> None: # arc-gone has no scale set any more but a pod made from its old template runs on the node: the role lists the namespace for as long as the pruner holds a Role there. _, store = self.run_store([KEY_A, KEY_B], scale_sets=[{"namespace": "arc-a", "store_keys": [KEY_B]}], pods=[self.store_pod([KEY_A], namespace="arc-gone")], STORE_NAMESPACES="arc-a arc-gone") - self.assertEqual(self.remaining(store), [KEY_A, KEY_B]) + self.assertEqual(self.remaining(store), sorted([KEY_A, KEY_B])) def test_a_pod_in_a_namespace_that_is_not_listed_keeps_nothing(self) -> None: # What the old scan did to arc-gone: the namespace dropped out of STORE_NAMESPACES and the entry went under the running pod. From a6174c25f93857ac49be69817baae6a8d20c447e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 14:15:20 +0100 Subject: [PATCH 38/43] test(arc): count lines in the capabilities test without relying on wc's padding --- tests/capabilities/run.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/capabilities/run.sh b/tests/capabilities/run.sh index 6017282..ca6c2b9 100755 --- a/tests/capabilities/run.sh +++ b/tests/capabilities/run.sh @@ -28,6 +28,8 @@ work="$(mktemp -d "${TMPDIR:-/tmp}/grtest-capabilities.XXXXXX")" export KUBECONFIG="$work/kubeconfig" log() { echo "==> $*"; } +# The number of lines in "$1", without padding: BSD wc (macOS) pads its count with spaces, which a string comparison does not ignore. +line_count() { printf '%s\n' "$1" | wc -l | tr -d '[:space:]'; } fail() { echo "FAIL: $*" >&2 kubectl -n "$namespace" get pods -o wide >&2 || true @@ -347,7 +349,7 @@ kubectl -n "$namespace" apply -f "$work/store-pod-store-a.yaml" >/dev/null wait_for_log store-a STORE-STEP-DONE expected_entries="$(kubectl -n "$namespace" get pod store-a -o jsonpath='{.metadata.annotations.github-runner\.exadev\.io/capability-store-keys}' | tr ' ' '\n' | LC_ALL=C sort)" -[ "$(printf '%s\n' "$expected_entries" | wc -l)" = 3 ] || fail "the pod's annotation does not list the tool, the library and the merged sysroot: '$expected_entries'" +[ "$(line_count "$expected_entries")" = 3 ] || fail "the pod's annotation does not list the tool, the library and the merged sysroot: '$expected_entries'" [ "$(store_entries)" = "$expected_entries" ] || fail "the store holds '$(store_entries)', not exactly the entries the pod's annotation lists: '$expected_entries'" [ -z "$(docker exec "$node" ls -A "$store_host_path/.incoming")" ] || fail "a copy was left in the incoming directory after the pod published it" [ "$(docker exec "$node" stat -c %u "$store_host_path/.incoming")" = 1001 ] || fail "the incoming directory is not the runner user's" @@ -359,7 +361,7 @@ kubectl -n "$namespace" apply -f "$work/store-pod-store-b.yaml" >/dev/null wait_for_log store-b STORE-STEP-DONE [ "$(store_state)" = "$state_before" ] || fail "the second pod changed the store, which it should only have read: $(diff <(echo "$state_before") <(store_state))" listings="$(for pod in store-a store-b; do kubectl -n "$namespace" logs "$pod" -c runner | sed -n 's/^STORE-LISTING-[A-Z]* //p'; done | sort -u)" -[ "$(printf '%s\n' "$listings" | wc -l)" = 1 ] || fail "the two pods saw different content in the entry: $listings" +[ "$(line_count "$listings")" = 1 ] || fail "the two pods saw different content in the entry: $listings" [ "$(kubectl -n "$namespace" get pod store-b -o jsonpath='{.metadata.annotations.github-runner\.exadev\.io/capability-store-keys}' | tr ' ' '\n' | LC_ALL=C sort)" = "$expected_entries" ] || fail "the two pods list different store entries" kubectl -n "$namespace" logs store-a -c runner | grep -q '^tool-cache find: ' || fail "@actions/tool-cache did not find the tool in the pod's tool cache" From 2eb4d6dbe71cb8bcb15bdb85703faf15b677331d Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 15:19:58 +0100 Subject: [PATCH 39/43] test(arc): cover a tool alias that sorts before its target in the tool-cache link script --- tests/unit/test_capabilities.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 7dd5eff..e05d2d8 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -1550,6 +1550,27 @@ def test_a_tool_or_version_directory_that_is_a_link_is_copied_as_the_link_and_no self.assertEqual(self.snapshot(self.root), before) self.assertFalse((entry / "node/22.0.0/x64/created").exists()) + def test_a_tool_alias_that_sorts_before_its_target_is_copied_as_the_link_and_no_tree_is_made_through_it(self) -> None: + """The glob reaches lts/22/x64 through the links, and it sorts before node/22.0.0/x64, so the target is not built when the alias comes up: without the guard, making the alias's parent runs through the dangling copy of the link and the init container fails.""" + entry = self.linked_tool_names() + (entry / "nodejs").unlink() + (entry / "lts").symlink_to("node") + (entry / "other/1.0/x64/bin").mkdir(parents=True) + (entry / "other/1.0/x64/bin/other").write_text("other") + (entry / "node/stable").symlink_to("../other/1.0") + cache = self.work / "toolcache-pod" + cache.mkdir() + self.link_tool_cache(cache) + self.assertEqual((cache / "lts").readlink(), Path("node")) + self.assertEqual((cache / "node/22").readlink(), Path("22.0.0")) + self.assertEqual((cache / "node/stable").readlink(), Path("../other/1.0")) + # Exactly one real tree for each of the two images' versions, and nothing made through an alias. + real = sorted(Path(directory).relative_to(cache).as_posix() for directory, names, _ in os.walk(cache) if Path(directory).name == "x64") + self.assertEqual(real, ["node/22.0.0/x64", "other/1.0/x64"]) + self.assertEqual(sorted(path.name for path in cache.iterdir()), ["lts", "node", "other"]) + self.assertEqual(sorted(path.name for path in (cache / "node").iterdir()), ["22", "22.0.0", "stable"]) + self.assertEqual((cache / "lts/stable/x64/bin/other").read_text(), "other") + def test_a_tool_directory_that_is_a_link_alone_and_a_version_directory_that_is_a_link_alone_are_each_copied_as_the_link(self) -> None: for case, tool_link, version_link in (("tool", True, False), ("version", False, True)): with self.subTest(case): From 742db365fe7b28917518e45dbb63425314b484f3 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 15:20:22 +0100 Subject: [PATCH 40/43] refactor(arc): stop registering the capability store key function as a filter no task uses --- plugins/filter/arc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 69b0392..5e147df 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1929,7 +1929,6 @@ def filters(self) -> dict[str, Any]: "arc_capability_store_externals": arc_capability_store_externals, "arc_capability_store_host_path_errors": arc_capability_store_host_path_errors, "arc_capability_store_path_errors": arc_capability_store_path_errors, - "arc_capability_store_key": arc_capability_store_key, "arc_capability_store_image_errors": arc_capability_store_image_errors, "arc_capability_store_namespaces": arc_capability_store_namespaces, "arc_host_path_overlap_errors": arc_host_path_overlap_errors, From 7d55ede1cc9cc36f58440bca0a90c060446399e1 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 15:21:48 +0100 Subject: [PATCH 41/43] fix(arc): copy an image shared by two capabilities once into the store --- plugins/filter/arc.py | 6 +++++- tests/unit/test_capabilities.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 5e147df..8da9645 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1361,6 +1361,7 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s """Return the init containers that put a profile's tool-cache and system-library payloads in the store and give the pod its view of them, with the store entries the pod uses.""" root = store["path"].rstrip("/") copies: list[dict[str, Any]] = [] + copied: set[str] = set() tool_keys: list[str] = [] sysroot_keys: list[str] = [] for name in names: @@ -1372,7 +1373,10 @@ def _capability_store_init_containers(names: Sequence[str], catalogue: Mapping[s key = arc_capability_store_key(image, _STORE_KIND_TOOL_CACHE if is_tool else _STORE_KIND_SYSROOT) assert key is not None payload = _TOOL_IMAGE_PAYLOAD if is_tool else _SYSROOT_IMAGE_PAYLOAD - copies.append(_capability_store_copy_container(f"{_CAPABILITY_CONTAINER_PREFIX}{name}", image, payload, key, store, runner_user, runner_group)) + # Two capabilities on one image name one entry, so only the first copies it: a second copy would write the same bytes again to be deleted at publishing. + if key not in copied: + copied.add(key) + copies.append(_capability_store_copy_container(f"{_CAPABILITY_CONTAINER_PREFIX}{name}", image, payload, key, store, runner_user, runner_group)) (tool_keys if is_tool else sysroot_keys).append(key) # Two capabilities may name one image, and the merge links each entry's files into the merged directory, so an entry named twice would be linked over itself (cp reports the same file). Each entry stays at its last position, where its files win over the earlier entries' as they did when every capability was copied into the pod in order, so the merged entry's name, a hash of the order, is that of the order that merges the same files. sysroot_keys = list(reversed(list(dict.fromkeys(reversed(sysroot_keys))))) diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index e05d2d8..4887047 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -812,6 +812,20 @@ def test_a_system_library_entry_named_again_keeps_its_last_position_so_its_files self.assertEqual(merge["command"][6:], [KEY_C, KEY_B]) self.assertNotEqual(merge["command"][5], self.merge_of(["a", "b"], catalogue)["command"][5]) + def test_two_capabilities_on_one_image_get_one_copy_container_for_the_entry_and_the_first_names_it(self) -> None: + for field, payload in (("image", "/toolcache"), ("sysroot_image", "/sysroot")): + with self.subTest(field): + image = f"ghcr.io/example/shared:1@sha256:{DIGEST_A}" + catalogue = {"first": {field: image}, "second": {field: image}, "third": {field: f"ghcr.io/example/other:1@sha256:{DIGEST_B}"}} + result = store_pod(DEFAULT_VALUES, ["first", "second", "third"], catalogue) + self.assertEqual(result["errors"], []) + containers = result["values"]["template"]["spec"]["initContainers"] + copies = [container for container in containers if container["name"].startswith("capability-") and not container["name"].startswith("capability-store-")] + self.assertEqual([container["name"] for container in copies], ["capability-first", "capability-third"]) + self.assertEqual(copies[0]["command"][6:], [payload, "capability-first"]) + publish = next(container for container in containers if container["name"] == "capability-store-publish") + self.assertEqual(publish["command"][4:], ["/opt/capability-store", f"{'toolcache' if field == 'image' else 'sysroot'}-{DIGEST_A}", f"{'toolcache' if field == 'image' else 'sysroot'}-{DIGEST_B}"]) + def test_one_image_used_for_a_tool_cache_and_a_sysroot_is_two_entries_each_copied_from_its_own_payload(self) -> None: image = f"ghcr.io/example/both:1@sha256:{DIGEST_A}" result = store_pod(DEFAULT_VALUES, ["tools", "libraries"], {"tools": {"image": image}, "libraries": {"sysroot_image": image}}) From 58e396e58c723650fe937ab299fcaa862dbf0aa6 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 15:26:15 +0100 Subject: [PATCH 42/43] docs(arc): state the capability store's copy cost on a cold node and its disk footprint exactly --- plugins/filter/arc.py | 8 ++++---- roles/github_runner_arc/README.md | 14 +++++++------- roles/github_runner_arc/defaults/main.yml | 4 ++-- roles/github_runner_arc/meta/argument_specs.yml | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 8da9645..72c78c7 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -621,7 +621,7 @@ def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, Args: require_app_id: whether each org must set app_id. The role needs it only when it writes the App Secret; otherwise the App's id is in the existing App Secret, and an app_id that is set is only checked against it. - orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile may set capability_store (a boolean, default false) to copy its tool-cache and system-library payloads, and the dind sidecar's externals, once per node into the capability store instead of into every pod (see arc_capability_pod). A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. + orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label. A profile may set capability_store (a boolean, default false) to copy its tool-cache and system-library payloads, and the dind sidecar's externals, into the capability store, which the pods of a node share, instead of into every pod (see arc_capability_pod). A profile with sizing may set burst (node_label_key, node_label_values, tolerations, max_runners) to let its runner pods overflow onto burst nodes, and a profile without sizing may set burst with exclusive true to run on burst nodes only, with burst.max_runners as its maxRunners; see arc_runner_placement. Returns: A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, target (namespace/release, joined), app_secret, runner_image (its org's image), scale_set_labels, values_file, node_selector, autoscale, sizing, burst (the burst settings with defaults filled in, or None), burst_runners (burst.max_runners, 0 without burst), max_runners (the static maxRunners the role sets, including burst_runners, or None), capabilities (the names of the capabilities its runner pods carry, in order; see arc_capability_pod), capability_store (whether it uses the capability store), label and settings, the profile's own keys) and ``autoscaled`` (every profile flagged autoscale, sharing one autoscaler-managed memory pool across their combined scale sets; empty when none are). @@ -1118,7 +1118,7 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca # --- The capability store: one copy per node of what every runner pod would otherwise copy at its start --- -# A pod's tool-cache and system-library copies (and the runner's externals for the dind sidecar) are the same bytes on every pod of a profile, taken from digest-pinned images. With a profile's capability_store on, each image's payload is copied once per node into /-, where is the image's sha256 digest and the payload it holds (see arc_capability_store_key), and pods mount the store read-only instead of writing their own copy. The store root is a hostPath of its own, apart from the node cache, whose last component names it. +# A pod's tool-cache and system-library copies (and the runner's externals for the dind sidecar) are the same bytes on every pod of a profile, taken from digest-pinned images. With a profile's capability_store on, each image's payload is copied into /- (by every pod that starts before the entry is published, the first publish winning, and by none after it), where is the image's sha256 digest and the payload it holds (see arc_capability_store_key), and pods mount the store read-only instead of writing their own copy. The store root is a hostPath of its own, apart from the node cache, whose last component names it. _CAPABILITY_STORE_HOST_PATH_NAME = re.compile(r"^(?:capability-store|[a-z0-9][a-z0-9-]*-capability-store)$") # A copy is written under /.incoming/.. and published with one rename onto /: rename(2) onto a non-empty directory fails, so of two pods copying the same image at once the first rename wins and the other's copy is deleted, and a reader never sees a part-written entry, since an entry exists only once it is complete. The pod uid comes from the downward API and the container name keeps two copies of one image within a pod apart. _CAPABILITY_STORE_INCOMING = ".incoming" @@ -1272,7 +1272,7 @@ def _capability_store_copy_container(name: str, image: str, payload: str, key: s } -# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there, saying why the rename failed when it did: mv's message is held back while the entry may be there (a lost race fails the rename too, and is no error) and printed only when it is not. Every copy is flushed to disk (sync) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. That sync is sync(2), which writes out every dirty page on the host and not only this copy's, and waits for it: busybox's sync takes no argument (given one it prints a warning and ignores it, so it cannot be scoped to the store's filesystem), and its fsync applet syncs one file or directory per call, which for a tree of many small files is one journal commit each where sync is one pass over the dirty pages, so scoping the flush would cost more work than it saves. The host-wide flush is accepted because it runs only in a pod that copied the entry, once per entry per node and not once per pod (a pod that finds the entry in the store copies and flushes nothing). If the store image's busybox ever gains a scoped sync (sync -f on a path), use it here and in the sysroot merge. +# Publishes each key's copies from this pod: a rename onto / for the first, and deletion for a copy whose key another pod published first. Then fails unless every key is in the store, so a pod never starts on an entry that is not there, saying why the rename failed when it did: mv's message is held back while the entry may be there (a lost race fails the rename too, and is no error) and printed only when it is not. Every copy is flushed to disk (sync) before its rename: a rename that reaches the disk ahead of the data it names leaves, after a power loss or crash, an entry that exists but holds empty files, and since an entry is named for an image digest and never revalidated, every later pod would use it. With the data written first, an entry that exists is complete whatever the crash, and a rename that did not reach the disk leaves no entry, which the next pod copies again. A completeness marker the readers check was left out: it would restate what the order of writes already guarantees, and cost every pod a check at its start. That sync is sync(2), which writes out every dirty page on the host and not only this copy's, and waits for it: busybox's sync takes no argument (given one it prints a warning and ignores it, so it cannot be scoped to the store's filesystem), and its fsync applet syncs one file or directory per call, which for a tree of many small files is one journal commit each where sync is one pass over the dirty pages, so scoping the flush would cost more work than it saves. The host-wide flush is accepted because it runs only in a pod that copied the entry, which is every pod that starts before the entry is published and none after it (a pod that finds the entry in the store copies and flushes nothing). If the store image's busybox ever gains a scoped sync (sync -f on a path), use it here and in the sysroot merge. _PUBLISH_SCRIPT = ( 'root=$1; shift; for key in "$@"; do moved=;' ' for incoming in "$root/' + _CAPABILITY_STORE_INCOMING + '/$key.$POD_UID".*; do [ -e "$incoming" ] || continue; sync || exit 1; moved=$(mv -T "$incoming" "$root/$key" 2>&1) || { ' + _MAKE_INCOMING_WRITABLE + ' rm -rf "$incoming"; }; done;' @@ -1476,7 +1476,7 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay. Args: - values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it once per node instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, rebuilt as real directories holding links to the store's files (see _TOOL_CACHE_LINK_SCRIPT), the sysroot is the store's merged entry mounted read-only, the runner container mounts, read-only, only the tool-cache entries the tool tree links into (each at /) and that merged entry, never the store's root, and the pod template is annotated with the entries it uses. + values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. store: the profile's capability store settings ({host_path, path, image}: the store's root on the node, where it is mounted in the pod, and the digest-pinned busybox image that publishes and links its entries), or None when the profile copies into each pod. With the store, each tool-cache and system-library payload is copied into it, to be shared by the pods of a node, instead of into each pod (see arc_capability_store_externals for the store's layout): the tool cache stays the pod's own emptyDir, rebuilt as real directories holding links to the store's files (see _TOOL_CACHE_LINK_SCRIPT), the sysroot is the store's merged entry mounted read-only, the runner container mounts, read-only, only the tool-cache entries the tool tree links into (each at /) and that merged entry, never the store's root, and the pod template is annotated with the entries it uses. Returns: A dict with ``errors`` (empty when the values can carry the capabilities), ``values`` (the values with the capabilities added, or as given when there are none or errors) and ``hooks`` (the ConfigMap's data apart from the dispatcher, which the role adds from its own file; empty when the profile needs no ConfigMap). diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index ee1c354..e4eea1d 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -26,11 +26,11 @@ Before touching the cluster the role then checks the secrets it is about to writ - `autoscale`: `true` makes the profile's scale set one of those the autoscaler manages, pooled against one memory budget; at most one autoscaled profile in a cluster may set `sizing`. - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it bounds the ceiling the autoscaler works out from the nodes, leaving `maxRunners` as the floor. - `capabilities`: names of entries in `github_runner_arc_capabilities` that the profile's runner pods carry, in order (see [Capabilities](#capabilities)). - - `capability_store`: `true` copies the capabilities' tool-cache and system-library payloads once per node into the capability store instead of into every pod, and does the same for the dind sidecar's copy of the runner's externals when the profile's values write that sidecar out with an `init-dind-externals` init container (a values file, or the registry cache's wiring), but not when the chart builds the sidecar itself from `containerMode: dind` (see [The capability store](#the-capability-store)). Default `false`. + - `capability_store`: `true` copies the capabilities' tool-cache and system-library payloads into the capability store, which the pods of a node share, instead of into every pod, and does the same for the dind sidecar's copy of the runner's externals when the profile's values write that sidecar out with an `init-dind-externals` init container (a values file, or the registry cache's wiring), but not when the chart builds the sidecar itself from `containerMode: dind` (see [The capability store](#the-capability-store)). Default `false`. - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`. - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. -- `github_runner_arc_node_cache_path`, `github_runner_arc_node_cache_host_path`, `github_runner_arc_node_cache_size_limit_gib`, `github_runner_arc_node_cache_hard_limit_factor`, `github_runner_arc_node_cache_prune_seconds`: where a node-cache capability's volume is mounted on the runner container (`/node-cache`), the dedicated directory on each node under which each profile's cache lives (`/var/lib/github-runner/node-cache`), the most gibibytes a profile's cache may hold before the pruner empties it (20; 0 for no pruner, a negative value is an error), the hard limit as a multiple of that (2) past which a busy cache is reported on every round, and the seconds between the pruner's checks (300, at least 1). See [Capabilities](#capabilities). +- `github_runner_arc_node_cache_path`, `github_runner_arc_node_cache_host_path`, `github_runner_arc_node_cache_size_limit_gib`, `github_runner_arc_node_cache_hard_limit_factor`, `github_runner_arc_node_cache_prune_seconds`: where a node-cache capability's volume is mounted on the runner container (`/node-cache`), the dedicated directory on each node under which each profile's cache lives (`/var/lib/github-runner/node-cache`), the most gibibytes a profile's cache may hold before the pruner empties it (20; 0 for no pruner, which a profile with `capability_store` refuses, since the pruner removes the store's unused entries; a negative value is an error), the hard limit as a multiple of that (2) past which a busy cache is reported on every round, and the seconds between the pruner's checks (300, at least 1). See [Capabilities](#capabilities). - `github_runner_arc_capability_store_host_path`, `github_runner_arc_capability_store_path`, `github_runner_arc_capability_store_image`: the dedicated directory on each node the capability store lives in (`/var/lib/github-runner/capability-store`, with the same rules as the node cache's, its last component `capability-store` or ending in `-capability-store`, and neither the same directory as the node cache's nor inside it or containing it, since the pruner removes what it finds directly under each as root), where the init containers mount it (`/opt/capability-store`) and under which the runner container mounts, read-only, each entry its pod uses (`/opt/capability-store/`), and the digest-pinned busybox image its own init containers run. See [The capability store](#the-capability-store). - `github_runner_arc_registry_cache_enabled` (default `false`) and the other `github_runner_arc_registry_cache_*` variables: a pull-through registry cache every Docker-in-Docker profile pulls through, falling back to each registry when the cache is down (see [Registry cache](#registry-cache)). - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`. @@ -142,7 +142,7 @@ A pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image, o - **A bad setting stops the pruner before it does anything.** The pruner's start-up checks end it with status 2 and a message naming the variable, and the DaemonSet's pods then crash-loop with that message in their log. The role refuses the numeric settings it templates by the same pattern and maxima (the size limit, the hard limit factor and the check interval: positive integers without sign, leading zero or fraction, up to 1073741824 GiB, 1024 and 86400), so only a value changed some other way, such as by editing the DaemonSet, reaches that check. - **A failure never stops the others:** a size that cannot be read skips that profile for the round, and a pod list that fails or is not a pod list means no pruning (`NODE_CACHE_UNPRUNABLE` when it is refused as Forbidden, `NODE_CACHE_LIST_FAILED` otherwise), since an unknown set of pods may be using the cache. -The limits are per profile and the node's disk is shared by all of them, so a node's worst case is the sum of the limits of the profiles with a cache on it, times the hard limit factor. Size the limit so that sum fits the node's free disk. The pruner can list pods through a Role and RoleBinding per profile namespace (named for the release, not a cluster-wide grant) and, in each namespace the capability store's references are read from, list pods and `autoscalingrunnersets` through a second Role and RoleBinding (`node-cache-pruner:capability-store`); it has no access to nodes, and runs as root with only `DAC_OVERRIDE` and `FOWNER` (to delete files and sticky-bit directories the runner's user made), a read-only root filesystem and no privilege escalation. +The limits are per profile and the node's disk is shared by all of them, so a node's worst case is the sum of the limits of the profiles with a cache on it, times the hard limit factor. Size the limit so that sum fits the node's free disk. The capability store is outside that sum and that limit: it holds one entry for each image payload the scale sets' templates and unfinished pods on the node list, however many gibibytes those are, and the size limit does not bound it; the pruner removes every other entry (and any copy left under `.incoming` by a pod that has finished or gone) on each round, so the store's footprint is the referenced entries alone, until the last profile with `capability_store` switches it off and the operator deletes the directory (see [The capability store](#the-capability-store)). The pruner can list pods through a Role and RoleBinding per profile namespace (named for the release, not a cluster-wide grant) and, in each namespace the capability store's references are read from, list pods and `autoscalingrunnersets` through a second Role and RoleBinding (`node-cache-pruner:capability-store`); it has no access to nodes, and runs as root with only `DAC_OVERRIDE` and `FOWNER` (to delete files and sticky-bit directories the runner's user made), a read-only root filesystem and no privilege escalation. A read-only mount for sensitive profiles was considered and is not offered: a profile's cache is its own and only its own jobs fill it, so a read-only copy of it would never be filled. A profile that must not share a cache with other jobs of its kind should not have the capability. @@ -305,17 +305,17 @@ When a profile has a hook capability, a `path` or a system-library capability, t The tool cache, the sysroot, the node cache and the hook are on the runner container, so they serve the steps that run there: every step in the default container mode and in Docker-in-Docker mode (`container_mode: dind`). A job that runs in a `container:` reaches the tool cache only through the runner's own mapping of it, which in Docker-in-Docker mode points at the Docker daemon's filesystem, where the cache is not mounted, and it does not see the sysroot at all. The `kubernetes` container modes run every step in a separate pod, so the role refuses capabilities on a profile whose values use them. It also refuses them when the values define no runner container (the role's own template does), when the runner container already sets `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, when a volume already uses the name `tool-cache`, `sysroot` or `job-started-hooks`, or, for a profile with a system-library capability, when `github_runner_arc_sysroot_path` is not an absolute directory without a `:` or overlaps the tool cache or the hooks directory. During validation it refuses a system-library capability whose `env` sets one of the search paths, `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, a name that is not a variable name or a value that is not a single-line string (a line break would add a line to `GITHUB_ENV`), and two of a profile's system-library capabilities that set one variable to different values, since they share one sysroot. A values file's own init containers, volumes, environment and mounts are kept, and the capabilities' are added after them, by the `exadev.github_runner.arc_capability_pod` filter. `exadev.github_runner.arc_capability_errors` checks the catalogue and each profile's references during validation, before anything changes. -Each tool-cache and system-library capability adds its copy to every pod's start. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage, and every pod start writes all of it. A profile with `capability_store: true` writes it once per node instead (see [The capability store](#the-capability-store)). +Each tool-cache and system-library capability adds its copy to every pod's start. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage, and every pod start writes all of it. A profile with `capability_store: true` writes each payload at most once per node once its entry is published, instead (see [The capability store](#the-capability-store)). ### The capability store -Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied once per node into `github_runner_arc_capability_store_host_path` and the pods use that copy. +Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied into `github_runner_arc_capability_store_host_path`, at most once per node once its entry is published, and the pods use that copy. Pods that start together before an entry is published each copy it (and each runs the host-wide `sync` below), the first publish wins and the others delete their copies, so the first pods on a node still pay the copy; only later pods pay nothing. Switching a profile to the store restarts its idle and pending runners (ARC replaces them when the scale set's template changes; runners running a job are untouched), so the first wave on a node whose store is cold copies as before, all at once. Two capabilities that use one image share one copy container. An entry is named for its image's digest and the kind of payload copied from it (`toolcache-`, `sysroot-`, `externals-`), so one image used for two payloads, such as a runner image that is also a tool image, is two entries that cannot be taken for each other, and every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. `tasks/validate.yml` refuses a profile with `capability_store: true` whose capability image is not pinned by digest, whose store path overlaps the tool cache, sysroot, hooks or node cache directory, or, in dind mode with the registry cache and no values file, whose organisation's runner image is not pinned by digest (a values file's own externals copy is checked when it is rendered, since its content is not known earlier), so the failure comes before any install step. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. - Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. -- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, once per entry per node, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. +- `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, which is every pod that starts before the entry is published and none after it, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `merged-sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. A tool or version directory that is itself a symlink (an image aliasing `node` and `nodejs`, or `22` and `22.0.0`) is copied as the link too, and nothing below it is walked, so the pod holds one real tree and a job's write through one name shows through the other. An `` directory that is itself a symlink (an image aliasing `x64` and `amd64`) is copied as the link too, so it names the real directory the pod's tree holds beside it, and its `.complete` marker is copied like any other. Everything else stays a link, so a pod still writes only the launchers' bytes. @@ -325,7 +325,7 @@ What a job can change, with the store's entries mounted read-only on the runner `arc_capability_store_externals`, which the role runs last, after the registry cache, moves a values file's (or the registry cache's) `init-dind-externals` copy into the store the same way, named for the runner image's digest (`externals-`), and mounts that entry read-only into the dind container in place of the `dind-externals` volume, whether the dind container is an ordinary container or a native sidecar under `initContainers` (the layout the registry cache writes out). Values that leave the dind sidecar to the chart's own `containerMode: dind` have no copy in the values to move. -The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so while some profile still uses the store, a profile that has switched it off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. The store is pruned only while some profile has `capability_store: true`: once the last profile switches it off, the pruner, installed or not, no longer reads or mounts the store, so every node keeps its entries and `.incoming` until the operator deletes the directory (`sudo rm -rf ` on each node, once no pod made from a template that used the store is still running; the next profile to switch the store on recreates it). Emptying it automatically was left out: the role cannot tell which nodes hold a store, and the pruner could only do it safely while the grants that list scale sets and pods stayed in place for every namespace, which would keep a grant for a store no profile uses. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. +The pod template carries the entries it uses in the annotation `github-runner.exadev.io/capability-store-keys`. The node cache pruner removes an entry that no scale set's pod template lists and no unfinished pod on its node lists, in any of the namespaces a pod may still run in from an earlier template (every fleet profile's namespace and every namespace holding one of the pruner's per-profile Roles, so while some profile still uses the store, a profile that has switched it off or left the fleet keeps its entries until its pods are gone), after listing both again directly before the rename, as it does for a cache; it removes a copy left under `.incoming` once the pod whose uid it carries has finished or gone. A list that fails means no store pruning that round. The store is pruned only while some profile has `capability_store: true`: once the last profile switches it off, the pruner, installed or not, no longer reads or mounts the store, so every node keeps its entries and `.incoming` until the operator deletes the directory (`sudo rm -rf ` on each node, once no pod made from a template that used the store is still running; the next profile to switch the store on recreates it). Emptying it automatically was left out: the role cannot tell which nodes hold a store, and the pruner could only do it safely while the grants that list scale sets and pods stayed in place for every namespace, which would keep a grant for a store no profile uses. So the store needs the pruner, and a profile with `capability_store: true` is refused while `github_runner_arc_node_cache_size_limit_gib` is 0 or `github_runner_arc_node_label_key` is empty, since the pruner runs only on the nodes carrying that label. While some profile uses the store, each of those namespaces gets a second Role and RoleBinding, `node-cache-pruner:capability-store`, that lists pods and `autoscalingrunnersets.actions.github.com` and nothing else; the role removes it, and its binding, from every namespace the references are no longer read from, which is every one once no profile uses the store, so the pruner never keeps a grant to list scale sets for nothing. A namespace that is Terminating is left out of those namespaces and loses the grant, since the API server refuses to create a Role there and its pods are being deleted. A departed profile's own Role, which lists pods only, stays for as long as the pruner is installed, which is what keeps that profile's namespace among the namespaces read. The store case of `tests/capabilities/run.sh` runs real pods in kind, with the store image's own busybox and the runner user's real identity, which the unit tests cannot show: it pushes the two images to a registry container in the test (the store names an entry for an image's digest, which an image loaded into kind has none of) and runs two pods of a profile with `capability_store: true` on the same node, the first against a store root the runner user already owns with no incoming directory in it (the state the preparation's `DAC_OVERRIDE` is for), and checks that the entries are exactly the ones the pods' annotation lists, that the second pod changes nothing in the store, that both pods see the same content, that a job can change the tool cache's tree but is refused (read-only filesystem) a write through a link, into the store and into the sysroot, that the store path in the runner container holds only the pod's own tool entry, and that the node cache pruner, run once from the role's own rendered manifests in a pod of its own, keeps every entry the running pods list, removes an entry and a copy that nothing lists, and removes the pods' entries once the pods have finished. The unit tests run the same init container commands with the host's `sh` and coreutils (`tests/unit/test_capabilities.py`, which needs GNU's for `mv -T` and `cp -l` where the host's are not), the pruner's script with the host's `bash` (`tests/unit/test_node_cache_pruner.py`), and the role's tasks against a stand-in for the cluster, not busybox or a real one. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index 33d2270..6e2afc1 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -25,14 +25,14 @@ github_runner_arc_node_cache_path: "/node-cache" # Where on the node the cache lives, as the directory every profile's own directory (/) sits under, so profiles never share a cache. It must be a dedicated directory the role may create and empty as root: written without a repeated or trailing slash (every path of the cache is built from this one string), at least three levels deep, with a last component ending in node-cache, and not inside a system directory such as /etc or /var/lib/kubelet. A symbolic link at or above the path defeats this validation. Each profile's directory is a hostPath volume created when a pod first needs it. Its lifetime is the node's: it survives pods, and is emptied by the pruner below when it outgrows its limit. github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" -# Where on the node the capability store lives, for profiles with capability_store on: one entry per image a profile's capabilities (and the dind sidecar's externals) copy from, named for the image's digest and the kind of payload copied from it, written once per node and mounted read-only by the pods. A dedicated directory the role creates and the pruner removes entries from as root, with the same rules as the node cache's root, whose last component is capability-store or ends in -capability-store. +# Where on the node the capability store lives, for profiles with capability_store on: one entry per image a profile's capabilities (and the dind sidecar's externals) copy from, named for the image's digest and the kind of payload copied from it, shared by the pods of a node and mounted read-only by them. A dedicated directory the role creates and the pruner removes entries from as root, with the same rules as the node cache's root, whose last component is capability-store or ends in -capability-store. github_runner_arc_capability_store_host_path: "/var/lib/github-runner/capability-store" # Where the store's root is mounted in the init containers, and under which the runner container mounts, read-only, each entry its pod uses as / (the tool cache's links point into those): the runner container never mounts the root, so a job cannot read another profile's entries. github_runner_arc_capability_store_path: "/opt/capability-store" # The image the store's own init containers run (creating the root, flushing and publishing a copy with sync and mv -T, merging the sysroot with cp -al and linking the tool cache), pinned by digest since one of them runs as root on a hostPath. Its sh, mkdir, chmod, chown, sync, mv, cp, find and rm must be busybox's or GNU's: mv -T refuses to move onto a directory that is not empty, which publishing relies on. github_runner_arc_capability_store_image: "docker.io/library/busybox:1.37.0@sha256:bdf57e528e45e4433820e045b29b4597825a1c9e38353532d90a01445013f82e" -# The most gibibytes one profile's cache may hold on a node before the pruner empties it. The limit is per profile, so a node's worst case is the sum of the limits of the profiles with a cache on it, times github_runner_arc_node_cache_hard_limit_factor. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) empties a cache only while no runner pod of its profile runs on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow; a negative value, a leading zero, a fraction or a value above 1073741824 is an error (the pruner's own arithmetic bounds it, and the role refuses what the pruner would). +# The most gibibytes one profile's cache may hold on a node before the pruner empties it. The limit is per profile, so a node's worst case is the sum of the limits of the profiles with a cache on it, times github_runner_arc_node_cache_hard_limit_factor. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) empties a cache only while no runner pod of its profile runs on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow, which a profile with capability_store refuses (the pruner removes the store's unused entries); a negative value, a leading zero, a fraction or a value above 1073741824 is an error (the pruner's own arithmetic bounds it, and the role refuses what the pruner would). github_runner_arc_node_cache_size_limit_gib: 20 # The hard limit, as a multiple of the size limit. A cache that a runner pod keeps busy cannot be emptied, and one over the hard limit is reported on every round by a log line starting NODE_CACHE_OVER_HARD_LIMIT, for whatever collects the pruner's logs to alert on. The pruner changes nothing about the node to force the emptying: draining the node or lowering its runner count lets the next round do it, and kubelet's disk-pressure eviction still protects the node itself. diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index b279237..875dc97 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -13,7 +13,7 @@ argument_specs: description: - Orgs whose runner scale sets this host installs. Each entry has name, app_id (needed only when the role writes the App Secret; otherwise the existing Secret supplies it), image, private_key or private_key_op_reference, and optionally installation_id and app_secret_name (the App Secret's name, default -github-app). - Each entry's scale_set_profiles is a non-empty list of profiles, each with suffix and optionally namespace (default arc-runners-), release_name (default -runners), values_file, max_runners, min_runners, container_mode, resources, node_selector, runs_on_label, scale_set_labels, capabilities, capability_store, tolerations, autoscale, sizing and burst. - - capability_store, a boolean (default false), makes the profile's runner pods copy each tool-cache and system-library capability's payload, and the runner's externals for a Docker-in-Docker sidecar the values write out, once per node into the capability store (github_runner_arc_capability_store_host_path) instead of into every pod's own volumes, and mount the result read-only. Every image those copies come from must be pinned by digest, and the store needs the node cache pruner, so a profile that sets it is refused while github_runner_arc_node_cache_size_limit_gib is 0. A job that can reach a privileged Docker daemon can write to the store, so give it only to a profile whose workflows are all of one trust level. + - capability_store, a boolean (default false), makes the profile's runner pods copy each tool-cache and system-library capability's payload, and the runner's externals for a Docker-in-Docker sidecar the values write out, into the capability store (github_runner_arc_capability_store_host_path), which the pods of a node share, instead of into every pod's own volumes, and mount the result read-only. Every image those copies come from must be pinned by digest, and the store needs the node cache pruner, so a profile that sets it is refused while github_runner_arc_node_cache_size_limit_gib is 0 or github_runner_arc_node_label_key is empty, since the pruner runs only on the nodes carrying that label. A job that can reach a privileged Docker daemon can write to the store, so give it only to a profile whose workflows are all of one trust level. - max_runners, when set, is the release's maxRunners and takes precedence over its values file and its sizing. - scale_set_labels replaces runs_on_label with the whole list of scaleSetLabels; an empty list sets none, so jobs target the release name. - tolerations is a list of Kubernetes tolerations for the profile's runner pods, for nodes the eligibility label selects that also carry a taint to keep other workloads off. Measured sizing counts a node whose only repelling taints these match, as the pods can be placed there; without them it leaves such a node out. With burst they come before burst.tolerations. @@ -48,7 +48,7 @@ argument_specs: github_runner_arc_capability_store_host_path: type: str default: /var/lib/github-runner/capability-store - description: The directory on each node that holds the capability store of the profiles with capability_store, one entry per image payload, written once per node. It must be an absolute path with no colon or line break, no repeated slash and no trailing slash, at least three levels deep, whose last component is capability-store or ends with -capability-store, not a system directory or inside one, and neither equal to, inside nor containing github_runner_arc_node_cache_host_path. A symbolic link at or above it defeats the validation. The node cache pruner removes the entries no scale set or pod uses, as root, only while some profile has capability_store; once none has, the entries stay until the operator deletes the directory on each node. + description: The directory on each node that holds the capability store of the profiles with capability_store, one entry per image payload, shared by the pods of a node. It must be an absolute path with no colon or line break, no repeated slash and no trailing slash, at least three levels deep, whose last component is capability-store or ends with -capability-store, not a system directory or inside one, and neither equal to, inside nor containing github_runner_arc_node_cache_host_path. A symbolic link at or above it defeats the validation. The node cache pruner removes the entries no scale set or pod uses, as root, only while some profile has capability_store; once none has, the entries stay until the operator deletes the directory on each node. github_runner_arc_capability_store_path: type: str default: /opt/capability-store From 5d814ebae6bca4c3b1a02fb519cee5327e82f0aa Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 22 Oct 2026 15:44:13 +0100 Subject: [PATCH 43/43] docs(arc): state the store's copy container, first-wave copying and burst nodes as the code has them The copy container is shared by the capabilities that use one image and is named for the first. Pods whose copy container starts before an entry is published copy it, the container name in an incoming path only records which container wrote it, and burst nodes are not pruned for the store either. --- plugins/filter/arc.py | 4 ++-- roles/github_runner_arc/README.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 72c78c7..5ea3009 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1117,10 +1117,10 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca } -# --- The capability store: one copy per node of what every runner pod would otherwise copy at its start --- +# --- The capability store: a shared copy per node of what every runner pod would otherwise copy at its start --- # A pod's tool-cache and system-library copies (and the runner's externals for the dind sidecar) are the same bytes on every pod of a profile, taken from digest-pinned images. With a profile's capability_store on, each image's payload is copied into /- (by every pod that starts before the entry is published, the first publish winning, and by none after it), where is the image's sha256 digest and the payload it holds (see arc_capability_store_key), and pods mount the store read-only instead of writing their own copy. The store root is a hostPath of its own, apart from the node cache, whose last component names it. _CAPABILITY_STORE_HOST_PATH_NAME = re.compile(r"^(?:capability-store|[a-z0-9][a-z0-9-]*-capability-store)$") -# A copy is written under /.incoming/.. and published with one rename onto /: rename(2) onto a non-empty directory fails, so of two pods copying the same image at once the first rename wins and the other's copy is deleted, and a reader never sees a part-written entry, since an entry exists only once it is complete. The pod uid comes from the downward API and the container name keeps two copies of one image within a pod apart. +# A copy is written under /.incoming/.. and published with one rename onto /: rename(2) onto a non-empty directory fails, so of two pods copying the same image at once the first rename wins and the other's copy is deleted, and a reader never sees a part-written entry, since an entry exists only once it is complete. The pod uid comes from the downward API, which keeps one pod's copy apart from another's, and the container name only records which of the pod's copy containers wrote it. _CAPABILITY_STORE_INCOMING = ".incoming" _CAPABILITY_STORE_VOLUME = "capability-store" _CAPABILITY_STORE_PREPARE = "capability-store-prepare" diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index e4eea1d..9358940 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -138,11 +138,11 @@ A pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image, o - **The credential must be able to read cluster roles.** Release 1.31.0 installed the pruner with a cluster-wide `ClusterRole` and `ClusterRoleBinding` named `node-cache-pruner`; every run now looks for them by name and removes them, binding first, only if they are what 1.31.0 created (a role that lists pods and nothing else, not aggregated, and a binding of it to the pruner's account in the platform namespace and nothing else; 1.31.0 set no labels, so the content is the identity). The lookup needs the credential the role reaches the cluster with to be allowed to `get` ClusterRoles and ClusterRoleBindings (cluster-scoped, read-only), also on a fleet that never ran 1.31.0, and `delete` on them only when it finds the 1.31.0 objects. A role of that name whose binding is not the one 1.31.0 created (the platform namespace changed, another subject was added) is left in place with the binding and reported once per run with a line starting `NODE_CACHE_LEGACY_ACCESS_LEFT`, so a dangling binding can never regain cluster-wide pod listing; delete them by hand if they are not wanted. The cleanup goes in a later release (ExaDev/github-runner#92). A flag that skipped the lookup on a fleet that never ran 1.31.0 would be one more setting to maintain, so there is none. - **Upgrading from 1.31.0:** the host path root is now validated whether or not any profile has a node cache, so a configuration whose `github_runner_arc_node_cache_host_path` does not end in `node-cache` (the default does) fails validation even if it uses no node-cache capability; set the path to a conforming one (a new path starts an empty cache, and the old directory is yours to delete). - **A cluster that ran release 1.31.0** has a cluster-wide grant to list pods named `node-cache-pruner`; the role removes the 1.31.0 objects where it finds them (binding, then role), subject to the paragraph above: only what matches what 1.31.0 created, and a role whose binding is not the one 1.31.0 created is left with it and reported. -- **Burst nodes are not pruned.** The pruner runs only on nodes that carry `github_runner_arc_node_label_key`, and a burst node carries the burst label and taint but, by design, never the eligibility label. A profile that overflows onto burst nodes therefore also gets a cache directory there, which nothing empties. That is harmless where burst nodes are what they are meant to be, short-lived instances removed once idle, since the directory goes with the node; if burst nodes live long enough for a cache to outgrow their disk, give them the eligibility label so the pruner runs there. +- **Burst nodes are not pruned.** The pruner runs only on nodes that carry `github_runner_arc_node_label_key`, and a burst node carries the burst label and taint but, by design, never the eligibility label. A profile that overflows onto burst nodes therefore also gets a cache directory there, and, with `capability_store` on, store entries there, which nothing empties. That is harmless where burst nodes are what they are meant to be, short-lived instances removed once idle, since the directories go with the node; if burst nodes live long enough for a cache or the store to outgrow their disk, give them the eligibility label so the pruner runs there. - **A bad setting stops the pruner before it does anything.** The pruner's start-up checks end it with status 2 and a message naming the variable, and the DaemonSet's pods then crash-loop with that message in their log. The role refuses the numeric settings it templates by the same pattern and maxima (the size limit, the hard limit factor and the check interval: positive integers without sign, leading zero or fraction, up to 1073741824 GiB, 1024 and 86400), so only a value changed some other way, such as by editing the DaemonSet, reaches that check. - **A failure never stops the others:** a size that cannot be read skips that profile for the round, and a pod list that fails or is not a pod list means no pruning (`NODE_CACHE_UNPRUNABLE` when it is refused as Forbidden, `NODE_CACHE_LIST_FAILED` otherwise), since an unknown set of pods may be using the cache. -The limits are per profile and the node's disk is shared by all of them, so a node's worst case is the sum of the limits of the profiles with a cache on it, times the hard limit factor. Size the limit so that sum fits the node's free disk. The capability store is outside that sum and that limit: it holds one entry for each image payload the scale sets' templates and unfinished pods on the node list, however many gibibytes those are, and the size limit does not bound it; the pruner removes every other entry (and any copy left under `.incoming` by a pod that has finished or gone) on each round, so the store's footprint is the referenced entries alone, until the last profile with `capability_store` switches it off and the operator deletes the directory (see [The capability store](#the-capability-store)). The pruner can list pods through a Role and RoleBinding per profile namespace (named for the release, not a cluster-wide grant) and, in each namespace the capability store's references are read from, list pods and `autoscalingrunnersets` through a second Role and RoleBinding (`node-cache-pruner:capability-store`); it has no access to nodes, and runs as root with only `DAC_OVERRIDE` and `FOWNER` (to delete files and sticky-bit directories the runner's user made), a read-only root filesystem and no privilege escalation. +The limits are per profile and the node's disk is shared by all of them, so a node's worst case is the sum of the limits of the profiles with a cache on it, times the hard limit factor. Size the limit so that sum fits the node's free disk. The capability store is outside that sum and that limit: it holds one entry for each image payload the scale sets' templates and unfinished pods on the node list, however many gibibytes those are, and the size limit does not bound it; the pruner removes every other entry (and any copy left under `.incoming` by a pod that has finished or gone) on each round, so the store's footprint on a node the pruner runs on is the referenced entries alone, until the last profile with `capability_store` switches it off and the operator deletes the directory (see [The capability store](#the-capability-store)). The pruner can list pods through a Role and RoleBinding per profile namespace (named for the release, not a cluster-wide grant) and, in each namespace the capability store's references are read from, list pods and `autoscalingrunnersets` through a second Role and RoleBinding (`node-cache-pruner:capability-store`); it has no access to nodes, and runs as root with only `DAC_OVERRIDE` and `FOWNER` (to delete files and sticky-bit directories the runner's user made), a read-only root filesystem and no privilege escalation. A read-only mount for sensitive profiles was considered and is not offered: a profile's cache is its own and only its own jobs fill it, so a read-only copy of it would never be filled. A profile that must not share a cache with other jobs of its kind should not have the capability. @@ -309,12 +309,12 @@ Each tool-cache and system-library capability adds its copy to every pod's start ### The capability store -Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied into `github_runner_arc_capability_store_host_path`, at most once per node once its entry is published, and the pods use that copy. Pods that start together before an entry is published each copy it (and each runs the host-wide `sync` below), the first publish wins and the others delete their copies, so the first pods on a node still pay the copy; only later pods pay nothing. Switching a profile to the store restarts its idle and pending runners (ARC replaces them when the scale set's template changes; runners running a job are untouched), so the first wave on a node whose store is cold copies as before, all at once. Two capabilities that use one image share one copy container. +Every runner pod of a profile copies the same bytes at its start: each capability image's payload, and, when the values write the dind sidecar out, the runner's externals for it. On a node with a slow disk those copies are a large share of what the disk serves, and a pod sits in its init containers while jobs beside it wait on the same disk. With a profile's `capability_store: true`, each payload is copied into `github_runner_arc_capability_store_host_path`, at most once per node once its entry is published, and the pods use that copy. Pods whose copy container starts before an entry is published each copy it (and each runs the host-wide `sync` below), the first publish wins and the others delete their copies, so the first pods on a node still pay the copy; only later pods pay nothing. Switching a profile to the store restarts its idle and pending runners (ARC replaces them when the scale set's template changes; runners running a job are untouched), so the first wave on a node whose store is cold copies as before, all at once. Two capabilities that use one image share one copy container. An entry is named for its image's digest and the kind of payload copied from it (`toolcache-`, `sysroot-`, `externals-`), so one image used for two payloads, such as a runner image that is also a tool image, is two entries that cannot be taken for each other, and every image a profile's capabilities copy from must be pinned by digest, and a new image is a new entry rather than a change to one a running pod uses. `tasks/validate.yml` refuses a profile with `capability_store: true` whose capability image is not pinned by digest, whose store path overlaps the tool cache, sysroot, hooks or node cache directory, or, in dind mode with the registry cache and no values file, whose organisation's runner image is not pinned by digest (a values file's own externals copy is checked when it is rendered, since its content is not known earlier), so the failure comes before any install step. The init containers that write to the store and run `github_runner_arc_capability_store_image` are `capability-store-prepare`, `capability-store-publish`, `capability-store-publish-externals` and `capability-store-sysroot`; `capability-store-tool-cache` runs it too but only reads the store, which it mounts read-only, and writes the pod's own tool cache. The containers that copy a payload write to the store's `.incoming` directory and run the payload's own image, not the store image. Its `sh`, `mkdir`, `chmod`, `chown`, `sync`, `mv`, `cp`, `find` and `rm` must be busybox's or GNU's, and publishing relies on `mv -T` refusing to move onto a directory that is not empty. A pod's init containers do the work, in this order: - `capability-store-prepare` creates the store and its `.incoming` directory and gives them to the runner's user, as root with `CHOWN` and `DAC_OVERRIDE`, as the node cache's preparation does. -- Each capability's own container (`capability-`, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. +- Each distinct image's copy container (`capability-`, named for the first capability that uses the image, its image, as the runner's user) copies the image's payload to `.incoming/..`, where `` is the entry's name, unless the store already has ``. It removes this pod's own destination first, because the kubelet reruns a pod's init containers after a sandbox change, and a copy killed part-way leaves its destination behind, which a second `cp -R` would nest the payload inside. Every place a pod removes its own copy (that leftover, the copy that loses a publish race, a partial sysroot merge) first makes it owner-writable with `chmod -R u+rwx`, because a payload can hold directories without write permission (a Go module cache's are mode 0555) that `rm -rf` cannot empty as the runner's user, which would fail the init container on every retry. It then makes the copy's own top directory owner-writable (`chmod u+rwx`), because `cp -R` gives it the payload root's mode and renaming a directory into another parent needs write permission on that directory itself, which a root of mode 0555 (a tool cache after `chmod -R a-w`) lacks. - `capability-store-publish` flushes each copy to disk with `sync`, then renames it onto `` with `mv -T`, which fails when another pod published that entry first, in which case this pod's copy is deleted without a message. A rename that fails for any other reason leaves the entry missing, and the container then fails with `mv`'s own error in its message. An entry therefore appears only complete: two pods starting together never see each other's half-written copy, and, because the data is on disk before the rename can be, a power loss or crash leaves either the whole entry or none of it, never an entry of empty files that every later pod would use, since entries are named for an image digest and not revalidated. That `sync` flushes every dirty page on the node and not only the copy's, because the store image's busybox `sync` takes no argument (it ignores one) and its `fsync` applet works on one file or directory per call, which costs a journal commit for each file of a large tree; it is accepted because only a pod that copied an entry runs it, which is every pod that starts before the entry is published and none after it, and a pod that finds the entry in the store flushes nothing. The sysroot merge flushes the same way. The container fails unless every entry the pod needs is in the store. - `capability-store-sysroot` builds the profile's sysroot as `merged-sysroot-`: hard links (`cp -al`) to each system library's entry, a later library's file replacing an earlier one's as the per-pod copies do, flushed and published the same way, rebuilt from nothing when a rerun finds this pod's earlier partial merge. Only the merge's directories are made owner-writable (they take the entries' modes, and one of mode 0555 would refuse the next library's files and the rename), never its files, which are the entries' own. The runner container mounts it read-only at `github_runner_arc_sysroot_path` through a `subPath` of the store. - `capability-store-tool-cache` fills the pod's own tool cache, still an `emptyDir`, from the store: each `//` tree is rebuilt in the pod as real directories holding a symlink to each store file, each directory made owner-writable (`chmod -R u+rwx`) whatever mode the store's directory has, since a tree such as a Go module cache's (directories of mode 0555) would otherwise give the pod directories a job cannot write to, and the `.complete` marker is copied. Two kinds of store entry are copied into the pod instead of linked. A symlink in the store is copied as the symlink it is, so that a relative target such as `bin/npm` pointing at `../lib/node_modules/npm/bin/npm-cli.js` resolves inside the pod's tree, not the store's. A regular file directly in `/bin` is copied in full, since a launcher such as `node`, `python` or `go` finds its own tree (npm's global prefix, a Python's `site-packages`) from its resolved executable path, which a link would put in the read-only store. A tool or version directory that is itself a symlink (an image aliasing `node` and `nodejs`, or `22` and `22.0.0`) is copied as the link too, and nothing below it is walked, so the pod holds one real tree and a job's write through one name shows through the other. An `` directory that is itself a symlink (an image aliasing `x64` and `amd64`) is copied as the link too, so it names the real directory the pod's tree holds beside it, and its `.complete` marker is copied like any other. Everything else stays a link, so a pod still writes only the launchers' bytes.