feat(arc): let a profile keep its capability copies in RAM

plugins/filter/arc.py

@@ -92,6 +92,9 @@
 _PATH_COMPONENT = re.compile(r"^[A-Za-z0-9_+-][A-Za-z0-9._+-]*$")
 # A variable a system-library capability may set: a shell and POSIX environment variable name.
 _ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+# The media an emptyDir can sit on for the copies the capabilities make: the node's disk (the empty string, the Kubernetes default) or RAM (Memory, a tmpfs). A RAM-backed copy writes nothing to the node's disk, and its pages count against the pod's memory, so a profile that uses it raises its pods' memory requests and limits by what the copies hold.
+_CAPABILITY_VOLUME_MEDIA = ("", "Memory")
+
 # The pod volumes the capabilities add: the tool cache every tool-cache capability copies into, the sysroot every system-library capability copies into, and the ConfigMap holding the job-started hook.
 _TOOL_CACHE_VOLUME = "tool-cache"
 _SYSROOT_VOLUME = "sysroot"
@@ -463,6 +466,15 @@ def _capability_names(profile: Mapping[str, Any], where: str, errors: list[str])
     return list(dict.fromkeys(names))
 
 
+def _capability_volume_medium(profile: Mapping[str, Any], where: str, errors: list[str]) -> str:
+    """Return the medium a profile's tool cache and sysroot sit on, recording an error when it is not one of _CAPABILITY_VOLUME_MEDIA."""
+    medium = profile.get("capability_volume_medium", "")
+    if medium not in _CAPABILITY_VOLUME_MEDIA:
+        errors.append(f"{where}.capability_volume_medium must be one of {', '.join(repr(value) for value in _CAPABILITY_VOLUME_MEDIA)}")
+        return ""
+    return str(medium)
+
+
 def _explicit_max_runners(profile: Mapping[str, Any], where: str, errors: list[str]) -> int | None:
     """Return the profile's own max_runners as a whole number, None when unset, recording an error when it is not a whole number of at least 0."""
     value = profile.get("max_runners")
@@ -595,6 +607,7 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er
         "burst_runners": burst_runners,
         "max_runners": max_runners,
         "capabilities": _capability_names(profile, where, errors),
+        "capability_volume_medium": _capability_volume_medium(profile, where, errors),
         "label": f"{org_name}{suffix}",
         "settings": dict(profile),
     }
@@ -1031,13 +1044,13 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca
     }
 
 
-def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str) -> dict[str, Any]:
+def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str, volume_medium: str) -> dict[str, Any]:
     """Add a profile's capabilities to its scale set's Helm values.
 
     Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay.
 
     Args:
-        values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace.
+        values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. volume_medium: the emptyDir medium of the tool cache and the sysroot, one of _CAPABILITY_VOLUME_MEDIA, which the profile's capability_volume_medium sets (the node cache is a hostPath and keeps its own).
 
     Returns:
         A dict with ``errors`` (empty when the values can carry the capabilities), ``values`` (the values with the capabilities added, or as given when there are none or errors) and ``hooks`` (the ConfigMap's data apart from the dispatcher, which the role adds from its own file; empty when the profile needs no ConfigMap).
@@ -1074,12 +1087,13 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu
     volumes: list[dict[str, Any]] = []
     mounts: list[dict[str, Any]] = []
     env: list[dict[str, Any]] = []
+    copy_volume: dict[str, Any] = {"medium": volume_medium} if volume_medium else {}
     if tool_capabilities:
-        volumes.append({"name": _TOOL_CACHE_VOLUME, "emptyDir": {}})
+        volumes.append({"name": _TOOL_CACHE_VOLUME, "emptyDir": dict(copy_volume)})
         mounts.append({"name": _TOOL_CACHE_VOLUME, "mountPath": tool_cache_path})
         env.append({"name": _TOOL_CACHE_ENV, "value": tool_cache_path})
     if sysroot_capabilities:
-        volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": {}})
+        volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": dict(copy_volume)})
         mounts.append({"name": _SYSROOT_VOLUME, "mountPath": sysroot_path})
     if node_cache_capabilities:
         volumes.append({"name": _NODE_CACHE_VOLUME, "hostPath": {"path": node_cache_host_path, "type": "DirectoryOrCreate"}})

roles/github_runner_arc/README.md

@@ -24,6 +24,7 @@ Before touching the cluster the role then checks the secrets it is about to writ
   - `autoscale`: `true` on at most one profile in the whole inventory makes it the scale set the autoscaler manages.
   - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it sets the autoscaler's ceiling, leaving `maxRunners` as the floor.
   - `capabilities`: names of entries in `github_runner_arc_capabilities` that the profile's runner pods carry, in order (see [Capabilities](#capabilities)).
+  - `capability_volume_medium`: what the tool cache and the sysroot the capabilities copy into sit on: unset or `""` for the node's disk, or `Memory` for RAM (see [Where the copies live](#where-the-copies-live)).
   - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`.
 - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from.
 - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as.
@@ -301,7 +302,11 @@ When a profile has a hook capability, a `path` or a system-library capability, t
 
 The tool cache, the sysroot, the node cache and the hook are on the runner container, so they serve the steps that run there: every step in the default container mode and in Docker-in-Docker mode (`container_mode: dind`). A job that runs in a `container:` reaches the tool cache only through the runner's own mapping of it, which in Docker-in-Docker mode points at the Docker daemon's filesystem, where the cache is not mounted, and it does not see the sysroot at all. The `kubernetes` container modes run every step in a separate pod, so the role refuses capabilities on a profile whose values use them. It also refuses them when the values define no runner container (the role's own template does), when the runner container already sets `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, when a volume already uses the name `tool-cache`, `sysroot` or `job-started-hooks`, or, for a profile with a system-library capability, when `github_runner_arc_sysroot_path` is not an absolute directory without a `:` or overlaps the tool cache or the hooks directory. During validation it refuses a system-library capability whose `env` sets one of the search paths, `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, a name that is not a variable name or a value that is not a single-line string (a line break would add a line to `GITHUB_ENV`), and two of a profile's system-library capabilities that set one variable to different values, since they share one sysroot. A values file's own init containers, volumes, environment and mounts are kept, and the capabilities' are added after them, by the `exadev.github_runner.arc_capability_pod` filter. `exadev.github_runner.arc_capability_errors` checks the catalogue and each profile's references during validation, before anything changes.
 
-Each tool-cache and system-library capability adds its copy to every pod's start, which is not yet measured. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage.
+### Where the copies live
+
+Each tool-cache and system-library capability copies its payload into the pod every time a pod starts, so every runner pod writes the whole tool set and sysroot before its job begins. By default the `tool-cache` and `sysroot` volumes are `emptyDir` volumes on the node's disk for the pod's lifetime, so they count against the node's ephemeral storage, and on a node whose disk is slow to write the copies alone can keep it busy while jobs beside them wait on it.
+
+A profile with `capability_volume_medium: Memory` puts both volumes on RAM (`emptyDir.medium: Memory`, a tmpfs), so the copies write nothing to the node's disk. Their pages count as memory instead, charged to the container that writes them. The copies are written by the capabilities' init containers, which set no limit, so once those exit the pages sit in the pod's own cgroup: they count towards the pod's usage, which the kubelet compares with its requests when the node runs short of memory, but against no container's limit. What a job adds to the tool cache (a setup action installs into it) is written by the runner container and counts against its limit. A profile that sets it therefore raises its runner pods' memory request by what the copies hold plus what its jobs add, and the runner container's memory limit by what its jobs add, both measured from running pods (`du -sk` of the volumes under `/var/lib/kubelet/pods/<uid>/volumes/kubernetes.io~empty-dir/`). The higher request also lowers the ceiling measured sizing derives. The node cache is a `hostPath` and stays on the node's disk either way.
 
 `tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the tool images or the test.
 

roles/github_runner_arc/tasks/render_scale_set_values.yml

@@ -12,7 +12,7 @@
             profile.capabilities, github_runner_arc_capabilities, github_runner_arc_tool_cache_path, github_runner_arc_sysroot_path,
             github_runner_arc_node_cache_path, github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_dir(profile.namespace, profile.release),
             github_runner_arc_runner_uid, github_runner_arc_runner_gid,
-            github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap)
+            github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap, profile.capability_volume_medium)
       }}
   vars:
     github_runner_arc_values_source: >-