From 36a6934ade4a4745428d347a2789d5360310391f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 17:07:19 +0100 Subject: [PATCH 1/2] feat(arc): let a profile keep its capability copies in RAM Every runner pod copies its tool cache and sysroot payloads before its job starts, and on a node with a slow disk those copies keep the disk busy while the jobs beside them wait on it. A profile's capability_volume_medium: Memory puts both volumes on a tmpfs so the copies write nothing to the disk; the README says how to raise the pods' memory requests and limits by what the copies hold. The node cache stays a hostPath on the disk. --- plugins/filter/arc.py | 22 +++++++++-- roles/github_runner_arc/README.md | 7 +++- .../tasks/render_scale_set_values.yml | 2 +- tests/unit/test_capabilities.py | 39 +++++++++++++++---- tests/unit/test_node_cache_pruner_role.py | 4 +- 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index b78fedb..ca250b7 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -92,6 +92,9 @@ _PATH_COMPONENT = re.compile(r"^[A-Za-z0-9_+-][A-Za-z0-9._+-]*$") # A variable a system-library capability may set: a shell and POSIX environment variable name. _ENV_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +# The media an emptyDir can sit on for the copies the capabilities make: the node's disk (the empty string, the Kubernetes default) or RAM (Memory, a tmpfs). A RAM-backed copy writes nothing to the node's disk, and its pages count against the pod's memory, so a profile that uses it raises its pods' memory requests and limits by what the copies hold. +_CAPABILITY_VOLUME_MEDIA = ("", "Memory") + # The pod volumes the capabilities add: the tool cache every tool-cache capability copies into, the sysroot every system-library capability copies into, and the ConfigMap holding the job-started hook. _TOOL_CACHE_VOLUME = "tool-cache" _SYSROOT_VOLUME = "sysroot" @@ -463,6 +466,15 @@ def _capability_names(profile: Mapping[str, Any], where: str, errors: list[str]) return list(dict.fromkeys(names)) +def _capability_volume_medium(profile: Mapping[str, Any], where: str, errors: list[str]) -> str: + """Return the medium a profile's tool cache and sysroot sit on, recording an error when it is not one of _CAPABILITY_VOLUME_MEDIA.""" + medium = profile.get("capability_volume_medium", "") + if medium not in _CAPABILITY_VOLUME_MEDIA: + errors.append(f"{where}.capability_volume_medium must be one of {', '.join(repr(value) for value in _CAPABILITY_VOLUME_MEDIA)}") + return "" + return str(medium) + + def _explicit_max_runners(profile: Mapping[str, Any], where: str, errors: list[str]) -> int | None: """Return the profile's own max_runners as a whole number, None when unset, recording an error when it is not a whole number of at least 0.""" value = profile.get("max_runners") @@ -595,6 +607,7 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er "burst_runners": burst_runners, "max_runners": max_runners, "capabilities": _capability_names(profile, where, errors), + "capability_volume_medium": _capability_volume_medium(profile, where, errors), "label": f"{org_name}{suffix}", "settings": dict(profile), } @@ -1031,13 +1044,13 @@ def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_ca } -def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str) -> dict[str, Any]: +def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str, volume_medium: str) -> dict[str, Any]: """Add a profile's capabilities to its scale set's Helm values. Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay. Args: - values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. + values: the release's merged Helm values (the profile's values with the role's overlay over them). names: the profile's capabilities, in order; the init containers and hook scripts run in this order. catalogue: github_runner_arc_capabilities, already checked by arc_capability_errors. tool_cache_path: where the tool cache is mounted on the runner container. sysroot_path: where the sysroot is mounted on the runner container. runner_user, runner_group: the runner container's user and group ids, which the init containers run as. hooks_dir: where the job-started ConfigMap is mounted on the runner container. hooks_configmap: the job-started ConfigMap's name, in the profile's namespace. volume_medium: the emptyDir medium of the tool cache and the sysroot, one of _CAPABILITY_VOLUME_MEDIA, which the profile's capability_volume_medium sets (the node cache is a hostPath and keeps its own). Returns: A dict with ``errors`` (empty when the values can carry the capabilities), ``values`` (the values with the capabilities added, or as given when there are none or errors) and ``hooks`` (the ConfigMap's data apart from the dispatcher, which the role adds from its own file; empty when the profile needs no ConfigMap). @@ -1074,12 +1087,13 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu volumes: list[dict[str, Any]] = [] mounts: list[dict[str, Any]] = [] env: list[dict[str, Any]] = [] + copy_volume: dict[str, Any] = {"medium": volume_medium} if volume_medium else {} if tool_capabilities: - volumes.append({"name": _TOOL_CACHE_VOLUME, "emptyDir": {}}) + volumes.append({"name": _TOOL_CACHE_VOLUME, "emptyDir": dict(copy_volume)}) mounts.append({"name": _TOOL_CACHE_VOLUME, "mountPath": tool_cache_path}) env.append({"name": _TOOL_CACHE_ENV, "value": tool_cache_path}) if sysroot_capabilities: - volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": {}}) + volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": dict(copy_volume)}) mounts.append({"name": _SYSROOT_VOLUME, "mountPath": sysroot_path}) if node_cache_capabilities: volumes.append({"name": _NODE_CACHE_VOLUME, "hostPath": {"path": node_cache_host_path, "type": "DirectoryOrCreate"}}) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 74af91e..172f0b6 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -24,6 +24,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `autoscale`: `true` on at most one profile in the whole inventory makes it the scale set the autoscaler manages. - `sizing`: derive a runner ceiling from node capacity, given as uniform figures or measured from each eligible node (see Sizing). On an ordinary profile without `max_runners` it sets `maxRunners`; on the autoscaled profile it sets the autoscaler's ceiling, leaving `maxRunners` as the floor. - `capabilities`: names of entries in `github_runner_arc_capabilities` that the profile's runner pods carry, in order (see [Capabilities](#capabilities)). + - `capability_volume_medium`: what the tool cache and the sysroot the capabilities copy into sit on: unset or `""` for the node's disk, or `Memory` for RAM (see [Where the copies live](#where-the-copies-live)). - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`. - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. @@ -301,7 +302,11 @@ When a profile has a hook capability, a `path` or a system-library capability, t The tool cache, the sysroot, the node cache and the hook are on the runner container, so they serve the steps that run there: every step in the default container mode and in Docker-in-Docker mode (`container_mode: dind`). A job that runs in a `container:` reaches the tool cache only through the runner's own mapping of it, which in Docker-in-Docker mode points at the Docker daemon's filesystem, where the cache is not mounted, and it does not see the sysroot at all. The `kubernetes` container modes run every step in a separate pod, so the role refuses capabilities on a profile whose values use them. It also refuses them when the values define no runner container (the role's own template does), when the runner container already sets `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, when a volume already uses the name `tool-cache`, `sysroot` or `job-started-hooks`, or, for a profile with a system-library capability, when `github_runner_arc_sysroot_path` is not an absolute directory without a `:` or overlaps the tool cache or the hooks directory. During validation it refuses a system-library capability whose `env` sets one of the search paths, `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, a name that is not a variable name or a value that is not a single-line string (a line break would add a line to `GITHUB_ENV`), and two of a profile's system-library capabilities that set one variable to different values, since they share one sysroot. A values file's own init containers, volumes, environment and mounts are kept, and the capabilities' are added after them, by the `exadev.github_runner.arc_capability_pod` filter. `exadev.github_runner.arc_capability_errors` checks the catalogue and each profile's references during validation, before anything changes. -Each tool-cache and system-library capability adds its copy to every pod's start, which is not yet measured. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage. +### Where the copies live + +Each tool-cache and system-library capability copies its payload into the pod every time a pod starts, so every runner pod writes the whole tool set and sysroot before its job begins. By default the `tool-cache` and `sysroot` volumes are `emptyDir` volumes on the node's disk for the pod's lifetime, so they count against the node's ephemeral storage, and on a node whose disk is slow to write the copies alone can keep it busy while jobs beside them wait on it. + +A profile with `capability_volume_medium: Memory` puts both volumes on RAM (`emptyDir.medium: Memory`, a tmpfs), so the copies write nothing to the node's disk. Their pages count against the pod's memory instead: the kubelet charges a RAM-backed `emptyDir` to the pod, and a pod over its memory limit is killed while one over its request is among the first evicted when the node runs short. A profile that sets it therefore raises its runner pods' memory requests and limits by what the copies hold, measured from a running pod (`du -sk` of the two volumes under `/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/`), plus what its jobs add to the tool cache, since a setup action installs into it. The higher request also lowers the ceiling measured sizing derives. The node cache is a `hostPath` and stays on the node's disk either way. `tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the tool images or the test. diff --git a/roles/github_runner_arc/tasks/render_scale_set_values.yml b/roles/github_runner_arc/tasks/render_scale_set_values.yml index 84fa876..9c37b1a 100644 --- a/roles/github_runner_arc/tasks/render_scale_set_values.yml +++ b/roles/github_runner_arc/tasks/render_scale_set_values.yml @@ -12,7 +12,7 @@ profile.capabilities, github_runner_arc_capabilities, github_runner_arc_tool_cache_path, github_runner_arc_sysroot_path, github_runner_arc_node_cache_path, github_runner_arc_node_cache_host_path | exadev.github_runner.arc_node_cache_dir(profile.namespace, profile.release), github_runner_arc_runner_uid, github_runner_arc_runner_gid, - github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap) + github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap, profile.capability_volume_medium) }} vars: github_runner_arc_values_source: >- diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 673483c..a6fb962 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -54,7 +54,7 @@ def expanded(profile: dict[str, Any]) -> dict[str, Any]: def pod(values: dict[str, Any], names: list[str]) -> dict[str, Any]: """Add names' capabilities from CATALOGUE to values with the role's default settings.""" - result: dict[str, Any] = arc.arc_capability_pod(values, names, CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result: dict[str, Any] = arc.arc_capability_pod(values, names, CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "") return result @@ -77,6 +77,16 @@ def test_capabilities_must_be_a_list_of_distinct_names(self) -> None: result = arc.arc_profiles([{"name": "Example", "image": "i:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": capabilities}]}], require_app_id=False) self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) + def test_a_profile_copies_onto_the_nodes_disk_unless_it_names_a_medium(self) -> None: + self.assertEqual(expanded({"max_runners": 1})["capability_volume_medium"], "") + self.assertEqual(expanded({"max_runners": 1, "capability_volume_medium": "Memory"})["capability_volume_medium"], "Memory") + + def test_a_capability_volume_medium_the_copies_cannot_use_is_an_error(self) -> None: + for medium in ("memory", "HugePages", 1, None): + with self.subTest(medium=medium): + result = arc.arc_profiles([{"name": "Example", "image": "i:1", "scale_set_profiles": [{"max_runners": 1, "capability_volume_medium": medium}]}], require_app_id=False) + self.assertTrue(any("capability_volume_medium must be one of '', 'Memory'" in error for error in result["errors"]), result["errors"]) + class CatalogueTest(unittest.TestCase): def errors(self, catalogue: Any, capabilities: list[str] | None = None) -> list[str]: @@ -305,8 +315,21 @@ def test_every_kind_together_keeps_the_profiles_order_and_shares_one_sysroot(sel self.assertEqual([variable["name"] for variable in runner(result["values"])["env"]], ["RUNNER_TOOL_CACHE", "ACTIONS_RUNNER_HOOK_JOB_STARTED"]) self.assertTrue(result["hooks"]["sysroot-env"].endswith("CMAKE_PREFIX_PATH+=/opt/sysroot\nOPAL_PREFIX=/opt/sysroot\n")) + def test_a_memory_medium_puts_the_tool_cache_and_the_sysroot_in_ram_and_leaves_the_node_cache_on_the_node(self) -> None: + result = arc.arc_capability_pod(DEFAULT_VALUES, ["node", "mpi", "pnpm-store"], CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "Memory") + self.assertEqual(result["errors"], []) + volumes = {volume["name"]: volume for volume in result["values"]["template"]["spec"]["volumes"]} + self.assertEqual(volumes["tool-cache"]["emptyDir"], {"medium": "Memory"}) + self.assertEqual(volumes["sysroot"]["emptyDir"], {"medium": "Memory"}) + self.assertNotIn("emptyDir", volumes["node-cache"]) + + def test_without_a_medium_the_copies_sit_on_the_nodes_disk(self) -> None: + volumes = {volume["name"]: volume for volume in pod(DEFAULT_VALUES, ["node", "mpi"])["values"]["template"]["spec"]["volumes"]} + self.assertEqual(volumes["tool-cache"]["emptyDir"], {}) + self.assertEqual(volumes["sysroot"]["emptyDir"], {}) + def test_the_sysroot_follows_its_configured_path(self) -> None: - result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], {"mpi": {"sysroot_image": "ghcr.io/example/m:1"}}, TOOL_CACHE, "/srv/libs/", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], {"mpi": {"sysroot_image": "ghcr.io/example/m:1"}}, TOOL_CACHE, "/srv/libs/", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "") self.assertEqual(result["errors"], []) self.assertEqual(result["values"]["template"]["spec"]["initContainers"][0]["command"][-1], "/srv/libs/") self.assertEqual(result["hooks"]["sysroot-env"].splitlines()[:2], ["PATH+=/srv/libs/bin", "LD_LIBRARY_PATH+=/srv/libs/lib"]) @@ -322,12 +345,12 @@ def test_a_sysroot_path_that_cannot_carry_a_sysroot_is_an_error(self) -> None: } for case, (path, message) in cases.items(): with self.subTest(case): - result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], CATALOGUE, TOOL_CACHE, path, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], CATALOGUE, TOOL_CACHE, path, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "") self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) self.assertIs(result["values"], DEFAULT_VALUES) def test_the_sysroot_path_is_not_checked_for_a_profile_without_a_system_library(self) -> None: - self.assertEqual(arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, "relative", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP)["errors"], []) + self.assertEqual(arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, "relative", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "")["errors"], []) def test_a_volume_already_named_sysroot_is_an_error(self) -> None: values = {"template": {"spec": {"volumes": [{"name": "sysroot", "emptyDir": {}}], "containers": [{"name": "runner"}]}}} @@ -371,7 +394,7 @@ def test_the_init_container_command_makes_and_gives_away_every_directory_it_name def test_a_node_cache_capability_shares_one_volume_with_every_capability_of_the_profile(self) -> None: catalogue = {**CATALOGUE, "yarn-cache": {"node_cache": {"init_image": BUSYBOX, "dirs": {"YARN_CACHE_FOLDER": "yarn"}}}} - result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store", "yarn-cache", "node"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store", "yarn-cache", "node"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "") self.assertEqual(result["errors"], []) spec = result["values"]["template"]["spec"] self.assertEqual([volume["name"] for volume in spec["volumes"]], ["tool-cache", "node-cache"]) @@ -379,7 +402,7 @@ def test_a_node_cache_capability_shares_one_volume_with_every_capability_of_the_ self.assertEqual({variable["name"] for variable in runner(result["values"])["env"]}, {"RUNNER_TOOL_CACHE", "pnpm_config_store_dir", "pnpm_config_enable_global_virtual_store", "YARN_CACHE_FOLDER"}) def test_a_profile_without_a_node_cache_capability_gets_no_node_volume_and_its_path_is_not_checked(self) -> None: - result = arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, SYSROOT, "relative", "also/relative", 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, SYSROOT, "relative", "also/relative", 1001, 1001, HOOKS_DIR, CONFIGMAP, "") self.assertEqual(result["errors"], []) self.assertNotIn("node-cache", [volume["name"] for volume in result["values"]["template"]["spec"]["volumes"]]) @@ -397,7 +420,7 @@ def test_node_cache_paths_that_cannot_carry_a_cache_are_errors(self) -> None: } for case, (mount, host, message) in cases.items(): with self.subTest(case): - result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store"], CATALOGUE, TOOL_CACHE, SYSROOT, mount, host, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store"], CATALOGUE, TOOL_CACHE, SYSROOT, mount, host, 1001, 1001, HOOKS_DIR, CONFIGMAP, "") self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) def test_two_node_cache_capabilities_setting_one_variable_differently_is_an_error(self) -> None: @@ -405,7 +428,7 @@ def test_two_node_cache_capabilities_setting_one_variable_differently_is_an_erro "a": {"node_cache": {"init_image": BUSYBOX, "dirs": {"STORE": "one"}}}, "b": {"node_cache": {"init_image": BUSYBOX, "dirs": {"STORE": "two"}}}, } - result = arc.arc_capability_pod(DEFAULT_VALUES, ["a", "b"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["a", "b"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP, "") self.assertEqual(result["errors"], ["node-cache capabilities a and b set STORE to different values"]) def test_a_volume_already_named_node_cache_is_an_error(self) -> None: diff --git a/tests/unit/test_node_cache_pruner_role.py b/tests/unit/test_node_cache_pruner_role.py index aa31905..676aee3 100644 --- a/tests/unit/test_node_cache_pruner_role.py +++ b/tests/unit/test_node_cache_pruner_role.py @@ -540,12 +540,12 @@ def test_the_readmes_pnpm_example_validates_and_renders_into_a_pod(self) -> None profile = arc.arc_profiles([{"name": "Example", "image": "ghcr.io/example/runner:1", "scale_set_profiles": [{"max_runners": 1, "capabilities": ["pnpm-store"]}]}], require_app_id=False)["profiles"][0] self.assertEqual(arc.arc_capability_errors([profile], catalogue), []) values = {"template": {"spec": {"containers": [{"name": "runner", "command": ["/home/runner/run.sh"]}]}}} - result = arc.arc_capability_pod(values, ["pnpm-store"], catalogue, "/opt/hostedtoolcache", "/opt/sysroot", "/node-cache", "/var/lib/github-runner/node-cache/arc/main", 1001, 1001, "/etc/github-runner/hooks", "job-started") + result = arc.arc_capability_pod(values, ["pnpm-store"], catalogue, "/opt/hostedtoolcache", "/opt/sysroot", "/node-cache", "/var/lib/github-runner/node-cache/arc/main", 1001, 1001, "/etc/github-runner/hooks", "job-started", "") self.assertEqual(result["errors"], []) def test_the_example_names_the_store_with_a_variable_pnpm_does_not_read(self) -> None: # A pod-wide pnpm_config_store_dir reaches pnpm/action-setup's version switch, which corrupts a shared global virtual store (README, Capabilities), so the example names the directory for a workflow to pass to its install step. - env = {item["name"]: item["value"] for item in arc.arc_capability_pod({"template": {"spec": {"containers": [{"name": "runner"}]}}}, ["pnpm-store"], self.example()["github_runner_arc_capabilities"], "/a", "/b", "/node-cache", "/var/lib/x/node-cache/a/b", 1001, 1001, "/h", "c")["values"]["template"]["spec"]["containers"][0]["env"]} + env = {item["name"]: item["value"] for item in arc.arc_capability_pod({"template": {"spec": {"containers": [{"name": "runner"}]}}}, ["pnpm-store"], self.example()["github_runner_arc_capabilities"], "/a", "/b", "/node-cache", "/var/lib/x/node-cache/a/b", 1001, 1001, "/h", "c", "")["values"]["template"]["spec"]["containers"][0]["env"]} self.assertEqual(env["NODE_CACHE_PNPM_STORE_DIR"], "/node-cache/pnpm") self.assertEqual([name for name in env if name.lower().startswith(("pnpm_config_", "npm_config_"))], []) readme = (ROLE / "README.md").read_text() From 8aa73d658c6a744902fdd12ce1b59f769680453d Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 17:13:08 +0100 Subject: [PATCH 2/2] docs(arc): say which memory a RAM-backed capability copy is charged to The init containers that copy the payloads set no limit, so their pages land in the pod's cgroup and count towards eviction against its requests but against no limit; what a job adds to the tool cache counts against the runner container's limit. --- roles/github_runner_arc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index 172f0b6..8094460 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -306,7 +306,7 @@ The tool cache, the sysroot, the node cache and the hook are on the runner conta Each tool-cache and system-library capability copies its payload into the pod every time a pod starts, so every runner pod writes the whole tool set and sysroot before its job begins. By default the `tool-cache` and `sysroot` volumes are `emptyDir` volumes on the node's disk for the pod's lifetime, so they count against the node's ephemeral storage, and on a node whose disk is slow to write the copies alone can keep it busy while jobs beside them wait on it. -A profile with `capability_volume_medium: Memory` puts both volumes on RAM (`emptyDir.medium: Memory`, a tmpfs), so the copies write nothing to the node's disk. Their pages count against the pod's memory instead: the kubelet charges a RAM-backed `emptyDir` to the pod, and a pod over its memory limit is killed while one over its request is among the first evicted when the node runs short. A profile that sets it therefore raises its runner pods' memory requests and limits by what the copies hold, measured from a running pod (`du -sk` of the two volumes under `/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/`), plus what its jobs add to the tool cache, since a setup action installs into it. The higher request also lowers the ceiling measured sizing derives. The node cache is a `hostPath` and stays on the node's disk either way. +A profile with `capability_volume_medium: Memory` puts both volumes on RAM (`emptyDir.medium: Memory`, a tmpfs), so the copies write nothing to the node's disk. Their pages count as memory instead, charged to the container that writes them. The copies are written by the capabilities' init containers, which set no limit, so once those exit the pages sit in the pod's own cgroup: they count towards the pod's usage, which the kubelet compares with its requests when the node runs short of memory, but against no container's limit. What a job adds to the tool cache (a setup action installs into it) is written by the runner container and counts against its limit. A profile that sets it therefore raises its runner pods' memory request by what the copies hold plus what its jobs add, and the runner container's memory limit by what its jobs add, both measured from running pods (`du -sk` of the volumes under `/var/lib/kubelet/pods//volumes/kubernetes.io~empty-dir/`). The higher request also lowers the ceiling measured sizing derives. The node cache is a `hostPath` and stays on the node's disk either way. `tests/capabilities/run.sh` exercises all of this in a kind cluster: it builds the reference Node.js and libpq images, renders a profile through the role's own tasks, renders the scale-set chart with the result and checks it against the ARC CRDs, then starts a pod from the chart's runner pod template and checks that the init containers filled the tool cache and the sysroot, that the hook ran, put the tool on `PATH` and the library on its search paths with its `env`, and that `@actions/tool-cache`, the library behind `actions/setup-node`'s lookup, finds the tool. `.github/workflows/capabilities.yml` runs it on pull requests that touch the role, its filters, the tool images or the test.