From eff01b1b0ba7d99a2819fc475dad5fbcd222c645 Mon Sep 17 00:00:00 2001 From: Nic O'Connor Date: Wed, 8 Jul 2026 11:53:46 -0400 Subject: [PATCH] Add docker-rootless-cgroup module for cgroup v2 delegation Rootless Docker only gets cgroup v2 controllers delegated when systemd sets Delegate=yes on the user session. Kasm images have no systemd, so the daemon comes up with CgroupDriver=none and `kind create cluster` fails. This module reproduces that delegation by hand: evacuates the cgroup-namespace root, delegates controllers into a user-owned /rootless slice, installs a PATH shim that gives the daemon a private cgroup leaf so it can self-evacuate, and restarts it. Guarded to no-op on systemd, rootful, and non-Docker hosts; wired into bootstrap.sh last. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 6 ++ bin/install.sh | 2 + bootstrap.sh | 3 + docker-rootless-cgroup/install.sh | 164 ++++++++++++++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100755 docker-rootless-cgroup/install.sh diff --git a/README.md b/README.md index b3f7064..173c489 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,12 @@ Already have a clone? Just run it directly: `~/.local/bin`. Persists across redeploys; only rebuilt when the venv is missing or broken. Building it may install `python3`/`python3-venv` via the system package manager (the one place this module uses sudo). +- `docker-rootless-cgroup/` — on systemd-less hosts (e.g. Kasm), hand-delegates + cgroup v2 controllers to the rootless Docker daemon so per-container cgroups + work and `kind` can create clusters (rootless Docker otherwise comes up with + `CgroupDriver=none`). Uses sudo for host cgroup writes and restarts the + daemon; no-ops on systemd or non-rootless hosts. Re-run at each bootstrap + (the cgroup layout resets on redeploy); its PATH shim lives in `$HOME`. - `mcp-config/` — shared MCP gateway config sync for laptops (LiteLLM / opencode). Independent of `bootstrap.sh`; see its own README. diff --git a/bin/install.sh b/bin/install.sh index f413acc..db8ffe0 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -69,6 +69,8 @@ install_k9s() { # kind — run local Kubernetes clusters in Docker (single static binary). Gated # on rootless Docker at the call site since it is useless without a runtime. +# On systemd-less hosts the rootless daemon also needs cgroup delegation before +# kind can create clusters; see the docker-rootless-cgroup/ module. install_kind() { ver=v0.32.0 have_version kind "$ver" && { fsay "kind $ver already installed."; return 0; } diff --git a/bootstrap.sh b/bootstrap.sh index 337ac13..b6001d1 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -33,3 +33,6 @@ fi "$SCRIPT_DIR/git/install.sh" "$SCRIPT_DIR/bin/install.sh" "$SCRIPT_DIR/ansible/install.sh" +# Runs last: needs rootless Docker already up (started at boot) so it can +# delegate cgroups to it. No-ops where that does not apply. +"$SCRIPT_DIR/docker-rootless-cgroup/install.sh" diff --git a/docker-rootless-cgroup/install.sh b/docker-rootless-cgroup/install.sh new file mode 100755 index 0000000..0110de9 --- /dev/null +++ b/docker-rootless-cgroup/install.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env sh +# docker-rootless-cgroup/install.sh — give rootless Docker cgroup v2 controller +# delegation on a systemd-less host (e.g. Kasm), so per-container cgroups work +# and `kind` can create clusters. +# +# Why: rootless Docker only receives delegated cgroup v2 controllers when +# systemd sets `Delegate=yes` on the user session. Kasm images run no systemd +# (PID 1 is a VNC startup script), so the daemon comes up with CgroupDriver=none +# and `kind create cluster` fails with: +# "running kind with rootless provider requires setting systemd property +# Delegate=yes" +# The suggested fix can't be applied — there is no systemd unit to set it on. +# +# What this does, reproducing that delegation by hand (needs passwordless sudo +# for the host cgroup writes): +# 1. Evacuate the cgroup-namespace root into an /init leaf, then enable the +# controllers in the root's subtree_control. cgroup v2's "no internal +# processes" rule forbids enabling controllers while a cgroup still holds +# processes, so the evacuation has to come first. (In a container, +# /sys/fs/cgroup is the namespace root but a real nested cgroup on the +# host, so it is subject to that rule; only the true system root is exempt.) +# 2. Create a user-owned /rootless slice with the controllers delegated down. +# 3. Install a PATH shim at ~/.local/bin/dockerd-rootless.sh that drops the +# daemon into a PRIVATE leaf (/rootless/daemon) before exec'ing the real +# launcher. Rootless dockerd enables cgroups by self-evacuating (moving its +# own process into a child cgroup, then enabling subtree_control); it can +# only do that in a cgroup it does not share with foreign processes. The +# Kasm supervisor forks the daemon and so shares its launch cgroup, which +# blocks self-evacuation — hence the private leaf. +# 4. Move the daemon's supervisor into the /rootless subtree and restart the +# daemon so it comes back up through the shim, in the private leaf, and +# flips its CgroupDriver from none to cgroupfs. +# +# The cgroup layout is host state a Kasm redeploy wipes, so this is meant to be +# re-run at each bootstrap; the shim lives in $HOME and survives. It no-ops +# (Docker untouched) unless it finds rootless Docker on a cgroup v2 host with +# delegation actually missing — safe on systemd hosts and non-Docker hosts. +set -eu + +say() { printf '%s\n' "$*" >&2; } + +CG=/sys/fs/cgroup +SLICE="$CG/rootless" +BIN_DIR="$HOME/.local/bin" +SHIM="$BIN_DIR/dockerd-rootless.sh" +CONTROLLERS='+cpu +cpuset +io +memory +pids' +NEED='cpu cpuset io memory pids' + +# --- guards: act only where this is both needed and possible ---------------- + +# Unified cgroup v2? +if [ ! -e "$CG/cgroup.controllers" ]; then + say "docker-rootless-cgroup: not a unified cgroup v2 host; skipping." + exit 0 +fi + +if ! command -v docker >/dev/null 2>&1; then + say "docker-rootless-cgroup: docker not installed; skipping." + exit 0 +fi + +# Rootless daemon? (systemd/rootful hosts manage their own cgroups.) +case "$(docker info -f '{{.SecurityOptions}}' 2>/dev/null || true)" in + *rootless*) : ;; + *) say "docker-rootless-cgroup: Docker is not rootless; the host handles cgroups. Skipping." + exit 0 ;; +esac + +# Already delegated? Then there is nothing to do. +driver="$(docker info -f '{{.CgroupDriver}}' 2>/dev/null || true)" +if [ "$driver" != "none" ] && [ -n "$driver" ]; then + say "docker-rootless-cgroup: CgroupDriver=$driver already; delegation present. Nothing to do." + exit 0 +fi + +# The host must actually have delegated the controllers to this container; if +# it withheld them there is nothing we can do from inside. +avail=" $(cat "$CG/cgroup.controllers") " +for c in $NEED; do + case "$avail" in + *" $c "*) : ;; + *) say "docker-rootless-cgroup: host did not delegate '$c' to this container (have:$avail); cannot proceed. Skipping." + exit 0 ;; + esac +done + +# Locate the running rootless daemon and its supervisor (its parent process, +# e.g. Kasm's custom_startup.sh loop that relaunches dockerd when it dies). +rk="$(pgrep -x rootlesskit | head -1 || true)" +if [ -z "$rk" ]; then + say "docker-rootless-cgroup: rootlesskit not running; is dockerd-rootless up? Skipping." + exit 0 +fi +sup="$(ps -o ppid= -p "$rk" 2>/dev/null | tr -d ' ' || true)" + +say "docker-rootless-cgroup: rootless Docker has CgroupDriver=none; delegating cgroup v2 controllers..." + +# --- 1. evacuate the ns-root and delegate its controllers ------------------- +sudo mkdir -p "$CG/init" +# Snapshot the pid list first (moving processes mutates cgroup.procs as we go). +for p in $(cat "$CG/cgroup.procs"); do + echo "$p" | sudo tee "$CG/init/cgroup.procs" >/dev/null 2>&1 || true +done +printf '%s' "$CONTROLLERS" | sudo tee "$CG/cgroup.subtree_control" >/dev/null +say " delegated controllers at ns-root: $(cat "$CG/cgroup.subtree_control")" + +# --- 2. user-owned /rootless slice with controllers, private leaves --------- +sudo mkdir -p "$SLICE/init" "$SLICE/daemon" +printf '%s' "$CONTROLLERS" | sudo tee "$SLICE/cgroup.subtree_control" >/dev/null +sudo chown -R "$(id -u):$(id -g)" "$SLICE" +say " created delegated slice $SLICE (controllers: $(cat "$SLICE/cgroup.controllers"))" + +# --- 3. PATH shim that gives the daemon a private cgroup leaf ---------------- +mkdir -p "$BIN_DIR" +tmp="$(mktemp)" +cat > "$tmp" <<'SH' +#!/bin/sh +# Managed by dotfiles (docker-rootless-cgroup). Shadows /usr/bin/dockerd-rootless.sh +# on PATH. Places the rootless Docker daemon in a private, delegated cgroup v2 +# leaf so it can enable controllers (needed by `kind`), then execs the real +# launcher. No-op if the delegated slice is absent, so it is safe anywhere. +_cg=/sys/fs/cgroup/rootless/daemon +if [ -w /sys/fs/cgroup/rootless ] && [ -d "$_cg" ]; then + echo $$ > "$_cg/cgroup.procs" 2>/dev/null || true +fi +exec /usr/bin/dockerd-rootless.sh "$@" +SH +chmod +x "$tmp" +if [ -f "$SHIM" ] && cmp -s "$tmp" "$SHIM"; then + rm -f "$tmp" + say " daemon cgroup shim already current at $SHIM" +else + mv "$tmp" "$SHIM" + chmod +x "$SHIM" + say " installed daemon cgroup shim at $SHIM" +fi + +# --- 4. relocate the supervisor and restart the daemon ---------------------- +# Put the supervisor inside the /rootless subtree so the daemon it relaunches +# (via the shim) starts there too — then the shim's move into /rootless/daemon +# needs no privilege (the common ancestor /rootless is user-owned). +if [ -n "$sup" ]; then + echo "$sup" | sudo tee "$SLICE/init/cgroup.procs" >/dev/null 2>&1 || true +fi + +say " restarting rootless Docker so it comes back up in the private leaf..." +kill "$rk" 2>/dev/null || true +i=0 +while [ "$i" -lt 30 ]; do + if docker info >/dev/null 2>&1; then break; fi + i=$((i + 1)) + sleep 1 +done + +# --- verify: report honestly, do not claim success on none ------------------ +driver="$(docker info -f '{{.CgroupDriver}}' 2>/dev/null || true)" +if [ "$driver" = "none" ] || [ -z "$driver" ]; then + say "docker-rootless-cgroup: WARNING — CgroupDriver is still '${driver:-unreachable}'." + say " kind will not work yet. Inspect the new dockerd's cgroup:" + say " cat /proc/\$(pgrep -x dockerd | head -1)/cgroup # expect .../rootless/daemon" + exit 0 +fi +say "docker-rootless-cgroup: done. CgroupDriver=$driver — rootless Docker now has" +say " delegated cgroups; \`kind create cluster\` can run."