vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound the runner's containers and keep its disk free
4e3759da08a603ede50107e89a23ff6d5432b7ac · Keenan Nemetz · 2h ago · parent c4ad0d5a
Bound the runner's containers and keep its disk free The container boundary was good and the resource the job consumes was not. A job could fork without a bound, write /tmp until the machine's RAM was gone, and fill the disk its workspace sits on — and the runner would only notice when a build died. The isolation protects the forge's data; this protects the machine that runs the code. - `--container-pids` now defaults to 2048 and zero means the default, not "no limit": a bound nobody set is the easiest one to be missing, and a job that can fork without one is a fork bomb away from taking the machine. - `--container-tmpfs` is new and defaults to 1G. /tmp is a tmpfs, so its size is memory, and an unsized one is a build that fills RAM without writing a byte to the workspace. It is passed to service containers too. - `--container-memory` stays the operator's setting rather than a default, because the right number is the machine's size and a wrong default would show up as an OOM kill in someone's build. All three places a runner is configured now set it: the packaged runner.env example, the manual steps in the deploy guide, and the cloud-init template (RUNNER_MEMORY, default 4G, alongside RUNNER_PIDS, RUNNER_TMPFS and RUNNER_DISK_RESERVE). - `--disk-reserve` (default 2G) is how the job tree is bounded. It is a host bind mount, so the runtime cannot quota it and a filesystem quota needs root; a walker over the tree would cost more than it protects. What the runner can promise is that it notices: one statfs per second while a command runs, and the job is stopped below the reserve with the numbers in its log. `--dir` on its own sized filesystem is the hard bound, and the docs say so. - The runner unit gets `MemoryMax=90%` as the backstop under all of it, for a job that gets past its own cgroup. It is a percentage so the unit does not have to know the machine. TasksMax is deliberately not set: the container's own --container-pids already applies, and a unit value would have to track --jobs and --container-pids and would fail builds, looking like the workflow's fault, when they drifted apart. A workflow still cannot raise or remove any of this: the limits are flags on the runner and the unit, never fields a job can set. scripts/check-deploy.py now fails when the rendered runner arguments stop carrying the memory, pids, tmpfs or disk limits, so the first-boot file cannot quietly lose them. Closes 01M6S798BH0BK00R2B5N1KDJYM
ci/container/container.go +43−7
| @@ -43,6 +43,16 @@ import ( | |||
| 43 | 43 | // and the cap keeps three services from crowding out the steps. | |
| 44 | 44 | const MaxServiceLogBytes = 256 << 10 | |
| 45 | 45 | ||
| 46 | // DefaultPIDs bounds a container's processes when the operator does not. A build | ||
| 47 | // that needs more than this many threads or children is misconfigured; the point of | ||
| 48 | // the default is that a fork bomb costs the job, not the machine. | ||
| 49 | const DefaultPIDs = 2048 | ||
| 50 | |||
| 51 | // DefaultTmpfs is the size of the job's /tmp when --container-tmpfs is not given. It | ||
| 52 | // is a tmpfs, so it is memory, and an unbounded one is a job that can fill the | ||
| 53 | // machine's RAM without writing a single byte to the workspace. | ||
| 54 | const DefaultTmpfs = "1g" | ||
| 55 | |||
| 46 | 56 | // KeepAlive is PID 1's command. Steps run with exec, so the container has to stay | |
| 47 | 57 | // up between them. `/bin/sh` is also what the steps need, which keeps the | |
| 48 | 58 | // requirement to one the image must satisfy anyway. | |
| @@ -60,6 +70,11 @@ type Config struct { | |||
| 60 | 70 | Memory string | |
| 61 | 71 | CPUs string | |
| 62 | 72 | PIDs int | |
| 73 | // Tmpfs is the size of the job's /tmp, which is a tmpfs in RAM: a compile that | ||
| 74 | // writes its output there and then runs it must not be able to take the machine | ||
| 75 | // with it. It cannot exceed Memory when both are set, because tmpfs pages are | ||
| 76 | // charged to the container's own cgroup. | ||
| 77 | Tmpfs string | ||
| 63 | 78 | } | |
| 64 | 79 | ||
| 65 | 80 | // Validate fills in the defaults and refuses settings the runtime would reject. | |
| @@ -111,6 +126,18 @@ func Validate(cfg Config) (Config, error) { | |||
| 111 | 126 | if cfg.PIDs < 0 { | |
| 112 | 127 | return Config{}, fmt.Errorf("--container-pids must not be negative") | |
| 113 | 128 | } | |
| 129 | // Zero means the default, not "no limit": a job that can fork without a bound is | ||
| 130 | // a fork bomb away from taking the machine, and a bound nobody set is the easiest | ||
| 131 | // one to be missing. An operator who wants a different number sets it. | ||
| 132 | if cfg.PIDs == 0 { | ||
| 133 | cfg.PIDs = DefaultPIDs | ||
| 134 | } | ||
| 135 | if cfg.Tmpfs == "" { | ||
| 136 | cfg.Tmpfs = DefaultTmpfs | ||
| 137 | } | ||
| 138 | if strings.ContainsAny(cfg.Tmpfs, " \t\r\n") { | ||
| 139 | return Config{}, fmt.Errorf("--container-tmpfs %q is not a size", cfg.Tmpfs) | ||
| 140 | } | ||
| 114 | 141 | return cfg, nil | |
| 115 | 142 | } | |
| 116 | 143 | ||
| @@ -238,6 +265,20 @@ func ValidatePaths(s Spec) error { | |||
| 238 | 265 | return nil | |
| 239 | 266 | } | |
| 240 | 267 | ||
| 268 | // tmpfsArg is the /tmp mount, with its own size. exec is explicit because the | ||
| 269 | // runtime mounts a tmpfs noexec by default; compilers write their output here and | ||
| 270 | // then run it, so noexec breaks every Go workflow. | ||
| 271 | func tmpfsArg(cfg Config) string { | ||
| 272 | size := cfg.Tmpfs | ||
| 273 | if size == "" { | ||
| 274 | // Validate fills this in; a Config built by hand still gets a mounted /tmp with | ||
| 275 | // a size rather than one that is unbounded, because "no size" is the failure | ||
| 276 | // mode this exists to prevent. | ||
| 277 | size = DefaultTmpfs | ||
| 278 | } | ||
| 279 | return "/tmp:rw,nosuid,nodev,exec,mode=1777,size=" + size | ||
| 280 | } | ||
| 281 | |||
| 241 | 282 | // RunArgs is the argument list for `docker run`. It is pure so it can be checked | |
| 242 | 283 | // without a daemon. | |
| 243 | 284 | func RunArgs(cfg Config, s Spec) []string { | |
| @@ -282,12 +323,7 @@ func RunArgs(cfg Config, s Spec) []string { | |||
| 282 | 323 | args = append(args, "--volume", m.Host+":"+m.Path+":rw") | |
| 283 | 324 | } | |
| 284 | 325 | args = append(args, | |
| 285 | // exec is explicit because the runtime mounts a tmpfs noexec by default. | ||
| 286 | // Compilers write their output here and then run it: `go test` and | ||
| 287 | // `go run` both exec from GOCACHE under /tmp, so noexec breaks every Go | ||
| 288 | // workflow. It costs no isolation a job does not already have, since the | ||
| 289 | // workspace mount is writable and executable too. | ||
| 290 | "--tmpfs", "/tmp:rw,nosuid,nodev,exec,mode=1777", | ||
| 326 | "--tmpfs", tmpfsArg(cfg), | ||
| 291 | 327 | "--cap-drop", "ALL", | |
| 292 | 328 | "--security-opt", "no-new-privileges", | |
| 293 | 329 | "--init", | |
| @@ -490,7 +526,7 @@ func ServiceRunArgs(cfg Config, s ServiceSpec) []string { | |||
| 490 | 526 | args = append(args, "--env", name+"="+s.Env[name]) | |
| 491 | 527 | } | |
| 492 | 528 | args = append(args, | |
| 493 | "--tmpfs", "/tmp:rw,nosuid,nodev,exec,mode=1777", | ||
| 529 | "--tmpfs", tmpfsArg(cfg), | ||
| 494 | 530 | "--cap-drop", "ALL", | |
| 495 | 531 | "--security-opt", "no-new-privileges", | |
| 496 | 532 | "--init", | |