vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound the runner's containers and keep its disk free
4e3759da08a603ede50107e89a23ff6d5432b7ac · Keenan Nemetz · 16m ago · parent c4ad0d5a
Bound the runner's containers and keep its disk free The container boundary was good and the resource the job consumes was not. A job could fork without a bound, write /tmp until the machine's RAM was gone, and fill the disk its workspace sits on — and the runner would only notice when a build died. The isolation protects the forge's data; this protects the machine that runs the code. - `--container-pids` now defaults to 2048 and zero means the default, not "no limit": a bound nobody set is the easiest one to be missing, and a job that can fork without one is a fork bomb away from taking the machine. - `--container-tmpfs` is new and defaults to 1G. /tmp is a tmpfs, so its size is memory, and an unsized one is a build that fills RAM without writing a byte to the workspace. It is passed to service containers too. - `--container-memory` stays the operator's setting rather than a default, because the right number is the machine's size and a wrong default would show up as an OOM kill in someone's build. All three places a runner is configured now set it: the packaged runner.env example, the manual steps in the deploy guide, and the cloud-init template (RUNNER_MEMORY, default 4G, alongside RUNNER_PIDS, RUNNER_TMPFS and RUNNER_DISK_RESERVE). - `--disk-reserve` (default 2G) is how the job tree is bounded. It is a host bind mount, so the runtime cannot quota it and a filesystem quota needs root; a walker over the tree would cost more than it protects. What the runner can promise is that it notices: one statfs per second while a command runs, and the job is stopped below the reserve with the numbers in its log. `--dir` on its own sized filesystem is the hard bound, and the docs say so. - The runner unit gets `MemoryMax=90%` as the backstop under all of it, for a job that gets past its own cgroup. It is a percentage so the unit does not have to know the machine. TasksMax is deliberately not set: the container's own --container-pids already applies, and a unit value would have to track --jobs and --container-pids and would fail builds, looking like the workflow's fault, when they drifted apart. A workflow still cannot raise or remove any of this: the limits are flags on the runner and the unit, never fields a job can set. scripts/check-deploy.py now fails when the rendered runner arguments stop carrying the memory, pids, tmpfs or disk limits, so the first-boot file cannot quietly lose them. Closes 01M6S798BH0BK00R2B5N1KDJYM
ci/agent/agent.go +39−3
| @@ -283,11 +283,21 @@ type Options struct { | |||
| 283 | 283 | // keep-id for rootless Podman, which is what makes a bind-mounted job tree | |
| 284 | 284 | // writable by the runner user. | |
| 285 | 285 | ContainerUserns string | |
| 286 | // ContainerMemory, ContainerCPUs, and ContainerPIDs are optional resource limits | ||
| 287 | // passed to the container runtime. Zero or empty leaves the runtime's default. | ||
| 286 | // ContainerMemory and ContainerCPUs are optional resource limits passed to the | ||
| 287 | // container runtime. Empty leaves the runtime's own behavior. | ||
| 288 | 288 | ContainerMemory string | |
| 289 | 289 | ContainerCPUs string | |
| 290 | ContainerPIDs int | ||
| 290 | // ContainerPIDs bounds one container's processes. Zero means the default in | ||
| 291 | // ci/container, not "no limit": a job that can fork without a bound is a fork bomb | ||
| 292 | // away from taking the machine. | ||
| 293 | ContainerPIDs int | ||
| 294 | // ContainerTmpfs is the size of the job's /tmp. Empty means the default; it is a | ||
| 295 | // tmpfs, so this is memory. | ||
| 296 | ContainerTmpfs string | ||
| 297 | // DiskReserve is how much free space the runner insists on keeping under Dir. A | ||
| 298 | // job that takes the filesystem below it is stopped, because a job tree is a host | ||
| 299 | // bind mount with no quota and the machine is shared. Zero disables the check. | ||
| 300 | DiskReserve int64 | ||
| 291 | 301 | ||
| 292 | 302 | // NoCache turns off the per-repository cache that lets a job reuse downloads | |
| 293 | 303 | // and compiled output between runs. A runner without one is slower and has | |
| @@ -505,6 +515,7 @@ func (sl *slot) connect() error { | |||
| 505 | 515 | sess.repos, sess.labels = sl.opts.Repos, sl.opts.Labels | |
| 506 | 516 | sess.network = sl.box.Network | |
| 507 | 517 | sess.cacheRoot, sess.cacheReuse = sl.cacheRoot, !sl.opts.NoCache | |
| 518 | sess.diskRoot, sess.diskReserve = sl.cache, sl.opts.DiskReserve | ||
| 508 | 519 | sl.sess, sl.who = sess, who | |
| 509 | 520 | return nil | |
| 510 | 521 | } | |
| @@ -725,6 +736,11 @@ type runnerSession struct { | |||
| 725 | 736 | // network is the network this runner can give, sent with every lease so the server | |
| 726 | 737 | // only hands it a job whose `network:` it can satisfy. | |
| 727 | 738 | network string | |
| 739 | // diskRoot is the filesystem a job's tree is written to, and diskReserve is how | ||
| 740 | // much of it must stay free. Together they are the only bound on a job tree: it is | ||
| 741 | // a host bind mount, so the runtime cannot quota it. | ||
| 742 | diskRoot string | ||
| 743 | diskReserve int64 | ||
| 728 | 744 | } | |
| 729 | 745 | ||
| 730 | 746 | // describe is the one line a runner prints when it starts work, because "why is | |
| @@ -1676,6 +1692,10 @@ func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID str | |||
| 1676 | 1692 | } | |
| 1677 | 1693 | tick := time.NewTicker(interval) | |
| 1678 | 1694 | defer tick.Stop() | |
| 1695 | // The disk is checked on its own beat, not the heartbeat's: a build that writes | ||
| 1696 | // gigabytes a second must not get a whole lease interval to do it. | ||
| 1697 | disk := time.NewTicker(time.Second) | ||
| 1698 | defer disk.Stop() | ||
| 1679 | 1699 | ||
| 1680 | 1700 | stop := func() { | |
| 1681 | 1701 | if onTimeout != nil { | |
| @@ -1703,6 +1723,22 @@ func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID str | |||
| 1703 | 1723 | case <-ctx.Done(): | |
| 1704 | 1724 | stop() | |
| 1705 | 1725 | return 1, false, false | |
| 1726 | case <-disk.C: | ||
| 1727 | free, low, err := diskLow(s.diskRoot, s.diskReserve) | ||
| 1728 | if err != nil { | ||
| 1729 | // One failed check is worth saying once; a stat on a directory that | ||
| 1730 | // vanished would otherwise fill the log with the same line. | ||
| 1731 | fmt.Fprintf(out, "\ncannot check the runner's free space: %v\n", err) | ||
| 1732 | disk.Stop() | ||
| 1733 | continue | ||
| 1734 | } | ||
| 1735 | if low { | ||
| 1736 | fmt.Fprintf(out, "\nthe runner has %s free, below its %s reserve; stopping the job "+ | ||
| 1737 | "so the machine does not fill (--disk-reserve)\n", | ||
| 1738 | format.Bytes(free), format.Bytes(s.diskReserve)) | ||
| 1739 | stop() | ||
| 1740 | return 1, false, false | ||
| 1741 | } | ||
| 1706 | 1742 | case <-tick.C: | |
| 1707 | 1743 | rep, err := s.call(runnerRequest{Op: "heartbeat", Run: runID}) | |
| 1708 | 1744 | if err != nil || !rep.OK { | |