vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Add packaging, a CI workflow, and containerised local runs
3b39a84c913cabe15bfa6a60dd176ff2dd28d241 · Keenan Nemetz · 14d ago · parent 30a72490
Add packaging, a CI workflow, and containerised local runs A v* tag now builds a .deb and a .rpm for amd64 and arm64 with nfpm and attaches them, with checksums, to the release. The package ships the systemd unit, a config at /etc/vole/vole.toml, and a postinstall that creates the service account and prepares the instance that config names; `vole admin init --config` is what makes that possible, and the binary's version is stamped from the tag instead of the source tree. scripts/ci.sh becomes .vole/workflows/ci.yaml, so `vole workflow run ci` executes exactly what a push runs, and the Markdown link check moves to scripts/check-links.py. Local runs now always execute in the job image and share ci/container with the runner; --no-container is gone, because a run in a different environment proves nothing about CI. The runner mounts a per-repository namespace at /cache and sets no variable for it: a job opts in through its image's env or the new job-level env, which is pinned with the queue and delivered with the lease. Whole namespaces are evicted least-recently-written first so a module cache is never half-deleted, and contributor branches get their own namespace so an untrusted tip cannot seed the cache a protected branch builds into. Running the suite in that container found four bugs worth naming: /tmp was mounted noexec, which broke `go test` and `go run`; the read-only root is gone, because images make toolchain directories world-writable deliberately and the mount set is what protects the host; OpenSSH needs a passwd entry for the job's uid; and a lost metadata push reported as "cannot lock ref" was not classified as a race, so concurrent CLI writes gave up on the first collision. That retry now backs off exponentially with jitter.
ci/agent/agent.go +107−12
| @@ -28,9 +28,12 @@ import ( | |||
| 28 | 28 | "time" | |
| 29 | 29 | ||
| 30 | 30 | "github.com/rodent-software/vole/ci" | |
| 31 | jobcache "github.com/rodent-software/vole/ci/cache" | ||
| 32 | "github.com/rodent-software/vole/ci/container" | ||
| 31 | 33 | "github.com/rodent-software/vole/ci/runs" | |
| 32 | 34 | "github.com/rodent-software/vole/client" | |
| 33 | 35 | "github.com/rodent-software/vole/core/lfs" | |
| 36 | "github.com/rodent-software/vole/forge/access" | ||
| 34 | 37 | "github.com/rodent-software/vole/internal/format" | |
| 35 | 38 | "github.com/rodent-software/vole/metadata" | |
| 36 | 39 | ) | |
| @@ -73,6 +76,8 @@ type runnerJob struct { | |||
| 73 | 76 | ImageFile string `json:"image_file,omitempty"` | |
| 74 | 77 | ImageDefinition string `json:"image_definition,omitempty"` | |
| 75 | 78 | ImageSHA string `json:"image_sha,omitempty"` | |
| 79 | // Env is the job's declared environment, pinned when the run was queued. | ||
| 80 | Env map[string]string `json:"env,omitempty"` | ||
| 76 | 81 | // Artifacts are path patterns, expanded in the checkout after the steps pass. | |
| 77 | 82 | Artifacts []string `json:"artifacts,omitempty"` | |
| 78 | 83 | // Tag is set for a tag-triggered run; TagAnnotated says whether it carries | |
| @@ -164,6 +169,13 @@ type Options struct { | |||
| 164 | 169 | ContainerMemory string | |
| 165 | 170 | ContainerCPUs string | |
| 166 | 171 | ContainerPIDs int | |
| 172 | |||
| 173 | // NoCache turns off the per-repository cache that lets a job reuse downloads | ||
| 174 | // and compiled output between runs. A runner without one is slower and has | ||
| 175 | // nothing to poison, which is the trade. | ||
| 176 | NoCache bool | ||
| 177 | // CacheMaxBytes caps that cache. Zero means the default; see ci/cache. | ||
| 178 | CacheMaxBytes int64 | ||
| 167 | 179 | } | |
| 168 | 180 | ||
| 169 | 181 | // Run leases and executes CI jobs until the connection fails, the context ends, | |
| @@ -195,6 +207,27 @@ func Run(ctx context.Context, opts Options) error { | |||
| 195 | 207 | cleanupJobTrees(cache) | |
| 196 | 208 | cleanupStaleContainers(box) | |
| 197 | 209 | ||
| 210 | // The directory is mounted whether or not it is reused: an image that sets | ||
| 211 | // GOCACHE=/cache/go-build must work under --no-cache too, and a non-root job | ||
| 212 | // cannot create /cache in the image's own (root-owned) tree. | ||
| 213 | cacheRoot := filepath.Join(cache, "cache") | ||
| 214 | if err := os.MkdirAll(cacheRoot, 0o755); err != nil { | ||
| 215 | return fmt.Errorf("prepare the job cache: %w", err) | ||
| 216 | } | ||
| 217 | cacheMax := opts.CacheMaxBytes | ||
| 218 | if cacheMax <= 0 { | ||
| 219 | cacheMax = jobcache.DefaultBytes | ||
| 220 | } | ||
| 221 | if opts.NoCache { | ||
| 222 | // Nothing survives, so nothing needs bounding; empty what a killed runner | ||
| 223 | // left behind. | ||
| 224 | _ = jobcache.Clear(cacheRoot) | ||
| 225 | } else if removed, err := jobcache.Trim(cacheRoot, cacheMax); err == nil && removed > 0 { | ||
| 226 | // A killed runner leaves whatever the last job wrote; bound it before work | ||
| 227 | // starts rather than after the disk is full. | ||
| 228 | fmt.Fprintf(os.Stderr, "vole runner: trimmed %s from the job cache\n", format.Bytes(removed)) | ||
| 229 | } | ||
| 230 | |||
| 198 | 231 | // A label has to be one the workflow validator would accept, or the machine would | |
| 199 | 232 | // advertise something no job can ask for — a typo with no symptom at all. | |
| 200 | 233 | for _, label := range opts.Labels { | |
| @@ -227,8 +260,20 @@ func Run(ctx context.Context, opts Options) error { | |||
| 227 | 260 | } | |
| 228 | 261 | sess.account, sess.name = who.Account, who.Name | |
| 229 | 262 | sess.repos, sess.labels = opts.Repos, opts.Labels | |
| 263 | sess.cacheRoot, sess.cacheMax, sess.cacheReuse = cacheRoot, cacheMax, !opts.NoCache | ||
| 264 | if opts.NoCache { | ||
| 265 | fmt.Fprintf(os.Stderr, "vole runner: job cache in %s, emptied after every job\n", cacheRoot) | ||
| 266 | } else { | ||
| 267 | fmt.Fprintf(os.Stderr, "vole runner: job cache in %s, capped at %s\n", | ||
| 268 | cacheRoot, format.Bytes(cacheMax)) | ||
| 269 | } | ||
| 230 | 270 | fmt.Fprintf(os.Stderr, "vole runner: %s\n", sess.describe(who.Name)) | |
| 231 | fmt.Fprintf(os.Stderr, "vole runner: jobs run in the workflow's image via %s\n", box.runtime) | ||
| 271 | fmt.Fprintf(os.Stderr, "vole runner: jobs run in the workflow's image via %s\n", box.Runtime) | ||
| 272 | if container.RunsAsRoot(box.Config) { | ||
| 273 | fmt.Fprintln(os.Stderr, "vole runner: warning: jobs run as root in their container, so a job can "+ | ||
| 274 | "rewrite its own image; run the runner as an unprivileged user (or pass --container-user) "+ | ||
| 275 | "to have the image's permissions keep it immutable") | ||
| 276 | } | ||
| 232 | 277 | for { | |
| 233 | 278 | // A resident runner parks on the server's lease: the reply arrives when work is | |
| 234 | 279 | // queued, not on a fixed beat, so an idle pool costs one open channel per | |
| @@ -288,6 +333,13 @@ type runnerSession struct { | |||
| 288 | 333 | // asked for, not the effective list: the server intersects it with the grant on | |
| 289 | 334 | // each request, so an allowlist edit takes effect without a restart. | |
| 290 | 335 | repos []string | |
| 336 | // cacheRoot is where this runner keeps the caches it mounts into jobs. cacheMax | ||
| 337 | // is the cap, in bytes. cacheReuse says whether a job keeps what the last one | ||
| 338 | // wrote; with --no-cache the directory is still mounted and emptied instead, so | ||
| 339 | // an image that points at /cache works either way. | ||
| 340 | cacheRoot string | ||
| 341 | cacheMax int64 | ||
| 342 | cacheReuse bool | ||
| 291 | 343 | // labels is what this machine is (--label). A job is only given to a runner that | |
| 292 | 344 | // carries every label the job asked for, so an unlabelled runner only sees jobs | |
| 293 | 345 | // that ask for nothing. | |
| @@ -403,7 +455,7 @@ func (s *runnerSession) close() { | |||
| 403 | 455 | ||
| 404 | 456 | // run executes a leased job and reports the outcome. A lost lease ends the job | |
| 405 | 457 | // without reporting a result: another runner owns it now. | |
| 406 | func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Duration, box containerConfig) { | ||
| 458 | func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Duration, box runnerBox) { | ||
| 407 | 459 | log := &strings.Builder{} | |
| 408 | 460 | co, err := prepareWorktree(remote, cache, job.Repo, job.SHA, job.ID, job.Attempt) | |
| 409 | 461 | if err != nil { | |
| @@ -428,7 +480,7 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 428 | 480 | s.finish(remote, co, job, log, 1, nil, nil, box) | |
| 429 | 481 | } | |
| 430 | 482 | fmt.Fprintf(log, "=== %s · %s · %s @ %s\n", job.Repo, job.Job, job.Branch, format.SHA(job.SHA)) | |
| 431 | fmt.Fprintf(log, "environment: %s via %s\n", job.Image, box.runtime) | ||
| 483 | fmt.Fprintf(log, "environment: %s via %s\n", job.Image, box.Runtime) | ||
| 432 | 484 | ||
| 433 | 485 | image, err := s.resolveImage(job, box, log, lease) | |
| 434 | 486 | if err != nil { | |
| @@ -439,14 +491,35 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 439 | 491 | fail("%s", err) | |
| 440 | 492 | return | |
| 441 | 493 | } | |
| 442 | sandbox, err := startContainer(box, image, job, co.Work) | ||
| 494 | mounts := s.cacheFor(job) | ||
| 495 | if s.cacheRoot != "" { | ||
| 496 | defer func() { | ||
| 497 | if !s.cacheReuse { | ||
| 498 | for _, m := range mounts { | ||
| 499 | _ = os.RemoveAll(m.Host) | ||
| 500 | } | ||
| 501 | return | ||
| 502 | } | ||
| 503 | if removed, err := jobcache.Trim(s.cacheRoot, s.cacheMax); err == nil && removed > 0 { | ||
| 504 | fmt.Fprintf(os.Stderr, "vole runner: trimmed %s from the job cache\n", format.Bytes(removed)) | ||
| 505 | } | ||
| 506 | }() | ||
| 507 | } | ||
| 508 | sandbox, err := container.Start(box.Config, container.Spec{ | ||
| 509 | Image: image, | ||
| 510 | Name: containerName(box, job), | ||
| 511 | Work: co.Work, | ||
| 512 | Labels: box.labels(job), | ||
| 513 | Mounts: mounts, | ||
| 514 | Env: job.Env, | ||
| 515 | }) | ||
| 443 | 516 | if err != nil { | |
| 444 | 517 | fail("%s", err) | |
| 445 | 518 | return | |
| 446 | 519 | } | |
| 447 | 520 | defer func() { | |
| 448 | 521 | if sandbox != nil { | |
| 449 | sandbox.stop() | ||
| 522 | sandbox.Stop() | ||
| 450 | 523 | } | |
| 451 | 524 | }() | |
| 452 | 525 | ||
| @@ -460,7 +533,7 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 460 | 533 | } | |
| 461 | 534 | if code != 0 { | |
| 462 | 535 | fmt.Fprintf(log, "\nstep %q exited %d\n", step.Name, code) | |
| 463 | sandbox.stop() | ||
| 536 | sandbox.Stop() | ||
| 464 | 537 | sandbox = nil | |
| 465 | 538 | s.finish(remote, co, job, log, code, nil, nil, box) | |
| 466 | 539 | return | |
| @@ -469,7 +542,7 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 469 | 542 | // The steps are done. Stop the container before reading artifacts or deploying a | |
| 470 | 543 | // site, so a process the workflow left in the background cannot change the files | |
| 471 | 544 | // while the runner is publishing them. | |
| 472 | sandbox.stop() | ||
| 545 | sandbox.Stop() | ||
| 473 | 546 | sandbox = nil | |
| 474 | 547 | arts, releaseArts, err := publishArtifacts(remote, co.Work, job, log) | |
| 475 | 548 | if err != nil { | |
| @@ -491,6 +564,28 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 491 | 564 | s.finish(remote, co, job, log, 0, arts, releaseArts, box) | |
| 492 | 565 | } | |
| 493 | 566 | ||
| 567 | // cacheFor returns the mount for one job's cache, or nothing when the runner has | ||
| 568 | // caching turned off. It sets no environment: a job opts in by pointing its own | ||
| 569 | // variables at Mount. The namespace separates repositories from each other, and | ||
| 570 | // contributor branches from everything else: a cache is shared mutable state, and a | ||
| 571 | // branch nobody has vouched for must not be able to seed the one a protected branch | ||
| 572 | // builds into. | ||
| 573 | func (s *runnerSession) cacheFor(job *runnerJob) []container.Mount { | ||
| 574 | if s.cacheRoot == "" { | ||
| 575 | return nil | ||
| 576 | } | ||
| 577 | trust := "direct" | ||
| 578 | if _, contributor := access.ContributorRefName(job.Ref); contributor { | ||
| 579 | trust = "contrib" | ||
| 580 | } | ||
| 581 | ns := jobcache.Namespace(s.cacheRoot, job.Repo, trust, jobcache.ImageKey(job.ImageSHA, job.Image)) | ||
| 582 | if err := os.MkdirAll(ns, 0o755); err != nil { | ||
| 583 | fmt.Fprintf(os.Stderr, "vole runner: cannot prepare a cache for %s: %v\n", job.Repo, err) | ||
| 584 | return nil | ||
| 585 | } | ||
| 586 | return []container.Mount{{Host: ns, Path: jobcache.Mount}} | ||
| 587 | } | ||
| 588 | |||
| 494 | 589 | // finish uploads this attempt's log, commits its record and pushes it, and only then | |
| 495 | 590 | // tells the server it is done. | |
| 496 | 591 | // | |
| @@ -504,7 +599,7 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 504 | 599 | // A log that cannot be stored does not lose the result: the record then names no log | |
| 505 | 600 | // and the failure is reported on stderr rather than swallowed. A record that cannot be | |
| 506 | 601 | // pushed is not reported at all, because there is nothing for the server to read. | |
| 507 | func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *strings.Builder, exit int, arts, releaseArts []runnerArtifact, box containerConfig) { | ||
| 602 | func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *strings.Builder, exit int, arts, releaseArts []runnerArtifact, box runnerBox) { | ||
| 508 | 603 | oid, size, err := uploadLog(remote, job.Repo, log.String()) | |
| 509 | 604 | if err != nil { | |
| 510 | 605 | fmt.Fprintf(os.Stderr, "vole runner: could not upload the log for %s: %v\n", job.ID, err) | |
| @@ -524,7 +619,7 @@ func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log * | |||
| 524 | 619 | OS: runtime.GOOS, Arch: runtime.GOARCH, | |
| 525 | 620 | } | |
| 526 | 621 | rec.Image, rec.ImageFile, rec.ImageSHA = job.Image, job.ImageFile, job.ImageSHA | |
| 527 | rec.Runtime = box.runtime | ||
| 622 | rec.Runtime = box.Runtime | ||
| 528 | 623 | if oid != "" { | |
| 529 | 624 | rec.Log = &lfs.Pointer{OID: oid, Size: size} | |
| 530 | 625 | } | |
| @@ -830,7 +925,7 @@ func (w *logBuffer) String() string { | |||
| 830 | 925 | ||
| 831 | 926 | // execStep runs one shell step, heartbeating while it works. It returns the exit | |
| 832 | 927 | // code, the output produced, and whether the lease was lost. | |
| 833 | func (s *runnerSession) execStep(runID string, sandbox *container, step runnerStep, lease time.Duration) (int, string, bool) { | ||
| 928 | func (s *runnerSession) execStep(runID string, sandbox *container.Container, step runnerStep, lease time.Duration) (int, string, bool) { | ||
| 834 | 929 | timeout := 10 * time.Minute | |
| 835 | 930 | if d, err := time.ParseDuration(step.Timeout); err == nil && d > 0 { | |
| 836 | 931 | timeout = d | |
| @@ -838,8 +933,8 @@ func (s *runnerSession) execStep(runID string, sandbox *container, step runnerSt | |||
| 838 | 933 | ctx, cancel := context.WithTimeout(context.Background(), timeout) | |
| 839 | 934 | defer cancel() | |
| 840 | 935 | ||
| 841 | cmd := sandbox.execCommand(ctx, step) | ||
| 842 | code, chunk, lost := s.runCommand(ctx, cmd, runID, lease, sandbox.stop) | ||
| 936 | cmd := sandbox.Exec(ctx, step.Run) | ||
| 937 | code, chunk, lost := s.runCommand(ctx, cmd, runID, lease, sandbox.Stop) | ||
| 843 | 938 | if ctx.Err() == context.DeadlineExceeded { | |
| 844 | 939 | // The heartbeat will have extended the lease this far; report the timeout as | |
| 845 | 940 | // a failure rather than leaving the job hanging. | |