vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Run CI jobs in a container chosen by the runner
7da9111ecabb236d48fc48592cd3d1d45f9afe78 · Keenan Nemetz · 15d ago · parent 474a812d
Run CI jobs in a container chosen by the runner Each lease gets a disposable checkout and, when --image is set, its own container from that image. The runner keeps the SSH key and trusted Git cache outside the job, so workflow code cannot reach either; older caches are rebuilt because a previous job may have written to them. The record names the image and runtime, and the queue mirrors them for the run page. Without --image the runner still executes on the host and says so.
ci/agent/agent.go +115−79
| @@ -1,10 +1,11 @@ | |||
| 1 | 1 | // Package agent implements the Vole CI agent. | |
| 2 | 2 | // | |
| 3 | 3 | // A runner connects to the instance over SSH with the key its account pushes | |
| 4 | // with, leases one job at a time, runs its steps in a persistent checkout, then | ||
| 5 | // uploads the log and artifacts and publishes the attempt as a signed commit. | ||
| 6 | // This package is the agent; argument parsing and the CLI's messages stay in | ||
| 7 | // cmd/vole. | ||
| 4 | // with, leases one job at a time, and gives the job a disposable checkout. Steps | ||
| 5 | // run either on the host or in a container from the operator's image; the runner's | ||
| 6 | // key and trusted Git cache stay outside both. When the job ends the runner uploads | ||
| 7 | // the log and artifacts and publishes the attempt as a signed commit. This package | ||
| 8 | // is the agent; argument parsing and the CLI's messages stay in cmd/vole. | ||
| 8 | 9 | package agent | |
| 9 | 10 | ||
| 10 | 11 | import ( | |
| @@ -133,6 +134,29 @@ type Options struct { | |||
| 133 | 134 | Labels []string | |
| 134 | 135 | // Once runs one job and exits, which is what a scheduled container wants. | |
| 135 | 136 | Once bool | |
| 137 | // Image runs each job's steps in a container built from this image. Empty runs | ||
| 138 | // them directly on the runner host. The image belongs to the machine, not to the | ||
| 139 | // workflow: a workflow asks for labels, and the operator decides what those | ||
| 140 | // labels mean. | ||
| 141 | Image string | ||
| 142 | // ContainerRuntime is the CLI used to run containers: docker, podman, or nerdctl. | ||
| 143 | ContainerRuntime string | ||
| 144 | // ContainerNetwork is the network mode: none (the default) or bridge. | ||
| 145 | ContainerNetwork string | ||
| 146 | // ContainerPull is the image pull policy: missing (the default), always, or never. | ||
| 147 | ContainerPull string | ||
| 148 | // ContainerUser is the user the job runs as inside the container, uid:gid. | ||
| 149 | // Empty means the runner user's uid:gid. | ||
| 150 | ContainerUser string | ||
| 151 | // ContainerUserns is the user namespace mode passed to the runtime. Empty uses | ||
| 152 | // keep-id for rootless Podman, which is what makes a bind-mounted job tree | ||
| 153 | // writable by the runner user. | ||
| 154 | ContainerUserns string | ||
| 155 | // ContainerMemory, ContainerCPUs, and ContainerPIDs are optional resource limits | ||
| 156 | // passed to the container runtime. Zero or empty leaves the runtime's default. | ||
| 157 | ContainerMemory string | ||
| 158 | ContainerCPUs string | ||
| 159 | ContainerPIDs int | ||
| 136 | 160 | } | |
| 137 | 161 | ||
| 138 | 162 | // Run leases and executes CI jobs until the connection fails, the context ends, | |
| @@ -149,10 +173,21 @@ func Run(ctx context.Context, opts Options) error { | |||
| 149 | 173 | } | |
| 150 | 174 | cache = filepath.Join(base, "vole-runner") | |
| 151 | 175 | } | |
| 176 | cache, err := filepath.Abs(cache) | ||
| 177 | if err != nil { | ||
| 178 | return err | ||
| 179 | } | ||
| 152 | 180 | if err := os.MkdirAll(cache, 0o755); err != nil { | |
| 153 | 181 | return err | |
| 154 | 182 | } | |
| 155 | 183 | ||
| 184 | box, err := containerOptions(opts, cache) | ||
| 185 | if err != nil { | ||
| 186 | return err | ||
| 187 | } | ||
| 188 | cleanupJobTrees(cache) | ||
| 189 | cleanupStaleContainers(box) | ||
| 190 | |||
| 156 | 191 | // A label has to be one the workflow validator would accept, or the machine would | |
| 157 | 192 | // advertise something no job can ask for — a typo with no symptom at all. | |
| 158 | 193 | for _, label := range opts.Labels { | |
| @@ -186,6 +221,11 @@ func Run(ctx context.Context, opts Options) error { | |||
| 186 | 221 | sess.account, sess.name = who.Account, who.Name | |
| 187 | 222 | sess.repos, sess.labels = opts.Repos, opts.Labels | |
| 188 | 223 | fmt.Fprintf(os.Stderr, "vole runner: %s\n", sess.describe(who.Name)) | |
| 224 | if box.enabled() { | ||
| 225 | fmt.Fprintf(os.Stderr, "vole runner: jobs run in %s via %s\n", box.image, box.runtime) | ||
| 226 | } else { | ||
| 227 | fmt.Fprintln(os.Stderr, "vole runner: no --image given; jobs run directly on this host") | ||
| 228 | } | ||
| 189 | 229 | for { | |
| 190 | 230 | // A resident runner parks on the server's lease: the reply arrives when work is | |
| 191 | 231 | // queued, not on a fixed beat, so an idle pool costs one open channel per | |
| @@ -221,7 +261,7 @@ func Run(ctx context.Context, opts Options) error { | |||
| 221 | 261 | if lease <= 0 { | |
| 222 | 262 | lease = 60 * time.Second | |
| 223 | 263 | } | |
| 224 | sess.run(opts.Remote, cache, rep.Run, lease) | ||
| 264 | sess.run(opts.Remote, cache, rep.Run, lease, box) | ||
| 225 | 265 | if opts.Once { | |
| 226 | 266 | return nil | |
| 227 | 267 | } | |
| @@ -360,9 +400,9 @@ func (s *runnerSession) close() { | |||
| 360 | 400 | ||
| 361 | 401 | // run executes a leased job and reports the outcome. A lost lease ends the job | |
| 362 | 402 | // without reporting a result: another runner owns it now. | |
| 363 | func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Duration) { | ||
| 403 | func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Duration, box containerConfig) { | ||
| 364 | 404 | log := &strings.Builder{} | |
| 365 | work, err := prepareWorktree(remote, cache, job.Repo, job.SHA) | ||
| 405 | co, err := prepareWorktree(remote, cache, job.Repo, job.SHA, job.ID, job.Attempt) | ||
| 366 | 406 | if err != nil { | |
| 367 | 407 | // No checkout means no branch to commit a record to, so there is nothing to | |
| 368 | 408 | // report: the lease expires and the queue gives the job to a runner that can. | |
| @@ -370,17 +410,39 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 370 | 410 | job.Repo, format.SHA(job.SHA), err) | |
| 371 | 411 | return | |
| 372 | 412 | } | |
| 413 | defer co.cleanup() | ||
| 414 | // recordBox is set only after the container actually starts. A failed start is | ||
| 415 | // not evidence that the job ran in that image, so its record stays on the host. | ||
| 416 | recordBox := containerConfig{} | ||
| 373 | 417 | fail := func(format string, args ...any) { | |
| 374 | 418 | msg := fmt.Sprintf(format, args...) | |
| 375 | 419 | fmt.Fprintf(os.Stderr, "vole runner: %s\n", msg) | |
| 376 | 420 | fmt.Fprintf(log, "\n%s\n", msg) | |
| 377 | s.finish(remote, work, job, log, 1, nil, nil) | ||
| 421 | s.finish(remote, co, job, log, 1, nil, nil, recordBox) | ||
| 378 | 422 | } | |
| 379 | 423 | fmt.Fprintf(log, "=== %s · %s · %s @ %s\n", job.Repo, job.Job, job.Branch, format.SHA(job.SHA)) | |
| 424 | if box.enabled() { | ||
| 425 | fmt.Fprintf(log, "environment: %s via %s\n", box.image, box.runtime) | ||
| 426 | } | ||
| 427 | |||
| 428 | var sandbox *container | ||
| 429 | if box.enabled() { | ||
| 430 | sandbox, err = startContainer(box, job, co.Work) | ||
| 431 | if err != nil { | ||
| 432 | fail("%s", err) | ||
| 433 | return | ||
| 434 | } | ||
| 435 | recordBox = box | ||
| 436 | defer func() { | ||
| 437 | if sandbox != nil { | ||
| 438 | sandbox.stop() | ||
| 439 | } | ||
| 440 | }() | ||
| 441 | } | ||
| 380 | 442 | ||
| 381 | 443 | for _, step := range job.Steps { | |
| 382 | 444 | fmt.Fprintf(log, "\n--- %s\n", step.Name) | |
| 383 | code, chunk, lost := s.execStep(job.ID, work, step, lease) | ||
| 445 | code, chunk, lost := s.execStep(job.ID, co, sandbox, step, lease) | ||
| 384 | 446 | log.WriteString(chunk) | |
| 385 | 447 | if lost { | |
| 386 | 448 | fmt.Fprintf(os.Stderr, "vole runner: lease lost while running %q; stopping\n", step.Name) | |
| @@ -388,11 +450,22 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 388 | 450 | } | |
| 389 | 451 | if code != 0 { | |
| 390 | 452 | fmt.Fprintf(log, "\nstep %q exited %d\n", step.Name, code) | |
| 391 | s.finish(remote, work, job, log, code, nil, nil) | ||
| 453 | if sandbox != nil { | ||
| 454 | sandbox.stop() | ||
| 455 | sandbox = nil | ||
| 456 | } | ||
| 457 | s.finish(remote, co, job, log, code, nil, nil, recordBox) | ||
| 392 | 458 | return | |
| 393 | 459 | } | |
| 394 | 460 | } | |
| 395 | arts, releaseArts, err := publishArtifacts(remote, work, job, log) | ||
| 461 | // The steps are done. Stop the container before reading artifacts or deploying a | ||
| 462 | // site, so a process the workflow left in the background cannot change the files | ||
| 463 | // while the runner is publishing them. | ||
| 464 | if sandbox != nil { | ||
| 465 | sandbox.stop() | ||
| 466 | sandbox = nil | ||
| 467 | } | ||
| 468 | arts, releaseArts, err := publishArtifacts(remote, co.Work, job, log) | ||
| 396 | 469 | if err != nil { | |
| 397 | 470 | fail("%s", err) | |
| 398 | 471 | return | |
| @@ -405,11 +478,11 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 405 | 478 | return | |
| 406 | 479 | } | |
| 407 | 480 | } | |
| 408 | if err := publishSite(remote, work, job, log); err != nil { | ||
| 481 | if err := publishSite(remote, co, job, log); err != nil { | ||
| 409 | 482 | fail("%s", err) | |
| 410 | 483 | return | |
| 411 | 484 | } | |
| 412 | s.finish(remote, work, job, log, 0, arts, releaseArts) | ||
| 485 | s.finish(remote, co, job, log, 0, arts, releaseArts, recordBox) | ||
| 413 | 486 | } | |
| 414 | 487 | ||
| 415 | 488 | // finish uploads this attempt's log, commits its record and pushes it, and only then | |
| @@ -425,7 +498,7 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 425 | 498 | // A log that cannot be stored does not lose the result: the record then names no log | |
| 426 | 499 | // and the failure is reported on stderr rather than swallowed. A record that cannot be | |
| 427 | 500 | // pushed is not reported at all, because there is nothing for the server to read. | |
| 428 | func (s *runnerSession) finish(remote, work string, job *runnerJob, log *strings.Builder, exit int, arts, releaseArts []runnerArtifact) { | ||
| 501 | func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *strings.Builder, exit int, arts, releaseArts []runnerArtifact, box containerConfig) { | ||
| 429 | 502 | oid, size, err := uploadLog(remote, job.Repo, log.String()) | |
| 430 | 503 | if err != nil { | |
| 431 | 504 | fmt.Fprintf(os.Stderr, "vole runner: could not upload the log for %s: %v\n", job.ID, err) | |
| @@ -444,6 +517,9 @@ func (s *runnerSession) finish(remote, work string, job *runnerJob, log *strings | |||
| 444 | 517 | // and a platform quirk for a fact the binary already knows. | |
| 445 | 518 | OS: runtime.GOOS, Arch: runtime.GOARCH, | |
| 446 | 519 | } | |
| 520 | if box.enabled() { | ||
| 521 | rec.Image, rec.Runtime = box.image, box.runtime | ||
| 522 | } | ||
| 447 | 523 | if oid != "" { | |
| 448 | 524 | rec.Log = &lfs.Pointer{OID: oid, Size: size} | |
| 449 | 525 | } | |
| @@ -458,7 +534,7 @@ func (s *runnerSession) finish(remote, work string, job *runnerJob, log *strings | |||
| 458 | 534 | return | |
| 459 | 535 | } | |
| 460 | 536 | ||
| 461 | if err := publishRecord(remote, work, job, rec, releaseArts, log); err != nil { | ||
| 537 | if err := publishRecord(remote, co, job, rec, releaseArts, log); err != nil { | ||
| 462 | 538 | fmt.Fprintf(os.Stderr, "vole runner: could not publish the record for %s: %v\n", job.ID, err) | |
| 463 | 539 | return | |
| 464 | 540 | } | |
| @@ -481,16 +557,14 @@ func (s *runnerSession) finish(remote, work string, job *runnerJob, log *strings | |||
| 481 | 557 | // commit, so a release cannot carry bytes whose build has no record and a run does not | |
| 482 | 558 | // pay a push per output. A lost race against somebody else's issue is re-applied on | |
| 483 | 559 | // the new tip, the same way every other metadata write is. | |
| 484 | func publishRecord(remote, work string, job *runnerJob, rec runs.Record, rel []runnerArtifact, log *strings.Builder) error { | ||
| 560 | func publishRecord(remote string, co checkout, job *runnerJob, rec runs.Record, rel []runnerArtifact, log *strings.Builder) error { | ||
| 485 | 561 | url, err := client.RepoURL(remote, job.Repo) | |
| 486 | 562 | if err != nil { | |
| 487 | 563 | return err | |
| 488 | 564 | } | |
| 489 | gitDir, err := client.GitDirOf(work) | ||
| 490 | if err != nil { | ||
| 491 | return err | ||
| 492 | } | ||
| 493 | c, err := client.OpenBranch(gitDir, url, strings.TrimPrefix(metadata.Ref, "refs/heads/"), "vole-runner") | ||
| 565 | // Use the trusted cache, never the job tree: the workflow may have edited the | ||
| 566 | // job tree's .git, and a push from there could run a hook the workflow wrote. | ||
| 567 | c, err := client.OpenBranch(co.GitDir, url, strings.TrimPrefix(metadata.Ref, "refs/heads/"), "vole-runner") | ||
| 494 | 568 | if err != nil { | |
| 495 | 569 | return err | |
| 496 | 570 | } | |
| @@ -622,7 +696,7 @@ func publishArtifacts(remote, work string, job *runnerJob, log *strings.Builder) | |||
| 622 | 696 | // newer commit has landed would quietly revert the site to an older build. Refusing | |
| 623 | 697 | // is the honest outcome — the log says why, and the job fails instead of publishing | |
| 624 | 698 | // a stale tree that nobody notices for a week. | |
| 625 | func publishSite(remote, work string, job *runnerJob, log *strings.Builder) error { | ||
| 699 | func publishSite(remote string, co checkout, job *runnerJob, log *strings.Builder) error { | ||
| 626 | 700 | if job.Pages == nil || job.Pages.Path == "" { | |
| 627 | 701 | return nil | |
| 628 | 702 | } | |
| @@ -630,7 +704,7 @@ func publishSite(remote, work string, job *runnerJob, log *strings.Builder) erro | |||
| 630 | 704 | if err != nil { | |
| 631 | 705 | return err | |
| 632 | 706 | } | |
| 633 | tip, err := client.LSRemote(work, url, "refs/heads/"+job.Branch) | ||
| 707 | tip, err := client.LSRemote(co.GitDir, url, "refs/heads/"+job.Branch) | ||
| 634 | 708 | if err != nil { | |
| 635 | 709 | return fmt.Errorf("check %s before deploying the site: %w", job.Branch, err) | |
| 636 | 710 | } | |
| @@ -642,15 +716,11 @@ func publishSite(remote, work string, job *runnerJob, log *strings.Builder) erro | |||
| 642 | 716 | format.SHA(job.SHA), job.Branch, format.SHA(tip)) | |
| 643 | 717 | } | |
| 644 | 718 | ||
| 645 | gitDir, err := client.GitDirOf(work) | ||
| 646 | if err != nil { | ||
| 647 | return err | ||
| 648 | } | ||
| 649 | c, err := client.OpenBranch(gitDir, url, job.Pages.Ref, "vole-runner") | ||
| 719 | c, err := client.OpenBranch(co.GitDir, url, job.Pages.Ref, "vole-runner") | ||
| 650 | 720 | if err != nil { | |
| 651 | 721 | return err | |
| 652 | 722 | } | |
| 653 | dir := filepath.Join(work, filepath.FromSlash(job.Pages.Path)) | ||
| 723 | dir := filepath.Join(co.Work, filepath.FromSlash(job.Pages.Path)) | ||
| 654 | 724 | msg := fmt.Sprintf("site: %s from run %s (%s)", job.Branch, format.SHA(job.SHA), job.ID) | |
| 655 | 725 | if len(msg) > 200 { | |
| 656 | 726 | msg = fmt.Sprintf("site: %s from run %s", job.Branch, job.ID) | |
| @@ -755,7 +825,7 @@ func (w *logBuffer) String() string { | |||
| 755 | 825 | ||
| 756 | 826 | // execStep runs one shell step, heartbeating while it works. It returns the exit | |
| 757 | 827 | // code, the output produced, and whether the lease was lost. | |
| 758 | func (s *runnerSession) execStep(runID, work string, step runnerStep, lease time.Duration) (int, string, bool) { | ||
| 828 | func (s *runnerSession) execStep(runID string, co checkout, sandbox *container, step runnerStep, lease time.Duration) (int, string, bool) { | ||
| 759 | 829 | timeout := 10 * time.Minute | |
| 760 | 830 | if d, err := time.ParseDuration(step.Timeout); err == nil && d > 0 { | |
| 761 | 831 | timeout = d | |
| @@ -763,9 +833,15 @@ func (s *runnerSession) execStep(runID, work string, step runnerStep, lease time | |||
| 763 | 833 | ctx, cancel := context.WithTimeout(context.Background(), timeout) | |
| 764 | 834 | defer cancel() | |
| 765 | 835 | ||
| 766 | cmd := exec.CommandContext(ctx, "sh", "-c", step.Run) | ||
| 767 | cmd.Dir = work | ||
| 768 | cmd.Env = append(os.Environ(), "CI=true", "VOLE_CI=1") | ||
| 836 | var cmd *exec.Cmd | ||
| 837 | if sandbox != nil { | ||
| 838 | cmd = sandbox.execCommand(ctx, step) | ||
| 839 | cmd.Dir = co.Work | ||
| 840 | } else { | ||
| 841 | cmd = exec.CommandContext(ctx, "sh", "-c", step.Run) | ||
| 842 | cmd.Dir = co.Work | ||
| 843 | cmd.Env = append(os.Environ(), "CI=true", "VOLE_CI=1") | ||
| 844 | } | ||
| 769 | 845 | // Own process group: a step that spawns children must not outlive its timeout. | |
| 770 | 846 | cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} | |
| 771 | 847 | w := &logBuffer{} | |
| @@ -802,6 +878,12 @@ func (s *runnerSession) execStep(runID, work string, step runnerStep, lease time | |||
| 802 | 878 | } | |
| 803 | 879 | return 1, chunk + fmt.Sprintf("\n%v\n", err), false | |
| 804 | 880 | case <-ctx.Done(): | |
| 881 | // Killing the docker exec client is not enough: the process inside the | ||
| 882 | // container would keep running. Remove the whole container, which also | ||
| 883 | // makes the timeout the end of this job rather than one hung step. | ||
| 884 | if sandbox != nil { | ||
| 885 | sandbox.stop() | ||
| 886 | } | ||
| 805 | 887 | if cmd.Process != nil { | |
| 806 | 888 | _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) | |
| 807 | 889 | } | |
| @@ -817,49 +899,3 @@ func (s *runnerSession) execStep(runID, work string, step runnerStep, lease time | |||
| 817 | 899 | } | |
| 818 | 900 | } | |
| 819 | 901 | } | |
| 820 | |||
| 821 | // prepareWorktree fetches the commit to run and checks it out, from a cache that | ||
| 822 | // persists between jobs so a second run of the same repository is cheap. | ||
| 823 | func prepareWorktree(remote, cache, repo, sha string) (string, error) { | ||
| 824 | cloneURL, err := client.RepoURL(remote, repo) | ||
| 825 | if err != nil { | ||
| 826 | return "", err | ||
| 827 | } | ||
| 828 | dir := filepath.Join(cache, repo) | ||
| 829 | |||
| 830 | if _, err := os.Stat(filepath.Join(dir, ".git")); err != nil { | ||
| 831 | if err := os.RemoveAll(dir); err != nil { | ||
| 832 | return "", err | ||
| 833 | } | ||
| 834 | if out, err := exec.Command("git", "clone", "--quiet", cloneURL, dir).CombinedOutput(); err != nil { | ||
| 835 | return "", fmt.Errorf("git clone: %v: %s", err, out) | ||
| 836 | } | ||
| 837 | } else { | ||
| 838 | cmd := exec.Command("git", "-C", dir, "fetch", "--quiet", "--prune", "origin", | ||
| 839 | "+refs/heads/*:refs/remotes/origin/*") | ||
| 840 | if out, err := cmd.CombinedOutput(); err != nil { | ||
| 841 | return "", fmt.Errorf("git fetch: %v: %s", err, out) | ||
| 842 | } | ||
| 843 | } | ||
| 844 | // Start from the exact commit, with no leftovers from a previous run. | ||
| 845 | for _, args := range [][]string{ | ||
| 846 | {"checkout", "--quiet", "--force", "--detach", sha}, | ||
| 847 | {"clean", "-qffdx"}, | ||
| 848 | } { | ||
| 849 | cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) | ||
| 850 | if out, err := cmd.CombinedOutput(); err != nil { | ||
| 851 | // The commit may not be reachable from any branch yet: fetch it by id. | ||
| 852 | if args[0] == "checkout" { | ||
| 853 | if out2, err2 := exec.Command("git", "-C", dir, "fetch", "--quiet", "origin", sha).CombinedOutput(); err2 == nil { | ||
| 854 | if out3, err3 := exec.Command("git", "-C", dir, "checkout", "--quiet", "--force", "--detach", sha).CombinedOutput(); err3 == nil { | ||
| 855 | _ = out2 | ||
| 856 | _ = out3 | ||
| 857 | continue | ||
| 858 | } | ||
| 859 | } | ||
| 860 | } | ||
| 861 | return "", fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, out) | ||
| 862 | } | ||
| 863 | } | ||
| 864 | return dir, nil | ||
| 865 | } | ||