Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Bound what a CI job can consume

6bfc42949be4d40b3b74de8cc17c6d903cfe531e · Keenan Nemetz · 12d ago · parent f5456470 

Bound what a CI job can consume

A job runs untrusted build code, and everything it produces is either
stored by the instance or kept on the runner. None of those channels had
a bound of its own.

The attempt log was a strings.Builder: the whole run in memory, then
written to a temp file, then read twice by the LFS transfer (once to hash,
once to send). It is now a spool file on disk from the first byte, capped
at 32 MiB, uploaded once at the end as an ordinary LFS object. A capped
log reports the whole write as taken so the producing process is not
killed, and gets a line saying it was truncated. Step offsets are
computed from the spool's length, which never exceeds the cap, so a
truncated log still has valid ranges. Nothing is streamed to the server:
a log's content-addressed id cannot be known before its last byte, so
streaming would only buy a second protocol.

A run may publish at most 256 artifact files totalling 4 GiB. The
workflow limit is on declarations, and a declaration is a pattern that
can match a directory tree, so the bound on what it expands to has to be
where the tree is walked.

Contributor caches are now namespaced per account, not one shared
"contrib" namespace: one contributor's untrusted branch must not be able
to seed another contributor's cache. A contributor keeps their cache
across their own branches.

Container volume paths are checked before the argument list is built: a
colon is the field separator, and a NUL or newline would split the
argument, so a path carrying one is refused rather than passed to the
runtime. RunArgs is unchanged; the refusal is in Start, and the closed-set
test now also pins --cap-add, --device, --pid/--ipc host and the podman
socket.

Image builds run through the configured runtime, so a rootless Podman
runner builds rootless. The definition is pinned by digest already; the
docs now say how the build gets its network and recommend pinning the
base image in from: by digest for a reproducible build.

Tests: the spool keeps what fits, truncates and says so, and removes its
file; artifact count and byte caps (the byte cap uses a sparse file so it
does not write 4 GiB); contributor caches are separated by account while
one account's branches share; volume paths with a colon, NUL or newline
are refused.

ci/agent/agent.go +77−70

@@ -15,6 +15,7 @@ import (
15 15 "encoding/json"
16 16 "errors"
17 17 "fmt"
18 "io"
18 19 "net/url"
19 20 "os"
20 21 "os/exec"
@@ -461,7 +462,12 @@ func (s *runnerSession) close() {
461 462 // run executes a leased job and reports the outcome. A lost lease ends the job
462 463 // without reporting a result: another runner owns it now.
463 464 func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Duration, box runnerBox) {
464 log := &strings.Builder{}
465 log, err := newSpool(MaxLogBytes)
466 if err != nil {
467 fmt.Fprintf(os.Stderr, "vole runner: %v\n", err)
468 return
469 }
470 defer log.Close()
465 471 co, err := prepareWorktree(remote, cache, job.Repo, job.SHA, job.ID, job.Attempt)
466 472 if err != nil {
467 473 // No checkout means no branch to commit a record to, so there is nothing to
@@ -532,14 +538,13 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur
532 538 for _, step := range job.Steps {
533 539 // The step's section of the log starts at its header, so a reader can open
534 540 // one step without holding the image build that ran before it.
535 logStart := int64(log.Len())
541 logStart := log.Len()
536 542 started := time.Now().UTC()
537 543 fmt.Fprintf(log, "\n--- %s\n", step.Name)
538 code, chunk, lost := s.execStep(job.ID, sandbox, step, lease)
539 log.WriteString(chunk)
544 code, lost := s.execStep(job.ID, sandbox, step, lease, log)
540 545 result := runs.StepResult{
541 546 Name: step.Name, Started: started, Finished: time.Now().UTC(), Exit: code,
542 LogOffset: logStart, LogSize: int64(log.Len()) - logStart,
547 LogOffset: logStart, LogSize: log.Len() - logStart,
543 548 }
544 549 if lost {
545 550 fmt.Fprintf(os.Stderr, "vole runner: lease lost while running %q; stopping\n", step.Name)
@@ -547,7 +552,7 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur
547 552 }
548 553 if code != 0 {
549 554 fmt.Fprintf(log, "\nstep %q exited %d\n", step.Name, code)
550 result.LogSize = int64(log.Len()) - logStart
555 result.LogSize = log.Len() - logStart
551 556 result.Finished = time.Now().UTC()
552 557 steps = append(steps, result)
553 558 sandbox.Stop()
@@ -592,9 +597,13 @@ func (s *runnerSession) cacheFor(job *runnerJob) []container.Mount {
592 597 if s.cacheRoot == "" {
593 598 return nil
594 599 }
600 // A contributor branch is untrusted code, and a cache is shared mutable state:
601 // one contributor's branch must not be able to seed another's cache or the one a
602 // protected branch builds into. Namespacing by account means a contributor's
603 // cache follows them across their branches and nobody else can write it.
595 604 trust := "direct"
596 if _, contributor := access.ContributorRefName(job.Ref); contributor {
597 trust = "contrib"
605 if name, contributor := access.ContributorRefName(job.Ref); contributor {
606 trust = "contrib-" + name
598 607 }
599 608 ns := jobcache.Namespace(s.cacheRoot, job.Repo, trust, jobcache.ImageKey(job.ImageSHA, job.Image))
600 609 if err := os.MkdirAll(ns, 0o755); err != nil {
@@ -617,8 +626,8 @@ func (s *runnerSession) cacheFor(job *runnerJob) []container.Mount {
617 626 // A log that cannot be stored does not lose the result: the record then names no log
618 627 // and the failure is reported on stderr rather than swallowed. A record that cannot be
619 628 // pushed is not reported at all, because there is nothing for the server to read.
620 func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *strings.Builder, exit int, steps []runs.StepResult, arts, releaseArts []runnerArtifact, box runnerBox) {
621 oid, size, err := uploadLog(remote, job.Repo, log.String())
629 func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *spool, exit int, steps []runs.StepResult, arts, releaseArts []runnerArtifact, box runnerBox) {
630 oid, size, err := uploadLog(remote, job.Repo, log)
622 631 if err != nil {
623 632 fmt.Fprintf(os.Stderr, "vole runner: could not upload the log for %s: %v\n", job.ID, err)
624 633 }
@@ -682,7 +691,7 @@ func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *
682 691 // commit, so a release cannot carry bytes whose build has no record and a run does not
683 692 // pay a push per output. A lost race against somebody else's issue is re-applied on
684 693 // the new tip, the same way every other metadata write is.
685 func publishRecord(remote string, co checkout, job *runnerJob, rec runs.Record, rel []runnerArtifact, log *strings.Builder) error {
694 func publishRecord(remote string, co checkout, job *runnerJob, rec runs.Record, rel []runnerArtifact, log io.Writer) error {
686 695 url, err := client.RepoURL(remote, job.Repo)
687 696 if err != nil {
688 697 return err
@@ -733,14 +742,18 @@ func checkReleaseJob(job *runnerJob) error {
733 742 return nil
734 743 }
735 744
736 // uploadLog puts the log's bytes in the instance's object store over the same SSH
737 // transfer the artifacts use, and returns the id the store computed. An empty log
738 // uploads nothing and reports no object, so a run with no output has no log rather
739 // than an empty file.
740 func uploadLog(remote, repo, text string) (oid string, size int64, err error) {
741 if text == "" {
745 // uploadLog puts the attempt's log in the instance's object store over the same
746 // SSH transfer the artifacts use, and returns the id the store computed. The spool
747 // is already a file, so it is sent as it is: no copy through memory and no second
748 // pass to hash it. An empty log uploads nothing and reports no object, so a run
749 // with no output has no log rather than an empty file.
750 func uploadLog(remote, repo string, log *spool) (oid string, size int64, err error) {
751 if log == nil || log.Len() == 0 {
742 752 return "", 0, nil
743 753 }
754 if err := log.seal(); err != nil {
755 return "", 0, err
756 }
744 757 url, err := client.RepoURL(remote, repo)
745 758 if err != nil {
746 759 return "", 0, err
@@ -750,21 +763,7 @@ func uploadLog(remote, repo, text string) (oid string, size int64, err error) {
750 763 return "", 0, err
751 764 }
752 765 defer tx.Close()
753
754 f, err := os.CreateTemp("", "vole-log-*")
755 if err != nil {
756 return "", 0, err
757 }
758 name := f.Name()
759 defer os.Remove(name)
760 if _, err := f.WriteString(text); err != nil {
761 f.Close()
762 return "", 0, err
763 }
764 if err := f.Close(); err != nil {
765 return "", 0, err
766 }
767 ptr, err := tx.Upload(name)
766 ptr, err := tx.Upload(log.Path())
768 767 if err != nil {
769 768 return "", 0, err
770 769 }
@@ -776,7 +775,7 @@ func uploadLog(remote, repo, text string) (oid string, size int64, err error) {
776 775 // in before the finish, so the server can refuse a claim it cannot serve.
777 776 // publishArtifacts uploads every declared output and returns them twice: once as
778 777 // the run's outputs, once as the subset that belongs to the tag's release.
779 func publishArtifacts(remote, work string, job *runnerJob, log *strings.Builder) (all, release []runnerArtifact, err error) {
778 func publishArtifacts(remote, work string, job *runnerJob, log io.Writer) (all, release []runnerArtifact, err error) {
780 779 if len(job.Artifacts) == 0 {
781 780 return nil, nil, nil
782 781 }
@@ -784,6 +783,9 @@ func publishArtifacts(remote, work string, job *runnerJob, log *strings.Builder)
784 783 if err != nil {
785 784 return nil, nil, err
786 785 }
786 if err := checkArtifactLimits(work, files); err != nil {
787 return nil, nil, err
788 }
787 789 isRelease := map[string]bool{}
788 790 for _, pat := range job.ReleaseArtifacts {
789 791 isRelease[pat] = true
@@ -821,7 +823,7 @@ func publishArtifacts(remote, work string, job *runnerJob, log *strings.Builder)
821 823 // newer commit has landed would quietly revert the site to an older build. Refusing
822 824 // is the honest outcome — the log says why, and the job fails instead of publishing
823 825 // a stale tree that nobody notices for a week.
824 func publishSite(remote string, co checkout, job *runnerJob, log *strings.Builder) error {
826 func publishSite(remote string, co checkout, job *runnerJob, log io.Writer) error {
825 827 if job.Pages == nil || job.Pages.Path == "" {
826 828 return nil
827 829 }
@@ -874,6 +876,31 @@ type artifactFile struct {
874 876 // nothing is an error rather than an empty artifact list: a build that stopped
875 877 // producing the file is the failure a declaration is supposed to catch, and a
876 878 // silent publish of nothing is how that goes unnoticed for a week.
879 // checkArtifactLimits bounds what one run may publish. The workflow caps the number
880 // of declarations; a declaration is a pattern, and a pattern can match a directory
881 // tree, so the bound on what they expand to has to be here. An artifact set is
882 // uploaded into the instance's object store, and a job choosing its size is a job
883 // choosing the instance's disk.
884 func checkArtifactLimits(work string, files []artifactFile) error {
885 if len(files) > MaxRunArtifactFiles {
886 return fmt.Errorf("this run publishes %d artifact files, limit %d; narrow the patterns",
887 len(files), MaxRunArtifactFiles)
888 }
889 var total int64
890 for _, f := range files {
891 info, err := os.Stat(filepath.Join(work, filepath.FromSlash(f.Path)))
892 if err != nil {
893 continue
894 }
895 total += info.Size()
896 }
897 if total > MaxRunArtifactBytes {
898 return fmt.Errorf("this run publishes %s of artifacts, limit %s; narrow the patterns",
899 format.Bytes(total), format.Bytes(MaxRunArtifactBytes))
900 }
901 return nil
902 }
903
877 904 func collectArtifactFiles(root string, patterns []string) ([]artifactFile, error) {
878 905 matched := make([]int, len(patterns))
879 906 var files []artifactFile
@@ -928,30 +955,9 @@ func collectArtifactFiles(root string, patterns []string) ([]artifactFile, error
928 955 return files, nil
929 956 }
930 957
931 // logBuffer collects a step's output. It is a sync buffer because os/exec copies
932 // stdout and stderr in separate goroutines. Nothing is shipped while the step runs:
933 // the whole attempt's log is uploaded once, when the attempt is over, and the step
934 // records byte ranges into it rather than uploading a second object.
935 type logBuffer struct {
936 mu sync.Mutex
937 buf strings.Builder
938 }
939
940 func (w *logBuffer) Write(p []byte) (int, error) {
941 w.mu.Lock()
942 defer w.mu.Unlock()
943 return w.buf.Write(p)
944 }
945
946 func (w *logBuffer) String() string {
947 w.mu.Lock()
948 defer w.mu.Unlock()
949 return w.buf.String()
950 }
951
952 958 // execStep runs one shell step, heartbeating while it works. It returns the exit
953 959 // code, the output produced, and whether the lease was lost.
954 func (s *runnerSession) execStep(runID string, sandbox *container.Container, step runnerStep, lease time.Duration) (int, string, bool) {
960 func (s *runnerSession) execStep(runID string, sandbox *container.Container, step runnerStep, lease time.Duration, out io.Writer) (int, bool) {
955 961 timeout := 10 * time.Minute
956 962 if d, err := time.ParseDuration(step.Timeout); err == nil && d > 0 {
957 963 timeout = d
@@ -960,26 +966,27 @@ func (s *runnerSession) execStep(runID string, sandbox *container.Container, ste
960 966 defer cancel()
961 967
962 968 cmd := sandbox.Exec(ctx, step.Run)
963 code, chunk, lost := s.runCommand(ctx, cmd, runID, lease, sandbox.Stop)
969 code, lost := s.runCommand(ctx, cmd, runID, lease, sandbox.Stop, out)
964 970 if ctx.Err() == context.DeadlineExceeded {
965 971 // The heartbeat will have extended the lease this far; report the timeout as
966 972 // a failure rather than leaving the job hanging.
967 return 1, chunk + fmt.Sprintf("\nstep timed out after %s\n", timeout), lost
973 fmt.Fprintf(out, "\nstep timed out after %s\n", timeout)
974 return 1, lost
968 975 }
969 return code, chunk, lost
976 return code, lost
970 977 }
971 978
972 979 // runCommand starts one host-side command and heartbeats while it runs. It is used
973 980 // for docker exec and for building a custom image; onTimeout is what closes the
974 981 // sandbox when the command runs out of time.
975 func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID string, lease time.Duration, onTimeout func()) (int, string, bool) {
982 func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID string, lease time.Duration, onTimeout func(), out io.Writer) (int, bool) {
976 983 // Own process group: a command that spawns children must not outlive its timeout.
977 984 cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
978 w := &logBuffer{}
979 cmd.Stdout, cmd.Stderr = w, w
985 cmd.Stdout, cmd.Stderr = out, out
980 986
981 987 if err := cmd.Start(); err != nil {
982 return 1, fmt.Sprintf("cannot start: %v\n", err), false
988 fmt.Fprintf(out, "cannot start: %v\n", err)
989 return 1, false
983 990 }
984 991 done := make(chan error, 1)
985 992 go func() { done <- cmd.Wait() }()
@@ -994,15 +1001,15 @@ func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID str
994 1001 for {
995 1002 select {
996 1003 case err := <-done:
997 chunk := w.String()
998 1004 if err == nil {
999 return 0, chunk, false
1005 return 0, false
1000 1006 }
1001 1007 var ee *exec.ExitError
1002 1008 if errors.As(err, &ee) {
1003 return ee.ExitCode(), chunk, false
1009 return ee.ExitCode(), false
1004 1010 }
1005 return 1, chunk + fmt.Sprintf("\n%v\n", err), false
1011 fmt.Fprintf(out, "\n%v\n", err)
1012 return 1, false
1006 1013 case <-ctx.Done():
1007 1014 if onTimeout != nil {
1008 1015 onTimeout()
@@ -1010,14 +1017,14 @@ func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID str
1010 1017 if cmd.Process != nil {
1011 1018 _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
1012 1019 }
1013 // Wait for the reader goroutine before returning the output.
1020 // Wait for the copy goroutines before the caller closes the log.
1014 1021 <-done
1015 return 1, w.String(), false
1022 return 1, false
1016 1023 case <-tick.C:
1017 1024 // The lease is the only thing a heartbeat carries.
1018 1025 rep, err := s.call(runnerRequest{Op: "heartbeat", Run: runID})
1019 1026 if err != nil || !rep.OK {
1020 return 1, "", true
1027 return 1, true
1021 1028 }
1022 1029 }
1023 1030 }