vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Bound what a CI job can consume

6bfc42949be4d40b3b74de8cc17c6d903cfe531e · Keenan Nemetz · 13d ago · parent f5456470

Bound what a CI job can consume

A job runs untrusted build code, and everything it produces is either
stored by the instance or kept on the runner. None of those channels had
a bound of its own.

The attempt log was a strings.Builder: the whole run in memory, then
written to a temp file, then read twice by the LFS transfer (once to hash,
once to send). It is now a spool file on disk from the first byte, capped
at 32 MiB, uploaded once at the end as an ordinary LFS object. A capped
log reports the whole write as taken so the producing process is not
killed, and gets a line saying it was truncated. Step offsets are
computed from the spool's length, which never exceeds the cap, so a
truncated log still has valid ranges. Nothing is streamed to the server:
a log's content-addressed id cannot be known before its last byte, so
streaming would only buy a second protocol.

A run may publish at most 256 artifact files totalling 4 GiB. The
workflow limit is on declarations, and a declaration is a pattern that
can match a directory tree, so the bound on what it expands to has to be
where the tree is walked.

Contributor caches are now namespaced per account, not one shared
"contrib" namespace: one contributor's untrusted branch must not be able
to seed another contributor's cache. A contributor keeps their cache
across their own branches.

Container volume paths are checked before the argument list is built: a
colon is the field separator, and a NUL or newline would split the
argument, so a path carrying one is refused rather than passed to the
runtime. RunArgs is unchanged; the refusal is in Start, and the closed-set
test now also pins --cap-add, --device, --pid/--ipc host and the podman
socket.

Image builds run through the configured runtime, so a rootless Podman
runner builds rootless. The definition is pinned by digest already; the
docs now say how the build gets its network and recommend pinning the
base image in from: by digest for a reproducible build.

Tests: the spool keeps what fits, truncates and says so, and removes its
file; artifact count and byte caps (the byte cap uses a sparse file so it
does not write 4 GiB); contributor caches are separated by account while
one account's branches share; volume paths with a colon, NUL or newline
are refused.

ci/agent/spool_test.go +120

@@ -0,0 +1,120 @@
	1	package agent
	2
	3	import (
	4	"os"
	5	"strings"
	6	"testing"
	7	)
	8
	9	func TestSpoolKeepsWhatFits(t *testing.T) {
	10	t.Parallel()
	11	s, err := newSpool(1024)
	12	if err != nil {
	13	t.Fatal(err)
	14	}
	15	defer s.Close()
	16	if _, err := s.Write([]byte("hello\n")); err != nil {
	17	t.Fatal(err)
	18	}
	19	if s.Len() != 6 {
	20	t.Fatalf("Len = %d, want 6", s.Len())
	21	}
	22	if err := s.seal(); err != nil {
	23	t.Fatal(err)
	24	}
	25	raw, err := os.ReadFile(s.Path())
	26	if err != nil {
	27	t.Fatal(err)
	28	}
	29	if string(raw) != "hello\n" {
	30	t.Fatalf("log = %q", raw)
	31	}
	32	}
	33
	34	// A capped log reports the whole write as taken, because a short write would kill
	35	// the process producing it; the excess is dropped and the file says so.
	36	func TestSpoolTruncatesAndSaysSo(t *testing.T) {
	37	t.Parallel()
	38	s, err := newSpool(16)
	39	if err != nil {
	40	t.Fatal(err)
	41	}
	42	defer s.Close()
	43	if n, err := s.Write(make([]byte, 100)); err != nil \|\| n != 100 {
	44	t.Fatalf("Write = %d, %v; want 100, nil", n, err)
	45	}
	46	if s.Len() != 16 {
	47	t.Fatalf("Len = %d, want the cap", s.Len())
	48	}
	49	if err := s.seal(); err != nil {
	50	t.Fatal(err)
	51	}
	52	raw, err := os.ReadFile(s.Path())
	53	if err != nil {
	54	t.Fatal(err)
	55	}
	56	if len(raw) <= 16 {
	57	t.Fatalf("the truncation note was not appended: %d bytes", len(raw))
	58	}
	59	if !strings.Contains(string(raw), "log truncated") {
	60	t.Fatalf("the log does not say it was truncated: %q", raw)
	61	}
	62	// Step ranges are computed from Len, which never exceeds the cap, so a range can
	63	// never name bytes the note overwrote.
	64	if s.Len() > 16 {
	65	t.Fatalf("Len = %d after sealing, want at most the cap", s.Len())
	66	}
	67	}
	68
	69	func TestSpoolCloseRemovesTheFile(t *testing.T) {
	70	t.Parallel()
	71	s, err := newSpool(1024)
	72	if err != nil {
	73	t.Fatal(err)
	74	}
	75	if _, err := s.Write([]byte("x")); err != nil {
	76	t.Fatal(err)
	77	}
	78	name := s.Path()
	79	if err := s.Close(); err != nil {
	80	t.Fatal(err)
	81	}
	82	if _, err := os.Stat(name); !os.IsNotExist(err) {
	83	t.Fatalf("the log file survived Close: %v", err)
	84	}
	85	}
	86
	87	func TestCheckArtifactLimits(t *testing.T) {
	88	t.Parallel()
	89	work := t.TempDir()
	90	write := func(name, body string) {
	91	if err := os.WriteFile(work+"/"+name, []byte(body), 0o644); err != nil {
	92	t.Fatal(err)
	93	}
	94	}
	95	write("a", "12345")
	96	files := []artifactFile{{Path: "a", Name: "a"}}
	97	if err := checkArtifactLimits(work, files); err != nil {
	98	t.Fatalf("a small artifact set was refused: %v", err)
	99	}
	100	many := make([]artifactFile, MaxRunArtifactFiles+1)
	101	for i := range many {
	102	many[i] = artifactFile{Path: "a", Name: "a"}
	103	}
	104	if err := checkArtifactLimits(work, many); err == nil {
	105	t.Fatal("too many artifact files were accepted")
	106	}
	107	// A sparse file is a 4 GiB stat result that costs no disk, which is how the byte
	108	// limit can be tested without writing 4 GiB.
	109	bigPath := work + "/big"
	110	if err := os.WriteFile(bigPath, nil, 0o644); err != nil {
	111	t.Fatal(err)
	112	}
	113	if err := os.Truncate(bigPath, MaxRunArtifactBytes+1); err != nil {
	114	t.Skipf("this filesystem does not support a sparse file: %v", err)
	115	}
	116	big := []artifactFile{{Path: "big", Name: "big"}}
	117	if err := checkArtifactLimits(work, big); err == nil {
	118	t.Fatal("an oversized artifact set was accepted")
	119	}
	120	}