vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Hold a job's cache namespace while the job runs

a29ce1f76aaf7ebc4b24afad36c56a0f7f01254e · Keenan Nemetz · 3d ago · parent b492b557

Hold a job's cache namespace while the job runs

A namespace was only safe while one job existed per runner. Two runners
sharing a --dir already had the bug: the trimmer deletes whole namespaces
and `--no-cache` empties one, either of which can land on a tree another
process has mounted. A Go module cache that loses a file mid-build is a
checksum error deep in a build, which is the case the whole-namespace
eviction rule exists to avoid.

A job now holds its namespace for as long as the container may be inside
it — shared, so two builds of the same repository still run together —
and eviction takes the same lock with LOCK_NB and leaves what is in use
for a later pass. `--no-cache` takes the namespace exclusively, because
emptying it afterwards must not land on somebody else's build.

The runner also stops trimming at the end of each job: the cap is
enforced at startup and on a timer, so eviction no longer belongs to one
job's lifetime. Lock files live under <root>/.locks, mirroring the
namespace, so a cache tree holds namespaces and nothing else; they are
never deleted, for the reason gitcache's are not.

`vole workflow run` gets the same lock, so two local runs in one checkout
cannot delete each other's cache either.

Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)

ci/agent/cache_test.go +61−9

@@ -4,6 +4,8 @@ import (
	4	4	"path/filepath"
	5	5	"strings"
	6	6	"testing"
		7
		8	jobcache "github.com/rodent-software/vole/ci/cache"
	7	9	)
	8	10
	9	11	// A cache is shared mutable state, so which namespace a job gets is a policy
@@ -12,9 +14,10 @@ import (
	12	14	func TestCacheNamespacesByRepoAndTrust(t *testing.T) {
	13	15	t.Parallel()
	14	16	root := t.TempDir()
	15		s := &runnerSession{cacheRoot: root}
		17	s := &runnerSession{cacheRoot: root, cacheReuse: true}
	16	18
	17		mounts := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/main", Image: "alpine:3.22"})
		19	mounts, release := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/main", Image: "alpine:3.22"})
		20	defer release()
	18	21	if len(mounts) != 1 \|\| mounts[0].Path != "/cache" {
	19	22	t.Fatalf("mounts = %+v, want one at /cache", mounts)
	20	23	}
@@ -25,14 +28,16 @@ func TestCacheNamespacesByRepoAndTrust(t *testing.T) {
	25	28	t.Fatalf("cache %q is not namespaced by repository", mounts[0].Host)
	26	29	}
	27	30
	28		contribMounts := s.cacheFor(&runnerJob{
		31	contribMounts, releaseContrib := s.cacheFor(&runnerJob{
	29	32	Repo: "demo", Ref: "refs/heads/contrib/alice/patch", Image: "alpine:3.22",
	30	33	})
		34	defer releaseContrib()
	31	35	if contribMounts[0].Host == mounts[0].Host {
	32	36	t.Fatalf("a contributor branch shares the protected branch's cache: %s", mounts[0].Host)
	33	37	}
	34	38
	35		otherMounts := s.cacheFor(&runnerJob{Repo: "other", Ref: "refs/heads/main", Image: "alpine:3.22"})
		39	otherMounts, releaseOther := s.cacheFor(&runnerJob{Repo: "other", Ref: "refs/heads/main", Image: "alpine:3.22"})
		40	defer releaseOther()
	36	41	if otherMounts[0].Host == mounts[0].Host {
	37	42	t.Fatal("two repositories share a cache namespace")
	38	43	}
@@ -41,7 +46,9 @@ func TestCacheNamespacesByRepoAndTrust(t *testing.T) {
	41	46	func TestCacheCanBeTurnedOff(t *testing.T) {
	42	47	t.Parallel()
	43	48	s := &runnerSession{}
	44		if mounts := s.cacheFor(&runnerJob{Repo: "demo"}); mounts != nil {
		49	mounts, release := s.cacheFor(&runnerJob{Repo: "demo"})
		50	defer release()
		51	if mounts != nil {
	45	52	t.Fatalf("a disabled cache produced %v", mounts)
	46	53	}
	47	54	}
@@ -51,16 +58,61 @@ func TestCacheCanBeTurnedOff(t *testing.T) {
	51	58	// branch inherits whatever the first one wrote.
	52	59	func TestContributorCachesAreSeparatedByAccount(t *testing.T) {
	53	60	t.Parallel()
	54		s := &runnerSession{cacheRoot: t.TempDir()}
	55		alice := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/contrib/alice/one", Image: "alpine:3.22"})
	56		bob := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/contrib/bob/one", Image: "alpine:3.22"})
		61	s := &runnerSession{cacheRoot: t.TempDir(), cacheReuse: true}
		62	alice, releaseAlice := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/contrib/alice/one", Image: "alpine:3.22"})
		63	defer releaseAlice()
		64	bob, releaseBob := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/contrib/bob/one", Image: "alpine:3.22"})
		65	defer releaseBob()
	57	66	if alice[0].Host == bob[0].Host {
	58	67	t.Fatalf("two contributors share a cache: %s", alice[0].Host)
	59	68	}
	60	69	// The same contributor's second branch reuses their cache, which is the point of
	61	70	// namespacing by account rather than by branch.
	62		again := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/contrib/alice/two", Image: "alpine:3.22"})
		71	again, releaseAgain := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/contrib/alice/two", Image: "alpine:3.22"})
		72	defer releaseAgain()
	63	73	if again[0].Host != alice[0].Host {
	64	74	t.Fatalf("one contributor's branches got different caches: %s vs %s", alice[0].Host, again[0].Host)
	65	75	}
	66	76	}
		77
		78	// A job holds its namespace, so eviction has to skip it: the trimmer runs on the
		79	// runner's own beat and may pass a mounted namespace in the middle of a build.
		80	func TestAReusedCacheIsHeldAgainstEviction(t *testing.T) {
		81	t.Parallel()
		82	s := &runnerSession{cacheRoot: t.TempDir(), cacheReuse: true}
		83	mounts, release := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/main", Image: "alpine:3.22"})
		84	defer release()
		85	if _, held := jobcache.TryLockExclusive(s.cacheRoot, mounts[0].Host); held {
		86	t.Fatal("eviction could take a namespace a job has mounted")
		87	}
		88	}
		89
		90	// Two jobs of the same repository share their namespace: they are meant to run at
		91	// once, and the tools inside them provide their own locking. Taking the second
		92	// shared lock returning at all is the assertion.
		93	func TestTwoJobsMayShareOneNamespace(t *testing.T) {
		94	t.Parallel()
		95	s := &runnerSession{cacheRoot: t.TempDir(), cacheReuse: true}
		96	job := &runnerJob{Repo: "demo", Ref: "refs/heads/main", Image: "alpine:3.22"}
		97	first, releaseFirst := s.cacheFor(job)
		98	defer releaseFirst()
		99	second, releaseSecond := s.cacheFor(job)
		100	defer releaseSecond()
		101	if first[0].Host != second[0].Host {
		102	t.Fatalf("one repository got two caches: %s vs %s", first[0].Host, second[0].Host)
		103	}
		104	}
		105
		106	// --no-cache empties the namespace when the job ends, so it must not run while
		107	// anybody else is inside it.
		108	func TestNoCacheHoldsTheNamespaceExclusively(t *testing.T) {
		109	t.Parallel()
		110	s := &runnerSession{cacheRoot: t.TempDir(), cacheReuse: false}
		111	mounts, release := s.cacheFor(&runnerJob{Repo: "demo", Ref: "refs/heads/main", Image: "alpine:3.22"})
		112	defer release()
		113	other, held := jobcache.TryLockExclusive(s.cacheRoot, mounts[0].Host)
		114	if held {
		115	other()
		116	t.Fatal("a --no-cache job did not hold its namespace exclusively")
		117	}
		118	}