vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Bound and reclaim the runner's saved checkouts

f594f1a16f6f8a8a3ee2867a44249b72a2c76c96 · Keenan Nemetz · 5d ago · parent de84c32b

Bound and reclaim the runner's saved checkouts

The runner already downloaded each repository once into --dir and cloned every
job from that copy, but nothing kept the copies bounded: they grew with every
tag, every branch, and every commit a job pinned by id, and a force-push left the
old history reachable through the remote-tracking reflogs. A long-lived runner
filled its disk and then could not check anything out at all.

Repack a copy that is most of the cap, dropping the objects no branch or tag can
reach and the temporary refs a killed runner leaves behind; then delete the least
recently used copy when the cap is still reached. --git-cache-size sets the cap
(10G by default, 0 keeps every copy). The repository this runner is about to
build is never the one deleted, and a copy another process holds is passed over.

Reading and writing a saved checkout now takes a per-repository lock, so a repack
or a deletion in one runner process cannot run underneath another's clone, or
underneath the site and release pushes that are written from the same cache.
The first clone skips the working-tree checkout, which nothing ever read.

ci/agent/gitcache_test.go +165

@@ -0,0 +1,165 @@
	1	package agent
	2
	3	import (
	4	"os"
	5	"path/filepath"
	6	"strings"
	7	"testing"
	8	"time"
	9	)
	10
	11	// The saved checkout exists to save a download, so its budget must not throw one
	12	// away while another job could still be using it, and must throw one away when the
	13	// disk is the thing at risk.
	14	func writeSavedCheckout(t *testing.T, cache, repo string, size int, used time.Time) string {
	15	t.Helper()
	16	dir := filepath.Join(cache, repo)
	17	if err := os.MkdirAll(filepath.Join(dir, ".git"), 0o755); err != nil {
	18	t.Fatal(err)
	19	}
	20	if err := os.WriteFile(filepath.Join(dir, ".git", "pack"), make([]byte, size), 0o644); err != nil {
	21	t.Fatal(err)
	22	}
	23	for _, name := range []string{cacheMarker, lastUsedStamp, repackStamp} {
	24	if err := os.WriteFile(filepath.Join(dir, name), []byte("1\n"), 0o644); err != nil {
	25	t.Fatal(err)
	26	}
	27	}
	28	stamp := filepath.Join(dir, lastUsedStamp)
	29	if err := os.Chtimes(stamp, used, used); err != nil {
	30	t.Fatal(err)
	31	}
	32	return dir
	33	}
	34
	35	func TestMaintainTrustedCachesDeletesTheLeastRecentlyUsedCheckout(t *testing.T) {
	36	cache := t.TempDir()
	37	now := time.Now()
	38	old := writeSavedCheckout(t, cache, "old", 1000, now.Add(-3024time.Hour))
	39	fresh := writeSavedCheckout(t, cache, "fresh", 1000, now)
	40
	41	maintainTrustedCaches(cache, 1500, "")
	42
	43	if _, err := os.Stat(old); !os.IsNotExist(err) {
	44	t.Fatalf("the least recently used checkout survived: %v", err)
	45	}
	46	if _, err := os.Stat(fresh); err != nil {
	47	t.Fatalf("the checkout a job just used was deleted: %v", err)
	48	}
	49	}
	50
	51	// The repository the next job will build from is never the one deleted, even when
	52	// it is the oldest: deleting it would turn a cache hit into a download for the very
	53	// job that asked.
	54	func TestMaintainTrustedCachesKeepsTheActiveCheckout(t *testing.T) {
	55	cache := t.TempDir()
	56	now := time.Now()
	57	active := writeSavedCheckout(t, cache, "active", 1000, now.Add(-3024time.Hour))
	58	other := writeSavedCheckout(t, cache, "other", 1000, now)
	59
	60	maintainTrustedCaches(cache, 1500, "active")
	61
	62	if _, err := os.Stat(active); err != nil {
	63	t.Fatalf("the active repository's checkout was deleted: %v", err)
	64	}
	65	if _, err := os.Stat(other); !os.IsNotExist(err) {
	66	t.Fatalf("the cache is over its cap and nothing else was deleted: %v", err)
	67	}
	68	}
	69
	70	// A checkout another process holds the lock on is in use, so its deletion is not
	71	// this process's to make.
	72	func TestDeleteSavedCheckoutDeclinesWhileAnotherProcessHoldsIt(t *testing.T) {
	73	cache := t.TempDir()
	74	dir := writeSavedCheckout(t, cache, "shared", 100, time.Now())
	75
	76	unlock, err := lockCache(cache, "shared")
	77	if err != nil {
	78	t.Fatal(err)
	79	}
	80	if _, ok := deleteSavedCheckout(cache, "shared"); ok {
	81	t.Fatal("a checkout held by another process was deleted")
	82	}
	83	if _, err := os.Stat(dir); err != nil {
	84	t.Fatalf("the checkout is gone although the lock was held: %v", err)
	85	}
	86	unlock()
	87
	88	if freed, ok := deleteSavedCheckout(cache, "shared"); !ok \|\| freed == 0 {
	89	t.Fatalf("delete after the lock was released = %d, %v; want freed bytes", freed, ok)
	90	}
	91	if _, err := os.Stat(dir); !os.IsNotExist(err) {
	92	t.Fatalf("the checkout survived a delete: %v", err)
	93	}
	94	}
	95
	96	// A repack is what keeps a long-lived runner from deleting a checkout it could
	97	// have kept: history no branch or tag reaches any more, and the temporary refs a
	98	// killed runner left behind, must go, while the tips stay so the next job is a
	99	// delta fetch.
	100	func TestRepackSavedCheckoutDropsUnreachableHistory(t *testing.T) {
	101	tmp := t.TempDir()
	102	src := filepath.Join(tmp, "src")
	103	remote := filepath.Join(tmp, "demo.git")
	104	cache := filepath.Join(tmp, "cache")
	105	cacheDir := filepath.Join(cache, "demo")
	106
	107	mustGit(t, "", "init", "-q", src)
	108	mustGit(t, src, "config", "user.email", "a@example.com")
	109	mustGit(t, src, "config", "user.name", "a")
	110	write := func(name, body string) {
	111	if err := os.WriteFile(filepath.Join(src, name), []byte(body), 0o644); err != nil {
	112	t.Fatal(err)
	113	}
	114	}
	115	write("file", "one\n")
	116	mustGit(t, src, "add", "file")
	117	mustGit(t, src, "commit", "-qm", "one")
	118	mustGit(t, "", "clone", "-q", "--bare", src, remote)
	119	tip := strings.TrimSpace(mustGit(t, src, "rev-parse", "HEAD"))
	120
	121	// A commit that only a branch carried, then lost: its only remaining claim is
	122	// the remote-tracking reflog, which is what a force-push leaves behind.
	123	write("file", "two\n")
	124	mustGit(t, src, "commit", "-qam", "two")
	125	gone := strings.TrimSpace(mustGit(t, src, "rev-parse", "HEAD"))
	126	mustGit(t, src, "push", "-q", remote, "HEAD:refs/heads/lost")
	127	mustGit(t, "", "clone", "-q", remote, cacheDir)
	128	mustGit(t, src, "push", "-q", remote, ":refs/heads/lost")
	129	mustGit(t, cacheDir, "fetch", "-q", "--prune", "origin")
	130
	131	// A commit that only a temporary job ref carries, as a killed runner leaves it.
	132	write("file", "three\n")
	133	mustGit(t, src, "commit", "-qam", "three")
	134	stale := strings.TrimSpace(mustGit(t, src, "rev-parse", "HEAD"))
	135	mustGit(t, src, "push", "-q", remote, "HEAD:refs/heads/stale")
	136	mustGit(t, cacheDir, "fetch", "-q", "origin", "refs/heads/stale:refs/remotes/origin/stale")
	137	mustGit(t, src, "push", "-q", remote, ":refs/heads/stale")
	138	mustGit(t, cacheDir, "update-ref", "-d", "refs/remotes/origin/stale")
	139	mustGit(t, cacheDir, "update-ref", "refs/vole/jobs/stale-1", stale)
	140	if err := os.WriteFile(filepath.Join(cacheDir, cacheMarker), []byte("1\n"), 0o644); err != nil {
	141	t.Fatal(err)
	142	}
	143
	144	for _, sha := range []string{tip, gone, stale} {
	145	if !hasCommit(cacheDir, sha) {
	146	t.Fatalf("setup: the cache does not hold %s", sha)
	147	}
	148	}
	149
	150	if _, err := repackSavedCheckout(cache, "demo"); err != nil {
	151	t.Fatalf("repackSavedCheckout: %v", err)
	152	}
	153
	154	if !hasCommit(cacheDir, tip) {
	155	t.Fatal("the repack dropped a commit a branch still reaches")
	156	}
	157	for name, sha := range map[string]string{"a branch that was deleted": gone, "a leftover job ref": stale} {
	158	if hasCommit(cacheDir, sha) {
	159	t.Fatalf("the repack kept a commit only %s carried", name)
	160	}
	161	}
	162	if refs := strings.TrimSpace(mustGit(t, cacheDir, "for-each-ref", "--format=%(refname)", "refs/vole/jobs")); refs != "" {
	163	t.Fatalf("leftover job refs survived the repack:\n%s", refs)
	164	}
	165	}