vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound and reclaim the runner's saved checkouts
f594f1a16f6f8a8a3ee2867a44249b72a2c76c96 · Keenan Nemetz · 5d ago · parent de84c32b
Bound and reclaim the runner's saved checkouts The runner already downloaded each repository once into --dir and cloned every job from that copy, but nothing kept the copies bounded: they grew with every tag, every branch, and every commit a job pinned by id, and a force-push left the old history reachable through the remote-tracking reflogs. A long-lived runner filled its disk and then could not check anything out at all. Repack a copy that is most of the cap, dropping the objects no branch or tag can reach and the temporary refs a killed runner leaves behind; then delete the least recently used copy when the cap is still reached. --git-cache-size sets the cap (10G by default, 0 keeps every copy). The repository this runner is about to build is never the one deleted, and a copy another process holds is passed over. Reading and writing a saved checkout now takes a per-repository lock, so a repack or a deletion in one runner process cannot run underneath another's clone, or underneath the site and release pushes that are written from the same cache. The first clone skips the working-tree checkout, which nothing ever read.
ci/agent/worktree.go +41−4
| @@ -7,6 +7,10 @@ package agent | |||
| 7 | 7 | // site pushes use the trusted cache, which is never mounted into the container. The | |
| 8 | 8 | // job tree is a clone of that cache with its own .git, so builds that use Git history | |
| 9 | 9 | // still work, and a poisoned hook dies with the job. | |
| 10 | // | ||
| 11 | // The cache is what saves the network: it is fetched once and then kept, so a later | ||
| 12 | // job for the same revision is a local copy and nothing more. Bounding it, and | ||
| 13 | // reclaiming what no job can reach, is gitcache.go. | ||
| 10 | 14 | ||
| 11 | 15 | import ( | |
| 12 | 16 | "fmt" | |
| @@ -32,12 +36,25 @@ type checkout struct { | |||
| 32 | 36 | // GitDir is the trusted cache's git directory. It is used only by the runner | |
| 33 | 37 | // process, never mounted into the job, and survives between jobs. | |
| 34 | 38 | GitDir string | |
| 39 | // Cache and Repo name the trusted cache the job may write to after its steps: | ||
| 40 | // the site and the release pointers are pushed from it, so those writes take | ||
| 41 | // its lock. See lockCache. | ||
| 42 | Cache string | ||
| 43 | Repo string | ||
| 35 | 44 | } | |
| 36 | 45 | ||
| 37 | 46 | func (c checkout) cleanup() { | |
| 38 | 47 | _ = os.RemoveAll(c.Work) | |
| 39 | 48 | } | |
| 40 | 49 | ||
| 50 | // lock takes the trusted cache's lock for the writes a job makes after its steps. | ||
| 51 | // The cache is shared with every other runner process using the same --dir, and | ||
| 52 | // those writes add objects and move refs; a repack or a deletion in another process | ||
| 53 | // must not run underneath them. | ||
| 54 | func (c checkout) lock() (func(), error) { | ||
| 55 | return lockCache(c.Cache, c.Repo) | ||
| 56 | } | ||
| 57 | |||
| 41 | 58 | // cleanupJobTrees removes job trees left by a crashed runner. It is age-based so a | |
| 42 | 59 | // second runner sharing the same --dir cannot delete work another process is using. | |
| 43 | 60 | func cleanupJobTrees(cache string) { | |
| @@ -61,6 +78,12 @@ func cleanupJobTrees(cache string) { | |||
| 61 | 78 | // prepareWorktree fetches the exact commit into the trusted cache, then makes a | |
| 62 | 79 | // disposable clone for the job. The cache is updated but never checked out for the | |
| 63 | 80 | // job; its .git is what a release push uses. | |
| 81 | // | ||
| 82 | // The whole preparation holds the repository's lock. It fetches into the clone and | ||
| 83 | // then copies out of it, so a repack or a deletion in another runner process — which | ||
| 84 | // take the same lock — cannot pull the object store out from under the copy. The | ||
| 85 | // lock is per repository, so jobs for different repositories never wait for each | ||
| 86 | // other. | ||
| 64 | 87 | func prepareWorktree(remote, cache, repo, sha, runID string, attempt int) (checkout, error) { | |
| 65 | 88 | cloneURL, err := client.RepoURL(remote, repo) | |
| 66 | 89 | if err != nil { | |
| @@ -68,6 +91,12 @@ func prepareWorktree(remote, cache, repo, sha, runID string, attempt int) (check | |||
| 68 | 91 | } | |
| 69 | 92 | cacheDir := filepath.Join(cache, repo) | |
| 70 | 93 | ||
| 94 | unlock, err := lockCache(cache, repo) | ||
| 95 | if err != nil { | ||
| 96 | return checkout{}, err | ||
| 97 | } | ||
| 98 | defer unlock() | ||
| 99 | |||
| 71 | 100 | _, gitErr := os.Stat(filepath.Join(cacheDir, ".git")) | |
| 72 | 101 | _, markerErr := os.Stat(filepath.Join(cacheDir, cacheMarker)) | |
| 73 | 102 | if gitErr != nil || markerErr != nil { | |
| @@ -77,19 +106,27 @@ func prepareWorktree(remote, cache, repo, sha, runID string, attempt int) (check | |||
| 77 | 106 | if err := os.MkdirAll(filepath.Dir(cacheDir), 0o755); err != nil { | |
| 78 | 107 | return checkout{}, err | |
| 79 | 108 | } | |
| 80 | cmd := exec.Command("git", "clone", "--quiet", cloneURL, cacheDir) | ||
| 109 | // Nothing reads the cache's working tree: the record and the site are | ||
| 110 | // written with plumbing, and the job gets a clone of its own. Checking it out | ||
| 111 | // would only spend the disk, twice. | ||
| 112 | cmd := exec.Command("git", "clone", "--quiet", "--no-checkout", cloneURL, cacheDir) | ||
| 81 | 113 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 82 | 114 | return checkout{}, fmt.Errorf("git clone: %v: %s", err, out) | |
| 83 | 115 | } | |
| 84 | 116 | } else { | |
| 117 | // The branch fetch is the delta and nothing else: the history the cache holds | ||
| 118 | // is already here. It is not skipped when the revision is cached, because a | ||
| 119 | // job's checkout carries these refs as origin/*, and a step that compares | ||
| 120 | // against origin/main must see the branch as it is now, not as it was the last | ||
| 121 | // time this cache fetched. | ||
| 85 | 122 | cmd := exec.Command("git", "-C", cacheDir, "fetch", "--quiet", "--prune", "--tags", | |
| 86 | 123 | "origin", "+refs/heads/*:refs/remotes/origin/*") | |
| 87 | 124 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 88 | 125 | return checkout{}, fmt.Errorf("git fetch: %v: %s", err, out) | |
| 89 | 126 | } | |
| 90 | 127 | } | |
| 91 | if err := os.WriteFile(filepath.Join(cacheDir, cacheMarker), []byte("1\n"), 0o644); err != nil { | ||
| 92 | return checkout{}, fmt.Errorf("mark the trusted cache: %w", err) | ||
| 128 | if err := touchCache(cacheDir); err != nil { | ||
| 129 | return checkout{}, err | ||
| 93 | 130 | } | |
| 94 | 131 | ||
| 95 | 132 | if !hasCommit(cacheDir, sha) { | |
| @@ -150,7 +187,7 @@ func prepareWorktree(remote, cache, repo, sha, runID string, attempt int) (check | |||
| 150 | 187 | return checkout{}, fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, out) | |
| 151 | 188 | } | |
| 152 | 189 | } | |
| 153 | return checkout{Work: jobDir, GitDir: gitDir}, nil | ||
| 190 | return checkout{Work: jobDir, GitDir: gitDir, Cache: cache, Repo: repo}, nil | ||
| 154 | 191 | } | |
| 155 | 192 | ||
| 156 | 193 | func hasCommit(dir, sha string) bool { | |