vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
7da9111e
Keenan Nemetz
15d ago
|
1 | package agent |
| 2 | ||
| 3 | // A job gets a disposable checkout, and the runner keeps a trusted clone beside it. | |
| 4 | // | |
| 5 | // The workflow may edit anything in the job tree, including its .git directory, so | |
| 6 | // the host must never run Git from that tree after the steps finish. The record and | |
| 7 | // site pushes use the trusted cache, which is never mounted into the container. The | |
| 8 | // job tree is a clone of that cache with its own .git, so builds that use Git history | |
| 9 | // still work, and a poisoned hook dies with the job. | |
|
f594f1a1
Keenan Nemetz
5d ago
|
10 | // |
| 11 | // The cache is what saves the network: it is fetched once and then kept, so a later | |
| 12 | // job for the same revision is a local copy and nothing more. Bounding it, and | |
| 13 | // reclaiming what no job can reach, is gitcache.go. | |
|
7da9111e
Keenan Nemetz
15d ago
|
14 | |
| 15 | import ( | |
| 16 | "fmt" | |
| 17 | "os" | |
| 18 | "os/exec" | |
| 19 | "path/filepath" | |
| 20 | "strconv" | |
| 21 | "strings" | |
| 22 | "time" | |
| 23 | ||
|
d37d301f
Keenan Nemetz
15h ago
|
24 | "rodent.software/vole/client" |
|
7da9111e
Keenan Nemetz
15d ago
|
25 | ) |
| 26 | ||
| 27 | // cacheMarker says the cache was created by a runner that never gave the job | |
| 28 | // access to it. A cache without the marker predates disposable job trees and may | |
| 29 | // have hooks or config written by a workflow, so it is rebuilt instead of trusted. | |
| 30 | const cacheMarker = ".vole-trusted-cache" | |
| 31 | ||
| 32 | type checkout struct { | |
| 33 | // Work is the disposable tree the steps run in and the container mounts. It is | |
| 34 | // removed when the job ends. | |
| 35 | Work string | |
| 36 | // GitDir is the trusted cache's git directory. It is used only by the runner | |
| 37 | // process, never mounted into the job, and survives between jobs. | |
| 38 | GitDir string | |
|
f594f1a1
Keenan Nemetz
5d ago
|
39 | // Cache and Repo name the trusted cache the job may write to after its steps: |
| 40 | // the site and the release pointers are pushed from it, so those writes take | |
| 41 | // its lock. See lockCache. | |
| 42 | Cache string | |
| 43 | Repo string | |
|
7da9111e
Keenan Nemetz
15d ago
|
44 | } |
| 45 | ||
| 46 | func (c checkout) cleanup() { | |
| 47 | _ = os.RemoveAll(c.Work) | |
| 48 | } | |
| 49 | ||
|
f594f1a1
Keenan Nemetz
5d ago
|
50 | // lock takes the trusted cache's lock for the writes a job makes after its steps. |
| 51 | // The cache is shared with every other runner process using the same --dir, and | |
| 52 | // those writes add objects and move refs; a repack or a deletion in another process | |
| 53 | // must not run underneath them. | |
| 54 | func (c checkout) lock() (func(), error) { | |
| 55 | return lockCache(c.Cache, c.Repo) | |
| 56 | } | |
| 57 | ||
|
7da9111e
Keenan Nemetz
15d ago
|
58 | // cleanupJobTrees removes job trees left by a crashed runner. It is age-based so a |
| 59 | // second runner sharing the same --dir cannot delete work another process is using. | |
| 60 | func cleanupJobTrees(cache string) { | |
| 61 | root := filepath.Join(cache, "jobs") | |
| 62 | entries, err := os.ReadDir(root) | |
| 63 | if err != nil { | |
| 64 | return | |
| 65 | } | |
| 66 | cutoff := time.Now().Add(-7 * 24 * time.Hour) | |
| 67 | for _, e := range entries { | |
| 68 | if !e.IsDir() { | |
| 69 | continue | |
| 70 | } | |
| 71 | info, err := e.Info() | |
| 72 | if err == nil && info.ModTime().Before(cutoff) { | |
| 73 | _ = os.RemoveAll(filepath.Join(root, e.Name())) | |
| 74 | } | |
| 75 | } | |
| 76 | } | |
| 77 | ||
| 78 | // prepareWorktree fetches the exact commit into the trusted cache, then makes a | |
| 79 | // disposable clone for the job. The cache is updated but never checked out for the | |
|
b312e166
Keenan Nemetz
10d ago
|
80 | // job; its .git is what a release push uses. |
|
f594f1a1
Keenan Nemetz
5d ago
|
81 | // |
| 82 | // The whole preparation holds the repository's lock. It fetches into the clone and | |
| 83 | // then copies out of it, so a repack or a deletion in another runner process — which | |
| 84 | // take the same lock — cannot pull the object store out from under the copy. The | |
| 85 | // lock is per repository, so jobs for different repositories never wait for each | |
| 86 | // other. | |
|
7da9111e
Keenan Nemetz
15d ago
|
87 | func prepareWorktree(remote, cache, repo, sha, runID string, attempt int) (checkout, error) { |
| 88 | cloneURL, err := client.RepoURL(remote, repo) | |
| 89 | if err != nil { | |
| 90 | return checkout{}, err | |
| 91 | } | |
| 92 | cacheDir := filepath.Join(cache, repo) | |
| 93 | ||
|
f594f1a1
Keenan Nemetz
5d ago
|
94 | unlock, err := lockCache(cache, repo) |
| 95 | if err != nil { | |
| 96 | return checkout{}, err | |
| 97 | } | |
| 98 | defer unlock() | |
| 99 | ||
|
7da9111e
Keenan Nemetz
15d ago
|
100 | _, gitErr := os.Stat(filepath.Join(cacheDir, ".git")) |
| 101 | _, markerErr := os.Stat(filepath.Join(cacheDir, cacheMarker)) | |
| 102 | if gitErr != nil || markerErr != nil { | |
| 103 | if err := os.RemoveAll(cacheDir); err != nil { | |
| 104 | return checkout{}, err | |
| 105 | } | |
| 106 | if err := os.MkdirAll(filepath.Dir(cacheDir), 0o755); err != nil { | |
| 107 | return checkout{}, err | |
| 108 | } | |
|
f594f1a1
Keenan Nemetz
5d ago
|
109 | // Nothing reads the cache's working tree: the record and the site are |
| 110 | // written with plumbing, and the job gets a clone of its own. Checking it out | |
| 111 | // would only spend the disk, twice. | |
| 112 | cmd := exec.Command("git", "clone", "--quiet", "--no-checkout", cloneURL, cacheDir) | |
|
7da9111e
Keenan Nemetz
15d ago
|
113 | if out, err := cmd.CombinedOutput(); err != nil { |
| 114 | return checkout{}, fmt.Errorf("git clone: %v: %s", err, out) | |
| 115 | } | |
| 116 | } else { | |
|
f594f1a1
Keenan Nemetz
5d ago
|
117 | // The branch fetch is the delta and nothing else: the history the cache holds |
| 118 | // is already here. It is not skipped when the revision is cached, because a | |
| 119 | // job's checkout carries these refs as origin/*, and a step that compares | |
| 120 | // against origin/main must see the branch as it is now, not as it was the last | |
| 121 | // time this cache fetched. | |
|
7da9111e
Keenan Nemetz
15d ago
|
122 | cmd := exec.Command("git", "-C", cacheDir, "fetch", "--quiet", "--prune", "--tags", |
| 123 | "origin", "+refs/heads/*:refs/remotes/origin/*") | |
| 124 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 125 | return checkout{}, fmt.Errorf("git fetch: %v: %s", err, out) | |
| 126 | } | |
| 127 | } | |
|
f594f1a1
Keenan Nemetz
5d ago
|
128 | if err := touchCache(cacheDir); err != nil { |
| 129 | return checkout{}, err | |
|
7da9111e
Keenan Nemetz
15d ago
|
130 | } |
| 131 | ||
| 132 | if !hasCommit(cacheDir, sha) { | |
| 133 | cmd := exec.Command("git", "-C", cacheDir, "fetch", "--quiet", "origin", sha) | |
| 134 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 135 | return checkout{}, fmt.Errorf("git fetch %s: %v: %s", sha, err, out) | |
| 136 | } | |
| 137 | } | |
| 138 | if !hasCommit(cacheDir, sha) { | |
| 139 | return checkout{}, fmt.Errorf("the cache does not have commit %s after fetching it", sha) | |
| 140 | } | |
| 141 | // A commit fetched by id may not be on a branch, and a local clone only copies | |
| 142 | // reachable objects. A temporary ref makes it reachable for the job clone; the | |
| 143 | // ref lives in the trusted cache and is removed before the job runs. | |
| 144 | jobRef := "refs/vole/jobs/" + runID + "-" + strconv.Itoa(attempt) | |
| 145 | if out, err := exec.Command("git", "-C", cacheDir, "update-ref", jobRef, sha).CombinedOutput(); err != nil { | |
| 146 | return checkout{}, fmt.Errorf("git update-ref %s: %v: %s", jobRef, err, out) | |
| 147 | } | |
| 148 | defer func() { | |
| 149 | _ = exec.Command("git", "-C", cacheDir, "update-ref", "-d", jobRef).Run() | |
| 150 | }() | |
| 151 | ||
| 152 | gitDir, err := client.GitDirOf(cacheDir) | |
| 153 | if err != nil { | |
| 154 | return checkout{}, err | |
| 155 | } | |
| 156 | ||
| 157 | jobDir := filepath.Join(cache, "jobs", repo, runID+"-"+strconv.Itoa(attempt)) | |
| 158 | if err := os.MkdirAll(filepath.Dir(jobDir), 0o755); err != nil { | |
| 159 | return checkout{}, err | |
| 160 | } | |
| 161 | if err := os.RemoveAll(jobDir); err != nil { | |
| 162 | return checkout{}, err | |
| 163 | } | |
| 164 | cmd := exec.Command("git", "clone", "--quiet", "--no-hardlinks", cacheDir, jobDir) | |
| 165 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 166 | return checkout{}, fmt.Errorf("git clone the job tree: %v: %s", err, out) | |
| 167 | } | |
| 168 | // The local clone may not carry a commit that is not on a branch. Fetch the | |
| 169 | // temporary ref from the cache before pointing origin at the server, which the | |
| 170 | // job cannot reach without the runner's key. | |
| 171 | if !hasCommit(jobDir, sha) { | |
| 172 | cmd := exec.Command("git", "-C", jobDir, "fetch", "--quiet", "origin", jobRef) | |
| 173 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 174 | _ = os.RemoveAll(jobDir) | |
| 175 | return checkout{}, fmt.Errorf("git fetch the job tree: %v: %s", err, out) | |
| 176 | } | |
| 177 | } | |
| 178 | _ = exec.Command("git", "-C", jobDir, "remote", "set-url", "origin", cloneURL).Run() | |
| 179 | ||
| 180 | for _, args := range [][]string{ | |
| 181 | {"checkout", "--quiet", "--force", "--detach", sha}, | |
| 182 | {"clean", "-qffdx"}, | |
| 183 | } { | |
| 184 | cmd := exec.Command("git", append([]string{"-C", jobDir}, args...)...) | |
| 185 | if out, err := cmd.CombinedOutput(); err != nil { | |
| 186 | _ = os.RemoveAll(jobDir) | |
| 187 | return checkout{}, fmt.Errorf("git %s: %v: %s", strings.Join(args, " "), err, out) | |
| 188 | } | |
| 189 | } | |
|
f594f1a1
Keenan Nemetz
5d ago
|
190 | return checkout{Work: jobDir, GitDir: gitDir, Cache: cache, Repo: repo}, nil |
|
7da9111e
Keenan Nemetz
15d ago
|
191 | } |
| 192 | ||
| 193 | func hasCommit(dir, sha string) bool { | |
| 194 | return exec.Command("git", "-C", dir, "cat-file", "-e", sha+"^{commit}").Run() == nil | |
| 195 | } |