vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 5: LFS objects, over SSH for writes and the proxy for reads
64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf
Phase 5: LFS objects, over SSH for writes and the proxy for reads Large files leave the git object database and land in one content-addressed store per instance. Reads are anonymous like clone; writes ride the SSH key that already authenticates the push, so no token exists anywhere. - internal/lfs: sha256-addressed store that verifies content and size before an object becomes visible; parser for the pointer format a client writes - HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid> with the object id as ETag; upload over HTTP is refused and names SSH - SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so `git lfs push` needs nothing but the key at the handshake - pre-receive: a pointer whose object is missing, or whose size disagrees with the stored object, fails the push that introduced it; a binary over lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run before the protected-branch gate, because an admin's approval bypass is about review, not about whether the commit can be checked out - runs: a finished log becomes a content-addressed object in the same write that records the status, served at /r/<repo>/runs/<id>/log - vole gc [--dry-run] [--grace]: marks pointer targets from every ref in every repository plus archived logs, sweeps the rest. Mark-and-sweep rather than reference counts: a counter that is one too low deletes an artifact, a walk over the refs cannot be wrong about what is reachable
internal/server/lfs_test.go +240
| @@ -0,0 +1,240 @@ | |||
| 1 | package server_test | ||
| 2 | |||
| 3 | // End-to-end LFS tests. The client-driven one runs the real git-lfs binary when | ||
| 4 | // it is installed and skips otherwise; the protocol tests speak HTTP directly so | ||
| 5 | // the server's half is always checked. | ||
| 6 | |||
| 7 | import ( | ||
| 8 | "bytes" | ||
| 9 | "crypto/sha256" | ||
| 10 | "encoding/hex" | ||
| 11 | "encoding/json" | ||
| 12 | "io" | ||
| 13 | "net/http" | ||
| 14 | "os" | ||
| 15 | "os/exec" | ||
| 16 | "path/filepath" | ||
| 17 | "strconv" | ||
| 18 | "strings" | ||
| 19 | "testing" | ||
| 20 | |||
| 21 | "github.com/rodent-software/vole/internal/lfs" | ||
| 22 | ) | ||
| 23 | |||
| 24 | func needGitLFS(t *testing.T) { | ||
| 25 | t.Helper() | ||
| 26 | if _, err := exec.LookPath("git-lfs"); err != nil { | ||
| 27 | t.Skip("git-lfs is not installed; install it to exercise the LFS client protocol") | ||
| 28 | } | ||
| 29 | } | ||
| 30 | |||
| 31 | // lfsRepo prepares a working clone that tracks *.bin through LFS. | ||
| 32 | func lfsRepo(t *testing.T, f *fixture) string { | ||
| 33 | t.Helper() | ||
| 34 | dir := workRepo(t, map[string]string{"readme.md": "lfs test\n"}) | ||
| 35 | if _, errs, err := f.git(dir, "lfs", "install", "--local"); err != nil { | ||
| 36 | t.Fatalf("git lfs install: %v\n%s", err, errs) | ||
| 37 | } | ||
| 38 | if _, errs, err := f.git(dir, "lfs", "track", "*.bin"); err != nil { | ||
| 39 | t.Fatalf("git lfs track: %v\n%s", err, errs) | ||
| 40 | } | ||
| 41 | return dir | ||
| 42 | } | ||
| 43 | |||
| 44 | // TestLFSClientPushThenClone is the phase's first exit criterion: a clone pulls | ||
| 45 | // real objects through the HTTP proxy, and the push that put them there happened | ||
| 46 | // over SSH with no token of any kind. | ||
| 47 | func TestLFSClientPushThenClone(t *testing.T) { | ||
| 48 | needGitLFS(t) | ||
| 49 | f := newFixture(t) | ||
| 50 | f.publish(f.repo) | ||
| 51 | |||
| 52 | dir := lfsRepo(t, f) | ||
| 53 | payload := make([]byte, 256<<10) | ||
| 54 | for i := range payload { | ||
| 55 | payload[i] = byte(i * 7) | ||
| 56 | } | ||
| 57 | if err := os.WriteFile(filepath.Join(dir, "artifact.bin"), payload, 0o644); err != nil { | ||
| 58 | t.Fatal(err) | ||
| 59 | } | ||
| 60 | sum := sha256.Sum256(payload) | ||
| 61 | wantOID := hex.EncodeToString(sum[:]) | ||
| 62 | |||
| 63 | if _, errs, err := f.git(dir, "add", "-A"); err != nil { | ||
| 64 | t.Fatalf("add: %v\n%s", err, errs) | ||
| 65 | } | ||
| 66 | if _, errs, err := f.git(dir, "commit", "-qm", "artifact"); err != nil { | ||
| 67 | t.Fatalf("commit: %v\n%s", err, errs) | ||
| 68 | } | ||
| 69 | if out, errs, err := f.git(dir, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 70 | t.Fatalf("push: %v\n%s%s", err, out, errs) | ||
| 71 | } | ||
| 72 | |||
| 73 | // The pointer, not the bytes, is what git received. | ||
| 74 | if !f.srv.LFS.Has(wantOID) { | ||
| 75 | t.Fatalf("LFS object %s is not in the store after the push", wantOID) | ||
| 76 | } | ||
| 77 | |||
| 78 | // A fresh anonymous clone over HTTP, then a pull: the only source of the | ||
| 79 | // bytes is this server. | ||
| 80 | dst := filepath.Join(t.TempDir(), "clone") | ||
| 81 | if _, errs, err := runGit(t, "", "clone", "-q", f.httpURL(f.repo), dst); err != nil { | ||
| 82 | t.Fatalf("clone: %v\n%s", err, errs) | ||
| 83 | } | ||
| 84 | if _, errs, err := runGit(t, dst, "lfs", "install", "--local"); err != nil { | ||
| 85 | t.Fatalf("git lfs install in the clone: %v\n%s", err, errs) | ||
| 86 | } | ||
| 87 | if _, errs, err := runGit(t, dst, "lfs", "pull"); err != nil { | ||
| 88 | t.Fatalf("git lfs pull: %v\n%s", err, errs) | ||
| 89 | } | ||
| 90 | got, err := os.ReadFile(filepath.Join(dst, "artifact.bin")) | ||
| 91 | if err != nil { | ||
| 92 | t.Fatal(err) | ||
| 93 | } | ||
| 94 | if !bytes.Equal(got, payload) { | ||
| 95 | t.Fatalf("cloned artifact is %d bytes and does not match the pushed %d", len(got), len(payload)) | ||
| 96 | } | ||
| 97 | } | ||
| 98 | |||
| 99 | // TestLFSPointerWithoutObjectIsRejected is the second exit criterion. --no-verify | ||
| 100 | // skips the client's own pre-push check, which is the only way to test the | ||
| 101 | // server's rule: the pointer is committed and pushed, its object never uploaded. | ||
| 102 | func TestLFSPointerWithoutObjectIsRejected(t *testing.T) { | ||
| 103 | f := newFixture(t) | ||
| 104 | f.publish(f.repo) | ||
| 105 | |||
| 106 | data := []byte("content that was never uploaded") | ||
| 107 | missing := sha256.Sum256(data) | ||
| 108 | oid := hex.EncodeToString(missing[:]) | ||
| 109 | pointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + | ||
| 110 | "\nsize " + strconv.Itoa(len(data)) + "\n" | ||
| 111 | |||
| 112 | dir := workRepo(t, map[string]string{"artifacts/missing.bin": pointer}) | ||
| 113 | out, errs, err := f.git(dir, "push", "--no-verify", f.sshURL(f.repo), "main:refs/heads/main") | ||
| 114 | if err == nil { | ||
| 115 | t.Fatalf("push of a pointer with no object was accepted:\n%s%s", out, errs) | ||
| 116 | } | ||
| 117 | for _, want := range []string{oid[:12], "not on this server", "git lfs push"} { | ||
| 118 | if !strings.Contains(errs, want) { | ||
| 119 | t.Fatalf("rejection does not mention %q:\n%s", want, errs) | ||
| 120 | } | ||
| 121 | } | ||
| 122 | |||
| 123 | // Nothing moved: the ref the push would have created is still absent. | ||
| 124 | remote, _, err := f.git(dir, "ls-remote", f.sshURL(f.repo)) | ||
| 125 | if err != nil { | ||
| 126 | t.Fatalf("ls-remote: %v", err) | ||
| 127 | } | ||
| 128 | if strings.Contains(remote, "refs/heads/main") { | ||
| 129 | t.Fatalf("ref moved despite the rejection:\n%s", remote) | ||
| 130 | } | ||
| 131 | |||
| 132 | // After the object is uploaded, the same push is accepted. | ||
| 133 | if _, _, err := f.srv.LFS.Put(oid, int64(len(data)), bytes.NewReader(data)); err != nil { | ||
| 134 | t.Fatal(err) | ||
| 135 | } | ||
| 136 | if out, errs, err := f.git(dir, "push", "--no-verify", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 137 | t.Fatalf("push rejected even though the object is present: %v\nstdout:\n%s\nstderr:\n%s", err, out, errs) | ||
| 138 | } | ||
| 139 | } | ||
| 140 | |||
| 141 | // TestLFSBatchProtocol covers the JSON side without a client, including the two | ||
| 142 | // refusals that keep the read-only-web / SSH-write split intact. | ||
| 143 | func TestLFSBatchProtocol(t *testing.T) { | ||
| 144 | f := newFixture(t) | ||
| 145 | f.publish(f.repo) | ||
| 146 | |||
| 147 | content := []byte("downloadable artifact") | ||
| 148 | oid, err := f.srv.LFS.PutBytes(content) | ||
| 149 | if err != nil { | ||
| 150 | t.Fatal(err) | ||
| 151 | } | ||
| 152 | missing := strings.Repeat("a", 64) | ||
| 153 | |||
| 154 | post := func(body lfs.BatchRequest) (int, string) { | ||
| 155 | t.Helper() | ||
| 156 | raw, _ := json.Marshal(body) | ||
| 157 | req, err := http.NewRequest(http.MethodPost, f.httpURL(f.repo)+"/info/lfs/objects/batch", bytes.NewReader(raw)) | ||
| 158 | if err != nil { | ||
| 159 | t.Fatal(err) | ||
| 160 | } | ||
| 161 | req.Header.Set("Content-Type", lfs.MediaType) | ||
| 162 | resp, err := http.DefaultClient.Do(req) | ||
| 163 | if err != nil { | ||
| 164 | t.Fatal(err) | ||
| 165 | } | ||
| 166 | defer resp.Body.Close() | ||
| 167 | got, _ := io.ReadAll(resp.Body) | ||
| 168 | return resp.StatusCode, string(got) | ||
| 169 | } | ||
| 170 | |||
| 171 | code, body := post(lfs.BatchRequest{ | ||
| 172 | Operation: "download", | ||
| 173 | Objects: []lfs.Object{{OID: oid, Size: int64(len(content))}, {OID: missing, Size: 4}}, | ||
| 174 | }) | ||
| 175 | if code != 200 { | ||
| 176 | t.Fatalf("download batch = %d: %s", code, body) | ||
| 177 | } | ||
| 178 | var res lfs.BatchResponse | ||
| 179 | if err := json.Unmarshal([]byte(body), &res); err != nil { | ||
| 180 | t.Fatalf("batch response is not JSON: %v\n%s", err, body) | ||
| 181 | } | ||
| 182 | if len(res.Objects) != 2 { | ||
| 183 | t.Fatalf("batch answered %d objects, want 2: %s", len(res.Objects), body) | ||
| 184 | } | ||
| 185 | if act, ok := res.Objects[0].Actions["download"]; !ok { | ||
| 186 | t.Fatalf("no download action for a stored object: %s", body) | ||
| 187 | } else { | ||
| 188 | // The href must be fetchable as given: that is the whole contract. | ||
| 189 | resp, err := http.Get(act.Href) | ||
| 190 | if err != nil { | ||
| 191 | t.Fatal(err) | ||
| 192 | } | ||
| 193 | got, _ := io.ReadAll(resp.Body) | ||
| 194 | resp.Body.Close() | ||
| 195 | if resp.StatusCode != 200 || !bytes.Equal(got, content) { | ||
| 196 | t.Fatalf("GET %s = %d, %d bytes", act.Href, resp.StatusCode, len(got)) | ||
| 197 | } | ||
| 198 | if etag := resp.Header.Get("ETag"); etag != `"`+oid+`"` { | ||
| 199 | t.Fatalf("ETag = %q, want the object id", etag) | ||
| 200 | } | ||
| 201 | } | ||
| 202 | if res.Objects[1].Error == nil || res.Objects[1].Error.Code != 404 { | ||
| 203 | t.Fatalf("missing object should carry a 404 error: %+v", res.Objects[1]) | ||
| 204 | } | ||
| 205 | |||
| 206 | // Uploads are not accepted over HTTP, and say where they are accepted. | ||
| 207 | code, body = post(lfs.BatchRequest{Operation: "upload", Objects: []lfs.Object{{OID: oid, Size: 1}}}) | ||
| 208 | if code != http.StatusForbidden || !strings.Contains(body, "SSH") { | ||
| 209 | t.Fatalf("upload batch = %d: %s", code, body) | ||
| 210 | } | ||
| 211 | |||
| 212 | // The object endpoint refuses unknown ids rather than looking them up. | ||
| 213 | if code, _ := f.get("/git/" + f.repo + ".git/info/lfs/objects/" + strings.Repeat("b", 64)); code != 404 { | ||
| 214 | t.Fatalf("GET of an absent object = %d, want 404", code) | ||
| 215 | } | ||
| 216 | } | ||
| 217 | |||
| 218 | // TestLFSIsHiddenInPrivateRepos mirrors the clone rule: a private repository | ||
| 219 | // answers 404 for LFS too, so the read surface never confirms that it exists. | ||
| 220 | func TestLFSIsHiddenInPrivateRepos(t *testing.T) { | ||
| 221 | f := newFixture(t) // deliberately not published | ||
| 222 | oid, err := f.srv.LFS.PutBytes([]byte("private artifact")) | ||
| 223 | if err != nil { | ||
| 224 | t.Fatal(err) | ||
| 225 | } | ||
| 226 | if code, body := f.get("/git/" + f.repo + ".git/info/lfs/objects/" + oid); code != 404 { | ||
| 227 | t.Fatalf("object GET in a private repo = %d: %s", code, body) | ||
| 228 | } | ||
| 229 | raw, _ := json.Marshal(lfs.BatchRequest{Operation: "download", Objects: []lfs.Object{{OID: oid}}}) | ||
| 230 | req, _ := http.NewRequest(http.MethodPost, f.httpURL(f.repo)+"/info/lfs/objects/batch", bytes.NewReader(raw)) | ||
| 231 | req.Header.Set("Content-Type", lfs.MediaType) | ||
| 232 | resp, err := http.DefaultClient.Do(req) | ||
| 233 | if err != nil { | ||
| 234 | t.Fatal(err) | ||
| 235 | } | ||
| 236 | defer resp.Body.Close() | ||
| 237 | if resp.StatusCode != 404 { | ||
| 238 | t.Fatalf("batch in a private repo = %d, want 404", resp.StatusCode) | ||
| 239 | } | ||
| 240 | } | ||