vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
64ac5071
Keenan Nemetz
17d ago
|
1 | package server_test |
| 2 | ||
| 3 | // End-to-end LFS tests. The client-driven one runs the real git-lfs binary when | |
| 4 | // it is installed and skips otherwise; the protocol tests speak HTTP directly so | |
| 5 | // the server's half is always checked. | |
| 6 | ||
| 7 | import ( | |
| 8 | "bytes" | |
| 9 | "crypto/sha256" | |
| 10 | "encoding/hex" | |
| 11 | "encoding/json" | |
|
65b88b3b
Keenan Nemetz
16d ago
|
12 | "fmt" |
|
64ac5071
Keenan Nemetz
17d ago
|
13 | "io" |
| 14 | "net/http" | |
| 15 | "os" | |
| 16 | "os/exec" | |
| 17 | "path/filepath" | |
|
d37d301f
Keenan Nemetz
15h ago
|
18 | "rodent.software/vole/forge" |
|
64ac5071
Keenan Nemetz
17d ago
|
19 | "strconv" |
| 20 | "strings" | |
| 21 | "testing" | |
| 22 | ||
|
d37d301f
Keenan Nemetz
15h ago
|
23 | "rodent.software/vole/core/lfs" |
|
64ac5071
Keenan Nemetz
17d ago
|
24 | ) |
| 25 | ||
| 26 | func needGitLFS(t *testing.T) { | |
| 27 | t.Helper() | |
| 28 | if _, err := exec.LookPath("git-lfs"); err != nil { | |
| 29 | t.Skip("git-lfs is not installed; install it to exercise the LFS client protocol") | |
| 30 | } | |
| 31 | } | |
| 32 | ||
| 33 | // lfsRepo prepares a working clone that tracks *.bin through LFS. | |
| 34 | func lfsRepo(t *testing.T, f *fixture) string { | |
| 35 | t.Helper() | |
| 36 | dir := workRepo(t, map[string]string{"readme.md": "lfs test\n"}) | |
| 37 | if _, errs, err := f.git(dir, "lfs", "install", "--local"); err != nil { | |
| 38 | t.Fatalf("git lfs install: %v\n%s", err, errs) | |
| 39 | } | |
| 40 | if _, errs, err := f.git(dir, "lfs", "track", "*.bin"); err != nil { | |
| 41 | t.Fatalf("git lfs track: %v\n%s", err, errs) | |
| 42 | } | |
| 43 | return dir | |
| 44 | } | |
| 45 | ||
|
520470e1
Keenan Nemetz
16d ago
|
46 | // TestLFSClientPushThenClone checks the main LFS flow: a clone pulls |
|
64ac5071
Keenan Nemetz
17d ago
|
47 | // real objects through the HTTP proxy, and the push that put them there happened |
| 48 | // over SSH with no token of any kind. | |
| 49 | func TestLFSClientPushThenClone(t *testing.T) { | |
|
44530e74
Keenan Nemetz
16d ago
|
50 | t.Parallel() |
|
64ac5071
Keenan Nemetz
17d ago
|
51 | needGitLFS(t) |
| 52 | f := newFixture(t) | |
| 53 | f.publish(f.repo) | |
| 54 | ||
| 55 | dir := lfsRepo(t, f) | |
| 56 | payload := make([]byte, 256<<10) | |
| 57 | for i := range payload { | |
| 58 | payload[i] = byte(i * 7) | |
| 59 | } | |
| 60 | if err := os.WriteFile(filepath.Join(dir, "artifact.bin"), payload, 0o644); err != nil { | |
| 61 | t.Fatal(err) | |
| 62 | } | |
| 63 | sum := sha256.Sum256(payload) | |
| 64 | wantOID := hex.EncodeToString(sum[:]) | |
| 65 | ||
| 66 | if _, errs, err := f.git(dir, "add", "-A"); err != nil { | |
| 67 | t.Fatalf("add: %v\n%s", err, errs) | |
| 68 | } | |
| 69 | if _, errs, err := f.git(dir, "commit", "-qm", "artifact"); err != nil { | |
| 70 | t.Fatalf("commit: %v\n%s", err, errs) | |
| 71 | } | |
| 72 | if out, errs, err := f.git(dir, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | |
| 73 | t.Fatalf("push: %v\n%s%s", err, out, errs) | |
| 74 | } | |
| 75 | ||
| 76 | // The pointer, not the bytes, is what git received. | |
|
65b88b3b
Keenan Nemetz
16d ago
|
77 | if !f.srv.LFS.Has(f.repo, wantOID) { |
|
64ac5071
Keenan Nemetz
17d ago
|
78 | t.Fatalf("LFS object %s is not in the store after the push", wantOID) |
| 79 | } | |
| 80 | ||
| 81 | // A fresh anonymous clone over HTTP, then a pull: the only source of the | |
| 82 | // bytes is this server. | |
| 83 | dst := filepath.Join(t.TempDir(), "clone") | |
| 84 | if _, errs, err := runGit(t, "", "clone", "-q", f.httpURL(f.repo), dst); err != nil { | |
| 85 | t.Fatalf("clone: %v\n%s", err, errs) | |
| 86 | } | |
| 87 | if _, errs, err := runGit(t, dst, "lfs", "install", "--local"); err != nil { | |
| 88 | t.Fatalf("git lfs install in the clone: %v\n%s", err, errs) | |
| 89 | } | |
| 90 | if _, errs, err := runGit(t, dst, "lfs", "pull"); err != nil { | |
| 91 | t.Fatalf("git lfs pull: %v\n%s", err, errs) | |
| 92 | } | |
| 93 | got, err := os.ReadFile(filepath.Join(dst, "artifact.bin")) | |
| 94 | if err != nil { | |
| 95 | t.Fatal(err) | |
| 96 | } | |
| 97 | if !bytes.Equal(got, payload) { | |
| 98 | t.Fatalf("cloned artifact is %d bytes and does not match the pushed %d", len(got), len(payload)) | |
| 99 | } | |
| 100 | } | |
| 101 | ||
| 102 | // TestLFSPointerWithoutObjectIsRejected is the second exit criterion. --no-verify | |
| 103 | // skips the client's own pre-push check, which is the only way to test the | |
| 104 | // server's rule: the pointer is committed and pushed, its object never uploaded. | |
| 105 | func TestLFSPointerWithoutObjectIsRejected(t *testing.T) { | |
|
44530e74
Keenan Nemetz
16d ago
|
106 | t.Parallel() |
|
64ac5071
Keenan Nemetz
17d ago
|
107 | f := newFixture(t) |
| 108 | f.publish(f.repo) | |
| 109 | ||
| 110 | data := []byte("content that was never uploaded") | |
| 111 | missing := sha256.Sum256(data) | |
| 112 | oid := hex.EncodeToString(missing[:]) | |
| 113 | pointer := "version https://git-lfs.github.com/spec/v1\noid sha256:" + oid + | |
| 114 | "\nsize " + strconv.Itoa(len(data)) + "\n" | |
| 115 | ||
| 116 | dir := workRepo(t, map[string]string{"artifacts/missing.bin": pointer}) | |
| 117 | out, errs, err := f.git(dir, "push", "--no-verify", f.sshURL(f.repo), "main:refs/heads/main") | |
| 118 | if err == nil { | |
| 119 | t.Fatalf("push of a pointer with no object was accepted:\n%s%s", out, errs) | |
| 120 | } | |
| 121 | for _, want := range []string{oid[:12], "not on this server", "git lfs push"} { | |
| 122 | if !strings.Contains(errs, want) { | |
| 123 | t.Fatalf("rejection does not mention %q:\n%s", want, errs) | |
| 124 | } | |
| 125 | } | |
| 126 | ||
| 127 | // Nothing moved: the ref the push would have created is still absent. | |
| 128 | remote, _, err := f.git(dir, "ls-remote", f.sshURL(f.repo)) | |
| 129 | if err != nil { | |
| 130 | t.Fatalf("ls-remote: %v", err) | |
| 131 | } | |
| 132 | if strings.Contains(remote, "refs/heads/main") { | |
| 133 | t.Fatalf("ref moved despite the rejection:\n%s", remote) | |
| 134 | } | |
| 135 | ||
| 136 | // After the object is uploaded, the same push is accepted. | |
|
65b88b3b
Keenan Nemetz
16d ago
|
137 | if _, _, err := f.srv.LFS.Put(f.repo, oid, int64(len(data)), bytes.NewReader(data)); err != nil { |
|
64ac5071
Keenan Nemetz
17d ago
|
138 | t.Fatal(err) |
| 139 | } | |
| 140 | if out, errs, err := f.git(dir, "push", "--no-verify", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | |
| 141 | t.Fatalf("push rejected even though the object is present: %v\nstdout:\n%s\nstderr:\n%s", err, out, errs) | |
| 142 | } | |
| 143 | } | |
| 144 | ||
| 145 | // TestLFSBatchProtocol covers the JSON side without a client, including the two | |
| 146 | // refusals that keep the read-only-web / SSH-write split intact. | |
| 147 | func TestLFSBatchProtocol(t *testing.T) { | |
|
44530e74
Keenan Nemetz
16d ago
|
148 | t.Parallel() |
|
64ac5071
Keenan Nemetz
17d ago
|
149 | f := newFixture(t) |
| 150 | f.publish(f.repo) | |
| 151 | ||
| 152 | content := []byte("downloadable artifact") | |
|
65b88b3b
Keenan Nemetz
16d ago
|
153 | oid, err := f.srv.LFS.PutBytes(f.repo, content) |
|
64ac5071
Keenan Nemetz
17d ago
|
154 | if err != nil { |
| 155 | t.Fatal(err) | |
| 156 | } | |
| 157 | missing := strings.Repeat("a", 64) | |
| 158 | ||
| 159 | post := func(body lfs.BatchRequest) (int, string) { | |
| 160 | t.Helper() | |
| 161 | raw, _ := json.Marshal(body) | |
| 162 | req, err := http.NewRequest(http.MethodPost, f.httpURL(f.repo)+"/info/lfs/objects/batch", bytes.NewReader(raw)) | |
| 163 | if err != nil { | |
| 164 | t.Fatal(err) | |
| 165 | } | |
| 166 | req.Header.Set("Content-Type", lfs.MediaType) | |
| 167 | resp, err := http.DefaultClient.Do(req) | |
| 168 | if err != nil { | |
| 169 | t.Fatal(err) | |
| 170 | } | |
| 171 | defer resp.Body.Close() | |
| 172 | got, _ := io.ReadAll(resp.Body) | |
| 173 | return resp.StatusCode, string(got) | |
| 174 | } | |
| 175 | ||
| 176 | code, body := post(lfs.BatchRequest{ | |
| 177 | Operation: "download", | |
| 178 | Objects: []lfs.Object{{OID: oid, Size: int64(len(content))}, {OID: missing, Size: 4}}, | |
| 179 | }) | |
| 180 | if code != 200 { | |
| 181 | t.Fatalf("download batch = %d: %s", code, body) | |
| 182 | } | |
| 183 | var res lfs.BatchResponse | |
| 184 | if err := json.Unmarshal([]byte(body), &res); err != nil { | |
| 185 | t.Fatalf("batch response is not JSON: %v\n%s", err, body) | |
| 186 | } | |
| 187 | if len(res.Objects) != 2 { | |
| 188 | t.Fatalf("batch answered %d objects, want 2: %s", len(res.Objects), body) | |
| 189 | } | |
| 190 | if act, ok := res.Objects[0].Actions["download"]; !ok { | |
| 191 | t.Fatalf("no download action for a stored object: %s", body) | |
| 192 | } else { | |
| 193 | // The href must be fetchable as given: that is the whole contract. | |
| 194 | resp, err := http.Get(act.Href) | |
| 195 | if err != nil { | |
| 196 | t.Fatal(err) | |
| 197 | } | |
| 198 | got, _ := io.ReadAll(resp.Body) | |
| 199 | resp.Body.Close() | |
| 200 | if resp.StatusCode != 200 || !bytes.Equal(got, content) { | |
| 201 | t.Fatalf("GET %s = %d, %d bytes", act.Href, resp.StatusCode, len(got)) | |
| 202 | } | |
| 203 | if etag := resp.Header.Get("ETag"); etag != `"`+oid+`"` { | |
| 204 | t.Fatalf("ETag = %q, want the object id", etag) | |
| 205 | } | |
| 206 | } | |
| 207 | if res.Objects[1].Error == nil || res.Objects[1].Error.Code != 404 { | |
| 208 | t.Fatalf("missing object should carry a 404 error: %+v", res.Objects[1]) | |
| 209 | } | |
| 210 | ||
| 211 | // Uploads are not accepted over HTTP, and say where they are accepted. | |
| 212 | code, body = post(lfs.BatchRequest{Operation: "upload", Objects: []lfs.Object{{OID: oid, Size: 1}}}) | |
| 213 | if code != http.StatusForbidden || !strings.Contains(body, "SSH") { | |
| 214 | t.Fatalf("upload batch = %d: %s", code, body) | |
| 215 | } | |
| 216 | ||
| 217 | // The object endpoint refuses unknown ids rather than looking them up. | |
|
ed03b3d1
Keenan Nemetz
2h ago
|
218 | if code, _ := f.get("/" + f.repo + ".git/info/lfs/objects/" + strings.Repeat("b", 64)); code != 404 { |
|
64ac5071
Keenan Nemetz
17d ago
|
219 | t.Fatalf("GET of an absent object = %d, want 404", code) |
| 220 | } | |
| 221 | } | |
| 222 | ||
| 223 | // TestLFSIsHiddenInPrivateRepos mirrors the clone rule: a private repository | |
| 224 | // answers 404 for LFS too, so the read surface never confirms that it exists. | |
| 225 | func TestLFSIsHiddenInPrivateRepos(t *testing.T) { | |
|
44530e74
Keenan Nemetz
16d ago
|
226 | t.Parallel() |
|
64ac5071
Keenan Nemetz
17d ago
|
227 | f := newFixture(t) // deliberately not published |
|
65b88b3b
Keenan Nemetz
16d ago
|
228 | oid, err := f.srv.LFS.PutBytes(f.repo, []byte("private artifact")) |
|
64ac5071
Keenan Nemetz
17d ago
|
229 | if err != nil { |
| 230 | t.Fatal(err) | |
| 231 | } | |
|
ed03b3d1
Keenan Nemetz
2h ago
|
232 | if code, body := f.get("/" + f.repo + ".git/info/lfs/objects/" + oid); code != 404 { |
|
64ac5071
Keenan Nemetz
17d ago
|
233 | t.Fatalf("object GET in a private repo = %d: %s", code, body) |
| 234 | } | |
| 235 | raw, _ := json.Marshal(lfs.BatchRequest{Operation: "download", Objects: []lfs.Object{{OID: oid}}}) | |
| 236 | req, _ := http.NewRequest(http.MethodPost, f.httpURL(f.repo)+"/info/lfs/objects/batch", bytes.NewReader(raw)) | |
| 237 | req.Header.Set("Content-Type", lfs.MediaType) | |
| 238 | resp, err := http.DefaultClient.Do(req) | |
| 239 | if err != nil { | |
| 240 | t.Fatal(err) | |
| 241 | } | |
| 242 | defer resp.Body.Close() | |
| 243 | if resp.StatusCode != 404 { | |
| 244 | t.Fatalf("batch in a private repo = %d, want 404", resp.StatusCode) | |
| 245 | } | |
| 246 | } | |
|
65b88b3b
Keenan Nemetz
16d ago
|
247 | |
| 248 | // The LFS object endpoint takes an arbitrary object id, so it is the one read path where | |
| 249 | // "may this repository see these bytes" is a real question. A repository that does not | |
| 250 | // hold the object answers 404 even though the store has the bytes and the id is exact. | |
| 251 | func TestAnObjectIsNotFetchableThroughAnotherRepository(t *testing.T) { | |
| 252 | t.Parallel() | |
| 253 | f := newFixture(t) | |
| 254 | f.publish(f.repo) | |
| 255 | ||
| 256 | secret := []byte("an asset that belongs to one repository") | |
| 257 | oid, err := f.srv.LFS.PutBytes("somewhere-else", secret) | |
| 258 | if err != nil { | |
| 259 | t.Fatal(err) | |
| 260 | } | |
| 261 | ||
| 262 | // The bytes are in the shared store. | |
| 263 | if _, err := os.Stat(f.srv.LFS.ObjectPath(oid)); err != nil { | |
| 264 | t.Fatalf("the fixture did not store the object: %v", err) | |
| 265 | } | |
| 266 | // But not through this repository. | |
|
ed03b3d1
Keenan Nemetz
2h ago
|
267 | code, _ := f.get("/" + f.repo + ".git/info/lfs/objects/" + oid) |
|
65b88b3b
Keenan Nemetz
16d ago
|
268 | if code != 404 { |
| 269 | t.Fatalf("another repository's object answered %d, want 404", code) | |
| 270 | } | |
| 271 | ||
| 272 | // Once the repository holds it, it is served — and deduplicated, not copied. | |
| 273 | if _, _, err := f.srv.LFS.Put(f.repo, oid, int64(len(secret)), bytes.NewReader(secret)); err != nil { | |
| 274 | t.Fatal(err) | |
| 275 | } | |
|
ed03b3d1
Keenan Nemetz
2h ago
|
276 | code, body := f.get("/" + f.repo + ".git/info/lfs/objects/" + oid) |
|
65b88b3b
Keenan Nemetz
16d ago
|
277 | if code != 200 || body != string(secret) { |
| 278 | t.Fatalf("own object = %d %q", code, body) | |
| 279 | } | |
| 280 | } | |
| 281 | ||
| 282 | // An instance upgrading from the unscoped store has a pool and no namespaces. Rather than | |
| 283 | // leaving every fetch broken until somebody reads an upgrade note, the server links the | |
| 284 | // store on first boot. | |
| 285 | func TestAnUnscopedStoreIsLinkedOnStartup(t *testing.T) { | |
| 286 | // Not parallel: it mutates the fixture's store layout before the server starts. | |
| 287 | f := newFixture(t) | |
| 288 | f.publish(f.repo) | |
| 289 | ||
| 290 | // A committed pointer, and the bytes in the shared pool, but no link — exactly what an | |
| 291 | // older instance looks like. | |
| 292 | secret := []byte("bytes an older instance already had") | |
| 293 | oid := fmt.Sprintf("%x", sha256.Sum256(secret)) | |
| 294 | if _, _, err := f.srv.LFS.Put(f.repo, oid, int64(len(secret)), bytes.NewReader(secret)); err != nil { | |
| 295 | t.Fatal(err) | |
| 296 | } | |
| 297 | pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(secret)) | |
| 298 | work := workRepo(t, map[string]string{"art.psd": pointer}) | |
| 299 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | |
| 300 | t.Fatalf("push: %s", errs) | |
| 301 | } | |
| 302 | // Remove the namespace, as an instance that never had one would have none. | |
| 303 | if err := os.RemoveAll(filepath.Join(f.cfg.LFSDir(), "repos")); err != nil { | |
| 304 | t.Fatal(err) | |
| 305 | } | |
| 306 | if f.srv.LFS.NamespaceExists() { | |
| 307 | t.Fatal("the namespace still exists") | |
| 308 | } | |
| 309 | // The repair pass, which is what the server runs at startup. | |
| 310 | repos, err := f.srv.Repos.List() | |
| 311 | if err != nil { | |
| 312 | t.Fatal(err) | |
| 313 | } | |
|
c5902451
Keenan Nemetz
16d ago
|
314 | scan := make([]forge.GCRepo, 0, len(repos)) |
|
65b88b3b
Keenan Nemetz
16d ago
|
315 | for _, m := range repos { |
|
c5902451
Keenan Nemetz
16d ago
|
316 | scan = append(scan, forge.GCRepo{Name: m.Name, Dir: m.Dir}) |
|
65b88b3b
Keenan Nemetz
16d ago
|
317 | } |
|
c5902451
Keenan Nemetz
16d ago
|
318 | if _, err := forge.GCLink(forge.GCOptions{Repos: scan, Store: f.srv.LFS}); err != nil { |
|
65b88b3b
Keenan Nemetz
16d ago
|
319 | t.Fatal(err) |
| 320 | } | |
| 321 | if !f.srv.LFS.Has(f.repo, oid) { | |
| 322 | t.Fatal("the repair did not restore the repository's view of its object") | |
| 323 | } | |
| 324 | } |