vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 7 slice 3: artifacts attach over SSH and are one pointer deep
85cc89fe9eafd1793702d143ea2eb41db635fbe2 · Keenan Nemetz · 17d ago · parent c56911e2
Phase 7 slice 3: artifacts attach over SSH and are one pointer deep A released binary is a pointer blob at releases/<tag>/<name> on the metadata branch. That is the whole record, and it was chosen because the machinery that already exists does the work: gc marks LFS objects named by pointers in any tree reachable from any ref (refs/heads/vole is one), and pre-receive's pointer hygiene already refuses a pointer whose bytes are missing. A JSON manifest of sha256 strings would have parsed as no pointer and been swept once the grace period passed. - cmd/vole attach speaks the pure-SSH LFS transfer protocol itself (version, batch, put-object, verify-object), so bytes move over the key that already writes: no token, no second endpoint, no new write surface - upload happens before the pointer commit, because the reverse order publishes a download that 404s and receive-pack cannot see bytes that were never in the pack. A test pushes a pointer with no object and asserts the push is refused - the artifact list is read from the pointer tree (internal/releases), which the metadata validator now knows about; a file under releases/ that is not a pointer is refused at push instead of silently unservable - downloads: /r/<repo>/releases/<tag>/artifacts/<name> out of the store, ETag = object id, always Content-Disposition: attachment with nosniff, ServeContent for Range and 304, clone read rule so private artifacts are 404 - the "not on this server" hook message now names vole release attach as well as git lfs push, because it fires for both now Measured end to end through the binary: two 2.9 MiB files attached over SSH in 390 ms, the identical second file deduplicating to one object; both downloaded in 13.9 ms with matching sha256, 304 on If-None-Match, 206 on Range; vole gc --dry-run reports the published artifact reachable and removes nothing. The planned releases/<tag>.json record and drafts are dropped with it: the pointer directory is the artifact list and being in the ref is what published means, so the file would only have duplicated the tag.
internal/server/releases_test.go +162
| @@ -8,12 +8,16 @@ import ( | |||
| 8 | 8 | "archive/tar" | |
| 9 | 9 | "bytes" | |
| 10 | 10 | "compress/gzip" | |
| 11 | "fmt" | ||
| 11 | 12 | "io" | |
| 12 | 13 | "net/http" | |
| 13 | 14 | "os" | |
| 14 | 15 | "path/filepath" | |
| 15 | 16 | "strings" | |
| 16 | 17 | "testing" | |
| 18 | "time" | ||
| 19 | |||
| 20 | "github.com/rodent-software/vole/internal/gc" | ||
| 17 | 21 | ) | |
| 18 | 22 | ||
| 19 | 23 | // releaseFixture builds a repository with one commit tagged v0.9.0 and a second | |
| @@ -302,3 +306,161 @@ func tarContents(t *testing.T, r io.Reader, gzipped bool) map[string]string { | |||
| 302 | 306 | } | |
| 303 | 307 | return out | |
| 304 | 308 | } | |
| 309 | |||
| 310 | // attachArtifact publishes one file as a release artifact the way the CLI does: | ||
| 311 | // upload the bytes to the store, then push a pointer that names them. The order | ||
| 312 | // matters, and the hook enforces it — see the ordering test below. | ||
| 313 | func attachArtifact(t *testing.T, f *fixture, tag, name string, data []byte) (string, string) { | ||
| 314 | t.Helper() | ||
| 315 | oid, err := f.srv.LFS.PutBytes(data) | ||
| 316 | if err != nil { | ||
| 317 | t.Fatalf("store the artifact: %v", err) | ||
| 318 | } | ||
| 319 | meta := f.newMetaRepo(t) | ||
| 320 | pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %d\n", oid, len(data)) | ||
| 321 | if err := meta.apply(map[string]string{"releases/" + tag + "/" + name: pointer}, "attach "+name); err != nil { | ||
| 322 | t.Fatalf("push the pointer: %v", err) | ||
| 323 | } | ||
| 324 | return oid, pointer | ||
| 325 | } | ||
| 326 | |||
| 327 | func TestReleaseArtifactDownload(t *testing.T) { | ||
| 328 | f := releaseFixture(t) | ||
| 329 | data := []byte("a release binary, not source code\n") | ||
| 330 | oid, _ := attachArtifact(t, f, "v1.0.0", "tool-linux-amd64", data) | ||
| 331 | |||
| 332 | // The page names the artifact and links to it, so a reader never has to guess | ||
| 333 | // a URL or hold a sha256. | ||
| 334 | code, page := f.get("/r/" + f.repo + "/releases/v1.0.0") | ||
| 335 | if code != 200 { | ||
| 336 | t.Fatalf("release page = %d", code) | ||
| 337 | } | ||
| 338 | for _, want := range []string{"tool-linux-amd64", "/artifacts/tool-linux-amd64", "SHA-256"} { | ||
| 339 | if !strings.Contains(page, want) { | ||
| 340 | t.Fatalf("release page does not show %q:\n%s", want, page) | ||
| 341 | } | ||
| 342 | } | ||
| 343 | |||
| 344 | url := f.http.URL + "/r/" + f.repo + "/releases/v1.0.0/artifacts/tool-linux-amd64" | ||
| 345 | resp, err := http.Get(url) | ||
| 346 | if err != nil { | ||
| 347 | t.Fatal(err) | ||
| 348 | } | ||
| 349 | body, _ := io.ReadAll(resp.Body) | ||
| 350 | resp.Body.Close() | ||
| 351 | if resp.StatusCode != 200 { | ||
| 352 | t.Fatalf("artifact download = %d", resp.StatusCode) | ||
| 353 | } | ||
| 354 | if !bytes.Equal(body, data) { | ||
| 355 | t.Fatalf("downloaded %q, want %q", body, data) | ||
| 356 | } | ||
| 357 | // The validator is the content id, and the disposition is always an | ||
| 358 | // attachment: an artifact must never render on this origin. | ||
| 359 | if got, want := resp.Header.Get("ETag"), `"`+oid+`"`; got != want { | ||
| 360 | t.Fatalf("ETag = %q, want %q", got, want) | ||
| 361 | } | ||
| 362 | if cd := resp.Header.Get("Content-Disposition"); !strings.Contains(cd, "attachment") { | ||
| 363 | t.Fatalf("Content-Disposition = %q", cd) | ||
| 364 | } | ||
| 365 | if resp.Header.Get("X-Content-Type-Options") != "nosniff" { | ||
| 366 | t.Fatal("artifact is served without nosniff") | ||
| 367 | } | ||
| 368 | |||
| 369 | // A conditional request costs no bytes. | ||
| 370 | req, _ := http.NewRequest(http.MethodGet, url, nil) | ||
| 371 | req.Header.Set("If-None-Match", `"`+oid+`"`) | ||
| 372 | again, err := http.DefaultClient.Do(req) | ||
| 373 | if err != nil { | ||
| 374 | t.Fatal(err) | ||
| 375 | } | ||
| 376 | again.Body.Close() | ||
| 377 | if again.StatusCode != http.StatusNotModified { | ||
| 378 | t.Fatalf("conditional artifact request = %d, want 304", again.StatusCode) | ||
| 379 | } | ||
| 380 | |||
| 381 | // Range support is what makes a large artifact resumable. | ||
| 382 | req, _ = http.NewRequest(http.MethodGet, url, nil) | ||
| 383 | req.Header.Set("Range", "bytes=0-3") | ||
| 384 | part, err := http.DefaultClient.Do(req) | ||
| 385 | if err != nil { | ||
| 386 | t.Fatal(err) | ||
| 387 | } | ||
| 388 | sliced, _ := io.ReadAll(part.Body) | ||
| 389 | part.Body.Close() | ||
| 390 | if part.StatusCode != http.StatusPartialContent || string(sliced) != string(data[:4]) { | ||
| 391 | t.Fatalf("range request = %d %q, want 206 %q", part.StatusCode, sliced, data[:4]) | ||
| 392 | } | ||
| 393 | |||
| 394 | if code, _ := f.get("/r/" + f.repo + "/releases/v1.0.0/artifacts/nope"); code != 404 { | ||
| 395 | t.Fatalf("unknown artifact = %d, want 404", code) | ||
| 396 | } | ||
| 397 | } | ||
| 398 | |||
| 399 | // TestReleasePointerWithoutObjectIsRefused is the reason attach uploads first. | ||
| 400 | // receive-pack cannot notice — the bytes were never in the pack — so the hook has | ||
| 401 | // to read the pointer and check the store, or the release would advertise a | ||
| 402 | // download that 404s. | ||
| 403 | func TestReleasePointerWithoutObjectIsRefused(t *testing.T) { | ||
| 404 | f := releaseFixture(t) | ||
| 405 | meta := f.newMetaRepo(t) | ||
| 406 | missing := strings.Repeat("ab", 32) | ||
| 407 | pointer := fmt.Sprintf("version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize 10\n", missing) | ||
| 408 | err := meta.apply(map[string]string{"releases/v1.0.0/ghost": pointer}, "attach a ghost") | ||
| 409 | if err == nil { | ||
| 410 | t.Fatal("the push accepted a pointer whose object was never uploaded") | ||
| 411 | } | ||
| 412 | if !strings.Contains(err.Error(), missing[:12]) { | ||
| 413 | t.Fatalf("the refusal does not name the missing object: %v", err) | ||
| 414 | } | ||
| 415 | // And the release page does not show it, because the ref never moved. | ||
| 416 | if code, _ := f.get("/r/" + f.repo + "/releases/v1.0.0/artifacts/ghost"); code != 404 { | ||
| 417 | t.Fatalf("ghost artifact = %d, want 404", code) | ||
| 418 | } | ||
| 419 | } | ||
| 420 | |||
| 421 | // TestGCCountsReleaseArtifactsAsReachable pins the design decision that made the | ||
| 422 | // pointer the whole record: the sweep marks objects named by pointer blobs in any | ||
| 423 | // tree reachable from any ref, and the metadata branch is one of those refs. | ||
| 424 | func TestGCCountsReleaseArtifactsAsReachable(t *testing.T) { | ||
| 425 | f := releaseFixture(t) | ||
| 426 | keep := []byte("published artifact") | ||
| 427 | drop := []byte("an upload nobody referenced") | ||
| 428 | keepOID, _ := attachArtifact(t, f, "v1.0.0", "keep.bin", keep) | ||
| 429 | dropOID, err := f.srv.LFS.PutBytes(drop) | ||
| 430 | if err != nil { | ||
| 431 | t.Fatal(err) | ||
| 432 | } | ||
| 433 | |||
| 434 | res, err := gc.Run(gc.Options{ | ||
| 435 | Repos: []string{filepath.Join(f.cfg.ReposDir(), f.repo+".git")}, | ||
| 436 | Store: f.srv.LFS, | ||
| 437 | // Far enough ahead that the grace period protects nothing. | ||
| 438 | Now: time.Now().Add(48 * time.Hour), | ||
| 439 | }) | ||
| 440 | if err != nil { | ||
| 441 | t.Fatalf("gc: %v", err) | ||
| 442 | } | ||
| 443 | if !f.srv.LFS.Has(keepOID) { | ||
| 444 | t.Fatalf("gc swept the published artifact (%+v)", res) | ||
| 445 | } | ||
| 446 | if f.srv.LFS.Has(dropOID) { | ||
| 447 | t.Fatalf("gc kept an object nothing reaches (%+v)", res) | ||
| 448 | } | ||
| 449 | } | ||
| 450 | |||
| 451 | func TestReleaseArtifactsArePrivateForPrivateRepos(t *testing.T) { | ||
| 452 | f := newFixture(t) | ||
| 453 | work := workRepo(t, map[string]string{"readme.md": "secret\n"}) | ||
| 454 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 455 | t.Fatalf("push: %s", errs) | ||
| 456 | } | ||
| 457 | if _, errs, err := runGit(t, work, "tag", "-a", "v1.0.0", "-m", "notes\n"); err != nil { | ||
| 458 | t.Fatalf("tag: %s", errs) | ||
| 459 | } | ||
| 460 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "refs/tags/v1.0.0:refs/tags/v1.0.0"); err != nil { | ||
| 461 | t.Fatalf("push tag: %s", errs) | ||
| 462 | } | ||
| 463 | if code, _ := f.get("/r/" + f.repo + "/releases/v1.0.0/artifacts/anything"); code != 404 { | ||
| 464 | t.Fatalf("private artifact = %d, want 404", code) | ||
| 465 | } | ||
| 466 | } | ||