vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 5: LFS objects, over SSH for writes and the proxy for reads
64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf
Phase 5: LFS objects, over SSH for writes and the proxy for reads Large files leave the git object database and land in one content-addressed store per instance. Reads are anonymous like clone; writes ride the SSH key that already authenticates the push, so no token exists anywhere. - internal/lfs: sha256-addressed store that verifies content and size before an object becomes visible; parser for the pointer format a client writes - HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid> with the object id as ETag; upload over HTTP is refused and names SSH - SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so `git lfs push` needs nothing but the key at the handshake - pre-receive: a pointer whose object is missing, or whose size disagrees with the stored object, fails the push that introduced it; a binary over lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run before the protected-branch gate, because an admin's approval bypass is about review, not about whether the commit can be checked out - runs: a finished log becomes a content-addressed object in the same write that records the status, served at /r/<repo>/runs/<id>/log - vole gc [--dry-run] [--grace]: marks pointer targets from every ref in every repository plus archived logs, sweeps the rest. Mark-and-sweep rather than reference counts: a counter that is one too low deletes an artifact, a walk over the refs cannot be wrong about what is reachable
internal/gc/gc_test.go +167
| @@ -0,0 +1,167 @@ | |||
| 1 | package gc | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "bytes" | ||
| 5 | "crypto/sha256" | ||
| 6 | "encoding/hex" | ||
| 7 | "testing" | ||
| 8 | "time" | ||
| 9 | |||
| 10 | "github.com/rodent-software/vole/internal/gitx" | ||
| 11 | "github.com/rodent-software/vole/internal/lfs" | ||
| 12 | ) | ||
| 13 | |||
| 14 | // repoWithPointer builds a bare repository whose main branch commits a pointer | ||
| 15 | // naming oid, then a second commit that replaces it with plain text. The object | ||
| 16 | // is therefore reachable only through history — which is still reachable. | ||
| 17 | func repoWithPointer(t *testing.T, oid string) string { | ||
| 18 | t.Helper() | ||
| 19 | dir := t.TempDir() + "/repo.git" | ||
| 20 | if err := gitx.InitBare(dir, "main"); err != nil { | ||
| 21 | t.Fatal(err) | ||
| 22 | } | ||
| 23 | g := gitx.Open(dir) | ||
| 24 | pointer, err := g.HashObject(lfs.Pointer{OID: oid, Size: 12}.Encode()) | ||
| 25 | if err != nil { | ||
| 26 | t.Fatal(err) | ||
| 27 | } | ||
| 28 | text, err := g.HashObject([]byte("readme\n")) | ||
| 29 | if err != nil { | ||
| 30 | t.Fatal(err) | ||
| 31 | } | ||
| 32 | tree1, err := g.MkTree([]gitx.Entry{ | ||
| 33 | {Mode: "100644", Type: "blob", SHA: pointer, Name: "artifact.bin"}, | ||
| 34 | {Mode: "100644", Type: "blob", SHA: text, Name: "README.md"}, | ||
| 35 | }) | ||
| 36 | if err != nil { | ||
| 37 | t.Fatal(err) | ||
| 38 | } | ||
| 39 | c1, err := g.CommitTree(tree1, "", "add the artifact") | ||
| 40 | if err != nil { | ||
| 41 | t.Fatal(err) | ||
| 42 | } | ||
| 43 | tree2, err := g.MkTree([]gitx.Entry{ | ||
| 44 | {Mode: "100644", Type: "blob", SHA: text, Name: "README.md"}, | ||
| 45 | }) | ||
| 46 | if err != nil { | ||
| 47 | t.Fatal(err) | ||
| 48 | } | ||
| 49 | c2, err := g.CommitTree(tree2, c1, "drop the artifact") | ||
| 50 | if err != nil { | ||
| 51 | t.Fatal(err) | ||
| 52 | } | ||
| 53 | if err := g.UpdateRef("refs/heads/main", c2); err != nil { | ||
| 54 | t.Fatal(err) | ||
| 55 | } | ||
| 56 | return dir | ||
| 57 | } | ||
| 58 | |||
| 59 | func storeWith(t *testing.T, blobs ...[]byte) (*lfs.Store, []string) { | ||
| 60 | t.Helper() | ||
| 61 | s := lfs.New(t.TempDir(), 1<<20) | ||
| 62 | var oids []string | ||
| 63 | for _, b := range blobs { | ||
| 64 | sum := sha256.Sum256(b) | ||
| 65 | oid := hex.EncodeToString(sum[:]) | ||
| 66 | if _, _, err := s.Put(oid, int64(len(b)), bytes.NewReader(b)); err != nil { | ||
| 67 | t.Fatal(err) | ||
| 68 | } | ||
| 69 | oids = append(oids, oid) | ||
| 70 | } | ||
| 71 | return s, oids | ||
| 72 | } | ||
| 73 | |||
| 74 | func TestReachableThroughHistorySurvivesAndGarbageDoesNot(t *testing.T) { | ||
| 75 | kept := []byte("the release artifact") | ||
| 76 | garbage := []byte("nobody points at this") | ||
| 77 | store, oids := storeWith(t, kept, garbage) | ||
| 78 | reachable, orphan := oids[0], oids[1] | ||
| 79 | dir := repoWithPointer(t, reachable) | ||
| 80 | |||
| 81 | res, err := Run(Options{ | ||
| 82 | Repos: []string{dir}, | ||
| 83 | Store: store, | ||
| 84 | Grace: 0, | ||
| 85 | Now: time.Now().Add(time.Hour), | ||
| 86 | }) | ||
| 87 | if err != nil { | ||
| 88 | t.Fatal(err) | ||
| 89 | } | ||
| 90 | if !store.Has(reachable) { | ||
| 91 | t.Fatal("swept an object a live ref reaches through history") | ||
| 92 | } | ||
| 93 | if store.Has(orphan) { | ||
| 94 | t.Fatal("kept an object nothing references") | ||
| 95 | } | ||
| 96 | if res.Swept != 1 || res.Marked != 1 { | ||
| 97 | t.Fatalf("result %+v; want 1 marked, 1 swept", res) | ||
| 98 | } | ||
| 99 | } | ||
| 100 | |||
| 101 | func TestGraceProtectsAFreshUpload(t *testing.T) { | ||
| 102 | kept := []byte("the release artifact") | ||
| 103 | fresh := []byte("uploaded a moment ago") | ||
| 104 | store, oids := storeWith(t, kept, fresh) | ||
| 105 | dir := repoWithPointer(t, oids[0]) | ||
| 106 | |||
| 107 | res, err := Run(Options{Repos: []string{dir}, Store: store, Grace: time.Hour}) | ||
| 108 | if err != nil { | ||
| 109 | t.Fatal(err) | ||
| 110 | } | ||
| 111 | if !store.Has(oids[1]) { | ||
| 112 | t.Fatal("an upload inside the grace period was swept") | ||
| 113 | } | ||
| 114 | if res.Within != 1 || res.Swept != 0 { | ||
| 115 | t.Fatalf("result %+v; want 1 within grace, 0 swept", res) | ||
| 116 | } | ||
| 117 | // The same store, an hour later: the grace has run out. | ||
| 118 | res, err = Run(Options{ | ||
| 119 | Repos: []string{dir}, Store: store, Grace: time.Hour, | ||
| 120 | Now: time.Now().Add(2 * time.Hour), | ||
| 121 | }) | ||
| 122 | if err != nil { | ||
| 123 | t.Fatal(err) | ||
| 124 | } | ||
| 125 | if store.Has(oids[1]) || res.Swept != 1 { | ||
| 126 | t.Fatalf("result %+v; want the stale object swept", res) | ||
| 127 | } | ||
| 128 | if !store.Has(oids[0]) { | ||
| 129 | t.Fatal("swept the reachable object along with the stale one") | ||
| 130 | } | ||
| 131 | } | ||
| 132 | |||
| 133 | func TestArchivedLogIsARoot(t *testing.T) { | ||
| 134 | log := []byte("$ go test ./...\nok\n") | ||
| 135 | store, oids := storeWith(t, log) | ||
| 136 | |||
| 137 | res, err := Run(Options{ | ||
| 138 | Repos: nil, | ||
| 139 | Store: store, | ||
| 140 | LogOIDs: oids, | ||
| 141 | Grace: 0, | ||
| 142 | Now: time.Now().Add(time.Hour), | ||
| 143 | }) | ||
| 144 | if err != nil { | ||
| 145 | t.Fatal(err) | ||
| 146 | } | ||
| 147 | if !store.Has(oids[0]) { | ||
| 148 | t.Fatal("swept an archived CI log; no git ref reaches one, so it must be a root") | ||
| 149 | } | ||
| 150 | if res.Swept != 0 || res.Marked != 1 { | ||
| 151 | t.Fatalf("result %+v", res) | ||
| 152 | } | ||
| 153 | } | ||
| 154 | |||
| 155 | func TestDryRunRemovesNothing(t *testing.T) { | ||
| 156 | store, oids := storeWith(t, []byte("garbage")) | ||
| 157 | res, err := Run(Options{Store: store, Grace: 0, DryRun: true, Now: time.Now().Add(time.Hour)}) | ||
| 158 | if err != nil { | ||
| 159 | t.Fatal(err) | ||
| 160 | } | ||
| 161 | if !store.Has(oids[0]) { | ||
| 162 | t.Fatal("--dry-run deleted an object") | ||
| 163 | } | ||
| 164 | if res.Swept != 1 { | ||
| 165 | t.Fatalf("dry run reported %d removals; want 1", res.Swept) | ||
| 166 | } | ||
| 167 | } | ||