Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf 

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/hook/lfs_test.go +200

@@ -0,0 +1,200 @@
1 package hook
2
3 import (
4 "bytes"
5 "crypto/sha256"
6 "encoding/hex"
7 "strings"
8 "testing"
9
10 "github.com/rodent-software/vole/internal/acl"
11 "github.com/rodent-software/vole/internal/gitx"
12 "github.com/rodent-software/vole/internal/lfs"
13 "github.com/rodent-software/vole/internal/repo"
14 )
15
16 // filesFixture builds a repository whose first commit contains every file, which
17 // is enough to test what pre-receive sees at a pushed tip.
18 func filesFixture(t *testing.T, files map[string][]byte) *fixture {
19 t.Helper()
20 st := repo.New(t.TempDir(), "/bin/true")
21 m, err := st.Create("demo", "owner", "")
22 if err != nil {
23 t.Fatal(err)
24 }
25 a, err := m.ACL()
26 if err != nil {
27 t.Fatal(err)
28 }
29 if err := a.SetEntry("writer", "write"); err != nil {
30 t.Fatal(err)
31 }
32 if err := m.SetACL(a, "test"); err != nil {
33 t.Fatal(err)
34 }
35
36 g := m.Git()
37 tree, err := gitx.WriteFlatTree(g, files)
38 if err != nil {
39 t.Fatal(err)
40 }
41 commit, err := g.CommitTree(tree, "", "initial")
42 if err != nil {
43 t.Fatal(err)
44 }
45 return &fixture{t: t, m: m, head: commit}
46 }
47
48 func pointerFor(content []byte) (lfs.Pointer, []byte) {
49 sum := sha256.Sum256(content)
50 p := lfs.Pointer{OID: hex.EncodeToString(sum[:]), Size: int64(len(content))}
51 return p, p.Encode()
52 }
53
54 func testStore(t *testing.T) *lfs.Store { return lfs.New(t.TempDir(), 1<<20) }
55
56 func TestPointerWithoutObjectIsAProblem(t *testing.T) {
57 p, pointer := pointerFor([]byte("artifact bytes nobody uploaded"))
58 f := filesFixture(t, map[string][]byte{"artifact.bin": pointer})
59
60 code, out := f.runWith(Config{Actor: "writer", LFS: testStore(t)}, "refs/heads/main")
61 if code == 0 {
62 t.Fatalf("push accepted although the pointer names an object the server lacks:\n%s", out)
63 }
64 for _, want := range []string{p.OID[:12], "not on this server", "git lfs push"} {
65 if !strings.Contains(out, want) {
66 t.Fatalf("rejection does not mention %q:\n%s", want, out)
67 }
68 }
69 }
70
71 func TestPointerWithObjectIsAccepted(t *testing.T) {
72 content := []byte("artifact bytes that were uploaded")
73 p, pointer := pointerFor(content)
74 store := testStore(t)
75 if _, _, err := store.Put(p.OID, p.Size, bytes.NewReader(content)); err != nil {
76 t.Fatal(err)
77 }
78 f := filesFixture(t, map[string][]byte{"artifact.bin": pointer})
79
80 if code, out := f.runWith(Config{Actor: "writer", LFS: store}, "refs/heads/main"); code != 0 {
81 t.Fatalf("push rejected although the object is present:\n%s", out)
82 }
83 }
84
85 func TestPointerSizeMismatchIsAProblem(t *testing.T) {
86 content := []byte("artifact bytes that were uploaded")
87 p, _ := pointerFor(content)
88 store := testStore(t)
89 if _, _, err := store.Put(p.OID, p.Size, bytes.NewReader(content)); err != nil {
90 t.Fatal(err)
91 }
92 // Same object id, a size that does not describe it: a client would stall or
93 // truncate, so the pointer is not trusted.
94 lying := lfs.Pointer{OID: p.OID, Size: p.Size + 10}.Encode()
95 f := filesFixture(t, map[string][]byte{"artifact.bin": lying})
96
97 code, out := f.runWith(Config{Actor: "writer", LFS: store}, "refs/heads/main")
98 if code == 0 {
99 t.Fatalf("push accepted with a pointer that misstates the size:\n%s", out)
100 }
101 if !strings.Contains(out, "bytes") {
102 t.Fatalf("rejection does not explain the size mismatch:\n%s", out)
103 }
104 }
105
106 func TestLargeBinaryIsWarnedAboutNotRejected(t *testing.T) {
107 binary := append(bytes.Repeat([]byte{0x00, 0x01, 0x02, 0x03}, 2000), 0xff)
108 f := filesFixture(t, map[string][]byte{
109 "assets/logo.png": binary,
110 "README.md": []byte("text\n"),
111 })
112
113 code, out := f.runWith(Config{Actor: "writer", LFS: testStore(t), LFSWarnBytes: 1024}, "refs/heads/main")
114 if code != 0 {
115 t.Fatalf("a large binary should be allowed with a warning, not rejected:\n%s", out)
116 }
117 for _, want := range []string{"warning", "assets/logo.png", `git lfs track "*.png"`, "git lfs migrate"} {
118 if !strings.Contains(out, want) {
119 t.Fatalf("warning does not mention %q:\n%s", want, out)
120 }
121 }
122 }
123
124 func TestTextFileIsNotWarnedAbout(t *testing.T) {
125 text := []byte(strings.Repeat("a line of prose\n", 500))
126 f := filesFixture(t, map[string][]byte{"notes.md": text})
127 code, out := f.runWith(Config{Actor: "writer", LFS: testStore(t), LFSWarnBytes: 1024}, "refs/heads/main")
128 if code != 0 {
129 t.Fatalf("push rejected: %s", out)
130 }
131 if strings.Contains(out, "warning") {
132 t.Fatalf("a text file was flagged as a binary to move to LFS:\n%s", out)
133 }
134 }
135
136 func TestDeletedPointerDoesNotBlockItsOwnRemoval(t *testing.T) {
137 // A broken pointer already in the tree must be removable; otherwise the only
138 // way out of the mistake is rewriting history.
139 st := repo.New(t.TempDir(), "/bin/true")
140 m, err := st.Create("demo", "owner", "")
141 if err != nil {
142 t.Fatal(err)
143 }
144 a, err := m.ACL()
145 if err != nil {
146 t.Fatal(err)
147 }
148 if err := a.SetEntry("writer", "write"); err != nil {
149 t.Fatal(err)
150 }
151 if err := m.SetACL(a, "test"); err != nil {
152 t.Fatal(err)
153 }
154 g := m.Git()
155 _, pointer := pointerFor([]byte("never uploaded"))
156 badBlob, err := g.HashObject(pointer)
157 if err != nil {
158 t.Fatal(err)
159 }
160 goodBlob, err := g.HashObject([]byte("text\n"))
161 if err != nil {
162 t.Fatal(err)
163 }
164 clutter, err := g.MkTree([]gitx.Entry{
165 {Mode: "100644", Type: "blob", SHA: badBlob, Name: "artifact.bin"},
166 {Mode: "100644", Type: "blob", SHA: goodBlob, Name: "README.md"},
167 })
168 if err != nil {
169 t.Fatal(err)
170 }
171 c1, err := g.CommitTree(clutter, "", "add a pointer with no object")
172 if err != nil {
173 t.Fatal(err)
174 }
175 clean, err := g.MkTree([]gitx.Entry{
176 {Mode: "100644", Type: "blob", SHA: goodBlob, Name: "README.md"},
177 })
178 if err != nil {
179 t.Fatal(err)
180 }
181 c2, err := g.CommitTree(clean, c1, "remove it")
182 if err != nil {
183 t.Fatal(err)
184 }
185
186 var out bytes.Buffer
187 code := PreReceive(Config{
188 RepoDir: m.Dir,
189 Actor: "writer",
190 LFS: testStore(t),
191 In: strings.NewReader(gitx.ZeroSHA + " " + c2 + " refs/heads/main\n"),
192 Out: &out,
193 })
194 if code != 0 {
195 t.Fatalf("removing a broken pointer was rejected:\n%s", out.String())
196 }
197 }
198
199 // The ACL helper is used above only through repo.Meta; keep the import honest.
200 var _ = acl.Write