Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf 

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/lfs/lfs.go +306

@@ -0,0 +1,306 @@
1 // Package lfs stores large files outside the git object database.
2 //
3 // A repository commits a small pointer file (the format git-lfs writes) in place
4 // of the bytes; the bytes live here, keyed by their sha256. The store is shared
5 // by every repository on the instance, so pushing the same release artifact in
6 // two repositories costs one copy.
7 //
8 // The store is derived, not authoritative: git is. An object that no live ref
9 // points at is garbage, which is why `vole gc` marks from the refs rather than
10 // trusting a counter that a crash could have left wrong.
11 package lfs
12
13 import (
14 "crypto/sha256"
15 "encoding/hex"
16 "errors"
17 "fmt"
18 "io"
19 "io/fs"
20 "os"
21 "path/filepath"
22 "regexp"
23 "strconv"
24 "strings"
25 "time"
26 )
27
28 const (
29 // PointerVersion is the first line of every git-lfs pointer.
30 PointerVersion = "https://git-lfs.github.com/spec/v1"
31 // MaxPointerBytes bounds what could be a pointer. Anything larger is a real
32 // blob, so scanning a tree never reads a whole asset to find that out.
33 MaxPointerBytes = 1024
34 // MediaType is the git-lfs JSON content type.
35 MediaType = "application/vnd.git-lfs+json"
36 )
37
38 var (
39 ErrBadOID = errors.New("lfs: malformed object id")
40 ErrMismatch = errors.New("lfs: content does not match its object id")
41 ErrTooLarge = errors.New("lfs: object is over the size limit")
42 ErrNotFound = errors.New("lfs: object not found")
43 )
44
45 var oidRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
46
47 // ValidOID reports whether s is a sha256 object id in the form the store uses.
48 func ValidOID(s string) bool { return oidRE.MatchString(s) }
49
50 // Pointer is the committed stub: what the file says, not the bytes it stands for.
51 type Pointer struct {
52 OID string
53 Size int64
54 }
55
56 // Encode renders the canonical three-line pointer. The first line is
57 // `version <url>`, not the bare url: git-lfs's own output has the key there, and
58 // a pointer without it is not a pointer to any client.
59 func (p Pointer) Encode() []byte {
60 return []byte(fmt.Sprintf("version %s\noid sha256:%s\nsize %d\n", PointerVersion, p.OID, p.Size))
61 }
62
63 // ParsePointer reads a git-lfs pointer. It accepts unknown keys (git-lfs writes
64 // `ext-` lines) but requires the version, oid and size, and refuses anything that
65 // does not look like an object id: a pointer that cannot name its object is not
66 // a pointer, it is a corrupt file.
67 func ParsePointer(b []byte) (Pointer, bool) {
68 if len(b) == 0 || len(b) > MaxPointerBytes {
69 return Pointer{}, false
70 }
71 var (
72 p Pointer
73 oidOK bool
74 sizeOK bool
75 )
76 lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n")
77 // The first line carries the format version as a key/value pair. A pointer
78 // that does not say which version it is written in is not readable, and the
79 // URL alone is what .gitattributes uses for a filter, not what an object file
80 // starts with.
81 if len(lines) == 0 {
82 return Pointer{}, false
83 }
84 if key, value, ok := strings.Cut(lines[0], " "); !ok || key != "version" || value != PointerVersion {
85 return Pointer{}, false
86 }
87 for _, line := range lines[1:] {
88 key, value, ok := strings.Cut(line, " ")
89 if !ok {
90 return Pointer{}, false
91 }
92 switch key {
93 case "oid":
94 algo, hexPart, ok := strings.Cut(value, ":")
95 if !ok || algo != "sha256" || !ValidOID(hexPart) {
96 return Pointer{}, false
97 }
98 p.OID = hexPart
99 oidOK = true
100 case "size":
101 n, err := strconv.ParseInt(value, 10, 64)
102 if err != nil || n < 0 {
103 return Pointer{}, false
104 }
105 p.Size = n
106 sizeOK = true
107 }
108 }
109 return p, oidOK && sizeOK
110 }
111
112 // Store is the instance's content-addressed object store.
113 type Store struct {
114 // Root holds objects/ and tmp/.
115 Root string
116 // MaxBytes caps a single object. Zero means no cap, which is only useful in
117 // tests: an unbounded object directory is a denial of service waiting.
118 MaxBytes int64
119 }
120
121 func New(root string, maxBytes int64) *Store { return &Store{Root: root, MaxBytes: maxBytes} }
122
123 // ObjectPath is the on-disk location of an object. Two levels of sharding keeps
124 // any one directory readable at a million objects; the file keeps the whole id so
125 // a store can be read by eye and a sweep never has to reassemble a name.
126 func (s *Store) ObjectPath(oid string) string {
127 return filepath.Join(s.Root, "objects", oid[:2], oid[2:4], oid)
128 }
129
130 // Stat returns the stored size, or ErrNotFound.
131 func (s *Store) Stat(oid string) (int64, error) {
132 if !ValidOID(oid) {
133 return 0, ErrBadOID
134 }
135 st, err := os.Stat(s.ObjectPath(oid))
136 if err != nil {
137 if errors.Is(err, fs.ErrNotExist) {
138 return 0, ErrNotFound
139 }
140 return 0, err
141 }
142 return st.Size(), nil
143 }
144
145 // Has reports whether the object is present.
146 func (s *Store) Has(oid string) bool {
147 _, err := s.Stat(oid)
148 return err == nil
149 }
150
151 // StoredAt is when an object was written, used by the sweep's grace period.
152 func (s *Store) StoredAt(oid string) (time.Time, error) {
153 if !ValidOID(oid) {
154 return time.Time{}, ErrBadOID
155 }
156 st, err := os.Stat(s.ObjectPath(oid))
157 if err != nil {
158 if errors.Is(err, fs.ErrNotExist) {
159 return time.Time{}, ErrNotFound
160 }
161 return time.Time{}, err
162 }
163 return st.ModTime(), nil
164 }
165
166 // Open returns the object for reading with its size. The caller closes it.
167 func (s *Store) Open(oid string) (*os.File, int64, error) {
168 if !ValidOID(oid) {
169 return nil, 0, ErrBadOID
170 }
171 f, err := os.Open(s.ObjectPath(oid))
172 if err != nil {
173 if errors.Is(err, fs.ErrNotExist) {
174 return nil, 0, ErrNotFound
175 }
176 return nil, 0, err
177 }
178 st, err := f.Stat()
179 if err != nil {
180 f.Close()
181 return nil, 0, err
182 }
183 return f, st.Size(), nil
184 }
185
186 // Put streams r into the store as oid. It verifies the sha256 and the size
187 // before the object becomes visible: a store whose key does not match its
188 // content is worse than an empty one, because every reader trusts it.
189 // It returns the number of bytes written and whether the object was new.
190 func (s *Store) Put(oid string, size int64, r io.Reader) (int64, bool, error) {
191 if !ValidOID(oid) {
192 return 0, false, ErrBadOID
193 }
194 tmpDir := filepath.Join(s.Root, "tmp")
195 if err := os.MkdirAll(tmpDir, 0o755); err != nil {
196 return 0, false, err
197 }
198 if size > 0 && s.MaxBytes > 0 && size > s.MaxBytes {
199 return 0, false, fmt.Errorf("%w: %d bytes, limit %d", ErrTooLarge, size, s.MaxBytes)
200 }
201 f, err := os.CreateTemp(tmpDir, "put-*")
202 if err != nil {
203 return 0, false, err
204 }
205 tmp := f.Name()
206 defer func() {
207 f.Close()
208 os.Remove(tmp) // no-op once renamed
209 }()
210
211 limit := int64(-1)
212 if s.MaxBytes > 0 {
213 limit = s.MaxBytes + 1
214 }
215 h := sha256.New()
216 var src io.Reader = r
217 if limit >= 0 {
218 src = io.LimitReader(r, limit)
219 }
220 n, err := io.Copy(io.MultiWriter(f, h), src)
221 if err != nil {
222 return 0, false, err
223 }
224 if limit >= 0 && n > s.MaxBytes {
225 return 0, false, fmt.Errorf("%w: over %d bytes", ErrTooLarge, s.MaxBytes)
226 }
227 if got := hex.EncodeToString(h.Sum(nil)); got != oid {
228 return 0, false, fmt.Errorf("%w: got %s, want %s", ErrMismatch, got, oid)
229 }
230 if size > 0 && n != size {
231 return 0, false, fmt.Errorf("%w: received %d bytes, expected %d", ErrMismatch, n, size)
232 }
233 if err := f.Sync(); err != nil {
234 return 0, false, err
235 }
236 if err := f.Close(); err != nil {
237 return 0, false, err
238 }
239
240 dst := s.ObjectPath(oid)
241 if st, err := os.Stat(dst); err == nil && st.Size() == n {
242 return n, false, nil // deduplicated
243 }
244 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
245 return 0, false, err
246 }
247 if err := os.Rename(tmp, dst); err != nil {
248 return 0, false, err
249 }
250 return n, true, nil
251 }
252
253 // PutBytes is Put for an in-memory payload (CI logs, small artifacts).
254 func (s *Store) PutBytes(data []byte) (string, error) {
255 sum := sha256.Sum256(data)
256 oid := hex.EncodeToString(sum[:])
257 _, _, err := s.Put(oid, int64(len(data)), strings.NewReader(string(data)))
258 return oid, err
259 }
260
261 // Remove deletes an object. Missing is not an error: a sweep races with nothing
262 // that recreates an object, but two sweeps can overlap.
263 func (s *Store) Remove(oid string) error {
264 if !ValidOID(oid) {
265 return ErrBadOID
266 }
267 err := os.Remove(s.ObjectPath(oid))
268 if errors.Is(err, fs.ErrNotExist) {
269 return nil
270 }
271 return err
272 }
273
274 // Walk visits every stored object.
275 func (s *Store) Walk(fn func(oid string, size int64) error) error {
276 root := filepath.Join(s.Root, "objects")
277 return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
278 if err != nil {
279 if errors.Is(err, fs.ErrNotExist) {
280 return nil
281 }
282 return err
283 }
284 if d.IsDir() {
285 return nil
286 }
287 if !ValidOID(d.Name()) {
288 return nil
289 }
290 info, err := d.Info()
291 if err != nil {
292 return err
293 }
294 return fn(d.Name(), info.Size())
295 })
296 }
297
298 // Usage totals what the store holds, for `vole gc` and the repository page.
299 func (s *Store) Usage() (objects int, bytes int64, err error) {
300 err = s.Walk(func(_ string, size int64) error {
301 objects++
302 bytes += size
303 return nil
304 })
305 return objects, bytes, err
306 }