Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.

vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · core/lfs/lfs.go

Line attribution on main · 487 line(s)

64ac5071
Keenan Nemetz
17d ago
1 // Package lfs stores large files outside the git object database.
2 //
3 // A repository commits a small pointer file (the format git-lfs writes) in place
4 // of the bytes; the bytes live here, keyed by their sha256. The store is shared
5 // by every repository on the instance, so pushing the same release artifact in
6 // two repositories costs one copy.
7 //
8 // The store is derived, not authoritative: git is. An object that no live ref
3abbe749
Keenan Nemetz
16d ago
9 // points at is garbage, which is why `vole admin gc` marks from the refs rather than
64ac5071
Keenan Nemetz
17d ago
10 // trusting a counter that a crash could have left wrong.
11 package lfs
12
13 import (
14 "crypto/sha256"
15 "encoding/hex"
16 "errors"
17 "fmt"
18 "io"
19 "io/fs"
20 "os"
21 "path/filepath"
22 "regexp"
23 "strconv"
24 "strings"
25 "time"
26 )
27
28 const (
29 // PointerVersion is the first line of every git-lfs pointer.
30 PointerVersion = "https://git-lfs.github.com/spec/v1"
31 // MaxPointerBytes bounds what could be a pointer. Anything larger is a real
32 // blob, so scanning a tree never reads a whole asset to find that out.
33 MaxPointerBytes = 1024
34 // MediaType is the git-lfs JSON content type.
35 MediaType = "application/vnd.git-lfs+json"
36 )
37
38 var (
39 ErrBadOID = errors.New("lfs: malformed object id")
40 ErrMismatch = errors.New("lfs: content does not match its object id")
41 ErrTooLarge = errors.New("lfs: object is over the size limit")
42 ErrNotFound = errors.New("lfs: object not found")
43 )
44
45 var oidRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
46
47 // ValidOID reports whether s is a sha256 object id in the form the store uses.
48 func ValidOID(s string) bool { return oidRE.MatchString(s) }
49
50 // Pointer is the committed stub: what the file says, not the bytes it stands for.
51 type Pointer struct {
52 OID string
53 Size int64
54 }
55
56 // Encode renders the canonical three-line pointer. The first line is
57 // `version <url>`, not the bare url: git-lfs's own output has the key there, and
58 // a pointer without it is not a pointer to any client.
59 func (p Pointer) Encode() []byte {
60 return []byte(fmt.Sprintf("version %s\noid sha256:%s\nsize %d\n", PointerVersion, p.OID, p.Size))
61 }
62
63 // ParsePointer reads a git-lfs pointer. It accepts unknown keys (git-lfs writes
64 // `ext-` lines) but requires the version, oid and size, and refuses anything that
65 // does not look like an object id: a pointer that cannot name its object is not
66 // a pointer, it is a corrupt file.
67 func ParsePointer(b []byte) (Pointer, bool) {
68 if len(b) == 0 || len(b) > MaxPointerBytes {
69 return Pointer{}, false
70 }
71 var (
72 p Pointer
73 oidOK bool
74 sizeOK bool
75 )
76 lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n")
77 // The first line carries the format version as a key/value pair. A pointer
78 // that does not say which version it is written in is not readable, and the
79 // URL alone is what .gitattributes uses for a filter, not what an object file
80 // starts with.
81 if len(lines) == 0 {
82 return Pointer{}, false
83 }
84 if key, value, ok := strings.Cut(lines[0], " "); !ok || key != "version" || value != PointerVersion {
85 return Pointer{}, false
86 }
87 for _, line := range lines[1:] {
88 key, value, ok := strings.Cut(line, " ")
89 if !ok {
90 return Pointer{}, false
91 }
92 switch key {
93 case "oid":
94 algo, hexPart, ok := strings.Cut(value, ":")
95 if !ok || algo != "sha256" || !ValidOID(hexPart) {
96 return Pointer{}, false
97 }
98 p.OID = hexPart
99 oidOK = true
100 case "size":
101 n, err := strconv.ParseInt(value, 10, 64)
102 if err != nil || n < 0 {
103 return Pointer{}, false
104 }
105 p.Size = n
106 sizeOK = true
107 }
108 }
109 return p, oidOK && sizeOK
110 }
111
112 // Store is the instance's content-addressed object store.
113 type Store struct {
114 // Root holds objects/ and tmp/.
115 Root string
116 // MaxBytes caps a single object. Zero means no cap, which is only useful in
117 // tests: an unbounded object directory is a denial of service waiting.
118 MaxBytes int64
119 }
120
121 func New(root string, maxBytes int64) *Store { return &Store{Root: root, MaxBytes: maxBytes} }
122
65b88b3b
Keenan Nemetz
16d ago
123 // ObjectPath is the on-disk location of an object in the shared pool. Two levels of
124 // sharding keeps any one directory readable at a million objects; the file keeps the
125 // whole id so a store can be read by eye and a sweep never has to reassemble a name.
64ac5071
Keenan Nemetz
17d ago
126 func (s *Store) ObjectPath(oid string) string {
127 return filepath.Join(s.Root, "objects", oid[:2], oid[2:4], oid)
128 }
129
65b88b3b
Keenan Nemetz
16d ago
130 // RepoPath is the same object as one repository sees it: a hard link into that
131 // repository's namespace.
132 //
133 // Why a link at all: the pool is content-addressed, so a plain lookup by id would let
134 // anyone who knows an id read an object through any repository whose read rule they pass.
135 // A repository's namespace is the set of objects it may serve, and a fetch reads only
136 // that, so "which repository owns these bytes" stops being a database question that can
137 // be stale and becomes a fact about the filesystem. The link costs no bytes: the same
138 // artifact uploaded to two repositories is still stored once, and removing one
139 // repository's link leaves the other's data alone.
140 func (s *Store) RepoPath(repo, oid string) string {
141 return filepath.Join(s.Root, "repos", repo, oid[:2], oid[2:4], oid)
142 }
143
144 // Link gives repo its view of an object already in the pool. It is idempotent, and a
145 // no-op when the object is not stored: the caller decides whether that is an error.
146 func (s *Store) Link(repo, oid string) error {
147 if !ValidOID(oid) {
148 return ErrBadOID
149 }
150 if err := validRepo(repo); err != nil {
151 return err
152 }
153 dst := s.RepoPath(repo, oid)
154 if _, err := os.Lstat(dst); err == nil {
155 return nil
156 }
157 src := s.ObjectPath(oid)
158 if _, err := os.Stat(src); err != nil {
159 if errors.Is(err, fs.ErrNotExist) {
160 return ErrNotFound
161 }
162 return err
163 }
164 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
165 return err
166 }
167 if err := os.Link(src, dst); err != nil {
168 if errors.Is(err, fs.ErrExist) {
169 return nil
170 }
171 return err
172 }
173 return nil
174 }
175
176 // Unlink removes repo's view. The bytes survive while any other name refers to them,
177 // which is what lets two repositories share one artifact.
178 func (s *Store) Unlink(repo, oid string) error {
179 if !ValidOID(oid) {
180 return ErrBadOID
181 }
182 if err := validRepo(repo); err != nil {
183 return err
184 }
185 err := os.Remove(s.RepoPath(repo, oid))
186 if errors.Is(err, fs.ErrNotExist) {
187 return nil
188 }
189 return err
190 }
191
192 // validRepo refuses a repository name that could escape the namespace. Repository names
193 // are already restricted to [a-z0-9._-], so this is a belt for callers that are not the
194 // repository store.
195 func validRepo(repo string) error {
196 switch {
197 case repo == "":
198 return fmt.Errorf("lfs: a repository name is required")
199 case repo == "." || repo == ".." || strings.ContainsAny(repo, "/\\\x00"):
200 return fmt.Errorf("lfs: %q is not a repository name", repo)
201 }
202 return nil
203 }
204
205 // Stat returns the stored size as repo sees it, or ErrNotFound.
206 func (s *Store) Stat(repo, oid string) (int64, error) {
64ac5071
Keenan Nemetz
17d ago
207 if !ValidOID(oid) {
208 return 0, ErrBadOID
209 }
65b88b3b
Keenan Nemetz
16d ago
210 st, err := os.Stat(s.RepoPath(repo, oid))
64ac5071
Keenan Nemetz
17d ago
211 if err != nil {
212 if errors.Is(err, fs.ErrNotExist) {
213 return 0, ErrNotFound
214 }
215 return 0, err
216 }
217 return st.Size(), nil
218 }
219
65b88b3b
Keenan Nemetz
16d ago
220 // Has reports whether repo may serve the object.
221 func (s *Store) Has(repo, oid string) bool {
222 _, err := s.Stat(repo, oid)
64ac5071
Keenan Nemetz
17d ago
223 return err == nil
224 }
225
65b88b3b
Keenan Nemetz
16d ago
226 // StoredAt is when an object was written, used by the sweep's grace period. A link shares
227 // the pool object's inode, so this reads the same time either way.
228 func (s *Store) StoredAt(repo, oid string) (time.Time, error) {
64ac5071
Keenan Nemetz
17d ago
229 if !ValidOID(oid) {
230 return time.Time{}, ErrBadOID
231 }
65b88b3b
Keenan Nemetz
16d ago
232 st, err := os.Stat(s.RepoPath(repo, oid))
64ac5071
Keenan Nemetz
17d ago
233 if err != nil {
234 if errors.Is(err, fs.ErrNotExist) {
235 return time.Time{}, ErrNotFound
236 }
237 return time.Time{}, err
238 }
239 return st.ModTime(), nil
240 }
241
65b88b3b
Keenan Nemetz
16d ago
242 // Open returns the object for reading with its size, as repo sees it. The caller closes it.
243 func (s *Store) Open(repo, oid string) (*os.File, int64, error) {
64ac5071
Keenan Nemetz
17d ago
244 if !ValidOID(oid) {
245 return nil, 0, ErrBadOID
246 }
65b88b3b
Keenan Nemetz
16d ago
247 f, err := os.Open(s.RepoPath(repo, oid))
64ac5071
Keenan Nemetz
17d ago
248 if err != nil {
249 if errors.Is(err, fs.ErrNotExist) {
250 return nil, 0, ErrNotFound
251 }
252 return nil, 0, err
253 }
254 st, err := f.Stat()
255 if err != nil {
256 f.Close()
257 return nil, 0, err
258 }
259 return f, st.Size(), nil
260 }
261
65b88b3b
Keenan Nemetz
16d ago
262 // Put streams r into the store as oid on behalf of repo. It verifies the sha256 and the
263 // size before the object becomes visible: a store whose key does not match its content is
264 // worse than an empty one, because every reader trusts it.
265 //
266 // It returns the number of bytes written and whether this is new to *this repository* —
267 // which is the answer that matters, since a second repository referring to an object that
268 // is already stored costs a link and no transfer.
269 func (s *Store) Put(repo, oid string, size int64, r io.Reader) (int64, bool, error) {
64ac5071
Keenan Nemetz
17d ago
270 if !ValidOID(oid) {
271 return 0, false, ErrBadOID
272 }
65b88b3b
Keenan Nemetz
16d ago
273 if err := validRepo(repo); err != nil {
274 return 0, false, err
275 }
64ac5071
Keenan Nemetz
17d ago
276 tmpDir := filepath.Join(s.Root, "tmp")
277 if err := os.MkdirAll(tmpDir, 0o755); err != nil {
278 return 0, false, err
279 }
280 if size > 0 && s.MaxBytes > 0 && size > s.MaxBytes {
281 return 0, false, fmt.Errorf("%w: %d bytes, limit %d", ErrTooLarge, size, s.MaxBytes)
282 }
283 f, err := os.CreateTemp(tmpDir, "put-*")
284 if err != nil {
285 return 0, false, err
286 }
287 tmp := f.Name()
288 defer func() {
289 f.Close()
290 os.Remove(tmp) // no-op once renamed
291 }()
292
293 limit := int64(-1)
294 if s.MaxBytes > 0 {
295 limit = s.MaxBytes + 1
296 }
297 h := sha256.New()
298 var src io.Reader = r
299 if limit >= 0 {
300 src = io.LimitReader(r, limit)
301 }
302 n, err := io.Copy(io.MultiWriter(f, h), src)
303 if err != nil {
304 return 0, false, err
305 }
306 if limit >= 0 && n > s.MaxBytes {
307 return 0, false, fmt.Errorf("%w: over %d bytes", ErrTooLarge, s.MaxBytes)
308 }
309 if got := hex.EncodeToString(h.Sum(nil)); got != oid {
310 return 0, false, fmt.Errorf("%w: got %s, want %s", ErrMismatch, got, oid)
311 }
312 if size > 0 && n != size {
313 return 0, false, fmt.Errorf("%w: received %d bytes, expected %d", ErrMismatch, n, size)
314 }
315 if err := f.Sync(); err != nil {
316 return 0, false, err
317 }
318 if err := f.Close(); err != nil {
319 return 0, false, err
320 }
321
322 dst := s.ObjectPath(oid)
65b88b3b
Keenan Nemetz
16d ago
323 createdPool := false
64ac5071
Keenan Nemetz
17d ago
324 if st, err := os.Stat(dst); err == nil && st.Size() == n {
65b88b3b
Keenan Nemetz
16d ago
325 // Already stored: the bytes are deduplicated and only the link is new.
326 } else {
327 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
328 return 0, false, err
329 }
330 if err := os.Rename(tmp, dst); err != nil {
331 return 0, false, err
332 }
333 createdPool = true
64ac5071
Keenan Nemetz
17d ago
334 }
65b88b3b
Keenan Nemetz
16d ago
335 // The repository's view is what a reader will look up, so an object that is stored
336 // but not linked is an object nobody can fetch. A failure here fails the upload: a
337 // successful put that leaves the object unreachable is the worse outcome.
338 hadLink := s.Has(repo, oid)
339 if err := s.Link(repo, oid); err != nil {
64ac5071
Keenan Nemetz
17d ago
340 return 0, false, err
341 }
65b88b3b
Keenan Nemetz
16d ago
342 return n, !hadLink && (createdPool || !hadLink), nil
64ac5071
Keenan Nemetz
17d ago
343 }
344
345 // PutBytes is Put for an in-memory payload (CI logs, small artifacts).
65b88b3b
Keenan Nemetz
16d ago
346 func (s *Store) PutBytes(repo string, data []byte) (string, error) {
64ac5071
Keenan Nemetz
17d ago
347 sum := sha256.Sum256(data)
348 oid := hex.EncodeToString(sum[:])
65b88b3b
Keenan Nemetz
16d ago
349 _, _, err := s.Put(repo, oid, int64(len(data)), strings.NewReader(string(data)))
64ac5071
Keenan Nemetz
17d ago
350 return oid, err
351 }
352
65b88b3b
Keenan Nemetz
16d ago
353 // Remove deletes an object everywhere: the pool's bytes and every repository's link to
354 // them. Missing is not an error, because two sweeps can overlap.
355 //
356 // A sweep has usually unlinked each repository's name already — it must, to tell "still
357 // reachable in that repository" from "not" — so in practice this drops the pool name and
358 // finds nothing else. It is written the slow, complete way anyway, because `Remove` means
359 // the object is gone: a function that left a link behind would leave something fetchable
360 // that the caller believes it deleted.
64ac5071
Keenan Nemetz
17d ago
361 func (s *Store) Remove(oid string) error {
362 if !ValidOID(oid) {
363 return ErrBadOID
364 }
65b88b3b
Keenan Nemetz
16d ago
365 var firstErr error
366 names, err := os.ReadDir(filepath.Join(s.Root, "repos"))
367 if err != nil && !errors.Is(err, fs.ErrNotExist) {
368 return err
64ac5071
Keenan Nemetz
17d ago
369 }
65b88b3b
Keenan Nemetz
16d ago
370 for _, e := range names {
371 if !e.IsDir() {
372 continue
373 }
374 if err := os.Remove(s.RepoPath(e.Name(), oid)); err != nil &&
375 !errors.Is(err, fs.ErrNotExist) && firstErr == nil {
376 firstErr = err
377 }
378 }
379 if err := os.Remove(s.ObjectPath(oid)); err != nil && !errors.Is(err, fs.ErrNotExist) && firstErr == nil {
380 firstErr = err
381 }
382 return firstErr
383 }
384
385 // PoolStoredAt is when the pool object was written, whatever any repository thinks. The
386 // sweep's grace period is about the bytes, not about who has a link to them.
387 func (s *Store) PoolStoredAt(oid string) (time.Time, error) {
388 if !ValidOID(oid) {
389 return time.Time{}, ErrBadOID
390 }
391 st, err := os.Stat(s.ObjectPath(oid))
392 if err != nil {
393 if errors.Is(err, fs.ErrNotExist) {
394 return time.Time{}, ErrNotFound
395 }
396 return time.Time{}, err
397 }
398 return st.ModTime(), nil
399 }
400
401 // RepoWalk visits every object one repository may serve.
402 func (s *Store) RepoWalk(repo string, fn func(oid string, size int64) error) error {
403 if err := validRepo(repo); err != nil {
404 return err
405 }
406 root := filepath.Join(s.Root, "repos", repo)
407 return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
408 if err != nil {
409 if errors.Is(err, fs.ErrNotExist) {
410 return nil
411 }
412 return err
413 }
414 if d.IsDir() {
415 return nil
416 }
417 if !ValidOID(d.Name()) {
418 return nil
419 }
420 info, err := d.Info()
421 if err != nil {
422 return err
423 }
424 return fn(d.Name(), info.Size())
425 })
426 }
427
428 // NamespaceExists reports whether this store has ever been scoped by repository. It is
429 // how a server knows to run the repair pass once on an instance that predates it.
430 func (s *Store) NamespaceExists() bool {
431 _, err := os.Stat(filepath.Join(s.Root, "repos"))
432 return err == nil
433 }
434
435 // RenameRepo moves a repository's namespace, so a rename does not strand every object it
436 // can serve. Missing is not an error: a repository with no objects has no namespace.
437 func (s *Store) RenameRepo(oldName, newName string) error {
438 if err := validRepo(oldName); err != nil {
439 return err
440 }
441 if err := validRepo(newName); err != nil {
442 return err
443 }
444 from := filepath.Join(s.Root, "repos", oldName)
445 to := filepath.Join(s.Root, "repos", newName)
446 if _, err := os.Stat(from); err != nil {
447 if errors.Is(err, fs.ErrNotExist) {
448 return nil
449 }
450 return err
451 }
452 return os.Rename(from, to)
64ac5071
Keenan Nemetz
17d ago
453 }
454
455 // Walk visits every stored object.
456 func (s *Store) Walk(fn func(oid string, size int64) error) error {
457 root := filepath.Join(s.Root, "objects")
458 return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
459 if err != nil {
460 if errors.Is(err, fs.ErrNotExist) {
461 return nil
462 }
463 return err
464 }
465 if d.IsDir() {
466 return nil
467 }
468 if !ValidOID(d.Name()) {
469 return nil
470 }
471 info, err := d.Info()
472 if err != nil {
473 return err
474 }
475 return fn(d.Name(), info.Size())
476 })
477 }
478
3abbe749
Keenan Nemetz
16d ago
479 // Usage totals what the store holds, for `vole admin gc` and the repository page.
64ac5071
Keenan Nemetz
17d ago
480 func (s *Store) Usage() (objects int, bytes int64, err error) {
481 err = s.Walk(func(_ string, size int64) error {
482 objects++
483 bytes += size
484 return nil
485 })
486 return objects, bytes, err
487 }