Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

392 commit(s) on main

Compare History
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys keenan · 1h ago 4b8c9e1d
history blame raw 14.6 KiB
1// Package lfs stores large files outside the git object database.
2//
3// A repository commits a small pointer file (the format git-lfs writes) in place
4// of the bytes; the bytes live here, keyed by their sha256. The store is shared
5// by every repository on the instance, so pushing the same release artifact in
6// two repositories costs one copy.
7//
8// The store is derived, not authoritative: git is. An object that no live ref
9// points at is garbage, which is why `vole admin gc` marks from the refs rather than
10// trusting a counter that a crash could have left wrong.
11package lfs
12
13import (
14 "crypto/sha256"
15 "encoding/hex"
16 "errors"
17 "fmt"
18 "io"
19 "io/fs"
20 "os"
21 "path/filepath"
22 "regexp"
23 "strconv"
24 "strings"
25 "time"
26)
27
28const (
29 // PointerVersion is the first line of every git-lfs pointer.
30 PointerVersion = "https://git-lfs.github.com/spec/v1"
31 // MaxPointerBytes bounds what could be a pointer. Anything larger is a real
32 // blob, so scanning a tree never reads a whole asset to find that out.
33 MaxPointerBytes = 1024
34 // MediaType is the git-lfs JSON content type.
35 MediaType = "application/vnd.git-lfs+json"
36)
37
38var (
39 ErrBadOID = errors.New("lfs: malformed object id")
40 ErrMismatch = errors.New("lfs: content does not match its object id")
41 ErrTooLarge = errors.New("lfs: object is over the size limit")
42 ErrNotFound = errors.New("lfs: object not found")
43)
44
45var oidRE = regexp.MustCompile(`^[0-9a-f]{64}$`)
46
47// ValidOID reports whether s is a sha256 object id in the form the store uses.
48func ValidOID(s string) bool { return oidRE.MatchString(s) }
49
50// Pointer is the committed stub: what the file says, not the bytes it stands for.
51type Pointer struct {
52 OID string
53 Size int64
54}
55
56// Encode renders the canonical three-line pointer. The first line is
57// `version <url>`, not the bare url: git-lfs's own output has the key there, and
58// a pointer without it is not a pointer to any client.
59func (p Pointer) Encode() []byte {
60 return []byte(fmt.Sprintf("version %s\noid sha256:%s\nsize %d\n", PointerVersion, p.OID, p.Size))
61}
62
63// ParsePointer reads a git-lfs pointer. It accepts unknown keys (git-lfs writes
64// `ext-` lines) but requires the version, oid and size, and refuses anything that
65// does not look like an object id: a pointer that cannot name its object is not
66// a pointer, it is a corrupt file.
67func ParsePointer(b []byte) (Pointer, bool) {
68 if len(b) == 0 || len(b) > MaxPointerBytes {
69 return Pointer{}, false
70 }
71 var (
72 p Pointer
73 oidOK bool
74 sizeOK bool
75 )
76 lines := strings.Split(strings.TrimRight(string(b), "\n"), "\n")
77 // The first line carries the format version as a key/value pair. A pointer
78 // that does not say which version it is written in is not readable, and the
79 // URL alone is what .gitattributes uses for a filter, not what an object file
80 // starts with.
81 if len(lines) == 0 {
82 return Pointer{}, false
83 }
84 if key, value, ok := strings.Cut(lines[0], " "); !ok || key != "version" || value != PointerVersion {
85 return Pointer{}, false
86 }
87 for _, line := range lines[1:] {
88 key, value, ok := strings.Cut(line, " ")
89 if !ok {
90 return Pointer{}, false
91 }
92 switch key {
93 case "oid":
94 algo, hexPart, ok := strings.Cut(value, ":")
95 if !ok || algo != "sha256" || !ValidOID(hexPart) {
96 return Pointer{}, false
97 }
98 p.OID = hexPart
99 oidOK = true
100 case "size":
101 n, err := strconv.ParseInt(value, 10, 64)
102 if err != nil || n < 0 {
103 return Pointer{}, false
104 }
105 p.Size = n
106 sizeOK = true
107 }
108 }
109 return p, oidOK && sizeOK
110}
111
112// Store is the instance's content-addressed object store.
113type Store struct {
114 // Root holds objects/ and tmp/.
115 Root string
116 // MaxBytes caps a single object. Zero means no cap, which is only useful in
117 // tests: an unbounded object directory is a denial of service waiting.
118 MaxBytes int64
119}
120
121func New(root string, maxBytes int64) *Store { return &Store{Root: root, MaxBytes: maxBytes} }
122
123// ObjectPath is the on-disk location of an object in the shared pool. Two levels of
124// sharding keeps any one directory readable at a million objects; the file keeps the
125// whole id so a store can be read by eye and a sweep never has to reassemble a name.
126func (s *Store) ObjectPath(oid string) string {
127 return filepath.Join(s.Root, "objects", oid[:2], oid[2:4], oid)
128}
129
130// RepoPath is the same object as one repository sees it: a hard link into that
131// repository's namespace.
132//
133// Why a link at all: the pool is content-addressed, so a plain lookup by id would let
134// anyone who knows an id read an object through any repository whose read rule they pass.
135// A repository's namespace is the set of objects it may serve, and a fetch reads only
136// that, so "which repository owns these bytes" stops being a database question that can
137// be stale and becomes a fact about the filesystem. The link costs no bytes: the same
138// artifact uploaded to two repositories is still stored once, and removing one
139// repository's link leaves the other's data alone.
140func (s *Store) RepoPath(repo, oid string) string {
141 return filepath.Join(s.Root, "repos", repo, oid[:2], oid[2:4], oid)
142}
143
144// Link gives repo its view of an object already in the pool. It is idempotent, and a
145// no-op when the object is not stored: the caller decides whether that is an error.
146func (s *Store) Link(repo, oid string) error {
147 if !ValidOID(oid) {
148 return ErrBadOID
149 }
150 if err := validRepo(repo); err != nil {
151 return err
152 }
153 dst := s.RepoPath(repo, oid)
154 if _, err := os.Lstat(dst); err == nil {
155 return nil
156 }
157 src := s.ObjectPath(oid)
158 if _, err := os.Stat(src); err != nil {
159 if errors.Is(err, fs.ErrNotExist) {
160 return ErrNotFound
161 }
162 return err
163 }
164 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
165 return err
166 }
167 if err := os.Link(src, dst); err != nil {
168 if errors.Is(err, fs.ErrExist) {
169 return nil
170 }
171 return err
172 }
173 return nil
174}
175
176// Unlink removes repo's view. The bytes survive while any other name refers to them,
177// which is what lets two repositories share one artifact.
178func (s *Store) Unlink(repo, oid string) error {
179 if !ValidOID(oid) {
180 return ErrBadOID
181 }
182 if err := validRepo(repo); err != nil {
183 return err
184 }
185 err := os.Remove(s.RepoPath(repo, oid))
186 if errors.Is(err, fs.ErrNotExist) {
187 return nil
188 }
189 return err
190}
191
192// validRepo refuses a repository name that could escape the namespace. Repository names
193// are already restricted to [a-z0-9._-], so this is a belt for callers that are not the
194// repository store.
195func validRepo(repo string) error {
196 switch {
197 case repo == "":
198 return fmt.Errorf("lfs: a repository name is required")
199 case repo == "." || repo == ".." || strings.ContainsAny(repo, "/\\\x00"):
200 return fmt.Errorf("lfs: %q is not a repository name", repo)
201 }
202 return nil
203}
204
205// Stat returns the stored size as repo sees it, or ErrNotFound.
206func (s *Store) Stat(repo, oid string) (int64, error) {
207 if !ValidOID(oid) {
208 return 0, ErrBadOID
209 }
210 st, err := os.Stat(s.RepoPath(repo, oid))
211 if err != nil {
212 if errors.Is(err, fs.ErrNotExist) {
213 return 0, ErrNotFound
214 }
215 return 0, err
216 }
217 return st.Size(), nil
218}
219
220// Has reports whether repo may serve the object.
221func (s *Store) Has(repo, oid string) bool {
222 _, err := s.Stat(repo, oid)
223 return err == nil
224}
225
226// StoredAt is when an object was written, used by the sweep's grace period. A link shares
227// the pool object's inode, so this reads the same time either way.
228func (s *Store) StoredAt(repo, oid string) (time.Time, error) {
229 if !ValidOID(oid) {
230 return time.Time{}, ErrBadOID
231 }
232 st, err := os.Stat(s.RepoPath(repo, oid))
233 if err != nil {
234 if errors.Is(err, fs.ErrNotExist) {
235 return time.Time{}, ErrNotFound
236 }
237 return time.Time{}, err
238 }
239 return st.ModTime(), nil
240}
241
242// Open returns the object for reading with its size, as repo sees it. The caller closes it.
243func (s *Store) Open(repo, oid string) (*os.File, int64, error) {
244 if !ValidOID(oid) {
245 return nil, 0, ErrBadOID
246 }
247 f, err := os.Open(s.RepoPath(repo, oid))
248 if err != nil {
249 if errors.Is(err, fs.ErrNotExist) {
250 return nil, 0, ErrNotFound
251 }
252 return nil, 0, err
253 }
254 st, err := f.Stat()
255 if err != nil {
256 f.Close()
257 return nil, 0, err
258 }
259 return f, st.Size(), nil
260}
261
262// Put streams r into the store as oid on behalf of repo. It verifies the sha256 and the
263// size before the object becomes visible: a store whose key does not match its content is
264// worse than an empty one, because every reader trusts it.
265//
266// It returns the number of bytes written and whether this is new to *this repository* —
267// which is the answer that matters, since a second repository referring to an object that
268// is already stored costs a link and no transfer.
269func (s *Store) Put(repo, oid string, size int64, r io.Reader) (int64, bool, error) {
270 if !ValidOID(oid) {
271 return 0, false, ErrBadOID
272 }
273 if err := validRepo(repo); err != nil {
274 return 0, false, err
275 }
276 tmpDir := filepath.Join(s.Root, "tmp")
277 if err := os.MkdirAll(tmpDir, 0o755); err != nil {
278 return 0, false, err
279 }
280 if size > 0 && s.MaxBytes > 0 && size > s.MaxBytes {
281 return 0, false, fmt.Errorf("%w: %d bytes, limit %d", ErrTooLarge, size, s.MaxBytes)
282 }
283 f, err := os.CreateTemp(tmpDir, "put-*")
284 if err != nil {
285 return 0, false, err
286 }
287 tmp := f.Name()
288 defer func() {
289 f.Close()
290 os.Remove(tmp) // no-op once renamed
291 }()
292
293 limit := int64(-1)
294 if s.MaxBytes > 0 {
295 limit = s.MaxBytes + 1
296 }
297 h := sha256.New()
298 var src io.Reader = r
299 if limit >= 0 {
300 src = io.LimitReader(r, limit)
301 }
302 n, err := io.Copy(io.MultiWriter(f, h), src)
303 if err != nil {
304 return 0, false, err
305 }
306 if limit >= 0 && n > s.MaxBytes {
307 return 0, false, fmt.Errorf("%w: over %d bytes", ErrTooLarge, s.MaxBytes)
308 }
309 if got := hex.EncodeToString(h.Sum(nil)); got != oid {
310 return 0, false, fmt.Errorf("%w: got %s, want %s", ErrMismatch, got, oid)
311 }
312 if size > 0 && n != size {
313 return 0, false, fmt.Errorf("%w: received %d bytes, expected %d", ErrMismatch, n, size)
314 }
315 if err := f.Sync(); err != nil {
316 return 0, false, err
317 }
318 if err := f.Close(); err != nil {
319 return 0, false, err
320 }
321
322 dst := s.ObjectPath(oid)
323 createdPool := false
324 if st, err := os.Stat(dst); err == nil && st.Size() == n {
325 // Already stored: the bytes are deduplicated and only the link is new.
326 } else {
327 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
328 return 0, false, err
329 }
330 if err := os.Rename(tmp, dst); err != nil {
331 return 0, false, err
332 }
333 createdPool = true
334 }
335 // The repository's view is what a reader will look up, so an object that is stored
336 // but not linked is an object nobody can fetch. A failure here fails the upload: a
337 // successful put that leaves the object unreachable is the worse outcome.
338 hadLink := s.Has(repo, oid)
339 if err := s.Link(repo, oid); err != nil {
340 return 0, false, err
341 }
342 return n, !hadLink && (createdPool || !hadLink), nil
343}
344
345// PutBytes is Put for an in-memory payload (CI logs, small artifacts).
346func (s *Store) PutBytes(repo string, data []byte) (string, error) {
347 sum := sha256.Sum256(data)
348 oid := hex.EncodeToString(sum[:])
349 _, _, err := s.Put(repo, oid, int64(len(data)), strings.NewReader(string(data)))
350 return oid, err
351}
352
353// Remove deletes an object everywhere: the pool's bytes and every repository's link to
354// them. Missing is not an error, because two sweeps can overlap.
355//
356// A sweep has usually unlinked each repository's name already — it must, to tell "still
357// reachable in that repository" from "not" — so in practice this drops the pool name and
358// finds nothing else. It is written the slow, complete way anyway, because `Remove` means
359// the object is gone: a function that left a link behind would leave something fetchable
360// that the caller believes it deleted.
361func (s *Store) Remove(oid string) error {
362 if !ValidOID(oid) {
363 return ErrBadOID
364 }
365 var firstErr error
366 names, err := os.ReadDir(filepath.Join(s.Root, "repos"))
367 if err != nil && !errors.Is(err, fs.ErrNotExist) {
368 return err
369 }
370 for _, e := range names {
371 if !e.IsDir() {
372 continue
373 }
374 if err := os.Remove(s.RepoPath(e.Name(), oid)); err != nil &&
375 !errors.Is(err, fs.ErrNotExist) && firstErr == nil {
376 firstErr = err
377 }
378 }
379 if err := os.Remove(s.ObjectPath(oid)); err != nil && !errors.Is(err, fs.ErrNotExist) && firstErr == nil {
380 firstErr = err
381 }
382 return firstErr
383}
384
385// PoolStoredAt is when the pool object was written, whatever any repository thinks. The
386// sweep's grace period is about the bytes, not about who has a link to them.
387func (s *Store) PoolStoredAt(oid string) (time.Time, error) {
388 if !ValidOID(oid) {
389 return time.Time{}, ErrBadOID
390 }
391 st, err := os.Stat(s.ObjectPath(oid))
392 if err != nil {
393 if errors.Is(err, fs.ErrNotExist) {
394 return time.Time{}, ErrNotFound
395 }
396 return time.Time{}, err
397 }
398 return st.ModTime(), nil
399}
400
401// RepoWalk visits every object one repository may serve.
402func (s *Store) RepoWalk(repo string, fn func(oid string, size int64) error) error {
403 if err := validRepo(repo); err != nil {
404 return err
405 }
406 root := filepath.Join(s.Root, "repos", repo)
407 return filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
408 if err != nil {
409 if errors.Is(err, fs.ErrNotExist) {
410 return nil
411 }
412 return err
413 }
414 if d.IsDir() {
415 return nil
416 }
417 if !ValidOID(d.Name()) {
418 return nil
419 }
420 info, err := d.Info()
421 if err != nil {
422 return err
423 }
424 return fn(d.Name(), info.Size())
425 })
426}
427
428// NamespaceExists reports whether this store has ever been scoped by repository. It is
429// how a server knows to run the repair pass once on an instance that predates it.
430func (s *Store) NamespaceExists() bool {
431 _, err := os.Stat(filepath.Join(s.Root, "repos"))
432 return err == nil
433}
434
435// RenameRepo moves a repository's namespace, so a rename does not strand every object it
436// can serve. Missing is not an error: a repository with no objects has no namespace.
437func (s *Store) RenameRepo(oldName, newName string) error {
438 if err := validRepo(oldName); err != nil {
439 return err
440 }
441 if err := validRepo(newName); err != nil {
442 return err
443 }
444 from := filepath.Join(s.Root, "repos", oldName)
445 to := filepath.Join(s.Root, "repos", newName)
446 if _, err := os.Stat(from); err != nil {
447 if errors.Is(err, fs.ErrNotExist) {
448 return nil
449 }
450 return err
451 }
452 return os.Rename(from, to)
453}
454
455// Walk visits every stored object.
456func (s *Store) Walk(fn func(oid string, size int64) error) error {
457 root := filepath.Join(s.Root, "objects")
458 return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
459 if err != nil {
460 if errors.Is(err, fs.ErrNotExist) {
461 return nil
462 }
463 return err
464 }
465 if d.IsDir() {
466 return nil
467 }
468 if !ValidOID(d.Name()) {
469 return nil
470 }
471 info, err := d.Info()
472 if err != nil {
473 return err
474 }
475 return fn(d.Name(), info.Size())
476 })
477}
478
479// Usage totals what the store holds, for `vole admin gc` and the repository page.
480func (s *Store) Usage() (objects int, bytes int64, err error) {
481 err = s.Walk(func(_ string, size int64) error {
482 objects++
483 bytes += size
484 return nil
485 })
486 return objects, bytes, err
487}
488