vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/gc/gc.go +123

@@ -0,0 +1,123 @@
	1	// Package gc sweeps the LFS store.
	2	//
	3	// The rule the sweep must never break: an object reachable from a live ref is
	4	// kept. That is why nothing here trusts a counter — counters drift after a crash,
	5	// a failed upload or an interrupted push, and a counter that is one too low
	6	// deletes a release artifact. Instead the sweep marks what it can prove is
	7	// reachable, and deletes only what remains.
	8	//
	9	// Two kinds of root:
	10	//
	11	// - Pointer files in any tree reachable from any ref, in any repository. A
	12	// pointer names one object by sha256; that is the whole reference.
	13	// - Archived CI logs recorded in the run queue. No git ref reaches them, so a
	14	// sweep that only marked git would delete every finished log.
	15	//
	16	// Anything unmarked and younger than the grace period is kept: an upload that
	17	// lands while the sweep is marking is not reachable yet, but is not garbage
	18	// either.
	19	package gc
	20
	21	import (
	22	"fmt"
	23	"time"
	24
	25	"github.com/rodent-software/vole/internal/gitx"
	26	"github.com/rodent-software/vole/internal/lfs"
	27	)
	28
	29	type Options struct {
	30	// Repos are bare repository directories.
	31	Repos []string
	32	Store *lfs.Store
	33	// LogOIDs are objects referenced by run records.
	34	LogOIDs []string
	35	// Grace protects recent objects that a concurrent upload has not made
	36	// reachable yet.
	37	Grace time.Duration
	38	DryRun bool
	39	Now time.Time
	40	}
	41
	42	type Result struct {
	43	Marked int
	44	Swept int
	45	SweptB int64
	46	Within int // unmarked but inside the grace period
	47	Scanned int
	48	Left int64
	49	Problems []string
	50	}
	51
	52	// Run marks, then sweeps. A repository that cannot be scanned aborts the sweep:
	53	// an unreadable ref means "I do not know what is reachable", and guessing costs
	54	// somebody's data.
	55	func Run(opts Options) (Result, error) {
	56	var res Result
	57	if opts.Store == nil {
	58	return res, fmt.Errorf("gc: no store")
	59	}
	60	if opts.Now.IsZero() {
	61	opts.Now = time.Now()
	62	}
	63	marked := map[string]bool{}
	64	for _, oid := range opts.LogOIDs {
	65	marked[oid] = true
	66	}
	67
	68	for _, dir := range opts.Repos {
	69	g := gitx.Open(dir)
	70	blobs, err := g.AllBlobs()
	71	if err != nil {
	72	return res, fmt.Errorf("gc: scan %s: %w", dir, err)
	73	}
	74	for sha, size := range blobs {
	75	if size > lfs.MaxPointerBytes {
	76	continue // too big to be a pointer
	77	}
	78	head, err := g.BlobPrefix(sha, int(lfs.MaxPointerBytes))
	79	if err != nil {
	80	return res, fmt.Errorf("gc: read %s in %s: %w", sha, dir, err)
	81	}
	82	p, ok := lfs.ParsePointer(head)
	83	if !ok {
	84	continue
	85	}
	86	marked[p.OID] = true
	87	}
	88	}
	89	res.Marked = len(marked)
	90
	91	err := opts.Store.Walk(func(oid string, size int64) error {
	92	res.Scanned++
	93	res.Left += size
	94	if marked[oid] {
	95	return nil
	96	}
	97	age, err := opts.Store.StoredAt(oid)
	98	if err != nil {
	99	res.Problems = append(res.Problems, fmt.Sprintf("%s: %v", oid, err))
	100	return nil
	101	}
	102	if opts.Now.Sub(age) < opts.Grace {
	103	res.Within++
	104	return nil
	105	}
	106	if opts.DryRun {
	107	res.Swept++
	108	res.SweptB += size
	109	return nil
	110	}
	111	if err := opts.Store.Remove(oid); err != nil {
	112	res.Problems = append(res.Problems, fmt.Sprintf("%s: %v", oid, err))
	113	return nil
	114	}
	115	res.Swept++
	116	res.SweptB += size
	117	return nil
	118	})
	119	if err != nil {
	120	return res, fmt.Errorf("gc: sweep: %w", err)
	121	}
	122	return res, nil
	123	}