vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · forge/gc.go

Line attribution on main · 218 line(s)

View file History

c5902451 Keenan Nemetz 16d ago	1	// GC sweeps the LFS store.
64ac5071 Keenan Nemetz 17d ago	2	//
65b88b3b Keenan Nemetz 16d ago	3	// The rule the sweep must never break: an object reachable from a live ref is kept. That
	4	// is why nothing here trusts a counter — counters drift after a crash, a failed upload or
	5	// an interrupted push, and a counter that is one too low deletes a release artifact.
	6	// Instead the sweep marks what it can prove is reachable, and deletes only what remains.
64ac5071 Keenan Nemetz 17d ago	7	//
	8	// Two kinds of root:
	9	//
65b88b3b Keenan Nemetz 16d ago	10	// - Pointer files in any tree reachable from any ref, in any repository. A pointer
	11	// names one object by sha256; that is the whole reference.
	12	// - Objects named by the run queue: archived CI logs and published run artifacts. No
	13	// git ref reaches them, so a sweep that only marked git would delete every finished
	14	// log and every build output a run published.
64ac5071 Keenan Nemetz 17d ago	15	//
65b88b3b Keenan Nemetz 16d ago	16	// Reachability is per repository, because an object is served through the repository that
	17	// owns it: the same bytes may be linked into two namespaces, and one repository dropping
	18	// its pointer must not remove the other's link. So the sweep does three things in order —
	19	// link what is reachable into the repository that reaches it, unlink what that repository
	20	// no longer reaches, and finally drop pool objects that no namespace refers to any more.
	21	// The middle step is what makes this pass the repair for a store that predates scoping,
	22	// or one whose namespace was deleted: nothing else creates the links.
	23	//
	24	// Anything unmarked and younger than the grace period is kept: an upload that lands while
	25	// the sweep is marking is not reachable yet, but is not garbage either.
c5902451 Keenan Nemetz 16d ago	26	package forge
64ac5071 Keenan Nemetz 17d ago	27
	28	import (
	29	"fmt"
	30	"time"
	31
d37d301f Keenan Nemetz 17h ago	32	"rodent.software/vole/core/git"
d37d301f Keenan Nemetz 17h ago	33	"rodent.software/vole/core/lfs"
64ac5071 Keenan Nemetz 17d ago	34	)
64ac5071 Keenan Nemetz 17d ago	35
65b88b3b Keenan Nemetz 16d ago	36	// Repo is one repository to scan: the name its objects are served under, and the bare
65b88b3b Keenan Nemetz 16d ago	37	// directory to read refs from.
c5902451 Keenan Nemetz 16d ago	38	type GCRepo struct {
65b88b3b Keenan Nemetz 16d ago	39	Name string
	40	Dir string
	41	}
	42
c5902451 Keenan Nemetz 16d ago	43	type GCOptions struct {
c5902451 Keenan Nemetz 16d ago	44	Repos []GCRepo
64ac5071 Keenan Nemetz 17d ago	45	Store *lfs.Store
5ece70a5 Keenan Nemetz 10d ago	46	// RunOIDs are objects named by the run queue, per repository: published run
	47	// artifacts. They are not reachable from a git ref, so a sweep that only marked
	48	// git would delete every build output a run published. Logs are files under the
	49	// instance's log directory and are not part of this pool.
65b88b3b Keenan Nemetz 16d ago	50	RunOIDs map[string][]string
65b88b3b Keenan Nemetz 16d ago	51	// Grace protects recent objects that a concurrent upload has not made reachable yet.
64ac5071 Keenan Nemetz 17d ago	52	Grace time.Duration
64ac5071 Keenan Nemetz 17d ago	53	DryRun bool
65b88b3b Keenan Nemetz 16d ago	54	// LinkOnly creates the missing links and deletes nothing. It is what an instance
	55	// upgrading onto repository-scoped storage runs once, and it is the safe half of
	56	// this package: it can only make more objects reachable.
	57	LinkOnly bool
	58	Now time.Time
64ac5071 Keenan Nemetz 17d ago	59	}
64ac5071 Keenan Nemetz 17d ago	60
c5902451 Keenan Nemetz 16d ago	61	type GCResult struct {
64ac5071 Keenan Nemetz 17d ago	62	Marked int
65b88b3b Keenan Nemetz 16d ago	63	Linked int // links created, which is repair work when it is not zero
65b88b3b Keenan Nemetz 16d ago	64	Unlinked int
64ac5071 Keenan Nemetz 17d ago	65	Swept int
	66	SweptB int64
	67	Within int // unmarked but inside the grace period
	68	Scanned int
	69	Left int64
	70	Problems []string
	71	}
	72
65b88b3b Keenan Nemetz 16d ago	73	// Run marks, links, unlinks, then sweeps. A repository that cannot be scanned aborts the
65b88b3b Keenan Nemetz 16d ago	74	// sweep: an unreadable ref means "I do not know what is reachable", and guessing costs
64ac5071 Keenan Nemetz 17d ago	75	// somebody's data.
c5902451 Keenan Nemetz 16d ago	76	func GC(opts GCOptions) (GCResult, error) {
c5902451 Keenan Nemetz 16d ago	77	var res GCResult
64ac5071 Keenan Nemetz 17d ago	78	if opts.Store == nil {
	79	return res, fmt.Errorf("gc: no store")
	80	}
	81	if opts.Now.IsZero() {
	82	opts.Now = time.Now()
	83	}
	84
65b88b3b Keenan Nemetz 16d ago	85	// Two sets: one per repository, for the links that make an object fetchable, and the
	86	// union, which is what decides whether the pool may forget the bytes.
	87	reachable := make(map[string]map[string]bool, len(opts.Repos))
	88	all := map[string]bool{}
	89	for _, repo := range opts.Repos {
c5902451 Keenan Nemetz 16d ago	90	g := git.Open(repo.Dir)
64ac5071 Keenan Nemetz 17d ago	91	blobs, err := g.AllBlobs()
64ac5071 Keenan Nemetz 17d ago	92	if err != nil {
65b88b3b Keenan Nemetz 16d ago	93	return res, fmt.Errorf("gc: scan %s: %w", repo.Dir, err)
	94	}
	95	set := map[string]bool{}
	96	for _, oid := range opts.RunOIDs[repo.Name] {
	97	set[oid] = true
64ac5071 Keenan Nemetz 17d ago	98	}
	99	for sha, size := range blobs {
	100	if size > lfs.MaxPointerBytes {
	101	continue // too big to be a pointer
	102	}
	103	head, err := g.BlobPrefix(sha, int(lfs.MaxPointerBytes))
	104	if err != nil {
65b88b3b Keenan Nemetz 16d ago	105	return res, fmt.Errorf("gc: read %s in %s: %w", sha, repo.Dir, err)
64ac5071 Keenan Nemetz 17d ago	106	}
	107	p, ok := lfs.ParsePointer(head)
	108	if !ok {
	109	continue
	110	}
65b88b3b Keenan Nemetz 16d ago	111	set[p.OID] = true
	112	}
	113	reachable[repo.Name] = set
	114	for oid := range set {
	115	all[oid] = true
	116	}
	117	}
	118	// A run's objects belong to a repository that may not exist any more; those are
	119	// still marked, so the sweep does not delete bytes a row still names.
	120	for _, oids := range opts.RunOIDs {
	121	for _, oid := range oids {
	122	all[oid] = true
	123	}
	124	}
	125	res.Marked = len(all)
	126
	127	// Link what each repository can reach, and (unless this is a repair-only pass)
	128	// unlink what it cannot. Grace applies to the unlink exactly as it does to the sweep:
	129	// a link created by an upload whose push has not landed yet is not garbage.
	130	for _, repo := range opts.Repos {
	131	set := reachable[repo.Name]
	132	for oid := range set {
	133	if opts.Store.Has(repo.Name, oid) {
	134	continue
	135	}
	136	res.Linked++
	137	if opts.DryRun {
	138	continue
	139	}
	140	if err := opts.Store.Link(repo.Name, oid); err != nil {
	141	res.Problems = append(res.Problems, fmt.Sprintf("%s: link %s: %v", repo.Name, oid, err))
	142	}
	143	}
	144	if opts.LinkOnly {
	145	continue
	146	}
	147	err := opts.Store.RepoWalk(repo.Name, func(oid string, size int64) error {
	148	if set[oid] {
	149	return nil
	150	}
	151	age, err := opts.Store.StoredAt(repo.Name, oid)
	152	if err != nil {
	153	res.Problems = append(res.Problems, fmt.Sprintf("%s: %s: %v", repo.Name, oid, err))
	154	return nil
	155	}
	156	if opts.Now.Sub(age) < opts.Grace {
	157	// Counted once, by the pool sweep: an object kept by grace here is kept
	158	// there too unless another repository still reaches it.
	159	return nil
	160	}
	161	res.Unlinked++
	162	if opts.DryRun {
	163	return nil
	164	}
	165	if err := opts.Store.Unlink(repo.Name, oid); err != nil {
	166	res.Problems = append(res.Problems, fmt.Sprintf("%s: unlink %s: %v", repo.Name, oid, err))
	167	}
	168	return nil
	169	})
	170	if err != nil {
	171	return res, fmt.Errorf("gc: reconcile links for %s: %w", repo.Name, err)
64ac5071 Keenan Nemetz 17d ago	172	}
64ac5071 Keenan Nemetz 17d ago	173	}
65b88b3b Keenan Nemetz 16d ago	174	if opts.LinkOnly {
	175	return res, nil
	176	}
64ac5071 Keenan Nemetz 17d ago	177
65b88b3b Keenan Nemetz 16d ago	178	// Finally the pool: an object no repository reaches, and that nothing refers to any
65b88b3b Keenan Nemetz 16d ago	179	// more, is the only thing that may be deleted.
64ac5071 Keenan Nemetz 17d ago	180	err := opts.Store.Walk(func(oid string, size int64) error {
	181	res.Scanned++
	182	res.Left += size
65b88b3b Keenan Nemetz 16d ago	183	if all[oid] {
64ac5071 Keenan Nemetz 17d ago	184	return nil
64ac5071 Keenan Nemetz 17d ago	185	}
65b88b3b Keenan Nemetz 16d ago	186	age, err := opts.Store.PoolStoredAt(oid)
64ac5071 Keenan Nemetz 17d ago	187	if err != nil {
	188	res.Problems = append(res.Problems, fmt.Sprintf("%s: %v", oid, err))
	189	return nil
	190	}
	191	if opts.Now.Sub(age) < opts.Grace {
	192	res.Within++
	193	return nil
	194	}
65b88b3b Keenan Nemetz 16d ago	195	if !opts.DryRun {
	196	if err := opts.Store.Remove(oid); err != nil {
	197	res.Problems = append(res.Problems, fmt.Sprintf("%s: %v", oid, err))
	198	return nil
	199	}
64ac5071 Keenan Nemetz 17d ago	200	}
	201	res.Swept++
	202	res.SweptB += size
	203	return nil
	204	})
	205	if err != nil {
	206	return res, fmt.Errorf("gc: sweep: %w", err)
	207	}
	208	return res, nil
	209	}
65b88b3b Keenan Nemetz 16d ago	210
	211	// Link is the repair pass: it creates the links a repository is missing and deletes
	212	// nothing. An instance that predates repository-scoped storage runs it once so its
	213	// existing objects become fetchable again; so does a namespace somebody deleted.
c5902451 Keenan Nemetz 16d ago	214	func GCLink(opts GCOptions) (GCResult, error) {
65b88b3b Keenan Nemetz 16d ago	215	opts.LinkOnly = true
65b88b3b Keenan Nemetz 16d ago	216	opts.DryRun = false
c5902451 Keenan Nemetz 16d ago	217	return GC(opts)
65b88b3b Keenan Nemetz 16d ago	218	}