Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

392 commit(s) on main

Compare History
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys keenan · 59m ago 4b8c9e1d
history blame raw 7.0 KiB
1// GC sweeps the LFS store.
2//
3// The rule the sweep must never break: an object reachable from a live ref is kept. That
4// is why nothing here trusts a counter — counters drift after a crash, a failed upload or
5// an interrupted push, and a counter that is one too low deletes a release artifact.
6// Instead the sweep marks what it can prove is reachable, and deletes only what remains.
7//
8// Two kinds of root:
9//
10// - Pointer files in any tree reachable from any ref, in any repository. A pointer
11// names one object by sha256; that is the whole reference.
12// - Objects named by the run queue: archived CI logs and published run artifacts. No
13// git ref reaches them, so a sweep that only marked git would delete every finished
14// log and every build output a run published.
15//
16// Reachability is per repository, because an object is served through the repository that
17// owns it: the same bytes may be linked into two namespaces, and one repository dropping
18// its pointer must not remove the other's link. So the sweep does three things in order —
19// link what is reachable into the repository that reaches it, unlink what that repository
20// no longer reaches, and finally drop pool objects that no namespace refers to any more.
21// The middle step is what makes this pass the repair for a store that predates scoping,
22// or one whose namespace was deleted: nothing else creates the links.
23//
24// Anything unmarked and younger than the grace period is kept: an upload that lands while
25// the sweep is marking is not reachable yet, but is not garbage either.
26package forge
27
28import (
29 "fmt"
30 "time"
31
32 "rodent.software/vole/core/git"
33 "rodent.software/vole/core/lfs"
34)
35
36// Repo is one repository to scan: the name its objects are served under, and the bare
37// directory to read refs from.
38type GCRepo struct {
39 Name string
40 Dir string
41}
42
43type GCOptions struct {
44 Repos []GCRepo
45 Store *lfs.Store
46 // RunOIDs are objects named by the run queue, per repository: published run
47 // artifacts. They are not reachable from a git ref, so a sweep that only marked
48 // git would delete every build output a run published. Logs are files under the
49 // instance's log directory and are not part of this pool.
50 RunOIDs map[string][]string
51 // Grace protects recent objects that a concurrent upload has not made reachable yet.
52 Grace time.Duration
53 DryRun bool
54 // LinkOnly creates the missing links and deletes nothing. It is what an instance
55 // upgrading onto repository-scoped storage runs once, and it is the safe half of
56 // this package: it can only make more objects reachable.
57 LinkOnly bool
58 Now time.Time
59}
60
61type GCResult struct {
62 Marked int
63 Linked int // links created, which is repair work when it is not zero
64 Unlinked int
65 Swept int
66 SweptB int64
67 Within int // unmarked but inside the grace period
68 Scanned int
69 Left int64
70 Problems []string
71}
72
73// Run marks, links, unlinks, then sweeps. A repository that cannot be scanned aborts the
74// sweep: an unreadable ref means "I do not know what is reachable", and guessing costs
75// somebody's data.
76func GC(opts GCOptions) (GCResult, error) {
77 var res GCResult
78 if opts.Store == nil {
79 return res, fmt.Errorf("gc: no store")
80 }
81 if opts.Now.IsZero() {
82 opts.Now = time.Now()
83 }
84
85 // Two sets: one per repository, for the links that make an object fetchable, and the
86 // union, which is what decides whether the pool may forget the bytes.
87 reachable := make(map[string]map[string]bool, len(opts.Repos))
88 all := map[string]bool{}
89 for _, repo := range opts.Repos {
90 g := git.Open(repo.Dir)
91 blobs, err := g.AllBlobs()
92 if err != nil {
93 return res, fmt.Errorf("gc: scan %s: %w", repo.Dir, err)
94 }
95 set := map[string]bool{}
96 for _, oid := range opts.RunOIDs[repo.Name] {
97 set[oid] = true
98 }
99 for sha, size := range blobs {
100 if size > lfs.MaxPointerBytes {
101 continue // too big to be a pointer
102 }
103 head, err := g.BlobPrefix(sha, int(lfs.MaxPointerBytes))
104 if err != nil {
105 return res, fmt.Errorf("gc: read %s in %s: %w", sha, repo.Dir, err)
106 }
107 p, ok := lfs.ParsePointer(head)
108 if !ok {
109 continue
110 }
111 set[p.OID] = true
112 }
113 reachable[repo.Name] = set
114 for oid := range set {
115 all[oid] = true
116 }
117 }
118 // A run's objects belong to a repository that may not exist any more; those are
119 // still marked, so the sweep does not delete bytes a row still names.
120 for _, oids := range opts.RunOIDs {
121 for _, oid := range oids {
122 all[oid] = true
123 }
124 }
125 res.Marked = len(all)
126
127 // Link what each repository can reach, and (unless this is a repair-only pass)
128 // unlink what it cannot. Grace applies to the unlink exactly as it does to the sweep:
129 // a link created by an upload whose push has not landed yet is not garbage.
130 for _, repo := range opts.Repos {
131 set := reachable[repo.Name]
132 for oid := range set {
133 if opts.Store.Has(repo.Name, oid) {
134 continue
135 }
136 res.Linked++
137 if opts.DryRun {
138 continue
139 }
140 if err := opts.Store.Link(repo.Name, oid); err != nil {
141 res.Problems = append(res.Problems, fmt.Sprintf("%s: link %s: %v", repo.Name, oid, err))
142 }
143 }
144 if opts.LinkOnly {
145 continue
146 }
147 err := opts.Store.RepoWalk(repo.Name, func(oid string, size int64) error {
148 if set[oid] {
149 return nil
150 }
151 age, err := opts.Store.StoredAt(repo.Name, oid)
152 if err != nil {
153 res.Problems = append(res.Problems, fmt.Sprintf("%s: %s: %v", repo.Name, oid, err))
154 return nil
155 }
156 if opts.Now.Sub(age) < opts.Grace {
157 // Counted once, by the pool sweep: an object kept by grace here is kept
158 // there too unless another repository still reaches it.
159 return nil
160 }
161 res.Unlinked++
162 if opts.DryRun {
163 return nil
164 }
165 if err := opts.Store.Unlink(repo.Name, oid); err != nil {
166 res.Problems = append(res.Problems, fmt.Sprintf("%s: unlink %s: %v", repo.Name, oid, err))
167 }
168 return nil
169 })
170 if err != nil {
171 return res, fmt.Errorf("gc: reconcile links for %s: %w", repo.Name, err)
172 }
173 }
174 if opts.LinkOnly {
175 return res, nil
176 }
177
178 // Finally the pool: an object no repository reaches, and that nothing refers to any
179 // more, is the only thing that may be deleted.
180 err := opts.Store.Walk(func(oid string, size int64) error {
181 res.Scanned++
182 res.Left += size
183 if all[oid] {
184 return nil
185 }
186 age, err := opts.Store.PoolStoredAt(oid)
187 if err != nil {
188 res.Problems = append(res.Problems, fmt.Sprintf("%s: %v", oid, err))
189 return nil
190 }
191 if opts.Now.Sub(age) < opts.Grace {
192 res.Within++
193 return nil
194 }
195 if !opts.DryRun {
196 if err := opts.Store.Remove(oid); err != nil {
197 res.Problems = append(res.Problems, fmt.Sprintf("%s: %v", oid, err))
198 return nil
199 }
200 }
201 res.Swept++
202 res.SweptB += size
203 return nil
204 })
205 if err != nil {
206 return res, fmt.Errorf("gc: sweep: %w", err)
207 }
208 return res, nil
209}
210
211// Link is the repair pass: it creates the links a repository is missing and deletes
212// nothing. An instance that predates repository-scoped storage runs it once so its
213// existing objects become fetchable again; so does a namespace somebody deleted.
214func GCLink(opts GCOptions) (GCResult, error) {
215 opts.LinkOnly = true
216 opts.DryRun = false
217 return GC(opts)
218}
219