vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Hold a job's cache namespace while the job runs
a29ce1f76aaf7ebc4b24afad36c56a0f7f01254e · Keenan Nemetz · 2d ago · parent b492b557
Hold a job's cache namespace while the job runs A namespace was only safe while one job existed per runner. Two runners sharing a --dir already had the bug: the trimmer deletes whole namespaces and `--no-cache` empties one, either of which can land on a tree another process has mounted. A Go module cache that loses a file mid-build is a checksum error deep in a build, which is the case the whole-namespace eviction rule exists to avoid. A job now holds its namespace for as long as the container may be inside it — shared, so two builds of the same repository still run together — and eviction takes the same lock with LOCK_NB and leaves what is in use for a later pass. `--no-cache` takes the namespace exclusively, because emptying it afterwards must not land on somebody else's build. The runner also stops trimming at the end of each job: the cap is enforced at startup and on a timer, so eviction no longer belongs to one job's lifetime. Lock files live under <root>/.locks, mirroring the namespace, so a cache tree holds namespaces and nothing else; they are never deleted, for the reason gitcache's are not. `vole workflow run` gets the same lock, so two local runs in one checkout cannot delete each other's cache either. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/cache/cache.go +101−7
| @@ -11,7 +11,9 @@ | |||
| 11 | 11 | // | |
| 12 | 12 | // Each job sees exactly one directory. Namespaces are whole directories so | |
| 13 | 13 | // eviction never leaves half a module cache behind: a namespace is either there or | |
| 14 | // it is not. | ||
| 14 | // it is not. A job holds its namespace while it runs and eviction takes the same | ||
| 15 | // lock, so a build is never reading a tree the runner is deleting: the trimmer | ||
| 16 | // passes over what is in use and takes it on a later pass. | ||
| 15 | 17 | package cache | |
| 16 | 18 | ||
| 17 | 19 | import ( | |
| @@ -24,6 +26,7 @@ import ( | |||
| 24 | 26 | "sort" | |
| 25 | 27 | "strconv" | |
| 26 | 28 | "strings" | |
| 29 | "sync" | ||
| 27 | 30 | "time" | |
| 28 | 31 | ) | |
| 29 | 32 | ||
| @@ -64,9 +67,83 @@ func ImageKey(imageSHA, imageRef string) string { | |||
| 64 | 67 | return "ref-" + segment(name) + "-" + hex.EncodeToString(sum[:4]) | |
| 65 | 68 | } | |
| 66 | 69 | ||
| 70 | // The namespace locks. A job holds its namespace for as long as a container may be | ||
| 71 | // inside it, and eviction takes the same lock, so either a tree is not in use or it | ||
| 72 | // is one eviction cannot have. | ||
| 73 | // | ||
| 74 | // The lock files live under <root>/.locks, mirroring the namespace, rather than | ||
| 75 | // beside or inside it. Inside would mean removing a namespace removes the lock its | ||
| 76 | // holder is using; a sibling directory would sit in the cache tree, where a sweep | ||
| 77 | // walking it could mistake bookkeeping for a cache and where a `--no-cache` run | ||
| 78 | // would look like it had left something behind. A lock file is never deleted for | ||
| 79 | // the same reason gitcache's are not: removing it races with the next holder | ||
| 80 | // opening it, and a dead process releases the kernel lock when its descriptors | ||
| 81 | // close. | ||
| 82 | func lockPath(root, ns string) (string, error) { | ||
| 83 | rel, err := filepath.Rel(root, ns) | ||
| 84 | if err != nil { | ||
| 85 | return "", fmt.Errorf("lock %s: it is not under %s: %w", ns, root, err) | ||
| 86 | } | ||
| 87 | if rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { | ||
| 88 | return "", fmt.Errorf("lock %s: it is not under %s", ns, root) | ||
| 89 | } | ||
| 90 | return filepath.Join(root, ".locks", rel+".lock"), nil | ||
| 91 | } | ||
| 92 | |||
| 93 | // Lock shares the namespace with any other job that has it. Two jobs of the same | ||
| 94 | // repository, trust and image may run at once — their tool caches are built for | ||
| 95 | // that — and what they cannot survive is the tree being deleted underneath them. | ||
| 96 | func Lock(root, ns string) (func(), error) { | ||
| 97 | release, _, err := holdNamespace(root, ns, false, true) | ||
| 98 | return release, err | ||
| 99 | } | ||
| 100 | |||
| 101 | // LockExclusive owns the namespace, waiting for any job already inside it. A runner | ||
| 102 | // that does not reuse its cache takes this because it empties the namespace when | ||
| 103 | // the job ends, and that must not land on somebody else's build. | ||
| 104 | func LockExclusive(root, ns string) (func(), error) { | ||
| 105 | release, _, err := holdNamespace(root, ns, true, true) | ||
| 106 | return release, err | ||
| 107 | } | ||
| 108 | |||
| 109 | // TryLockExclusive takes the namespace exclusively or reports false when a job has | ||
| 110 | // it. Eviction uses this: a namespace in use is left for a later pass rather than | ||
| 111 | // waited on, and a lock file that cannot be opened is treated the same way, because | ||
| 112 | // skipping a deletion is always the safe answer. | ||
| 113 | func TryLockExclusive(root, ns string) (func(), bool) { | ||
| 114 | release, held, err := holdNamespace(root, ns, true, false) | ||
| 115 | if err != nil || !held { | ||
| 116 | return nil, false | ||
| 117 | } | ||
| 118 | return release, true | ||
| 119 | } | ||
| 120 | |||
| 121 | func holdNamespace(root, ns string, exclusive, wait bool) (func(), bool, error) { | ||
| 122 | path, err := lockPath(root, ns) | ||
| 123 | if err != nil { | ||
| 124 | return nil, false, err | ||
| 125 | } | ||
| 126 | if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { | ||
| 127 | return nil, false, fmt.Errorf("prepare the lock for %s: %w", ns, err) | ||
| 128 | } | ||
| 129 | f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) | ||
| 130 | if err != nil { | ||
| 131 | return nil, false, fmt.Errorf("open the lock for %s: %w", ns, err) | ||
| 132 | } | ||
| 133 | unlock, held, err := lockFile(f, exclusive, wait) | ||
| 134 | if err != nil || !held { | ||
| 135 | _ = f.Close() | ||
| 136 | return nil, false, err | ||
| 137 | } | ||
| 138 | var once sync.Once | ||
| 139 | return func() { once.Do(func() { unlock(); _ = f.Close() }) }, true, nil | ||
| 140 | } | ||
| 141 | |||
| 67 | 142 | // Trim deletes whole namespaces, least recently written first, until root holds at | |
| 68 | 143 | // most maxBytes. It returns how many bytes it removed. A namespace that cannot be | |
| 69 | // removed is skipped rather than leaving the cache in an unknown state. | ||
| 144 | // removed, or that a job has mounted, is skipped rather than leaving the cache in an | ||
| 145 | // unknown state; a skipped namespace still counts against the cap, so the next pass | ||
| 146 | // finds it again. | ||
| 70 | 147 | // | |
| 71 | 148 | // Whole namespaces rather than files: a Go module cache with one file missing is | |
| 72 | 149 | // worse than no Go module cache, because the failure is a checksum error deep in a | |
| @@ -101,7 +178,13 @@ func Trim(root string, maxBytes int64) (int64, error) { | |||
| 101 | 178 | if total-removed <= maxBytes { | |
| 102 | 179 | break | |
| 103 | 180 | } | |
| 104 | if err := forceRemoveAll(ns.path); err != nil { | ||
| 181 | release, held := TryLockExclusive(root, ns.path) | ||
| 182 | if !held { | ||
| 183 | continue // a job has it mounted; the next pass can have it | ||
| 184 | } | ||
| 185 | err := forceRemoveAll(ns.path) | ||
| 186 | release() | ||
| 187 | if err != nil { | ||
| 105 | 188 | continue | |
| 106 | 189 | } | |
| 107 | 190 | removed += ns.size | |
| @@ -111,13 +194,21 @@ func Trim(root string, maxBytes int64) (int64, error) { | |||
| 111 | 194 | ||
| 112 | 195 | // Clear removes every namespace under root. It is for a runner that does not reuse | |
| 113 | 196 | // the cache: the directory still has to be mounted, because an image may point at | |
| 114 | // it, but nothing from a previous job should be there. | ||
| 197 | // it, but nothing from a previous job should be there. A namespace a job has | ||
| 198 | // mounted is skipped, so a second runner process sharing this --dir is not emptied | ||
| 199 | // under its own build. | ||
| 115 | 200 | func Clear(root string) error { | |
| 116 | 201 | var first error | |
| 117 | 202 | for _, repo := range dirs(root) { | |
| 118 | 203 | for _, trust := range dirs(repo) { | |
| 119 | 204 | for _, image := range dirs(trust) { | |
| 120 | if err := forceRemoveAll(image); err != nil && first == nil { | ||
| 205 | release, held := TryLockExclusive(root, image) | ||
| 206 | if !held { | ||
| 207 | continue // a job has it mounted, or the lock cannot be opened | ||
| 208 | } | ||
| 209 | err := forceRemoveAll(image) | ||
| 210 | release() | ||
| 211 | if err != nil && first == nil { | ||
| 121 | 212 | first = err | |
| 122 | 213 | } | |
| 123 | 214 | } | |
| @@ -160,7 +251,10 @@ func ParseSize(s string) (int64, error) { | |||
| 160 | 251 | } | |
| 161 | 252 | ||
| 162 | 253 | // dirs lists the subdirectories of p, ignoring anything that cannot be read. A | |
| 163 | // cache is best-effort: a permission problem there must not fail a job. | ||
| 254 | // cache is best-effort: a permission problem there must not fail a job. Hidden | ||
| 255 | // directories are skipped because the runner keeps its lock files under .locks, and | ||
| 256 | // bookkeeping must never be measured or evicted as if it were a cache. Namespace | ||
| 257 | // parts never start with a dot: segment strips them. | ||
| 164 | 258 | func dirs(p string) []string { | |
| 165 | 259 | entries, err := os.ReadDir(p) | |
| 166 | 260 | if err != nil { | |
| @@ -168,7 +262,7 @@ func dirs(p string) []string { | |||
| 168 | 262 | } | |
| 169 | 263 | out := make([]string, 0, len(entries)) | |
| 170 | 264 | for _, e := range entries { | |
| 171 | if e.IsDir() { | ||
| 265 | if e.IsDir() && !strings.HasPrefix(e.Name(), ".") { | ||
| 172 | 266 | out = append(out, filepath.Join(p, e.Name())) | |
| 173 | 267 | } | |
| 174 | 268 | } | |