vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Hold a job's cache namespace while the job runs
a29ce1f76aaf7ebc4b24afad36c56a0f7f01254e · Keenan Nemetz · 2d ago · parent b492b557
Hold a job's cache namespace while the job runs A namespace was only safe while one job existed per runner. Two runners sharing a --dir already had the bug: the trimmer deletes whole namespaces and `--no-cache` empties one, either of which can land on a tree another process has mounted. A Go module cache that loses a file mid-build is a checksum error deep in a build, which is the case the whole-namespace eviction rule exists to avoid. A job now holds its namespace for as long as the container may be inside it — shared, so two builds of the same repository still run together — and eviction takes the same lock with LOCK_NB and leaves what is in use for a later pass. `--no-cache` takes the namespace exclusively, because emptying it afterwards must not land on somebody else's build. The runner also stops trimming at the end of each job: the cap is enforced at startup and on a timer, so eviction no longer belongs to one job's lifetime. Lock files live under <root>/.locks, mirroring the namespace, so a cache tree holds namespaces and nothing else; they are never deleted, for the reason gitcache's are not. `vole workflow run` gets the same lock, so two local runs in one checkout cannot delete each other's cache either. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/cache/cache_test.go +192−2
| @@ -3,6 +3,7 @@ package cache | |||
| 3 | 3 | import ( | |
| 4 | 4 | "os" | |
| 5 | 5 | "path/filepath" | |
| 6 | "strings" | ||
| 6 | 7 | "testing" | |
| 7 | 8 | "time" | |
| 8 | 9 | ) | |
| @@ -150,9 +151,29 @@ func TestClearRemovesEverything(t *testing.T) { | |||
| 150 | 151 | if err := Clear(root); err != nil { | |
| 151 | 152 | t.Fatalf("clear: %v", err) | |
| 152 | 153 | } | |
| 153 | if matches, _ := filepath.Glob(filepath.Join(root, "*", "*", "*")); len(matches) != 0 { | ||
| 154 | t.Fatalf("namespaces survived Clear: %v", matches) | ||
| 154 | // The lock files under .locks are bookkeeping, not cache content: they stay, | ||
| 155 | // because removing one races with the next holder opening it. Everything a job | ||
| 156 | // could have written is gone. | ||
| 157 | var survived []string | ||
| 158 | for _, m := range globAll(t, filepath.Join(root, "*", "*", "*")) { | ||
| 159 | if !strings.Contains(m, string(filepath.Separator)+".locks"+string(filepath.Separator)) { | ||
| 160 | survived = append(survived, m) | ||
| 161 | } | ||
| 162 | } | ||
| 163 | if len(survived) != 0 { | ||
| 164 | t.Fatalf("namespaces survived Clear: %v", survived) | ||
| 165 | } | ||
| 166 | } | ||
| 167 | |||
| 168 | // globAll is filepath.Glob for a test, which turns a bad pattern into a failure | ||
| 169 | // rather than a silently empty list. | ||
| 170 | func globAll(t *testing.T, pattern string) []string { | ||
| 171 | t.Helper() | ||
| 172 | matches, err := filepath.Glob(pattern) | ||
| 173 | if err != nil { | ||
| 174 | t.Fatalf("glob %q: %v", pattern, err) | ||
| 155 | 175 | } | |
| 176 | return matches | ||
| 156 | 177 | } | |
| 157 | 178 | ||
| 158 | 179 | func TestTrimIsANoOpWithoutACap(t *testing.T) { | |
| @@ -169,3 +190,172 @@ func TestTrimIsANoOpWithoutACap(t *testing.T) { | |||
| 169 | 190 | t.Fatalf("a namespace was removed with no cap set: %v", err) | |
| 170 | 191 | } | |
| 171 | 192 | } | |
| 193 | |||
| 194 | // writeNamespace fills a namespace with one file of the given size and stamps it | ||
| 195 | // with an age, which is what Trim sorts by. | ||
| 196 | func writeNamespace(t *testing.T, ns string, bytes int, age time.Duration) { | ||
| 197 | t.Helper() | ||
| 198 | if err := os.MkdirAll(ns, 0o755); err != nil { | ||
| 199 | t.Fatal(err) | ||
| 200 | } | ||
| 201 | file := filepath.Join(ns, "content") | ||
| 202 | if err := os.WriteFile(file, make([]byte, bytes), 0o644); err != nil { | ||
| 203 | t.Fatal(err) | ||
| 204 | } | ||
| 205 | when := time.Now().Add(-age) | ||
| 206 | if err := os.Chtimes(file, when, when); err != nil { | ||
| 207 | t.Fatal(err) | ||
| 208 | } | ||
| 209 | } | ||
| 210 | |||
| 211 | // A job holds its namespace while it runs, so a trim that reaches it must move on | ||
| 212 | // to the next candidate rather than delete a module cache out from under a build. | ||
| 213 | func TestTrimSkipsANamespaceInUse(t *testing.T) { | ||
| 214 | t.Parallel() | ||
| 215 | root := t.TempDir() | ||
| 216 | old := Namespace(root, "old", "direct", "ref-a") | ||
| 217 | fresh := Namespace(root, "fresh", "direct", "ref-b") | ||
| 218 | writeNamespace(t, old, 1000, 2*time.Hour) | ||
| 219 | writeNamespace(t, fresh, 1000, time.Minute) | ||
| 220 | |||
| 221 | release, err := Lock(root, old) | ||
| 222 | if err != nil { | ||
| 223 | t.Fatalf("lock: %v", err) | ||
| 224 | } | ||
| 225 | |||
| 226 | // The cap wants 500 bytes gone. The oldest namespace is the one in use, so the | ||
| 227 | // trimmer takes the next one instead. | ||
| 228 | removed, err := Trim(root, 1500) | ||
| 229 | if err != nil || removed != 1000 { | ||
| 230 | t.Fatalf("trim = %d, %v; want 1000 bytes from the newest namespace", removed, err) | ||
| 231 | } | ||
| 232 | if _, err := os.Stat(old); err != nil { | ||
| 233 | t.Fatalf("a mounted namespace was removed: %v", err) | ||
| 234 | } | ||
| 235 | if _, err := os.Stat(fresh); !os.IsNotExist(err) { | ||
| 236 | t.Fatalf("the trimmer did not fall through to the next namespace: %v", err) | ||
| 237 | } | ||
| 238 | |||
| 239 | // Once the job is done the namespace is a candidate again. | ||
| 240 | release() | ||
| 241 | if removed, err := Trim(root, 500); err != nil || removed != 1000 { | ||
| 242 | t.Fatalf("trim after release = %d, %v; want 1000 bytes", removed, err) | ||
| 243 | } | ||
| 244 | if _, err := os.Stat(old); !os.IsNotExist(err) { | ||
| 245 | t.Fatalf("the namespace survived once nobody held it: %v", err) | ||
| 246 | } | ||
| 247 | } | ||
| 248 | |||
| 249 | // Clear is what --no-cache uses. It empties what nobody has mounted and leaves a | ||
| 250 | // build alone, which is the difference between a fresh cache and a broken one. | ||
| 251 | func TestClearSkipsANamespaceInUse(t *testing.T) { | ||
| 252 | t.Parallel() | ||
| 253 | root := t.TempDir() | ||
| 254 | mine := Namespace(root, "mine", "direct", "ref-a") | ||
| 255 | theirs := Namespace(root, "theirs", "direct", "ref-b") | ||
| 256 | writeNamespace(t, mine, 10, time.Minute) | ||
| 257 | writeNamespace(t, theirs, 10, time.Minute) | ||
| 258 | |||
| 259 | release, err := Lock(root, theirs) | ||
| 260 | if err != nil { | ||
| 261 | t.Fatalf("lock: %v", err) | ||
| 262 | } | ||
| 263 | if err := Clear(root); err != nil { | ||
| 264 | t.Fatalf("clear: %v", err) | ||
| 265 | } | ||
| 266 | if _, err := os.Stat(mine); !os.IsNotExist(err) { | ||
| 267 | t.Fatalf("Clear left an unused namespace behind: %v", err) | ||
| 268 | } | ||
| 269 | if _, err := os.Stat(theirs); err != nil { | ||
| 270 | t.Fatalf("Clear emptied a namespace a job has mounted: %v", err) | ||
| 271 | } | ||
| 272 | |||
| 273 | release() | ||
| 274 | if err := Clear(root); err != nil { | ||
| 275 | t.Fatalf("clear after release: %v", err) | ||
| 276 | } | ||
| 277 | if _, err := os.Stat(theirs); !os.IsNotExist(err) { | ||
| 278 | t.Fatalf("the namespace survived once nobody held it: %v", err) | ||
| 279 | } | ||
| 280 | } | ||
| 281 | |||
| 282 | // The lock file lives beside the namespace, not inside it: removing a namespace | ||
| 283 | // must not remove the lock its holder is using, and the sweep must not mistake a | ||
| 284 | // lock for a cache. | ||
| 285 | func TestTheLockFileOutlivesItsNamespace(t *testing.T) { | ||
| 286 | t.Parallel() | ||
| 287 | root := t.TempDir() | ||
| 288 | ns := Namespace(root, "demo", "direct", "ref-a") | ||
| 289 | writeNamespace(t, ns, 10, time.Minute) | ||
| 290 | |||
| 291 | lock, err := lockPath(root, ns) | ||
| 292 | if err != nil { | ||
| 293 | t.Fatalf("lock path: %v", err) | ||
| 294 | } | ||
| 295 | if strings.HasPrefix(lock, ns+string(filepath.Separator)) { | ||
| 296 | t.Fatalf("the lock %q is inside the namespace it protects", lock) | ||
| 297 | } | ||
| 298 | |||
| 299 | release, err := Lock(root, ns) | ||
| 300 | if err != nil { | ||
| 301 | t.Fatalf("lock: %v", err) | ||
| 302 | } | ||
| 303 | release() | ||
| 304 | if _, err := os.Stat(lock); err != nil { | ||
| 305 | t.Fatalf("no lock file beside the namespace: %v", err) | ||
| 306 | } | ||
| 307 | |||
| 308 | if removed, err := Trim(root, 1); err != nil || removed != 10 { | ||
| 309 | t.Fatalf("trim = %d, %v; want 10 bytes", removed, err) | ||
| 310 | } | ||
| 311 | if _, err := os.Stat(lock); err != nil { | ||
| 312 | t.Fatalf("trimming the namespace removed the lock beside it: %v", err) | ||
| 313 | } | ||
| 314 | // Removing a lock file races with the next holder opening it, so it stays; a | ||
| 315 | // later job has to be able to take it. | ||
| 316 | again, err := Lock(root, ns) | ||
| 317 | if err != nil { | ||
| 318 | t.Fatalf("lock after the namespace was evicted: %v", err) | ||
| 319 | } | ||
| 320 | again() | ||
| 321 | } | ||
| 322 | |||
| 323 | // Two jobs and a trimmer on one root: neither holder may lose its tree, however | ||
| 324 | // often the disk-pressure check runs. | ||
| 325 | func TestATrimmerNeverTakesANamespaceInUse(t *testing.T) { | ||
| 326 | t.Parallel() | ||
| 327 | root := t.TempDir() | ||
| 328 | first := Namespace(root, "one", "direct", "ref-a") | ||
| 329 | second := Namespace(root, "two", "direct", "ref-b") | ||
| 330 | writeNamespace(t, first, 100, time.Minute) | ||
| 331 | writeNamespace(t, second, 100, time.Minute) | ||
| 332 | |||
| 333 | releaseFirst, err := Lock(root, first) | ||
| 334 | if err != nil { | ||
| 335 | t.Fatalf("lock: %v", err) | ||
| 336 | } | ||
| 337 | defer releaseFirst() | ||
| 338 | releaseSecond, err := Lock(root, second) | ||
| 339 | if err != nil { | ||
| 340 | t.Fatalf("lock: %v", err) | ||
| 341 | } | ||
| 342 | defer releaseSecond() | ||
| 343 | |||
| 344 | done := make(chan struct{}) | ||
| 345 | go func() { | ||
| 346 | defer close(done) | ||
| 347 | for i := 0; i < 20; i++ { | ||
| 348 | // A cap that wants everything gone: every pass has to skip both. | ||
| 349 | if _, err := Trim(root, 1); err != nil { | ||
| 350 | return | ||
| 351 | } | ||
| 352 | } | ||
| 353 | }() | ||
| 354 | <-done | ||
| 355 | |||
| 356 | for _, ns := range []string{first, second} { | ||
| 357 | if _, err := os.Stat(ns); err != nil { | ||
| 358 | t.Fatalf("the trimmer removed a mounted namespace: %v", err) | ||
| 359 | } | ||
| 360 | } | ||
| 361 | } | ||