vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Hold a job's cache namespace while the job runs
a29ce1f76aaf7ebc4b24afad36c56a0f7f01254e · Keenan Nemetz · 2d ago · parent b492b557
Hold a job's cache namespace while the job runs A namespace was only safe while one job existed per runner. Two runners sharing a --dir already had the bug: the trimmer deletes whole namespaces and `--no-cache` empties one, either of which can land on a tree another process has mounted. A Go module cache that loses a file mid-build is a checksum error deep in a build, which is the case the whole-namespace eviction rule exists to avoid. A job now holds its namespace for as long as the container may be inside it — shared, so two builds of the same repository still run together — and eviction takes the same lock with LOCK_NB and leaves what is in use for a later pass. `--no-cache` takes the namespace exclusively, because emptying it afterwards must not land on somebody else's build. The runner also stops trimming at the end of each job: the cap is enforced at startup and on a timer, so eviction no longer belongs to one job's lifetime. Lock files live under <root>/.locks, mirroring the namespace, so a cache tree holds namespaces and nothing else; they are never deleted, for the reason gitcache's are not. `vole workflow run` gets the same lock, so two local runs in one checkout cannot delete each other's cache either. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/agent/agent.go +84−33
| @@ -44,6 +44,11 @@ import ( | |||
| 44 | 44 | // invisible in the queue's load. | |
| 45 | 45 | const leaseWait = 25 * time.Second | |
| 46 | 46 | ||
| 47 | // cacheTrimInterval is how often the runner bounds the job cache between jobs. A | ||
| 48 | // namespace that is past the cap while a job has it mounted is skipped and taken on | ||
| 49 | // a later pass, so a longer gap costs disk, not correctness. | ||
| 50 | const cacheTrimInterval = 5 * time.Minute | ||
| 51 | |||
| 47 | 52 | // errLeaseLost means the queue took the job away while the runner was still | |
| 48 | 53 | // preparing its environment. There is nothing to report in that case. | |
| 49 | 54 | var errLeaseLost = errors.New("lease lost") | |
| @@ -324,10 +329,17 @@ func Run(ctx context.Context, opts Options) error { | |||
| 324 | 329 | // Nothing survives, so nothing needs bounding; empty what a killed runner | |
| 325 | 330 | // left behind. | |
| 326 | 331 | _ = jobcache.Clear(cacheRoot) | |
| 327 | } else if removed, err := jobcache.Trim(cacheRoot, cacheMax); err == nil && removed > 0 { | ||
| 328 | // A killed runner leaves whatever the last job wrote; bound it before work | ||
| 329 | // starts rather than after the disk is full. | ||
| 330 | fmt.Fprintf(os.Stderr, "vole runner: trimmed %s from the job cache\n", format.Bytes(removed)) | ||
| 332 | } else { | ||
| 333 | if removed, err := jobcache.Trim(cacheRoot, cacheMax); err == nil && removed > 0 { | ||
| 334 | // A killed runner leaves whatever the last job wrote; bound it before work | ||
| 335 | // starts rather than after the disk is full. | ||
| 336 | fmt.Fprintf(os.Stderr, "vole runner: trimmed %s from the job cache\n", format.Bytes(removed)) | ||
| 337 | } | ||
| 338 | // Bound it between jobs too, but on its own beat rather than at the end of | ||
| 339 | // every job: eviction is whole-namespace and passes over a namespace a job | ||
| 340 | // has mounted, so it is a disk-pressure check rather than the last step of a | ||
| 341 | // build. | ||
| 342 | go trimJobCache(ctx, cacheRoot, cacheMax) | ||
| 331 | 343 | } | |
| 332 | 344 | ||
| 333 | 345 | // A label has to be one the workflow validator would accept, or the machine would | |
| @@ -363,7 +375,7 @@ func Run(ctx context.Context, opts Options) error { | |||
| 363 | 375 | sess.name = who.Name | |
| 364 | 376 | sess.repos, sess.labels = opts.Repos, opts.Labels | |
| 365 | 377 | sess.network = box.Network | |
| 366 | sess.cacheRoot, sess.cacheMax, sess.cacheReuse = cacheRoot, cacheMax, !opts.NoCache | ||
| 378 | sess.cacheRoot, sess.cacheReuse = cacheRoot, !opts.NoCache | ||
| 367 | 379 | if opts.NoCache { | |
| 368 | 380 | fmt.Fprintf(os.Stderr, "vole runner: job cache in %s, emptied after every job\n", cacheRoot) | |
| 369 | 381 | } else { | |
| @@ -450,12 +462,12 @@ type runnerSession struct { | |||
| 450 | 462 | // asked for, not the effective list: the server intersects it with the grant on | |
| 451 | 463 | // each request, so an allowlist edit takes effect without a restart. | |
| 452 | 464 | repos []string | |
| 453 | // cacheRoot is where this runner keeps the caches it mounts into jobs. cacheMax | ||
| 454 | // is the cap, in bytes. cacheReuse says whether a job keeps what the last one | ||
| 455 | // wrote; with --no-cache the directory is still mounted and emptied instead, so | ||
| 456 | // an image that points at /cache works either way. | ||
| 465 | // cacheRoot is where this runner keeps the caches it mounts into jobs. cacheReuse | ||
| 466 | // says whether a job keeps what the last one wrote; with --no-cache the directory | ||
| 467 | // is still mounted and emptied instead, so an image that points at /cache works | ||
| 468 | // either way. The cap and the trimmer live in Run: a cache is bounded on the | ||
| 469 | // runner's own beat, not per job. | ||
| 457 | 470 | cacheRoot string | |
| 458 | cacheMax int64 | ||
| 459 | 471 | cacheReuse bool | |
| 460 | 472 | // labels is what this machine is (--label). A job is only given to a runner that | |
| 461 | 473 | // carries every label the job asked for, so an unlabelled runner only sees jobs | |
| @@ -606,6 +618,12 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 606 | 618 | return | |
| 607 | 619 | } | |
| 608 | 620 | defer co.cleanup() | |
| 621 | // The cache is held from here until after the container is gone, so eviction | ||
| 622 | // cannot take a namespace this job has mounted. The lock is taken before the | ||
| 623 | // container starts and released after it stops: the other order would leave a | ||
| 624 | // window where the tree is up for eviction but the job can still write to it. | ||
| 625 | cacheMounts, releaseCache := s.cacheFor(job) | ||
| 626 | defer releaseCache() | ||
| 609 | 627 | if job.Image == "" { | |
| 610 | 628 | // A run queued before images were required has no environment to start. | |
| 611 | 629 | // Fail it with a record rather than guessing at the host. | |
| @@ -711,12 +729,11 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 711 | 729 | } | |
| 712 | 730 | } | |
| 713 | 731 | } | |
| 714 | cacheMounts := s.cacheFor(job) | ||
| 715 | mounts := cacheMounts | ||
| 716 | 732 | // The run directory is mounted at /run/vole for the pinned event payload and for | |
| 717 | 733 | // the job's secrets. Both arrive as files rather than as interpolated commands: a | |
| 718 | 734 | // large issue body cannot make an argument oversized, and a secret never becomes | |
| 719 | 735 | // an argument at all. | |
| 736 | mounts := cacheMounts | ||
| 720 | 737 | env := job.Env | |
| 721 | 738 | runDir := "" | |
| 722 | 739 | if job.EventPayload != "" || len(job.Secrets) > 0 || job.Credential != nil { | |
| @@ -777,18 +794,14 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 777 | 794 | } | |
| 778 | 795 | fmt.Fprintf(log, "acting as %s for this run\n", job.Bot) | |
| 779 | 796 | } | |
| 780 | if s.cacheRoot != "" { | ||
| 797 | if s.cacheRoot != "" && !s.cacheReuse { | ||
| 781 | 798 | defer func() { | |
| 782 | if !s.cacheReuse { | ||
| 783 | // Only the cache mounts: the run directory is the job's own and is | ||
| 784 | // removed by the defer above. | ||
| 785 | for _, m := range cacheMounts { | ||
| 786 | _ = os.RemoveAll(m.Host) | ||
| 787 | } | ||
| 788 | return | ||
| 789 | } | ||
| 790 | if removed, err := jobcache.Trim(s.cacheRoot, s.cacheMax); err == nil && removed > 0 { | ||
| 791 | fmt.Fprintf(os.Stderr, "vole runner: trimmed %s from the job cache\n", format.Bytes(removed)) | ||
| 799 | // The namespace is held exclusively for this job and this defer runs before | ||
| 800 | // the lock is released, so nothing else is inside it while it is emptied. | ||
| 801 | // Only the cache mount: the run directory is the job's own and is removed | ||
| 802 | // by the defer above. | ||
| 803 | for _, m := range cacheMounts { | ||
| 804 | _ = os.RemoveAll(m.Host) | ||
| 792 | 805 | } | |
| 793 | 806 | }() | |
| 794 | 807 | } | |
| @@ -886,15 +899,18 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 886 | 899 | s.finish(remote, co, job, log, 0, steps, arts, releaseArts, box) | |
| 887 | 900 | } | |
| 888 | 901 | ||
| 889 | // cacheFor returns the mount for one job's cache, or nothing when the runner has | ||
| 890 | // caching turned off. It sets no environment: a job opts in by pointing its own | ||
| 891 | // variables at Mount. The namespace separates repositories from each other, and | ||
| 892 | // contributor branches from everything else: a cache is shared mutable state, and a | ||
| 893 | // branch nobody has vouched for must not be able to seed the one a protected branch | ||
| 894 | // builds into. | ||
| 895 | func (s *runnerSession) cacheFor(job *runnerJob) []container.Mount { | ||
| 902 | // cacheFor returns the mount for one job's cache and the lock that keeps eviction | ||
| 903 | // off the namespace until release is called. The release function is always | ||
| 904 | // non-nil, so a caller can defer it without asking whether there is a cache: a | ||
| 905 | // runner with none returns no mounts and a no-op release. | ||
| 906 | // | ||
| 907 | // It sets no environment: a job opts in by pointing its own variables at Mount. The | ||
| 908 | // namespace separates repositories from each other, and contributor branches from | ||
| 909 | // everything else: a cache is shared mutable state, and a branch nobody has vouched | ||
| 910 | // for must not be able to seed the one a protected branch builds into. | ||
| 911 | func (s *runnerSession) cacheFor(job *runnerJob) ([]container.Mount, func()) { | ||
| 896 | 912 | if s.cacheRoot == "" { | |
| 897 | return nil | ||
| 913 | return nil, func() {} | ||
| 898 | 914 | } | |
| 899 | 915 | // A contributor branch is untrusted code, and a cache is shared mutable state: | |
| 900 | 916 | // one contributor's branch must not be able to seed another's cache or the one a | |
| @@ -902,11 +918,46 @@ func (s *runnerSession) cacheFor(job *runnerJob) []container.Mount { | |||
| 902 | 918 | // cache follows them across their branches and nobody else can write it. | |
| 903 | 919 | trust := access.TrustNamespace(job.Ref) | |
| 904 | 920 | ns := jobcache.Namespace(s.cacheRoot, job.Repo, trust, jobcache.ImageKey(job.ImageSHA, job.Image)) | |
| 921 | // Take the lock before creating the directory, so an eviction between the two | ||
| 922 | // cannot remove what this job has just made. The shared lock is what lets two | ||
| 923 | // jobs of the same repository run at once while eviction passes over both; | ||
| 924 | // --no-cache takes it exclusively, because emptying the namespace afterwards must | ||
| 925 | // not land on somebody else's build. | ||
| 926 | var release func() | ||
| 927 | var err error | ||
| 928 | if s.cacheReuse { | ||
| 929 | release, err = jobcache.Lock(s.cacheRoot, ns) | ||
| 930 | } else { | ||
| 931 | release, err = jobcache.LockExclusive(s.cacheRoot, ns) | ||
| 932 | } | ||
| 933 | if err != nil { | ||
| 934 | fmt.Fprintf(os.Stderr, "vole runner: cannot lock a cache for %s: %v\n", job.Repo, err) | ||
| 935 | return nil, func() {} | ||
| 936 | } | ||
| 905 | 937 | if err := os.MkdirAll(ns, 0o755); err != nil { | |
| 938 | release() | ||
| 906 | 939 | fmt.Fprintf(os.Stderr, "vole runner: cannot prepare a cache for %s: %v\n", job.Repo, err) | |
| 907 | return nil | ||
| 940 | return nil, func() {} | ||
| 941 | } | ||
| 942 | return []container.Mount{{Host: ns, Path: jobcache.Mount}}, release | ||
| 943 | } | ||
| 944 | |||
| 945 | // trimJobCache bounds the job cache until the runner stops. Each pass skips a | ||
| 946 | // namespace a job has mounted; the next pass tries again, so the cap is a bound on | ||
| 947 | // what the runner keeps rather than a promise about the moment a job ends. | ||
| 948 | func trimJobCache(ctx context.Context, root string, max int64) { | ||
| 949 | t := time.NewTicker(cacheTrimInterval) | ||
| 950 | defer t.Stop() | ||
| 951 | for { | ||
| 952 | select { | ||
| 953 | case <-ctx.Done(): | ||
| 954 | return | ||
| 955 | case <-t.C: | ||
| 956 | if removed, err := jobcache.Trim(root, max); err == nil && removed > 0 { | ||
| 957 | fmt.Fprintf(os.Stderr, "vole runner: trimmed %s from the job cache\n", format.Bytes(removed)) | ||
| 958 | } | ||
| 959 | } | ||
| 908 | 960 | } | |
| 909 | return []container.Mount{{Host: ns, Path: jobcache.Mount}} | ||
| 910 | 961 | } | |
| 911 | 962 | ||
| 912 | 963 | // finish uploads this attempt's log, sends its record to the server with the | |