vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Build one image-file digest once per runner process
77056ba834ba243104726b91e7ff19498d4e8337 · Keenan Nemetz · 2d ago · parent 18c559ea
Build one image-file digest once per runner process Two slots leased jobs that name the same image-file and both built it: the same bytes, the same tag, twice the CPU and twice the disk, with two runtimes racing to tag the result. The first slot now builds it and the others wait on the same answer, which is shared by the process rather than by the session, because that is what --jobs is. Waiting is not free: a build can run for half an hour, and a runner that stops heartbeating loses the run it is holding. So a waiter heartbeats on the same beat its own build would have, and stops if the lease goes away or a maintainer cancels the run. A failed build fails the jobs waiting on it, since the same definition would fail the same way. Two runner processes sharing a --dir can still both build the digest. That is waste, not damage: the definition is pinned by digest, so both builds produce the same image under the same tag, and serialising that too would mean a lock file per digest beside the caches. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/agent/build.go +136−9
| @@ -9,11 +9,13 @@ package agent | |||
| 9 | 9 | ||
| 10 | 10 | import ( | |
| 11 | 11 | "context" | |
| 12 | "errors" | ||
| 12 | 13 | "fmt" | |
| 13 | 14 | "io" | |
| 14 | 15 | "os" | |
| 15 | 16 | "os/exec" | |
| 16 | 17 | "path/filepath" | |
| 18 | "sync" | ||
| 17 | 19 | "time" | |
| 18 | 20 | ||
| 19 | 21 | "github.com/rodent-software/vole/ci" | |
| @@ -22,6 +24,58 @@ import ( | |||
| 22 | 24 | ||
| 23 | 25 | const imageBuildTimeout = 30 * time.Minute | |
| 24 | 26 | ||
| 27 | // One build per digest, per process. | ||
| 28 | // | ||
| 29 | // With --jobs, two slots can lease jobs that name the same image-file: same bytes, | ||
| 30 | // same tag, twice the CPU and twice the disk, and two processes racing to tag the | ||
| 31 | // result. The first slot builds; the others wait on the same answer. | ||
| 32 | // | ||
| 33 | // Waiting is not free. A build can run for half an hour, and a runner that stops | ||
| 34 | // hearing from the queue loses the run it is holding, so a waiter keeps its lease | ||
| 35 | // alive exactly as a build of its own would have, and stops when the lease or the | ||
| 36 | // run goes away. | ||
| 37 | // | ||
| 38 | // Two runners sharing a --dir can still both build the digest. That is waste, not | ||
| 39 | // damage: the definition is pinned by digest, so both builds produce the same image | ||
| 40 | // under the same tag. Serialising that too would need a lock file per digest beside | ||
| 41 | // the caches, which is more machinery than the case is worth. | ||
| 42 | |||
| 43 | // imageBuild is one in-flight build of a digest. done closes when the attempt has an | ||
| 44 | // answer, success or failure; err is that answer. | ||
| 45 | type imageBuild struct { | ||
| 46 | done chan struct{} | ||
| 47 | err error | ||
| 48 | } | ||
| 49 | |||
| 50 | // imageBuilds is the set of builds this process has in flight, shared by its slots. | ||
| 51 | type imageBuilds struct { | ||
| 52 | mu sync.Mutex | ||
| 53 | running map[string]*imageBuild | ||
| 54 | } | ||
| 55 | |||
| 56 | // join makes this caller the builder of tag, or hands it the build already running. | ||
| 57 | func (b *imageBuilds) join(tag string) (build *imageBuild, mine bool) { | ||
| 58 | b.mu.Lock() | ||
| 59 | defer b.mu.Unlock() | ||
| 60 | if cur, ok := b.running[tag]; ok { | ||
| 61 | return cur, false | ||
| 62 | } | ||
| 63 | if b.running == nil { | ||
| 64 | b.running = make(map[string]*imageBuild) | ||
| 65 | } | ||
| 66 | build = &imageBuild{done: make(chan struct{})} | ||
| 67 | b.running[tag] = build | ||
| 68 | return build, true | ||
| 69 | } | ||
| 70 | |||
| 71 | // leave forgets a finished build, so the next job that wants the digest asks the | ||
| 72 | // runtime again — which is also how an image somebody deleted gets rebuilt. | ||
| 73 | func (b *imageBuilds) leave(tag string) { | ||
| 74 | b.mu.Lock() | ||
| 75 | defer b.mu.Unlock() | ||
| 76 | delete(b.running, tag) | ||
| 77 | } | ||
| 78 | |||
| 25 | 79 | // resolveImage returns the reference to run. A registry image is used as written; | |
| 26 | 80 | // an image-file definition is built locally and cached under a digest tag. | |
| 27 | 81 | func (s *runnerSession) resolveImage(job *runnerJob, box runnerBox, log io.Writer, lease time.Duration) (string, error) { | |
| @@ -42,18 +96,63 @@ func (s *runnerSession) resolveImage(job *runnerJob, box runnerBox, log io.Write | |||
| 42 | 96 | fmt.Fprintf(log, "image-file %s: using cached %s\n", job.ImageFile, tag) | |
| 43 | 97 | return tag, nil | |
| 44 | 98 | } | |
| 99 | if box.builds == nil { | ||
| 100 | // A runnerBox built by hand (a test, or a caller that never validated) has no | ||
| 101 | // shared state to join; building alone is what it did before --jobs. | ||
| 102 | if err := s.buildImageFile(job, tag, box, log, lease); err != nil { | ||
| 103 | return "", err | ||
| 104 | } | ||
| 105 | return tag, nil | ||
| 106 | } | ||
| 107 | |||
| 108 | build, mine := box.builds.join(tag) | ||
| 109 | if mine { | ||
| 110 | err := s.buildImageFile(job, tag, box, log, lease) | ||
| 111 | build.err = err | ||
| 112 | // Leave before closing, so a job that arrives after this one takes the | ||
| 113 | // finished entry starts a fresh build instead of reading an answer about a | ||
| 114 | // digest it did not ask for. Waiters already on done are unaffected. | ||
| 115 | box.builds.leave(tag) | ||
| 116 | close(build.done) | ||
| 117 | if err != nil { | ||
| 118 | return "", err | ||
| 119 | } | ||
| 120 | return tag, nil | ||
| 121 | } | ||
| 122 | |||
| 123 | // Another slot is building it. The line goes in this job's log so a reader knows | ||
| 124 | // why the steps have not started. | ||
| 125 | fmt.Fprintf(log, "image-file %s: %s is being built by another job; waiting\n", job.ImageFile, tag) | ||
| 126 | if err := s.waitForBuild(build, lease, job.ID, log); err != nil { | ||
| 127 | return "", err | ||
| 128 | } | ||
| 129 | if build.err != nil { | ||
| 130 | // A failed build is this job's failure too: the same definition would fail | ||
| 131 | // the same way, and a second attempt would only spend the time again. A lease | ||
| 132 | // that went away belongs to the builder, though, and means only that nobody | ||
| 133 | // may run this image on this runner's word. | ||
| 134 | if errors.Is(build.err, errLeaseLost) || errors.Is(build.err, errCanceled) { | ||
| 135 | return "", build.err | ||
| 136 | } | ||
| 137 | return "", fmt.Errorf("image-file %q: %w", job.ImageFile, build.err) | ||
| 138 | } | ||
| 139 | return tag, nil | ||
| 140 | } | ||
| 45 | 141 | ||
| 142 | // buildImageFile renders the pinned definition, builds it, and tags the result by | ||
| 143 | // its digest. | ||
| 144 | func (s *runnerSession) buildImageFile(job *runnerJob, tag string, box runnerBox, log io.Writer, lease time.Duration) error { | ||
| 46 | 145 | dockerfile, err := ci.RenderImage(job.ImageDefinition) | |
| 47 | 146 | if err != nil { | |
| 48 | return "", fmt.Errorf("image-file %q: %w", job.ImageFile, err) | ||
| 147 | return fmt.Errorf("image-file %q: %w", job.ImageFile, err) | ||
| 49 | 148 | } | |
| 50 | 149 | dir, err := os.MkdirTemp("", "vole-image-*") | |
| 51 | 150 | if err != nil { | |
| 52 | return "", err | ||
| 151 | return err | ||
| 53 | 152 | } | |
| 54 | 153 | defer os.RemoveAll(dir) | |
| 55 | 154 | if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o600); err != nil { | |
| 56 | return "", err | ||
| 155 | return err | ||
| 57 | 156 | } | |
| 58 | 157 | ||
| 59 | 158 | args := []string{"build", "-t", tag, "-f", filepath.Join(dir, "Dockerfile"), dir} | |
| @@ -68,20 +167,48 @@ func (s *runnerSession) resolveImage(job *runnerJob, box runnerBox, log io.Write | |||
| 68 | 167 | cmd := exec.CommandContext(ctx, box.Runtime, args...) | |
| 69 | 168 | code, lost, canceled := s.runCommand(ctx, cmd, job.ID, lease, nil, log) | |
| 70 | 169 | if lost { | |
| 71 | return "", errLeaseLost | ||
| 170 | return errLeaseLost | ||
| 72 | 171 | } | |
| 73 | 172 | if canceled { | |
| 74 | return "", errCanceled | ||
| 173 | return errCanceled | ||
| 75 | 174 | } | |
| 76 | 175 | if ctx.Err() == context.DeadlineExceeded { | |
| 77 | return "", fmt.Errorf("building image-file %q timed out after %s", job.ImageFile, imageBuildTimeout) | ||
| 176 | return fmt.Errorf("building image-file %q timed out after %s", job.ImageFile, imageBuildTimeout) | ||
| 78 | 177 | } | |
| 79 | 178 | if code != 0 { | |
| 80 | return "", fmt.Errorf("building image-file %q exited %d", job.ImageFile, code) | ||
| 179 | return fmt.Errorf("building image-file %q exited %d", job.ImageFile, code) | ||
| 81 | 180 | } | |
| 82 | 181 | if !container.Exists(box.Runtime, tag) { | |
| 83 | return "", fmt.Errorf("building image-file %q produced no image tagged %s", job.ImageFile, tag) | ||
| 182 | return fmt.Errorf("building image-file %q produced no image tagged %s", job.ImageFile, tag) | ||
| 84 | 183 | } | |
| 85 | 184 | fmt.Fprintf(log, "\nimage-file %s: built %s\n", job.ImageFile, tag) | |
| 86 | return tag, nil | ||
| 185 | return nil | ||
| 186 | } | ||
| 187 | |||
| 188 | // waitForBuild waits for another slot's build, keeping this job's lease alive for | ||
| 189 | // as long as a build of its own would have. A waiter that went quiet would be | ||
| 190 | // requeued to somebody else mid-wait, and would then go on to run a job the queue | ||
| 191 | // no longer considers its own. | ||
| 192 | func (s *runnerSession) waitForBuild(build *imageBuild, lease time.Duration, runID string, log io.Writer) error { | ||
| 193 | interval := lease / 3 | ||
| 194 | if interval < time.Second { | ||
| 195 | interval = time.Second | ||
| 196 | } | ||
| 197 | tick := time.NewTicker(interval) | ||
| 198 | defer tick.Stop() | ||
| 199 | for { | ||
| 200 | select { | ||
| 201 | case <-build.done: | ||
| 202 | return nil | ||
| 203 | case <-tick.C: | ||
| 204 | rep, err := s.call(runnerRequest{Op: "heartbeat", Run: runID}) | ||
| 205 | if err != nil || !rep.OK { | ||
| 206 | return errLeaseLost | ||
| 207 | } | ||
| 208 | if rep.Cancel != "" { | ||
| 209 | fmt.Fprintf(log, "\nthe run was canceled by %s; stopping\n", rep.Cancel) | ||
| 210 | return errCanceled | ||
| 211 | } | ||
| 212 | } | ||
| 213 | } | ||
| 87 | 214 | } | |