vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys
keenan · 59m ago
4b8c9e1d
| 1 | package agent |
| 2 | |
| 3 | // Building a custom image from a .vole/images definition. |
| 4 | // |
| 5 | // The job carries the pinned definition text and its digest, so the runner does |
| 6 | // not need to read the repository again and cannot build a definition that changed |
| 7 | // after the job was queued. The image is tagged by that digest, which makes a |
| 8 | // second job with the same definition reuse the first build. |
| 9 | |
| 10 | import ( |
| 11 | "context" |
| 12 | "errors" |
| 13 | "fmt" |
| 14 | "io" |
| 15 | "os" |
| 16 | "os/exec" |
| 17 | "path/filepath" |
| 18 | "sync" |
| 19 | "time" |
| 20 | |
| 21 | "rodent.software/vole/ci" |
| 22 | "rodent.software/vole/ci/container" |
| 23 | ) |
| 24 | |
| 25 | const imageBuildTimeout = 30 * time.Minute |
| 26 | |
| 27 | // One build per digest, per process. |
| 28 | // |
| 29 | // With --jobs, two slots can lease jobs that name the same image-file: same bytes, |
| 30 | // same tag, twice the CPU and twice the disk, and two processes racing to tag the |
| 31 | // result. The first slot builds; the others wait on the same answer. |
| 32 | // |
| 33 | // Waiting is not free. A build can run for half an hour, and a runner that stops |
| 34 | // hearing from the queue loses the run it is holding, so a waiter keeps its lease |
| 35 | // alive exactly as a build of its own would have, and stops when the lease or the |
| 36 | // run goes away. |
| 37 | // |
| 38 | // Two runners sharing a --dir can still both build the digest. That is waste, not |
| 39 | // damage: the definition is pinned by digest, so both builds produce the same image |
| 40 | // under the same tag. Serialising that too would need a lock file per digest beside |
| 41 | // the caches, which is more machinery than the case is worth. |
| 42 | |
| 43 | // imageBuild is one in-flight build of a digest. done closes when the attempt has an |
| 44 | // answer, success or failure; err is that answer. |
| 45 | type imageBuild struct { |
| 46 | done chan struct{} |
| 47 | err error |
| 48 | } |
| 49 | |
| 50 | // imageBuilds is the set of builds this process has in flight, shared by its slots. |
| 51 | type imageBuilds struct { |
| 52 | mu sync.Mutex |
| 53 | running map[string]*imageBuild |
| 54 | } |
| 55 | |
| 56 | // join makes this caller the builder of tag, or hands it the build already running. |
| 57 | func (b *imageBuilds) join(tag string) (build *imageBuild, mine bool) { |
| 58 | b.mu.Lock() |
| 59 | defer b.mu.Unlock() |
| 60 | if cur, ok := b.running[tag]; ok { |
| 61 | return cur, false |
| 62 | } |
| 63 | if b.running == nil { |
| 64 | b.running = make(map[string]*imageBuild) |
| 65 | } |
| 66 | build = &imageBuild{done: make(chan struct{})} |
| 67 | b.running[tag] = build |
| 68 | return build, true |
| 69 | } |
| 70 | |
| 71 | // leave forgets a finished build, so the next job that wants the digest asks the |
| 72 | // runtime again — which is also how an image somebody deleted gets rebuilt. |
| 73 | func (b *imageBuilds) leave(tag string) { |
| 74 | b.mu.Lock() |
| 75 | defer b.mu.Unlock() |
| 76 | delete(b.running, tag) |
| 77 | } |
| 78 | |
| 79 | // resolveImage returns the reference to run. A registry image is used as written; |
| 80 | // an image-file definition is built locally and cached under a digest tag. |
| 81 | func (s *runnerSession) resolveImage(job *runnerJob, box runnerBox, log io.Writer, lease time.Duration) (string, error) { |
| 82 | if job.ImageFile == "" { |
| 83 | if err := ci.ValidImageRef(job.Image); err != nil { |
| 84 | return "", fmt.Errorf("the job image: %w", err) |
| 85 | } |
| 86 | return job.Image, nil |
| 87 | } |
| 88 | if job.ImageDefinition == "" { |
| 89 | return "", fmt.Errorf("image-file %q has no pinned definition", job.ImageFile) |
| 90 | } |
| 91 | if job.ImageSHA == "" { |
| 92 | return "", fmt.Errorf("image-file %q has no definition digest", job.ImageFile) |
| 93 | } |
| 94 | tag := container.ImageTag(job.ImageFile, job.ImageSHA) |
| 95 | if box.Pull != "always" && container.Exists(box.Runtime, tag) { |
| 96 | fmt.Fprintf(log, "image-file %s: using cached %s\n", job.ImageFile, tag) |
| 97 | return tag, nil |
| 98 | } |
| 99 | if box.builds == nil { |
| 100 | // A runnerBox built by hand (a test, or a caller that never validated) has no |
| 101 | // shared state to join; building alone is what it did before --jobs. |
| 102 | if err := s.buildImageFile(job, tag, box, log, lease); err != nil { |
| 103 | return "", err |
| 104 | } |
| 105 | return tag, nil |
| 106 | } |
| 107 | |
| 108 | build, mine := box.builds.join(tag) |
| 109 | if mine { |
| 110 | err := s.buildImageFile(job, tag, box, log, lease) |
| 111 | build.err = err |
| 112 | // Leave before closing, so a job that arrives after this one takes the |
| 113 | // finished entry starts a fresh build instead of reading an answer about a |
| 114 | // digest it did not ask for. Waiters already on done are unaffected. |
| 115 | box.builds.leave(tag) |
| 116 | close(build.done) |
| 117 | if err != nil { |
| 118 | return "", err |
| 119 | } |
| 120 | return tag, nil |
| 121 | } |
| 122 | |
| 123 | // Another slot is building it. The line goes in this job's log so a reader knows |
| 124 | // why the steps have not started. |
| 125 | fmt.Fprintf(log, "image-file %s: %s is being built by another job; waiting\n", job.ImageFile, tag) |
| 126 | if err := s.waitForBuild(build, lease, job.ID, log); err != nil { |
| 127 | return "", err |
| 128 | } |
| 129 | if build.err != nil { |
| 130 | // A failed build is this job's failure too: the same definition would fail |
| 131 | // the same way, and a second attempt would only spend the time again. A lease |
| 132 | // that went away belongs to the builder, though, and means only that nobody |
| 133 | // may run this image on this runner's word. |
| 134 | if errors.Is(build.err, errLeaseLost) || errors.Is(build.err, errCanceled) { |
| 135 | return "", build.err |
| 136 | } |
| 137 | return "", fmt.Errorf("image-file %q: %w", job.ImageFile, build.err) |
| 138 | } |
| 139 | return tag, nil |
| 140 | } |
| 141 | |
| 142 | // buildImageFile renders the pinned definition, builds it, and tags the result by |
| 143 | // its digest. |
| 144 | func (s *runnerSession) buildImageFile(job *runnerJob, tag string, box runnerBox, log io.Writer, lease time.Duration) error { |
| 145 | dockerfile, err := ci.RenderImage(job.ImageDefinition) |
| 146 | if err != nil { |
| 147 | return fmt.Errorf("image-file %q: %w", job.ImageFile, err) |
| 148 | } |
| 149 | dir, err := os.MkdirTemp("", "vole-image-*") |
| 150 | if err != nil { |
| 151 | return err |
| 152 | } |
| 153 | defer os.RemoveAll(dir) |
| 154 | if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o600); err != nil { |
| 155 | return err |
| 156 | } |
| 157 | |
| 158 | args := []string{"build", "-t", tag, "-f", filepath.Join(dir, "Dockerfile"), dir} |
| 159 | switch { |
| 160 | case box.Pull == "always" && box.Runtime == "podman": |
| 161 | args = append(args, "--pull=always") |
| 162 | case box.Pull == "always": |
| 163 | args = append(args, "--pull") |
| 164 | } |
| 165 | ctx, cancel := context.WithTimeout(context.Background(), imageBuildTimeout) |
| 166 | defer cancel() |
| 167 | cmd := exec.CommandContext(ctx, box.Runtime, args...) |
| 168 | code, lost, canceled := s.runCommand(ctx, cmd, job.ID, lease, nil, log) |
| 169 | if lost { |
| 170 | return errLeaseLost |
| 171 | } |
| 172 | if canceled { |
| 173 | return errCanceled |
| 174 | } |
| 175 | if ctx.Err() == context.DeadlineExceeded { |
| 176 | return fmt.Errorf("building image-file %q timed out after %s", job.ImageFile, imageBuildTimeout) |
| 177 | } |
| 178 | if code != 0 { |
| 179 | return fmt.Errorf("building image-file %q exited %d", job.ImageFile, code) |
| 180 | } |
| 181 | if !container.Exists(box.Runtime, tag) { |
| 182 | return fmt.Errorf("building image-file %q produced no image tagged %s", job.ImageFile, tag) |
| 183 | } |
| 184 | fmt.Fprintf(log, "\nimage-file %s: built %s\n", job.ImageFile, tag) |
| 185 | return nil |
| 186 | } |
| 187 | |
| 188 | // waitForBuild waits for another slot's build, keeping this job's lease alive for |
| 189 | // as long as a build of its own would have. A waiter that went quiet would be |
| 190 | // requeued to somebody else mid-wait, and would then go on to run a job the queue |
| 191 | // no longer considers its own. |
| 192 | func (s *runnerSession) waitForBuild(build *imageBuild, lease time.Duration, runID string, log io.Writer) error { |
| 193 | interval := lease / 3 |
| 194 | if interval < time.Second { |
| 195 | interval = time.Second |
| 196 | } |
| 197 | tick := time.NewTicker(interval) |
| 198 | defer tick.Stop() |
| 199 | for { |
| 200 | select { |
| 201 | case <-build.done: |
| 202 | return nil |
| 203 | case <-tick.C: |
| 204 | rep, err := s.call(runnerRequest{Op: "heartbeat", Run: runID}) |
| 205 | if err != nil || !rep.OK { |
| 206 | return errLeaseLost |
| 207 | } |
| 208 | if rep.Cancel != "" { |
| 209 | fmt.Fprintf(log, "\nthe run was canceled by %s; stopping\n", rep.Cancel) |
| 210 | return errCanceled |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 |