vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Build one image-file digest once per runner process
77056ba834ba243104726b91e7ff19498d4e8337 · Keenan Nemetz · 3d ago · parent 18c559ea
Build one image-file digest once per runner process Two slots leased jobs that name the same image-file and both built it: the same bytes, the same tag, twice the CPU and twice the disk, with two runtimes racing to tag the result. The first slot now builds it and the others wait on the same answer, which is shared by the process rather than by the session, because that is what --jobs is. Waiting is not free: a build can run for half an hour, and a runner that stops heartbeating loses the run it is holding. So a waiter heartbeats on the same beat its own build would have, and stops if the lease goes away or a maintainer cancels the run. A failed build fails the jobs waiting on it, since the same definition would fail the same way. Two runner processes sharing a --dir can still both build the digest. That is waste, not damage: the definition is pinned by digest, so both builds produce the same image under the same tag, and serialising that too would mean a lock file per digest beside the caches. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/agent/build_test.go +177−9
| @@ -1,18 +1,23 @@ | |||
| 1 | 1 | package agent | |
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | "errors" | ||
| 4 | 5 | "os" | |
| 5 | 6 | "path/filepath" | |
| 6 | 7 | "strings" | |
| 8 | "sync" | ||
| 7 | 9 | "testing" | |
| 8 | 10 | "time" | |
| 9 | 11 | ||
| 10 | 12 | "github.com/rodent-software/vole/ci" | |
| 11 | 13 | ) | |
| 12 | 14 | ||
| 13 | // A definition is built once, tagged by its digest, and reused by a later job. | ||
| 14 | // The fake runtime records the Dockerfile the agent generated and the build count. | ||
| 15 | func TestResolveImageBuildsAndCachesDefinition(t *testing.T) { | ||
| 15 | // fakeImageRuntime puts a `docker` on PATH that answers the two verbs image-file | ||
| 16 | // building uses: whether a tag exists, and a build that records the Dockerfile and | ||
| 17 | // the fact that it ran. A build can be made slow (so two callers overlap) or made | ||
| 18 | // to fail. It returns the directory holding what the fake was asked to do. | ||
| 19 | func fakeImageRuntime(t *testing.T, sleep string, fail bool) string { | ||
| 20 | t.Helper() | ||
| 16 | 21 | dir := t.TempDir() | |
| 17 | 22 | state := filepath.Join(dir, "state") | |
| 18 | 23 | if err := os.MkdirAll(state, 0o755); err != nil { | |
| @@ -40,8 +45,10 @@ case "$cmd" in | |||
| 40 | 45 | cp "$df" "$state/Dockerfile.captured" | |
| 41 | 46 | m=$(printf '%s' "$tag" | tr '/:' '__') | |
| 42 | 47 | printf 'build\n' >> "$state/builds" | |
| 43 | : > "$state/$m" | ||
| 44 | 48 | echo "fake build" | |
| 49 | sleep "${FAKE_BUILD_SLEEP:-0}" | ||
| 50 | [ "${FAKE_BUILD_FAIL:-0}" = 0 ] || exit 1 | ||
| 51 | : > "$state/$m" | ||
| 45 | 52 | exit 0 | |
| 46 | 53 | ;; | |
| 47 | 54 | *) exit 0 ;; | |
| @@ -52,16 +59,43 @@ esac | |||
| 52 | 59 | } | |
| 53 | 60 | t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) | |
| 54 | 61 | t.Setenv("FAKE_STATE", state) | |
| 62 | t.Setenv("FAKE_BUILD_SLEEP", sleep) | ||
| 63 | if fail { | ||
| 64 | t.Setenv("FAKE_BUILD_FAIL", "1") | ||
| 65 | } | ||
| 66 | return state | ||
| 67 | } | ||
| 55 | 68 | ||
| 56 | cfg, err := containerOptions(Options{ContainerRuntime: "docker"}, t.TempDir()) | ||
| 69 | func buildCount(t *testing.T, state string) int { | ||
| 70 | t.Helper() | ||
| 71 | builds, err := os.ReadFile(filepath.Join(state, "builds")) | ||
| 57 | 72 | if err != nil { | |
| 73 | if os.IsNotExist(err) { | ||
| 74 | return 0 | ||
| 75 | } | ||
| 58 | 76 | t.Fatal(err) | |
| 59 | 77 | } | |
| 78 | return strings.Count(string(builds), "build") | ||
| 79 | } | ||
| 80 | |||
| 81 | func imageJob() *runnerJob { | ||
| 60 | 82 | content := "version: 1\nfrom: alpine:3.22\nenv:\n X: \"1\"\nrun:\n - echo hello\n" | |
| 61 | job := &runnerJob{ | ||
| 83 | return &runnerJob{ | ||
| 62 | 84 | ID: "run1", Attempt: 1, | |
| 63 | 85 | Image: "file:go", ImageFile: "go", ImageDefinition: content, ImageSHA: ci.ImageDigest(content), | |
| 64 | 86 | } | |
| 87 | } | ||
| 88 | |||
| 89 | // A definition is built once, tagged by its digest, and reused by a later job. | ||
| 90 | // The fake runtime records the Dockerfile the agent generated and the build count. | ||
| 91 | func TestResolveImageBuildsAndCachesDefinition(t *testing.T) { | ||
| 92 | state := fakeImageRuntime(t, "0", false) | ||
| 93 | |||
| 94 | cfg, err := containerOptions(Options{ContainerRuntime: "docker"}, t.TempDir()) | ||
| 95 | if err != nil { | ||
| 96 | t.Fatal(err) | ||
| 97 | } | ||
| 98 | job := imageJob() | ||
| 65 | 99 | log := &strings.Builder{} | |
| 66 | 100 | s := &runnerSession{} | |
| 67 | 101 | tag, err := s.resolveImage(job, cfg, log, time.Hour) | |
| @@ -90,11 +124,145 @@ esac | |||
| 90 | 124 | if !strings.Contains(log.String(), "using cached") { | |
| 91 | 125 | t.Fatalf("second build was not cached:\n%s", log) | |
| 92 | 126 | } | |
| 93 | builds, err := os.ReadFile(filepath.Join(state, "builds")) | ||
| 127 | if n := buildCount(t, state); n != 1 { | ||
| 128 | t.Fatalf("build count = %d, want 1", n) | ||
| 129 | } | ||
| 130 | } | ||
| 131 | |||
| 132 | // Two slots, one digest: the first builds it, the second joins that build instead | ||
| 133 | // of spending the time and the disk again. Both end up with the same tag. | ||
| 134 | func TestTwoJobsBuildOneImage(t *testing.T) { | ||
| 135 | state := fakeImageRuntime(t, "0.3", false) | ||
| 136 | cfg, err := containerOptions(Options{ContainerRuntime: "docker"}, t.TempDir()) | ||
| 137 | if err != nil { | ||
| 138 | t.Fatal(err) | ||
| 139 | } | ||
| 140 | job := imageJob() | ||
| 141 | |||
| 142 | var ( | ||
| 143 | wg sync.WaitGroup | ||
| 144 | tags [2]string | ||
| 145 | errs [2]error | ||
| 146 | logs [2]strings.Builder | ||
| 147 | ) | ||
| 148 | start := make(chan struct{}) | ||
| 149 | for i := range tags { | ||
| 150 | wg.Add(1) | ||
| 151 | go func(i int) { | ||
| 152 | defer wg.Done() | ||
| 153 | <-start | ||
| 154 | // Two sessions, as two slots have: the lease is long enough that neither | ||
| 155 | // has to heartbeat during a build this short. | ||
| 156 | tags[i], errs[i] = (&runnerSession{}).resolveImage(job, cfg, &logs[i], time.Hour) | ||
| 157 | }(i) | ||
| 158 | } | ||
| 159 | close(start) | ||
| 160 | wg.Wait() | ||
| 161 | |||
| 162 | for i := range tags { | ||
| 163 | if errs[i] != nil { | ||
| 164 | t.Fatalf("job %d: %v", i, errs[i]) | ||
| 165 | } | ||
| 166 | if tags[i] != tags[0] { | ||
| 167 | t.Fatalf("two jobs got two tags: %q and %q", tags[0], tags[i]) | ||
| 168 | } | ||
| 169 | } | ||
| 170 | if n := buildCount(t, state); n != 1 { | ||
| 171 | t.Fatalf("%d builds for one digest, want 1", n) | ||
| 172 | } | ||
| 173 | // One of them waited, and said so in its own log: a reader of that job's log has | ||
| 174 | // to be able to tell why the steps had not started. | ||
| 175 | waited := strings.Contains(logs[0].String(), "being built by another job") || | ||
| 176 | strings.Contains(logs[1].String(), "being built by another job") | ||
| 177 | if !waited { | ||
| 178 | t.Fatalf("neither job said it was waiting:\n%s\n%s", logs[0].String(), logs[1].String()) | ||
| 179 | } | ||
| 180 | } | ||
| 181 | |||
| 182 | // A build that fails fails every job waiting on it: the same definition would fail | ||
| 183 | // the same way, and a second attempt only spends the time again. | ||
| 184 | func TestAFailedBuildFailsTheJobsWaitingOnIt(t *testing.T) { | ||
| 185 | fakeImageRuntime(t, "0.2", true) | ||
| 186 | cfg, err := containerOptions(Options{ContainerRuntime: "docker"}, t.TempDir()) | ||
| 94 | 187 | if err != nil { | |
| 95 | 188 | t.Fatal(err) | |
| 96 | 189 | } | |
| 97 | if strings.Count(string(builds), "build") != 1 { | ||
| 98 | t.Fatalf("build count = %d, want 1:\n%s", strings.Count(string(builds), "build"), builds) | ||
| 190 | job := imageJob() | ||
| 191 | |||
| 192 | var ( | ||
| 193 | wg sync.WaitGroup | ||
| 194 | errs [2]error | ||
| 195 | ) | ||
| 196 | start := make(chan struct{}) | ||
| 197 | for i := range errs { | ||
| 198 | wg.Add(1) | ||
| 199 | go func(i int) { | ||
| 200 | defer wg.Done() | ||
| 201 | <-start | ||
| 202 | _, errs[i] = (&runnerSession{}).resolveImage(job, cfg, &strings.Builder{}, time.Hour) | ||
| 203 | }(i) | ||
| 204 | } | ||
| 205 | close(start) | ||
| 206 | wg.Wait() | ||
| 207 | |||
| 208 | for i := range errs { | ||
| 209 | if errs[i] == nil { | ||
| 210 | t.Fatalf("job %d saw a successful build", i) | ||
| 211 | } | ||
| 212 | if !strings.Contains(errs[i].Error(), "go") { | ||
| 213 | t.Fatalf("job %d: %v does not name the image-file", i, errs[i]) | ||
| 214 | } | ||
| 215 | } | ||
| 216 | } | ||
| 217 | |||
| 218 | // The map is the whole mechanism, so its edges are worth pinning: one builder at a | ||
| 219 | // time, and a finished build is forgotten rather than answered forever. | ||
| 220 | func TestImageBuildsJoinAndLeave(t *testing.T) { | ||
| 221 | t.Parallel() | ||
| 222 | builds := &imageBuilds{} | ||
| 223 | first, mine := builds.join("tag") | ||
| 224 | if !mine { | ||
| 225 | t.Fatal("the first join is not the builder") | ||
| 226 | } | ||
| 227 | second, mine := builds.join("tag") | ||
| 228 | if mine || second != first { | ||
| 229 | t.Fatal("the second join started its own build of the same digest") | ||
| 230 | } | ||
| 231 | other, mine := builds.join("another") | ||
| 232 | if !mine || other == first { | ||
| 233 | t.Fatal("a different digest joined the wrong build") | ||
| 234 | } | ||
| 235 | builds.leave("tag") | ||
| 236 | again, mine := builds.join("tag") | ||
| 237 | if !mine || again == first { | ||
| 238 | t.Fatal("a forgotten digest was not rebuilt") | ||
| 239 | } | ||
| 240 | } | ||
| 241 | |||
| 242 | // waitForBuild is what keeps a waiting job's lease alive. It has to end for each of | ||
| 243 | // the three reasons a wait can end: the build finished, the lease went away, or a | ||
| 244 | // maintainer stopped the run. | ||
| 245 | func TestWaitForBuildEnds(t *testing.T) { | ||
| 246 | t.Parallel() | ||
| 247 | lease := 3 * time.Second // a one-second heartbeat interval | ||
| 248 | |||
| 249 | done := &imageBuild{done: make(chan struct{})} | ||
| 250 | close(done.done) | ||
| 251 | if err := (&runnerSession{}).waitForBuild(done, lease, "run", &strings.Builder{}); err != nil { | ||
| 252 | t.Fatalf("a finished build returned %v", err) | ||
| 253 | } | ||
| 254 | |||
| 255 | lost := pipeSession(t, runnerReply{Op: "heartbeat", OK: false}) | ||
| 256 | if err := lost.waitForBuild(&imageBuild{done: make(chan struct{})}, lease, "run", &strings.Builder{}); !errors.Is(err, errLeaseLost) { | ||
| 257 | t.Fatalf("a lost lease returned %v, want errLeaseLost", err) | ||
| 258 | } | ||
| 259 | |||
| 260 | canceled := pipeSession(t, runnerReply{Op: "heartbeat", OK: true, Cancel: "maintainer"}) | ||
| 261 | log := &strings.Builder{} | ||
| 262 | if err := canceled.waitForBuild(&imageBuild{done: make(chan struct{})}, lease, "run", log); !errors.Is(err, errCanceled) { | ||
| 263 | t.Fatalf("a canceled run returned %v, want errCanceled", err) | ||
| 264 | } | ||
| 265 | if !strings.Contains(log.String(), "maintainer") { | ||
| 266 | t.Fatalf("the log does not say who stopped the run:\n%s", log) | ||
| 99 | 267 | } | |
| 100 | 268 | } | |