vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Stop a running job at its runner's next heartbeat
0c14f0789af1ab0c43f99043acd7507eec4956e4 · Keenan Nemetz · 6d ago · parent 7e0b5950
Stop a running job at its runner's next heartbeat A maintainer could cancel a queued run and nothing else: CancelRun only retired waiting work, so a build that was clearly wrong or expensive held its runner until its budget ran out, or until somebody killed the container by hand. The queue cannot kill a process it does not own, but it can ask. Canceling a running run now marks the queue row and leaves the lease alone, because the runner still holds it and its partial attempt has to be acceptable. The runner sees the request on the heartbeat it already sends while a step or an image build runs, stops the container, writes one line to the log, and reports what it did. RecordAttempt and Finish then keep the canceled status instead of mapping the nonzero exit code to failed, so the partial log and step timings survive and the run is not a red build. A stop request whose runner dies first becomes canceled on the expiry sweep rather than being leased again, and a rerun clears the request: it is a new decision. The run page says who asked, distinguishes a run stopped while running from one canceled in the queue, and shows the interrupted step as stopped rather than failed. Closes #162.
server/workflows_test.go +65
| @@ -127,3 +127,68 @@ jobs: | |||
| 127 | 127 | t.Fatalf("cancel without CSRF = %d, want 403", resp.StatusCode) | |
| 128 | 128 | } | |
| 129 | 129 | } | |
| 130 | |||
| 131 | // A running run is stopped from the page by asking its runner. The page says who | ||
| 132 | // asked while the runner is still working, and says the run was stopped rather than | ||
| 133 | // failed once the attempt comes back. | ||
| 134 | func TestStoppingARunningRunFromTheBrowser(t *testing.T) { | ||
| 135 | t.Parallel() | ||
| 136 | f := newFixture(t) | ||
| 137 | f.publish(f.repo) | ||
| 138 | |||
| 139 | const id = "01M5R8XR12EZ5WYJDM0F1928D2" | ||
| 140 | now := time.Now().UTC() | ||
| 141 | run := runs.Run{ | ||
| 142 | ID: id, Repo: f.repo, Ref: "refs/heads/main", Branch: "main", | ||
| 143 | SHA: strings.Repeat("a", 40), Workflow: ".vole/workflows/ci.yaml", | ||
| 144 | WorkflowSHA: strings.Repeat("b", 40), StepsSHA: strings.Repeat("c", 40), | ||
| 145 | Job: "test", Status: runs.StatusQueued, Created: now, | ||
| 146 | Steps: []runs.Step{{Name: "build", Run: "go build ./...", Timeout: "1m0s"}}, | ||
| 147 | } | ||
| 148 | if _, err := f.srv.Runs.Enqueue(run, nil); err != nil { | ||
| 149 | t.Fatal(err) | ||
| 150 | } | ||
| 151 | leased, err := f.srv.Runs.Lease(runs.LeaseRequest{ | ||
| 152 | Owner: f.ownerID, Repos: []string{f.repo}, TTL: time.Hour, | ||
| 153 | }, now) | ||
| 154 | if err != nil || leased == nil { | ||
| 155 | t.Fatalf("lease = %+v, %v", leased, err) | ||
| 156 | } | ||
| 157 | |||
| 158 | sess, cookie := f.browserSession(t, f.ownerID) | ||
| 159 | resp, body := f.browserGet(t, "/r/"+f.repo+"/runs/"+id, cookie) | ||
| 160 | if resp.StatusCode != 200 || !strings.Contains(body, "Stop running run") { | ||
| 161 | t.Fatalf("running run page = %d:\n%s", resp.StatusCode, body) | ||
| 162 | } | ||
| 163 | |||
| 164 | resp, _ = f.browserPost(t, "/r/"+f.repo+"/runs/"+id+"/cancel", cookie, | ||
| 165 | url.Values{"csrf": {sess.CSRF}}) | ||
| 166 | if resp.StatusCode != 303 { | ||
| 167 | t.Fatalf("stop running run = %d", resp.StatusCode) | ||
| 168 | } | ||
| 169 | got, err := f.srv.Runs.Get(id) | ||
| 170 | if err != nil { | ||
| 171 | t.Fatal(err) | ||
| 172 | } | ||
| 173 | if got.Status != runs.StatusRunning || !got.CancelRequested || got.CancelActor != f.ownerID { | ||
| 174 | t.Fatalf("run after the request = %+v", got) | ||
| 175 | } | ||
| 176 | |||
| 177 | // The page says the stop is on its way, and does not offer the button again. | ||
| 178 | resp, body = f.browserGet(t, "/r/"+f.repo+"/runs/"+id, cookie) | ||
| 179 | if resp.StatusCode != 200 || !strings.Contains(body, "Stopping") || | ||
| 180 | !strings.Contains(body, f.ownerID) || strings.Contains(body, "Stop running run") { | ||
| 181 | t.Fatalf("page while stopping = %d:\n%s", resp.StatusCode, body) | ||
| 182 | } | ||
| 183 | |||
| 184 | // The runner stops and reports. The exit code is nonzero because the process was | ||
| 185 | // killed, and the page still says the run was canceled rather than failed. | ||
| 186 | if ok, err := f.srv.Runs.Finish(id, f.ownerID, 1, now.Add(time.Minute)); err != nil || !ok { | ||
| 187 | t.Fatalf("finish = %t, %v", ok, err) | ||
| 188 | } | ||
| 189 | resp, body = f.browserGet(t, "/r/"+f.repo+"/runs/"+id, cookie) | ||
| 190 | if resp.StatusCode != 200 || !strings.Contains(body, "canceled") || | ||
| 191 | !strings.Contains(body, "Stopped while running") || !strings.Contains(body, f.ownerID) { | ||
| 192 | t.Fatalf("page after the stop = %d:\n%s", resp.StatusCode, body) | ||
| 193 | } | ||
| 194 | } | ||