vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Record a job that never started as could-not-run, not as a failure
70c9ad4eb3f3d0cd39159e647cde77afba75b944 · Keenan Nemetz · 7h ago · parent 9704d735
Record a job that never started as could-not-run, not as a failure A run that never executed a step was recorded exit=1, steps=nil, and the run page said the same thing it says when a test fails. With checks required on a protected branch, a runner that could not build an image red-flagged a commit whose tests pass, and the only way back was a rerun by hand. Two facts were collapsed into one: the job ran and something failed, which is the repository's problem, and the job never ran, which is the machine's. The runner knows the difference exactly once, and it does not need to guess whose fault it was: if no step ran, nothing about the commit was tested. - A record can now say where it stopped: Failure=setup means the attempt ended before the job's first step, with the runner's own one-line reason in FailureMessage, masked the way the log is. The reason is what the page shows instead of making a reader open the log to learn the machine broke. - A run whose newest attempt has that origin is StatusError, which the pages call "could not run". It is not StatusFailed: it is not evidence against the commit. - The gate reads it as a wait, not a refusal. Landing stays blocked because nothing is verified, but the commit is not blamed, and the gap names the reason and the rerun command. - The run list counts and filters it, the run page explains it, and the change list says "CI could not run" rather than "CI failed". A step that exits non-zero is still a failure and still reds the check, because that one is the code's. `vole runner` still reports a setup failure with exit 1; the origin is what carries the meaning, so an old runner and an old instance keep talking. Not done here, on purpose: a run whose attempts all vanished before reporting (the lease-loss exhaustion in RequeueExpired) still ends as failed. That run has no record to say what ran, and a job that repeatedly kills its runner is not distinguishable from a machine problem from the queue's side; issue #243 is where that gets a signal. Nor is a setup failure requeued automatically: MaxAttempts is about flaky tests, not broken runners, and #247 already stops a runner whose runtime cannot start a container from serving at all. A rerun is a button. The finish path now takes an attemptOutcome instead of six positional arguments, which is also what lets a failure after the last step keep the steps that ran instead of dropping them. Closes 01M6Z05K5PM58EE3H6VE2P2EYD
forge/landing/gate.go +43−17
| @@ -252,23 +252,8 @@ func (g Gate) CheckSummary(sha, branch string) ([]string, bool, error) { | |||
| 252 | 252 | for _, wf := range required { | |
| 253 | 253 | for _, job := range wf.Jobs { | |
| 254 | 254 | run := results[runs.Key(wf.Path, job.ID)] | |
| 255 | switch { | ||
| 256 | case run == nil: | ||
| 257 | add(fmt.Sprintf( | ||
| 258 | "%s: job %q has no run for %s — push the branch and let CI run on it "+ | ||
| 259 | "(this job filters on [%s]; a commit tested only on other branches has no result here)", | ||
| 260 | wf.Path, job.ID, format.SHA(sha), strings.Join(wf.Branches, " ")), true) | ||
| 261 | case run.Status == runs.StatusFailed: | ||
| 262 | add(fmt.Sprintf( | ||
| 263 | "%s: job %q failed for %s (exit %d)", wf.Path, job.ID, format.SHA(sha), run.ExitCode), false) | ||
| 264 | case run.Status == runs.StatusCanceled: | ||
| 265 | add(fmt.Sprintf( | ||
| 266 | "%s: job %q was canceled for %s", wf.Path, job.ID, format.SHA(sha)), false) | ||
| 267 | case run.Status != runs.StatusSuccess: | ||
| 268 | add(fmt.Sprintf( | ||
| 269 | "%s: job %q is %s for %s; wait for it to finish", wf.Path, job.ID, run.Status, format.SHA(sha)), true) | ||
| 270 | } | ||
| 271 | if run == nil || run.Status != runs.StatusSuccess { | ||
| 255 | if problem, wait, gap := checkGap(wf, job, sha, run); gap { | ||
| 256 | add(problem, wait) | ||
| 272 | 257 | continue | |
| 273 | 258 | } | |
| 274 | 259 | // The result has to be attributable to somebody the host allowed, or it | |
| @@ -296,6 +281,47 @@ func (g Gate) CheckSummary(sha, branch string) ([]string, bool, error) { | |||
| 296 | 281 | return gaps, len(gaps) > 0 && pending, nil | |
| 297 | 282 | } | |
| 298 | 283 | ||
| 284 | // checkGap is what one job's newest run means for a landing: why it is not | ||
| 285 | // satisfied, and whether that reason is a wait. Success returns no gap, and the | ||
| 286 | // caller then asks who reported it, which is the one question a status cannot answer. | ||
| 287 | // | ||
| 288 | // A setup failure is a wait rather than a refusal: the job never ran, so the commit is | ||
| 289 | // not what failed, and a required check must not red it. The run page says which | ||
| 290 | // machine stopped it and why, and a rerun is one command away. | ||
| 291 | func checkGap(wf *ci.Workflow, job ci.Job, sha string, run *runs.Run) (problem string, wait, gap bool) { | ||
| 292 | switch { | ||
| 293 | case run == nil: | ||
| 294 | return fmt.Sprintf( | ||
| 295 | "%s: job %q has no run for %s — push the branch and let CI run on it "+ | ||
| 296 | "(this job filters on [%s]; a commit tested only on other branches has no result here)", | ||
| 297 | wf.Path, job.ID, format.SHA(sha), strings.Join(wf.Branches, " ")), true, true | ||
| 298 | case run.Status == runs.StatusFailed: | ||
| 299 | return fmt.Sprintf("%s: job %q failed for %s (exit %d)", | ||
| 300 | wf.Path, job.ID, format.SHA(sha), run.ExitCode), false, true | ||
| 301 | case run.Status == runs.StatusCanceled: | ||
| 302 | return fmt.Sprintf("%s: job %q was canceled for %s", | ||
| 303 | wf.Path, job.ID, format.SHA(sha)), false, true | ||
| 304 | case run.Status == runs.StatusError: | ||
| 305 | return fmt.Sprintf( | ||
| 306 | "%s: job %q could not run for %s (%s); the job never started, so nothing was "+ | ||
| 307 | "tested. Re-run it with `vole run rerun %s` once the machine can build", | ||
| 308 | wf.Path, job.ID, format.SHA(sha), failureReason(run), run.ID), true, true | ||
| 309 | case run.Status != runs.StatusSuccess: | ||
| 310 | return fmt.Sprintf("%s: job %q is %s for %s; wait for it to finish", | ||
| 311 | wf.Path, job.ID, run.Status, format.SHA(sha)), true, true | ||
| 312 | } | ||
| 313 | return "", false, false | ||
| 314 | } | ||
| 315 | |||
| 316 | // failureReason is the runner's own words for a run that never started, or a | ||
| 317 | // sentence saying it left none. It is one line by construction, so it fits a gap. | ||
| 318 | func failureReason(run *runs.Run) string { | ||
| 319 | if run.FailureMessage != "" { | ||
| 320 | return run.FailureMessage | ||
| 321 | } | ||
| 322 | return "the runner recorded no reason" | ||
| 323 | } | ||
| 324 | |||
| 299 | 325 | // RunnerSummary renders the allowlist for a message. Admins are named as implicit | |
| 300 | 326 | // because an operator reading the refusal needs to know who already counts. | |
| 301 | 327 | func RunnerSummary(a *access.ACL) string { | |