vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Renew the lease at every step boundary
539d6e6cd19ab148c18467d3b9dc9746f30d87b7 · Keenan Nemetz · 21h ago · parent c980fe53
Renew the lease at every step boundary A step heartbeats while it runs, but its ticker starts with the step, so a step that finishes inside one interval never heartbeats at all. An attempt whose steps add up to more than the lease therefore expired between two of them, and the next heartbeat — the first tick of the long step at the end — arrived to find the queue had already given the job away. Twelve short steps in front of `go test` is exactly that shape: it killed attempts at 61s, 65s and 77s with no result, five times per run, and the retry re-ran every step from the start. Send one heartbeat before each step, which is the only place a step shorter than the interval can renew. A lease that is really gone still stops the attempt, and the command ticker keeps covering a step that runs longer than one interval. Verified on a runner at the default lease_seconds = 60: a nine-step workflow that died four times at the boundary ran to completion inside one lease, and failed only on the ci/cache test that #236 covers.
ci/agent/agent.go +22
| @@ -822,6 +822,19 @@ func (s *runnerSession) call(req runnerRequest) (runnerReply, error) { | |||
| 822 | 822 | return rep, nil | |
| 823 | 823 | } | |
| 824 | 824 | ||
| 825 | // heartbeat renews the lease and asks whether a maintainer has stopped the run. It | ||
| 826 | // is for the places that are not a running command: a command heartbeats on its own | ||
| 827 | // ticker while it runs, and this is what keeps the lease across the steps that are | ||
| 828 | // too short to have one. A broken connection answers false, so an attempt that can | ||
| 829 | // no longer reach the instance stops instead of working without a lease. | ||
| 830 | func (s *runnerSession) heartbeat(runID string) bool { | ||
| 831 | rep, err := s.call(runnerRequest{Op: "heartbeat", Run: runID}) | ||
| 832 | if err != nil { | ||
| 833 | return false | ||
| 834 | } | ||
| 835 | return rep.OK | ||
| 836 | } | ||
| 837 | |||
| 825 | 838 | func (s *runnerSession) close() { | |
| 826 | 839 | s.mu.Lock() | |
| 827 | 840 | _ = s.enc.Encode(runnerRequest{Op: "bye"}) | |
| @@ -1084,6 +1097,15 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 1084 | 1097 | fmt.Fprintf(log, "\nthe job's %s budget ran out before this step\n", job.Timeout) | |
| 1085 | 1098 | code = 1 | |
| 1086 | 1099 | } else { | |
| 1100 | // Renew before the step, not only during it: a step heartbeats while it runs, | ||
| 1101 | // but its ticker starts with the step. Steps that each finish inside one | ||
| 1102 | // interval never heartbeat at all, so an attempt whose steps add up to more | ||
| 1103 | // than the lease expires between two of them and the next heartbeat arrives | ||
| 1104 | // to find the queue has already given the job away. | ||
| 1105 | if !s.heartbeat(job.ID) { | ||
| 1106 | fmt.Fprintf(os.Stderr, "vole runner: lease lost before %q; stopping\n", step.Name) | ||
| 1107 | return | ||
| 1108 | } | ||
| 1087 | 1109 | code, lost, canceled = s.execStep(job.ID, sandbox, step, limit, lease, log) | |
| 1088 | 1110 | } | |
| 1089 | 1111 | // The mask holds back a few bytes so a value split across writes is caught; | |