vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Stop a runner by draining instead of killing its jobs
f642e0aa232df7a284b9d9026fe17b34eada634c · Keenan Nemetz · 3d ago · parent 77056ba8
Stop a runner by draining instead of killing its jobs SIGTERM ended the process mid-step, so a build that was nearly done was thrown away and its run sat unclaimed until the lease expired. With --jobs that is N jobs instead of one. A runner now stops taking work on a signal and lets the jobs it already has finish and report. The one wait it cannot interrupt from inside is a parked lease — the reply is read by the session's decoder, and nothing else can break that read — so the session is hung up from outside its lock and the slot exits. That costs nothing: a slot parked on a lease is holding no job. A second signal ends the process at once, which is what signal.Stop restores. The packaged unit allows 60 seconds for the drain (TimeoutStopSec), so a longer build is still killed; docs/runner.md and docs/deploy/runner.md say how to raise it and why. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/agent/slot_test.go +54
| @@ -111,3 +111,57 @@ func TestAnEmptyLeaseIsNotARefusal(t *testing.T) { | |||
| 111 | 111 | t.Fatalf("serve = %v, want no error from an empty one-shot lease", err) | |
| 112 | 112 | } | |
| 113 | 113 | } | |
| 114 | |||
| 115 | // silentSession is a session whose reader never gets a reply: what a resident runner | ||
| 116 | // looks like while it is parked on a lease. | ||
| 117 | func silentSession(t *testing.T) *runnerSession { | ||
| 118 | t.Helper() | ||
| 119 | reqR, reqW := io.Pipe() | ||
| 120 | repR, repW := io.Pipe() | ||
| 121 | go func() { _, _ = io.Copy(io.Discard, reqR) }() | ||
| 122 | t.Cleanup(func() { | ||
| 123 | _ = reqW.Close() | ||
| 124 | _ = repW.Close() | ||
| 125 | }) | ||
| 126 | return &runnerSession{ | ||
| 127 | in: bufio.NewWriter(reqW), | ||
| 128 | out: bufio.NewReader(repR), | ||
| 129 | enc: json.NewEncoder(reqW), | ||
| 130 | dec: json.NewDecoder(repR), | ||
| 131 | } | ||
| 132 | } | ||
| 133 | |||
| 134 | // A parked lease is the only wait a stopping runner cannot interrupt from inside, so | ||
| 135 | // it has to end when the runner is asked to stop — otherwise a shutdown sits out the | ||
| 136 | // whole wait window while nothing is running. | ||
| 137 | func TestAParkedLeaseEndsWhenTheRunnerStops(t *testing.T) { | ||
| 138 | t.Parallel() | ||
| 139 | sess := silentSession(t) | ||
| 140 | ctx, cancel := context.WithCancel(context.Background()) | ||
| 141 | done := make(chan error, 1) | ||
| 142 | go func() { | ||
| 143 | _, err := sess.lease(ctx, runnerRequest{Op: "lease", Wait: 25}) | ||
| 144 | done <- err | ||
| 145 | }() | ||
| 146 | cancel() | ||
| 147 | select { | ||
| 148 | case err := <-done: | ||
| 149 | if !errors.Is(err, errStopped) { | ||
| 150 | t.Fatalf("a canceled lease returned %v, want errStopped", err) | ||
| 151 | } | ||
| 152 | case <-time.After(5 * time.Second): | ||
| 153 | t.Fatal("a canceled lease stayed parked") | ||
| 154 | } | ||
| 155 | } | ||
| 156 | |||
| 157 | // A slot that was told to stop before it leased anything does not lease at all: the | ||
| 158 | // shutdown is not a last chance to pick up work. | ||
| 159 | func TestAStoppedSlotTakesNoWork(t *testing.T) { | ||
| 160 | t.Parallel() | ||
| 161 | sess := silentSession(t) | ||
| 162 | ctx, cancel := context.WithCancel(context.Background()) | ||
| 163 | cancel() | ||
| 164 | if err := (&slot{sess: sess}).serve(ctx); err != nil { | ||
| 165 | t.Fatalf("serve = %v, want no error from a stopped slot", err) | ||
| 166 | } | ||
| 167 | } | ||