vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Stop a runner by draining instead of killing its jobs
f642e0aa232df7a284b9d9026fe17b34eada634c · Keenan Nemetz · 2d ago · parent 77056ba8
Stop a runner by draining instead of killing its jobs SIGTERM ended the process mid-step, so a build that was nearly done was thrown away and its run sat unclaimed until the lease expired. With --jobs that is N jobs instead of one. A runner now stops taking work on a signal and lets the jobs it already has finish and report. The one wait it cannot interrupt from inside is a parked lease — the reply is read by the session's decoder, and nothing else can break that read — so the session is hung up from outside its lock and the slot exits. That costs nothing: a slot parked on a lease is holding no job. A second signal ends the process at once, which is what signal.Stop restores. The packaged unit allows 60 seconds for the drain (TimeoutStopSec), so a longer build is still killed; docs/runner.md and docs/deploy/runner.md say how to raise it and why. Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)
ci/agent/agent.go +46−1
| @@ -65,6 +65,11 @@ var errLeaseLost = errors.New("lease lost") | |||
| 65 | 65 | // reporting: a canceled run has a story, and the partial log and step timings are it. | |
| 66 | 66 | var errCanceled = errors.New("run canceled") | |
| 67 | 67 | ||
| 68 | // errStopped says this slot was asked to stop while it held no job. The session was | ||
| 69 | // hung up so a parked lease would answer instead of sitting out its wait, and nothing | ||
| 70 | // is lost: a slot that holds no job holds no work. | ||
| 71 | var errStopped = errors.New("the runner is stopping") | ||
| 72 | |||
| 68 | 73 | // refusedJob says the instance handed this runner a job whose shape this runner does | |
| 69 | 74 | // not accept. Reconnecting would ask the same instance the same question, so the | |
| 70 | 75 | // error stops every slot and the process says why: a runner that kept looping would | |
| @@ -544,6 +549,12 @@ func (sl *slot) run(ctx context.Context) error { | |||
| 544 | 549 | if err == nil { | |
| 545 | 550 | return nil | |
| 546 | 551 | } | |
| 552 | if ctx.Err() != nil { | ||
| 553 | // Stopping: the session went away because it was hung up, or a job | ||
| 554 | // finished after another slot stopped the runner. Either way this slot is | ||
| 555 | // done, and there is nothing to reconnect to. | ||
| 556 | return nil | ||
| 557 | } | ||
| 547 | 558 | var refused refusedJob | |
| 548 | 559 | if errors.As(err, &refused) || sl.opts.Once { | |
| 549 | 560 | return err | |
| @@ -575,7 +586,7 @@ func (sl *slot) serve(ctx context.Context) error { | |||
| 575 | 586 | if !sl.opts.Once { | |
| 576 | 587 | req.Wait = int(leaseWait.Seconds()) | |
| 577 | 588 | } | |
| 578 | rep, err := s.call(req) | ||
| 589 | rep, err := s.lease(ctx, req) | ||
| 579 | 590 | if err != nil { | |
| 580 | 591 | return err | |
| 581 | 592 | } | |
| @@ -617,6 +628,40 @@ func (sl *slot) serve(ctx context.Context) error { | |||
| 617 | 628 | } | |
| 618 | 629 | } | |
| 619 | 630 | ||
| 631 | // lease asks for a job and returns when the server answers or the runner is stopping. | ||
| 632 | // A resident runner parks on this call for up to leaseWait, and a shutdown that had | ||
| 633 | // to sit out the park would be half a minute of nothing happening. | ||
| 634 | // | ||
| 635 | // The reply is read by the session's decoder, which cannot be interrupted from | ||
| 636 | // inside it; ending the connection is what unblocks it, and that costs nothing here | ||
| 637 | // because a slot parked on a lease is holding no job. | ||
| 638 | func (s *runnerSession) lease(ctx context.Context, req runnerRequest) (runnerReply, error) { | ||
| 639 | type answer struct { | ||
| 640 | rep runnerReply | ||
| 641 | err error | ||
| 642 | } | ||
| 643 | done := make(chan answer, 1) | ||
| 644 | go func() { | ||
| 645 | rep, err := s.call(req) | ||
| 646 | done <- answer{rep, err} | ||
| 647 | }() | ||
| 648 | select { | ||
| 649 | case a := <-done: | ||
| 650 | return a.rep, a.err | ||
| 651 | case <-ctx.Done(): | ||
| 652 | s.hangup() | ||
| 653 | return runnerReply{}, errStopped | ||
| 654 | } | ||
| 655 | } | ||
| 656 | |||
| 657 | // hangup ends the connection from outside the session's lock: taking the lock would | ||
| 658 | // wait for the goroutine that holds it, which is the one parked on the read. | ||
| 659 | func (s *runnerSession) hangup() { | ||
| 660 | if s.cmd != nil && s.cmd.Process != nil { | ||
| 661 | _ = s.cmd.Process.Signal(syscall.SIGTERM) | ||
| 662 | } | ||
| 663 | } | ||
| 664 | |||
| 620 | 665 | func closeSlots(slots []*slot) { | |
| 621 | 666 | for _, sl := range slots { | |
| 622 | 667 | sl.close() | |