vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Run several jobs at once with --jobs

18c559ea0010dc22db13c889afda4fe9af3c15d7 · Keenan Nemetz · 3d ago · parent 91b47a22

Run several jobs at once with --jobs

A runner process leased one job and blocked on it, so the only way to get a
second job onto a machine was a second process: another host, another key,
another allowlist entry, and another --dir that learns the repositories cold.
The server was never the limit — a lease is taken per session and claimed by a
conditional update — so this is the runner catching up.

`--jobs N` is N slots in one process. Each slot owns a session and runs at most
one job, and everything that belongs to the process happens once, before the
slots start: the directories, the cache bounds, the leftover sweep, and the
connections. A slot that refuses a job stops every slot, because reconnecting
would ask the same instance the same question. A slot that loses its connection
reconnects with a doubling delay instead, so one flaky link does not take the
other slots' jobs with it; the job it was running is requeued when its lease
expires.

The log stream moves from the session to the job, which is what makes two jobs
on one session impossible to confuse: a slot has one, and it is the stream for
the job in front of it.

Deployment: `RUNNER_JOBS` in the first-boot renderer, the packaged argument
file, and the docs. The container limits are per job, so the machine has to
carry --jobs times what they allow; max_jobs_per_repo is what stops one
repository claiming every slot.

Refs: 01M6SCWCSX39QXS38QAHQSMVGX (#216)

ci/agent/slot_test.go +113

@@ -0,0 +1,113 @@
	1	package agent
	2
	3	// The slot is one worker: one session, one job at a time. --jobs N is N of them in
	4	// one process, which is where the failure policy matters: a lost connection belongs
	5	// to the slot, and a job the runner refuses belongs to the runner.
	6
	7	import (
	8	"bufio"
	9	"context"
	10	"encoding/json"
	11	"errors"
	12	"fmt"
	13	"io"
	14	"testing"
	15	"time"
	16	)
	17
	18	func TestNextDelayDoublesAndCaps(t *testing.T) {
	19	t.Parallel()
	20	for _, tc := range []struct{ from, want time.Duration }{
	21	{time.Second, 2 * time.Second},
	22	{20 * time.Second, reconnectMax},
	23	{reconnectMax, reconnectMax},
	24	} {
	25	if got := nextDelay(tc.from); got != tc.want {
	26	t.Errorf("nextDelay(%s) = %s, want %s", tc.from, got, tc.want)
	27	}
	28	}
	29	}
	30
	31	func TestSleepStopsWhenTheContextEnds(t *testing.T) {
	32	t.Parallel()
	33	ctx, cancel := context.WithCancel(context.Background())
	34	cancel()
	35	if sleep(ctx, time.Minute) {
	36	t.Fatal("a cancelled context slept the whole delay")
	37	}
	38	if !sleep(context.Background(), time.Millisecond) {
	39	t.Fatal("a live context did not finish a short wait")
	40	}
	41	}
	42
	43	// A refused job is the one error a slot must not retry: the runner and the instance
	44	// disagree about what a job may contain, and reconnecting asks the same question.
	45	func TestARefusedJobSurvivesWrapping(t *testing.T) {
	46	t.Parallel()
	47	refused := refusedJob{errors.New("run id is not a run id")}
	48	wrapped := fmt.Errorf("slot stopped: %w", error(refused))
	49	var got refusedJob
	50	if !errors.As(wrapped, &got) {
	51	t.Fatal("a wrapped refusal is no longer a refusal")
	52	}
	53	if got.Error() != "run id is not a run id" {
	54	t.Fatalf("the refusal lost its reason: %q", got.Error())
	55	}
	56	}
	57
	58	// pipeSession is a session with no ssh under it: requests go into a pipe, and the
	59	// answering goroutine writes one reply per request from the list. It is enough to
	60	// exercise what a slot does before it touches a checkout or a container.
	61	func pipeSession(t testing.T, replies ...runnerReply) runnerSession {
	62	t.Helper()
	63	reqR, reqW := io.Pipe()
	64	repR, repW := io.Pipe()
	65	go func() {
	66	defer repW.Close()
	67	dec := json.NewDecoder(reqR)
	68	enc := json.NewEncoder(repW)
	69	for _, reply := range replies {
	70	var req runnerRequest
	71	if err := dec.Decode(&req); err != nil {
	72	return
	73	}
	74	if err := enc.Encode(reply); err != nil {
	75	return
	76	}
	77	}
	78	}()
	79	t.Cleanup(func() { _ = reqW.Close() })
	80	return &runnerSession{
	81	in: bufio.NewWriter(reqW),
	82	out: bufio.NewReader(repR),
	83	enc: json.NewEncoder(reqW),
	84	dec: json.NewDecoder(repR),
	85	}
	86	}
	87
	88	// A leased job that does not have the shape this runner documents stops the slot
	89	// with a refusal, not with a retry loop that would lease it again.
	90	func TestASlotRefusesAJobItWillNotRun(t *testing.T) {
	91	t.Parallel()
	92	job := &runnerJob{ID: "not-a-ulid", Repo: "demo", SHA: "ab", Image: "alpine:3.22"}
	93	sess := pipeSession(t, runnerReply{Op: "lease", OK: true, Run: job})
	94	sl := &slot{sess: sess}
	95
	96	err := sl.serve(context.Background())
	97	var refused refusedJob
	98	if !errors.As(err, &refused) {
	99	t.Fatalf("serve = %v, want a refused job", err)
	100	}
	101	}
	102
	103	// A lease that answers "nothing to do" is not a failure: a one-shot slot leaves,
	104	// and a resident one asks again. Either way it is not a refusal.
	105	func TestAnEmptyLeaseIsNotARefusal(t *testing.T) {
	106	t.Parallel()
	107	sess := pipeSession(t, runnerReply{Op: "lease", OK: true})
	108	sl := &slot{sess: sess, opts: Options{Once: true}}
	109
	110	if err := sl.serve(context.Background()); err != nil {
	111	t.Fatalf("serve = %v, want no error from an empty one-shot lease", err)
	112	}
	113	}