vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Require an image for every CI job

70b702649b875d841e294f5a085ca784d43e5bc4 · Keenan Nemetz · 15d ago · parent 7da9111e

Require an image for every CI job

Runners now require an OCI runtime, and each workflow job must name either a registry image or an image-file definition under .vole/images. Definitions are validated at push time, pinned with the run, built locally by the runner, and cached by definition digest. Records and the run page carry the image and definition digest.

Split the CI guide into workflows and runners, and add reference pages for workflow and image definition fields.

server/integration_test.go +138−8

@@ -26,6 +26,7 @@ import (
	26	26
	27	27	"golang.org/x/crypto/ssh"
	28	28
		29	"github.com/rodent-software/vole/ci"
	29	30	"github.com/rodent-software/vole/ci/runs"
	30	31	"github.com/rodent-software/vole/core/git"
	31	32	"github.com/rodent-software/vole/forge/config"
@@ -40,6 +41,55 @@ import (
	40	41	// and written by none, which is what makes it safe for them to run concurrently.
	41	42	var binPath string
	42	43
		44	// fakeRuntimeDir holds a docker-shaped script for the integration tests. The real
		45	// runner now requires an OCI runtime, but these tests are about the queue, SSH
		46	// boundary and records, not about isolation. The script runs a step in the job tree
		47	// and otherwise answers the runtime commands the agent uses.
		48	var fakeRuntimeDir string
		49
		50	const fakeDocker = `#!/bin/sh
		51	# A Docker-shaped test runtime. It does not isolate anything; it runs the step in
		52	# the job tree so integration tests can exercise the runner without a daemon.
		53	cmd="$1"
		54	[ $# -gt 0 ] && shift
		55	case "$cmd" in
		56	ps)
		57	exit 0
		58	;;
		59	image)
		60	exit 0
		61	;;
		62	build)
		63	exit 0
		64	;;
		65	run)
		66	exit 0
		67	;;
		68	exec)
		69	while [ $# -gt 0 ]; do
		70	case "$1" in
		71	-w) shift 2 ;;
		72	--*) shift ;;
		73	-*) shift ;;
		74	*) break ;;
		75	esac
		76	done
		77	[ $# -gt 0 ] && shift
		78	[ $# -gt 0 ] && shift
		79	[ $# -gt 0 ] && shift
		80	script="$1"
		81	CI=true VOLE_CI=1 HOME=/tmp GIT_TERMINAL_PROMPT=0 /bin/sh -c "$script"
		82	exit $?
		83	;;
		84	rm)
		85	exit 0
		86	;;
		87	*)
		88	exit 0
		89	;;
		90	esac
		91	`
		92
	43	93	// Every test here calls t.Parallel(). These are integration tests: each one starts a
	44	94	// server, pushes over SSH and runs `git` subprocesses, so it spends almost all of its
	45	95	// time waiting on child processes and almost none of it on CPU. Serially that is ~150
@@ -67,6 +117,15 @@ func TestMain(m *testing.M) {
	67	117	fmt.Fprintln(os.Stderr, "build vole:", err)
	68	118	os.Exit(1)
	69	119	}
		120	fakeRuntimeDir = filepath.Join(dir, "runtime")
		121	if err := os.MkdirAll(fakeRuntimeDir, 0o755); err != nil {
		122	fmt.Fprintln(os.Stderr, err)
		123	os.Exit(1)
		124	}
		125	if err := os.WriteFile(filepath.Join(fakeRuntimeDir, "docker"), []byte(fakeDocker), 0o755); err != nil {
		126	fmt.Fprintln(os.Stderr, err)
		127	os.Exit(1)
		128	}
	70	129	code := m.Run()
	71	130	os.RemoveAll(dir)
	72	131	os.Exit(code)
@@ -2584,7 +2643,7 @@ func TestInvalidWorkflowRejectedAtPush(t *testing.T) {
	2584	2643	f.publish(f.repo)
	2585	2644
	2586	2645	broken := workRepo(t, map[string]string{
	2587		".vole/workflows/ci.yaml": "version: 1\njobs:\n test:\n steps:\n - name: forgot the command\n",
		2646	".vole/workflows/ci.yaml": "version: 1\njobs:\n test:\n image: alpine:3.22\n steps:\n - name: forgot the command\n",
	2588	2647	})
	2589	2648	_, errs, err := f.git(broken, "push", f.sshURL(f.repo), "main:refs/heads/main")
	2590	2649	if err == nil {
@@ -2598,7 +2657,7 @@ func TestInvalidWorkflowRejectedAtPush(t *testing.T) {
	2598	2657
	2599	2658	// The same push with a valid file is accepted.
	2600	2659	good := workRepo(t, map[string]string{
	2601		".vole/workflows/ci.yaml": "version: 1\non:\n push:\n branches: [main]\njobs:\n test:\n steps:\n - run: go test ./...\n",
		2660	".vole/workflows/ci.yaml": "version: 1\non:\n push:\n branches: [main]\njobs:\n test:\n image: alpine:3.22\n steps:\n - run: go test ./...\n",
	2602	2661	})
	2603	2662	if _, errs, err := f.git(good, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
	2604	2663	t.Fatalf("valid workflow rejected: %s", errs)
@@ -2612,8 +2671,17 @@ func (f *fixture) ciEnv() []string { return f.ciEnvFor(f.keyPath) }
	2612	2671
	2613	2672	func (f *fixture) ciEnvFor(keyPath string) []string {
	2614	2673	sshCmd := "ssh -i " + keyPath + " -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR"
	2615		return append(os.Environ(), "VOLE_SSH_COMMAND="+sshCmd, "GIT_SSH_COMMAND="+sshCmd,
		2674	env := append(os.Environ(), "VOLE_SSH_COMMAND="+sshCmd, "GIT_SSH_COMMAND="+sshCmd,
	2616	2675	"GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
		2676	// The runner requires an OCI runtime. Put the test script first so this package
		2677	// does not depend on Docker being installed or usable in a sandboxed test host.
		2678	for i, kv := range env {
		2679	if strings.HasPrefix(kv, "PATH=") {
		2680	env[i] = "PATH=" + fakeRuntimeDir + string(os.PathListSeparator) + strings.TrimPrefix(kv, "PATH=")
		2681	break
		2682	}
		2683	}
		2684	return env
	2617	2685	}
	2618	2686
	2619	2687	func (f *fixture) sshBase() string { return "ssh://git@" + f.sshAddr }
@@ -2653,6 +2721,7 @@ func TestPushRunsWorkflow(t *testing.T) {
	2653	2721	work := workRepo(t, workflowFile(`version: 1
	2654	2722	jobs:
	2655	2723	test:
		2724	image: alpine:3.22
	2656	2725	steps:
	2657	2726	- name: say hello
	2658	2727	run: echo "hello from the runner"
@@ -2727,6 +2796,63 @@ jobs:
	2727	2796	}
	2728	2797	}
	2729	2798
		2799	// A workflow can name a definition under .vole/images. The queue pins the
		2800	// definition, the runner builds or finds the digest tag, and the record says which
		2801	// definition ran.
		2802	func TestWorkflowUsesAFileImage(t *testing.T) {
		2803	t.Parallel()
		2804	f := newFixtureWith(t, func(c *config.Config) { c.LeaseSeconds = 2 })
		2805	f.publish(f.repo)
		2806
		2807	definition := "version: 1\nfrom: alpine:3.22\nrun:\n - echo image-built\n"
		2808	work := workRepo(t, map[string]string{
		2809	".vole/workflows/ci.yaml": `version: 1
		2810	jobs:
		2811	test:
		2812	image-file: go
		2813	steps:
		2814	- name: say hello
		2815	run: echo "hello from a file image"
		2816	`,
		2817	".vole/images/go.yaml": definition,
		2818	})
		2819	if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
		2820	t.Fatalf("push: %s", errs)
		2821	}
		2822
		2823	queued := f.waitForRun(t, 5time.Second, func(r runs.Run) bool { return r.Status == runs.StatusQueued })
		2824	wantSHA := ci.ImageDigest(definition)
		2825	if queued.Image != "file:go" \|\| queued.ImageFile != "go" \|\| queued.ImageSHA != wantSHA \|\|
		2826	queued.ImageDefinition != definition {
		2827	t.Fatalf("queued image = %+v, want file:go with digest %s", queued, wantSHA)
		2828	}
		2829
		2830	cmd := exec.Command(binPath, "runner", "--remote", f.sshBase(), "--once", "--dir", t.TempDir())
		2831	cmd.Env = f.ciEnv()
		2832	if out, err := cmd.CombinedOutput(); err != nil {
		2833	t.Fatalf("runner: %v\n%s", err, out)
		2834	}
		2835
		2836	done := f.waitForRun(t, 10time.Second, func(r runs.Run) bool {
		2837	return r.Status != runs.StatusQueued && r.Status != runs.StatusRunning
		2838	})
		2839	if done.Status != runs.StatusSuccess \|\| done.Image != "file:go" \|\| done.ImageFile != "go" \|\|
		2840	done.ImageSHA != wantSHA {
		2841	t.Fatalf("finished run = %+v", done)
		2842	}
		2843	m, err := f.srv.Repos.Open(f.repo)
		2844	if err != nil {
		2845	t.Fatal(err)
		2846	}
		2847	rec, err := runs.ReadRecord(m.Git(), metadata.Ref, done.ID)
		2848	if err != nil {
		2849	t.Fatal(err)
		2850	}
		2851	if rec == nil \|\| rec.Image != "file:go" \|\| rec.ImageFile != "go" \|\| rec.ImageSHA != wantSHA {
		2852	t.Fatalf("record = %+v", rec)
		2853	}
		2854	}
		2855
	2730	2856	// A runner started with --repos for something its account may not run refuses at
	2731	2857	// startup. That is the whole point of checking it before the loop: a filter that
	2732	2858	// matches nothing looks exactly like a quiet instance, so a runner that started
@@ -2761,6 +2887,7 @@ func TestRunRecordIsPortable(t *testing.T) {
	2761	2887	work := workRepo(t, workflowFile(`version: 1
	2762	2888	jobs:
	2763	2889	test:
		2890	image: alpine:3.22
	2764	2891	steps:
	2765	2892	- name: say hello
	2766	2893	run: echo "recorded, not just queued"
@@ -2890,6 +3017,7 @@ func TestKilledRunnerIsRequeued(t *testing.T) {
	2890	3017	work := workRepo(t, workflowFile(`version: 1
	2891	3018	jobs:
	2892	3019	slow:
		3020	image: alpine:3.22
	2893	3021	steps:
	2894	3022	- name: hold the lease
	2895	3023	# Long enough that the runner is certainly still working when it is killed
@@ -2956,6 +3084,7 @@ func TestQueuedRunUsesTheWorkflowItWasQueuedFrom(t *testing.T) {
	2956	3084	work := workRepo(t, workflowFile(`version: 1
	2957	3085	jobs:
	2958	3086	test:
		3087	image: alpine:3.22
	2959	3088	steps:
	2960	3089	- run: while [ ! -f `+sentinel+` ]; do sleep 0.1; done; echo first-version
	2961	3090	`))
@@ -2979,7 +3108,7 @@ jobs:
	2979	3108
	2980	3109	// Second push rewrites the workflow, while the first run is executing.
	2981	3110	if err := os.WriteFile(filepath.Join(work, ".vole/workflows/ci.yaml"),
	2982		[]byte("version: 1\njobs:\n test:\n steps:\n - run: echo second-version\n"), 0o644); err != nil {
		3111	[]byte("version: 1\njobs:\n test:\n image: alpine:3.22\n steps:\n - run: echo second-version\n"), 0o644); err != nil {
	2983	3112	t.Fatal(err)
	2984	3113	}
	2985	3114	if _, errs, err := runGit(t, work, "add", "-A"); err != nil {
@@ -3086,7 +3215,7 @@ func TestProtectedBranchRequiresGreenChecks(t *testing.T) {
	3086	3215	t.Fatal(err)
	3087	3216	}
	3088	3217
	3089		feature, head := f.cloneWithChange(t, workflowFile("version: 1\njobs:\n test:\n steps:\n - run: echo ok\n"), "ci")
		3218	feature, head := f.cloneWithChange(t, workflowFile("version: 1\njobs:\n test:\n image: alpine:3.22\n steps:\n - run: echo ok\n"), "ci")
	3090	3219	if _, errs, err := f.gitWithKey(writerKey, feature, "push", f.sshURL(f.repo), "HEAD:refs/heads/feature"); err != nil {
	3091	3220	t.Fatalf("feature: %s", errs)
	3092	3221	}
@@ -3122,7 +3251,7 @@ func TestOnlyAllowedRunnersGreenLightABranch(t *testing.T) {
	3122	3251	t.Fatal(err)
	3123	3252	}
	3124	3253
	3125		feature, head := f.cloneWithChange(t, workflowFile("version: 1\njobs:\n test:\n steps:\n - run: echo ok\n"), "ci")
		3254	feature, head := f.cloneWithChange(t, workflowFile("version: 1\njobs:\n test:\n image: alpine:3.22\n steps:\n - run: echo ok\n"), "ci")
	3126	3255	if _, errs, err := f.gitWithKey(writerKey, feature, "push", f.sshURL(f.repo), "HEAD:refs/heads/feature"); err != nil {
	3127	3256	t.Fatalf("feature: %s", errs)
	3128	3257	}
@@ -3291,7 +3420,7 @@ func TestFailingCheckBlocksTheMerge(t *testing.T) {
	3291	3420	t.Fatal(err)
	3292	3421	}
	3293	3422
	3294		feature, head := f.cloneWithChange(t, workflowFile("version: 1\njobs:\n test:\n steps:\n - run: exit 3\n"), "ci")
		3423	feature, head := f.cloneWithChange(t, workflowFile("version: 1\njobs:\n test:\n image: alpine:3.22\n steps:\n - run: exit 3\n"), "ci")
	3295	3424	if _, errs, err := f.gitWithKey(writerKey, feature, "push", f.sshURL(f.repo), "HEAD:refs/heads/feature"); err != nil {
	3296	3425	t.Fatalf("feature: %s", errs)
	3297	3426	}
@@ -3330,7 +3459,7 @@ func TestMissingRunBlocksWithAHint(t *testing.T) {
	3330	3459	}
	3331	3460
	3332	3461	// This workflow filters to main, so pushing a feature branch queues nothing.
	3333		feature, head := f.cloneWithChange(t, workflowFile("version: 1\non:\n push:\n branches: [main]\njobs:\n test:\n steps:\n - run: echo ok\n"), "ci")
		3462	feature, head := f.cloneWithChange(t, workflowFile("version: 1\non:\n push:\n branches: [main]\njobs:\n test:\n image: alpine:3.22\n steps:\n - run: echo ok\n"), "ci")
	3334	3463	if _, errs, err := f.gitWithKey(writerKey, feature, "push", f.sshURL(f.repo), "HEAD:refs/heads/feature"); err != nil {
	3335	3464	t.Fatalf("feature: %s", errs)
	3336	3465	}
@@ -3391,6 +3520,7 @@ func TestRunsOnWaitsForARunnerThatCarriesTheLabel(t *testing.T) {
	3391	3520	jobs:
	3392	3521	build:
	3393	3522	runs-on: [linux, fast]
		3523	image: alpine:3.22
	3394	3524	steps:
	3395	3525	- run: echo "on the right machine"
	3396	3526	`))