vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Require an image for every CI job

70b702649b875d841e294f5a085ca784d43e5bc4 · Keenan Nemetz · 15d ago · parent 7da9111e

Require an image for every CI job

Runners now require an OCI runtime, and each workflow job must name either a registry image or an image-file definition under .vole/images. Definitions are validated at push time, pinned with the run, built locally by the runner, and cached by definition digest. Records and the run page carry the image and definition digest.

Split the CI guide into workflows and runners, and add reference pages for workflow and image definition fields.

ci/workflow.go +178−33

@@ -10,6 +10,7 @@ package ci
	10	10
	11	11	import (
	12	12	"fmt"
		13	"io/fs"
	13	14	"os"
	14	15	"path"
	15	16	"path/filepath"
@@ -91,6 +92,18 @@ type Job struct {
	91	92	// had before targeting existed. A job is only leased to a runner carrying every
	92	93	// label here, so this is a filter on machines, not a preference among them.
	93	94	Labels []string
		95	// Image is the image the job runs in. It is either a registry reference from
		96	// `image:` or `file:<name>` for a definition under .vole/images.
		97	Image string
		98	// ImageFile names the custom definition when `image-file:` was used. It is empty
		99	// for a registry image.
		100	ImageFile string
		101	// ImageDefinition is the pinned definition text for an image-file job. It is set
		102	// when the workflow is validated against the tree, so a queued run keeps the
		103	// definition it was queued from.
		104	ImageDefinition string
		105	// ImageSHA is the digest of ImageDefinition.
		106	ImageSHA string
	94	107	// Artifacts are the files this job publishes as the run's outputs.
	95	108	Artifacts []Artifact
	96	109	// Pages, when set, publishes a directory as the repository's site. It is on the
@@ -164,6 +177,8 @@ type rawPages struct {
	164	177	type rawJob struct {
	165	178	Steps []rawStep `yaml:"steps"`
	166	179	RunsOn rawLabels `yaml:"runs-on"`
		180	Image string `yaml:"image"`
		181	ImageFile string `yaml:"image-file"`
	167	182	Artifacts []rawArtifact `yaml:"artifacts"`
	168	183	Pages *rawPages `yaml:"pages"`
	169	184	}
@@ -292,6 +307,32 @@ func Parse(file string, content string) (*Workflow, []string) {
	292	307	bad("jobs.%s: %d steps, limit %d", id, len(jr.Steps), MaxSteps)
	293	308	}
	294	309	job := Job{ID: id}
		310	// Every job names the environment it runs in. Either a registry image or a
		311	// definition under .vole/images; one or the other, never both and never
		312	// neither. The definition content is resolved in CheckWithImages, where the
		313	// rest of the tree is available.
		314	image := strings.TrimSpace(jr.Image)
		315	imageFile := strings.TrimSpace(jr.ImageFile)
		316	switch {
		317	case image != "" && imageFile != "":
		318	bad("jobs.%s: image and image-file are mutually exclusive", id)
		319	case image == "" && imageFile == "":
		320	bad("jobs.%s: image or image-file is required", id)
		321	case image != "":
		322	if err := ValidImageRef(image); err != nil {
		323	bad("jobs.%s.image: %v", id, err)
		324	} else {
		325	job.Image = image
		326	}
		327	default:
		328	name, err := ImageName(imageFile)
		329	if err != nil {
		330	bad("jobs.%s.image-file: %v", id, err)
		331	} else {
		332	job.ImageFile = name
		333	job.Image = "file:" + name
		334	}
		335	}
	295	336	// runs-on is validated here rather than at lease time: a label a job asks for and
	296	337	// no runner can ever carry leaves the job queued forever, which looks like a busy
	297	338	// pool rather than a mistake. A typo fails the push that introduced it.
@@ -412,8 +453,17 @@ func Parse(file string, content string) (*Workflow, []string) {
	412	453	return wf, nil
	413	454	}
	414	455
	415		// Check validates workflow files given as path to contents, e.g. a pushed tree.
		456	// Check validates workflow files given as path to contents, e.g. a pushed tree. It
		457	// has no image definitions to resolve, so a workflow that uses image-file is
		458	// reported as missing its definition. Callers with a tree use CheckWithImages.
	416	459	func Check(files map[string]string) ([]*Workflow, []string) {
		460	return CheckWithImages(files, nil)
		461	}
		462
		463	// CheckWithImages validates workflows and the .vole/images definitions they
		464	// reference. Every image file is parsed, even when unreferenced, so a broken
		465	// definition fails the push that introduced it rather than a later build.
		466	func CheckWithImages(files, images map[string]string) ([]*Workflow, []string) {
	417	467	var out []*Workflow
	418	468	var problems []string
	419	469
@@ -444,6 +494,69 @@ func Check(files map[string]string) ([]*Workflow, []string) {
	444	494	out = append(out, wf)
	445	495	}
	446	496	}
		497
		498	// One name per definition. A go.yaml and a go.yml would make image-file: go
		499	// ambiguous, which is a mistake better caught here than at build time.
		500	type foundImage struct {
		501	path string
		502	content string
		503	}
		504	byName := map[string]foundImage{}
		505	imagePaths := make([]string, 0, len(images))
		506	for p := range images {
		507	imagePaths = append(imagePaths, p)
		508	}
		509	sort.Strings(imagePaths)
		510	for _, p := range imagePaths {
		511	if !strings.HasPrefix(p, ImageDir+"/") {
		512	continue
		513	}
		514	rel := strings.TrimPrefix(p, ImageDir+"/")
		515	if strings.Contains(rel, "/") {
		516	problems = append(problems, fmt.Sprintf(
		517	"%s: image definitions live directly in %s, not in subdirectories", p, ImageDir))
		518	continue
		519	}
		520	if !strings.HasSuffix(rel, ".yaml") && !strings.HasSuffix(rel, ".yml") {
		521	problems = append(problems, fmt.Sprintf(
		522	"%s: an image definition must end in .yaml or .yml", p))
		523	continue
		524	}
		525	name, err := ImageName(rel)
		526	if err != nil {
		527	problems = append(problems, fmt.Sprintf("%s: %v", p, err))
		528	continue
		529	}
		530	def, probs := ParseImage(p, images[p])
		531	problems = append(problems, probs...)
		532	if def == nil {
		533	continue
		534	}
		535	if prev, exists := byName[name]; exists {
		536	problems = append(problems, fmt.Sprintf(
		537	"%s: image name %q is already defined by %s", p, name, prev.path))
		538	continue
		539	}
		540	byName[name] = foundImage{path: p, content: images[p]}
		541	}
		542
		543	for _, wf := range out {
		544	for i := range wf.Jobs {
		545	job := &wf.Jobs[i]
		546	if job.ImageFile == "" {
		547	continue
		548	}
		549	found, ok := byName[job.ImageFile]
		550	if !ok {
		551	problems = append(problems, fmt.Sprintf(
		552	"%s: jobs.%s.image-file: %q has no %s",
		553	wf.Path, job.ID, job.ImageFile, ImagePath(job.ImageFile)))
		554	continue
		555	}
		556	job.ImageDefinition = found.content
		557	job.ImageSHA = ImageDigest(found.content)
		558	}
		559	}
	447	560	return out, problems
	448	561	}
	449	562
@@ -456,6 +569,7 @@ func tidyYAMLError(err error) string {
	456	569	" not found in type ci.rawJob",
	457	570	" not found in type ci.rawStep",
	458	571	" not found in type ci.rawWorkflow",
		572	" not found in type ci.rawImageDef",
	459	573	" not found in type ci.raw",
	460	574	} {
	461	575	msg = strings.ReplaceAll(msg, suffix, " is not a known key here")
@@ -486,19 +600,26 @@ func At(g git.Repo, commit string) ([]Workflow, []string) {
	486	600	return nil, []string{fmt.Sprintf("cannot read %s: %v", Dir, err)}
	487	601	}
	488	602	files := map[string]string{}
		603	images := map[string]string{}
	489	604	shas := map[string]string{}
	490	605	for _, e := range blobs {
	491		if !strings.HasPrefix(e.Name, Dir+"/") {
	492		continue
	493		}
	494		data, err := g.BlobBySHA(e.SHA)
	495		if err != nil {
	496		return nil, []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)}
		606	switch {
		607	case strings.HasPrefix(e.Name, Dir+"/"):
		608	data, err := g.BlobBySHA(e.SHA)
		609	if err != nil {
		610	return nil, []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)}
		611	}
		612	files[e.Name] = string(data)
		613	shas[e.Name] = e.SHA
		614	case strings.HasPrefix(e.Name, ImageDir+"/"):
		615	data, err := g.BlobBySHA(e.SHA)
		616	if err != nil {
		617	return nil, []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)}
		618	}
		619	images[e.Name] = string(data)
	497	620	}
	498		files[e.Name] = string(data)
	499		shas[e.Name] = e.SHA
	500	621	}
	501		wfs, problems := Check(files)
		622	wfs, problems := CheckWithImages(files, images)
	502	623	for _, wf := range wfs {
	503	624	wf.BlobSHA = shas[wf.Path]
	504	625	}
@@ -516,39 +637,63 @@ func ValidateTree(g *git.Repo, commit string) []string {
	516	637	return []string{fmt.Sprintf("cannot read %s: %v", Dir, err)}
	517	638	}
	518	639	files := map[string]string{}
		640	images := map[string]string{}
	519	641	for _, e := range blobs {
	520		if !strings.HasPrefix(e.Name, Dir+"/") {
	521		continue
	522		}
	523		data, err := g.BlobBySHA(e.SHA)
	524		if err != nil {
	525		return []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)}
		642	switch {
		643	case strings.HasPrefix(e.Name, Dir+"/"):
		644	data, err := g.BlobBySHA(e.SHA)
		645	if err != nil {
		646	return []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)}
		647	}
		648	files[e.Name] = string(data)
		649	case strings.HasPrefix(e.Name, ImageDir+"/"):
		650	data, err := g.BlobBySHA(e.SHA)
		651	if err != nil {
		652	return []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)}
		653	}
		654	images[e.Name] = string(data)
	526	655	}
	527		files[e.Name] = string(data)
	528	656	}
	529		_, problems := Check(files)
		657	_, problems := CheckWithImages(files, images)
	530	658	return problems
	531	659	}
	532	660
	533	661	// CheckDir validates the workflows in a working tree, for `vole workflow check`.
	534	662	func CheckDir(root string) ([]*Workflow, []string, error) {
	535		pattern := filepath.Join(root, filepath.FromSlash(Dir), ".yml")
	536		paths, err := filepath.Glob(pattern)
	537		if err != nil {
	538		return nil, nil, err
		663	read := func(dir string, into map[string]string) error {
		664	return filepath.WalkDir(filepath.Join(root, filepath.FromSlash(dir)), func(p string, d fs.DirEntry, err error) error {
		665	if err != nil {
		666	if os.IsNotExist(err) {
		667	return nil
		668	}
		669	return err
		670	}
		671	if d.IsDir() {
		672	return nil
		673	}
		674	if !strings.HasSuffix(d.Name(), ".yaml") && !strings.HasSuffix(d.Name(), ".yml") {
		675	return nil
		676	}
		677	data, err := os.ReadFile(p)
		678	if err != nil {
		679	return err
		680	}
		681	rel, err := filepath.Rel(root, p)
		682	if err != nil {
		683	rel = p
		684	}
		685	into[filepath.ToSlash(rel)] = string(data)
		686	return nil
		687	})
	539	688	}
	540	689	files := map[string]string{}
	541		for _, p := range paths {
	542		data, err := os.ReadFile(p)
	543		if err != nil {
	544		return nil, nil, err
	545		}
	546		rel, err := filepath.Rel(root, p)
	547		if err != nil {
	548		rel = p
	549		}
	550		files[filepath.ToSlash(rel)] = string(data)
		690	images := map[string]string{}
		691	if err := read(Dir, files); err != nil {
		692	return nil, nil, err
		693	}
		694	if err := read(ImageDir, images); err != nil {
		695	return nil, nil, err
	551	696	}
	552		wfs, problems := Check(files)
		697	wfs, problems := CheckWithImages(files, images)
	553	698	return wfs, problems, nil
	554	699	}