vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · ci/agent/build.go

Line attribution on main · 214 line(s)

View file History

70b70264 Keenan Nemetz 15d ago	1	package agent
	2
	3	// Building a custom image from a .vole/images definition.
	4	//
	5	// The job carries the pinned definition text and its digest, so the runner does
	6	// not need to read the repository again and cannot build a definition that changed
	7	// after the job was queued. The image is tagged by that digest, which makes a
	8	// second job with the same definition reuse the first build.
	9
	10	import (
	11	"context"
77056ba8 Keenan Nemetz 2d ago	12	"errors"
70b70264 Keenan Nemetz 15d ago	13	"fmt"
6bfc4294 Keenan Nemetz 12d ago	14	"io"
70b70264 Keenan Nemetz 15d ago	15	"os"
	16	"os/exec"
	17	"path/filepath"
77056ba8 Keenan Nemetz 2d ago	18	"sync"
70b70264 Keenan Nemetz 15d ago	19	"time"
70b70264 Keenan Nemetz 15d ago	20
d37d301f Keenan Nemetz 16h ago	21	"rodent.software/vole/ci"
d37d301f Keenan Nemetz 16h ago	22	"rodent.software/vole/ci/container"
70b70264 Keenan Nemetz 15d ago	23	)
	24
	25	const imageBuildTimeout = 30 * time.Minute
	26
77056ba8 Keenan Nemetz 2d ago	27	// One build per digest, per process.
	28	//
	29	// With --jobs, two slots can lease jobs that name the same image-file: same bytes,
	30	// same tag, twice the CPU and twice the disk, and two processes racing to tag the
	31	// result. The first slot builds; the others wait on the same answer.
	32	//
	33	// Waiting is not free. A build can run for half an hour, and a runner that stops
	34	// hearing from the queue loses the run it is holding, so a waiter keeps its lease
	35	// alive exactly as a build of its own would have, and stops when the lease or the
	36	// run goes away.
	37	//
	38	// Two runners sharing a --dir can still both build the digest. That is waste, not
	39	// damage: the definition is pinned by digest, so both builds produce the same image
	40	// under the same tag. Serialising that too would need a lock file per digest beside
	41	// the caches, which is more machinery than the case is worth.
	42
	43	// imageBuild is one in-flight build of a digest. done closes when the attempt has an
	44	// answer, success or failure; err is that answer.
	45	type imageBuild struct {
	46	done chan struct{}
	47	err error
	48	}
	49
	50	// imageBuilds is the set of builds this process has in flight, shared by its slots.
	51	type imageBuilds struct {
	52	mu sync.Mutex
	53	running map[string]*imageBuild
	54	}
	55
	56	// join makes this caller the builder of tag, or hands it the build already running.
	57	func (b imageBuilds) join(tag string) (build imageBuild, mine bool) {
	58	b.mu.Lock()
	59	defer b.mu.Unlock()
	60	if cur, ok := b.running[tag]; ok {
	61	return cur, false
	62	}
	63	if b.running == nil {
	64	b.running = make(map[string]*imageBuild)
	65	}
	66	build = &imageBuild{done: make(chan struct{})}
	67	b.running[tag] = build
	68	return build, true
	69	}
	70
	71	// leave forgets a finished build, so the next job that wants the digest asks the
	72	// runtime again — which is also how an image somebody deleted gets rebuilt.
	73	func (b *imageBuilds) leave(tag string) {
	74	b.mu.Lock()
	75	defer b.mu.Unlock()
	76	delete(b.running, tag)
	77	}
	78
70b70264 Keenan Nemetz 15d ago	79	// resolveImage returns the reference to run. A registry image is used as written;
70b70264 Keenan Nemetz 15d ago	80	// an image-file definition is built locally and cached under a digest tag.
6bfc4294 Keenan Nemetz 12d ago	81	func (s runnerSession) resolveImage(job runnerJob, box runnerBox, log io.Writer, lease time.Duration) (string, error) {
70b70264 Keenan Nemetz 15d ago	82	if job.ImageFile == "" {
	83	if err := ci.ValidImageRef(job.Image); err != nil {
	84	return "", fmt.Errorf("the job image: %w", err)
	85	}
	86	return job.Image, nil
	87	}
	88	if job.ImageDefinition == "" {
	89	return "", fmt.Errorf("image-file %q has no pinned definition", job.ImageFile)
	90	}
	91	if job.ImageSHA == "" {
	92	return "", fmt.Errorf("image-file %q has no definition digest", job.ImageFile)
	93	}
3b39a84c Keenan Nemetz 14d ago	94	tag := container.ImageTag(job.ImageFile, job.ImageSHA)
3b39a84c Keenan Nemetz 14d ago	95	if box.Pull != "always" && container.Exists(box.Runtime, tag) {
70b70264 Keenan Nemetz 15d ago	96	fmt.Fprintf(log, "image-file %s: using cached %s\n", job.ImageFile, tag)
	97	return tag, nil
	98	}
77056ba8 Keenan Nemetz 2d ago	99	if box.builds == nil {
	100	// A runnerBox built by hand (a test, or a caller that never validated) has no
	101	// shared state to join; building alone is what it did before --jobs.
	102	if err := s.buildImageFile(job, tag, box, log, lease); err != nil {
	103	return "", err
	104	}
	105	return tag, nil
	106	}
	107
	108	build, mine := box.builds.join(tag)
	109	if mine {
	110	err := s.buildImageFile(job, tag, box, log, lease)
	111	build.err = err
	112	// Leave before closing, so a job that arrives after this one takes the
	113	// finished entry starts a fresh build instead of reading an answer about a
	114	// digest it did not ask for. Waiters already on done are unaffected.
	115	box.builds.leave(tag)
	116	close(build.done)
	117	if err != nil {
	118	return "", err
	119	}
	120	return tag, nil
	121	}
	122
	123	// Another slot is building it. The line goes in this job's log so a reader knows
	124	// why the steps have not started.
	125	fmt.Fprintf(log, "image-file %s: %s is being built by another job; waiting\n", job.ImageFile, tag)
	126	if err := s.waitForBuild(build, lease, job.ID, log); err != nil {
	127	return "", err
	128	}
	129	if build.err != nil {
	130	// A failed build is this job's failure too: the same definition would fail
	131	// the same way, and a second attempt would only spend the time again. A lease
	132	// that went away belongs to the builder, though, and means only that nobody
	133	// may run this image on this runner's word.
	134	if errors.Is(build.err, errLeaseLost) \|\| errors.Is(build.err, errCanceled) {
	135	return "", build.err
	136	}
	137	return "", fmt.Errorf("image-file %q: %w", job.ImageFile, build.err)
	138	}
	139	return tag, nil
	140	}
70b70264 Keenan Nemetz 15d ago	141
77056ba8 Keenan Nemetz 2d ago	142	// buildImageFile renders the pinned definition, builds it, and tags the result by
	143	// its digest.
	144	func (s runnerSession) buildImageFile(job runnerJob, tag string, box runnerBox, log io.Writer, lease time.Duration) error {
70b70264 Keenan Nemetz 15d ago	145	dockerfile, err := ci.RenderImage(job.ImageDefinition)
70b70264 Keenan Nemetz 15d ago	146	if err != nil {
77056ba8 Keenan Nemetz 2d ago	147	return fmt.Errorf("image-file %q: %w", job.ImageFile, err)
70b70264 Keenan Nemetz 15d ago	148	}
	149	dir, err := os.MkdirTemp("", "vole-image-*")
	150	if err != nil {
77056ba8 Keenan Nemetz 2d ago	151	return err
70b70264 Keenan Nemetz 15d ago	152	}
	153	defer os.RemoveAll(dir)
	154	if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o600); err != nil {
77056ba8 Keenan Nemetz 2d ago	155	return err
70b70264 Keenan Nemetz 15d ago	156	}
	157
	158	args := []string{"build", "-t", tag, "-f", filepath.Join(dir, "Dockerfile"), dir}
	159	switch {
3b39a84c Keenan Nemetz 14d ago	160	case box.Pull == "always" && box.Runtime == "podman":
70b70264 Keenan Nemetz 15d ago	161	args = append(args, "--pull=always")
3b39a84c Keenan Nemetz 14d ago	162	case box.Pull == "always":
70b70264 Keenan Nemetz 15d ago	163	args = append(args, "--pull")
	164	}
	165	ctx, cancel := context.WithTimeout(context.Background(), imageBuildTimeout)
	166	defer cancel()
3b39a84c Keenan Nemetz 14d ago	167	cmd := exec.CommandContext(ctx, box.Runtime, args...)
0c14f078 Keenan Nemetz 6d ago	168	code, lost, canceled := s.runCommand(ctx, cmd, job.ID, lease, nil, log)
70b70264 Keenan Nemetz 15d ago	169	if lost {
77056ba8 Keenan Nemetz 2d ago	170	return errLeaseLost
70b70264 Keenan Nemetz 15d ago	171	}
0c14f078 Keenan Nemetz 6d ago	172	if canceled {
77056ba8 Keenan Nemetz 2d ago	173	return errCanceled
0c14f078 Keenan Nemetz 6d ago	174	}
70b70264 Keenan Nemetz 15d ago	175	if ctx.Err() == context.DeadlineExceeded {
77056ba8 Keenan Nemetz 2d ago	176	return fmt.Errorf("building image-file %q timed out after %s", job.ImageFile, imageBuildTimeout)
70b70264 Keenan Nemetz 15d ago	177	}
70b70264 Keenan Nemetz 15d ago	178	if code != 0 {
77056ba8 Keenan Nemetz 2d ago	179	return fmt.Errorf("building image-file %q exited %d", job.ImageFile, code)
70b70264 Keenan Nemetz 15d ago	180	}
3b39a84c Keenan Nemetz 14d ago	181	if !container.Exists(box.Runtime, tag) {
77056ba8 Keenan Nemetz 2d ago	182	return fmt.Errorf("building image-file %q produced no image tagged %s", job.ImageFile, tag)
70b70264 Keenan Nemetz 15d ago	183	}
70b70264 Keenan Nemetz 15d ago	184	fmt.Fprintf(log, "\nimage-file %s: built %s\n", job.ImageFile, tag)
77056ba8 Keenan Nemetz 2d ago	185	return nil
	186	}
	187
	188	// waitForBuild waits for another slot's build, keeping this job's lease alive for
	189	// as long as a build of its own would have. A waiter that went quiet would be
	190	// requeued to somebody else mid-wait, and would then go on to run a job the queue
	191	// no longer considers its own.
	192	func (s runnerSession) waitForBuild(build imageBuild, lease time.Duration, runID string, log io.Writer) error {
	193	interval := lease / 3
	194	if interval < time.Second {
	195	interval = time.Second
	196	}
	197	tick := time.NewTicker(interval)
	198	defer tick.Stop()
	199	for {
	200	select {
	201	case <-build.done:
	202	return nil
	203	case <-tick.C:
	204	rep, err := s.call(runnerRequest{Op: "heartbeat", Run: runID})
	205	if err != nil \|\| !rep.OK {
	206	return errLeaseLost
	207	}
	208	if rep.Cancel != "" {
	209	fmt.Fprintf(log, "\nthe run was canceled by %s; stopping\n", rep.Cancel)
	210	return errCanceled
	211	}
	212	}
	213	}
70b70264 Keenan Nemetz 15d ago	214	}