Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

392 commit(s) on main

Compare History
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys keenan · 52m ago 4b8c9e1d
history blame raw 18.8 KiB
1package main
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "io"
8 "os"
9 "os/exec"
10 "path/filepath"
11 "strings"
12 "time"
13
14 "rodent.software/vole/ci"
15 jobcache "rodent.software/vole/ci/cache"
16 "rodent.software/vole/ci/container"
17 "rodent.software/vole/ci/redact"
18 "rodent.software/vole/client"
19 "rodent.software/vole/internal/format"
20)
21
22// workflowJob is one job selected for a local run.
23type workflowJob struct {
24 workflow *ci.Workflow
25 job ci.Job
26}
27
28// localExitError carries the status of the step that stopped a local run, so the
29// command exits with the same code a shell pipeline would see.
30type localExitError struct {
31 code int
32 msg string
33}
34
35func (e localExitError) Error() string { return e.msg }
36func (e localExitError) ExitCode() int { return e.code }
37
38// cmdWorkflowRun executes workflows from the working tree. It needs no server and
39// no runner: the point is to check the steps before a push. They run in the image
40// the workflow names, so the answer here is the answer CI gives.
41func cmdWorkflowRun(dir string, f *flags) error {
42 root, err := filepath.Abs(dir)
43 if err != nil {
44 return err
45 }
46 wfs, problems, err := ci.CheckDir(root)
47 if err != nil {
48 return err
49 }
50 for _, p := range problems {
51 fmt.Fprintln(os.Stderr, p)
52 }
53 if len(problems) > 0 {
54 return fmt.Errorf("%d problem(s) in %s", len(problems), ci.Dir)
55 }
56 if len(wfs) == 0 {
57 fmt.Fprintf(os.Stderr, "no workflows in %s\n",
58 filepath.Join(root, filepath.FromSlash(ci.Dir)))
59 return nil
60 }
61
62 targets, err := selectWorkflowJobs(wfs, f.rest, f.all("job"))
63 if err != nil {
64 return err
65 }
66 cfg, err := localContainerConfig(f)
67 if err != nil {
68 return err
69 }
70 // The run is about to build and start a container, so find out now whether this
71 // machine can, rather than at the first step with the runtime's exit code.
72 if err := container.Probe(context.Background(), cfg); err != nil {
73 return err
74 }
75 if container.RunsAsRoot(cfg) {
76 fmt.Fprintln(os.Stderr, "note: running as root, so a step can rewrite its own container; "+
77 "run vole as an unprivileged user to have the image's permissions keep it immutable")
78 }
79 cacheRoot, reuse, err := localCacheRoot(f)
80 if err != nil {
81 return err
82 }
83 return runWorkflowJobs(root, targets, cfg, cacheRoot, reuse)
84}
85
86// localContainerConfig reads the container flags. The default network is bridge
87// here, not none: CI's own runner needs the network to fetch modules, and a local
88// run that cannot build is not the run CI performs. The isolation a pushed job
89// needs is still the same container, so nothing about how a step executes differs.
90// This is only the default: a job that declares network: decides for itself, which
91// localNetwork applies per job.
92func localContainerConfig(f *flags) (container.Config, error) {
93 network := f.one("container-network")
94 if network == "" {
95 network = "bridge"
96 }
97 return container.Validate(container.Config{
98 Runtime: f.one("container-runtime"),
99 Network: network,
100 Pull: f.one("container-pull"),
101 User: f.one("container-user"),
102 Userns: f.one("container-userns"),
103 })
104}
105
106// localNetwork applies the job's `network:` to a local run the way a runner applies
107// it to a pushed one: the job decides, and the default only fills in for a job that
108// says nothing. A job that asks for the bridge is refused here when this run was
109// started without one, because a local run that silently has less than CI does is
110// the difference this command exists to remove.
111func localNetwork(cfg container.Config, job ci.Job) (container.Config, error) {
112 if len(job.Services) > 0 && job.Network != ci.NetworkBridge {
113 return cfg, fmt.Errorf("%s declares services but not network: bridge", job.ID)
114 }
115 if job.Network == "" {
116 return cfg, nil
117 }
118 if err := ci.ValidNetwork(job.Network); err != nil {
119 return cfg, fmt.Errorf("%s.network: %w", job.ID, err)
120 }
121 if job.Network == ci.NetworkBridge && cfg.Network != ci.NetworkBridge {
122 return cfg, fmt.Errorf("%s asks for network: bridge, but this run was started with "+
123 "--container-network %s; run it again with --container-network bridge", job.ID, cfg.Network)
124 }
125 cfg.Network = job.Network
126 return cfg, nil
127}
128
129// localCacheRoot is where a local run keeps the caches it shares between runs. It
130// is the same layout the runner uses, under the user cache directory, so the second
131// run of a job is as fast here as it is on a runner. The directory is mounted even
132// with --no-cache, because an image that points at /cache must work either way;
133// reuse says whether anything survives the job.
134func localCacheRoot(f *flags) (root string, reuse bool, err error) {
135 max := jobcache.DefaultBytes
136 if raw := f.one("cache-size"); raw != "" {
137 parsed, err := jobcache.ParseSize(raw)
138 if err != nil {
139 return "", false, usagef("--cache-size %v", err)
140 }
141 max = parsed
142 }
143 base, err := os.UserCacheDir()
144 if err != nil {
145 base = os.TempDir()
146 }
147 root = filepath.Join(base, "vole", "workflows")
148 if err := os.MkdirAll(root, 0o755); err != nil {
149 return "", false, fmt.Errorf("prepare the job cache: %w", err)
150 }
151 if f.has("no-cache") {
152 if err := jobcache.Clear(root); err != nil {
153 return "", false, fmt.Errorf("empty the job cache: %w", err)
154 }
155 return root, false, nil
156 }
157 if removed, err := jobcache.Trim(root, max); err == nil && removed > 0 {
158 fmt.Fprintf(os.Stderr, "note: trimmed %s from the job cache\n", format.Bytes(removed))
159 }
160 return root, true, nil
161}
162
163// selectWorkflowJobs applies the positional workflow names and the --job
164// filter. A filter that matches nothing is a usage mistake, and the error lists
165// what could have matched: a local run that silently does nothing looks exactly
166// like one that passed.
167func selectWorkflowJobs(wfs []*ci.Workflow, workflows, jobs []string) ([]workflowJob, error) {
168 selected := make([]*ci.Workflow, 0, len(wfs))
169 for _, wf := range wfs {
170 if workflowMatches(wf, workflows) {
171 selected = append(selected, wf)
172 }
173 }
174 if len(selected) == 0 {
175 names := make([]string, 0, len(wfs))
176 for _, wf := range wfs {
177 names = append(names, filepath.Base(wf.Path))
178 }
179 return nil, usagef("no workflow matches %s; available: %s (use --job to select a job)",
180 strings.Join(workflows, ", "), strings.Join(names, ", "))
181 }
182
183 var out []workflowJob
184 for _, wf := range selected {
185 for _, job := range wf.Jobs {
186 if jobMatches(job, jobs) {
187 out = append(out, workflowJob{workflow: wf, job: job})
188 }
189 }
190 }
191 if len(out) > 0 {
192 return out, nil
193 }
194
195 available := make([]string, 0, len(selected))
196 for _, wf := range selected {
197 ids := make([]string, 0, len(wf.Jobs))
198 for _, job := range wf.Jobs {
199 ids = append(ids, job.ID)
200 }
201 available = append(available, fmt.Sprintf("%s: %s", filepath.Base(wf.Path), strings.Join(ids, ", ")))
202 }
203 return nil, usagef("no job matches %s; available: %s",
204 strings.Join(jobs, ", "), strings.Join(available, "; "))
205}
206
207func workflowMatches(wf *ci.Workflow, filters []string) bool {
208 if len(filters) == 0 {
209 return true
210 }
211 base := filepath.Base(wf.Path)
212 stem := workflowStem(base)
213 for _, raw := range filters {
214 f := strings.TrimSpace(raw)
215 if f == "" {
216 continue
217 }
218 f = filepath.ToSlash(filepath.Clean(f))
219 if f == wf.Path {
220 return true
221 }
222 f = filepath.Base(f)
223 if f == base || workflowStem(f) == stem {
224 return true
225 }
226 }
227 return false
228}
229
230func workflowStem(name string) string {
231 return strings.TrimSuffix(strings.TrimSuffix(name, ".yaml"), ".yml")
232}
233
234func jobMatches(job ci.Job, filters []string) bool {
235 if len(filters) == 0 {
236 return true
237 }
238 for _, raw := range filters {
239 if strings.TrimSpace(raw) == job.ID {
240 return true
241 }
242 }
243 return false
244}
245
246// runWorkflowJobs runs the selected jobs in declaration order. It stops at the
247// first failing step: a local run is for finding the problem, not for collecting
248// every failure.
249func runWorkflowJobs(root string, targets []workflowJob, cfg container.Config, cacheRoot string, cacheReuse bool) error {
250 for _, target := range targets {
251 fmt.Fprintf(os.Stderr, "=== %s · %s\n", target.workflow.Path, target.job.ID)
252 if err := runJobInContainer(root, target, cfg, cacheRoot, cacheReuse); err != nil {
253 return err
254 }
255 if len(target.job.Artifacts) > 0 {
256 fmt.Fprintln(os.Stderr, "note: a local run does not publish artifacts")
257 }
258 if target.job.Pages != nil {
259 fmt.Fprintln(os.Stderr, "note: a local run does not deploy the site")
260 }
261 }
262 fmt.Fprintf(os.Stderr, "ok: %d job(s)\n", len(targets))
263 return nil
264}
265
266// runJobInContainer runs the job's steps in the image the workflow names. The
267// working tree is bound into the container, so the run sees uncommitted edits and
268// leaves its outputs where the steps would have put them.
269func runJobInContainer(root string, target workflowJob, cfg container.Config, cacheRoot string, cacheReuse bool) error {
270 cfg, err := localNetwork(cfg, target.job)
271 if err != nil {
272 return err
273 }
274 image, err := resolveLocalImage(target.job, cfg)
275 if err != nil {
276 return err
277 }
278 // A local run starts the same service containers on the same kind of private
279 // network a runner would, so the answer here is the answer CI gives. The job
280 // stops before the services; a failed run keeps their output in the terminal.
281 var localNetworkValue *container.Network
282 var localServices []*container.Container
283 var localServiceIDs []string
284 succeeded := false
285 if len(target.job.Services) > 0 {
286 netName := fmt.Sprintf("vole-workflow-run-%d-%s-net", os.Getpid(), strings.ToLower(target.job.ID))
287 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
288 network, err := container.CreateNetwork(ctx, cfg, netName, map[string]string{"vole.workflowrun": "1"})
289 cancel()
290 if err != nil {
291 return err
292 }
293 localNetworkValue = network
294 cfg.Network = netName
295 defer func() {
296 if !succeeded {
297 for i, svc := range localServices {
298 ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
299 data, logErr := svc.Logs(ctx, container.MaxServiceLogBytes)
300 cancel()
301 if logErr != nil || len(data) == 0 {
302 continue
303 }
304 fmt.Fprintf(os.Stderr, "\n=== service %s ===\n", localServiceIDs[i])
305 _, _ = os.Stderr.Write(data)
306 if data[len(data)-1] != '\n' {
307 fmt.Fprintln(os.Stderr)
308 }
309 }
310 }
311 for _, svc := range localServices {
312 svc.Stop()
313 }
314 if localNetworkValue != nil {
315 ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
316 _ = localNetworkValue.Remove(ctx)
317 cancel()
318 }
319 }()
320 for _, svc := range target.job.Services {
321 c, err := container.StartService(cfg, container.ServiceSpec{
322 Name: fmt.Sprintf("vole-workflow-run-%d-%s-svc-%s", os.Getpid(), strings.ToLower(target.job.ID), svc.ID),
323 Alias: svc.ID,
324 Image: svc.Image,
325 Env: svc.Env,
326 Labels: map[string]string{"vole.workflowrun": "1"},
327 })
328 if err != nil {
329 return err
330 }
331 localServices = append(localServices, c)
332 localServiceIDs = append(localServiceIDs, svc.ID)
333 }
334 for i, c := range localServices {
335 running, err := c.Running(context.Background())
336 if err != nil {
337 return fmt.Errorf("service %s: %w", target.job.Services[i].ID, err)
338 }
339 if !running {
340 return fmt.Errorf("service %s exited before the steps started", target.job.Services[i].ID)
341 }
342 }
343 }
344 name := fmt.Sprintf("vole-workflow-run-%d-%s", os.Getpid(), strings.ToLower(target.job.ID))
345 mounts, releaseCache := localCache(cacheRoot, root, target.job, cacheReuse)
346 defer releaseCache()
347 if !cacheReuse {
348 defer func() {
349 // This defer runs before the namespace lock is released, so nothing else
350 // is inside a local run's cache while it is emptied.
351 for _, m := range mounts {
352 _ = os.RemoveAll(m.Host)
353 }
354 }()
355 }
356 // A local run sees the secrets the pushed job would see, read with the
357 // developer's own credential. What it cannot read is left unset with a note, the
358 // same answer a runner gives, because the alternative is a step that runs with a
359 // value nobody chose.
360 values, notes := localSecrets(root, target.job)
361 for _, note := range notes {
362 fmt.Fprintln(os.Stderr, "note: "+note)
363 }
364 output := stepOutput{stdout: os.Stdout, stderr: os.Stderr}
365 if len(values) > 0 {
366 // The same redaction a runner applies to its log, so a step that prints a
367 // secret does not leave it in the terminal's scrollback either.
368 list := make([]string, 0, len(values))
369 for _, value := range values {
370 list = append(list, value)
371 }
372 stdout := redact.NewWriter(os.Stdout, list)
373 stderr := redact.NewWriter(os.Stderr, list)
374 output = stepOutput{stdout: stdout, stderr: stderr, flush: func() {
375 _ = stdout.Flush()
376 _ = stderr.Flush()
377 }}
378 }
379 secretsFile := ""
380 if len(values) > 0 {
381 runDir, err := os.MkdirTemp("", "vole-workflow-run-*")
382 if err != nil {
383 return err
384 }
385 defer os.RemoveAll(runDir)
386 env, err := container.SecretsEnv(values)
387 if err != nil {
388 return err
389 }
390 if err := os.WriteFile(filepath.Join(runDir, "secrets.env"), env, 0o600); err != nil {
391 return err
392 }
393 mounts = append(mounts, container.Mount{Host: runDir, Path: "/run/vole"})
394 secretsFile = "/run/vole/secrets.env"
395 }
396 sandbox, err := container.Start(cfg, container.Spec{
397 Image: image,
398 Name: name,
399 Work: root,
400 Labels: map[string]string{"vole.workflowrun": "1"},
401 Mounts: mounts,
402 Env: target.job.Env,
403 SecretsFile: secretsFile,
404 })
405 if err != nil {
406 return err
407 }
408 defer sandbox.Stop()
409
410 // The job's budget starts with its first step, the way a runner spends it: an
411 // image build has its own cap and does not come out of this clock.
412 var deadline time.Time
413 if target.job.Timeout > 0 {
414 deadline = time.Now().Add(target.job.Timeout)
415 }
416 for _, step := range target.job.Steps {
417 fmt.Fprintf(os.Stderr, "--- %s\n", step.Name)
418 limit := ci.StepLimit(ci.BudgetLeft(deadline, time.Now()), step.Timeout)
419 if limit <= 0 {
420 fmt.Fprintf(os.Stderr, "the job's %s budget ran out before step %q\n", target.job.Timeout, step.Name)
421 return localExitError{
422 code: 1,
423 msg: fmt.Sprintf("job %q ran out of its %s budget at step %q", target.job.ID, target.job.Timeout, step.Name),
424 }
425 }
426 if err := runStep(target.job.ID, step, sandbox, limit, output); err != nil {
427 return err
428 }
429 }
430 succeeded = true
431 return nil
432}
433
434// localSecrets reads the secrets a job declared, with the account the developer
435// pushes as. A tree with no remote, or an instance that cannot be reached, leaves
436// the variables unset and says which ones: a local run is for finding problems, and
437// a missing staging token is a note rather than a refused run.
438func localSecrets(root string, job ci.Job) (map[string]string, []string) {
439 if len(job.Secrets) == 0 {
440 return nil, nil
441 }
442 _, _, path, err := client.SSHTarget(root, "origin")
443 repoName := ""
444 if err == nil {
445 repoName = client.RepoNameFromPath(path)
446 }
447 if err != nil || repoName == "" {
448 notes := make([]string, 0, len(job.Secrets))
449 for _, name := range job.Secrets {
450 notes = append(notes, name+" is not set for this repository, so this job runs without it")
451 }
452 return nil, notes
453 }
454 values := make(map[string]string, len(job.Secrets))
455 var notes []string
456 for _, name := range job.Secrets {
457 value, err := client.GetSecret(root, "origin", repoName, name)
458 if err != nil {
459 notes = append(notes, fmt.Sprintf("%s cannot be read (%v), so this job runs without it", name, err))
460 continue
461 }
462 values[name] = value
463 }
464 return values, notes
465}
466
467// localCache is the mount one local job gets, mirroring what a runner does: one
468// directory per repository and image, so two checkouts do not share a namespace by
469// accident. As on a runner, the job decides whether to use it.
470// localCache is the runner's cacheFor for a local run: one namespace, held for as
471// long as the container may be inside it, so two `vole workflow run`s in the same
472// checkout do not delete each other's cache. The release function is always
473// non-nil.
474func localCache(cacheRoot, root string, job ci.Job, reuse bool) ([]container.Mount, func()) {
475 if cacheRoot == "" {
476 return nil, func() {}
477 }
478 ns := jobcache.Namespace(cacheRoot, filepath.Base(root), "local",
479 jobcache.ImageKey(job.ImageSHA, job.Image))
480 var release func()
481 var err error
482 if reuse {
483 release, err = jobcache.Lock(cacheRoot, ns)
484 } else {
485 release, err = jobcache.LockExclusive(cacheRoot, ns)
486 }
487 if err != nil {
488 fmt.Fprintf(os.Stderr, "note: cannot lock a job cache: %v\n", err)
489 return nil, func() {}
490 }
491 if err := os.MkdirAll(ns, 0o755); err != nil {
492 release()
493 fmt.Fprintf(os.Stderr, "note: cannot prepare a job cache: %v\n", err)
494 return nil, func() {}
495 }
496 return []container.Mount{{Host: ns, Path: jobcache.Mount}}, release
497}
498
499// resolveLocalImage returns the reference to run, building an image-file
500// definition when the local cache does not already hold it.
501func resolveLocalImage(job ci.Job, cfg container.Config) (string, error) {
502 if job.ImageFile == "" {
503 if err := ci.ValidImageRef(job.Image); err != nil {
504 return "", fmt.Errorf("the job image: %w", err)
505 }
506 fmt.Fprintf(os.Stderr, "--- image %s\n", job.Image)
507 return job.Image, nil
508 }
509 if job.ImageDefinition == "" || job.ImageSHA == "" {
510 return "", fmt.Errorf("image-file %q has no definition in %s", job.ImageFile, ci.ImageDir)
511 }
512 tag := container.ImageTag(job.ImageFile, job.ImageSHA)
513 if cfg.Pull != "always" && container.Exists(cfg.Runtime, tag) {
514 fmt.Fprintf(os.Stderr, "--- image %s (cached)\n", tag)
515 return tag, nil
516 }
517 fmt.Fprintf(os.Stderr, "--- image %s (building)\n", tag)
518 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
519 defer cancel()
520 if err := container.Build(ctx, cfg, job.ImageDefinition, tag, os.Stderr); err != nil {
521 return "", err
522 }
523 return tag, nil
524}
525
526// stepOutput is where a local step's output goes. A job with secrets masks the
527// values first; flush writes whatever the mask held back at the end of the step.
528type stepOutput struct {
529 stdout io.Writer
530 stderr io.Writer
531 flush func()
532}
533
534// runStep applies the step's limit and turns its exit status into the error the
535// command reports. limit is what is left of the job's budget or the step's own
536// timeout, already resolved by StepLimit. Stdin is left unattached so a command that
537// asks a question fails instead of parking a build on a prompt.
538func runStep(jobID string, step ci.Step, sandbox *container.Container, limit time.Duration, output stepOutput) error {
539 ctx, cancel := context.WithTimeout(context.Background(), limit)
540 defer cancel()
541
542 c := sandbox.Exec(ctx, step.Run)
543 c.Stdout = output.stdout
544 c.Stderr = output.stderr
545 err := c.Run()
546 if output.flush != nil {
547 output.flush()
548 }
549
550 if err == nil {
551 return nil
552 }
553 if ctx.Err() == context.DeadlineExceeded {
554 fmt.Fprintf(os.Stderr, "step %q timed out after %s\n", step.Name, limit.Round(time.Millisecond))
555 return localExitError{
556 code: 1,
557 msg: fmt.Sprintf("job %q timed out at step %q", jobID, step.Name),
558 }
559 }
560 var exit *exec.ExitError
561 if errors.As(err, &exit) {
562 code := exit.ExitCode()
563 if code <= 0 {
564 code = 1
565 }
566 fmt.Fprintf(os.Stderr, "step %q exited %d\n", step.Name, code)
567 return localExitError{
568 code: code,
569 msg: fmt.Sprintf("job %q failed at step %q", jobID, step.Name),
570 }
571 }
572 return fmt.Errorf("run step %q: %w", step.Name, err)
573}
574