vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Let a job declare the secrets it reads
4ac42680eca3181efc70099f65d6e2c9f45e9dd2 · Keenan Nemetz · 7d ago · parent 0c40a1d3
Let a job declare the secrets it reads
A job now names the repository secrets it needs, and the instance delivers
those values when the job is leased. The workflow carries names only, so the
queue row, the metadata branch and the run record never hold a value: the
record names the version of each value the job ran with, which is what makes a
result say which one it used.
jobs:
deploy:
image: alpine:3.22
secrets: [DEPLOY_KEY]
A name the repository has no value for is not a failure. The variable is left
unset rather than empty, the step's shell can tell the difference, and the
runner writes one line to the log saying which value was missing. That keeps
an optional secret optional and leaves "required" with the step, where someone
can write the message: ${DEPLOY_KEY:?DEPLOY_KEY is not set}.
The value reaches the step through a file the runner writes into the run
directory it already mounts at /run/vole for an event payload, and each step is
prefixed with `. /run/vole/secrets.env`. That is deliberately not an env-file
argument to the runtime, which cannot carry a line break, and not --env, which
puts the value in the process's argument list where every process on the host
can read it. A local `vole workflow run` writes the same file, reading the
values with the developer's own credential, and says which ones it could not
read.
The name rule and the 128 KiB limit moved to ci, next to the environment rules
they mirror, so the store refuses a name no workflow could declare and the
workflow refuses a name no process environment could carry.
Slice 2 of #154.
ci/agent/agent.go +88−18
| @@ -84,6 +84,12 @@ type runnerJob struct { | |||
| 84 | 84 | ImageSHA string `json:"image_sha,omitempty"` | |
| 85 | 85 | // Env is the job's declared environment, pinned when the run was queued. | |
| 86 | 86 | Env map[string]string `json:"env,omitempty"` | |
| 87 | // Secrets are the declared secrets this repository has, and Missing are the | ||
| 88 | // names it declared that it does not. A missing one leaves its variable unset and | ||
| 89 | // gets a line in the log; it is not a failure, because a workflow is allowed an | ||
| 90 | // optional secret. | ||
| 91 | Secrets map[string]runnerSecret `json:"secrets,omitempty"` | ||
| 92 | Missing []runnerSecretIssue `json:"missing_secrets,omitempty"` | ||
| 87 | 93 | // Artifacts are path patterns, expanded in the checkout after the steps pass. | |
| 88 | 94 | Artifacts []string `json:"artifacts,omitempty"` | |
| 89 | 95 | // Tag is set for a tag-triggered run; TagAnnotated says whether it carries | |
| @@ -96,6 +102,21 @@ type runnerJob struct { | |||
| 96 | 102 | Pages *runnerPages `json:"pages,omitempty"` | |
| 97 | 103 | } | |
| 98 | 104 | ||
| 105 | // runnerSecret is one delivered value and the version it came from. The version is | ||
| 106 | // what the attempt record reports, so a result says which value ran without | ||
| 107 | // carrying it. | ||
| 108 | type runnerSecret struct { | ||
| 109 | Version int `json:"version"` | ||
| 110 | Value string `json:"value"` | ||
| 111 | } | ||
| 112 | |||
| 113 | // runnerSecretIssue is a declared secret the instance could not supply. Reason is | ||
| 114 | // empty when the only reason is that nobody has stored that name. | ||
| 115 | type runnerSecretIssue struct { | ||
| 116 | Name string `json:"name"` | ||
| 117 | Reason string `json:"reason,omitempty"` | ||
| 118 | } | ||
| 119 | |||
| 99 | 120 | type runnerPages struct { | |
| 100 | 121 | Path string `json:"path"` | |
| 101 | 122 | Ref string `json:"ref"` | |
| @@ -518,6 +539,17 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 518 | 539 | } | |
| 519 | 540 | fmt.Fprintf(log, "=== %s · %s · %s @ %s\n", job.Repo, job.Job, job.Branch, format.SHA(job.SHA)) | |
| 520 | 541 | fmt.Fprintf(log, "environment: %s via %s\n", job.Image, box.Runtime) | |
| 542 | // A declared secret the repository does not have is not a failure: the variable | ||
| 543 | // is left unset and the log says which one, because a workflow is allowed an | ||
| 544 | // optional secret and a required one is ${NAME:?} in the step. | ||
| 545 | for _, issue := range job.Missing { | ||
| 546 | if issue.Reason != "" { | ||
| 547 | fmt.Fprintf(log, "vole: %s cannot be read (%s), so this job runs without it.\n", | ||
| 548 | issue.Name, issue.Reason) | ||
| 549 | continue | ||
| 550 | } | ||
| 551 | fmt.Fprintf(log, "vole: %s is not set for this repository, so this job runs without it.\n", issue.Name) | ||
| 552 | } | ||
| 521 | 553 | ||
| 522 | 554 | image, err := s.resolveImage(job, box, log, lease) | |
| 523 | 555 | if err != nil { | |
| @@ -528,23 +560,28 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 528 | 560 | fail("%s", err) | |
| 529 | 561 | return | |
| 530 | 562 | } | |
| 531 | mounts := s.cacheFor(job) | ||
| 532 | // An event run gets its pinned payload as a mounted file, not as an interpolated | ||
| 533 | // command. The workflow reads it deliberately, and a large issue or comment body | ||
| 534 | // cannot make the environment argument oversized. | ||
| 563 | cacheMounts := s.cacheFor(job) | ||
| 564 | mounts := cacheMounts | ||
| 565 | // The run directory is mounted at /run/vole for the pinned event payload and for | ||
| 566 | // the job's secrets. Both arrive as files rather than as interpolated commands: a | ||
| 567 | // large issue body cannot make an argument oversized, and a secret never becomes | ||
| 568 | // an argument at all. | ||
| 535 | 569 | env := job.Env | |
| 536 | if job.EventPayload != "" { | ||
| 537 | eventDir := filepath.Join(filepath.Dir(co.Work), "event-"+job.ID+"-"+fmt.Sprint(job.Attempt)) | ||
| 538 | if err := os.MkdirAll(eventDir, 0o700); err != nil { | ||
| 539 | fail("cannot prepare the event payload: %s", err) | ||
| 570 | runDir := "" | ||
| 571 | if job.EventPayload != "" || len(job.Secrets) > 0 { | ||
| 572 | runDir = filepath.Join(filepath.Dir(co.Work), "run-"+job.ID+"-"+fmt.Sprint(job.Attempt)) | ||
| 573 | if err := os.MkdirAll(runDir, 0o700); err != nil { | ||
| 574 | fail("cannot prepare the run directory: %s", err) | ||
| 540 | 575 | return | |
| 541 | 576 | } | |
| 542 | defer os.RemoveAll(eventDir) | ||
| 543 | if err := os.WriteFile(filepath.Join(eventDir, "event.json"), []byte(job.EventPayload), 0o600); err != nil { | ||
| 577 | defer os.RemoveAll(runDir) | ||
| 578 | mounts = append(mounts, container.Mount{Host: runDir, Path: "/run/vole"}) | ||
| 579 | } | ||
| 580 | if job.EventPayload != "" { | ||
| 581 | if err := os.WriteFile(filepath.Join(runDir, "event.json"), []byte(job.EventPayload), 0o600); err != nil { | ||
| 544 | 582 | fail("cannot write the event payload: %s", err) | |
| 545 | 583 | return | |
| 546 | 584 | } | |
| 547 | mounts = append(mounts, container.Mount{Host: eventDir, Path: "/run/vole"}) | ||
| 548 | 585 | env = make(map[string]string, len(job.Env)+3) | |
| 549 | 586 | for name, value := range job.Env { | |
| 550 | 587 | env[name] = value | |
| @@ -553,10 +590,25 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 553 | 590 | env["VOLE_EVENT_ACTION"] = job.TriggerAction | |
| 554 | 591 | env["VOLE_EVENT_PATH"] = "/run/vole/event.json" | |
| 555 | 592 | } | |
| 593 | secretsFile := "" | ||
| 594 | if len(job.Secrets) > 0 { | ||
| 595 | script, err := secretsScript(job.Secrets) | ||
| 596 | if err != nil { | ||
| 597 | fail("%s", err) | ||
| 598 | return | ||
| 599 | } | ||
| 600 | if err := os.WriteFile(filepath.Join(runDir, "secrets.env"), script, 0o600); err != nil { | ||
| 601 | fail("cannot prepare the secrets: %s", err) | ||
| 602 | return | ||
| 603 | } | ||
| 604 | secretsFile = "/run/vole/secrets.env" | ||
| 605 | } | ||
| 556 | 606 | if s.cacheRoot != "" { | |
| 557 | 607 | defer func() { | |
| 558 | 608 | if !s.cacheReuse { | |
| 559 | for _, m := range mounts { | ||
| 609 | // Only the cache mounts: the run directory is the job's own and is | ||
| 610 | // removed by the defer above. | ||
| 611 | for _, m := range cacheMounts { | ||
| 560 | 612 | _ = os.RemoveAll(m.Host) | |
| 561 | 613 | } | |
| 562 | 614 | return | |
| @@ -567,12 +619,13 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 567 | 619 | }() | |
| 568 | 620 | } | |
| 569 | 621 | sandbox, err := container.Start(box.Config, container.Spec{ | |
| 570 | Image: image, | ||
| 571 | Name: containerName(box, job), | ||
| 572 | Work: co.Work, | ||
| 573 | Labels: box.labels(job), | ||
| 574 | Mounts: mounts, | ||
| 575 | Env: env, | ||
| 622 | Image: image, | ||
| 623 | Name: containerName(box, job), | ||
| 624 | Work: co.Work, | ||
| 625 | Labels: box.labels(job), | ||
| 626 | Mounts: mounts, | ||
| 627 | Env: env, | ||
| 628 | SecretsFile: secretsFile, | ||
| 576 | 629 | }) | |
| 577 | 630 | if err != nil { | |
| 578 | 631 | fail("%s", err) | |
| @@ -701,6 +754,12 @@ func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log * | |||
| 701 | 754 | } | |
| 702 | 755 | rec.Image, rec.ImageFile, rec.ImageSHA = job.Image, job.ImageFile, job.ImageSHA | |
| 703 | 756 | rec.Runtime = box.Runtime | |
| 757 | // The record says which version of each secret the job ran with, never the value. | ||
| 758 | // The names are sorted so the same attempt reported twice produces the same bytes. | ||
| 759 | for name, delivered := range job.Secrets { | ||
| 760 | rec.Secrets = append(rec.Secrets, runs.SecretRef{Name: name, Version: delivered.Version}) | ||
| 761 | } | ||
| 762 | sort.Slice(rec.Secrets, func(i, j int) bool { return rec.Secrets[i].Name < rec.Secrets[j].Name }) | ||
| 704 | 763 | if streamErr == nil { | |
| 705 | 764 | rec.LogSize = log.Len() | |
| 706 | 765 | } | |
| @@ -1064,3 +1123,14 @@ func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID str | |||
| 1064 | 1123 | } | |
| 1065 | 1124 | } | |
| 1066 | 1125 | } | |
| 1126 | |||
| 1127 | // secretsScript renders a job's delivered values as the file every step sources. | ||
| 1128 | // The rendering lives in ci/container because that is the package that sources it, | ||
| 1129 | // so a local run writes exactly the same file a runner does. | ||
| 1130 | func secretsScript(secrets map[string]runnerSecret) ([]byte, error) { | ||
| 1131 | values := make(map[string]string, len(secrets)) | ||
| 1132 | for name, delivered := range secrets { | ||
| 1133 | values[name] = delivered.Value | ||
| 1134 | } | ||
| 1135 | return container.SecretsEnv(values) | ||
| 1136 | } | ||