Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.

vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Let a job declare the secrets it reads

4ac42680eca3181efc70099f65d6e2c9f45e9dd2 · Keenan Nemetz · 7d ago · parent 0c40a1d3

Let a job declare the secrets it reads

A job now names the repository secrets it needs, and the instance delivers
those values when the job is leased. The workflow carries names only, so the
queue row, the metadata branch and the run record never hold a value: the
record names the version of each value the job ran with, which is what makes a
result say which one it used.

    jobs:
      deploy:
        image: alpine:3.22
        secrets: [DEPLOY_KEY]

A name the repository has no value for is not a failure. The variable is left
unset rather than empty, the step's shell can tell the difference, and the
runner writes one line to the log saying which value was missing. That keeps
an optional secret optional and leaves "required" with the step, where someone
can write the message: ${DEPLOY_KEY:?DEPLOY_KEY is not set}.

The value reaches the step through a file the runner writes into the run
directory it already mounts at /run/vole for an event payload, and each step is
prefixed with `. /run/vole/secrets.env`. That is deliberately not an env-file
argument to the runtime, which cannot carry a line break, and not --env, which
puts the value in the process's argument list where every process on the host
can read it. A local `vole workflow run` writes the same file, reading the
values with the developer's own credential, and says which ones it could not
read.

The name rule and the 128 KiB limit moved to ci, next to the environment rules
they mirror, so the store refuses a name no workflow could declare and the
workflow refuses a name no process environment could carry.

Slice 2 of #154.

ci/agent/agent.go +88−18

@@ -84,6 +84,12 @@ type runnerJob struct {
84 84 ImageSHA string `json:"image_sha,omitempty"`
85 85 // Env is the job's declared environment, pinned when the run was queued.
86 86 Env map[string]string `json:"env,omitempty"`
87 // Secrets are the declared secrets this repository has, and Missing are the
88 // names it declared that it does not. A missing one leaves its variable unset and
89 // gets a line in the log; it is not a failure, because a workflow is allowed an
90 // optional secret.
91 Secrets map[string]runnerSecret `json:"secrets,omitempty"`
92 Missing []runnerSecretIssue `json:"missing_secrets,omitempty"`
87 93 // Artifacts are path patterns, expanded in the checkout after the steps pass.
88 94 Artifacts []string `json:"artifacts,omitempty"`
89 95 // Tag is set for a tag-triggered run; TagAnnotated says whether it carries
@@ -96,6 +102,21 @@ type runnerJob struct {
96 102 Pages *runnerPages `json:"pages,omitempty"`
97 103 }
98 104
105 // runnerSecret is one delivered value and the version it came from. The version is
106 // what the attempt record reports, so a result says which value ran without
107 // carrying it.
108 type runnerSecret struct {
109 Version int `json:"version"`
110 Value string `json:"value"`
111 }
112
113 // runnerSecretIssue is a declared secret the instance could not supply. Reason is
114 // empty when the only reason is that nobody has stored that name.
115 type runnerSecretIssue struct {
116 Name string `json:"name"`
117 Reason string `json:"reason,omitempty"`
118 }
119
99 120 type runnerPages struct {
100 121 Path string `json:"path"`
101 122 Ref string `json:"ref"`
@@ -518,6 +539,17 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur
518 539 }
519 540 fmt.Fprintf(log, "=== %s · %s · %s @ %s\n", job.Repo, job.Job, job.Branch, format.SHA(job.SHA))
520 541 fmt.Fprintf(log, "environment: %s via %s\n", job.Image, box.Runtime)
542 // A declared secret the repository does not have is not a failure: the variable
543 // is left unset and the log says which one, because a workflow is allowed an
544 // optional secret and a required one is ${NAME:?} in the step.
545 for _, issue := range job.Missing {
546 if issue.Reason != "" {
547 fmt.Fprintf(log, "vole: %s cannot be read (%s), so this job runs without it.\n",
548 issue.Name, issue.Reason)
549 continue
550 }
551 fmt.Fprintf(log, "vole: %s is not set for this repository, so this job runs without it.\n", issue.Name)
552 }
521 553
522 554 image, err := s.resolveImage(job, box, log, lease)
523 555 if err != nil {
@@ -528,23 +560,28 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur
528 560 fail("%s", err)
529 561 return
530 562 }
531 mounts := s.cacheFor(job)
532 // An event run gets its pinned payload as a mounted file, not as an interpolated
533 // command. The workflow reads it deliberately, and a large issue or comment body
534 // cannot make the environment argument oversized.
563 cacheMounts := s.cacheFor(job)
564 mounts := cacheMounts
565 // The run directory is mounted at /run/vole for the pinned event payload and for
566 // the job's secrets. Both arrive as files rather than as interpolated commands: a
567 // large issue body cannot make an argument oversized, and a secret never becomes
568 // an argument at all.
535 569 env := job.Env
536 if job.EventPayload != "" {
537 eventDir := filepath.Join(filepath.Dir(co.Work), "event-"+job.ID+"-"+fmt.Sprint(job.Attempt))
538 if err := os.MkdirAll(eventDir, 0o700); err != nil {
539 fail("cannot prepare the event payload: %s", err)
570 runDir := ""
571 if job.EventPayload != "" || len(job.Secrets) > 0 {
572 runDir = filepath.Join(filepath.Dir(co.Work), "run-"+job.ID+"-"+fmt.Sprint(job.Attempt))
573 if err := os.MkdirAll(runDir, 0o700); err != nil {
574 fail("cannot prepare the run directory: %s", err)
540 575 return
541 576 }
542 defer os.RemoveAll(eventDir)
543 if err := os.WriteFile(filepath.Join(eventDir, "event.json"), []byte(job.EventPayload), 0o600); err != nil {
577 defer os.RemoveAll(runDir)
578 mounts = append(mounts, container.Mount{Host: runDir, Path: "/run/vole"})
579 }
580 if job.EventPayload != "" {
581 if err := os.WriteFile(filepath.Join(runDir, "event.json"), []byte(job.EventPayload), 0o600); err != nil {
544 582 fail("cannot write the event payload: %s", err)
545 583 return
546 584 }
547 mounts = append(mounts, container.Mount{Host: eventDir, Path: "/run/vole"})
548 585 env = make(map[string]string, len(job.Env)+3)
549 586 for name, value := range job.Env {
550 587 env[name] = value
@@ -553,10 +590,25 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur
553 590 env["VOLE_EVENT_ACTION"] = job.TriggerAction
554 591 env["VOLE_EVENT_PATH"] = "/run/vole/event.json"
555 592 }
593 secretsFile := ""
594 if len(job.Secrets) > 0 {
595 script, err := secretsScript(job.Secrets)
596 if err != nil {
597 fail("%s", err)
598 return
599 }
600 if err := os.WriteFile(filepath.Join(runDir, "secrets.env"), script, 0o600); err != nil {
601 fail("cannot prepare the secrets: %s", err)
602 return
603 }
604 secretsFile = "/run/vole/secrets.env"
605 }
556 606 if s.cacheRoot != "" {
557 607 defer func() {
558 608 if !s.cacheReuse {
559 for _, m := range mounts {
609 // Only the cache mounts: the run directory is the job's own and is
610 // removed by the defer above.
611 for _, m := range cacheMounts {
560 612 _ = os.RemoveAll(m.Host)
561 613 }
562 614 return
@@ -567,12 +619,13 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur
567 619 }()
568 620 }
569 621 sandbox, err := container.Start(box.Config, container.Spec{
570 Image: image,
571 Name: containerName(box, job),
572 Work: co.Work,
573 Labels: box.labels(job),
574 Mounts: mounts,
575 Env: env,
622 Image: image,
623 Name: containerName(box, job),
624 Work: co.Work,
625 Labels: box.labels(job),
626 Mounts: mounts,
627 Env: env,
628 SecretsFile: secretsFile,
576 629 })
577 630 if err != nil {
578 631 fail("%s", err)
@@ -701,6 +754,12 @@ func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *
701 754 }
702 755 rec.Image, rec.ImageFile, rec.ImageSHA = job.Image, job.ImageFile, job.ImageSHA
703 756 rec.Runtime = box.Runtime
757 // The record says which version of each secret the job ran with, never the value.
758 // The names are sorted so the same attempt reported twice produces the same bytes.
759 for name, delivered := range job.Secrets {
760 rec.Secrets = append(rec.Secrets, runs.SecretRef{Name: name, Version: delivered.Version})
761 }
762 sort.Slice(rec.Secrets, func(i, j int) bool { return rec.Secrets[i].Name < rec.Secrets[j].Name })
704 763 if streamErr == nil {
705 764 rec.LogSize = log.Len()
706 765 }
@@ -1064,3 +1123,14 @@ func (s *runnerSession) runCommand(ctx context.Context, cmd *exec.Cmd, runID str
1064 1123 }
1065 1124 }
1066 1125 }
1126
1127 // secretsScript renders a job's delivered values as the file every step sources.
1128 // The rendering lives in ci/container because that is the package that sources it,
1129 // so a local run writes exactly the same file a runner does.
1130 func secretsScript(secrets map[string]runnerSecret) ([]byte, error) {
1131 values := make(map[string]string, len(secrets))
1132 for name, delivered := range secrets {
1133 values[name] = delivered.Value
1134 }
1135 return container.SecretsEnv(values)
1136 }