vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 7 slice 6: a job declares its outputs, the runner publishes them
213f4d6dc6707cc9199f97acdacd617740bfb84d · Keenan Nemetz · 17d ago · parent 4c0f2aef
Phase 7 slice 6: a job declares its outputs, the runner publishes them
Workflow artifacts are a declaration, not a CLI call in a step. A workflow is
validated at the push that introduces it, so a bad pattern fails the push; a
shell command calling vole would fail at run time, need the binary on a runner's
PATH, and get idempotency wrong under retries.
- workflow: artifacts: [{path: dist/*}] on a job, validated at push — relative,
no '..', no '**', no trailing slash, a glob that parses. Patterns are pinned
into the queued run beside the pinned steps
- runner: expands the patterns in its checkout after the steps pass (one segment
at a time, .git excluded), uploads through the same SSH object transfer git-lfs
uses, and reports the list with its finish. A pattern matching nothing fails the
job: a silent empty artifact is the failure found a week later
- server: the artifact list rides the finish message, so a run's outputs and its
result are one fact; nothing is recorded unless the lease is still held, and
every oid must be in the store at the stated size, or the finish is refused
- storage: run_artifacts rows point at objects in the existing store. A build
output is machine-written state, so it does not touch the metadata branch; the
queue is its only index, and gc marks it from there the way it marks logs
- web: a run page (status, job, sha, log, artifacts) and
/r/<repo>/runs/<id>/artifacts/<name> served from the store, ETag = oid, always
an attachment, nosniff, Range. The CI table now links the run rather than only
its log
Measured through the binary on a real instance: a two-file dist/* declaration ran
on a real runner, both files uploaded over SSH, the run page served them with a
matching sha256, and vole gc --dry-run reported 3 objects, 3 reachable roots (the
log and both artifacts), removing nothing.
internal/workflow/workflow.go +54−3
| @@ -30,6 +30,10 @@ const ( | |||
| 30 | 30 | MaxStepBytes = 64 << 10 | |
| 31 | 31 | MaxJobs = 50 | |
| 32 | 32 | MaxSteps = 100 | |
| 33 | MaxArtifacts = 32 | ||
| 34 | // MaxArtifactPathBytes bounds one pattern. A declaration is a path, not a | ||
| 35 | // manifest: the files it matches are discovered when the job runs. | ||
| 36 | MaxArtifactPathBytes = 256 | ||
| 33 | 37 | // DefaultTimeout applies to a step that does not set one. A hung step must not | |
| 34 | 38 | // hold a runner forever. | |
| 35 | 39 | DefaultTimeout = 10 * time.Minute | |
| @@ -49,9 +53,18 @@ type Step struct { | |||
| 49 | 53 | Timeout time.Duration | |
| 50 | 54 | } | |
| 51 | 55 | ||
| 56 | // Artifact declares an output of a job: a path pattern whose matches are uploaded | ||
| 57 | // when the steps succeed. It is a declaration rather than a step that calls the | ||
| 58 | // CLI, because a workflow is validated when it is pushed — a typo here fails the | ||
| 59 | // push, while a shell command would fail at run time, possibly much later. | ||
| 60 | type Artifact struct { | ||
| 61 | Path string | ||
| 62 | } | ||
| 63 | |||
| 52 | 64 | type Job struct { | |
| 53 | ID string | ||
| 54 | Steps []Step | ||
| 65 | ID string | ||
| 66 | Steps []Step | ||
| 67 | Artifacts []Artifact | ||
| 55 | 68 | } | |
| 56 | 69 | ||
| 57 | 70 | type Workflow struct { | |
| @@ -80,8 +93,13 @@ type rawStep struct { | |||
| 80 | 93 | Timeout string `yaml:"timeout"` | |
| 81 | 94 | } | |
| 82 | 95 | ||
| 96 | type rawArtifact struct { | ||
| 97 | Path string `yaml:"path"` | ||
| 98 | } | ||
| 99 | |||
| 83 | 100 | type rawJob struct { | |
| 84 | Steps []rawStep `yaml:"steps"` | ||
| 101 | Steps []rawStep `yaml:"steps"` | ||
| 102 | Artifacts []rawArtifact `yaml:"artifacts"` | ||
| 85 | 103 | } | |
| 86 | 104 | ||
| 87 | 105 | type rawWorkflow struct { | |
| @@ -189,6 +207,39 @@ func Parse(file string, content string) (*Workflow, []string) { | |||
| 189 | 207 | } | |
| 190 | 208 | job.Steps = append(job.Steps, step) | |
| 191 | 209 | } | |
| 210 | if len(jr.Artifacts) > MaxArtifacts { | ||
| 211 | bad("jobs.%s: %d artifact declarations, limit %d", id, len(jr.Artifacts), MaxArtifacts) | ||
| 212 | } | ||
| 213 | for i, ar := range jr.Artifacts { | ||
| 214 | where := fmt.Sprintf("jobs.%s.artifacts[%d]", id, i) | ||
| 215 | p := strings.TrimSpace(ar.Path) | ||
| 216 | switch { | ||
| 217 | case p == "": | ||
| 218 | bad("%s: path is required (a glob relative to the checkout, e.g. dist/*)", where) | ||
| 219 | continue | ||
| 220 | case len(p) > MaxArtifactPathBytes: | ||
| 221 | bad("%s: path is %d bytes, limit %d", where, len(p), MaxArtifactPathBytes) | ||
| 222 | continue | ||
| 223 | case strings.HasPrefix(p, "/") || filepath.IsAbs(p): | ||
| 224 | bad("%s: %q must be relative to the checkout", where, p) | ||
| 225 | continue | ||
| 226 | case strings.Contains("/"+p+"/", "/../") || p == "..": | ||
| 227 | bad("%s: %q climbs out of the checkout", where, p) | ||
| 228 | continue | ||
| 229 | case strings.HasSuffix(p, "/"): | ||
| 230 | bad("%s: %q names a directory; declare the files, e.g. %s*", where, p, p) | ||
| 231 | continue | ||
| 232 | case strings.Contains(p, "**"): | ||
| 233 | bad("%s: %q uses **, which does not mean what it does elsewhere; use %s*", | ||
| 234 | where, p, strings.SplitN(p, "**", 2)[0]) | ||
| 235 | continue | ||
| 236 | } | ||
| 237 | if _, err := path.Match(p, "probe"); err != nil { | ||
| 238 | bad("%s: %q is not a glob", where, p) | ||
| 239 | continue | ||
| 240 | } | ||
| 241 | job.Artifacts = append(job.Artifacts, Artifact{Path: p}) | ||
| 242 | } | ||
| 192 | 243 | wf.Jobs = append(wf.Jobs, job) | |
| 193 | 244 | } | |
| 194 | 245 | if len(problems) > 0 { | |