vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 7 slice 6: a job declares its outputs, the runner publishes them
213f4d6dc6707cc9199f97acdacd617740bfb84d · Keenan Nemetz · 17d ago · parent 4c0f2aef
Phase 7 slice 6: a job declares its outputs, the runner publishes them
Workflow artifacts are a declaration, not a CLI call in a step. A workflow is
validated at the push that introduces it, so a bad pattern fails the push; a
shell command calling vole would fail at run time, need the binary on a runner's
PATH, and get idempotency wrong under retries.
- workflow: artifacts: [{path: dist/*}] on a job, validated at push — relative,
no '..', no '**', no trailing slash, a glob that parses. Patterns are pinned
into the queued run beside the pinned steps
- runner: expands the patterns in its checkout after the steps pass (one segment
at a time, .git excluded), uploads through the same SSH object transfer git-lfs
uses, and reports the list with its finish. A pattern matching nothing fails the
job: a silent empty artifact is the failure found a week later
- server: the artifact list rides the finish message, so a run's outputs and its
result are one fact; nothing is recorded unless the lease is still held, and
every oid must be in the store at the stated size, or the finish is refused
- storage: run_artifacts rows point at objects in the existing store. A build
output is machine-written state, so it does not touch the metadata branch; the
queue is its only index, and gc marks it from there the way it marks logs
- web: a run page (status, job, sha, log, artifacts) and
/r/<repo>/runs/<id>/artifacts/<name> served from the store, ETag = oid, always
an attachment, nosniff, Range. The CI table now links the run rather than only
its log
Measured through the binary on a real instance: a two-file dist/* declaration ran
on a real runner, both files uploaded over SSH, the run page served them with a
matching sha256, and vole gc --dry-run reported 3 objects, 3 reachable roots (the
log and both artifacts), removing nothing.
internal/sshserver/runner.go +78−1
| @@ -2,12 +2,19 @@ package sshserver | |||
| 2 | 2 | ||
| 3 | 3 | import ( | |
| 4 | 4 | "encoding/json" | |
| 5 | "fmt" | ||
| 5 | 6 | "time" | |
| 6 | 7 | ||
| 7 | 8 | "github.com/rodent-software/vole/internal/acl" | |
| 9 | "github.com/rodent-software/vole/internal/lfs" | ||
| 10 | "github.com/rodent-software/vole/internal/runs" | ||
| 8 | 11 | "golang.org/x/crypto/ssh" | |
| 9 | 12 | ) | |
| 10 | 13 | ||
| 14 | // maxRunArtifacts bounds one run's output list. It is a page of downloads, not a | ||
| 15 | // package index. | ||
| 16 | const maxRunArtifacts = 64 | ||
| 17 | |||
| 11 | 18 | // The runner protocol. | |
| 12 | 19 | // | |
| 13 | 20 | // A runner connects over the same SSH transport as everything else, so it | |
| @@ -27,6 +34,16 @@ type runnerRequest struct { | |||
| 27 | 34 | Run string `json:"run"` | |
| 28 | 35 | Log string `json:"log"` | |
| 29 | 36 | Exit int `json:"exit"` | |
| 37 | // Artifacts arrive with the finish, not as a separate call: a job's outputs | ||
| 38 | // and its result are one fact, so there is no window in which a successful run | ||
| 39 | // has no artifacts or a failed one has some. | ||
| 40 | Artifacts []runnerArtifact `json:"artifacts,omitempty"` | ||
| 41 | } | ||
| 42 | |||
| 43 | type runnerArtifact struct { | ||
| 44 | Name string `json:"name"` | ||
| 45 | OID string `json:"oid"` | ||
| 46 | Size int64 `json:"size"` | ||
| 30 | 47 | } | |
| 31 | 48 | ||
| 32 | 49 | type runnerStep struct { | |
| @@ -43,6 +60,9 @@ type runnerJob struct { | |||
| 43 | 60 | Workflow string `json:"workflow"` | |
| 44 | 61 | Job string `json:"job"` | |
| 45 | 62 | Steps []runnerStep `json:"steps"` | |
| 63 | // Artifacts are the declared patterns the runner expands and uploads after the | ||
| 64 | // steps succeed. | ||
| 65 | Artifacts []string `json:"artifacts,omitempty"` | ||
| 46 | 66 | } | |
| 47 | 67 | ||
| 48 | 68 | type runnerReply struct { | |
| @@ -92,7 +112,7 @@ func (s *Server) serveRunner(ch ssh.Channel, user, accountID string) { | |||
| 92 | 112 | } | |
| 93 | 113 | job := &runnerJob{ | |
| 94 | 114 | ID: run.ID, Repo: run.Repo, SHA: run.SHA, Branch: run.Branch, | |
| 95 | Workflow: run.Workflow, Job: run.Job, | ||
| 115 | Workflow: run.Workflow, Job: run.Job, Artifacts: run.Artifacts, | ||
| 96 | 116 | } | |
| 97 | 117 | for _, st := range run.Steps { | |
| 98 | 118 | job.Steps = append(job.Steps, runnerStep{Name: st.Name, Run: st.Run, Timeout: st.Timeout}) | |
| @@ -110,6 +130,26 @@ func (s *Server) serveRunner(ch ssh.Channel, user, accountID string) { | |||
| 110 | 130 | _ = enc.Encode(runnerReply{Op: "heartbeat", OK: ok}) | |
| 111 | 131 | ||
| 112 | 132 | case "finish": | |
| 133 | // Artifacts are recorded while the lease is still held, and only if the | ||
| 134 | // objects are really in the store: a runner naming bytes it never | ||
| 135 | // uploaded would otherwise produce a download that 404s. | ||
| 136 | if len(req.Artifacts) > 0 { | ||
| 137 | arts, err := s.checkArtifacts(req.Artifacts) | ||
| 138 | if err != nil { | ||
| 139 | _ = enc.Encode(runnerReply{Op: "finish", Error: err.Error()}) | ||
| 140 | continue | ||
| 141 | } | ||
| 142 | ok, err := cfg.Runs.RecordArtifacts(req.Run, owner, arts) | ||
| 143 | if err != nil { | ||
| 144 | _ = enc.Encode(runnerReply{Op: "finish", Error: err.Error()}) | ||
| 145 | continue | ||
| 146 | } | ||
| 147 | if !ok { | ||
| 148 | // The lease moved: report nothing, exactly like a lost finish. | ||
| 149 | _ = enc.Encode(runnerReply{Op: "finish", OK: false}) | ||
| 150 | continue | ||
| 151 | } | ||
| 152 | } | ||
| 113 | 153 | // A finished log never changes again, so it becomes a content-addressed | |
| 114 | 154 | // object and the queue row stops carrying text. The object is written | |
| 115 | 155 | // first: if this runner has lost the lease, the row is left alone and | |
| @@ -151,6 +191,43 @@ func (s *Server) serveRunner(ch ssh.Channel, user, accountID string) { | |||
| 151 | 191 | } | |
| 152 | 192 | } | |
| 153 | 193 | ||
| 194 | // checkArtifacts validates what a runner claims it uploaded. The store is the | ||
| 195 | // authority on whether the bytes exist; this only refuses claims that are not | ||
| 196 | // downloadable at all. | ||
| 197 | func (s *Server) checkArtifacts(in []runnerArtifact) ([]runs.Artifact, error) { | ||
| 198 | if len(in) > maxRunArtifacts { | ||
| 199 | return nil, fmt.Errorf("%d artifacts, limit %d", len(in), maxRunArtifacts) | ||
| 200 | } | ||
| 201 | seen := make(map[string]bool, len(in)) | ||
| 202 | out := make([]runs.Artifact, 0, len(in)) | ||
| 203 | for _, a := range in { | ||
| 204 | if err := runs.ValidArtifactName(a.Name); err != nil { | ||
| 205 | return nil, err | ||
| 206 | } | ||
| 207 | if seen[a.Name] { | ||
| 208 | return nil, fmt.Errorf("two artifacts are named %q", a.Name) | ||
| 209 | } | ||
| 210 | seen[a.Name] = true | ||
| 211 | if !lfs.ValidOID(a.OID) { | ||
| 212 | return nil, fmt.Errorf("artifact %q: %q is not a sha256 object id", a.Name, a.OID) | ||
| 213 | } | ||
| 214 | if s.cfg.LFS == nil { | ||
| 215 | return nil, fmt.Errorf("this instance has no object store, so artifacts cannot be published") | ||
| 216 | } | ||
| 217 | n, err := s.cfg.LFS.Stat(a.OID) | ||
| 218 | if err != nil { | ||
| 219 | return nil, fmt.Errorf("artifact %q names object %s, which is not in the store: "+ | ||
| 220 | "upload it before finishing the run", a.Name, a.OID[:12]) | ||
| 221 | } | ||
| 222 | if a.Size != n { | ||
| 223 | return nil, fmt.Errorf("artifact %q says %d bytes but the stored object is %d", | ||
| 224 | a.Name, a.Size, n) | ||
| 225 | } | ||
| 226 | out = append(out, runs.Artifact{Name: a.Name, OID: a.OID, Size: n}) | ||
| 227 | } | ||
| 228 | return out, nil | ||
| 229 | } | ||
| 230 | |||
| 154 | 231 | // runnableRepos lists the repositories this account may execute jobs for. | |
| 155 | 232 | func (s *Server) runnableRepos(accountID string) ([]string, error) { | |
| 156 | 233 | repos, err := s.cfg.Repos.List() | |