vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 7 slice 6: a job declares its outputs, the runner publishes them
213f4d6dc6707cc9199f97acdacd617740bfb84d · Keenan Nemetz · 17d ago · parent 4c0f2aef
Phase 7 slice 6: a job declares its outputs, the runner publishes them
Workflow artifacts are a declaration, not a CLI call in a step. A workflow is
validated at the push that introduces it, so a bad pattern fails the push; a
shell command calling vole would fail at run time, need the binary on a runner's
PATH, and get idempotency wrong under retries.
- workflow: artifacts: [{path: dist/*}] on a job, validated at push — relative,
no '..', no '**', no trailing slash, a glob that parses. Patterns are pinned
into the queued run beside the pinned steps
- runner: expands the patterns in its checkout after the steps pass (one segment
at a time, .git excluded), uploads through the same SSH object transfer git-lfs
uses, and reports the list with its finish. A pattern matching nothing fails the
job: a silent empty artifact is the failure found a week later
- server: the artifact list rides the finish message, so a run's outputs and its
result are one fact; nothing is recorded unless the lease is still held, and
every oid must be in the store at the stated size, or the finish is refused
- storage: run_artifacts rows point at objects in the existing store. A build
output is machine-written state, so it does not touch the metadata branch; the
queue is its only index, and gc marks it from there the way it marks logs
- web: a run page (status, job, sha, log, artifacts) and
/r/<repo>/runs/<id>/artifacts/<name> served from the store, ETag = oid, always
an attachment, nosniff, Range. The CI table now links the run rather than only
its log
Measured through the binary on a real instance: a two-file dist/* declaration ran
on a real runner, both files uploaded over SSH, the run page served them with a
matching sha256, and vole gc --dry-run reported 3 objects, 3 reachable roots (the
log and both artifacts), removing nothing.
internal/server/artifacts_test.go +160
| @@ -0,0 +1,160 @@ | |||
| 1 | package server_test | ||
| 2 | |||
| 3 | // Workflow artifacts, end to end: a push declares outputs, a real runner expands | ||
| 4 | // the patterns in its checkout, uploads the bytes over SSH, and reports them with | ||
| 5 | // the result. The run page and the download are then read-only, like everything | ||
| 6 | // else on the web. | ||
| 7 | |||
| 8 | import ( | ||
| 9 | "io" | ||
| 10 | "net/http" | ||
| 11 | "path/filepath" | ||
| 12 | "strings" | ||
| 13 | "testing" | ||
| 14 | "time" | ||
| 15 | |||
| 16 | "github.com/rodent-software/vole/internal/config" | ||
| 17 | "github.com/rodent-software/vole/internal/gc" | ||
| 18 | "github.com/rodent-software/vole/internal/runs" | ||
| 19 | ) | ||
| 20 | |||
| 21 | func artifactWorkflow() map[string]string { | ||
| 22 | return workflowFile(`version: 1 | ||
| 23 | jobs: | ||
| 24 | build: | ||
| 25 | steps: | ||
| 26 | - name: build two files | ||
| 27 | run: mkdir -p dist && printf 'binary bytes\n' > dist/tool-linux && printf 'notes\n' > dist/NOTES.txt | ||
| 28 | artifacts: | ||
| 29 | - path: dist/* | ||
| 30 | `) | ||
| 31 | } | ||
| 32 | |||
| 33 | func TestWorkflowArtifactsArePublished(t *testing.T) { | ||
| 34 | f := newFixtureWith(t, func(c *config.Config) { c.LeaseSeconds = 5 }) | ||
| 35 | f.publish(f.repo) | ||
| 36 | |||
| 37 | work := workRepo(t, artifactWorkflow()) | ||
| 38 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 39 | t.Fatalf("push: %s", errs) | ||
| 40 | } | ||
| 41 | f.waitForRun(t, 5*time.Second, func(r *runs.Run) bool { return r.Status == runs.StatusQueued }) | ||
| 42 | f.runOnce(t) | ||
| 43 | |||
| 44 | done := f.waitForRun(t, 10*time.Second, func(r *runs.Run) bool { return r.Status != runs.StatusQueued && r.Status != runs.StatusRunning }) | ||
| 45 | if done.Status != runs.StatusSuccess { | ||
| 46 | t.Fatalf("run status = %s (exit %d)", done.Status, done.ExitCode) | ||
| 47 | } | ||
| 48 | arts, err := f.srv.Runs.Artifacts(done.ID) | ||
| 49 | if err != nil { | ||
| 50 | t.Fatal(err) | ||
| 51 | } | ||
| 52 | if len(arts) != 2 || arts[0].Name != "NOTES.txt" || arts[1].Name != "tool-linux" { | ||
| 53 | t.Fatalf("published %+v, want the two files from dist/", arts) | ||
| 54 | } | ||
| 55 | |||
| 56 | // The run page names them, so a download has a description of its build. | ||
| 57 | code, page := f.get("/r/" + f.repo + "/runs/" + done.ID) | ||
| 58 | if code != 200 { | ||
| 59 | t.Fatalf("run page = %d", code) | ||
| 60 | } | ||
| 61 | for _, want := range []string{"tool-linux", "NOTES.txt", "download", done.Job} { | ||
| 62 | if !strings.Contains(page, want) { | ||
| 63 | t.Fatalf("run page does not show %q:\n%s", want, page) | ||
| 64 | } | ||
| 65 | } | ||
| 66 | |||
| 67 | // The bytes come back out of the store, with the object id as the validator. | ||
| 68 | url := f.http.URL + "/r/" + f.repo + "/runs/" + done.ID + "/artifacts/tool-linux" | ||
| 69 | resp, err := http.Get(url) | ||
| 70 | if err != nil { | ||
| 71 | t.Fatal(err) | ||
| 72 | } | ||
| 73 | body, _ := io.ReadAll(resp.Body) | ||
| 74 | resp.Body.Close() | ||
| 75 | if resp.StatusCode != 200 || string(body) != "binary bytes\n" { | ||
| 76 | t.Fatalf("artifact download = %d %q", resp.StatusCode, body) | ||
| 77 | } | ||
| 78 | if got, want := resp.Header.Get("ETag"), `"`+arts[1].OID+`"`; got != want { | ||
| 79 | t.Fatalf("ETag = %q, want %q", got, want) | ||
| 80 | } | ||
| 81 | if cd := resp.Header.Get("Content-Disposition"); !strings.Contains(cd, "attachment") { | ||
| 82 | t.Fatalf("Content-Disposition = %q", cd) | ||
| 83 | } | ||
| 84 | req, _ := http.NewRequest(http.MethodGet, url, nil) | ||
| 85 | req.Header.Set("If-None-Match", `"`+arts[1].OID+`"`) | ||
| 86 | again, err := http.DefaultClient.Do(req) | ||
| 87 | if err != nil { | ||
| 88 | t.Fatal(err) | ||
| 89 | } | ||
| 90 | again.Body.Close() | ||
| 91 | if again.StatusCode != http.StatusNotModified { | ||
| 92 | t.Fatalf("conditional artifact request = %d, want 304", again.StatusCode) | ||
| 93 | } | ||
| 94 | |||
| 95 | if code, _ := f.get("/r/" + f.repo + "/runs/" + done.ID + "/artifacts/nope"); code != 404 { | ||
| 96 | t.Fatalf("unknown artifact = %d, want 404", code) | ||
| 97 | } | ||
| 98 | // The repository page links the run, so the artifacts are reachable by clicking. | ||
| 99 | if _, repoPage := f.get("/r/" + f.repo); !strings.Contains(repoPage, "/runs/"+done.ID) { | ||
| 100 | t.Fatalf("repository page does not link the run:\n%s", repoPage) | ||
| 101 | } | ||
| 102 | } | ||
| 103 | |||
| 104 | // TestRunArtifactsSurviveTheSweep pins the reachability rule: nothing in git names | ||
| 105 | // a build output, so the run queue is the only thing keeping it. | ||
| 106 | func TestRunArtifactsSurviveTheSweep(t *testing.T) { | ||
| 107 | f := newFixtureWith(t, func(c *config.Config) { c.LeaseSeconds = 5 }) | ||
| 108 | f.publish(f.repo) | ||
| 109 | work := workRepo(t, artifactWorkflow()) | ||
| 110 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 111 | t.Fatalf("push: %s", errs) | ||
| 112 | } | ||
| 113 | f.waitForRun(t, 5*time.Second, func(r *runs.Run) bool { return r.Status == runs.StatusQueued }) | ||
| 114 | f.runOnce(t) | ||
| 115 | done := f.waitForRun(t, 10*time.Second, func(r *runs.Run) bool { return r.Status == runs.StatusSuccess }) | ||
| 116 | arts, err := f.srv.Runs.Artifacts(done.ID) | ||
| 117 | if err != nil || len(arts) == 0 { | ||
| 118 | t.Fatalf("no artifacts to protect: %+v %v", arts, err) | ||
| 119 | } | ||
| 120 | |||
| 121 | oids, err := f.srv.Runs.ArtifactOIDs() | ||
| 122 | if err != nil { | ||
| 123 | t.Fatal(err) | ||
| 124 | } | ||
| 125 | res, err := gc.Run(gc.Options{ | ||
| 126 | Repos: []string{filepath.Join(f.cfg.ReposDir(), f.repo+".git")}, | ||
| 127 | Store: f.srv.LFS, | ||
| 128 | RunOIDs: oids, | ||
| 129 | Now: time.Now().Add(48 * time.Hour), // past every grace period | ||
| 130 | }) | ||
| 131 | if err != nil { | ||
| 132 | t.Fatalf("gc: %v", err) | ||
| 133 | } | ||
| 134 | for _, a := range arts { | ||
| 135 | if !f.srv.LFS.Has(a.OID) { | ||
| 136 | t.Fatalf("gc swept run artifact %q (%+v)", a.Name, res) | ||
| 137 | } | ||
| 138 | } | ||
| 139 | } | ||
| 140 | |||
| 141 | func TestRunArtifactsArePrivateForPrivateRepos(t *testing.T) { | ||
| 142 | f := newFixtureWith(t, func(c *config.Config) { c.LeaseSeconds = 5 }) | ||
| 143 | // Deliberately not published: an anonymous reader may not see it exists. | ||
| 144 | work := workRepo(t, artifactWorkflow()) | ||
| 145 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 146 | t.Fatalf("push: %s", errs) | ||
| 147 | } | ||
| 148 | f.waitForRun(t, 5*time.Second, func(r *runs.Run) bool { return r.Status == runs.StatusQueued }) | ||
| 149 | f.runOnce(t) | ||
| 150 | done := f.waitForRun(t, 10*time.Second, func(r *runs.Run) bool { return r.Status == runs.StatusSuccess }) | ||
| 151 | |||
| 152 | for _, path := range []string{ | ||
| 153 | "/r/" + f.repo + "/runs/" + done.ID, | ||
| 154 | "/r/" + f.repo + "/runs/" + done.ID + "/artifacts/tool-linux", | ||
| 155 | } { | ||
| 156 | if code, _ := f.get(path); code != 404 { | ||
| 157 | t.Fatalf("%s on a private repo = %d, want 404", path, code) | ||
| 158 | } | ||
| 159 | } | ||
| 160 | } | ||