vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Store CI run records in the database, not the repository
b312e1660ac3d570fc35350a22b3514c56ab466f · Keenan Nemetz · 11d ago · parent db5e4382
Store CI run records in the database, not the repository A finished attempt was a set of files on the vole branch: a record, step ranges, and artifact pointers, pushed as a commit. At 959 files, CI records were 849 of them, and every human metadata write had to carry them. The runner now sends the record with its finish message over the SSH channel that holds the lease. The server checks the account, the job, and the log and artifact pointers, then writes the attempt, steps, artifacts, attestation, and run result in one transaction in data/runs.db. A tagged build still pushes its release pointers to the branch after the record is stored. Pre-receive refuses a new runs/ file. vole admin index rebuild imports records still on an old branch; records written since are never deleted by a rebuild. The run page reads the stored attempt, and runs.db is now a source that must be backed up. This stops new CI history from entering Git. Existing commits keep the records they already have; removing those would require rewriting the protected vole branch, which this change does not do.
server/hook/records.go +16−105
| @@ -1,127 +1,38 @@ | |||
| 1 | 1 | package hook | |
| 2 | 2 | ||
| 3 | // Who may report a run result. | ||
| 3 | // CI run records no longer live on the metadata branch. A runner sends the result | ||
| 4 | // over its authenticated finish channel, and the server stores it in runs.db. The | ||
| 5 | // push boundary's only remaining job is to refuse a client that tries to put a | ||
| 6 | // record back: a record in Git would be ignored by the server and would grow the | ||
| 7 | // branch again. | ||
| 4 | 8 | // | |
| 5 | // A finished attempt's record is written by the runner that ran it: the party that | ||
| 6 | // did the work states what came of it, signs the commit and pushes it with the | ||
| 7 | // artifacts the build published. The gate's question is not "is this record well | ||
| 8 | // formed" but "was this account allowed to be the one that reported this attempt", | ||
| 9 | // and the answer is the same allowlist that decides who may lease a job — | ||
| 10 | // `vole admin acl runners`. Admins are implicit there for the same reason as everywhere | ||
| 11 | // else: they can rewrite the list. | ||
| 12 | // | ||
| 13 | // Two further rules make the record describe something that actually happened. The | ||
| 14 | // commit that adds it must be signed by a key the pushing account holds — that | ||
| 15 | // signature is the whole attestation, so an unsigned record is a claim nobody can | ||
| 16 | // check — and the record must describe the job the queue handed to that account: the | ||
| 17 | // same run, the same attempt, the same commit, workflow and job. Without the second | ||
| 18 | // rule an allowed runner could report a green result for a commit nobody asked it to | ||
| 19 | // build, or invent attempt 99 for a finished run and have the next rebuild believe it. | ||
| 9 | // Deletions are allowed so an operator can remove records written before the move. | ||
| 10 | // Additions, edits and renames are refused. | ||
| 20 | 11 | ||
| 21 | 12 | import ( | |
| 22 | 13 | "fmt" | |
| 23 | 14 | ||
| 24 | 15 | "github.com/rodent-software/vole/ci/runs" | |
| 25 | 16 | "github.com/rodent-software/vole/core/git" | |
| 26 | "github.com/rodent-software/vole/forge/access" | ||
| 27 | "github.com/rodent-software/vole/forge/landing" | ||
| 28 | 17 | ) | |
| 29 | 18 | ||
| 30 | func recordFindings(g *git.Repo, c change, a *access.ACL, actor, repoName, usersDB, runsDB string) []string { | ||
| 31 | added, problems, err := runs.CheckPush(g, c.old, c.new) | ||
| 19 | func recordFindings(g *git.Repo, c change) []string { | ||
| 20 | changes, err := runs.RecordChanges(g, c.old, c.new) | ||
| 32 | 21 | if err != nil { | |
| 33 | 22 | return []string{fmt.Sprintf("%s: cannot read the run records in this push: %v", c.ref, err)} | |
| 34 | 23 | } | |
| 35 | if len(added) == 0 { | ||
| 36 | return problems | ||
| 37 | } | ||
| 38 | if a == nil || !a.CanReport(actor) { | ||
| 39 | people := "none named, so only admins" | ||
| 40 | if a != nil { | ||
| 41 | people = landing.RunnerSummary(a) | ||
| 24 | var problems []string | ||
| 25 | for _, ch := range changes { | ||
| 26 | if ch.Status == "D" { | ||
| 27 | continue | ||
| 42 | 28 | } | |
| 43 | return append(problems, fmt.Sprintf( | ||
| 44 | "%s: %s reported a run result but is not a runner for %s (runners: %s). "+ | ||
| 45 | "An admin can allow that account with `vole admin acl runners add %s %s`", | ||
| 46 | c.ref, actor, repoName, people, repoName, actor)) | ||
| 47 | } | ||
| 48 | if runsDB == "" { | ||
| 49 | return append(problems, fmt.Sprintf( | ||
| 50 | "%s: this instance has no run queue, so a run result has nothing to be a result of", c.ref)) | ||
| 51 | } | ||
| 52 | store, err := runs.Open(runsDB) | ||
| 53 | if err != nil { | ||
| 54 | return append(problems, fmt.Sprintf("%s: cannot open the run queue: %v", c.ref, err)) | ||
| 55 | } | ||
| 56 | defer store.Close() | ||
| 57 | |||
| 58 | for _, rec := range added { | ||
| 59 | problems = append(problems, recordProblems(g, store, rec, actor, repoName)...) | ||
| 29 | problems = append(problems, fmt.Sprintf( | ||
| 30 | "%s: run records are stored in the instance database now; a runner reports a result with finish, not a push", | ||
| 31 | ch.Path)) | ||
| 60 | 32 | } | |
| 61 | 33 | return problems | |
| 62 | 34 | } | |
| 63 | 35 | ||
| 64 | // recordProblems checks one added record against the account that pushed it and the job | ||
| 65 | // the queue handed out. Attribution is the boundary's: the record has to name the | ||
| 66 | // account that pushed it, and that is the same rule an issue's author and an approval's | ||
| 67 | // owner follow. There is no signature to check, because there is no signature anywhere | ||
| 68 | // else either. | ||
| 69 | func recordProblems(g *git.Repo, store *runs.Store, rec runs.Record, actor, repoName string) []string { | ||
| 70 | where := runs.RecordPath(rec.RunID, rec.Attempt) | ||
| 71 | var out []string | ||
| 72 | |||
| 73 | // The signer is not the pusher's to choose: the file names an account, and the | ||
| 74 | // commit has to be signed by that account, or the name is decoration. | ||
| 75 | if rec.Signer != actor { | ||
| 76 | out = append(out, fmt.Sprintf( | ||
| 77 | "%s: the record names %s as the account that reported it, but %s pushed it; "+ | ||
| 78 | "a runner reports its own results", where, rec.Signer, actor)) | ||
| 79 | } | ||
| 80 | |||
| 81 | // The record has to be in a commit, because the commit is what the branch carries and | ||
| 82 | // what a reader follows back to the bytes. | ||
| 83 | if rec.Commit == "" { | ||
| 84 | out = append(out, fmt.Sprintf("%s: the commit that adds the record could not be found", where)) | ||
| 85 | } | ||
| 86 | |||
| 87 | if rec.Repo != repoName { | ||
| 88 | out = append(out, fmt.Sprintf("%s: the record says it ran in %q, not %s", where, rec.Repo, repoName)) | ||
| 89 | } | ||
| 90 | run, err := store.Get(rec.RunID) | ||
| 91 | if err != nil { | ||
| 92 | return append(out, fmt.Sprintf("%s: cannot read the run: %v", where, err)) | ||
| 93 | } | ||
| 94 | if run == nil { | ||
| 95 | return append(out, fmt.Sprintf("%s: %s is not a run this instance queued", where, rec.RunID)) | ||
| 96 | } | ||
| 97 | switch { | ||
| 98 | case run.Repo != repoName: | ||
| 99 | out = append(out, fmt.Sprintf("%s: run %s belongs to %s", where, rec.RunID, run.Repo)) | ||
| 100 | case run.Status != runs.StatusRunning || run.LeaseOwner != actor: | ||
| 101 | out = append(out, fmt.Sprintf( | ||
| 102 | "%s: %s is not running under %s's lease right now (%s), so only the runner holding it can report it", | ||
| 103 | where, rec.RunID, actor, describe(run))) | ||
| 104 | case run.Attempts != rec.Attempt: | ||
| 105 | out = append(out, fmt.Sprintf("%s: the queue is on attempt %d for %s, not %d", | ||
| 106 | where, run.Attempts, rec.RunID, rec.Attempt)) | ||
| 107 | case rec.SHA != run.SHA || rec.Ref != run.Ref || rec.Workflow != run.Workflow || | ||
| 108 | rec.WorkflowSHA != run.WorkflowSHA || rec.Job != run.Job || rec.StepsSHA != run.StepsSHA || | ||
| 109 | (run.Image != "" && (rec.Image != run.Image || rec.ImageFile != run.ImageFile || rec.ImageSHA != run.ImageSHA)): | ||
| 110 | out = append(out, fmt.Sprintf( | ||
| 111 | "%s: the record describes a different job than the one this instance queued for %s "+ | ||
| 112 | "(it must name the commit, workflow and job it was handed)", where, rec.RunID)) | ||
| 113 | } | ||
| 114 | return out | ||
| 115 | } | ||
| 116 | |||
| 117 | // describe renders a run's queue state for a refusal. | ||
| 118 | func describe(run *runs.Run) string { | ||
| 119 | if run.LeaseOwner == "" { | ||
| 120 | return run.Status | ||
| 121 | } | ||
| 122 | return run.Status + " on " + run.LeaseOwner | ||
| 123 | } | ||
| 124 | |||
| 125 | 36 | // openRuns opens the run queue for one gate check. The pre-receive hook is a | |
| 126 | 37 | // process that answers one push and exits, so the store is opened per check and not | |
| 127 | 38 | // closed: the close would come after the answer, and the process ends there anyway. | |