vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Store CI run records in the database, not the repository
b312e1660ac3d570fc35350a22b3514c56ab466f · Keenan Nemetz · 11d ago · parent db5e4382
Store CI run records in the database, not the repository A finished attempt was a set of files on the vole branch: a record, step ranges, and artifact pointers, pushed as a commit. At 959 files, CI records were 849 of them, and every human metadata write had to carry them. The runner now sends the record with its finish message over the SSH channel that holds the lease. The server checks the account, the job, and the log and artifact pointers, then writes the attempt, steps, artifacts, attestation, and run result in one transaction in data/runs.db. A tagged build still pushes its release pointers to the branch after the record is stored. Pre-receive refuses a new runs/ file. vole admin index rebuild imports records still on an old branch; records written since are never deleted by a rebuild. The run page reads the stored attempt, and runs.db is now a source that must be backed up. This stops new CI history from entering Git. Existing commits keep the records they already have; removing those would require rewriting the protected vole branch, which this change does not do.
ci/agent/agent.go +54−47
| @@ -4,9 +4,9 @@ | |||
| 4 | 4 | // with, leases one job at a time, and gives the job a disposable checkout. Steps | |
| 5 | 5 | // run in a container built from the image the workflow names, or from a pinned | |
| 6 | 6 | // .vole/images definition; the runner's key and trusted Git cache stay outside. | |
| 7 | // When the job ends the runner uploads the log and artifacts and publishes the | ||
| 8 | // attempt as a signed commit. This package is the agent; argument parsing and the | ||
| 9 | // CLI's messages stay in cmd/vole. | ||
| 7 | // When the job ends the runner uploads the log and artifacts, sends the attempt to | ||
| 8 | // the server with its finish message, and attaches any release pointers. This | ||
| 9 | // package is the agent; argument parsing and the CLI's messages stay in cmd/vole. | ||
| 10 | 10 | package agent | |
| 11 | 11 | ||
| 12 | 12 | import ( | |
| @@ -102,12 +102,14 @@ type runnerArtifact struct { | |||
| 102 | 102 | Size int64 `json:"size"` | |
| 103 | 103 | } | |
| 104 | 104 | ||
| 105 | // runnerRequest is the wire. A finish carries nothing but the run id: the result | ||
| 106 | // went up as a git push before this was sent, because the record is the runner's | ||
| 107 | // statement and the push is what the gate checks. This channel only closes the lease. | ||
| 105 | // runnerRequest is the wire. A finish carries the attempt's record: the runner | ||
| 106 | // no longer publishes it to Git, and this channel is what the server checks and | ||
| 107 | // stores. | ||
| 108 | 108 | type runnerRequest struct { | |
| 109 | 109 | Op string `json:"op"` | |
| 110 | 110 | Run string `json:"run"` | |
| 111 | // Record is the attempt's result, sent with finish. | ||
| 112 | Record *runs.Record `json:"record,omitempty"` | ||
| 111 | 113 | // Repos is the --repos filter, sent with every lease. The server intersects it | |
| 112 | 114 | // with what the account's allowlists grant, so it can only narrow. | |
| 113 | 115 | Repos []string `json:"repos,omitempty"` | |
| @@ -613,19 +615,18 @@ func (s *runnerSession) cacheFor(job *runnerJob) []container.Mount { | |||
| 613 | 615 | return []container.Mount{{Host: ns, Path: jobcache.Mount}} | |
| 614 | 616 | } | |
| 615 | 617 | ||
| 616 | // finish uploads this attempt's log, commits its record and pushes it, and only then | ||
| 617 | // tells the server it is done. | ||
| 618 | // finish uploads this attempt's log, sends its record to the server with the | ||
| 619 | // finish message, and then attaches any release pointers the build published. | ||
| 618 | 620 | // | |
| 619 | 621 | // The bytes go in first, the same rule artifacts follow: a record is a set of | |
| 620 | // pointers, and a pointer to an object the store does not have would fail the push | ||
| 621 | // that carries it. Then one signed commit carries the whole run — the record and any | ||
| 622 | // release pointers the build published — and one push lands it. The server creates | ||
| 623 | // nothing here: `pre-receive` checks the signature and the runner allowlist, and the | ||
| 624 | // finish message only closes the lease. | ||
| 622 | // pointers, and the server refuses one whose log or artifact is not in the store. | ||
| 623 | // The record is not a commit; the server stores it with the run's result in one | ||
| 624 | // transaction. A tagged build's release pointers are product metadata and are pushed | ||
| 625 | // afterwards, so a release cannot exist without the build that produced it. | ||
| 625 | 626 | // | |
| 626 | 627 | // A log that cannot be stored does not lose the result: the record then names no log | |
| 627 | // and the failure is reported on stderr rather than swallowed. A record that cannot be | ||
| 628 | // pushed is not reported at all, because there is nothing for the server to read. | ||
| 628 | // and the failure is reported on stderr rather than swallowed. A record the server | ||
| 629 | // does not accept is not reported at all. | ||
| 629 | 630 | func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log *spool, exit int, steps []runs.StepResult, arts, releaseArts []runnerArtifact, box runnerBox) { | |
| 630 | 631 | oid, size, err := uploadLog(remote, job.Repo, log) | |
| 631 | 632 | if err != nil { | |
| @@ -658,8 +659,8 @@ func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log * | |||
| 658 | 659 | rec.Log = &lfs.Pointer{OID: oid, Size: size} | |
| 659 | 660 | } | |
| 660 | 661 | ||
| 661 | // Keep the lease through the push: the record is checked against it, and a slow | ||
| 662 | // upload must not turn a finished job into one that expired mid-report. | ||
| 662 | // Keep the lease through the report: the record is checked against it, and a | ||
| 663 | // slow upload must not turn a finished job into one that expired mid-report. | ||
| 663 | 664 | if rep, err := s.call(runnerRequest{Op: "heartbeat", Run: job.ID}); err != nil { | |
| 664 | 665 | fmt.Fprintf(os.Stderr, "vole runner: lost the connection while finishing %s: %v\n", job.ID, err) | |
| 665 | 666 | return | |
| @@ -668,49 +669,60 @@ func (s *runnerSession) finish(remote string, co checkout, job *runnerJob, log * | |||
| 668 | 669 | return | |
| 669 | 670 | } | |
| 670 | 671 | ||
| 671 | if err := publishRecord(remote, co, job, rec, releaseArts, log); err != nil { | ||
| 672 | fmt.Fprintf(os.Stderr, "vole runner: could not publish the record for %s: %v\n", job.ID, err) | ||
| 673 | return | ||
| 674 | } | ||
| 675 | |||
| 676 | rep, err := s.call(runnerRequest{Op: "finish", Run: job.ID}) | ||
| 672 | // The record goes to the server on the same channel that holds the lease. If | ||
| 673 | // this call is lost after the server stored the record, the retry sends the | ||
| 674 | // same bytes and the server answers ok again; a run record is append-only. | ||
| 675 | rep, err := s.call(runnerRequest{Op: "finish", Run: job.ID, Record: &rec}) | ||
| 677 | 676 | switch { | |
| 678 | 677 | case err != nil: | |
| 679 | fmt.Fprintf(os.Stderr, "vole runner: connection lost after publishing %s; it is on the branch\n", job.ID) | ||
| 678 | fmt.Fprintf(os.Stderr, "vole runner: connection lost while reporting %s; the result was not stored\n", job.ID) | ||
| 679 | return | ||
| 680 | 680 | case rep.Error != "": | |
| 681 | 681 | fmt.Fprintf(os.Stderr, "vole runner: %s\n", rep.Error) | |
| 682 | return | ||
| 682 | 683 | case !rep.OK: | |
| 683 | fmt.Fprintf(os.Stderr, "vole runner: the lease for %s moved; the record is on the branch but the queue did not take it\n", job.ID) | ||
| 684 | fmt.Fprintf(os.Stderr, "vole runner: the lease for %s moved; the result was not stored\n", job.ID) | ||
| 685 | return | ||
| 686 | } | ||
| 687 | |||
| 688 | // The result is stored. A tagged build's release pointers are product metadata | ||
| 689 | // and still live on the branch; attach them only after the record, so a release | ||
| 690 | // cannot exist without the build that produced it. | ||
| 691 | if len(releaseArts) > 0 { | ||
| 692 | if err := publishReleases(remote, co, job, releaseArts, log); err != nil { | ||
| 693 | fmt.Fprintf(os.Stderr, "vole runner: could not attach %d artifact(s) to %s: %v\n", | ||
| 694 | len(releaseArts), job.Tag, err) | ||
| 695 | fmt.Fprintf(log, "\nrelease: could not attach %d artifact(s) to %s: %v\n", | ||
| 696 | len(releaseArts), job.Tag, err) | ||
| 697 | } | ||
| 684 | 698 | } | |
| 685 | 699 | } | |
| 686 | 700 | ||
| 687 | // publishRecord commits the attempt's record, with the release pointers the build | ||
| 688 | // published beside it, and pushes once. | ||
| 701 | // publishReleases attaches a tagged build's release pointers to the metadata | ||
| 702 | // branch. The run record is already in the database, so a failure here loses the | ||
| 703 | // release attachment, not the result; the operator can attach it again with | ||
| 704 | // `vole release attach`. | ||
| 689 | 705 | // | |
| 690 | // One push per run is the point: the result and the artifacts it produced are one | ||
| 691 | // commit, so a release cannot carry bytes whose build has no record and a run does not | ||
| 692 | // pay a push per output. A lost race against somebody else's issue is re-applied on | ||
| 693 | // the new tip, the same way every other metadata write is. | ||
| 694 | func publishRecord(remote string, co checkout, job *runnerJob, rec runs.Record, rel []runnerArtifact, log io.Writer) error { | ||
| 706 | // The branch is pushed once, and a lost race against somebody else's issue is | ||
| 707 | // re-applied on the new tip the same way every other metadata write is. The | ||
| 708 | // runner's trusted cache is used, never the job tree: the workflow may have edited | ||
| 709 | // the job tree's .git, and a push from there could run a hook the workflow wrote. | ||
| 710 | func publishReleases(remote string, co checkout, job *runnerJob, rel []runnerArtifact, log io.Writer) error { | ||
| 695 | 711 | url, err := client.RepoURL(remote, job.Repo) | |
| 696 | 712 | if err != nil { | |
| 697 | 713 | return err | |
| 698 | 714 | } | |
| 699 | // Use the trusted cache, never the job tree: the workflow may have edited the | ||
| 700 | // job tree's .git, and a push from there could run a hook the workflow wrote. | ||
| 701 | 715 | c, err := client.OpenBranch(co.GitDir, url, strings.TrimPrefix(metadata.Ref, "refs/heads/"), "vole-runner") | |
| 702 | 716 | if err != nil { | |
| 703 | 717 | return err | |
| 704 | 718 | } | |
| 705 | add := func(files map[string][]byte) error { | ||
| 706 | for _, a := range rel { | ||
| 707 | files[metadata.ReleasePath(job.Tag, a.Name)] = lfs.Pointer{OID: a.OID, Size: a.Size}.Encode() | ||
| 708 | } | ||
| 709 | return nil | ||
| 719 | arts := make([]metadata.Artifact, 0, len(rel)) | ||
| 720 | for _, a := range rel { | ||
| 721 | arts = append(arts, metadata.Artifact{Name: a.Name, OID: a.OID, Size: a.Size}) | ||
| 710 | 722 | } | |
| 723 | message := fmt.Sprintf("release: attach %d artifact(s) to %s from run %s", len(arts), job.Tag, job.ID) | ||
| 711 | 724 | write := func() error { | |
| 712 | _, err := runs.WriteRecord(c.Repo, metadata.Ref, rec, c.Author, add) | ||
| 713 | return err | ||
| 725 | return metadata.WriteReleasePointers(c.Repo, metadata.Ref, job.Tag, arts, message, c.Author) | ||
| 714 | 726 | } | |
| 715 | 727 | if err := write(); err != nil { | |
| 716 | 728 | return err | |
| @@ -718,12 +730,7 @@ func publishRecord(remote string, co checkout, job *runnerJob, rec runs.Record, | |||
| 718 | 730 | if err := c.CommitAndPush(false, write); err != nil { | |
| 719 | 731 | return err | |
| 720 | 732 | } | |
| 721 | if rec.Attempt > 0 { | ||
| 722 | fmt.Fprintf(log, "\nrecord: pushed %s\n", runs.RecordPath(rec.RunID, rec.Attempt)) | ||
| 723 | } | ||
| 724 | if len(rel) > 0 { | ||
| 725 | fmt.Fprintf(log, "\nrelease: attached %d artifact(s) to %s\n", len(rel), job.Tag) | ||
| 726 | } | ||
| 733 | fmt.Fprintf(log, "\nrelease: attached %d artifact(s) to %s\n", len(arts), job.Tag) | ||
| 727 | 734 | return nil | |
| 728 | 735 | } | |
| 729 | 736 | ||