vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4 slice 2: the queue and the runner
b44cbfaf873fda01a458b527e5684b2386266117 · Keenan Nemetz · 18d ago · parent 8fe238d4
Phase 4 slice 2: the queue and the runner A push now runs a pipeline. post-receive spools one file per accepted ref update — the hook must never write to a database or fail a commit — and the server ingests those into data/runs.db, pinning each run to the commit and to the workflow blob it was queued from. The unique index on (repo, ref, sha, workflow, job) is what makes ingestion idempotent, so a spool file read twice after a crash does not run twice. vole runner connects over SSH with the key it would push with: no token, no stored secret, and the same identity the rest of the system uses. It may only run jobs for repositories where its account is an admin, because running a job executes that repository's code — a bigger power than writing to it. Delivery is at-least-once by construction: - A lease hands out a job for a fixed window; the runner heartbeats while it works. - An expired lease means the runner died, and the janitor returns the job to the queue for somebody else. - A runner whose lease is gone stops rather than reporting a result nobody will record, and a job that kills five runners in a row is failed instead of cycling. Three bugs found while building it, each worth the test that now covers it: - Background loops lived in Run(), so the tests — which serve the SSH listener themselves — never had ingestion or lease expiry. They start in New() now. - post-receive derived the repository name from GIT_DIR, which git sets to "." for hooks, so every event was queued for a repository called ".". The same trap had already appeared in a pre-receive message; both now resolve the absolute path. - Leases ordered only by creation time, so two pushes in the same second could be handed out in any order; the id breaks the tie, and ULIDs sort by time. Verified end to end on a live instance: a push queues a job, a runner leases it, clones the commit, prints "hi", and the run closes success with the log attached. The integration tests cover the two exit criteria that matter — a push runs a pipeline, and killing a runner mid-job requeues the work onto a second runner (attempts=2, success) — plus the property that editing a workflow after a push does not change what the already-queued run executes. Not built yet: statuses on change requests and required-check gating (slice 3), and signed run records (Phase 7). Runs live in data/runs.db and are visible there, not on the web.
internal/server/ci.go +137
| @@ -0,0 +1,137 @@ | |||
| 1 | package server | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "context" | ||
| 5 | "fmt" | ||
| 6 | "strings" | ||
| 7 | "time" | ||
| 8 | |||
| 9 | "github.com/rodent-software/vole/internal/id" | ||
| 10 | "github.com/rodent-software/vole/internal/runs" | ||
| 11 | "github.com/rodent-software/vole/internal/workflow" | ||
| 12 | ) | ||
| 13 | |||
| 14 | // watchSpool turns accepted pushes into queued CI runs. | ||
| 15 | // | ||
| 16 | // The hook cannot write to the database — it runs inside a push and must not hold | ||
| 17 | // a lock or fail a commit — so it drops one file per ref update and this loop | ||
| 18 | // consumes them. Ingestion is idempotent: a spool file read twice enqueues the | ||
| 19 | // same (repo, ref, sha, workflow, job) once, because the queue has a unique index | ||
| 20 | // on exactly that tuple. | ||
| 21 | func (s *Server) watchSpool(ctx context.Context) { | ||
| 22 | t := time.NewTicker(time.Second) | ||
| 23 | defer t.Stop() | ||
| 24 | for { | ||
| 25 | select { | ||
| 26 | case <-ctx.Done(): | ||
| 27 | return | ||
| 28 | case <-t.C: | ||
| 29 | entries, err := runs.Next(s.Cfg.SpoolDir()) | ||
| 30 | if err != nil { | ||
| 31 | s.Log.Warn("ci: cannot read the push spool", "err", err) | ||
| 32 | continue | ||
| 33 | } | ||
| 34 | for _, e := range entries { | ||
| 35 | if e.Err != nil { | ||
| 36 | s.Log.Warn("ci: dropping an unreadable spool entry", "path", e.Path, "err", e.Err) | ||
| 37 | } else { | ||
| 38 | s.enqueue(e.Event) | ||
| 39 | } | ||
| 40 | if err := runs.Done(e.Path); err != nil { | ||
| 41 | s.Log.Warn("ci: cannot clear a spool entry", "path", e.Path, "err", err) | ||
| 42 | } | ||
| 43 | } | ||
| 44 | } | ||
| 45 | } | ||
| 46 | } | ||
| 47 | |||
| 48 | // enqueue creates one run per job of every workflow that matches the pushed | ||
| 49 | // branch, pinned to the commit and the workflow blob that were pushed. | ||
| 50 | func (s *Server) enqueue(ev runs.Event) { | ||
| 51 | if !strings.HasPrefix(ev.Ref, "refs/heads/") { | ||
| 52 | return // tags and the metadata branch do not run CI | ||
| 53 | } | ||
| 54 | if ev.New == "" || strings.HasPrefix(ev.New, "0000000000000000000000000000000000000000") { | ||
| 55 | return // a branch deletion | ||
| 56 | } | ||
| 57 | m, err := s.Repos.Open(ev.Repo) | ||
| 58 | if err != nil { | ||
| 59 | s.Log.Warn("ci: cannot open the pushed repository", "repo", ev.Repo, "err", err) | ||
| 60 | return | ||
| 61 | } | ||
| 62 | // Trust the ref, not the spool: a push that lost a race may have left an event | ||
| 63 | // whose new commit never became the tip. | ||
| 64 | tip, err := m.Git().RevParse(ev.Ref) | ||
| 65 | if err != nil || tip != ev.New { | ||
| 66 | s.Log.Debug("ci: skipping a spool entry whose ref has moved", "repo", ev.Repo, "ref", ev.Ref) | ||
| 67 | return | ||
| 68 | } | ||
| 69 | wfs, problems := workflow.At(m.Git(), ev.New) | ||
| 70 | s.Log.Debug("ci: examined push", "repo", ev.Repo, "ref", ev.Ref, "workflows", len(wfs), "problems", len(problems)) | ||
| 71 | if len(problems) > 0 { | ||
| 72 | // pre-receive rejects these, so reaching here means a workflow was already | ||
| 73 | // broken: queue nothing rather than guess. | ||
| 74 | s.Log.Warn("ci: workflows in a pushed commit do not validate", "repo", ev.Repo, "problems", len(problems)) | ||
| 75 | return | ||
| 76 | } | ||
| 77 | branch := strings.TrimPrefix(ev.Ref, "refs/heads/") | ||
| 78 | for _, wf := range wfs { | ||
| 79 | if !wf.Applies(branch) { | ||
| 80 | continue | ||
| 81 | } | ||
| 82 | for _, job := range wf.Jobs { | ||
| 83 | run := runs.Run{ | ||
| 84 | Repo: ev.Repo, | ||
| 85 | Ref: ev.Ref, | ||
| 86 | Branch: branch, | ||
| 87 | SHA: ev.New, | ||
| 88 | Workflow: wf.Path, | ||
| 89 | WorkflowSHA: wf.BlobSHA, | ||
| 90 | Job: job.ID, | ||
| 91 | Steps: steps(job), | ||
| 92 | Created: time.Now().UTC(), | ||
| 93 | } | ||
| 94 | run.ID, _ = id.New(time.Now()) | ||
| 95 | created, err := s.Runs.Enqueue(run) | ||
| 96 | if err != nil { | ||
| 97 | s.Log.Warn("ci: cannot queue a run", "repo", ev.Repo, "job", job.ID, "err", err) | ||
| 98 | continue | ||
| 99 | } | ||
| 100 | if created { | ||
| 101 | s.Log.Info("ci: queued", "repo", ev.Repo, "job", job.ID, "sha", ev.New[:8], "run", run.ID) | ||
| 102 | } | ||
| 103 | } | ||
| 104 | } | ||
| 105 | } | ||
| 106 | |||
| 107 | func steps(job workflow.Job) []runs.Step { | ||
| 108 | out := make([]runs.Step, 0, len(job.Steps)) | ||
| 109 | for _, st := range job.Steps { | ||
| 110 | // The step list is pinned at enqueue time, so editing the workflow after the | ||
| 111 | // push cannot change what an already-queued run executes. | ||
| 112 | out = append(out, runs.Step{Name: st.Name, Run: st.Run, Timeout: st.Timeout.String()}) | ||
| 113 | } | ||
| 114 | return out | ||
| 115 | } | ||
| 116 | |||
| 117 | // watchLeases hands jobs from dead runners back to the queue. | ||
| 118 | func (s *Server) watchLeases(ctx context.Context) { | ||
| 119 | t := time.NewTicker(5 * time.Second) | ||
| 120 | defer t.Stop() | ||
| 121 | for { | ||
| 122 | select { | ||
| 123 | case <-ctx.Done(): | ||
| 124 | return | ||
| 125 | case <-t.C: | ||
| 126 | back, failed, err := s.Runs.RequeueExpired(time.Now().UTC()) | ||
| 127 | if err != nil { | ||
| 128 | s.Log.Warn("ci: cannot requeue expired leases", "err", err) | ||
| 129 | continue | ||
| 130 | } | ||
| 131 | if back > 0 || failed > 0 { | ||
| 132 | s.Log.Warn(fmt.Sprintf("ci: %d job(s) returned to the queue, %d failed after %d attempts", | ||
| 133 | back, failed, runs.MaxAttempts)) | ||
| 134 | } | ||
| 135 | } | ||
| 136 | } | ||
| 137 | } | ||