vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4 slice 2: the queue and the runner
b44cbfaf873fda01a458b527e5684b2386266117 · Keenan Nemetz · 17d ago · parent 8fe238d4
Phase 4 slice 2: the queue and the runner A push now runs a pipeline. post-receive spools one file per accepted ref update — the hook must never write to a database or fail a commit — and the server ingests those into data/runs.db, pinning each run to the commit and to the workflow blob it was queued from. The unique index on (repo, ref, sha, workflow, job) is what makes ingestion idempotent, so a spool file read twice after a crash does not run twice. vole runner connects over SSH with the key it would push with: no token, no stored secret, and the same identity the rest of the system uses. It may only run jobs for repositories where its account is an admin, because running a job executes that repository's code — a bigger power than writing to it. Delivery is at-least-once by construction: - A lease hands out a job for a fixed window; the runner heartbeats while it works. - An expired lease means the runner died, and the janitor returns the job to the queue for somebody else. - A runner whose lease is gone stops rather than reporting a result nobody will record, and a job that kills five runners in a row is failed instead of cycling. Three bugs found while building it, each worth the test that now covers it: - Background loops lived in Run(), so the tests — which serve the SSH listener themselves — never had ingestion or lease expiry. They start in New() now. - post-receive derived the repository name from GIT_DIR, which git sets to "." for hooks, so every event was queued for a repository called ".". The same trap had already appeared in a pre-receive message; both now resolve the absolute path. - Leases ordered only by creation time, so two pushes in the same second could be handed out in any order; the id breaks the tie, and ULIDs sort by time. Verified end to end on a live instance: a push queues a job, a runner leases it, clones the commit, prints "hi", and the run closes success with the log attached. The integration tests cover the two exit criteria that matter — a push runs a pipeline, and killing a runner mid-job requeues the work onto a second runner (attempts=2, success) — plus the property that editing a workflow after a push does not change what the already-queued run executes. Not built yet: statuses on change requests and required-check gating (slice 3), and signed run records (Phase 7). Runs live in data/runs.db and are visible there, not on the web.
internal/sshserver/runner.go +147
| @@ -0,0 +1,147 @@ | |||
| 1 | package sshserver | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "encoding/json" | ||
| 5 | "time" | ||
| 6 | |||
| 7 | "github.com/rodent-software/vole/internal/acl" | ||
| 8 | "golang.org/x/crypto/ssh" | ||
| 9 | ) | ||
| 10 | |||
| 11 | // The runner protocol. | ||
| 12 | // | ||
| 13 | // A runner connects over the same SSH transport as everything else, so it | ||
| 14 | // authenticates with a key and needs no token: the account it connects as decides | ||
| 15 | // which repositories it may run jobs for. One JSON object per line, one reply per | ||
| 16 | // request, on a single channel. | ||
| 17 | // | ||
| 18 | // {"op":"lease"} -> {"run":{...}} or {} | ||
| 19 | // {"op":"heartbeat","run":"<id>","log":"..."} -> {"ok":true|false} | ||
| 20 | // {"op":"finish","run":"<id>","exit":0} -> {"ok":true|false} | ||
| 21 | // | ||
| 22 | // "ok":false means the lease is gone — another runner has the job — and the runner | ||
| 23 | // must stop rather than report a result nobody will record. | ||
| 24 | |||
| 25 | type runnerRequest struct { | ||
| 26 | Op string `json:"op"` | ||
| 27 | Run string `json:"run"` | ||
| 28 | Log string `json:"log"` | ||
| 29 | Exit int `json:"exit"` | ||
| 30 | } | ||
| 31 | |||
| 32 | type runnerStep struct { | ||
| 33 | Name string `json:"name"` | ||
| 34 | Run string `json:"run"` | ||
| 35 | Timeout string `json:"timeout"` | ||
| 36 | } | ||
| 37 | |||
| 38 | type runnerJob struct { | ||
| 39 | ID string `json:"id"` | ||
| 40 | Repo string `json:"repo"` | ||
| 41 | SHA string `json:"sha"` | ||
| 42 | Branch string `json:"branch"` | ||
| 43 | Workflow string `json:"workflow"` | ||
| 44 | Job string `json:"job"` | ||
| 45 | Steps []runnerStep `json:"steps"` | ||
| 46 | } | ||
| 47 | |||
| 48 | type runnerReply struct { | ||
| 49 | Op string `json:"op"` | ||
| 50 | Error string `json:"error,omitempty"` | ||
| 51 | Run *runnerJob `json:"run,omitempty"` | ||
| 52 | OK bool `json:"ok"` | ||
| 53 | Lease int `json:"lease,omitempty"` | ||
| 54 | } | ||
| 55 | |||
| 56 | // serveRunner answers one runner session until it disconnects. | ||
| 57 | func (s *Server) serveRunner(ch ssh.Channel, user, accountID string) { | ||
| 58 | defer ch.Close() | ||
| 59 | cfg := s.cfg | ||
| 60 | if cfg.Runs == nil { | ||
| 61 | _ = json.NewEncoder(ch).Encode(runnerReply{Op: "error", Error: "the run queue is not configured on this server"}) | ||
| 62 | return | ||
| 63 | } | ||
| 64 | dec := json.NewDecoder(ch) | ||
| 65 | enc := json.NewEncoder(ch) | ||
| 66 | owner := accountID | ||
| 67 | cfg.Log.Info("runner connected", "user", user) | ||
| 68 | |||
| 69 | for { | ||
| 70 | var req runnerRequest | ||
| 71 | if err := dec.Decode(&req); err != nil { | ||
| 72 | return // disconnected | ||
| 73 | } | ||
| 74 | switch req.Op { | ||
| 75 | case "lease": | ||
| 76 | // A runner may only run jobs for repositories it administers: running a | ||
| 77 | // job executes that repository's code, which is a bigger power than | ||
| 78 | // writing to it. | ||
| 79 | repos, err := s.runnableRepos(accountID) | ||
| 80 | if err != nil { | ||
| 81 | _ = enc.Encode(runnerReply{Op: "lease", Error: err.Error()}) | ||
| 82 | continue | ||
| 83 | } | ||
| 84 | run, err := cfg.Runs.Lease(owner, repos, cfg.Lease, time.Now().UTC()) | ||
| 85 | if err != nil { | ||
| 86 | _ = enc.Encode(runnerReply{Op: "lease", Error: err.Error()}) | ||
| 87 | continue | ||
| 88 | } | ||
| 89 | if run == nil { | ||
| 90 | _ = enc.Encode(runnerReply{Op: "lease", OK: true}) | ||
| 91 | continue | ||
| 92 | } | ||
| 93 | job := &runnerJob{ | ||
| 94 | ID: run.ID, Repo: run.Repo, SHA: run.SHA, Branch: run.Branch, | ||
| 95 | Workflow: run.Workflow, Job: run.Job, | ||
| 96 | } | ||
| 97 | for _, st := range run.Steps { | ||
| 98 | job.Steps = append(job.Steps, runnerStep{Name: st.Name, Run: st.Run, Timeout: st.Timeout}) | ||
| 99 | } | ||
| 100 | cfg.Log.Info("runner leased a job", "user", user, "repo", run.Repo, | ||
| 101 | "job", run.Job, "sha", run.SHA[:8], "run", run.ID, "attempt", run.Attempts) | ||
| 102 | _ = enc.Encode(runnerReply{Op: "lease", OK: true, Run: job, Lease: int(cfg.Lease.Seconds())}) | ||
| 103 | |||
| 104 | case "heartbeat": | ||
| 105 | ok, err := cfg.Runs.Heartbeat(req.Run, owner, req.Log, cfg.Lease, time.Now().UTC()) | ||
| 106 | if err != nil { | ||
| 107 | _ = enc.Encode(runnerReply{Op: "heartbeat", Error: err.Error()}) | ||
| 108 | continue | ||
| 109 | } | ||
| 110 | _ = enc.Encode(runnerReply{Op: "heartbeat", OK: ok}) | ||
| 111 | |||
| 112 | case "finish": | ||
| 113 | ok, err := cfg.Runs.Finish(req.Run, owner, req.Exit, req.Log, time.Now().UTC()) | ||
| 114 | if err != nil { | ||
| 115 | _ = enc.Encode(runnerReply{Op: "finish", Error: err.Error()}) | ||
| 116 | continue | ||
| 117 | } | ||
| 118 | cfg.Log.Info("run finished", "user", user, "run", req.Run, "exit", req.Exit, "recorded", ok) | ||
| 119 | _ = enc.Encode(runnerReply{Op: "finish", OK: ok}) | ||
| 120 | |||
| 121 | case "ping": | ||
| 122 | _ = enc.Encode(runnerReply{Op: "ping", OK: true}) | ||
| 123 | |||
| 124 | default: | ||
| 125 | _ = enc.Encode(runnerReply{Op: req.Op, Error: "unknown op"}) | ||
| 126 | } | ||
| 127 | } | ||
| 128 | } | ||
| 129 | |||
| 130 | // runnableRepos lists the repositories this account may execute jobs for. | ||
| 131 | func (s *Server) runnableRepos(accountID string) ([]string, error) { | ||
| 132 | repos, err := s.cfg.Repos.List() | ||
| 133 | if err != nil { | ||
| 134 | return nil, err | ||
| 135 | } | ||
| 136 | var out []string | ||
| 137 | for _, m := range repos { | ||
| 138 | a, err := m.ACL() | ||
| 139 | if err != nil { | ||
| 140 | continue // a repository without readable rules grants nothing | ||
| 141 | } | ||
| 142 | if a.Can(accountID, acl.Admin) { | ||
| 143 | out = append(out, m.Name) | ||
| 144 | } | ||
| 145 | } | ||
| 146 | return out, nil | ||
| 147 | } | ||