vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4 slice 2: the queue and the runner
b44cbfaf873fda01a458b527e5684b2386266117 · Keenan Nemetz · 18d ago · parent 8fe238d4
Phase 4 slice 2: the queue and the runner A push now runs a pipeline. post-receive spools one file per accepted ref update — the hook must never write to a database or fail a commit — and the server ingests those into data/runs.db, pinning each run to the commit and to the workflow blob it was queued from. The unique index on (repo, ref, sha, workflow, job) is what makes ingestion idempotent, so a spool file read twice after a crash does not run twice. vole runner connects over SSH with the key it would push with: no token, no stored secret, and the same identity the rest of the system uses. It may only run jobs for repositories where its account is an admin, because running a job executes that repository's code — a bigger power than writing to it. Delivery is at-least-once by construction: - A lease hands out a job for a fixed window; the runner heartbeats while it works. - An expired lease means the runner died, and the janitor returns the job to the queue for somebody else. - A runner whose lease is gone stops rather than reporting a result nobody will record, and a job that kills five runners in a row is failed instead of cycling. Three bugs found while building it, each worth the test that now covers it: - Background loops lived in Run(), so the tests — which serve the SSH listener themselves — never had ingestion or lease expiry. They start in New() now. - post-receive derived the repository name from GIT_DIR, which git sets to "." for hooks, so every event was queued for a repository called ".". The same trap had already appeared in a pre-receive message; both now resolve the absolute path. - Leases ordered only by creation time, so two pushes in the same second could be handed out in any order; the id breaks the tie, and ULIDs sort by time. Verified end to end on a live instance: a push queues a job, a runner leases it, clones the commit, prints "hi", and the run closes success with the log attached. The integration tests cover the two exit criteria that matter — a push runs a pipeline, and killing a runner mid-job requeues the work onto a second runner (attempts=2, success) — plus the property that editing a workflow after a push does not change what the already-queued run executes. Not built yet: statuses on change requests and required-check gating (slice 3), and signed run records (Phase 7). Runs live in data/runs.db and are visible there, not on the web.
internal/hook/hook.go +45−3
| @@ -17,6 +17,7 @@ import ( | |||
| 17 | 17 | "github.com/rodent-software/vole/internal/issues" | |
| 18 | 18 | "github.com/rodent-software/vole/internal/repo" | |
| 19 | 19 | "github.com/rodent-software/vole/internal/requests" | |
| 20 | "github.com/rodent-software/vole/internal/runs" | ||
| 20 | 21 | "github.com/rodent-software/vole/internal/workflow" | |
| 21 | 22 | ) | |
| 22 | 23 | ||
| @@ -30,6 +31,8 @@ type Config struct { | |||
| 30 | 31 | // SystemRepo, when set, is the roster repository, used only to render account | |
| 31 | 32 | // ids as names in messages. | |
| 32 | 33 | SystemRepo string | |
| 34 | // SpoolDir receives one file per accepted ref update, for the CI queue. | ||
| 35 | SpoolDir string | ||
| 33 | 36 | // AuditPath receives lines about privileged metadata operations. Empty means | |
| 34 | 37 | // no audit sink, which is only acceptable for tests. | |
| 35 | 38 | AuditPath string | |
| @@ -49,9 +52,7 @@ func PreReceive(cfg Config) int { | |||
| 49 | 52 | if cfg.In == nil { | |
| 50 | 53 | cfg.In = strings.NewReader("") | |
| 51 | 54 | } | |
| 52 | // Clean first: receive-pack passes a path with a trailing separator, and | ||
| 53 | // filepath.Base("/x/y.git/") is ".". | ||
| 54 | m := &repo.Meta{Name: filepath.Base(filepath.Clean(cfg.RepoDir)), Dir: cfg.RepoDir} | ||
| 55 | m := &repo.Meta{Name: repoNameFromDir(cfg.RepoDir), Dir: cfg.RepoDir} | ||
| 55 | 56 | a, err := m.ACL() | |
| 56 | 57 | if err != nil { | |
| 57 | 58 | reject(cfg.Out, "cannot read repository access rules: %v", err) | |
| @@ -354,6 +355,18 @@ func touchesWorkflows(g *gitx.Repo, c change) (bool, error) { | |||
| 354 | 355 | return false, nil | |
| 355 | 356 | } | |
| 356 | 357 | ||
| 358 | // repoNameFromDir derives the repository name from the directory git handed the | ||
| 359 | // hook. It cannot use the path as given: hooks run with GIT_DIR="." (relative to | ||
| 360 | // the repository), and receive-pack may pass a trailing separator, so a bare | ||
| 361 | // filepath.Base returns "." for both. | ||
| 362 | func repoNameFromDir(dir string) string { | ||
| 363 | abs, err := filepath.Abs(dir) | ||
| 364 | if err != nil { | ||
| 365 | abs = dir | ||
| 366 | } | ||
| 367 | return strings.TrimSuffix(filepath.Base(abs), ".git") | ||
| 368 | } | ||
| 369 | |||
| 357 | 370 | // openNames returns a function rendering account ids as names, falling back to | |
| 358 | 371 | // the id when the roster is unavailable. Display only: nothing is authorized by | |
| 359 | 372 | // the result. | |
| @@ -391,3 +404,32 @@ func short(sha string) string { | |||
| 391 | 404 | func reject(w io.Writer, format string, args ...any) { | |
| 392 | 405 | fmt.Fprintf(w, "vole: "+format+"\n", args...) | |
| 393 | 406 | } | |
| 407 | |||
| 408 | // PostReceive hands accepted ref updates to the CI queue by writing one file per | ||
| 409 | // update into the spool directory. It runs after the refs have moved, so what it | ||
| 410 | // records is what actually landed, and it never fails a push: a spool write that | ||
| 411 | // does not happen costs a CI run, not a commit. | ||
| 412 | func PostReceive(cfg Config) int { | ||
| 413 | if cfg.SpoolDir == "" { | ||
| 414 | return 0 | ||
| 415 | } | ||
| 416 | sc := bufio.NewScanner(cfg.In) | ||
| 417 | for sc.Scan() { | ||
| 418 | fields := strings.Fields(sc.Text()) | ||
| 419 | if len(fields) != 3 { | ||
| 420 | continue | ||
| 421 | } | ||
| 422 | old, next, ref := fields[0], fields[1], fields[2] | ||
| 423 | event := runs.Event{ | ||
| 424 | Repo: repoNameFromDir(cfg.RepoDir), | ||
| 425 | Ref: ref, | ||
| 426 | Old: old, | ||
| 427 | New: next, | ||
| 428 | At: time.Now().UTC().Format(time.RFC3339Nano), | ||
| 429 | } | ||
| 430 | if err := runs.WriteSpool(cfg.SpoolDir, event); err != nil { | ||
| 431 | fmt.Fprintf(cfg.Out, "vole: could not queue CI for %s: %v\n", ref, err) | ||
| 432 | } | ||
| 433 | } | ||
| 434 | return 0 | ||
| 435 | } | ||