vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4 slice 2: the queue and the runner
b44cbfaf873fda01a458b527e5684b2386266117 · Keenan Nemetz · 17d ago · parent 8fe238d4
Phase 4 slice 2: the queue and the runner A push now runs a pipeline. post-receive spools one file per accepted ref update — the hook must never write to a database or fail a commit — and the server ingests those into data/runs.db, pinning each run to the commit and to the workflow blob it was queued from. The unique index on (repo, ref, sha, workflow, job) is what makes ingestion idempotent, so a spool file read twice after a crash does not run twice. vole runner connects over SSH with the key it would push with: no token, no stored secret, and the same identity the rest of the system uses. It may only run jobs for repositories where its account is an admin, because running a job executes that repository's code — a bigger power than writing to it. Delivery is at-least-once by construction: - A lease hands out a job for a fixed window; the runner heartbeats while it works. - An expired lease means the runner died, and the janitor returns the job to the queue for somebody else. - A runner whose lease is gone stops rather than reporting a result nobody will record, and a job that kills five runners in a row is failed instead of cycling. Three bugs found while building it, each worth the test that now covers it: - Background loops lived in Run(), so the tests — which serve the SSH listener themselves — never had ingestion or lease expiry. They start in New() now. - post-receive derived the repository name from GIT_DIR, which git sets to "." for hooks, so every event was queued for a repository called ".". The same trap had already appeared in a pre-receive message; both now resolve the absolute path. - Leases ordered only by creation time, so two pushes in the same second could be handed out in any order; the id breaks the tie, and ULIDs sort by time. Verified end to end on a live instance: a push queues a job, a runner leases it, clones the commit, prints "hi", and the run closes success with the log attached. The integration tests cover the two exit criteria that matter — a push runs a pipeline, and killing a runner mid-job requeues the work onto a second runner (attempts=2, success) — plus the property that editing a workflow after a push does not change what the already-queued run executes. Not built yet: statuses on change requests and required-check gating (slice 3), and signed run records (Phase 7). Runs live in data/runs.db and are visible there, not on the web.
internal/server/server.go +25−7
| @@ -16,6 +16,7 @@ import ( | |||
| 16 | 16 | "github.com/rodent-software/vole/internal/identity" | |
| 17 | 17 | "github.com/rodent-software/vole/internal/index" | |
| 18 | 18 | "github.com/rodent-software/vole/internal/repo" | |
| 19 | "github.com/rodent-software/vole/internal/runs" | ||
| 19 | 20 | "github.com/rodent-software/vole/internal/signing" | |
| 20 | 21 | "github.com/rodent-software/vole/internal/sshserver" | |
| 21 | 22 | "github.com/rodent-software/vole/internal/transport" | |
| @@ -23,9 +24,12 @@ import ( | |||
| 23 | 24 | ) | |
| 24 | 25 | ||
| 25 | 26 | type Server struct { | |
| 27 | ctx context.Context | ||
| 28 | cancel context.CancelFunc | ||
| 26 | 29 | Cfg *config.Config | |
| 27 | 30 | Repos *repo.Store | |
| 28 | 31 | Identity *identity.Store | |
| 32 | Runs *runs.Store | ||
| 29 | 33 | Index *index.Store | |
| 30 | 34 | Proxy *transport.Proxy | |
| 31 | 35 | SSH *sshserver.Server | |
| @@ -51,6 +55,10 @@ func New(cfg *config.Config, binPath string, log *slog.Logger) (*Server, error) | |||
| 51 | 55 | if err != nil { | |
| 52 | 56 | return nil, fmt.Errorf("open index: %w", err) | |
| 53 | 57 | } | |
| 58 | runStore, err := runs.Open(cfg.RunsDB()) | ||
| 59 | if err != nil { | ||
| 60 | return nil, fmt.Errorf("open run queue: %w", err) | ||
| 61 | } | ||
| 54 | 62 | rs := repo.New(cfg.ReposDir(), binPath) | |
| 55 | 63 | rs.MaxInputBytes = cfg.ReceiveMaxBytes | |
| 56 | 64 | if n, err := rs.InstallAllHooks(); err != nil { | |
| @@ -75,6 +83,9 @@ func New(cfg *config.Config, binPath string, log *slog.Logger) (*Server, error) | |||
| 75 | 83 | HostKeyPath: cfg.SSHHostKey(), | |
| 76 | 84 | MaxBlobBytes: cfg.MaxBlobBytes, | |
| 77 | 85 | AuditPath: cfg.AuditLog(), | |
| 86 | SpoolDir: cfg.SpoolDir(), | ||
| 87 | Runs: runStore, | ||
| 88 | Lease: cfg.Lease(), | ||
| 78 | 89 | }) | |
| 79 | 90 | if err != nil { | |
| 80 | 91 | return nil, fmt.Errorf("ssh: %w", err) | |
| @@ -86,10 +97,17 @@ func New(cfg *config.Config, binPath string, log *slog.Logger) (*Server, error) | |||
| 86 | 97 | wp.Logger = func(format string, args ...any) { log.Warn(fmt.Sprintf(format, args...)) } | |
| 87 | 98 | ||
| 88 | 99 | s := &Server{ | |
| 89 | Cfg: cfg, Repos: rs, Identity: ident, Index: ix, | ||
| 100 | Cfg: cfg, Repos: rs, Identity: ident, Runs: runStore, Index: ix, | ||
| 90 | 101 | Proxy: proxy, SSH: ssh, Key: key, Web: wp, Log: log, | |
| 91 | 102 | } | |
| 92 | 103 | s.mux = s.routes() | |
| 104 | // Background loops start here rather than in Run: a caller that serves the | ||
| 105 | // listeners itself (the tests, an embedder) must still get roster reloads, | ||
| 106 | // queue ingestion and lease expiry. | ||
| 107 | s.ctx, s.cancel = context.WithCancel(context.Background()) | ||
| 108 | go s.watchRoster(s.ctx) | ||
| 109 | go s.watchSpool(s.ctx) | ||
| 110 | go s.watchLeases(s.ctx) | ||
| 93 | 111 | return s, nil | |
| 94 | 112 | } | |
| 95 | 113 | ||
| @@ -152,12 +170,6 @@ func (s *Server) Run(ctx context.Context) error { | |||
| 152 | 170 | s.Log.Info("listening", "ssh", sshLn.Addr().String(), "http", httpLn.Addr().String(), | |
| 153 | 171 | "data", s.Cfg.DataDir, "users", s.Identity.Snapshot().Count) | |
| 154 | 172 | ||
| 155 | // The roster is a directory of files, so an operator may edit it directly. | ||
| 156 | // Polling makes "drop a key in and it works" true without a restart. A | ||
| 157 | // failed reload keeps the previous snapshot and logs loudly rather than | ||
| 158 | // locking everyone out over a half-finished edit. | ||
| 159 | go s.watchRoster(ctx) | ||
| 160 | |||
| 161 | 173 | select { | |
| 162 | 174 | case <-ctx.Done(): | |
| 163 | 175 | sh, cancel := context.WithTimeout(context.Background(), 20*time.Second) | |
| @@ -193,6 +205,12 @@ func (s *Server) watchRoster(ctx context.Context) { | |||
| 193 | 205 | } | |
| 194 | 206 | ||
| 195 | 207 | func (s *Server) Close() error { | |
| 208 | if s.cancel != nil { | ||
| 209 | s.cancel() | ||
| 210 | } | ||
| 211 | if s.Runs != nil { | ||
| 212 | _ = s.Runs.Close() | ||
| 213 | } | ||
| 196 | 214 | if s.Index != nil { | |
| 197 | 215 | return s.Index.Close() | |
| 198 | 216 | } | |