vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4: the queue is reconciled from the refs, not only from the spool
a945ae5581394065c78c3c1d927e1d558ef56fdc · Keenan Nemetz · 17d ago · parent 592710c2
Phase 4: the queue is reconciled from the refs, not only from the spool A push reached CI through exactly one statement: a spool file written by post-receive. The hook runs after the refs move and is allowed to fail that write without failing the push, so a crash or a full disk between the two left a live commit with no build and nothing anywhere that would ever notice. Deleting runs.db had the same shape: queued jobs vanished, while the package header claimed only in-flight work was lost. Both are the same question, so they now have the same answer: a ref tip is the durable statement that a commit is live. ref_cursors records the tip each ref had when its work was queued; every 30 seconds — and once at startup, so a wiped queue has no blind spot — reconcileRefs compares the refs against those cursors and queues whatever the queue has never seen. The spool stays exactly what it should be: the sub-second path, not the promise. Costs, stated rather than discovered later: a tip no cursor has recorded is work owed, so an instance upgrading onto this (or one whose runs.db was deleted) queues its current tips once. The unique index on (repo, ref, sha, workflow, job) stops a tip that already ran from running twice once its row is restored from the records on the metadata branch by `vole index rebuild`. Measured by TestReconcileQueuesATipTheQueueHasNeverSeen (a tip is queued once, a second pass queues nothing), TestReconcileRebuildsAQueueThatWasDeleted (runs.db and its WAL deleted, the live tip is queued again), TestReconcileIgnoresTheMetadataBranch and TestReconcileDropsTheCursorOfADeletedBranch.
internal/server/reconcile_test.go +202
| @@ -0,0 +1,202 @@ | |||
| 1 | // White-box tests for the reconciliation that makes the queue follow the refs. | ||
| 2 | package server | ||
| 3 | |||
| 4 | import ( | ||
| 5 | "io" | ||
| 6 | "log/slog" | ||
| 7 | "os" | ||
| 8 | "path/filepath" | ||
| 9 | "testing" | ||
| 10 | |||
| 11 | "github.com/rodent-software/vole/internal/access" | ||
| 12 | "github.com/rodent-software/vole/internal/gitx" | ||
| 13 | "github.com/rodent-software/vole/internal/identity" | ||
| 14 | "github.com/rodent-software/vole/internal/repo" | ||
| 15 | "github.com/rodent-software/vole/internal/runs" | ||
| 16 | ) | ||
| 17 | |||
| 18 | // reconcileHarness is the smallest server that can reconcile: the refs and the | ||
| 19 | // queue. Everything else the Server holds is for serving, not for deciding what work | ||
| 20 | // is owed. | ||
| 21 | type reconcileHarness struct { | ||
| 22 | s *Server | ||
| 23 | repo *repo.Meta | ||
| 24 | runs *runs.Store | ||
| 25 | dir string | ||
| 26 | } | ||
| 27 | |||
| 28 | func newReconcileHarness(t *testing.T) *reconcileHarness { | ||
| 29 | t.Helper() | ||
| 30 | dir := t.TempDir() | ||
| 31 | if err := identity.Init(filepath.Join(dir, "users.db")); err != nil { | ||
| 32 | t.Fatal(err) | ||
| 33 | } | ||
| 34 | acls, err := access.Open(filepath.Join(dir, "users.db")) | ||
| 35 | if err != nil { | ||
| 36 | t.Fatal(err) | ||
| 37 | } | ||
| 38 | t.Cleanup(func() { acls.Close() }) | ||
| 39 | |||
| 40 | repos := repo.New(filepath.Join(dir, "repos"), "vole-under-test", acls) | ||
| 41 | if err := os.MkdirAll(repos.Root, 0o755); err != nil { | ||
| 42 | t.Fatal(err) | ||
| 43 | } | ||
| 44 | m, err := repos.Create("demo", "owner-id", "test repo") | ||
| 45 | if err != nil { | ||
| 46 | t.Fatal(err) | ||
| 47 | } | ||
| 48 | st, err := runs.Open(filepath.Join(dir, "runs.db")) | ||
| 49 | if err != nil { | ||
| 50 | t.Fatal(err) | ||
| 51 | } | ||
| 52 | t.Cleanup(func() { st.Close() }) | ||
| 53 | |||
| 54 | log := slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError})) | ||
| 55 | return &reconcileHarness{ | ||
| 56 | s: &Server{Repos: repos, Runs: st, Log: log}, | ||
| 57 | repo: m, | ||
| 58 | runs: st, | ||
| 59 | dir: dir, | ||
| 60 | } | ||
| 61 | } | ||
| 62 | |||
| 63 | // commit pushes a tree containing a workflow to a ref, the way a push would. | ||
| 64 | func (h *reconcileHarness) commit(t *testing.T, ref, workflow string) string { | ||
| 65 | t.Helper() | ||
| 66 | g := h.repo.Git() | ||
| 67 | blob, err := g.HashObject([]byte(workflow)) | ||
| 68 | if err != nil { | ||
| 69 | t.Fatal(err) | ||
| 70 | } | ||
| 71 | file, err := g.MkTree([]gitx.Entry{{Mode: "100644", Type: "blob", SHA: blob, Name: "ci.yaml"}}) | ||
| 72 | if err != nil { | ||
| 73 | t.Fatal(err) | ||
| 74 | } | ||
| 75 | dir, err := g.MkTree([]gitx.Entry{{Mode: "040000", Type: "tree", SHA: file, Name: "workflows"}}) | ||
| 76 | if err != nil { | ||
| 77 | t.Fatal(err) | ||
| 78 | } | ||
| 79 | readme, err := g.HashObject([]byte("demo\n")) | ||
| 80 | if err != nil { | ||
| 81 | t.Fatal(err) | ||
| 82 | } | ||
| 83 | root, err := g.MkTree([]gitx.Entry{ | ||
| 84 | {Mode: "040000", Type: "tree", SHA: dir, Name: ".vole"}, | ||
| 85 | {Mode: "100644", Type: "blob", SHA: readme, Name: "README.md"}, | ||
| 86 | }) | ||
| 87 | if err != nil { | ||
| 88 | t.Fatal(err) | ||
| 89 | } | ||
| 90 | commit, err := g.CommitTree(root, "", "work") | ||
| 91 | if err != nil { | ||
| 92 | t.Fatal(err) | ||
| 93 | } | ||
| 94 | if err := g.UpdateRef(ref, commit); err != nil { | ||
| 95 | t.Fatal(err) | ||
| 96 | } | ||
| 97 | return commit | ||
| 98 | } | ||
| 99 | |||
| 100 | func (h *reconcileHarness) queued(t *testing.T) int { | ||
| 101 | t.Helper() | ||
| 102 | list, err := h.runs.Recent("demo", 50) | ||
| 103 | if err != nil { | ||
| 104 | t.Fatal(err) | ||
| 105 | } | ||
| 106 | return len(list) | ||
| 107 | } | ||
| 108 | |||
| 109 | const testWorkflow = `version: 1 | ||
| 110 | jobs: | ||
| 111 | build: | ||
| 112 | steps: | ||
| 113 | - name: test | ||
| 114 | run: printf 'ok\n' | ||
| 115 | ` | ||
| 116 | |||
| 117 | // A ref tip no cursor has recorded is work owed — that is the whole reason the queue | ||
| 118 | // is reconciled from git rather than from the spool file a crash may have eaten. | ||
| 119 | func TestReconcileQueuesATipTheQueueHasNeverSeen(t *testing.T) { | ||
| 120 | h := newReconcileHarness(t) | ||
| 121 | h.commit(t, "refs/heads/main", testWorkflow) | ||
| 122 | |||
| 123 | h.s.reconcileRefs() | ||
| 124 | if n := h.queued(t); n != 1 { | ||
| 125 | t.Fatalf("queued %d run(s) after reconciling, want 1", n) | ||
| 126 | } | ||
| 127 | |||
| 128 | // The cursor is the record that the tip was handled: a second pass must not queue | ||
| 129 | // the same work again. | ||
| 130 | h.s.reconcileRefs() | ||
| 131 | if n := h.queued(t); n != 1 { | ||
| 132 | t.Fatalf("queued %d run(s) after a second pass, want the tip to be handled once", n) | ||
| 133 | } | ||
| 134 | } | ||
| 135 | |||
| 136 | // The promise this buys: deleting the queue loses at most the job that was running. | ||
| 137 | // The refs still say what is live, so the work comes back. | ||
| 138 | func TestReconcileRebuildsAQueueThatWasDeleted(t *testing.T) { | ||
| 139 | h := newReconcileHarness(t) | ||
| 140 | h.commit(t, "refs/heads/main", testWorkflow) | ||
| 141 | h.s.reconcileRefs() | ||
| 142 | |||
| 143 | // What `rm data/runs.db` does, including the WAL files SQLite leaves beside it. | ||
| 144 | if err := h.runs.Close(); err != nil { | ||
| 145 | t.Fatal(err) | ||
| 146 | } | ||
| 147 | for _, name := range []string{"runs.db", "runs.db-wal", "runs.db-shm"} { | ||
| 148 | if err := os.Remove(filepath.Join(h.dir, name)); err != nil && !os.IsNotExist(err) { | ||
| 149 | t.Fatal(err) | ||
| 150 | } | ||
| 151 | } | ||
| 152 | st, err := runs.Open(filepath.Join(h.dir, "runs.db")) | ||
| 153 | if err != nil { | ||
| 154 | t.Fatal(err) | ||
| 155 | } | ||
| 156 | h.s.Runs = st | ||
| 157 | h.runs = st | ||
| 158 | if n := h.queued(t); n != 0 { | ||
| 159 | t.Fatalf("a fresh queue already has %d run(s)", n) | ||
| 160 | } | ||
| 161 | |||
| 162 | h.s.reconcileRefs() | ||
| 163 | if n := h.queued(t); n != 1 { | ||
| 164 | t.Fatalf("queued %d run(s) after the queue was deleted, want the live tip re-queued", n) | ||
| 165 | } | ||
| 166 | } | ||
| 167 | |||
| 168 | // The metadata branch is full of entities and run records; a build must never be | ||
| 169 | // queued for it, however its tip moves. | ||
| 170 | func TestReconcileIgnoresTheMetadataBranch(t *testing.T) { | ||
| 171 | h := newReconcileHarness(t) | ||
| 172 | commit := h.commit(t, "refs/heads/main", testWorkflow) | ||
| 173 | if err := h.repo.Git().UpdateRef("refs/heads/vole", commit); err != nil { | ||
| 174 | t.Fatal(err) | ||
| 175 | } | ||
| 176 | |||
| 177 | h.s.reconcileRefs() | ||
| 178 | if n := h.queued(t); n != 1 { | ||
| 179 | t.Fatalf("queued %d run(s), want only the branch run and nothing for refs/heads/vole", n) | ||
| 180 | } | ||
| 181 | } | ||
| 182 | |||
| 183 | // A deleted branch drops its cursor, so the branch coming back at a new tip is work | ||
| 184 | // owed rather than work suppressed by a note about a branch that no longer exists. | ||
| 185 | func TestReconcileDropsTheCursorOfADeletedBranch(t *testing.T) { | ||
| 186 | h := newReconcileHarness(t) | ||
| 187 | h.commit(t, "refs/heads/main", testWorkflow) | ||
| 188 | h.s.reconcileRefs() | ||
| 189 | |||
| 190 | if err := h.repo.Git().DeleteRef("refs/heads/main"); err != nil { | ||
| 191 | t.Fatal(err) | ||
| 192 | } | ||
| 193 | h.s.reconcileRefs() | ||
| 194 | |||
| 195 | cursors, err := h.runs.Cursors("demo") | ||
| 196 | if err != nil { | ||
| 197 | t.Fatal(err) | ||
| 198 | } | ||
| 199 | if _, ok := cursors["refs/heads/main"]; ok { | ||
| 200 | t.Fatalf("the cursor for a deleted branch survived: %v", cursors) | ||
| 201 | } | ||
| 202 | } | ||