vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4: the queue is reconciled from the refs, not only from the spool
a945ae5581394065c78c3c1d927e1d558ef56fdc · Keenan Nemetz · 17d ago · parent 592710c2
Phase 4: the queue is reconciled from the refs, not only from the spool A push reached CI through exactly one statement: a spool file written by post-receive. The hook runs after the refs move and is allowed to fail that write without failing the push, so a crash or a full disk between the two left a live commit with no build and nothing anywhere that would ever notice. Deleting runs.db had the same shape: queued jobs vanished, while the package header claimed only in-flight work was lost. Both are the same question, so they now have the same answer: a ref tip is the durable statement that a commit is live. ref_cursors records the tip each ref had when its work was queued; every 30 seconds — and once at startup, so a wiped queue has no blind spot — reconcileRefs compares the refs against those cursors and queues whatever the queue has never seen. The spool stays exactly what it should be: the sub-second path, not the promise. Costs, stated rather than discovered later: a tip no cursor has recorded is work owed, so an instance upgrading onto this (or one whose runs.db was deleted) queues its current tips once. The unique index on (repo, ref, sha, workflow, job) stops a tip that already ran from running twice once its row is restored from the records on the metadata branch by `vole index rebuild`. Measured by TestReconcileQueuesATipTheQueueHasNeverSeen (a tip is queued once, a second pass queues nothing), TestReconcileRebuildsAQueueThatWasDeleted (runs.db and its WAL deleted, the live tip is queued again), TestReconcileIgnoresTheMetadataBranch and TestReconcileDropsTheCursorOfADeletedBranch.
internal/server/ci.go +92
| @@ -7,6 +7,7 @@ import ( | |||
| 7 | 7 | "time" | |
| 8 | 8 | ||
| 9 | 9 | "github.com/rodent-software/vole/internal/id" | |
| 10 | "github.com/rodent-software/vole/internal/issues" | ||
| 10 | 11 | "github.com/rodent-software/vole/internal/runs" | |
| 11 | 12 | "github.com/rodent-software/vole/internal/workflow" | |
| 12 | 13 | ) | |
| @@ -134,6 +135,97 @@ func (s *Server) enqueue(ev runs.Event) { | |||
| 134 | 135 | } | |
| 135 | 136 | } | |
| 136 | 137 | ||
| 138 | // reconcileSeconds is how often the queue is checked against the refs. The spool | ||
| 139 | // delivers a push in well under a second; this is the safety net that makes the | ||
| 140 | // promise eventually true when the spool file was never written. | ||
| 141 | const reconcileSeconds = 30 | ||
| 142 | |||
| 143 | // watchRefs reconciles the queue from the refs once at startup and then on a timer. | ||
| 144 | // The startup pass is why a queue that was deleted comes back without a 30-second | ||
| 145 | // blind spot. | ||
| 146 | func (s *Server) watchRefs(ctx context.Context) { | ||
| 147 | s.reconcileRefs() | ||
| 148 | t := time.NewTicker(reconcileSeconds * time.Second) | ||
| 149 | defer t.Stop() | ||
| 150 | for { | ||
| 151 | select { | ||
| 152 | case <-ctx.Done(): | ||
| 153 | return | ||
| 154 | case <-t.C: | ||
| 155 | s.reconcileRefs() | ||
| 156 | } | ||
| 157 | } | ||
| 158 | } | ||
| 159 | |||
| 160 | // reconcileRefs queues work for every ref whose tip the queue has never seen. | ||
| 161 | // | ||
| 162 | // It is what makes the spool a latency optimisation instead of a promise. The hook | ||
| 163 | // runs after the refs move and may fail to write its spool file without failing the | ||
| 164 | // push (a crash between the two, a full disk), and `runs.db` can be deleted while | ||
| 165 | // jobs are queued. Both are answered the same way: a ref tip is the durable | ||
| 166 | // statement that a commit is live, so a tip no cursor has recorded is work owed. | ||
| 167 | // | ||
| 168 | // The trade is explicit: a tip nothing has ever recorded is queued, which includes | ||
| 169 | // every branch of an instance whose queue was emptied. The unique index stops a tip | ||
| 170 | // that already ran from running again once its row is restored from the records on | ||
| 171 | // the metadata branch, and `vole index rebuild` restores exactly that. | ||
| 172 | func (s *Server) reconcileRefs() { | ||
| 173 | repos, err := s.Repos.List() | ||
| 174 | if err != nil { | ||
| 175 | s.Log.Warn("ci: cannot list repositories to reconcile", "err", err) | ||
| 176 | return | ||
| 177 | } | ||
| 178 | now := time.Now().UTC() | ||
| 179 | reconciled := 0 | ||
| 180 | for _, m := range repos { | ||
| 181 | cursors, err := s.Runs.Cursors(m.Name) | ||
| 182 | if err != nil { | ||
| 183 | s.Log.Warn("ci: cannot read ref cursors", "repo", m.Name, "err", err) | ||
| 184 | continue | ||
| 185 | } | ||
| 186 | refs, err := m.Git().ListRefs("refs/") | ||
| 187 | if err != nil { | ||
| 188 | s.Log.Warn("ci: cannot list refs", "repo", m.Name, "err", err) | ||
| 189 | continue | ||
| 190 | } | ||
| 191 | for ref, tip := range refs { | ||
| 192 | // The metadata branch carries human-authored entities and run records; the | ||
| 193 | // runner's own push of a record must not queue a build of the tool's data. | ||
| 194 | if ref == issues.Ref { | ||
| 195 | continue | ||
| 196 | } | ||
| 197 | if !strings.HasPrefix(ref, "refs/heads/") && !strings.HasPrefix(ref, "refs/tags/") { | ||
| 198 | continue | ||
| 199 | } | ||
| 200 | if held, ok := cursors[ref]; ok && held == tip { | ||
| 201 | continue | ||
| 202 | } | ||
| 203 | // enqueue re-checks the tip against the ref, so an event that raced with | ||
| 204 | // another push is dropped there rather than trusted here. | ||
| 205 | s.enqueue(runs.Event{ | ||
| 206 | Repo: m.Name, Ref: ref, Old: cursors[ref], New: tip, | ||
| 207 | At: now.Format(time.RFC3339Nano), | ||
| 208 | }) | ||
| 209 | // The cursor moves after the work is queued: it is a note that a tip was | ||
| 210 | // handled, not a claim about work that is about to be attempted. | ||
| 211 | if err := s.Runs.SetCursor(m.Name, ref, tip, now); err != nil { | ||
| 212 | s.Log.Warn("ci: cannot record a ref cursor", "repo", m.Name, "ref", ref, "err", err) | ||
| 213 | } | ||
| 214 | reconciled++ | ||
| 215 | } | ||
| 216 | for ref := range cursors { | ||
| 217 | if _, ok := refs[ref]; !ok { | ||
| 218 | if err := s.Runs.DropCursor(m.Name, ref); err != nil { | ||
| 219 | s.Log.Warn("ci: cannot drop a stale cursor", "repo", m.Name, "ref", ref, "err", err) | ||
| 220 | } | ||
| 221 | } | ||
| 222 | } | ||
| 223 | } | ||
| 224 | if reconciled > 0 { | ||
| 225 | s.Log.Info("ci: reconciled refs with the queue", "refs", reconciled) | ||
| 226 | } | ||
| 227 | } | ||
| 228 | |||
| 137 | 229 | // releasePaths picks the patterns that also publish to the tag's release. | |
| 138 | 230 | func releasePaths(job workflow.Job) []string { | |
| 139 | 231 | var out []string | |