Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Materialise the activity feed in an events table

bca9ec24cbbb938f2d349d5f06e2c1f4fca06fcb · Keenan Nemetz · 10d ago · parent 73a732b1 

Materialise the activity feed in an events table

The previous commit read the feed as a UNION over the source tables. It
stopped materialising events in Go, but SQLite still scanned and sorted
the union to find a page, so a page cost the history.

This replaces it with the design the feed actually wants: an `events`
table with one row per event — issue and change opened, comment,
approval, state change, release — carrying the time, actor, title and
assignees the feed renders. Rows are indexed by (repository, time, id),
so a repository page is an index range read; the instance page walks the
global time index and filters by repository, and neither sorts history.
An unfiltered page total is a stored per-repository count.

Maintenance is part of the refresh that already exists:

- an entity's events are replaced when the entity is reloaded, which is
  how a title or assignee edit reaches old events;
- state changes are rebuilt into events for the entities the commits
  name, which also catches a close-and-reopen that leaves the tree where
  it started and an endpoint diff cannot see;
- releases are refreshed from the tags after every push (post-receive)
  and after a release written in the browser, because a tag is a ref and
  no metadata commit announces it;
- the stored count moves with the rows a refresh adds or removes.

Measured with 100,000 events across 50 repositories: a repository page
takes 0.3-0.5 ms (page 1 or page 20), an instance page 0.5-0.9 ms, and an
unfiltered total 0.1 ms. A filtered total is the one query that scans,
at 12 ms for 100,000 events.

On the load harness, instance activity rises from 222 to 482 req/s and
p50 from 18 ms to 8 ms; activity from 577 to 762 req/s. The repository
feed no longer reads tags per request, and nothing reads the metadata
branch to render a page.

forge/index/events_test.go +306

@@ -0,0 +1,306 @@
1 package index
2
3 // The events table is the feed's read model. These tests hold what it has to say
4 // and, just as important, what it must stop saying: an edit changes the title on
5 // old events, a deletion removes them, and a state change that leaves the tree
6 // where it started still has to appear.
7
8 import (
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/rodent-software/vole/core/git"
14 "github.com/rodent-software/vole/metadata"
15 )
16
17 func eventSummaries(t *testing.T, s *Store, repoID string) []string {
18 t.Helper()
19 rows, err := s.Events([]string{repoID}, "", "", "", 1000, 0)
20 if err != nil {
21 t.Fatal(err)
22 }
23 out := make([]string, 0, len(rows))
24 for _, r := range rows {
25 out = append(out, r.Kind+"/"+r.Type+"/"+r.Title)
26 }
27 return out
28 }
29
30 func TestEventsCoverTheFeed(t *testing.T) {
31 t.Parallel()
32 s, m, g := incrementalRepo(t)
33 now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
34 iss := createIndexedIssue(t, g, "An issue", now, nil, []string{"keenan"})
35 addIssueComment(t, g, iss.ID, "a note", now.Add(time.Minute))
36 req := createIndexedRequest(t, g, "A change", now.Add(2*time.Minute), nil)
37 comment, err := metadata.NewRequestComment("review note", "wilbur", now.Add(3*time.Minute))
38 if err != nil {
39 t.Fatal(err)
40 }
41 if err := metadata.AddRequestComment(g, req.ID, comment, "wilbur"); err != nil {
42 t.Fatal(err)
43 }
44 if _, err := metadata.ApproveRequest(g, req.ID, "wilbur", "main", requestHead, now.Add(4*time.Minute), "wilbur"); err != nil {
45 t.Fatal(err)
46 }
47 tip, err := g.RevParse(metadata.Ref)
48 if err != nil || tip == "" {
49 t.Fatalf("resolve metadata tip: %q, %v", tip, err)
50 }
51 if err := g.WriteAnnotatedTag("v1", tip, "notes\n", "keenan", false); err != nil {
52 t.Fatal(err)
53 }
54 if err := s.Ensure(m); err != nil {
55 t.Fatal(err)
56 }
57 if err := s.RefreshTags(m); err != nil {
58 t.Fatal(err)
59 }
60
61 got := eventSummaries(t, s, m.ID)
62 for _, want := range []string{
63 "issue/open/An issue",
64 "issue/comment/An issue",
65 "change/open/A change",
66 "change/comment/A change",
67 "change/approval/A change",
68 "release/release/v1",
69 } {
70 found := false
71 for _, g := range got {
72 if g == want {
73 found = true
74 break
75 }
76 }
77 if !found {
78 t.Fatalf("events are missing %q: %v", want, got)
79 }
80 }
81 var count int
82 if err := s.db.QueryRow(`SELECT events_count FROM repos WHERE id=?`, m.ID).Scan(&count); err != nil {
83 t.Fatal(err)
84 }
85 if count != len(got) {
86 t.Fatalf("events_count = %d, want %d", count, len(got))
87 }
88 if n, err := s.EventCount([]string{m.ID}, "", "", ""); err != nil || n != len(got) {
89 t.Fatalf("EventCount = %d, %v; want %d", n, err, len(got))
90 }
91
92 // The rebuild must produce the same feed.
93 if err := s.IndexRepo(m); err != nil {
94 t.Fatal(err)
95 }
96 rebuilt := eventSummaries(t, s, m.ID)
97 if strings.Join(rebuilt, "\n") != strings.Join(got, "\n") {
98 t.Fatalf("rebuilt events differ\n got: %v\nwant: %v", rebuilt, got)
99 }
100 }
101
102 func TestEventTitlesFollowTheEntityAndDeletionRemovesItsEvents(t *testing.T) {
103 t.Parallel()
104 s, m, g := incrementalRepo(t)
105 now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
106 iss := createIndexedIssue(t, g, "Old title", now, nil, nil)
107 addIssueComment(t, g, iss.ID, "a note", now.Add(time.Minute))
108 if err := s.Ensure(m); err != nil {
109 t.Fatal(err)
110 }
111 if _, err := metadata.UpdateIssue(g, iss.ID, func(iss *metadata.Issue) error {
112 iss.Title = "New title"
113 return nil
114 }, "keenan"); err != nil {
115 t.Fatal(err)
116 }
117 if err := s.Ensure(m); err != nil {
118 t.Fatal(err)
119 }
120 for _, ev := range eventSummaries(t, s, m.ID) {
121 if !strings.HasSuffix(ev, "/New title") {
122 t.Fatalf("event still carries the old title: %v", ev)
123 }
124 }
125
126 if _, err := git.CommitFiles(g, metadata.Ref, func(files map[string][]byte) error {
127 for path := range files {
128 if strings.HasPrefix(path, "issues/"+iss.ID+"/") {
129 delete(files, path)
130 }
131 }
132 return nil
133 }, "delete the issue", "keenan"); err != nil {
134 t.Fatal(err)
135 }
136 if err := s.Ensure(m); err != nil {
137 t.Fatal(err)
138 }
139 if got := eventSummaries(t, s, m.ID); len(got) != 0 {
140 t.Fatalf("events survived the deletion: %v", got)
141 }
142 if n, err := s.EventCount([]string{m.ID}, "", "", ""); err != nil || n != 0 {
143 t.Fatalf("EventCount after deletion = %d, %v", n, err)
144 }
145 }
146
147 // TestStateChangeEventsSurviveATreeRoundTrip is the case an endpoint diff cannot
148 // see: close and reopen inside one refresh leaves the file where it started, and
149 // both events still have to be in the feed.
150 func TestStateChangeEventsSurviveATreeRoundTrip(t *testing.T) {
151 t.Parallel()
152 s, m, g := incrementalRepo(t)
153 now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
154 iss := createIndexedIssue(t, g, "Round trip", now, nil, nil)
155 if err := s.Ensure(m); err != nil {
156 t.Fatal(err)
157 }
158 for _, state := range []string{"closed", "open"} {
159 if _, err := metadata.SetIssueState(g, iss.ID, state, "keenan"); err != nil {
160 t.Fatal(err)
161 }
162 }
163 if err := s.Ensure(m); err != nil {
164 t.Fatal(err)
165 }
166 var states int
167 rows, err := s.Events([]string{m.ID}, "issue", "", "", 100, 0)
168 if err != nil {
169 t.Fatal(err)
170 }
171 for _, r := range rows {
172 if r.Type == "state" {
173 states++
174 }
175 }
176 if states != 2 {
177 t.Fatalf("state events = %d, want 2: %+v", states, rows)
178 }
179 }
180
181 func TestRefreshTagsReplacesReleaseEvents(t *testing.T) {
182 t.Parallel()
183 s, m, g := incrementalRepo(t)
184 tip, err := g.RevParse(metadata.Ref)
185 if err != nil || tip == "" {
186 t.Fatalf("resolve metadata tip: %q, %v", tip, err)
187 }
188 if err := g.WriteAnnotatedTag("v1", tip, "notes\n", "keenan", false); err != nil {
189 t.Fatal(err)
190 }
191 if err := s.RefreshTags(m); err != nil {
192 t.Fatal(err)
193 }
194 if got := eventSummaries(t, s, m.ID); len(got) != 1 || got[0] != "release/release/v1" {
195 t.Fatalf("release events = %v", got)
196 }
197 if err := g.DeleteRef("refs/tags/v1"); err != nil {
198 t.Fatal(err)
199 }
200 if err := s.RefreshTags(m); err != nil {
201 t.Fatal(err)
202 }
203 if got := eventSummaries(t, s, m.ID); len(got) != 0 {
204 t.Fatalf("release events survived the tag deletion: %v", got)
205 }
206 }
207
208 func TestEventsPaginate(t *testing.T) {
209 t.Parallel()
210 s, m, g := incrementalRepo(t)
211 now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
212 for i := 0; i < 55; i++ {
213 createIndexedIssue(t, g, "issue", now.Add(time.Duration(i)*time.Minute), nil, nil)
214 }
215 if err := s.Ensure(m); err != nil {
216 t.Fatal(err)
217 }
218 first, err := s.Events([]string{m.ID}, "", "", "", 50, 0)
219 if err != nil || len(first) != 50 {
220 t.Fatalf("first page = %d, %v", len(first), err)
221 }
222 second, err := s.Events([]string{m.ID}, "", "", "", 50, 50)
223 if err != nil || len(second) != 5 {
224 t.Fatalf("second page = %d, %v", len(second), err)
225 }
226 if n, err := s.EventCount([]string{m.ID}, "", "", ""); err != nil || n != 55 {
227 t.Fatalf("count = %d, %v", n, err)
228 }
229 }
230
231 // TestEventPageDoesNotSortHistory pins the design: one repository is an index
232 // range, and the instance feed walks the time index and filters. If a later
233 // change makes SQLite sort the events, the page cost goes back to the history.
234 func TestEventPageDoesNotSortHistory(t *testing.T) {
235 t.Parallel()
236 s, err := Open(t.TempDir() + "/index.db")
237 if err != nil {
238 t.Fatal(err)
239 }
240 defer s.Close()
241 plan := func(repoIDs []string) string {
242 q, args := eventPageQuery(repoIDs, "", "", "", 50, 100)
243 rows, err := s.db.Query("EXPLAIN QUERY PLAN "+q, args...)
244 if err != nil {
245 t.Fatal(err)
246 }
247 defer rows.Close()
248 var b strings.Builder
249 for rows.Next() {
250 var id, parent, notused int
251 var detail string
252 if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
253 t.Fatal(err)
254 }
255 b.WriteString(detail + "\n")
256 }
257 return b.String()
258 }
259 if got := plan([]string{"r1"}); !strings.Contains(got, "events_repo") || strings.Contains(got, "TEMP B-TREE") {
260 t.Fatalf("one repository does not page by index:\n%s", got)
261 }
262 if got := plan([]string{"r1", "r2"}); !strings.Contains(got, "events_time") || strings.Contains(got, "TEMP B-TREE") {
263 t.Fatalf("several repositories do not page by time index:\n%s", got)
264 }
265 }
266
267 func TestEventsOrderAcrossRepositories(t *testing.T) {
268 t.Parallel()
269 s, m, g := incrementalRepo(t)
270 other, og := newSecondRepo(t, s, "other")
271 now := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC)
272 createIndexedIssue(t, g, "demo one", now, nil, nil)
273 createIndexedIssue(t, g, "demo two", now.Add(2*time.Minute), nil, nil)
274 first, err := metadata.NewIssue("other one", "body", "keenan", nil, nil, now.Add(time.Minute))
275 if err != nil {
276 t.Fatal(err)
277 }
278 if err := metadata.CreateIssue(og, first, "keenan"); err != nil {
279 t.Fatal(err)
280 }
281 if err := s.Ensure(m); err != nil {
282 t.Fatal(err)
283 }
284 if err := s.Ensure(other); err != nil {
285 t.Fatal(err)
286 }
287 rows, err := s.Events([]string{m.ID, other.ID}, "", "", "", 10, 0)
288 if err != nil {
289 t.Fatal(err)
290 }
291 if len(rows) != 3 {
292 t.Fatalf("events = %d, want 3: %+v", len(rows), rows)
293 }
294 for i := 1; i < len(rows); i++ {
295 if rows[i-1].At < rows[i].At {
296 t.Fatalf("events are not newest first: %+v", rows)
297 }
298 }
299 if n, err := s.EventCount([]string{m.ID, other.ID}, "", "", ""); err != nil || n != 3 {
300 t.Fatalf("count = %d, %v", n, err)
301 }
302 page, err := s.Events([]string{m.ID, other.ID}, "", "", "", 2, 2)
303 if err != nil || len(page) != 1 {
304 t.Fatalf("second page = %+v, %v", page, err)
305 }
306 }