Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Materialise the activity feed in an events table

bca9ec24cbbb938f2d349d5f06e2c1f4fca06fcb · Keenan Nemetz · 9d ago · parent 73a732b1 

Materialise the activity feed in an events table

The previous commit read the feed as a UNION over the source tables. It
stopped materialising events in Go, but SQLite still scanned and sorted
the union to find a page, so a page cost the history.

This replaces it with the design the feed actually wants: an `events`
table with one row per event — issue and change opened, comment,
approval, state change, release — carrying the time, actor, title and
assignees the feed renders. Rows are indexed by (repository, time, id),
so a repository page is an index range read; the instance page walks the
global time index and filters by repository, and neither sorts history.
An unfiltered page total is a stored per-repository count.

Maintenance is part of the refresh that already exists:

- an entity's events are replaced when the entity is reloaded, which is
  how a title or assignee edit reaches old events;
- state changes are rebuilt into events for the entities the commits
  name, which also catches a close-and-reopen that leaves the tree where
  it started and an endpoint diff cannot see;
- releases are refreshed from the tags after every push (post-receive)
  and after a release written in the browser, because a tag is a ref and
  no metadata commit announces it;
- the stored count moves with the rows a refresh adds or removes.

Measured with 100,000 events across 50 repositories: a repository page
takes 0.3-0.5 ms (page 1 or page 20), an instance page 0.5-0.9 ms, and an
unfiltered total 0.1 ms. A filtered total is the one query that scans,
at 12 ms for 100,000 events.

On the load harness, instance activity rises from 222 to 482 req/s and
p50 from 18 ms to 8 ms; activity from 577 to 762 req/s. The repository
feed no longer reads tags per request, and nothing reads the metadata
branch to render a page.

forge/index/refresh.go +66−8

@@ -94,6 +94,10 @@ func (s *Store) IndexRepo(m *repo.Meta) error {
94 94 return err
95 95 }
96 96 }
97 tags, err := g.Tags()
98 if err != nil {
99 return err
100 }
97 101
98 102 tx, err := s.db.Begin()
99 103 if err != nil {
@@ -103,6 +107,10 @@ func (s *Store) IndexRepo(m *repo.Meta) error {
103 107 if err := deleteRepoRows(tx, m.ID); err != nil {
104 108 return err
105 109 }
110 // State changes come before the entities whose events they produce.
111 if err := insertStateChanges(tx, m.ID, stateCommits); err != nil {
112 return err
113 }
106 114 for number := 1; number <= numbers.Len(); number++ {
107 115 id, kind, ok := numbers.At(number)
108 116 if !ok {
@@ -119,6 +127,9 @@ func (s *Store) IndexRepo(m *repo.Meta) error {
119 127 if err := replaceComments(tx, m.ID, commentKindIssue, iss.ID, iss.Title, iss.Comments); err != nil {
120 128 return err
121 129 }
130 if _, _, err := insertIssueEvents(tx, m.ID, iss); err != nil {
131 return err
132 }
122 133 }
123 134 for _, req := range requests {
124 135 if err := insertRequest(tx, m.ID, req); err != nil {
@@ -127,11 +138,18 @@ func (s *Store) IndexRepo(m *repo.Meta) error {
127 138 if err := replaceComments(tx, m.ID, commentKindRequest, req.ID, req.Title, req.Comments); err != nil {
128 139 return err
129 140 }
141 if _, _, err := insertRequestEvents(tx, m.ID, req); err != nil {
142 return err
143 }
130 144 }
131 145 if err := replaceArtifacts(tx, m.ID, artifacts); err != nil {
132 146 return err
133 147 }
134 if err := insertStateChanges(tx, m.ID, stateCommits); err != nil {
148 if _, _, err := replaceReleaseEvents(tx, m.ID, tags); err != nil {
149 return err
150 }
151 if _, err := tx.Exec(`UPDATE repos SET events_count=(SELECT COUNT(*) FROM events WHERE repo_id=?) WHERE id=?`,
152 m.ID, m.ID); err != nil {
135 153 return err
136 154 }
137 155 if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, commit, m.ID); err != nil {
@@ -150,8 +168,28 @@ func (s *Store) indexFrom(m *repo.Meta, from, to string) error {
150 168 if err != nil {
151 169 return err
152 170 }
171 // The commits in the range carry the state changes. They also name entities
172 // whose events must be rebuilt even when the tree diff has nothing to show: a
173 // close followed by a reopen inside the range leaves the file where it
174 // started, and an endpoint diff cannot see the two events in between.
175 stateCommits, err := g.CommitsBetween(from, to, 0)
176 if err != nil {
177 return err
178 }
153 179 issueIDs := map[string]bool{}
154 180 requestIDs := map[string]bool{}
181 for _, c := range stateCommits {
182 kind, id, _, ok := metadata.ParseStateSubject(c.Subject)
183 if !ok {
184 continue
185 }
186 switch kind {
187 case "issue":
188 issueIDs[id] = true
189 case "request":
190 requestIDs[id] = true
191 }
192 }
155 193 releasesChanged := false
156 194 for path := range changes {
157 195 switch {
@@ -200,12 +238,6 @@ func (s *Store) indexFrom(m *repo.Meta, from, to string) error {
200 238 // Numbers are historical: a deleted entity keeps its number, and a creation
201 239 // that was deleted again inside the range still consumed one. The additions
202 240 // are therefore taken from the commit range, not from the current tree.
203 // The commits in the range carry the state changes: a closed issue is a
204 // subject, not a file to diff.
205 stateCommits, err := g.CommitsBetween(from, to, 0)
206 if err != nil {
207 return err
208 }
209 241 groups, err := g.AddedPathsPerCommit(from+".."+to, metadata.IssuesDir, metadata.RequestsDir)
210 242 if err != nil {
211 243 return err
@@ -216,10 +248,20 @@ func (s *Store) indexFrom(m *repo.Meta, from, to string) error {
216 248 return err
217 249 }
218 250 defer tx.Rollback()
251 // State changes come first: an entity's events are rebuilt from them.
252 if err := insertStateChanges(tx, m.ID, stateCommits); err != nil {
253 return err
254 }
255 eventDelta := 0
219 256 for id := range issueIDs {
220 257 if _, found := issues[id]; found {
221 258 continue
222 259 }
260 deleted, err := deleteEventsForEntity(tx, m.ID, "issue", id)
261 if err != nil {
262 return err
263 }
264 eventDelta -= deleted
223 265 if err := deleteIssue(tx, m.ID, id); err != nil {
224 266 return err
225 267 }
@@ -234,11 +276,21 @@ func (s *Store) indexFrom(m *repo.Meta, from, to string) error {
234 276 if err := replaceComments(tx, m.ID, commentKindIssue, iss.ID, iss.Title, iss.Comments); err != nil {
235 277 return err
236 278 }
279 inserted, deleted, err := insertIssueEvents(tx, m.ID, iss)
280 if err != nil {
281 return err
282 }
283 eventDelta += inserted - deleted
237 284 }
238 285 for id := range requestIDs {
239 286 if _, found := requests[id]; found {
240 287 continue
241 288 }
289 deleted, err := deleteEventsForEntity(tx, m.ID, "change", id)
290 if err != nil {
291 return err
292 }
293 eventDelta -= deleted
242 294 if err := deleteRequest(tx, m.ID, id); err != nil {
243 295 return err
244 296 }
@@ -253,13 +305,18 @@ func (s *Store) indexFrom(m *repo.Meta, from, to string) error {
253 305 if err := replaceComments(tx, m.ID, commentKindRequest, req.ID, req.Title, req.Comments); err != nil {
254 306 return err
255 307 }
308 inserted, deleted, err := insertRequestEvents(tx, m.ID, req)
309 if err != nil {
310 return err
311 }
312 eventDelta += inserted - deleted
256 313 }
257 314 if releasesChanged {
258 315 if err := replaceArtifacts(tx, m.ID, artifacts); err != nil {
259 316 return err
260 317 }
261 318 }
262 if err := insertStateChanges(tx, m.ID, stateCommits); err != nil {
319 if err := bumpEventCount(tx, m.ID, eventDelta); err != nil {
263 320 return err
264 321 }
265 322 if err := addNumbers(tx, m.ID, metadata.Additions(groups)); err != nil {
@@ -285,6 +342,7 @@ func deleteRepoRows(tx execer, repoID string) error {
285 342 `DELETE FROM comments WHERE repo_id=?`,
286 343 `DELETE FROM artifacts WHERE repo_id=?`,
287 344 `DELETE FROM state_changes WHERE repo_id=?`,
345 `DELETE FROM events WHERE repo_id=?`,
288 346 `DELETE FROM entity_numbers WHERE repo_id=?`,
289 347 } {
290 348 if _, err := tx.Exec(q, repoID); err != nil {