Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.

vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Materialise the activity feed in an events table

bca9ec24cbbb938f2d349d5f06e2c1f4fca06fcb · Keenan Nemetz · 9d ago · parent 73a732b1

Materialise the activity feed in an events table

The previous commit read the feed as a UNION over the source tables. It
stopped materialising events in Go, but SQLite still scanned and sorted
the union to find a page, so a page cost the history.

This replaces it with the design the feed actually wants: an `events`
table with one row per event — issue and change opened, comment,
approval, state change, release — carrying the time, actor, title and
assignees the feed renders. Rows are indexed by (repository, time, id),
so a repository page is an index range read; the instance page walks the
global time index and filters by repository, and neither sorts history.
An unfiltered page total is a stored per-repository count.

Maintenance is part of the refresh that already exists:

- an entity's events are replaced when the entity is reloaded, which is
  how a title or assignee edit reaches old events;
- state changes are rebuilt into events for the entities the commits
  name, which also catches a close-and-reopen that leaves the tree where
  it started and an endpoint diff cannot see;
- releases are refreshed from the tags after every push (post-receive)
  and after a release written in the browser, because a tag is a ref and
  no metadata commit announces it;
- the stored count moves with the rows a refresh adds or removes.

Measured with 100,000 events across 50 repositories: a repository page
takes 0.3-0.5 ms (page 1 or page 20), an instance page 0.5-0.9 ms, and an
unfiltered total 0.1 ms. A filtered total is the one query that scans,
at 12 ms for 100,000 events.

On the load harness, instance activity rises from 222 to 482 req/s and
p50 from 18 ms to 8 ms; activity from 577 to 762 req/s. The repository
feed no longer reads tags per request, and nothing reads the metadata
branch to render a page.

server/web/activity_query.go +56−105

@@ -1,146 +1,97 @@
1 1 package web
2 2
3 // The activity feed as a stream.
3 // The activity feed as a page of events.
4 4 //
5 // The rows come from forge/index (`Activity` and `ActivityCount`), which orders
6 // and limits them in SQL: a page no longer costs the whole history in this
7 // process. Releases are the one event source that is not an index row — a tag is
8 // a ref, not metadata — so their events are read from git (one call per
9 // repository, only when the feed can show a release) and merged with the page in
10 // time order. Tags are few, so that part does not grow with history.
5 // The rows come from forge/index (`Events` and `EventCount`), which keeps a row
6 // per feed event with a (repository, time) index: a page is an index range read,
7 // and an unfiltered page total is a stored count. Releases are rows there too, so
8 // nothing is fetched from git to render a page.
11 9
12 10 import (
13 "sort"
14 11 "strings"
15 12 "time"
16 13
17 14 "github.com/rodent-software/vole/forge/index"
18 "github.com/rodent-software/vole/forge/repo"
19 15 )
20 16
21 // activityTagEvents reads the release events for the repositories in scope. A
22 // feed that cannot show a release, or is filtered by assignee (a tag has none),
23 // does not spend the git call.
24 func (w *Web) activityTagEvents(metas []*repo.Meta, kind, actor, assignee string, withRepo bool) []activityEvent {
25 if kind != "" && kind != "release" {
26 return nil
27 }
28 if assignee != "" {
29 return nil
30 }
31 var out []activityEvent
32 for _, m := range metas {
33 tags, err := m.Git().Tags()
34 if err != nil {
35 continue
36 }
37 for _, t := range tags {
38 if actor != "" && t.Tagger != actor {
39 continue
40 }
41 ev := activityEvent{
42 Kind: "release", Icon: "tag", Time: time.Unix(t.When, 0).UTC(),
43 Actor: t.Tagger, ActorName: t.Tagger, Title: t.Name, Detail: "released",
44 URL: "/r/" + m.Name + "/releases/" + t.Name,
45 }
46 if withRepo {
47 ev.Repo = m.Name
48 }
49 out = append(out, ev)
50 }
51 }
52 sort.SliceStable(out, func(i, j int) bool { return out[i].Time.After(out[j].Time) })
53 return out
54 }
55
56 // activityEventFromRow maps one database row to the event a template renders.
57 func (w *Web) activityEventFromRow(row index.ActivityRow, repoName string, withRepo bool) activityEvent {
17 // activityEventFromRow maps one indexed event to the event a template renders.
18 func (w *Web) activityEventFromRow(row index.EventRow, repoName string, withRepo bool) activityEvent {
58 19 ev := activityEvent{
59 Time: time.Unix(row.At, 0).UTC(), Actor: w.who(row.Actor), ActorName: row.Actor,
20 Time: time.Unix(row.At, 0).UTC(), ActorName: row.Actor,
60 21 Title: row.Title, Assignees: splitAssignees(row.Assignees),
61 22 }
62 23 if withRepo {
63 24 ev.Repo = repoName
64 25 }
65 26 base := "/r/" + repoName
66 switch row.Event {
67 case "issue_open":
68 ev.Kind, ev.Icon, ev.Detail = "issue", "circle-dot", "opened issue"
69 ev.URL = base + "/issues/" + row.ID
70 case "issue_comment":
71 ev.Kind, ev.Icon, ev.Detail = "issue", "message-square", "commented on issue"
72 ev.URL = base + "/issues/" + row.Parent + "#c-" + row.ID
73 case "request_open":
74 ev.Kind, ev.Icon, ev.Detail = "change", "git-pull-request", "opened change"
75 ev.URL = base + "/requests/" + row.ID
76 case "request_comment":
77 ev.Kind, ev.Icon, ev.Detail = "change", "message-square", "commented on change"
78 ev.URL = base + "/requests/" + row.Parent + "#c-" + row.ID
27 switch row.Kind {
28 case "release":
29 ev.Kind, ev.Icon, ev.Detail = "release", "tag", "released"
30 ev.Actor = row.Actor // the tagger identity, as the tag records it
31 ev.URL = base + "/releases/" + row.EntityID
32 return ev
33 case "issue":
34 ev.Kind, ev.Icon = "issue", "circle-dot"
35 case "change":
36 ev.Kind, ev.Icon = "change", "git-pull-request"
37 }
38 ev.Actor = w.who(row.Actor)
39 switch row.Type {
40 case "open":
41 if row.Kind == "issue" {
42 ev.Detail = "opened issue"
43 ev.URL = base + "/issues/" + row.EntityID
44 } else {
45 ev.Detail = "opened change"
46 ev.URL = base + "/requests/" + row.EntityID
47 }
48 case "comment":
49 ev.Icon = "message-square"
50 if row.Kind == "issue" {
51 ev.Detail = "commented on issue"
52 ev.URL = base + "/issues/" + row.ParentID + "#c-" + row.CommentID
53 } else {
54 ev.Detail = "commented on change"
55 ev.URL = base + "/requests/" + row.ParentID + "#c-" + row.CommentID
56 }
79 57 case "approval":
80 ev.Kind, ev.Icon, ev.Detail = "change", "check", "approved change"
81 ev.URL = base + "/requests/" + row.ID
82 case "state_issue":
83 ev.Kind, ev.Icon, ev.Detail = "issue", "circle-dot", issueStateDetail(row.State)
84 ev.URL = base + "/issues/" + row.ID
85 case "state_request":
86 ev.Kind, ev.Icon, ev.Detail = "change", "git-pull-request", requestStateDetail(row.State)
87 ev.URL = base + "/requests/" + row.ID
58 ev.Icon, ev.Detail = "check", "approved change"
59 ev.URL = base + "/requests/" + row.EntityID
60 case "state":
61 if row.Kind == "issue" {
62 ev.Detail = issueStateDetail(row.State)
63 ev.URL = base + "/issues/" + row.EntityID
64 } else {
65 ev.Detail = requestStateDetail(row.State)
66 ev.URL = base + "/requests/" + row.EntityID
67 }
88 68 }
89 69 return ev
90 70 }
91 71
92 // activityPage returns one page of the feed and the pager that describes it. The
93 // page is the SQL stream merged with the tag events, so the two sources share one
94 // chronological order and one total.
95 func (w *Web) activityPage(repoIDs []string, metas []*repo.Meta, names map[string]string,
72 // activityPage returns one page of the feed and the pager that describes it.
73 func (w *Web) activityPage(repoIDs []string, names map[string]string,
96 74 kind, actor, assignee string, withRepo bool, number, perPage int,
97 75 urlFor func(int) string, label string) ([]activityEvent, pager, error) {
98 count, err := w.Index.ActivityCount(repoIDs, kind, actor, assignee)
76 count, err := w.Index.EventCount(repoIDs, kind, actor, assignee)
99 77 if err != nil {
100 78 return nil, pager{}, err
101 79 }
102 tags := w.activityTagEvents(metas, kind, actor, assignee, withRepo)
103 p := newPager(count+len(tags), perPage, number, urlFor)
80 p := newPager(count, perPage, number, urlFor)
104 81 p.Label = label
105 82 if p.Total == 0 {
106 83 return nil, p, nil
107 84 }
108 from, to := p.rows()
109 rows, err := w.Index.Activity(repoIDs, kind, actor, assignee, to)
85 from, _ := p.rows()
86 rows, err := w.Index.Events(repoIDs, kind, actor, assignee, perPage, from)
110 87 if err != nil {
111 88 return nil, p, err
112 89 }
113 events := make([]activityEvent, 0, len(rows)+len(tags))
90 events := make([]activityEvent, 0, len(rows))
114 91 for _, row := range rows {
115 92 events = append(events, w.activityEventFromRow(row, names[row.RepoID], withRepo))
116 93 }
117 return mergeActivity(events, tags, from, to-from), p, nil
118 }
119
120 // mergeActivity interleaves two time-ordered lists and takes one page. SQL events
121 // come first on a tie, which is the order the in-memory feed used (releases were
122 // added last).
123 func mergeActivity(sqlEvents, tags []activityEvent, from, limit int) []activityEvent {
124 all := make([]activityEvent, 0, len(sqlEvents)+len(tags))
125 i, j := 0, 0
126 for i < len(sqlEvents) && j < len(tags) {
127 if tags[j].Time.After(sqlEvents[i].Time) {
128 all = append(all, tags[j])
129 j++
130 continue
131 }
132 all = append(all, sqlEvents[i])
133 i++
134 }
135 all = append(all, sqlEvents[i:]...)
136 all = append(all, tags[j:]...)
137 if from >= len(all) {
138 return nil
139 }
140 if from+limit > len(all) {
141 limit = len(all) - from
142 }
143 return all[from : from+limit]
94 return events, p, nil
144 95 }
145 96
146 97 func splitAssignees(s string) []string {