vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Materialise the activity feed in an events table
bca9ec24cbbb938f2d349d5f06e2c1f4fca06fcb · Keenan Nemetz · 9d ago · parent 73a732b1
Materialise the activity feed in an events table The previous commit read the feed as a UNION over the source tables. It stopped materialising events in Go, but SQLite still scanned and sorted the union to find a page, so a page cost the history. This replaces it with the design the feed actually wants: an `events` table with one row per event — issue and change opened, comment, approval, state change, release — carrying the time, actor, title and assignees the feed renders. Rows are indexed by (repository, time, id), so a repository page is an index range read; the instance page walks the global time index and filters by repository, and neither sorts history. An unfiltered page total is a stored per-repository count. Maintenance is part of the refresh that already exists: - an entity's events are replaced when the entity is reloaded, which is how a title or assignee edit reaches old events; - state changes are rebuilt into events for the entities the commits name, which also catches a close-and-reopen that leaves the tree where it started and an endpoint diff cannot see; - releases are refreshed from the tags after every push (post-receive) and after a release written in the browser, because a tag is a ref and no metadata commit announces it; - the stored count moves with the rows a refresh adds or removes. Measured with 100,000 events across 50 repositories: a repository page takes 0.3-0.5 ms (page 1 or page 20), an instance page 0.5-0.9 ms, and an unfiltered total 0.1 ms. A filtered total is the one query that scans, at 12 ms for 100,000 events. On the load harness, instance activity rises from 222 to 482 req/s and p50 from 18 ms to 8 ms; activity from 577 to 762 req/s. The repository feed no longer reads tags per request, and nothing reads the metadata branch to render a page.
server/web/activity_query.go +56−105
| @@ -1,146 +1,97 @@ | |||
| 1 | 1 | package web | |
| 2 | 2 | ||
| 3 | // The activity feed as a stream. | ||
| 3 | // The activity feed as a page of events. | ||
| 4 | 4 | // | |
| 5 | // The rows come from forge/index (`Activity` and `ActivityCount`), which orders | ||
| 6 | // and limits them in SQL: a page no longer costs the whole history in this | ||
| 7 | // process. Releases are the one event source that is not an index row — a tag is | ||
| 8 | // a ref, not metadata — so their events are read from git (one call per | ||
| 9 | // repository, only when the feed can show a release) and merged with the page in | ||
| 10 | // time order. Tags are few, so that part does not grow with history. | ||
| 5 | // The rows come from forge/index (`Events` and `EventCount`), which keeps a row | ||
| 6 | // per feed event with a (repository, time) index: a page is an index range read, | ||
| 7 | // and an unfiltered page total is a stored count. Releases are rows there too, so | ||
| 8 | // nothing is fetched from git to render a page. | ||
| 11 | 9 | ||
| 12 | 10 | import ( | |
| 13 | "sort" | ||
| 14 | 11 | "strings" | |
| 15 | 12 | "time" | |
| 16 | 13 | ||
| 17 | 14 | "github.com/rodent-software/vole/forge/index" | |
| 18 | "github.com/rodent-software/vole/forge/repo" | ||
| 19 | 15 | ) | |
| 20 | 16 | ||
| 21 | // activityTagEvents reads the release events for the repositories in scope. A | ||
| 22 | // feed that cannot show a release, or is filtered by assignee (a tag has none), | ||
| 23 | // does not spend the git call. | ||
| 24 | func (w *Web) activityTagEvents(metas []*repo.Meta, kind, actor, assignee string, withRepo bool) []activityEvent { | ||
| 25 | if kind != "" && kind != "release" { | ||
| 26 | return nil | ||
| 27 | } | ||
| 28 | if assignee != "" { | ||
| 29 | return nil | ||
| 30 | } | ||
| 31 | var out []activityEvent | ||
| 32 | for _, m := range metas { | ||
| 33 | tags, err := m.Git().Tags() | ||
| 34 | if err != nil { | ||
| 35 | continue | ||
| 36 | } | ||
| 37 | for _, t := range tags { | ||
| 38 | if actor != "" && t.Tagger != actor { | ||
| 39 | continue | ||
| 40 | } | ||
| 41 | ev := activityEvent{ | ||
| 42 | Kind: "release", Icon: "tag", Time: time.Unix(t.When, 0).UTC(), | ||
| 43 | Actor: t.Tagger, ActorName: t.Tagger, Title: t.Name, Detail: "released", | ||
| 44 | URL: "/r/" + m.Name + "/releases/" + t.Name, | ||
| 45 | } | ||
| 46 | if withRepo { | ||
| 47 | ev.Repo = m.Name | ||
| 48 | } | ||
| 49 | out = append(out, ev) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | sort.SliceStable(out, func(i, j int) bool { return out[i].Time.After(out[j].Time) }) | ||
| 53 | return out | ||
| 54 | } | ||
| 55 | |||
| 56 | // activityEventFromRow maps one database row to the event a template renders. | ||
| 57 | func (w *Web) activityEventFromRow(row index.ActivityRow, repoName string, withRepo bool) activityEvent { | ||
| 17 | // activityEventFromRow maps one indexed event to the event a template renders. | ||
| 18 | func (w *Web) activityEventFromRow(row index.EventRow, repoName string, withRepo bool) activityEvent { | ||
| 58 | 19 | ev := activityEvent{ | |
| 59 | Time: time.Unix(row.At, 0).UTC(), Actor: w.who(row.Actor), ActorName: row.Actor, | ||
| 20 | Time: time.Unix(row.At, 0).UTC(), ActorName: row.Actor, | ||
| 60 | 21 | Title: row.Title, Assignees: splitAssignees(row.Assignees), | |
| 61 | 22 | } | |
| 62 | 23 | if withRepo { | |
| 63 | 24 | ev.Repo = repoName | |
| 64 | 25 | } | |
| 65 | 26 | base := "/r/" + repoName | |
| 66 | switch row.Event { | ||
| 67 | case "issue_open": | ||
| 68 | ev.Kind, ev.Icon, ev.Detail = "issue", "circle-dot", "opened issue" | ||
| 69 | ev.URL = base + "/issues/" + row.ID | ||
| 70 | case "issue_comment": | ||
| 71 | ev.Kind, ev.Icon, ev.Detail = "issue", "message-square", "commented on issue" | ||
| 72 | ev.URL = base + "/issues/" + row.Parent + "#c-" + row.ID | ||
| 73 | case "request_open": | ||
| 74 | ev.Kind, ev.Icon, ev.Detail = "change", "git-pull-request", "opened change" | ||
| 75 | ev.URL = base + "/requests/" + row.ID | ||
| 76 | case "request_comment": | ||
| 77 | ev.Kind, ev.Icon, ev.Detail = "change", "message-square", "commented on change" | ||
| 78 | ev.URL = base + "/requests/" + row.Parent + "#c-" + row.ID | ||
| 27 | switch row.Kind { | ||
| 28 | case "release": | ||
| 29 | ev.Kind, ev.Icon, ev.Detail = "release", "tag", "released" | ||
| 30 | ev.Actor = row.Actor // the tagger identity, as the tag records it | ||
| 31 | ev.URL = base + "/releases/" + row.EntityID | ||
| 32 | return ev | ||
| 33 | case "issue": | ||
| 34 | ev.Kind, ev.Icon = "issue", "circle-dot" | ||
| 35 | case "change": | ||
| 36 | ev.Kind, ev.Icon = "change", "git-pull-request" | ||
| 37 | } | ||
| 38 | ev.Actor = w.who(row.Actor) | ||
| 39 | switch row.Type { | ||
| 40 | case "open": | ||
| 41 | if row.Kind == "issue" { | ||
| 42 | ev.Detail = "opened issue" | ||
| 43 | ev.URL = base + "/issues/" + row.EntityID | ||
| 44 | } else { | ||
| 45 | ev.Detail = "opened change" | ||
| 46 | ev.URL = base + "/requests/" + row.EntityID | ||
| 47 | } | ||
| 48 | case "comment": | ||
| 49 | ev.Icon = "message-square" | ||
| 50 | if row.Kind == "issue" { | ||
| 51 | ev.Detail = "commented on issue" | ||
| 52 | ev.URL = base + "/issues/" + row.ParentID + "#c-" + row.CommentID | ||
| 53 | } else { | ||
| 54 | ev.Detail = "commented on change" | ||
| 55 | ev.URL = base + "/requests/" + row.ParentID + "#c-" + row.CommentID | ||
| 56 | } | ||
| 79 | 57 | case "approval": | |
| 80 | ev.Kind, ev.Icon, ev.Detail = "change", "check", "approved change" | ||
| 81 | ev.URL = base + "/requests/" + row.ID | ||
| 82 | case "state_issue": | ||
| 83 | ev.Kind, ev.Icon, ev.Detail = "issue", "circle-dot", issueStateDetail(row.State) | ||
| 84 | ev.URL = base + "/issues/" + row.ID | ||
| 85 | case "state_request": | ||
| 86 | ev.Kind, ev.Icon, ev.Detail = "change", "git-pull-request", requestStateDetail(row.State) | ||
| 87 | ev.URL = base + "/requests/" + row.ID | ||
| 58 | ev.Icon, ev.Detail = "check", "approved change" | ||
| 59 | ev.URL = base + "/requests/" + row.EntityID | ||
| 60 | case "state": | ||
| 61 | if row.Kind == "issue" { | ||
| 62 | ev.Detail = issueStateDetail(row.State) | ||
| 63 | ev.URL = base + "/issues/" + row.EntityID | ||
| 64 | } else { | ||
| 65 | ev.Detail = requestStateDetail(row.State) | ||
| 66 | ev.URL = base + "/requests/" + row.EntityID | ||
| 67 | } | ||
| 88 | 68 | } | |
| 89 | 69 | return ev | |
| 90 | 70 | } | |
| 91 | 71 | ||
| 92 | // activityPage returns one page of the feed and the pager that describes it. The | ||
| 93 | // page is the SQL stream merged with the tag events, so the two sources share one | ||
| 94 | // chronological order and one total. | ||
| 95 | func (w *Web) activityPage(repoIDs []string, metas []*repo.Meta, names map[string]string, | ||
| 72 | // activityPage returns one page of the feed and the pager that describes it. | ||
| 73 | func (w *Web) activityPage(repoIDs []string, names map[string]string, | ||
| 96 | 74 | kind, actor, assignee string, withRepo bool, number, perPage int, | |
| 97 | 75 | urlFor func(int) string, label string) ([]activityEvent, pager, error) { | |
| 98 | count, err := w.Index.ActivityCount(repoIDs, kind, actor, assignee) | ||
| 76 | count, err := w.Index.EventCount(repoIDs, kind, actor, assignee) | ||
| 99 | 77 | if err != nil { | |
| 100 | 78 | return nil, pager{}, err | |
| 101 | 79 | } | |
| 102 | tags := w.activityTagEvents(metas, kind, actor, assignee, withRepo) | ||
| 103 | p := newPager(count+len(tags), perPage, number, urlFor) | ||
| 80 | p := newPager(count, perPage, number, urlFor) | ||
| 104 | 81 | p.Label = label | |
| 105 | 82 | if p.Total == 0 { | |
| 106 | 83 | return nil, p, nil | |
| 107 | 84 | } | |
| 108 | from, to := p.rows() | ||
| 109 | rows, err := w.Index.Activity(repoIDs, kind, actor, assignee, to) | ||
| 85 | from, _ := p.rows() | ||
| 86 | rows, err := w.Index.Events(repoIDs, kind, actor, assignee, perPage, from) | ||
| 110 | 87 | if err != nil { | |
| 111 | 88 | return nil, p, err | |
| 112 | 89 | } | |
| 113 | events := make([]activityEvent, 0, len(rows)+len(tags)) | ||
| 90 | events := make([]activityEvent, 0, len(rows)) | ||
| 114 | 91 | for _, row := range rows { | |
| 115 | 92 | events = append(events, w.activityEventFromRow(row, names[row.RepoID], withRepo)) | |
| 116 | 93 | } | |
| 117 | return mergeActivity(events, tags, from, to-from), p, nil | ||
| 118 | } | ||
| 119 | |||
| 120 | // mergeActivity interleaves two time-ordered lists and takes one page. SQL events | ||
| 121 | // come first on a tie, which is the order the in-memory feed used (releases were | ||
| 122 | // added last). | ||
| 123 | func mergeActivity(sqlEvents, tags []activityEvent, from, limit int) []activityEvent { | ||
| 124 | all := make([]activityEvent, 0, len(sqlEvents)+len(tags)) | ||
| 125 | i, j := 0, 0 | ||
| 126 | for i < len(sqlEvents) && j < len(tags) { | ||
| 127 | if tags[j].Time.After(sqlEvents[i].Time) { | ||
| 128 | all = append(all, tags[j]) | ||
| 129 | j++ | ||
| 130 | continue | ||
| 131 | } | ||
| 132 | all = append(all, sqlEvents[i]) | ||
| 133 | i++ | ||
| 134 | } | ||
| 135 | all = append(all, sqlEvents[i:]...) | ||
| 136 | all = append(all, tags[j:]...) | ||
| 137 | if from >= len(all) { | ||
| 138 | return nil | ||
| 139 | } | ||
| 140 | if from+limit > len(all) { | ||
| 141 | limit = len(all) - from | ||
| 142 | } | ||
| 143 | return all[from : from+limit] | ||
| 94 | return events, p, nil | ||
| 144 | 95 | } | |
| 145 | 96 | ||
| 146 | 97 | func splitAssignees(s string) []string { | |