vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Read the activity feed as a SQL stream
73a732b1b84a308f0739e4c8325e9c2a832097cb · Keenan Nemetz · 9d ago · parent c8cdb6ac
Read the activity feed as a SQL stream A page of the feed used to be built in memory: every issue, comment, approval and state change became an event, was sorted, and was then sliced to fifty rows. The page cost the whole history, and the instance feed did it for every visible repository. The sources are rows in the index, so the stream is a UNION over them (forge/index.Activity and ActivityCount): the database filters, orders and limits, and only the page's rows cross back into the process. Releases are the one event source that is not an index row — a tag is a ref — so they are read from git (one call per repository, only when the feed can show a release) and merged with the page in time order. The handlers refresh each repository before reading, because a browser write never passes through the push hook. The repository feed keeps the in-memory path as a fallback for a Web without an index. `visibleRepos` now builds a repository handle without a HEAD lookup, which removes one git process per repository from the instance activity page too. The count and the page each scan the union, so the database still sorts the events rather than reading an index in order; a materialised events table is the next step if a large history makes that visible. What is gone is the per-request materialisation of every event in Go, which is what this follow-up set out to remove.
server/web/activity_query.go +151
| @@ -0,0 +1,151 @@ | |||
| 1 | package web | ||
| 2 | |||
| 3 | // The activity feed as a stream. | ||
| 4 | // | ||
| 5 | // The rows come from forge/index (`Activity` and `ActivityCount`), which orders | ||
| 6 | // and limits them in SQL: a page no longer costs the whole history in this | ||
| 7 | // process. Releases are the one event source that is not an index row — a tag is | ||
| 8 | // a ref, not metadata — so their events are read from git (one call per | ||
| 9 | // repository, only when the feed can show a release) and merged with the page in | ||
| 10 | // time order. Tags are few, so that part does not grow with history. | ||
| 11 | |||
| 12 | import ( | ||
| 13 | "sort" | ||
| 14 | "strings" | ||
| 15 | "time" | ||
| 16 | |||
| 17 | "github.com/rodent-software/vole/forge/index" | ||
| 18 | "github.com/rodent-software/vole/forge/repo" | ||
| 19 | ) | ||
| 20 | |||
| 21 | // activityTagEvents reads the release events for the repositories in scope. A | ||
| 22 | // feed that cannot show a release, or is filtered by assignee (a tag has none), | ||
| 23 | // does not spend the git call. | ||
| 24 | func (w *Web) activityTagEvents(metas []*repo.Meta, kind, actor, assignee string, withRepo bool) []activityEvent { | ||
| 25 | if kind != "" && kind != "release" { | ||
| 26 | return nil | ||
| 27 | } | ||
| 28 | if assignee != "" { | ||
| 29 | return nil | ||
| 30 | } | ||
| 31 | var out []activityEvent | ||
| 32 | for _, m := range metas { | ||
| 33 | tags, err := m.Git().Tags() | ||
| 34 | if err != nil { | ||
| 35 | continue | ||
| 36 | } | ||
| 37 | for _, t := range tags { | ||
| 38 | if actor != "" && t.Tagger != actor { | ||
| 39 | continue | ||
| 40 | } | ||
| 41 | ev := activityEvent{ | ||
| 42 | Kind: "release", Icon: "tag", Time: time.Unix(t.When, 0).UTC(), | ||
| 43 | Actor: t.Tagger, ActorName: t.Tagger, Title: t.Name, Detail: "released", | ||
| 44 | URL: "/r/" + m.Name + "/releases/" + t.Name, | ||
| 45 | } | ||
| 46 | if withRepo { | ||
| 47 | ev.Repo = m.Name | ||
| 48 | } | ||
| 49 | out = append(out, ev) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | sort.SliceStable(out, func(i, j int) bool { return out[i].Time.After(out[j].Time) }) | ||
| 53 | return out | ||
| 54 | } | ||
| 55 | |||
| 56 | // activityEventFromRow maps one database row to the event a template renders. | ||
| 57 | func (w *Web) activityEventFromRow(row index.ActivityRow, repoName string, withRepo bool) activityEvent { | ||
| 58 | ev := activityEvent{ | ||
| 59 | Time: time.Unix(row.At, 0).UTC(), Actor: w.who(row.Actor), ActorName: row.Actor, | ||
| 60 | Title: row.Title, Assignees: splitAssignees(row.Assignees), | ||
| 61 | } | ||
| 62 | if withRepo { | ||
| 63 | ev.Repo = repoName | ||
| 64 | } | ||
| 65 | base := "/r/" + repoName | ||
| 66 | switch row.Event { | ||
| 67 | case "issue_open": | ||
| 68 | ev.Kind, ev.Icon, ev.Detail = "issue", "circle-dot", "opened issue" | ||
| 69 | ev.URL = base + "/issues/" + row.ID | ||
| 70 | case "issue_comment": | ||
| 71 | ev.Kind, ev.Icon, ev.Detail = "issue", "message-square", "commented on issue" | ||
| 72 | ev.URL = base + "/issues/" + row.Parent + "#c-" + row.ID | ||
| 73 | case "request_open": | ||
| 74 | ev.Kind, ev.Icon, ev.Detail = "change", "git-pull-request", "opened change" | ||
| 75 | ev.URL = base + "/requests/" + row.ID | ||
| 76 | case "request_comment": | ||
| 77 | ev.Kind, ev.Icon, ev.Detail = "change", "message-square", "commented on change" | ||
| 78 | ev.URL = base + "/requests/" + row.Parent + "#c-" + row.ID | ||
| 79 | case "approval": | ||
| 80 | ev.Kind, ev.Icon, ev.Detail = "change", "check", "approved change" | ||
| 81 | ev.URL = base + "/requests/" + row.ID | ||
| 82 | case "state_issue": | ||
| 83 | ev.Kind, ev.Icon, ev.Detail = "issue", "circle-dot", issueStateDetail(row.State) | ||
| 84 | ev.URL = base + "/issues/" + row.ID | ||
| 85 | case "state_request": | ||
| 86 | ev.Kind, ev.Icon, ev.Detail = "change", "git-pull-request", requestStateDetail(row.State) | ||
| 87 | ev.URL = base + "/requests/" + row.ID | ||
| 88 | } | ||
| 89 | return ev | ||
| 90 | } | ||
| 91 | |||
| 92 | // activityPage returns one page of the feed and the pager that describes it. The | ||
| 93 | // page is the SQL stream merged with the tag events, so the two sources share one | ||
| 94 | // chronological order and one total. | ||
| 95 | func (w *Web) activityPage(repoIDs []string, metas []*repo.Meta, names map[string]string, | ||
| 96 | kind, actor, assignee string, withRepo bool, number, perPage int, | ||
| 97 | urlFor func(int) string, label string) ([]activityEvent, pager, error) { | ||
| 98 | count, err := w.Index.ActivityCount(repoIDs, kind, actor, assignee) | ||
| 99 | if err != nil { | ||
| 100 | return nil, pager{}, err | ||
| 101 | } | ||
| 102 | tags := w.activityTagEvents(metas, kind, actor, assignee, withRepo) | ||
| 103 | p := newPager(count+len(tags), perPage, number, urlFor) | ||
| 104 | p.Label = label | ||
| 105 | if p.Total == 0 { | ||
| 106 | return nil, p, nil | ||
| 107 | } | ||
| 108 | from, to := p.rows() | ||
| 109 | rows, err := w.Index.Activity(repoIDs, kind, actor, assignee, to) | ||
| 110 | if err != nil { | ||
| 111 | return nil, p, err | ||
| 112 | } | ||
| 113 | events := make([]activityEvent, 0, len(rows)+len(tags)) | ||
| 114 | for _, row := range rows { | ||
| 115 | events = append(events, w.activityEventFromRow(row, names[row.RepoID], withRepo)) | ||
| 116 | } | ||
| 117 | return mergeActivity(events, tags, from, to-from), p, nil | ||
| 118 | } | ||
| 119 | |||
| 120 | // mergeActivity interleaves two time-ordered lists and takes one page. SQL events | ||
| 121 | // come first on a tie, which is the order the in-memory feed used (releases were | ||
| 122 | // added last). | ||
| 123 | func mergeActivity(sqlEvents, tags []activityEvent, from, limit int) []activityEvent { | ||
| 124 | all := make([]activityEvent, 0, len(sqlEvents)+len(tags)) | ||
| 125 | i, j := 0, 0 | ||
| 126 | for i < len(sqlEvents) && j < len(tags) { | ||
| 127 | if tags[j].Time.After(sqlEvents[i].Time) { | ||
| 128 | all = append(all, tags[j]) | ||
| 129 | j++ | ||
| 130 | continue | ||
| 131 | } | ||
| 132 | all = append(all, sqlEvents[i]) | ||
| 133 | i++ | ||
| 134 | } | ||
| 135 | all = append(all, sqlEvents[i:]...) | ||
| 136 | all = append(all, tags[j:]...) | ||
| 137 | if from >= len(all) { | ||
| 138 | return nil | ||
| 139 | } | ||
| 140 | if from+limit > len(all) { | ||
| 141 | limit = len(all) - from | ||
| 142 | } | ||
| 143 | return all[from : from+limit] | ||
| 144 | } | ||
| 145 | |||
| 146 | func splitAssignees(s string) []string { | ||
| 147 | if s == "" { | ||
| 148 | return nil | ||
| 149 | } | ||
| 150 | return strings.Split(s, "\x1f") | ||
| 151 | } | ||