vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Materialise the activity feed in an events table

bca9ec24cbbb938f2d349d5f06e2c1f4fca06fcb · Keenan Nemetz · 9d ago · parent 73a732b1

Materialise the activity feed in an events table

The previous commit read the feed as a UNION over the source tables. It
stopped materialising events in Go, but SQLite still scanned and sorted
the union to find a page, so a page cost the history.

This replaces it with the design the feed actually wants: an `events`
table with one row per event — issue and change opened, comment,
approval, state change, release — carrying the time, actor, title and
assignees the feed renders. Rows are indexed by (repository, time, id),
so a repository page is an index range read; the instance page walks the
global time index and filters by repository, and neither sorts history.
An unfiltered page total is a stored per-repository count.

Maintenance is part of the refresh that already exists:

- an entity's events are replaced when the entity is reloaded, which is
  how a title or assignee edit reaches old events;
- state changes are rebuilt into events for the entities the commits
  name, which also catches a close-and-reopen that leaves the tree where
  it started and an endpoint diff cannot see;
- releases are refreshed from the tags after every push (post-receive)
  and after a release written in the browser, because a tag is a ref and
  no metadata commit announces it;
- the stored count moves with the rows a refresh adds or removes.

Measured with 100,000 events across 50 repositories: a repository page
takes 0.3-0.5 ms (page 1 or page 20), an instance page 0.5-0.9 ms, and an
unfiltered total 0.1 ms. A filtered total is the one query that scans,
at 12 ms for 100,000 events.

On the load harness, instance activity rises from 222 to 482 req/s and
p50 from 18 ms to 8 ms; activity from 577 to 762 req/s. The repository
feed no longer reads tags per request, and nothing reads the metadata
branch to render a page.

forge/index/activity_stream.go −139

@@ -1,139 +0,0 @@
	1	package index
	2
	3	// The activity feed as a SQL stream.
	4	//
	5	// The feed used to be built in memory from every source row, sorted, and then
	6	// sliced to a page, so a page cost the whole history. The sources are rows in
	7	// this package, so the stream is a UNION over them, ordered and limited by the
	8	// database; only the page's rows come back to the caller.
	9	//
	10	// Releases are not rows here: a tag is a ref, so the caller reads those from git
	11	// and merges them. Everything this file selects is derived metadata.
	12
	13	import (
	14	"strings"
	15	)
	16
	17	// ActivityRow is one feed event as the database returns it. The SQL event names
	18	// and the fields the renderer needs are the whole contract.
	19	type ActivityRow struct {
	20	RepoID string
	21	Event string // issue_open \| issue_comment \| request_open \| request_comment \| approval \| state_issue \| state_request
	22	ID string
	23	Parent string
	24	Title string
	25	Actor string
	26	Assignees string
	27	State string
	28	At int64
	29	}
	30
	31	// activityCTE is one row per feed event: issues and changes opened, their
	32	// comments and approvals, and the state changes in the branch's history. The
	33	// title and assignees come from the current entity, which is what the feed has
	34	// always shown; an entity that was deleted takes its events with it (inner joins).
	35	const activityCTE = `WITH ev AS (
	36	SELECT i.repo_id, 'issue_open' AS event, i.id AS id, '' AS parent, i.created AS at,
	37	i.author AS actor, i.title AS title, i.assignees AS assignees, '' AS state
	38	FROM issues i
	39	UNION ALL
	40	SELECT c.repo_id, 'issue_comment', c.id, c.parent, c.created, c.author, i.title, i.assignees, ''
	41	FROM comments c JOIN issues i ON i.repo_id = c.repo_id AND i.id = c.parent
	42	WHERE c.kind = 'issue'
	43	UNION ALL
	44	SELECT r.repo_id, 'request_open', r.id, '', r.created, r.author, r.title, '', ''
	45	FROM requests r
	46	UNION ALL
	47	SELECT c.repo_id, 'request_comment', c.id, c.parent, c.created, c.author, r.title, '', ''
	48	FROM comments c JOIN requests r ON r.repo_id = c.repo_id AND r.id = c.parent
	49	WHERE c.kind = 'request'
	50	UNION ALL
	51	SELECT a.repo_id, 'approval', a.request_id, '', a.created, a.principal, r.title, '', ''
	52	FROM request_approvals a JOIN requests r ON r.repo_id = a.repo_id AND r.id = a.request_id
	53	UNION ALL
	54	SELECT s.repo_id, 'state_issue', s.id, '', s.at, s.actor, i.title, i.assignees, s.state
	55	FROM state_changes s JOIN issues i ON i.repo_id = s.repo_id AND i.id = s.id
	56	WHERE s.kind = 'issue'
	57	UNION ALL
	58	SELECT s.repo_id, 'state_request', s.id, '', s.at, s.actor, r.title, '', s.state
	59	FROM state_changes s JOIN requests r ON r.repo_id = s.repo_id AND r.id = s.id
	60	WHERE s.kind = 'request'
	61	) `
	62
	63	// activityFilter builds the WHERE clause the count and the page share. Kind picks
	64	// the event types. A release has no SQL rows at all and is the caller's tag read.
	65	func activityFilter(repoIDs []string, kind, actor, assignee string) (string, []any) {
	66	conds := []string{`repo_id IN (` + questionMarks(len(repoIDs)) + `)`}
	67	args := make([]any, 0, len(repoIDs)+2)
	68	for _, id := range repoIDs {
	69	args = append(args, id)
	70	}
	71	switch kind {
	72	case "issue":
	73	conds = append(conds, `event IN ('issue_open','issue_comment','state_issue')`)
	74	case "change":
	75	conds = append(conds, `event IN ('request_open','request_comment','approval','state_request')`)
	76	}
	77	if actor != "" {
	78	conds = append(conds, `actor = ?`)
	79	args = append(args, actor)
	80	}
	81	if assignee != "" {
	82	// Only issue events carry assignees. The column is a \x1f-joined list;
	83	// char(31) keeps the delimiter out of the query text.
	84	conds = append(conds,
	85	`event IN ('issue_open','issue_comment','state_issue')`,
	86	`instr(char(31)\|\|assignees\|\|char(31), char(31)\|\|?\|\|char(31)) > 0`)
	87	args = append(args, assignee)
	88	}
	89	return strings.Join(conds, " AND "), args
	90	}
	91
	92	// ActivityCount is how many events the filters select, before paging.
	93	func (s *Store) ActivityCount(repoIDs []string, kind, actor, assignee string) (int, error) {
	94	if kind == "release" \|\| len(repoIDs) == 0 {
	95	return 0, nil
	96	}
	97	where, args := activityFilter(repoIDs, kind, actor, assignee)
	98	var n int
	99	err := s.db.QueryRow(activityCTE+`SELECT COUNT(*) FROM ev WHERE `+where, args...).Scan(&n)
	100	return n, err
	101	}
	102
	103	// Activity reads the first limit events the filters select, newest first. It is
	104	// asked for the first event of the page, not the page itself, because the caller
	105	// merges the tag events in after the query; the extra rows keep that merge exact
	106	// when a release falls inside the page.
	107	func (s *Store) Activity(repoIDs []string, kind, actor, assignee string, limit int) ([]ActivityRow, error) {
	108	if kind == "release" \|\| len(repoIDs) == 0 \|\| limit <= 0 {
	109	return nil, nil
	110	}
	111	where, args := activityFilter(repoIDs, kind, actor, assignee)
	112	args = append(args, limit)
	113	q := activityCTE + `SELECT repo_id, event, id, parent, at, actor, title, assignees, state
	114	FROM ev WHERE ` + where + `
	115	ORDER BY at DESC, event, id, parent LIMIT ?`
	116	rows, err := s.db.Query(q, args...)
	117	if err != nil {
	118	return nil, err
	119	}
	120	defer rows.Close()
	121	var out []ActivityRow
	122	for rows.Next() {
	123	var r ActivityRow
	124	if err := rows.Scan(&r.RepoID, &r.Event, &r.ID, &r.Parent, &r.At, &r.Actor,
	125	&r.Title, &r.Assignees, &r.State); err != nil {
	126	return nil, err
	127	}
	128	out = append(out, r)
	129	}
	130	return out, rows.Err()
	131	}
	132
	133	// questionMarks renders "?,?,?" for an IN clause.
	134	func questionMarks(n int) string {
	135	if n <= 0 {
	136	return ""
	137	}
	138	return strings.TrimSuffix(strings.Repeat("?,", n), ",")
	139	}