vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Read the activity feed as a SQL stream

73a732b1b84a308f0739e4c8325e9c2a832097cb · Keenan Nemetz · 9d ago · parent c8cdb6ac

Read the activity feed as a SQL stream

A page of the feed used to be built in memory: every issue, comment,
approval and state change became an event, was sorted, and was then sliced
to fifty rows. The page cost the whole history, and the instance feed did
it for every visible repository.

The sources are rows in the index, so the stream is a UNION over them
(forge/index.Activity and ActivityCount): the database filters, orders and
limits, and only the page's rows cross back into the process. Releases are
the one event source that is not an index row — a tag is a ref — so they
are read from git (one call per repository, only when the feed can show a
release) and merged with the page in time order.

The handlers refresh each repository before reading, because a browser
write never passes through the push hook. The repository feed keeps the
in-memory path as a fallback for a Web without an index. `visibleRepos`
now builds a repository handle without a HEAD lookup, which removes one
git process per repository from the instance activity page too.

The count and the page each scan the union, so the database still sorts
the events rather than reading an index in order; a materialised events
table is the next step if a large history makes that visible. What is
gone is the per-request materialisation of every event in Go, which is
what this follow-up set out to remove.

forge/index/activity_stream.go +139

@@ -0,0 +1,139 @@
	1	package index
	2
	3	// The activity feed as a SQL stream.
	4	//
	5	// The feed used to be built in memory from every source row, sorted, and then
	6	// sliced to a page, so a page cost the whole history. The sources are rows in
	7	// this package, so the stream is a UNION over them, ordered and limited by the
	8	// database; only the page's rows come back to the caller.
	9	//
	10	// Releases are not rows here: a tag is a ref, so the caller reads those from git
	11	// and merges them. Everything this file selects is derived metadata.
	12
	13	import (
	14	"strings"
	15	)
	16
	17	// ActivityRow is one feed event as the database returns it. The SQL event names
	18	// and the fields the renderer needs are the whole contract.
	19	type ActivityRow struct {
	20	RepoID string
	21	Event string // issue_open \| issue_comment \| request_open \| request_comment \| approval \| state_issue \| state_request
	22	ID string
	23	Parent string
	24	Title string
	25	Actor string
	26	Assignees string
	27	State string
	28	At int64
	29	}
	30
	31	// activityCTE is one row per feed event: issues and changes opened, their
	32	// comments and approvals, and the state changes in the branch's history. The
	33	// title and assignees come from the current entity, which is what the feed has
	34	// always shown; an entity that was deleted takes its events with it (inner joins).
	35	const activityCTE = `WITH ev AS (
	36	SELECT i.repo_id, 'issue_open' AS event, i.id AS id, '' AS parent, i.created AS at,
	37	i.author AS actor, i.title AS title, i.assignees AS assignees, '' AS state
	38	FROM issues i
	39	UNION ALL
	40	SELECT c.repo_id, 'issue_comment', c.id, c.parent, c.created, c.author, i.title, i.assignees, ''
	41	FROM comments c JOIN issues i ON i.repo_id = c.repo_id AND i.id = c.parent
	42	WHERE c.kind = 'issue'
	43	UNION ALL
	44	SELECT r.repo_id, 'request_open', r.id, '', r.created, r.author, r.title, '', ''
	45	FROM requests r
	46	UNION ALL
	47	SELECT c.repo_id, 'request_comment', c.id, c.parent, c.created, c.author, r.title, '', ''
	48	FROM comments c JOIN requests r ON r.repo_id = c.repo_id AND r.id = c.parent
	49	WHERE c.kind = 'request'
	50	UNION ALL
	51	SELECT a.repo_id, 'approval', a.request_id, '', a.created, a.principal, r.title, '', ''
	52	FROM request_approvals a JOIN requests r ON r.repo_id = a.repo_id AND r.id = a.request_id
	53	UNION ALL
	54	SELECT s.repo_id, 'state_issue', s.id, '', s.at, s.actor, i.title, i.assignees, s.state
	55	FROM state_changes s JOIN issues i ON i.repo_id = s.repo_id AND i.id = s.id
	56	WHERE s.kind = 'issue'
	57	UNION ALL
	58	SELECT s.repo_id, 'state_request', s.id, '', s.at, s.actor, r.title, '', s.state
	59	FROM state_changes s JOIN requests r ON r.repo_id = s.repo_id AND r.id = s.id
	60	WHERE s.kind = 'request'
	61	) `
	62
	63	// activityFilter builds the WHERE clause the count and the page share. Kind picks
	64	// the event types. A release has no SQL rows at all and is the caller's tag read.
	65	func activityFilter(repoIDs []string, kind, actor, assignee string) (string, []any) {
	66	conds := []string{`repo_id IN (` + questionMarks(len(repoIDs)) + `)`}
	67	args := make([]any, 0, len(repoIDs)+2)
	68	for _, id := range repoIDs {
	69	args = append(args, id)
	70	}
	71	switch kind {
	72	case "issue":
	73	conds = append(conds, `event IN ('issue_open','issue_comment','state_issue')`)
	74	case "change":
	75	conds = append(conds, `event IN ('request_open','request_comment','approval','state_request')`)
	76	}
	77	if actor != "" {
	78	conds = append(conds, `actor = ?`)
	79	args = append(args, actor)
	80	}
	81	if assignee != "" {
	82	// Only issue events carry assignees. The column is a \x1f-joined list;
	83	// char(31) keeps the delimiter out of the query text.
	84	conds = append(conds,
	85	`event IN ('issue_open','issue_comment','state_issue')`,
	86	`instr(char(31)\|\|assignees\|\|char(31), char(31)\|\|?\|\|char(31)) > 0`)
	87	args = append(args, assignee)
	88	}
	89	return strings.Join(conds, " AND "), args
	90	}
	91
	92	// ActivityCount is how many events the filters select, before paging.
	93	func (s *Store) ActivityCount(repoIDs []string, kind, actor, assignee string) (int, error) {
	94	if kind == "release" \|\| len(repoIDs) == 0 {
	95	return 0, nil
	96	}
	97	where, args := activityFilter(repoIDs, kind, actor, assignee)
	98	var n int
	99	err := s.db.QueryRow(activityCTE+`SELECT COUNT(*) FROM ev WHERE `+where, args...).Scan(&n)
	100	return n, err
	101	}
	102
	103	// Activity reads the first limit events the filters select, newest first. It is
	104	// asked for the first event of the page, not the page itself, because the caller
	105	// merges the tag events in after the query; the extra rows keep that merge exact
	106	// when a release falls inside the page.
	107	func (s *Store) Activity(repoIDs []string, kind, actor, assignee string, limit int) ([]ActivityRow, error) {
	108	if kind == "release" \|\| len(repoIDs) == 0 \|\| limit <= 0 {
	109	return nil, nil
	110	}
	111	where, args := activityFilter(repoIDs, kind, actor, assignee)
	112	args = append(args, limit)
	113	q := activityCTE + `SELECT repo_id, event, id, parent, at, actor, title, assignees, state
	114	FROM ev WHERE ` + where + `
	115	ORDER BY at DESC, event, id, parent LIMIT ?`
	116	rows, err := s.db.Query(q, args...)
	117	if err != nil {
	118	return nil, err
	119	}
	120	defer rows.Close()
	121	var out []ActivityRow
	122	for rows.Next() {
	123	var r ActivityRow
	124	if err := rows.Scan(&r.RepoID, &r.Event, &r.ID, &r.Parent, &r.At, &r.Actor,
	125	&r.Title, &r.Assignees, &r.State); err != nil {
	126	return nil, err
	127	}
	128	out = append(out, r)
	129	}
	130	return out, rows.Err()
	131	}
	132
	133	// questionMarks renders "?,?,?" for an IN clause.
	134	func questionMarks(n int) string {
	135	if n <= 0 {
	136	return ""
	137	}
	138	return strings.TrimSuffix(strings.Repeat("?,", n), ",")
	139	}