vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Materialise the activity feed in an events table
bca9ec24cbbb938f2d349d5f06e2c1f4fca06fcb · Keenan Nemetz · 9d ago · parent 73a732b1
Materialise the activity feed in an events table The previous commit read the feed as a UNION over the source tables. It stopped materialising events in Go, but SQLite still scanned and sorted the union to find a page, so a page cost the history. This replaces it with the design the feed actually wants: an `events` table with one row per event — issue and change opened, comment, approval, state change, release — carrying the time, actor, title and assignees the feed renders. Rows are indexed by (repository, time, id), so a repository page is an index range read; the instance page walks the global time index and filters by repository, and neither sorts history. An unfiltered page total is a stored per-repository count. Maintenance is part of the refresh that already exists: - an entity's events are replaced when the entity is reloaded, which is how a title or assignee edit reaches old events; - state changes are rebuilt into events for the entities the commits name, which also catches a close-and-reopen that leaves the tree where it started and an endpoint diff cannot see; - releases are refreshed from the tags after every push (post-receive) and after a release written in the browser, because a tag is a ref and no metadata commit announces it; - the stored count moves with the rows a refresh adds or removes. Measured with 100,000 events across 50 repositories: a repository page takes 0.3-0.5 ms (page 1 or page 20), an instance page 0.5-0.9 ms, and an unfiltered total 0.1 ms. A filtered total is the one query that scans, at 12 ms for 100,000 events. On the load harness, instance activity rises from 222 to 482 req/s and p50 from 18 ms to 8 ms; activity from 577 to 762 req/s. The repository feed no longer reads tags per request, and nothing reads the metadata branch to render a page.
forge/index/index.go +35−4
| @@ -54,6 +54,9 @@ CREATE TABLE IF NOT EXISTS repos ( | |||
| 54 | 54 | // Added after the first release of the index; the table is disposable, so a | |
| 55 | 55 | // failed ALTER (already present) is not an error. | |
| 56 | 56 | _, _ = db.Exec(`ALTER TABLE repos ADD COLUMN issues_sha TEXT NOT NULL DEFAULT ''`) | |
| 57 | // events_count is the number of feed events a repository has, so an | ||
| 58 | // unfiltered pager gets its total without counting the table. | ||
| 59 | _, _ = db.Exec(`ALTER TABLE repos ADD COLUMN events_count INTEGER NOT NULL DEFAULT 0`) | ||
| 57 | 60 | if _, err := db.Exec(` | |
| 58 | 61 | CREATE TABLE IF NOT EXISTS issues ( | |
| 59 | 62 | repo_id TEXT NOT NULL, | |
| @@ -197,7 +200,34 @@ CREATE TABLE IF NOT EXISTS state_changes ( | |||
| 197 | 200 | ordinal INTEGER NOT NULL DEFAULT 0, | |
| 198 | 201 | PRIMARY KEY (repo_id, sha) | |
| 199 | 202 | ); | |
| 200 | CREATE INDEX IF NOT EXISTS state_changes_recent ON state_changes(repo_id, ordinal DESC);`); err != nil { | ||
| 203 | CREATE INDEX IF NOT EXISTS state_changes_recent ON state_changes(repo_id, ordinal DESC); | ||
| 204 | |||
| 205 | -- events is the activity feed as rows: one per issue, change and release, and | ||
| 206 | -- one per comment, approval and state change. It is the read model the feed | ||
| 207 | -- pages paginate; the row carries the title and assignees the feed shows, and | ||
| 208 | -- ids rather than URLs so a repository rename cannot leave a dead link. | ||
| 209 | CREATE TABLE IF NOT EXISTS events ( | ||
| 210 | repo_id TEXT NOT NULL, | ||
| 211 | id TEXT NOT NULL, | ||
| 212 | kind TEXT NOT NULL, -- issue | change | release | ||
| 213 | type TEXT NOT NULL, -- open | comment | approval | state | release | ||
| 214 | at INTEGER NOT NULL, | ||
| 215 | actor TEXT NOT NULL, | ||
| 216 | title TEXT NOT NULL, | ||
| 217 | assignees TEXT NOT NULL DEFAULT '', | ||
| 218 | entity_id TEXT NOT NULL, | ||
| 219 | parent_id TEXT NOT NULL DEFAULT '', | ||
| 220 | comment_id TEXT NOT NULL DEFAULT '', | ||
| 221 | state TEXT NOT NULL DEFAULT '', | ||
| 222 | PRIMARY KEY (repo_id, id) | ||
| 223 | ); | ||
| 224 | -- The feed's query: one repository, newest first. The id breaks a tie inside a | ||
| 225 | -- second, so a page boundary cannot repeat or skip an event. | ||
| 226 | CREATE INDEX IF NOT EXISTS events_repo ON events(repo_id, at DESC, id DESC); | ||
| 227 | -- The instance feed orders across repositories; this index lets SQLite scan in | ||
| 228 | -- time order and filter by repository rather than sorting every visible event. | ||
| 229 | CREATE INDEX IF NOT EXISTS events_time ON events(at DESC, id DESC); | ||
| 230 | CREATE INDEX IF NOT EXISTS events_entity ON events(repo_id, entity_id);`); err != nil { | ||
| 201 | 231 | db.Close() | |
| 202 | 232 | return nil, err | |
| 203 | 233 | } | |
| @@ -233,7 +263,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search USING fts5( | |||
| 233 | 263 | // and leaves every repo with an empty issues_sha, which makes the next read | |
| 234 | 264 | // rebuild that repo from git. A stale row is worse than a rebuild, and a rebuild is the | |
| 235 | 265 | // operation the index already has to survive. | |
| 236 | const schemaVersion = 7 | ||
| 266 | const schemaVersion = 8 | ||
| 237 | 267 | ||
| 238 | 268 | func (s *Store) migrate() error { | |
| 239 | 269 | var v int | |
| @@ -254,8 +284,9 @@ DELETE FROM request_approvals; | |||
| 254 | 284 | DELETE FROM comments; | |
| 255 | 285 | DELETE FROM artifacts; | |
| 256 | 286 | DELETE FROM state_changes; | |
| 287 | DELETE FROM events; | ||
| 257 | 288 | DELETE FROM search; | |
| 258 | UPDATE repos SET issues_sha = '';`); err != nil { | ||
| 289 | UPDATE repos SET issues_sha = '', events_count = 0;`); err != nil { | ||
| 259 | 290 | return err | |
| 260 | 291 | } | |
| 261 | 292 | _, err := s.db.Exec(fmt.Sprintf(`PRAGMA user_version = %d`, schemaVersion)) | |
| @@ -299,7 +330,7 @@ func (s *Store) Rebuild(st *repo.Store) (int, error) { | |||
| 299 | 330 | // longer exists must not survive it. | |
| 300 | 331 | for _, table := range []string{ | |
| 301 | 332 | "issues", "issue_labels", "issue_assignees", "requests", "request_closes", | |
| 302 | "request_approvals", "comments", "artifacts", "state_changes", "search", "entity_numbers", | ||
| 333 | "request_approvals", "comments", "artifacts", "state_changes", "events", "search", "entity_numbers", | ||
| 303 | 334 | } { | |
| 304 | 335 | if _, err := tx.Exec(`DELETE FROM ` + table); err != nil { | |
| 305 | 336 | return 0, err | |