vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Index the rest of the metadata branch

7f547ca3b8da218249120d3c910c73c9a1a1e52d · Keenan Nemetz · 10d ago · parent c9202f7f

Index the rest of the metadata branch

The index held issues and their numbers. Everything else on the branch —
change requests, their comments, approvals and closes, and release
artifacts — was read from git on every page that needed it, and the
repository header read the whole requests subtree just to print a count.

Add the tables and derive them from the same commit:

- requests, with the body preserved for search;
- request_closes and request_approvals, each in its own table so a
  backlink and a current-approval count are queries rather than a walk of
  every request;
- comments for both kinds of entity, with the anchor a review comment
  carries;
- artifacts, the release pointers, with the size from the pointer.

The refresh is still driven by the path delta between two commits: a
changed issue or request replaces its own rows, its comments and its sets,
and a deleted one removes them while keeping its `#42` number. Rebuild and
the first read go through IndexRepo, which is the definition the
incremental path has to reproduce; a rebuild now clears the derived tables
first so a deleted repository leaves nothing behind.

Schema version 4 drops the old rows on first open, which is the existing
recovery path: the index is disposable and built from the branch.

forge/index/refresh.go +272

@@ -0,0 +1,272 @@
	1	package index
	2
	3	// Refreshing the derived rows from the metadata branch.
	4	//
	5	// Ensure runs on read. It resolves the metadata ref, compares it with the commit
	6	// the rows were built from, and applies the difference when the branch moved
	7	// forward. The branch is the source of truth, so everything here can be thrown
	8	// away and rebuilt; the index only has to agree with one commit at a time.
	9
	10	import (
	11	"strings"
	12
	13	"github.com/rodent-software/vole/forge/repo"
	14	"github.com/rodent-software/vole/metadata"
	15	)
	16
	17	// Ensure refreshes everything a repo derives from its metadata branch: issues,
	18	// change requests, their comments and sets, the release artifacts, and the `#42`
	19	// numbering. Called on read, so a push is visible without a hook or a full
	20	// rebuild.
	21	//
	22	// A branch that moved forward is refreshed from the paths that changed. The full
	23	// read is reserved for the cases that need it: the first read, a rewritten
	24	// branch, or an incremental refresh that cannot be trusted.
	25	func (s Store) Ensure(m repo.Meta) error {
	26	current, err := m.Git().RevParse(metadata.Ref)
	27	if err != nil {
	28	return err
	29	}
	30	return s.EnsureAt(m, current)
	31	}
	32
	33	// EnsureAt refreshes the derived rows to the metadata commit the caller already
	34	// resolved. It is Ensure for a caller that needs the commit for something else
	35	// too, so one page does not resolve the same ref twice.
	36	func (s Store) EnsureAt(m repo.Meta, current string) error {
	37	var stored string
	38	err := s.db.QueryRow(`SELECT issues_sha FROM repos WHERE id=?`, m.ID).Scan(&stored)
	39	if err != nil {
	40	return err
	41	}
	42	if stored == current {
	43	return nil
	44	}
	45	if stored != "" && current != "" {
	46	if ok, err := m.Git().IsAncestor(stored, current); err == nil && ok {
	47	if err := s.indexFrom(m, stored, current); err == nil {
	48	return nil
	49	}
	50	// The delta could not be applied, so the rows and the stored commit
	51	// would disagree. Fall through to the reconstruction that defines the
	52	// index.
	53	}
	54	}
	55	return s.IndexRepo(m)
	56	}
	57
	58	// EnsureIssues keeps the older name used by the issue surfaces and the tests. It
	59	// is Ensure: the refresh covers every table, because they are derived together.
	60	func (s Store) EnsureIssues(m repo.Meta) error { return s.Ensure(m) }
	61
	62	// IndexRepo rebuilds every derived row for one repository from the metadata
	63	// branch. It is the definition of the index: the incremental path must produce
	64	// the same rows, and `vole admin index rebuild` is this function for every repo.
	65	func (s Store) IndexRepo(m repo.Meta) error {
	66	g := m.Git()
	67	commit, err := g.RevParse(metadata.Ref)
	68	if err != nil {
	69	return err
	70	}
	71	issues, err := metadata.LoadIssuesAt(g, commit)
	72	if err != nil {
	73	return err
	74	}
	75	requests, err := metadata.LoadRequestsAt(g, commit)
	76	if err != nil {
	77	return err
	78	}
	79	numbers, err := metadata.LoadNumbersAt(g, commit)
	80	if err != nil {
	81	return err
	82	}
	83	artifacts, err := metadata.LoadArtifactsAt(g, commit)
	84	if err != nil {
	85	return err
	86	}
	87
	88	tx, err := s.db.Begin()
	89	if err != nil {
	90	return err
	91	}
	92	defer tx.Rollback()
	93	if err := deleteRepoRows(tx, m.ID); err != nil {
	94	return err
	95	}
	96	for number := 1; number <= numbers.Len(); number++ {
	97	id, kind, ok := numbers.At(number)
	98	if !ok {
	99	continue
	100	}
	101	if err := insertNumber(tx, m.ID, number, kind, id); err != nil {
	102	return err
	103	}
	104	}
	105	for _, iss := range issues {
	106	if err := insertIssue(tx, m.ID, iss); err != nil {
	107	return err
	108	}
	109	if err := replaceComments(tx, m.ID, commentKindIssue, iss.ID, iss.Comments); err != nil {
	110	return err
	111	}
	112	}
	113	for _, req := range requests {
	114	if err := insertRequest(tx, m.ID, req); err != nil {
	115	return err
	116	}
	117	if err := replaceComments(tx, m.ID, commentKindRequest, req.ID, req.Comments); err != nil {
	118	return err
	119	}
	120	}
	121	if err := replaceArtifacts(tx, m.ID, artifacts); err != nil {
	122	return err
	123	}
	124	if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, commit, m.ID); err != nil {
	125	return err
	126	}
	127	return tx.Commit()
	128	}
	129
	130	// indexFrom applies the difference between two commits to the index. The caller
	131	// has already established that from is an ancestor of to, so every entity that
	132	// exists at to is either unchanged, changed in the range, or new.
	133	func (s Store) indexFrom(m repo.Meta, from, to string) error {
	134	g := m.Git()
	135	changes, err := g.DiffStatusPaths(from, to,
	136	metadata.IssuesDir, metadata.RequestsDir, metadata.ReleasesDir)
	137	if err != nil {
	138	return err
	139	}
	140	issueIDs := map[string]bool{}
	141	requestIDs := map[string]bool{}
	142	releasesChanged := false
	143	for path := range changes {
	144	switch {
	145	case strings.HasPrefix(path, metadata.IssuesDir+"/"):
	146	if id, ok := metadata.IssueIDFromPath(path); ok {
	147	issueIDs[id] = true
	148	}
	149	case strings.HasPrefix(path, metadata.RequestsDir+"/"):
	150	if id, ok := metadata.RequestIDFromPath(path); ok {
	151	requestIDs[id] = true
	152	}
	153	case strings.HasPrefix(path, metadata.ReleasesDir+"/"):
	154	releasesChanged = true
	155	}
	156	}
	157
	158	// Read every changed entity before opening the write transaction. `to` is a
	159	// commit, so a concurrent push cannot make the rows describe two tree states.
	160	issues := make(map[string]*metadata.Issue, len(issueIDs))
	161	for id := range issueIDs {
	162	iss, found, err := metadata.LoadIssueAtCommit(g, to, id)
	163	if err != nil {
	164	return err
	165	}
	166	if found {
	167	issues[id] = iss
	168	}
	169	}
	170	requests := make(map[string]*metadata.Request, len(requestIDs))
	171	for id := range requestIDs {
	172	req, found, err := metadata.LoadRequestAt(g, to, id)
	173	if err != nil {
	174	return err
	175	}
	176	if found {
	177	requests[id] = req
	178	}
	179	}
	180	var artifacts map[string][]metadata.Artifact
	181	if releasesChanged {
	182	if artifacts, err = metadata.LoadArtifactsAt(g, to); err != nil {
	183	return err
	184	}
	185	}
	186
	187	// Numbers are historical: a deleted entity keeps its number, and a creation
	188	// that was deleted again inside the range still consumed one. The additions
	189	// are therefore taken from the commit range, not from the current tree.
	190	groups, err := g.AddedPathsPerCommit(from+".."+to, metadata.IssuesDir, metadata.RequestsDir)
	191	if err != nil {
	192	return err
	193	}
	194
	195	tx, err := s.db.Begin()
	196	if err != nil {
	197	return err
	198	}
	199	defer tx.Rollback()
	200	for id := range issueIDs {
	201	if _, found := issues[id]; found {
	202	continue
	203	}
	204	if err := deleteIssue(tx, m.ID, id); err != nil {
	205	return err
	206	}
	207	}
	208	for _, iss := range issues {
	209	if err := deleteIssue(tx, m.ID, iss.ID); err != nil {
	210	return err
	211	}
	212	if err := insertIssue(tx, m.ID, iss); err != nil {
	213	return err
	214	}
	215	if err := replaceComments(tx, m.ID, commentKindIssue, iss.ID, iss.Comments); err != nil {
	216	return err
	217	}
	218	}
	219	for id := range requestIDs {
	220	if _, found := requests[id]; found {
	221	continue
	222	}
	223	if err := deleteRequest(tx, m.ID, id); err != nil {
	224	return err
	225	}
	226	}
	227	for _, req := range requests {
	228	if err := deleteRequest(tx, m.ID, req.ID); err != nil {
	229	return err
	230	}
	231	if err := insertRequest(tx, m.ID, req); err != nil {
	232	return err
	233	}
	234	if err := replaceComments(tx, m.ID, commentKindRequest, req.ID, req.Comments); err != nil {
	235	return err
	236	}
	237	}
	238	if releasesChanged {
	239	if err := replaceArtifacts(tx, m.ID, artifacts); err != nil {
	240	return err
	241	}
	242	}
	243	if err := addNumbers(tx, m.ID, metadata.Additions(groups)); err != nil {
	244	return err
	245	}
	246	if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, to, m.ID); err != nil {
	247	return err
	248	}
	249	return tx.Commit()
	250	}
	251
	252	// deleteRepoRows removes every derived row for one repository. It is what makes
	253	// the full rebuild a replacement rather than a merge, and it is why a deleted
	254	// repository cannot leave rows behind.
	255	func deleteRepoRows(tx execer, repoID string) error {
	256	for _, q := range []string{
	257	`DELETE FROM issues WHERE repo_id=?`,
	258	`DELETE FROM issue_labels WHERE repo_id=?`,
	259	`DELETE FROM issue_assignees WHERE repo_id=?`,
	260	`DELETE FROM requests WHERE repo_id=?`,
	261	`DELETE FROM request_closes WHERE repo_id=?`,
	262	`DELETE FROM request_approvals WHERE repo_id=?`,
	263	`DELETE FROM comments WHERE repo_id=?`,
	264	`DELETE FROM artifacts WHERE repo_id=?`,
	265	`DELETE FROM entity_numbers WHERE repo_id=?`,
	266	} {
	267	if _, err := tx.Exec(q, repoID); err != nil {
	268	return err
	269	}
	270	}
	271	return nil
	272	}