vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Refresh the issue index from what changed

34a404b394dd9abcce0a67e32e99342c69595413 · Keenan Nemetz · 10d ago · parent fdf2340f

Refresh the issue index from what changed

A page read after a write used to reload every issue and renumber the repository from the whole branch history. Ensure now applies the difference between the indexed commit and the tip: it reloads only changed issues, adds only the numbers the range created, and leaves every other row alone. A rewritten branch or a missing index still falls back to the full rebuild.

The detail page and the browser write handlers now read one issue instead of the whole issue tree. Issue writes no longer re-validate the whole metadata tree after a write the writer already validated; the pre-receive hook still owns that check for pushes.

forge/index/issues.go +179−37

@@ -1,6 +1,7 @@
	1	1	package index
	2	2
	3	3	import (
		4	"database/sql"
	4	5	"strings"
	5	6	"time"
	6	7
@@ -11,6 +12,10 @@ import (
	11	12	// Ensure refreshes everything a repo derives from its metadata branch: the issue
	12	13	// rows, the label and assignee filters, and the `#42` numbering. Called on read,
	13	14	// so a push is visible without a hook or a full rebuild.
		15	//
		16	// A branch that moved forward is refreshed from the paths that changed. The full
		17	// read is reserved for the cases that need it: the first read, a rewritten
		18	// branch, or an incremental refresh that cannot be trusted.
	14	19	func (s Store) Ensure(m repo.Meta) error {
	15	20	current := ""
	16	21	if sha, err := m.Git().RevParse(metadata.Ref); err == nil {
@@ -24,6 +29,15 @@ func (s Store) Ensure(m repo.Meta) error {
	24	29	if stored == current {
	25	30	return nil
	26	31	}
		32	if stored != "" && current != "" {
		33	if ok, err := m.Git().IsAncestor(stored, current); err == nil && ok {
		34	if err := s.indexIssuesFrom(m, stored, current); err == nil {
		35	return nil
		36	}
		37	// The delta could not be applied, so the rows and the stored sha would
		38	// disagree. Fall through to the reconstruction that defines the index.
		39	}
		40	}
	27	41	return s.IndexIssues(m)
	28	42	}
	29	43
@@ -52,58 +66,188 @@ func (s Store) IndexIssues(m repo.Meta) error {
	52	66	return err
	53	67	}
	54	68	defer tx.Rollback()
	55		if _, err := tx.Exec(`DELETE FROM issues WHERE repo_id=?`, m.ID); err != nil {
		69	for _, table := range []string{"issues", "issue_labels", "issue_assignees", "entity_numbers"} {
		70	if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_id=?`, m.ID); err != nil {
		71	return err
		72	}
		73	}
		74	for number := 1; number <= numbers.Len(); number++ {
		75	id, kind, ok := numbers.At(number)
		76	if !ok {
		77	continue
		78	}
		79	if err := insertNumber(tx, m.ID, number, kind, id); err != nil {
		80	return err
		81	}
		82	}
		83	for _, iss := range list {
		84	if err := insertIssue(tx, m.ID, iss); err != nil {
		85	return err
		86	}
		87	}
		88	if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, sha, m.ID); err != nil {
	56	89	return err
	57	90	}
	58		if _, err := tx.Exec(`DELETE FROM issue_labels WHERE repo_id=?`, m.ID); err != nil {
		91	return tx.Commit()
		92	}
		93
		94	// indexIssuesFrom applies the difference between two commits to the index. The
		95	// caller has already established that from is an ancestor of to, so every issue
		96	// that exists at to is either unchanged, changed in the range, or new.
		97	func (s Store) indexIssuesFrom(m repo.Meta, from, to string) error {
		98	changes, err := m.Git().DiffStatusPaths(from, to, metadata.IssuesDir)
		99	if err != nil {
	59	100	return err
	60	101	}
	61		if _, err := tx.Exec(`DELETE FROM issue_assignees WHERE repo_id=?`, m.ID); err != nil {
		102	ids := map[string]bool{}
		103	for path := range changes {
		104	if id, ok := metadata.IssueIDFromPath(path); ok {
		105	ids[id] = true
		106	}
		107	}
		108
		109	// Read every changed issue before opening the write transaction. `to` is a
		110	// commit, so a concurrent push cannot make the rows describe two different
		111	// tree states.
		112	issues := make(map[string]*metadata.Issue, len(ids))
		113	for id := range ids {
		114	iss, found, err := metadata.LoadIssueAt(m.Git(), to, id)
		115	if err != nil {
		116	return err
		117	}
		118	if found {
		119	issues[id] = iss
		120	}
		121	}
		122
		123	// Numbers are historical: a deleted entity keeps its number, and a creation
		124	// that was deleted again inside the range still consumed one. The additions
		125	// are therefore taken from the commit range, not from the current tree.
		126	groups, err := m.Git().AddedPathsPerCommit(from+".."+to, metadata.IssuesDir, metadata.RequestsDir)
		127	if err != nil {
	62	128	return err
	63	129	}
	64		if _, err := tx.Exec(`DELETE FROM entity_numbers WHERE repo_id=?`, m.ID); err != nil {
		130
		131	tx, err := s.db.Begin()
		132	if err != nil {
	65	133	return err
	66	134	}
	67		for number := 1; number <= numbers.Len(); number++ {
	68		id, kind, ok := numbers.At(number)
	69		if !ok {
		135	defer tx.Rollback()
		136	for id := range ids {
		137	if _, found := issues[id]; found {
	70	138	continue
	71	139	}
		140	if err := deleteIssue(tx, m.ID, id); err != nil {
		141	return err
		142	}
		143	}
		144	for _, iss := range issues {
		145	if err := deleteIssue(tx, m.ID, iss.ID); err != nil {
		146	return err
		147	}
		148	if err := insertIssue(tx, m.ID, iss); err != nil {
		149	return err
		150	}
		151	}
		152	if err := addNumbers(tx, m.ID, metadata.Additions(groups)); err != nil {
		153	return err
		154	}
		155	if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, to, m.ID); err != nil {
		156	return err
		157	}
		158	return tx.Commit()
		159	}
		160
		161	// insertIssue writes the row and filter tables for one issue. The caller decides
		162	// whether it is replacing an existing issue, so this stays one shape used by the
		163	// full rebuild and the incremental refresh alike.
		164	func insertIssue(tx execer, repoID string, iss *metadata.Issue) error {
		165	if _, err := tx.Exec(
		166	`INSERT INTO issues(repo_id, id, title, state, author, created, labels, assignees, comments, body)
		167	VALUES(?,?,?,?,?,?,?,?,?,?)`,
		168	repoID, iss.ID, iss.Title, iss.State, iss.Author,
		169	iss.Created.Unix(), strings.Join(iss.Labels, "\x1f"), strings.Join(iss.Assignees, "\x1f"),
		170	len(iss.Comments), iss.Description,
		171	); err != nil {
		172	return err
		173	}
		174	for _, l := range iss.Labels {
	72	175	if _, err := tx.Exec(
	73		`INSERT INTO entity_numbers(repo_id, number, kind, id) VALUES(?,?,?,?)`,
	74		m.ID, number, kind, id); err != nil {
		176	`INSERT OR IGNORE INTO issue_labels(repo_id, issue_id, label) VALUES(?,?,?)`,
		177	repoID, iss.ID, l); err != nil {
	75	178	return err
	76	179	}
	77	180	}
	78		for _, iss := range list {
		181	for _, a := range iss.Assignees {
	79	182	if _, err := tx.Exec(
	80		`INSERT INTO issues(repo_id, id, title, state, author, created, labels, assignees, comments, body)
	81		VALUES(?,?,?,?,?,?,?,?,?,?)`,
	82		m.ID, iss.ID, iss.Title, iss.State, iss.Author,
	83		iss.Created.Unix(), strings.Join(iss.Labels, "\x1f"), strings.Join(iss.Assignees, "\x1f"),
	84		len(iss.Comments), iss.Description,
	85		); err != nil {
		183	`INSERT OR IGNORE INTO issue_assignees(repo_id, issue_id, assignee) VALUES(?,?,?)`,
		184	repoID, iss.ID, a); err != nil {
	86	185	return err
	87	186	}
	88		for _, l := range iss.Labels {
	89		if _, err := tx.Exec(
	90		`INSERT OR IGNORE INTO issue_labels(repo_id, issue_id, label) VALUES(?,?,?)`,
	91		m.ID, iss.ID, l); err != nil {
	92		return err
	93		}
		187	}
		188	return nil
		189	}
		190
		191	// deleteIssue removes one issue's rows without touching its number. A number is
		192	// spent for good once history assigned it.
		193	func deleteIssue(tx execer, repoID, id string) error {
		194	for _, q := range []string{
		195	`DELETE FROM issues WHERE repo_id=? AND id=?`,
		196	`DELETE FROM issue_labels WHERE repo_id=? AND issue_id=?`,
		197	`DELETE FROM issue_assignees WHERE repo_id=? AND issue_id=?`,
		198	} {
		199	if _, err := tx.Exec(q, repoID, id); err != nil {
		200	return err
	94	201	}
	95		for _, a := range iss.Assignees {
	96		if _, err := tx.Exec(
	97		`INSERT OR IGNORE INTO issue_assignees(repo_id, issue_id, assignee) VALUES(?,?,?)`,
	98		m.ID, iss.ID, a); err != nil {
	99		return err
	100		}
		202	}
		203	return nil
		204	}
		205
		206	func insertNumber(tx execer, repoID string, number int, kind, id string) error {
		207	_, err := tx.Exec(
		208	`INSERT INTO entity_numbers(repo_id, number, kind, id) VALUES(?,?,?,?)`,
		209	repoID, number, kind, id)
		210	return err
		211	}
		212
		213	// addNumbers appends the entities a commit range created, continuing the
		214	// existing sequence. An id the index already knows keeps its number; this is
		215	// what makes a deleted-then-re-added entity cheap and stable.
		216	func addNumbers(tx *sql.Tx, repoID string, additions []metadata.Entity) error {
		217	rows, err := tx.Query(`SELECT id, number FROM entity_numbers WHERE repo_id=?`, repoID)
		218	if err != nil {
		219	return err
		220	}
		221	seen := map[string]bool{}
		222	next := 0
		223	for rows.Next() {
		224	var id string
		225	var number int
		226	if err := rows.Scan(&id, &number); err != nil {
		227	rows.Close()
		228	return err
		229	}
		230	seen[id] = true
		231	if number > next {
		232	next = number
	101	233	}
	102	234	}
	103		if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, sha, m.ID); err != nil {
		235	if err := rows.Err(); err != nil {
		236	rows.Close()
	104	237	return err
	105	238	}
	106		return tx.Commit()
		239	rows.Close()
		240	for _, e := range additions {
		241	if seen[e.ID] {
		242	continue
		243	}
		244	next++
		245	if err := insertNumber(tx, repoID, next, e.Kind, e.ID); err != nil {
		246	return err
		247	}
		248	seen[e.ID] = true
		249	}
		250	return nil
	107	251	}
	108	252
	109	253	// Issues returns a repo's indexed issues, newest first.
@@ -174,18 +318,16 @@ func (s *Store) Issues(repoID string, f Filter) ([]IssueRow, error) {
	174	318	}
	175	319
	176	320	// LoadIssue reads one issue straight from git, so the rendered body and comments
	177		// are always the committed ones.
		321	// are always the committed ones. Only that issue's files are read.
	178	322	func LoadIssue(m repo.Meta, id string) (metadata.Issue, error) {
	179		list, err := metadata.LoadIssues(m.Git(), metadata.Ref)
		323	iss, found, err := metadata.LoadIssueAt(m.Git(), metadata.Ref, id)
	180	324	if err != nil {
	181	325	return nil, err
	182	326	}
	183		for _, iss := range list {
	184		if iss.ID == id {
	185		return iss, nil
	186		}
		327	if !found {
		328	return nil, ErrNotFound
	187	329	}
	188		return nil, ErrNotFound
		330	return iss, nil
	189	331	}
	190	332
	191	333	var ErrNotFound = errNotFound{}