vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
7f547ca3
Keenan Nemetz
10d ago
|
1 | package index |
| 2 | ||
| 3 | // Refreshing the derived rows from the metadata branch. | |
| 4 | // | |
| 5 | // Ensure runs on read. It resolves the metadata ref, compares it with the commit | |
| 6 | // the rows were built from, and applies the difference when the branch moved | |
| 7 | // forward. The branch is the source of truth, so everything here can be thrown | |
| 8 | // away and rebuilt; the index only has to agree with one commit at a time. | |
| 9 | ||
| 10 | import ( | |
| 11 | "strings" | |
| 12 | ||
|
d37d301f
Keenan Nemetz
16h ago
|
13 | "rodent.software/vole/core/git" |
| 14 | "rodent.software/vole/forge/repo" | |
| 15 | "rodent.software/vole/metadata" | |
|
7f547ca3
Keenan Nemetz
10d ago
|
16 | ) |
| 17 | ||
| 18 | // Ensure refreshes everything a repo derives from its metadata branch: issues, | |
| 19 | // change requests, their comments and sets, the release artifacts, and the `#42` | |
| 20 | // numbering. Called on read, so a push is visible without a hook or a full | |
| 21 | // rebuild. | |
| 22 | // | |
| 23 | // A branch that moved forward is refreshed from the paths that changed. The full | |
| 24 | // read is reserved for the cases that need it: the first read, a rewritten | |
| 25 | // branch, or an incremental refresh that cannot be trusted. | |
| 26 | func (s *Store) Ensure(m *repo.Meta) error { | |
|
dced06da
Keenan Nemetz
9d ago
|
27 | current, err := m.Git().RefCommit(metadata.Ref) |
|
7f547ca3
Keenan Nemetz
10d ago
|
28 | if err != nil { |
| 29 | return err | |
| 30 | } | |
| 31 | return s.EnsureAt(m, current) | |
| 32 | } | |
| 33 | ||
| 34 | // EnsureAt refreshes the derived rows to the metadata commit the caller already | |
| 35 | // resolved. It is Ensure for a caller that needs the commit for something else | |
| 36 | // too, so one page does not resolve the same ref twice. | |
| 37 | func (s *Store) EnsureAt(m *repo.Meta, current string) error { | |
| 38 | var stored string | |
| 39 | err := s.db.QueryRow(`SELECT issues_sha FROM repos WHERE id=?`, m.ID).Scan(&stored) | |
| 40 | if err != nil { | |
| 41 | return err | |
| 42 | } | |
| 43 | if stored == current { | |
| 44 | return nil | |
| 45 | } | |
| 46 | if stored != "" && current != "" { | |
| 47 | if ok, err := m.Git().IsAncestor(stored, current); err == nil && ok { | |
| 48 | if err := s.indexFrom(m, stored, current); err == nil { | |
| 49 | return nil | |
| 50 | } | |
| 51 | // The delta could not be applied, so the rows and the stored commit | |
| 52 | // would disagree. Fall through to the reconstruction that defines the | |
| 53 | // index. | |
| 54 | } | |
| 55 | } | |
| 56 | return s.IndexRepo(m) | |
| 57 | } | |
| 58 | ||
| 59 | // EnsureIssues keeps the older name used by the issue surfaces and the tests. It | |
| 60 | // is Ensure: the refresh covers every table, because they are derived together. | |
| 61 | func (s *Store) EnsureIssues(m *repo.Meta) error { return s.Ensure(m) } | |
| 62 | ||
| 63 | // IndexRepo rebuilds every derived row for one repository from the metadata | |
| 64 | // branch. It is the definition of the index: the incremental path must produce | |
| 65 | // the same rows, and `vole admin index rebuild` is this function for every repo. | |
| 66 | func (s *Store) IndexRepo(m *repo.Meta) error { | |
| 67 | g := m.Git() | |
|
dced06da
Keenan Nemetz
9d ago
|
68 | commit, err := g.RefCommit(metadata.Ref) |
|
7f547ca3
Keenan Nemetz
10d ago
|
69 | if err != nil { |
| 70 | return err | |
| 71 | } | |
| 72 | issues, err := metadata.LoadIssuesAt(g, commit) | |
| 73 | if err != nil { | |
| 74 | return err | |
| 75 | } | |
| 76 | requests, err := metadata.LoadRequestsAt(g, commit) | |
| 77 | if err != nil { | |
| 78 | return err | |
| 79 | } | |
| 80 | numbers, err := metadata.LoadNumbersAt(g, commit) | |
| 81 | if err != nil { | |
| 82 | return err | |
| 83 | } | |
| 84 | artifacts, err := metadata.LoadArtifactsAt(g, commit) | |
| 85 | if err != nil { | |
| 86 | return err | |
| 87 | } | |
|
4d938fe8
Keenan Nemetz
9d ago
|
88 | // State changes are the metadata commits' subjects, so a rebuild reads the |
| 89 | // whole branch and takes every one of them. A repository that never pushed | |
| 90 | // metadata has no branch to read, which is not an error. | |
| 91 | var stateCommits []git.Commit | |
| 92 | if commit != "" { | |
| 93 | if stateCommits, err = g.Log(metadata.Ref, 0); err != nil { | |
| 94 | return err | |
| 95 | } | |
| 96 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
97 | tags, err := g.Tags() |
| 98 | if err != nil { | |
| 99 | return err | |
| 100 | } | |
|
7cc36ba0
Keenan Nemetz
4d ago
|
101 | // One `git log` dates every issue's and request's last activity. The branch has |
| 102 | // no updated field to read, and a process per entity would make a rebuild cost | |
| 103 | // the queue. | |
|
db4081ec
Keenan Nemetz
4d ago
|
104 | touchTimes, err := issueTouchTimes(g, commit) |
| 105 | if err != nil { | |
| 106 | return err | |
| 107 | } | |
|
7cc36ba0
Keenan Nemetz
4d ago
|
108 | requestTimes, err := requestTouchTimes(g, commit) |
| 109 | if err != nil { | |
| 110 | return err | |
| 111 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
112 | |
| 113 | tx, err := s.db.Begin() | |
| 114 | if err != nil { | |
| 115 | return err | |
| 116 | } | |
| 117 | defer tx.Rollback() | |
| 118 | if err := deleteRepoRows(tx, m.ID); err != nil { | |
| 119 | return err | |
| 120 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
121 | // State changes come before the entities whose events they produce. |
| 122 | if err := insertStateChanges(tx, m.ID, stateCommits); err != nil { | |
| 123 | return err | |
| 124 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
125 | for number := 1; number <= numbers.Len(); number++ { |
| 126 | id, kind, ok := numbers.At(number) | |
| 127 | if !ok { | |
| 128 | continue | |
| 129 | } | |
| 130 | if err := insertNumber(tx, m.ID, number, kind, id); err != nil { | |
| 131 | return err | |
| 132 | } | |
| 133 | } | |
| 134 | for _, iss := range issues { | |
|
db4081ec
Keenan Nemetz
4d ago
|
135 | if err := insertIssue(tx, m.ID, iss, touchTimes[iss.ID]); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
136 | return err |
| 137 | } | |
|
6762a520
Keenan Nemetz
10d ago
|
138 | if err := replaceComments(tx, m.ID, commentKindIssue, iss.ID, iss.Title, iss.Comments); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
139 | return err |
| 140 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
141 | if _, _, err := insertIssueEvents(tx, m.ID, iss); err != nil { |
| 142 | return err | |
| 143 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
144 | } |
| 145 | for _, req := range requests { | |
|
7cc36ba0
Keenan Nemetz
4d ago
|
146 | if err := insertRequest(tx, m.ID, req, requestTimes[req.ID]); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
147 | return err |
| 148 | } | |
|
6762a520
Keenan Nemetz
10d ago
|
149 | if err := replaceComments(tx, m.ID, commentKindRequest, req.ID, req.Title, req.Comments); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
150 | return err |
| 151 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
152 | if _, _, err := insertRequestEvents(tx, m.ID, req); err != nil { |
| 153 | return err | |
| 154 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
155 | } |
| 156 | if err := replaceArtifacts(tx, m.ID, artifacts); err != nil { | |
| 157 | return err | |
| 158 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
159 | if _, _, err := replaceReleaseEvents(tx, m.ID, tags); err != nil { |
| 160 | return err | |
| 161 | } | |
| 162 | if _, err := tx.Exec(`UPDATE repos SET events_count=(SELECT COUNT(*) FROM events WHERE repo_id=?) WHERE id=?`, | |
| 163 | m.ID, m.ID); err != nil { | |
|
4d938fe8
Keenan Nemetz
9d ago
|
164 | return err |
| 165 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
166 | if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, commit, m.ID); err != nil { |
| 167 | return err | |
| 168 | } | |
|
8dc56e4e
Keenan Nemetz
9d ago
|
169 | if err := tx.Commit(); err != nil { |
| 170 | return err | |
| 171 | } | |
|
a32a859a
Keenan Nemetz
4d ago
|
172 | // The metadata rebuild is a good moment to recompute what the overview reads |
| 173 | // from git: the head and its tree, and the tags. A failure leaves a section | |
| 174 | // empty, not a broken index; the next push or rebuild fills it. | |
|
2a8f7cef
Keenan Nemetz
9d ago
|
175 | _, _ = s.RefreshHead(m) |
| 176 | _ = s.RefreshTags(m) | |
|
8dc56e4e
Keenan Nemetz
9d ago
|
177 | return nil |
|
7f547ca3
Keenan Nemetz
10d ago
|
178 | } |
| 179 | ||
| 180 | // indexFrom applies the difference between two commits to the index. The caller | |
| 181 | // has already established that from is an ancestor of to, so every entity that | |
| 182 | // exists at to is either unchanged, changed in the range, or new. | |
| 183 | func (s *Store) indexFrom(m *repo.Meta, from, to string) error { | |
| 184 | g := m.Git() | |
| 185 | changes, err := g.DiffStatusPaths(from, to, | |
|
b492b557
Keenan Nemetz
3d ago
|
186 | metadata.IssuesDir, metadata.ChangesDir, metadata.ReleasesDir) |
|
7f547ca3
Keenan Nemetz
10d ago
|
187 | if err != nil { |
| 188 | return err | |
| 189 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
190 | // The commits in the range carry the state changes. They also name entities |
| 191 | // whose events must be rebuilt even when the tree diff has nothing to show: a | |
| 192 | // close followed by a reopen inside the range leaves the file where it | |
| 193 | // started, and an endpoint diff cannot see the two events in between. | |
| 194 | stateCommits, err := g.CommitsBetween(from, to, 0) | |
| 195 | if err != nil { | |
| 196 | return err | |
| 197 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
198 | issueIDs := map[string]bool{} |
| 199 | requestIDs := map[string]bool{} | |
|
bca9ec24
Keenan Nemetz
9d ago
|
200 | for _, c := range stateCommits { |
| 201 | kind, id, _, ok := metadata.ParseStateSubject(c.Subject) | |
| 202 | if !ok { | |
| 203 | continue | |
| 204 | } | |
| 205 | switch kind { | |
| 206 | case "issue": | |
| 207 | issueIDs[id] = true | |
| 208 | case "request": | |
| 209 | requestIDs[id] = true | |
| 210 | } | |
| 211 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
212 | releasesChanged := false |
| 213 | for path := range changes { | |
| 214 | switch { | |
| 215 | case strings.HasPrefix(path, metadata.IssuesDir+"/"): | |
| 216 | if id, ok := metadata.IssueIDFromPath(path); ok { | |
| 217 | issueIDs[id] = true | |
| 218 | } | |
|
b492b557
Keenan Nemetz
3d ago
|
219 | case strings.HasPrefix(path, metadata.ChangesDir+"/"): |
|
7f547ca3
Keenan Nemetz
10d ago
|
220 | if id, ok := metadata.RequestIDFromPath(path); ok { |
| 221 | requestIDs[id] = true | |
| 222 | } | |
| 223 | case strings.HasPrefix(path, metadata.ReleasesDir+"/"): | |
| 224 | releasesChanged = true | |
| 225 | } | |
| 226 | } | |
| 227 | ||
|
7cc36ba0
Keenan Nemetz
4d ago
|
228 | // The commits also date each entity's last activity, which is what the lists |
| 229 | // sort by. They name every issue and request the range touched, including one | |
| 230 | // edited and then edited back: the tree diff cannot see those, but the entity | |
| 231 | // still moved. | |
|
db4081ec
Keenan Nemetz
4d ago
|
232 | touchTimes, err := issueTouchTimes(g, from+".."+to) |
| 233 | if err != nil { | |
| 234 | return err | |
| 235 | } | |
| 236 | for id := range touchTimes { | |
| 237 | issueIDs[id] = true | |
| 238 | } | |
|
7cc36ba0
Keenan Nemetz
4d ago
|
239 | requestTimes, err := requestTouchTimes(g, from+".."+to) |
| 240 | if err != nil { | |
| 241 | return err | |
| 242 | } | |
| 243 | for id := range requestTimes { | |
| 244 | requestIDs[id] = true | |
| 245 | } | |
|
db4081ec
Keenan Nemetz
4d ago
|
246 | |
|
7f547ca3
Keenan Nemetz
10d ago
|
247 | // Read every changed entity before opening the write transaction. `to` is a |
| 248 | // commit, so a concurrent push cannot make the rows describe two tree states. | |
| 249 | issues := make(map[string]*metadata.Issue, len(issueIDs)) | |
| 250 | for id := range issueIDs { | |
| 251 | iss, found, err := metadata.LoadIssueAtCommit(g, to, id) | |
| 252 | if err != nil { | |
| 253 | return err | |
| 254 | } | |
| 255 | if found { | |
| 256 | issues[id] = iss | |
| 257 | } | |
| 258 | } | |
| 259 | requests := make(map[string]*metadata.Request, len(requestIDs)) | |
| 260 | for id := range requestIDs { | |
| 261 | req, found, err := metadata.LoadRequestAt(g, to, id) | |
| 262 | if err != nil { | |
| 263 | return err | |
| 264 | } | |
| 265 | if found { | |
| 266 | requests[id] = req | |
| 267 | } | |
| 268 | } | |
| 269 | var artifacts map[string][]metadata.Artifact | |
| 270 | if releasesChanged { | |
| 271 | if artifacts, err = metadata.LoadArtifactsAt(g, to); err != nil { | |
| 272 | return err | |
| 273 | } | |
| 274 | } | |
| 275 | ||
| 276 | // Numbers are historical: a deleted entity keeps its number, and a creation | |
| 277 | // that was deleted again inside the range still consumed one. The additions | |
| 278 | // are therefore taken from the commit range, not from the current tree. | |
|
b492b557
Keenan Nemetz
3d ago
|
279 | groups, err := g.AddedPathsPerCommit(from+".."+to, metadata.IssuesDir, metadata.ChangesDir) |
|
7f547ca3
Keenan Nemetz
10d ago
|
280 | if err != nil { |
| 281 | return err | |
| 282 | } | |
| 283 | ||
| 284 | tx, err := s.db.Begin() | |
| 285 | if err != nil { | |
| 286 | return err | |
| 287 | } | |
| 288 | defer tx.Rollback() | |
|
bca9ec24
Keenan Nemetz
9d ago
|
289 | // State changes come first: an entity's events are rebuilt from them. |
| 290 | if err := insertStateChanges(tx, m.ID, stateCommits); err != nil { | |
| 291 | return err | |
| 292 | } | |
| 293 | eventDelta := 0 | |
|
7f547ca3
Keenan Nemetz
10d ago
|
294 | for id := range issueIDs { |
| 295 | if _, found := issues[id]; found { | |
| 296 | continue | |
| 297 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
298 | deleted, err := deleteEventsForEntity(tx, m.ID, "issue", id) |
| 299 | if err != nil { | |
| 300 | return err | |
| 301 | } | |
| 302 | eventDelta -= deleted | |
|
7f547ca3
Keenan Nemetz
10d ago
|
303 | if err := deleteIssue(tx, m.ID, id); err != nil { |
| 304 | return err | |
| 305 | } | |
| 306 | } | |
| 307 | for _, iss := range issues { | |
| 308 | if err := deleteIssue(tx, m.ID, iss.ID); err != nil { | |
| 309 | return err | |
| 310 | } | |
|
db4081ec
Keenan Nemetz
4d ago
|
311 | if err := insertIssue(tx, m.ID, iss, touchTimes[iss.ID]); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
312 | return err |
| 313 | } | |
|
6762a520
Keenan Nemetz
10d ago
|
314 | if err := replaceComments(tx, m.ID, commentKindIssue, iss.ID, iss.Title, iss.Comments); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
315 | return err |
| 316 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
317 | inserted, deleted, err := insertIssueEvents(tx, m.ID, iss) |
| 318 | if err != nil { | |
| 319 | return err | |
| 320 | } | |
| 321 | eventDelta += inserted - deleted | |
|
7f547ca3
Keenan Nemetz
10d ago
|
322 | } |
| 323 | for id := range requestIDs { | |
| 324 | if _, found := requests[id]; found { | |
| 325 | continue | |
| 326 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
327 | deleted, err := deleteEventsForEntity(tx, m.ID, "change", id) |
| 328 | if err != nil { | |
| 329 | return err | |
| 330 | } | |
| 331 | eventDelta -= deleted | |
|
7f547ca3
Keenan Nemetz
10d ago
|
332 | if err := deleteRequest(tx, m.ID, id); err != nil { |
| 333 | return err | |
| 334 | } | |
| 335 | } | |
| 336 | for _, req := range requests { | |
| 337 | if err := deleteRequest(tx, m.ID, req.ID); err != nil { | |
| 338 | return err | |
| 339 | } | |
|
7cc36ba0
Keenan Nemetz
4d ago
|
340 | if err := insertRequest(tx, m.ID, req, requestTimes[req.ID]); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
341 | return err |
| 342 | } | |
|
6762a520
Keenan Nemetz
10d ago
|
343 | if err := replaceComments(tx, m.ID, commentKindRequest, req.ID, req.Title, req.Comments); err != nil { |
|
7f547ca3
Keenan Nemetz
10d ago
|
344 | return err |
| 345 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
346 | inserted, deleted, err := insertRequestEvents(tx, m.ID, req) |
| 347 | if err != nil { | |
| 348 | return err | |
| 349 | } | |
| 350 | eventDelta += inserted - deleted | |
|
7f547ca3
Keenan Nemetz
10d ago
|
351 | } |
| 352 | if releasesChanged { | |
| 353 | if err := replaceArtifacts(tx, m.ID, artifacts); err != nil { | |
| 354 | return err | |
| 355 | } | |
| 356 | } | |
|
bca9ec24
Keenan Nemetz
9d ago
|
357 | if err := bumpEventCount(tx, m.ID, eventDelta); err != nil { |
|
4d938fe8
Keenan Nemetz
9d ago
|
358 | return err |
| 359 | } | |
|
7f547ca3
Keenan Nemetz
10d ago
|
360 | if err := addNumbers(tx, m.ID, metadata.Additions(groups)); err != nil { |
| 361 | return err | |
| 362 | } | |
| 363 | if _, err := tx.Exec(`UPDATE repos SET issues_sha=? WHERE id=?`, to, m.ID); err != nil { | |
| 364 | return err | |
| 365 | } | |
| 366 | return tx.Commit() | |
| 367 | } | |
| 368 | ||
| 369 | // deleteRepoRows removes every derived row for one repository. It is what makes | |
| 370 | // the full rebuild a replacement rather than a merge, and it is why a deleted | |
| 371 | // repository cannot leave rows behind. | |
| 372 | func deleteRepoRows(tx execer, repoID string) error { | |
| 373 | for _, q := range []string{ | |
| 374 | `DELETE FROM issues WHERE repo_id=?`, | |
| 375 | `DELETE FROM issue_labels WHERE repo_id=?`, | |
| 376 | `DELETE FROM issue_assignees WHERE repo_id=?`, | |
| 377 | `DELETE FROM requests WHERE repo_id=?`, | |
| 378 | `DELETE FROM request_closes WHERE repo_id=?`, | |
| 379 | `DELETE FROM request_approvals WHERE repo_id=?`, | |
| 380 | `DELETE FROM comments WHERE repo_id=?`, | |
| 381 | `DELETE FROM artifacts WHERE repo_id=?`, | |
|
4d938fe8
Keenan Nemetz
9d ago
|
382 | `DELETE FROM state_changes WHERE repo_id=?`, |
|
bca9ec24
Keenan Nemetz
9d ago
|
383 | `DELETE FROM events WHERE repo_id=?`, |
|
7f547ca3
Keenan Nemetz
10d ago
|
384 | `DELETE FROM entity_numbers WHERE repo_id=?`, |
| 385 | } { | |
| 386 | if _, err := tx.Exec(q, repoID); err != nil { | |
| 387 | return err | |
| 388 | } | |
| 389 | } | |
| 390 | return nil | |
| 391 | } |