vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Store the overview's summaries when a push lands
2a8f7cef47240564179a45126863d1ccd8d9d362 · Keenan Nemetz · 10d ago · parent 8dc56e4e
Store the overview's summaries when a push lands The repository overview walked git on every read: `rev-parse` for the tip, `log` for the last five commits, `for-each-ref` for tags, `ls-tree` plus `shortlog` for languages and contributors, and a `cat-file` for the README. All of those describe a ref except the README, and a ref moves only when a push lands, so the push now records them. - Schema v10 adds `repo_languages`, `repo_commits`, `repo_contributors` and `repo_tags`. - `RefreshHead` stores the tip's newest commits and its language breakdown from one `ls-tree -r -l`, and returns whether the tip moved. - `RefreshContributors` stores the top authors; the hook calls it only when the tip changed, never on a metadata or tag push. - `RefreshTags` stores the tag rows beside the release events it already rewrote, and `IndexRepo` now refreshes tags and contributors too, so a rebuild restores everything a push would have written. - The overview reads those rows when the recorded head is the branch tip (`RefCommit` is a loose-ref file read) and falls back to git when it is not, so a missed hook shows the correct page rather than a stale one. If one stored section is empty on a repository with commits, that section alone is re-read from git. - The sidebar asks SQL for five open issues and changes instead of sorting the whole open queue in memory. - The rendered README is cached by repository, commit and ref under a byte budget, so the overview's last git spawn is gone after the first read of a commit. - The language fallback reads one `ls-tree -r -l` instead of `ls-tree` followed by a `cat-file --batch-check`. - `Remove` deletes every derived row for a repository, not only its row. Load harness at concurrency 4: overview goes from 473 to about 740 req/s, p50 from 8 ms to 5 ms, and the server runs no git child process for the warm requests.
forge/index/head.go +103−24
| @@ -1,11 +1,11 @@ | |||
| 1 | 1 | package index | |
| 2 | 2 | ||
| 3 | // The head summary: a repository's default branch tip and the language it is | ||
| 4 | // mostly made of. | ||
| 3 | // The head summary: a repository's default branch tip, the language it is mostly | ||
| 4 | // made of, its newest commits, and (in RefreshContributors) its authors. | ||
| 5 | 5 | // | |
| 6 | // The landing page showed these from git on every request: `symbolic-ref`, | ||
| 7 | // `rev-parse`, `log -1` and `ls-tree -r` per card. They change only when a push | ||
| 8 | // moves the default branch, so the push records them and a page reads them. | ||
| 6 | // The landing and overview pages showed these from git on every request, and the | ||
| 7 | // history and tree walks are the expensive part. They change only when a push | ||
| 8 | // moves the default branch, so the push records them and a page reads rows. | ||
| 9 | 9 | ||
| 10 | 10 | import ( | |
| 11 | 11 | "github.com/rodent-software/vole/forge/repo" | |
| @@ -21,11 +21,12 @@ type headSummary struct { | |||
| 21 | 21 | LangClass string | |
| 22 | 22 | } | |
| 23 | 23 | ||
| 24 | // RefreshHead recomputes a repository's default branch summary. It runs when a | ||
| 25 | // push lands (post-receive), on a rebuild, and after the default branch changes. | ||
| 26 | // If the recorded SHA is still the branch tip, nothing is written: a tag or | ||
| 24 | // RefreshHead recomputes a repository's default branch summary, its newest | ||
| 25 | // commits and its language breakdown. It runs when a push lands (post-receive), | ||
| 26 | // on a rebuild, and after the default branch changes. If the recorded SHA is | ||
| 27 | // still the branch tip, nothing is written and changed is false: a tag or | ||
| 27 | 28 | // metadata push then costs one rev-parse, not a tree walk. | |
| 28 | func (s *Store) RefreshHead(m *repo.Meta) error { | ||
| 29 | func (s *Store) RefreshHead(m *repo.Meta) (changed bool, err error) { | ||
| 29 | 30 | branch := m.DefaultBranch | |
| 30 | 31 | if branch == "" { | |
| 31 | 32 | // The hook only knows the directory and the id; the branch is the row's. | |
| @@ -42,28 +43,106 @@ func (s *Store) RefreshHead(m *repo.Meta) error { | |||
| 42 | 43 | } | |
| 43 | 44 | var stored string | |
| 44 | 45 | if err := s.db.QueryRow(`SELECT head_sha FROM repos WHERE id=?`, m.ID).Scan(&stored); err != nil { | |
| 45 | return err | ||
| 46 | return false, err | ||
| 46 | 47 | } | |
| 47 | 48 | if stored == sha { | |
| 48 | return nil | ||
| 49 | return false, nil | ||
| 49 | 50 | } | |
| 50 | if sha == "" { | ||
| 51 | // The branch was deleted (or the repository still has no commits). | ||
| 52 | return s.setHead(m.ID, headSummary{}) | ||
| 51 | |||
| 52 | var h headSummary | ||
| 53 | var commits []CommitRow | ||
| 54 | var shares []lang.Share | ||
| 55 | if sha != "" { | ||
| 56 | h.SHA = sha | ||
| 57 | if list, err := m.Git().Log(sha, 5); err == nil { | ||
| 58 | for i, c := range list { | ||
| 59 | if i == 0 { | ||
| 60 | h.Subject, h.Author, h.When = c.Subject, c.Author, c.When | ||
| 61 | } | ||
| 62 | commits = append(commits, CommitRow{SHA: c.SHA, Subject: c.Subject, Author: c.Author, When: c.When}) | ||
| 63 | } | ||
| 64 | } | ||
| 65 | if sizes, err := m.Git().LsTreeSizes(sha); err == nil { | ||
| 66 | shares = lang.Breakdown(sizes) | ||
| 67 | h.Language, h.LangClass = lang.TopSizes(sizes) | ||
| 68 | } | ||
| 53 | 69 | } | |
| 54 | h := headSummary{SHA: sha} | ||
| 55 | if commits, err := m.Git().Log(sha, 1); err == nil && len(commits) > 0 { | ||
| 56 | h.Subject, h.Author, h.When = commits[0].Subject, commits[0].Author, commits[0].When | ||
| 70 | return true, s.writeHead(m.ID, h, shares, commits) | ||
| 71 | } | ||
| 72 | |||
| 73 | // RefreshContributors records the authors of the default branch's history. It | ||
| 74 | // walks every commit, so it runs once per pushed tip and on a rebuild, never on | ||
| 75 | // a metadata refresh. | ||
| 76 | func (s *Store) RefreshContributors(m *repo.Meta) error { | ||
| 77 | var sha string | ||
| 78 | if err := s.db.QueryRow(`SELECT head_sha FROM repos WHERE id=?`, m.ID).Scan(&sha); err != nil { | ||
| 79 | return err | ||
| 57 | 80 | } | |
| 58 | if tree, err := m.Git().LsTreeR(sha); err == nil { | ||
| 59 | h.Language, h.LangClass = lang.Top(tree) | ||
| 81 | var list []ContributorRow | ||
| 82 | if sha != "" { | ||
| 83 | authors, err := m.Git().Contributors(sha) | ||
| 84 | if err != nil { | ||
| 85 | return err | ||
| 86 | } | ||
| 87 | for i, c := range authors { | ||
| 88 | if i == 8 { | ||
| 89 | break | ||
| 90 | } | ||
| 91 | list = append(list, ContributorRow{Name: c.Name, Commits: c.Commits}) | ||
| 92 | } | ||
| 93 | } | ||
| 94 | tx, err := s.db.Begin() | ||
| 95 | if err != nil { | ||
| 96 | return err | ||
| 97 | } | ||
| 98 | defer tx.Rollback() | ||
| 99 | if _, err := tx.Exec(`DELETE FROM repo_contributors WHERE repo_id=?`, m.ID); err != nil { | ||
| 100 | return err | ||
| 60 | 101 | } | |
| 61 | return s.setHead(m.ID, h) | ||
| 102 | for i, c := range list { | ||
| 103 | if _, err := tx.Exec( | ||
| 104 | `INSERT INTO repo_contributors(repo_id, pos, name, commits) VALUES(?,?,?,?)`, | ||
| 105 | m.ID, i, c.Name, c.Commits); err != nil { | ||
| 106 | return err | ||
| 107 | } | ||
| 108 | } | ||
| 109 | return tx.Commit() | ||
| 62 | 110 | } | |
| 63 | 111 | ||
| 64 | func (s *Store) setHead(repoID string, h headSummary) error { | ||
| 65 | _, err := s.db.Exec( | ||
| 112 | // writeHead replaces the head row and the summaries that belong to it in one | ||
| 113 | // transaction, so a reader never sees a new tip with an old tree. | ||
| 114 | func (s *Store) writeHead(repoID string, h headSummary, shares []lang.Share, commits []CommitRow) error { | ||
| 115 | tx, err := s.db.Begin() | ||
| 116 | if err != nil { | ||
| 117 | return err | ||
| 118 | } | ||
| 119 | defer tx.Rollback() | ||
| 120 | if _, err := tx.Exec( | ||
| 66 | 121 | `UPDATE repos SET head_sha=?, head_subject=?, head_author=?, head_when=?, language=?, lang_class=? WHERE id=?`, | |
| 67 | h.SHA, h.Subject, h.Author, h.When, h.Language, h.LangClass, repoID) | ||
| 68 | return err | ||
| 122 | h.SHA, h.Subject, h.Author, h.When, h.Language, h.LangClass, repoID); err != nil { | ||
| 123 | return err | ||
| 124 | } | ||
| 125 | // The old summaries describe the old tip. The authors are refilled by | ||
| 126 | // RefreshContributors; clearing them here means a failure there shows a | ||
| 127 | // missing section, not yesterday's authors under today's commit. | ||
| 128 | for _, table := range []string{"repo_languages", "repo_commits", "repo_contributors"} { | ||
| 129 | if _, err := tx.Exec(`DELETE FROM `+table+` WHERE repo_id=?`, repoID); err != nil { | ||
| 130 | return err | ||
| 131 | } | ||
| 132 | } | ||
| 133 | for _, sh := range shares { | ||
| 134 | if _, err := tx.Exec( | ||
| 135 | `INSERT INTO repo_languages(repo_id, name, bytes) VALUES(?,?,?)`, | ||
| 136 | repoID, sh.Name, sh.Bytes); err != nil { | ||
| 137 | return err | ||
| 138 | } | ||
| 139 | } | ||
| 140 | for i, c := range commits { | ||
| 141 | if _, err := tx.Exec( | ||
| 142 | `INSERT INTO repo_commits(repo_id, pos, sha, subject, author, when_at) VALUES(?,?,?,?,?,?)`, | ||
| 143 | repoID, i, c.SHA, c.Subject, c.Author, c.When); err != nil { | ||
| 144 | return err | ||
| 145 | } | ||
| 146 | } | ||
| 147 | return tx.Commit() | ||
| 69 | 148 | } | |