Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Store the overview's summaries when a push lands

2a8f7cef47240564179a45126863d1ccd8d9d362 · Keenan Nemetz · 10d ago · parent 8dc56e4e 

Store the overview's summaries when a push lands

The repository overview walked git on every read: `rev-parse` for the
tip, `log` for the last five commits, `for-each-ref` for tags, `ls-tree`
plus `shortlog` for languages and contributors, and a `cat-file` for the
README. All of those describe a ref except the README, and a ref moves
only when a push lands, so the push now records them.

- Schema v10 adds `repo_languages`, `repo_commits`, `repo_contributors`
  and `repo_tags`.
- `RefreshHead` stores the tip's newest commits and its language
  breakdown from one `ls-tree -r -l`, and returns whether the tip moved.
- `RefreshContributors` stores the top authors; the hook calls it only
  when the tip changed, never on a metadata or tag push.
- `RefreshTags` stores the tag rows beside the release events it already
  rewrote, and `IndexRepo` now refreshes tags and contributors too, so a
  rebuild restores everything a push would have written.
- The overview reads those rows when the recorded head is the branch tip
  (`RefCommit` is a loose-ref file read) and falls back to git when it is
  not, so a missed hook shows the correct page rather than a stale one.
  If one stored section is empty on a repository with commits, that
  section alone is re-read from git.
- The sidebar asks SQL for five open issues and changes instead of
  sorting the whole open queue in memory.
- The rendered README is cached by repository, commit and ref under a
  byte budget, so the overview's last git spawn is gone after the first
  read of a commit.
- The language fallback reads one `ls-tree -r -l` instead of `ls-tree`
  followed by a `cat-file --batch-check`.
- `Remove` deletes every derived row for a repository, not only its row.

Load harness at concurrency 4: overview goes from 473 to about 740 req/s,
p50 from 8 ms to 5 ms, and the server runs no git child process for the
warm requests.

server/web/overview.go +108−45

@@ -62,27 +62,11 @@ func (w *Web) Overview(rw http.ResponseWriter, r *http.Request) {
62 62 }
63 63 g := m.Git()
64 64 ref := m.DefaultBranch
65 commit, _ := g.RevParse(ref)
65 // A loose branch is a file read. RefCommit still falls back to rev-parse for
66 // a packed ref, and the answer is what the index's summary is checked against.
67 commit, _ := g.RefCommit("refs/heads/" + ref)
66 68
67 var commits []git.Commit
68 var languages []languageShare
69 if commit != "" {
70 commits, _ = g.Log(commit, 5)
71 languages = w.languages(g, commit)
72 }
73
74 var tags []git.Tag
75 if list, err := g.Tags(); err == nil {
76 if len(list) > 5 {
77 list = list[:5]
78 }
79 tags = list
80 }
81
82 contributors := w.contributors(g, commit)
83 if len(contributors) > 8 {
84 contributors = contributors[:8]
85 }
69 commits, languages, contributors, tags := w.overviewSummaries(memoOf(r), m, commit)
86 70
87 71 var recentRuns []*runs.Run
88 72 if w.Runs != nil {
@@ -124,6 +108,82 @@ func (w *Web) Overview(rw http.ResponseWriter, r *http.Request) {
124 108 }, 0)
125 109 }
126 110
111 // overviewSummaries returns the parts of the overview that describe the default
112 // branch: its newest commits, its languages, its authors, and the repository's
113 // tags. When the index has the summary for this exact tip they are rows; when it
114 // does not (no index, or a tip the push hook has not recorded) they are read from
115 // git, which is the fallback that keeps the page correct.
116 func (w *Web) overviewSummaries(memo *requestMemo, m *repo.Meta, commit string) (
117 commits []git.Commit, languages []languageShare, contributors []contributorView, tags []git.Tag) {
118 g := m.Git()
119 if w.Index != nil {
120 if err := w.ensureIndex(memo, m); err == nil {
121 if e, err := w.Index.Get(m.Name); err == nil && e.HeadSHA == commit {
122 if list, err := w.Index.RecentCommits(e.ID, 5); err == nil {
123 commits = make([]git.Commit, 0, len(list))
124 for _, c := range list {
125 commits = append(commits, git.Commit{SHA: c.SHA, Subject: c.Subject, Author: c.Author, When: c.When})
126 }
127 }
128 if shares, err := w.Index.Languages(e.ID); err == nil {
129 languages = languageShares(shares)
130 }
131 if list, err := w.Index.Contributors(e.ID, 8); err == nil {
132 contributors = make([]contributorView, 0, len(list))
133 for _, c := range list {
134 contributors = append(contributors, contributorView{
135 Name: c.Name, Commits: c.Commits, Initial: initial(c.Name), Tone: tone(c.Name),
136 })
137 }
138 }
139 if list, err := w.Index.Tags(e.ID, 5); err == nil {
140 tags = make([]git.Tag, 0, len(list))
141 for _, t := range list {
142 tags = append(tags, git.Tag{
143 Name: t.Name, OID: t.OID, Commit: t.Commit, Annotated: t.Annotated,
144 Tagger: t.Tagger, When: t.When, Subject: t.Subject,
145 })
146 }
147 }
148 // A repository with commits has at least one of each; an empty
149 // section means the summary was written without it (a tree walk or
150 // a shortlog that failed), so read that part from git rather than
151 // leave the page blank until the next push.
152 if commit != "" {
153 if len(commits) == 0 {
154 commits, _ = g.Log(commit, 5)
155 }
156 if len(languages) == 0 {
157 languages = w.languages(g, commit)
158 }
159 if len(contributors) == 0 {
160 contributors = w.contributors(g, commit)
161 if len(contributors) > 8 {
162 contributors = contributors[:8]
163 }
164 }
165 }
166 return commits, languages, contributors, tags
167 }
168 }
169 }
170 if commit != "" {
171 commits, _ = g.Log(commit, 5)
172 languages = w.languages(g, commit)
173 }
174 if list, err := g.Tags(); err == nil {
175 if len(list) > 5 {
176 list = list[:5]
177 }
178 tags = list
179 }
180 contributors = w.contributors(g, commit)
181 if len(contributors) > 8 {
182 contributors = contributors[:8]
183 }
184 return commits, languages, contributors, tags
185 }
186
127 187 // overviewRecent returns the newest open issues and change requests for the
128 188 // overview sidebar. Closed and merged records are history and already have a
129 189 // home in their own lists; the overview is a map of what is still being worked
@@ -131,7 +191,7 @@ func (w *Web) Overview(rw http.ResponseWriter, r *http.Request) {
131 191 func (w *Web) overviewRecent(memo *requestMemo, m *repo.Meta) (issues, requests []overviewEntity) {
132 192 if w.Index != nil {
133 193 if err := w.ensureIndex(memo, m); err == nil {
134 if rows, err := w.Index.Issues(m.ID, index.Filter{State: "open"}); err == nil {
194 if rows, err := w.Index.Issues(m.ID, index.Filter{State: "open", Limit: overviewSideLimit}); err == nil {
135 195 for _, row := range rows {
136 196 issues = append(issues, overviewEntity{
137 197 ID: row.ID, Title: row.Title, Author: row.Author,
@@ -139,7 +199,7 @@ func (w *Web) overviewRecent(memo *requestMemo, m *repo.Meta) (issues, requests
139 199 })
140 200 }
141 201 }
142 if rows, err := w.Index.Requests(m.ID, index.RequestFilter{State: "open"}); err == nil {
202 if rows, err := w.Index.Requests(m.ID, index.RequestFilter{State: "open", Limit: overviewSideLimit}); err == nil {
143 203 for _, row := range rows {
144 204 requests = append(requests, overviewEntity{
145 205 ID: row.ID, Title: row.Title, Author: row.Author,
@@ -187,6 +247,10 @@ func (w *Web) readmeAt(m *repo.Meta, ref, commit string) (template.HTML, string)
187 247 if commit == "" {
188 248 return "", ""
189 249 }
250 key := commitKey(m.Dir, commit) + "\x00" + ref
251 if v, ok := w.readmes.get(key); ok {
252 return v.html, v.name
253 }
190 254 for _, name := range []string{"README.md", "readme.md", "Readme.md", "README.markdown"} {
191 255 stream, err := m.Git().OpenBlobAt(commit, name)
192 256 if err != nil {
@@ -197,11 +261,13 @@ func (w *Web) readmeAt(m *repo.Meta, ref, commit string) (template.HTML, string)
197 261 stream.Close()
198 262 // Name the file and the size: a README block that is silently absent
199 263 // looks like a bug in the forge rather than a decision about the file.
200 return template.HTML(fmt.Sprintf(
264 note := template.HTML(fmt.Sprintf(
201 265 `<p class="muted">%s is %s, past the %s this page renders. <a href="/r/%s/blob/%s/%s">View the source</a>.</p>`,
202 266 template.HTMLEscapeString(name), format.Bytes(size), format.Bytes(maxReadmeBytes),
203 267 template.HTMLEscapeString(m.Name), template.HTMLEscapeString(ref),
204 template.HTMLEscapeString(name))), name
268 template.HTMLEscapeString(name)))
269 w.readmes.put(key, readmeAnswer{html: note, name: name})
270 return note, name
205 271 }
206 272 data, err := io.ReadAll(io.LimitReader(stream, maxReadmeBytes))
207 273 stream.Close()
@@ -209,39 +275,36 @@ func (w *Web) readmeAt(m *repo.Meta, ref, commit string) (template.HTML, string)
209 275 continue
210 276 }
211 277 doc := treeDoc{Repo: m.Name, Ref: ref, Path: name}
212 return w.renderTreeMarkdown(doc, string(data)), name
278 rendered := w.renderTreeMarkdown(doc, string(data))
279 w.readmes.put(key, readmeAnswer{html: rendered, name: name})
280 return rendered, name
213 281 }
282 // A repository without a README would otherwise look for four names on every
283 // visit; remember that the answer is none.
284 w.readmes.put(key, readmeAnswer{})
214 285 return "", ""
215 286 }
216 287
217 // languageShares groups the current tree by file extension and returns the
218 // largest groups. It is a rough map, not a linguist: the point is to show what a
219 // repository is mostly made of, not to classify every generated file.
220 func languageShares(blobs map[string]git.Blob) []languageShare {
221 byName := map[string]int64{}
288 // languageShares turns stored or freshly read language groups into the bar the
289 // overview draws: the six largest by bytes and everything else folded together.
290 // It is a rough map, not a linguist: the point is to show what a repository is
291 // mostly made of, not to classify every generated file.
292 func languageShares(groups []lang.Share) []languageShare {
222 293 var total int64
223 for _, b := range blobs {
224 name := lang.For(b.Path)
225 byName[name] += b.Size
226 total += b.Size
294 for _, g := range groups {
295 total += g.Bytes
227 296 }
228 297 if total == 0 {
229 298 return nil
230 299 }
231 list := make([]languageShare, 0, len(byName))
232 for name, n := range byName {
233 pct := int(n * 100 / total)
234 if pct == 0 && n > 0 {
300 list := make([]languageShare, 0, len(groups))
301 for _, g := range groups {
302 pct := int(g.Bytes * 100 / total)
303 if pct == 0 && g.Bytes > 0 {
235 304 pct = 1
236 305 }
237 list = append(list, languageShare{Name: name, Bytes: n, Percent: pct})
306 list = append(list, languageShare{Name: g.Name, Bytes: g.Bytes, Percent: pct})
238 307 }
239 sort.Slice(list, func(i, j int) bool {
240 if list[i].Bytes != list[j].Bytes {
241 return list[i].Bytes > list[j].Bytes
242 }
243 return list[i].Name < list[j].Name
244 })
245 308 if len(list) > 6 {
246 309 other := languageShare{Name: "Other"}
247 310 for _, l := range list[6:] {