Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

392 commit(s) on main

Compare History
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys keenan · 55m ago 4b8c9e1d
history blame raw 9.5 KiB
1package web
2
3// robots.txt and the sitemap.
4//
5// A crawler asks two questions: which paths are worth fetching, and where the
6// public pages are. robots.txt answers the first; the sitemap answers the second.
7// The sitemap is a tree of files rather than one document because a forge grows
8// past what one URL set may hold, and a crawler fetches only the parts it needs.
9//
10// Every URL here is built from the anonymous read rule, never from the viewer's
11// session. A sitemap is cacheable by a shared proxy, and an instance admin who
12// asks for one must not put a private repository's name into what everyone else
13// receives. Account profiles are deliberately absent: /u and /u/{name} carry
14// <meta name="robots" content="noindex">, and a robots.txt Disallow would hide
15// that request from the crawler that has to read it.
16
17import (
18 "encoding/xml"
19 "fmt"
20 "net/http"
21 "net/url"
22 "time"
23
24 "rodent.software/vole/forge/access"
25 "rodent.software/vole/forge/index"
26)
27
28const (
29 sitemapNS = "http://www.sitemaps.org/schemas/sitemap/0.9"
30 // sitemapMaxURLs is the protocol's limit for one file. A repository past it
31 // is truncated and the event is logged rather than split: the alternative is
32 // a second layer of pagination that an instance this size needs everywhere,
33 // not just here.
34 sitemapMaxURLs = 50000
35 // sitemapCacheControl lets a proxy reuse the answer while a new issue or
36 // release lands. The window is short because the sitemap is how a crawler
37 // learns about those pages in the first place.
38 sitemapCacheControl = "public, max-age=300"
39)
40
41type sitemapURLSet struct {
42 XMLName xml.Name `xml:"urlset"`
43 XMLNS string `xml:"xmlns,attr"`
44 URLs []sitemapURL `xml:"url"`
45}
46
47type sitemapURL struct {
48 Loc string `xml:"loc"`
49 LastMod string `xml:"lastmod,omitempty"`
50}
51
52type sitemapIndex struct {
53 XMLName xml.Name `xml:"sitemapindex"`
54 XMLNS string `xml:"xmlns,attr"`
55 Sitemaps []sitemapRef `xml:"sitemap"`
56}
57
58type sitemapRef struct {
59 Loc string `xml:"loc"`
60}
61
62// Robots serves GET /robots.txt. The public surface is allowed by default and the
63// account, administration, write and fragment paths are closed; a crawler that
64// follows a link into a settings form or a live fragment is wasting a request
65// that could have been a page. The sitemap line points at the index, which is
66// the discoverable list of repositories, issues, changes, releases and code.
67func (w *Web) Robots(rw http.ResponseWriter, r *http.Request) {
68 rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
69 rw.Header().Set("Cache-Control", "public, max-age=3600")
70 fmt.Fprintf(rw, `User-agent: *
71Allow: /
72
73# The forge's own pages live under /-/.
74Disallow: /-/account
75Disallow: /-/admin
76Disallow: /-/login
77Disallow: /-/register
78Disallow: /-/invite
79Disallow: /-/users
80Disallow: /-/dev/
81Disallow: /-/repos/new
82Disallow: /-/runners
83Disallow: /-/healthz
84Disallow: /-/search/
85
86# Repository writes, fragments and machine-readable responses.
87Disallow: /*.git/
88Disallow: /*/counts
89Disallow: /*/live
90Disallow: /*/raw/
91Disallow: /*/comment-form
92Disallow: /*/compare
93Disallow: /*/search
94Disallow: /*/issues/new
95Disallow: /*/issues/*/edit
96Disallow: /*/changes/new
97Disallow: /*/changes/*/edit
98Disallow: /*/changes/*/files
99Disallow: /*/changes/*/runs
100Disallow: /*/changes/*/live
101Disallow: /*/releases/new
102Disallow: /*/releases/edit
103Disallow: /*/releases/*/archive.tar
104Disallow: /*/releases/*/archive.tar.gz
105Disallow: /*/releases/*/artifacts/
106Disallow: /*/runs
107Disallow: /*/workflows/
108Disallow: /*/locks
109Disallow: /*/settings
110
111# /-/u and /-/u/{name} stay crawlable on purpose: those pages ask for noindex, and
112# a crawler cannot see that request through a Disallow line.
113Sitemap: %s/sitemap.xml
114`, w.canonicalBase(r))
115}
116
117// SitemapIndex serves GET /sitemap.xml: the instance-wide pages in one child and
118// each public repository in another. A private repository is absent, not listed
119// and refused, because the URL itself is the disclosure.
120func (w *Web) SitemapIndex(rw http.ResponseWriter, r *http.Request) {
121 base := w.canonicalBase(r)
122 entries, err := w.publicRepos()
123 if err != nil {
124 w.serverError(rw, r, "sitemap: "+err.Error())
125 return
126 }
127 sitemaps := make([]sitemapRef, 0, len(entries)+1)
128 sitemaps = append(sitemaps, sitemapRef{Loc: base + "/sitemap-pages.xml"})
129 for _, e := range entries {
130 sitemaps = append(sitemaps, sitemapRef{
131 Loc: base + "/" + url.PathEscape(e.Name) + "/sitemap.xml",
132 })
133 }
134 if len(sitemaps) > sitemapMaxURLs {
135 if w.Logger != nil {
136 w.Logger("sitemap: %d public repositories; keeping the first %d", len(entries), sitemapMaxURLs-1)
137 }
138 sitemaps = sitemaps[:sitemapMaxURLs]
139 }
140 w.writeSitemap(rw, sitemapIndex{XMLNS: sitemapNS, Sitemaps: sitemaps})
141}
142
143// SitemapPages serves GET /sitemap-pages.xml: the public destinations that do
144// not belong to a repository. The profile directory and a profile are absent;
145// they are public pages that ask not to be indexed.
146func (w *Web) SitemapPages(rw http.ResponseWriter, r *http.Request) {
147 base := w.canonicalBase(r)
148 w.writeSitemap(rw, sitemapURLSet{XMLNS: sitemapNS, URLs: []sitemapURL{
149 {Loc: base + "/"},
150 {Loc: base + "/-/activity"},
151 }})
152}
153
154// RepoSitemap serves GET /{name}/sitemap.xml. It uses the anonymous read rule,
155// not w.open, so a signed-in maintainer or admin gets the same file as everyone
156// else and a private repository's sitemap is a 404 rather than a session-scoped
157// document that a proxy could hand to a stranger.
158func (w *Web) RepoSitemap(rw http.ResponseWriter, r *http.Request) {
159 name := r.PathValue("name")
160 m, err := w.Repos.Meta(name)
161 if err != nil {
162 w.notFound(rw, r)
163 return
164 }
165 acl, err := m.ACL()
166 if err != nil || !acl.Can("", access.Read) {
167 w.notFound(rw, r)
168 return
169 }
170 if w.Index == nil {
171 w.serverError(rw, r, "sitemap: the index is unavailable")
172 return
173 }
174 entry, err := w.Index.Get(name)
175 if err != nil {
176 w.serverError(rw, r, "sitemap: "+err.Error())
177 return
178 }
179 if err := w.ensureIndex(memoOf(r), m); err != nil {
180 w.serverError(rw, r, "sitemap: cannot read metadata: "+err.Error())
181 return
182 }
183
184 base := w.canonicalBase(r)
185 root := "/" + url.PathEscape(entry.Name)
186 headMod := sitemapUnix(entry.HeadWhen)
187 urls := []sitemapURL{
188 {Loc: base + root, LastMod: headMod},
189 {Loc: base + root + "/activity", LastMod: headMod},
190 {Loc: base + root + "/code", LastMod: headMod},
191 {Loc: base + root + "/issues"},
192 {Loc: base + root + "/changes"},
193 {Loc: base + root + "/releases", LastMod: headMod},
194 }
195 if entry.DefaultBranch != "" && entry.HeadSHA != "" {
196 urls = append(urls, sitemapURL{
197 Loc: base + root + "/commits/" + url.PathEscape(entry.DefaultBranch),
198 LastMod: headMod,
199 })
200 }
201
202 // Releases first: the detail pages are the ones a search result is most
203 // likely to want, so a repository close to the URL limit keeps them.
204 tags, err := w.Index.Tags(entry.ID, 0)
205 if err != nil {
206 w.serverError(rw, r, "sitemap: cannot read releases: "+err.Error())
207 return
208 }
209 for _, t := range tags {
210 urls = append(urls, sitemapURL{
211 Loc: base + root + "/releases/" + url.PathEscape(t.Name),
212 LastMod: sitemapUnix(t.When),
213 })
214 }
215
216 issues, err := w.Index.Issues(entry.ID, index.Filter{})
217 if err != nil {
218 w.serverError(rw, r, "sitemap: cannot read issues: "+err.Error())
219 return
220 }
221 for _, iss := range issues {
222 urls = append(urls, sitemapURL{
223 Loc: base + root + "/issues/" + url.PathEscape(iss.ID),
224 LastMod: sitemapTime(iss.Updated),
225 })
226 }
227
228 requests, err := w.Index.Requests(entry.ID, index.RequestFilter{})
229 if err != nil {
230 w.serverError(rw, r, "sitemap: cannot read changes: "+err.Error())
231 return
232 }
233 for _, req := range requests {
234 urls = append(urls, sitemapURL{
235 Loc: base + root + "/changes/" + url.PathEscape(req.ID),
236 LastMod: sitemapTime(req.Updated),
237 })
238 }
239
240 if len(urls) > sitemapMaxURLs {
241 if w.Logger != nil {
242 w.Logger("sitemap: %s has %d URLs; keeping the first %d", entry.Name, len(urls), sitemapMaxURLs)
243 }
244 urls = urls[:sitemapMaxURLs]
245 }
246 w.writeSitemap(rw, sitemapURLSet{XMLNS: sitemapNS, URLs: urls})
247}
248
249// publicRepos returns the repositories an anonymous reader can open. It reads the
250// ACL rather than the request's principal, so the answer does not depend on who
251// asked.
252func (w *Web) publicRepos() ([]index.Entry, error) {
253 if w.Index == nil {
254 return nil, fmt.Errorf("the index is unavailable")
255 }
256 entries, err := w.Index.List()
257 if err != nil {
258 return nil, err
259 }
260 public := make([]index.Entry, 0, len(entries))
261 for _, e := range entries {
262 if w.Repos.Public(e.Name) {
263 public = append(public, e)
264 }
265 }
266 return public, nil
267}
268
269// writeSitemap renders a url set or an index. The body is built before the
270// headers are set so an encoding error is a 500 rather than a truncated XML
271// document that a crawler would remember as the sitemap.
272func (w *Web) writeSitemap(rw http.ResponseWriter, v any) {
273 body, err := xml.MarshalIndent(v, "", " ")
274 if err != nil {
275 if w.Logger != nil {
276 w.Logger("sitemap: %v", err)
277 }
278 http.Error(rw, "The sitemap could not be rendered.", http.StatusInternalServerError)
279 return
280 }
281 rw.Header().Set("Content-Type", "application/xml; charset=utf-8")
282 rw.Header().Set("Cache-Control", sitemapCacheControl)
283 _, _ = rw.Write([]byte(xml.Header))
284 _, _ = rw.Write(body)
285 _, _ = rw.Write([]byte("\n"))
286}
287
288// sitemapTime formats a page timestamp. An empty value is omitted by the URL
289// struct rather than written as the zero time, which is not a date a crawler can
290// compare.
291func sitemapTime(t time.Time) string {
292 if t.IsZero() {
293 return ""
294 }
295 return t.UTC().Format(time.RFC3339)
296}
297
298func sitemapUnix(sec int64) string {
299 if sec <= 0 {
300 return ""
301 }
302 return time.Unix(sec, 0).UTC().Format(time.RFC3339)
303}
304