Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Publish a robots.txt and sitemap for public pages

80300457bddaf6b90f472a336893ed5dee3c0a48 · Keenan Nemetz · 1d ago · parent c980fe53 

Publish a robots.txt and sitemap for public pages

A crawler needs to know which paths are worth fetching and where the public pages are. robots.txt allows the public surface, closes the account, administration, write, fragment and Git paths, and points at a sitemap index. The index splits the instance pages from each public repository; a repository sitemap lists its overview, activity, code browser, issues, changes, releases and default-branch commits, with lastmod on detail pages.

Visibility is the anonymous read rule, not the viewer's session: a private repository is absent from the index and its sitemap is a 404 even for an admin. Account profiles stay out of the sitemap and out of robots.txt so their existing noindex request remains visible to crawlers.

server/web/seo.go +303

@@ -0,0 +1,303 @@
1 package web
2
3 // robots.txt and the sitemap.
4 //
5 // A crawler asks two questions: which paths are worth fetching, and where the
6 // public pages are. robots.txt answers the first; the sitemap answers the second.
7 // The sitemap is a tree of files rather than one document because a forge grows
8 // past what one URL set may hold, and a crawler fetches only the parts it needs.
9 //
10 // Every URL here is built from the anonymous read rule, never from the viewer's
11 // session. A sitemap is cacheable by a shared proxy, and an instance admin who
12 // asks for one must not put a private repository's name into what everyone else
13 // receives. Account profiles are deliberately absent: /u and /u/{name} carry
14 // <meta name="robots" content="noindex">, and a robots.txt Disallow would hide
15 // that request from the crawler that has to read it.
16
17 import (
18 "encoding/xml"
19 "fmt"
20 "net/http"
21 "net/url"
22 "time"
23
24 "github.com/rodent-software/vole/forge/access"
25 "github.com/rodent-software/vole/forge/index"
26 )
27
28 const (
29 sitemapNS = "http://www.sitemaps.org/schemas/sitemap/0.9"
30 // sitemapMaxURLs is the protocol's limit for one file. A repository past it
31 // is truncated and the event is logged rather than split: the alternative is
32 // a second layer of pagination that an instance this size needs everywhere,
33 // not just here.
34 sitemapMaxURLs = 50000
35 // sitemapCacheControl lets a proxy reuse the answer while a new issue or
36 // release lands. The window is short because the sitemap is how a crawler
37 // learns about those pages in the first place.
38 sitemapCacheControl = "public, max-age=300"
39 )
40
41 type sitemapURLSet struct {
42 XMLName xml.Name `xml:"urlset"`
43 XMLNS string `xml:"xmlns,attr"`
44 URLs []sitemapURL `xml:"url"`
45 }
46
47 type sitemapURL struct {
48 Loc string `xml:"loc"`
49 LastMod string `xml:"lastmod,omitempty"`
50 }
51
52 type sitemapIndex struct {
53 XMLName xml.Name `xml:"sitemapindex"`
54 XMLNS string `xml:"xmlns,attr"`
55 Sitemaps []sitemapRef `xml:"sitemap"`
56 }
57
58 type sitemapRef struct {
59 Loc string `xml:"loc"`
60 }
61
62 // Robots serves GET /robots.txt. The public surface is allowed by default and the
63 // account, administration, write and fragment paths are closed; a crawler that
64 // follows a link into a settings form or a live fragment is wasting a request
65 // that could have been a page. The sitemap line points at the index, which is
66 // the discoverable list of repositories, issues, changes, releases and code.
67 func (w *Web) Robots(rw http.ResponseWriter, r *http.Request) {
68 rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
69 rw.Header().Set("Cache-Control", "public, max-age=3600")
70 fmt.Fprintf(rw, `User-agent: *
71 Allow: /
72
73 # Account and instance-administration surfaces.
74 Disallow: /account
75 Disallow: /admin
76 Disallow: /login
77 Disallow: /register
78 Disallow: /invite
79 Disallow: /users
80 Disallow: /dev/
81 Disallow: /repos/new
82 Disallow: /runners
83 Disallow: /healthz
84 Disallow: /search/
85
86 # Repository writes, fragments and machine-readable responses.
87 Disallow: /git/
88 Disallow: /r/*/counts
89 Disallow: /r/*/live
90 Disallow: /r/*/raw/
91 Disallow: /r/*/comment-form
92 Disallow: /r/*/compare
93 Disallow: /r/*/search
94 Disallow: /r/*/issues/new
95 Disallow: /r/*/issues/*/edit
96 Disallow: /r/*/changes/new
97 Disallow: /r/*/changes/*/edit
98 Disallow: /r/*/changes/*/files
99 Disallow: /r/*/changes/*/runs
100 Disallow: /r/*/changes/*/live
101 Disallow: /r/*/releases/new
102 Disallow: /r/*/releases/edit
103 Disallow: /r/*/releases/*/archive.tar
104 Disallow: /r/*/releases/*/archive.tar.gz
105 Disallow: /r/*/releases/*/artifacts/
106 Disallow: /r/*/runs
107 Disallow: /r/*/workflows/
108 Disallow: /r/*/locks
109 Disallow: /r/*/settings
110
111 # /u and /u/{name} stay crawlable on purpose: those pages ask for noindex, and a
112 # crawler cannot see that request through a Disallow line.
113 Sitemap: %s/sitemap.xml
114 `, w.canonicalBase(r))
115 }
116
117 // SitemapIndex serves GET /sitemap.xml: the instance-wide pages in one child and
118 // each public repository in another. A private repository is absent, not listed
119 // and refused, because the URL itself is the disclosure.
120 func (w *Web) SitemapIndex(rw http.ResponseWriter, r *http.Request) {
121 base := w.canonicalBase(r)
122 entries, err := w.publicRepos()
123 if err != nil {
124 w.serverError(rw, r, "sitemap: "+err.Error())
125 return
126 }
127 sitemaps := make([]sitemapRef, 0, len(entries)+1)
128 sitemaps = append(sitemaps, sitemapRef{Loc: base + "/sitemap-pages.xml"})
129 for _, e := range entries {
130 sitemaps = append(sitemaps, sitemapRef{
131 Loc: base + "/r/" + url.PathEscape(e.Name) + "/sitemap.xml",
132 })
133 }
134 if len(sitemaps) > sitemapMaxURLs {
135 if w.Logger != nil {
136 w.Logger("sitemap: %d public repositories; keeping the first %d", len(entries), sitemapMaxURLs-1)
137 }
138 sitemaps = sitemaps[:sitemapMaxURLs]
139 }
140 w.writeSitemap(rw, sitemapIndex{XMLNS: sitemapNS, Sitemaps: sitemaps})
141 }
142
143 // SitemapPages serves GET /sitemap-pages.xml: the public destinations that do
144 // not belong to a repository. The profile directory and a profile are absent;
145 // they are public pages that ask not to be indexed.
146 func (w *Web) SitemapPages(rw http.ResponseWriter, r *http.Request) {
147 base := w.canonicalBase(r)
148 w.writeSitemap(rw, sitemapURLSet{XMLNS: sitemapNS, URLs: []sitemapURL{
149 {Loc: base + "/"},
150 {Loc: base + "/activity"},
151 }})
152 }
153
154 // RepoSitemap serves GET /r/{name}/sitemap.xml. It uses the anonymous read rule,
155 // not w.open, so a signed-in maintainer or admin gets the same file as everyone
156 // else and a private repository's sitemap is a 404 rather than a session-scoped
157 // document that a proxy could hand to a stranger.
158 func (w *Web) RepoSitemap(rw http.ResponseWriter, r *http.Request) {
159 name := r.PathValue("name")
160 m, err := w.Repos.Meta(name)
161 if err != nil {
162 w.notFound(rw, r)
163 return
164 }
165 acl, err := m.ACL()
166 if err != nil || !acl.Can("", access.Read) {
167 w.notFound(rw, r)
168 return
169 }
170 if w.Index == nil {
171 w.serverError(rw, r, "sitemap: the index is unavailable")
172 return
173 }
174 entry, err := w.Index.Get(name)
175 if err != nil {
176 w.serverError(rw, r, "sitemap: "+err.Error())
177 return
178 }
179 if err := w.ensureIndex(memoOf(r), m); err != nil {
180 w.serverError(rw, r, "sitemap: cannot read metadata: "+err.Error())
181 return
182 }
183
184 base := w.canonicalBase(r)
185 root := "/r/" + url.PathEscape(entry.Name)
186 headMod := sitemapUnix(entry.HeadWhen)
187 urls := []sitemapURL{
188 {Loc: base + root, LastMod: headMod},
189 {Loc: base + root + "/activity", LastMod: headMod},
190 {Loc: base + root + "/code", LastMod: headMod},
191 {Loc: base + root + "/issues"},
192 {Loc: base + root + "/changes"},
193 {Loc: base + root + "/releases", LastMod: headMod},
194 }
195 if entry.DefaultBranch != "" && entry.HeadSHA != "" {
196 urls = append(urls, sitemapURL{
197 Loc: base + root + "/commits/" + url.PathEscape(entry.DefaultBranch),
198 LastMod: headMod,
199 })
200 }
201
202 // Releases first: the detail pages are the ones a search result is most
203 // likely to want, so a repository close to the URL limit keeps them.
204 tags, err := w.Index.Tags(entry.ID, 0)
205 if err != nil {
206 w.serverError(rw, r, "sitemap: cannot read releases: "+err.Error())
207 return
208 }
209 for _, t := range tags {
210 urls = append(urls, sitemapURL{
211 Loc: base + root + "/releases/" + url.PathEscape(t.Name),
212 LastMod: sitemapUnix(t.When),
213 })
214 }
215
216 issues, err := w.Index.Issues(entry.ID, index.Filter{})
217 if err != nil {
218 w.serverError(rw, r, "sitemap: cannot read issues: "+err.Error())
219 return
220 }
221 for _, iss := range issues {
222 urls = append(urls, sitemapURL{
223 Loc: base + root + "/issues/" + url.PathEscape(iss.ID),
224 LastMod: sitemapTime(iss.Updated),
225 })
226 }
227
228 requests, err := w.Index.Requests(entry.ID, index.RequestFilter{})
229 if err != nil {
230 w.serverError(rw, r, "sitemap: cannot read changes: "+err.Error())
231 return
232 }
233 for _, req := range requests {
234 urls = append(urls, sitemapURL{
235 Loc: base + root + "/changes/" + url.PathEscape(req.ID),
236 LastMod: sitemapTime(req.Updated),
237 })
238 }
239
240 if len(urls) > sitemapMaxURLs {
241 if w.Logger != nil {
242 w.Logger("sitemap: %s has %d URLs; keeping the first %d", entry.Name, len(urls), sitemapMaxURLs)
243 }
244 urls = urls[:sitemapMaxURLs]
245 }
246 w.writeSitemap(rw, sitemapURLSet{XMLNS: sitemapNS, URLs: urls})
247 }
248
249 // publicRepos returns the repositories an anonymous reader can open. It reads the
250 // ACL rather than the request's principal, so the answer does not depend on who
251 // asked.
252 func (w *Web) publicRepos() ([]index.Entry, error) {
253 if w.Index == nil {
254 return nil, fmt.Errorf("the index is unavailable")
255 }
256 entries, err := w.Index.List()
257 if err != nil {
258 return nil, err
259 }
260 public := make([]index.Entry, 0, len(entries))
261 for _, e := range entries {
262 if w.Repos.Public(e.Name) {
263 public = append(public, e)
264 }
265 }
266 return public, nil
267 }
268
269 // writeSitemap renders a url set or an index. The body is built before the
270 // headers are set so an encoding error is a 500 rather than a truncated XML
271 // document that a crawler would remember as the sitemap.
272 func (w *Web) writeSitemap(rw http.ResponseWriter, v any) {
273 body, err := xml.MarshalIndent(v, "", " ")
274 if err != nil {
275 if w.Logger != nil {
276 w.Logger("sitemap: %v", err)
277 }
278 http.Error(rw, "The sitemap could not be rendered.", http.StatusInternalServerError)
279 return
280 }
281 rw.Header().Set("Content-Type", "application/xml; charset=utf-8")
282 rw.Header().Set("Cache-Control", sitemapCacheControl)
283 _, _ = rw.Write([]byte(xml.Header))
284 _, _ = rw.Write(body)
285 _, _ = rw.Write([]byte("\n"))
286 }
287
288 // sitemapTime formats a page timestamp. An empty value is omitted by the URL
289 // struct rather than written as the zero time, which is not a date a crawler can
290 // compare.
291 func sitemapTime(t time.Time) string {
292 if t.IsZero() {
293 return ""
294 }
295 return t.UTC().Format(time.RFC3339)
296 }
297
298 func sitemapUnix(sec int64) string {
299 if sec <= 0 {
300 return ""
301 }
302 return time.Unix(sec, 0).UTC().Format(time.RFC3339)
303 }