vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
80300457
Keenan Nemetz
23h ago
|
1 | package web |
| 2 | ||
| 3 | // robots.txt and the sitemap. | |
| 4 | // | |
| 5 | // A crawler asks two questions: which paths are worth fetching, and where the | |
| 6 | // public pages are. robots.txt answers the first; the sitemap answers the second. | |
| 7 | // The sitemap is a tree of files rather than one document because a forge grows | |
| 8 | // past what one URL set may hold, and a crawler fetches only the parts it needs. | |
| 9 | // | |
| 10 | // Every URL here is built from the anonymous read rule, never from the viewer's | |
| 11 | // session. A sitemap is cacheable by a shared proxy, and an instance admin who | |
| 12 | // asks for one must not put a private repository's name into what everyone else | |
| 13 | // receives. Account profiles are deliberately absent: /u and /u/{name} carry | |
| 14 | // <meta name="robots" content="noindex">, and a robots.txt Disallow would hide | |
| 15 | // that request from the crawler that has to read it. | |
| 16 | ||
| 17 | import ( | |
| 18 | "encoding/xml" | |
| 19 | "fmt" | |
| 20 | "net/http" | |
| 21 | "net/url" | |
| 22 | "time" | |
| 23 | ||
|
d37d301f
Keenan Nemetz
16h ago
|
24 | "rodent.software/vole/forge/access" |
| 25 | "rodent.software/vole/forge/index" | |
|
80300457
Keenan Nemetz
23h ago
|
26 | ) |
| 27 | ||
| 28 | const ( | |
| 29 | sitemapNS = "http://www.sitemaps.org/schemas/sitemap/0.9" | |
| 30 | // sitemapMaxURLs is the protocol's limit for one file. A repository past it | |
| 31 | // is truncated and the event is logged rather than split: the alternative is | |
| 32 | // a second layer of pagination that an instance this size needs everywhere, | |
| 33 | // not just here. | |
| 34 | sitemapMaxURLs = 50000 | |
| 35 | // sitemapCacheControl lets a proxy reuse the answer while a new issue or | |
| 36 | // release lands. The window is short because the sitemap is how a crawler | |
| 37 | // learns about those pages in the first place. | |
| 38 | sitemapCacheControl = "public, max-age=300" | |
| 39 | ) | |
| 40 | ||
| 41 | type sitemapURLSet struct { | |
| 42 | XMLName xml.Name `xml:"urlset"` | |
| 43 | XMLNS string `xml:"xmlns,attr"` | |
| 44 | URLs []sitemapURL `xml:"url"` | |
| 45 | } | |
| 46 | ||
| 47 | type sitemapURL struct { | |
| 48 | Loc string `xml:"loc"` | |
| 49 | LastMod string `xml:"lastmod,omitempty"` | |
| 50 | } | |
| 51 | ||
| 52 | type sitemapIndex struct { | |
| 53 | XMLName xml.Name `xml:"sitemapindex"` | |
| 54 | XMLNS string `xml:"xmlns,attr"` | |
| 55 | Sitemaps []sitemapRef `xml:"sitemap"` | |
| 56 | } | |
| 57 | ||
| 58 | type sitemapRef struct { | |
| 59 | Loc string `xml:"loc"` | |
| 60 | } | |
| 61 | ||
| 62 | // Robots serves GET /robots.txt. The public surface is allowed by default and the | |
| 63 | // account, administration, write and fragment paths are closed; a crawler that | |
| 64 | // follows a link into a settings form or a live fragment is wasting a request | |
| 65 | // that could have been a page. The sitemap line points at the index, which is | |
| 66 | // the discoverable list of repositories, issues, changes, releases and code. | |
| 67 | func (w *Web) Robots(rw http.ResponseWriter, r *http.Request) { | |
| 68 | rw.Header().Set("Content-Type", "text/plain; charset=utf-8") | |
| 69 | rw.Header().Set("Cache-Control", "public, max-age=3600") | |
| 70 | fmt.Fprintf(rw, `User-agent: * | |
| 71 | Allow: / | |
| 72 | ||
|
ed03b3d1
Keenan Nemetz
3h ago
|
73 | # The forge's own pages live under /-/. |
| 74 | Disallow: /-/account | |
| 75 | Disallow: /-/admin | |
| 76 | Disallow: /-/login | |
| 77 | Disallow: /-/register | |
| 78 | Disallow: /-/invite | |
| 79 | Disallow: /-/users | |
| 80 | Disallow: /-/dev/ | |
| 81 | Disallow: /-/repos/new | |
| 82 | Disallow: /-/runners | |
| 83 | Disallow: /-/healthz | |
| 84 | Disallow: /-/search/ | |
|
80300457
Keenan Nemetz
23h ago
|
85 | |
| 86 | # Repository writes, fragments and machine-readable responses. | |
|
ed03b3d1
Keenan Nemetz
3h ago
|
87 | Disallow: /*.git/ |
| 88 | Disallow: /*/counts | |
| 89 | Disallow: /*/live | |
| 90 | Disallow: /*/raw/ | |
| 91 | Disallow: /*/comment-form | |
| 92 | Disallow: /*/compare | |
| 93 | Disallow: /*/search | |
| 94 | Disallow: /*/issues/new | |
| 95 | Disallow: /*/issues/*/edit | |
| 96 | Disallow: /*/changes/new | |
| 97 | Disallow: /*/changes/*/edit | |
| 98 | Disallow: /*/changes/*/files | |
| 99 | Disallow: /*/changes/*/runs | |
| 100 | Disallow: /*/changes/*/live | |
| 101 | Disallow: /*/releases/new | |
| 102 | Disallow: /*/releases/edit | |
| 103 | Disallow: /*/releases/*/archive.tar | |
| 104 | Disallow: /*/releases/*/archive.tar.gz | |
| 105 | Disallow: /*/releases/*/artifacts/ | |
| 106 | Disallow: /*/runs | |
| 107 | Disallow: /*/workflows/ | |
| 108 | Disallow: /*/locks | |
| 109 | Disallow: /*/settings | |
| 110 | ||
| 111 | # /-/u and /-/u/{name} stay crawlable on purpose: those pages ask for noindex, and | |
| 112 | # a crawler cannot see that request through a Disallow line. | |
|
80300457
Keenan Nemetz
23h ago
|
113 | Sitemap: %s/sitemap.xml |
| 114 | `, w.canonicalBase(r)) | |
| 115 | } | |
| 116 | ||
| 117 | // SitemapIndex serves GET /sitemap.xml: the instance-wide pages in one child and | |
| 118 | // each public repository in another. A private repository is absent, not listed | |
| 119 | // and refused, because the URL itself is the disclosure. | |
| 120 | func (w *Web) SitemapIndex(rw http.ResponseWriter, r *http.Request) { | |
| 121 | base := w.canonicalBase(r) | |
| 122 | entries, err := w.publicRepos() | |
| 123 | if err != nil { | |
| 124 | w.serverError(rw, r, "sitemap: "+err.Error()) | |
| 125 | return | |
| 126 | } | |
| 127 | sitemaps := make([]sitemapRef, 0, len(entries)+1) | |
| 128 | sitemaps = append(sitemaps, sitemapRef{Loc: base + "/sitemap-pages.xml"}) | |
| 129 | for _, e := range entries { | |
| 130 | sitemaps = append(sitemaps, sitemapRef{ | |
|
ed03b3d1
Keenan Nemetz
3h ago
|
131 | Loc: base + "/" + url.PathEscape(e.Name) + "/sitemap.xml", |
|
80300457
Keenan Nemetz
23h ago
|
132 | }) |
| 133 | } | |
| 134 | if len(sitemaps) > sitemapMaxURLs { | |
| 135 | if w.Logger != nil { | |
| 136 | w.Logger("sitemap: %d public repositories; keeping the first %d", len(entries), sitemapMaxURLs-1) | |
| 137 | } | |
| 138 | sitemaps = sitemaps[:sitemapMaxURLs] | |
| 139 | } | |
| 140 | w.writeSitemap(rw, sitemapIndex{XMLNS: sitemapNS, Sitemaps: sitemaps}) | |
| 141 | } | |
| 142 | ||
| 143 | // SitemapPages serves GET /sitemap-pages.xml: the public destinations that do | |
| 144 | // not belong to a repository. The profile directory and a profile are absent; | |
| 145 | // they are public pages that ask not to be indexed. | |
| 146 | func (w *Web) SitemapPages(rw http.ResponseWriter, r *http.Request) { | |
| 147 | base := w.canonicalBase(r) | |
| 148 | w.writeSitemap(rw, sitemapURLSet{XMLNS: sitemapNS, URLs: []sitemapURL{ | |
| 149 | {Loc: base + "/"}, | |
|
ed03b3d1
Keenan Nemetz
3h ago
|
150 | {Loc: base + "/-/activity"}, |
|
80300457
Keenan Nemetz
23h ago
|
151 | }}) |
| 152 | } | |
| 153 | ||
|
ed03b3d1
Keenan Nemetz
3h ago
|
154 | // RepoSitemap serves GET /{name}/sitemap.xml. It uses the anonymous read rule, |
|
80300457
Keenan Nemetz
23h ago
|
155 | // not w.open, so a signed-in maintainer or admin gets the same file as everyone |
| 156 | // else and a private repository's sitemap is a 404 rather than a session-scoped | |
| 157 | // document that a proxy could hand to a stranger. | |
| 158 | func (w *Web) RepoSitemap(rw http.ResponseWriter, r *http.Request) { | |
| 159 | name := r.PathValue("name") | |
| 160 | m, err := w.Repos.Meta(name) | |
| 161 | if err != nil { | |
| 162 | w.notFound(rw, r) | |
| 163 | return | |
| 164 | } | |
| 165 | acl, err := m.ACL() | |
| 166 | if err != nil || !acl.Can("", access.Read) { | |
| 167 | w.notFound(rw, r) | |
| 168 | return | |
| 169 | } | |
| 170 | if w.Index == nil { | |
| 171 | w.serverError(rw, r, "sitemap: the index is unavailable") | |
| 172 | return | |
| 173 | } | |
| 174 | entry, err := w.Index.Get(name) | |
| 175 | if err != nil { | |
| 176 | w.serverError(rw, r, "sitemap: "+err.Error()) | |
| 177 | return | |
| 178 | } | |
| 179 | if err := w.ensureIndex(memoOf(r), m); err != nil { | |
| 180 | w.serverError(rw, r, "sitemap: cannot read metadata: "+err.Error()) | |
| 181 | return | |
| 182 | } | |
| 183 | ||
| 184 | base := w.canonicalBase(r) | |
|
ed03b3d1
Keenan Nemetz
3h ago
|
185 | root := "/" + url.PathEscape(entry.Name) |
|
80300457
Keenan Nemetz
23h ago
|
186 | headMod := sitemapUnix(entry.HeadWhen) |
| 187 | urls := []sitemapURL{ | |
| 188 | {Loc: base + root, LastMod: headMod}, | |
| 189 | {Loc: base + root + "/activity", LastMod: headMod}, | |
| 190 | {Loc: base + root + "/code", LastMod: headMod}, | |
| 191 | {Loc: base + root + "/issues"}, | |
| 192 | {Loc: base + root + "/changes"}, | |
| 193 | {Loc: base + root + "/releases", LastMod: headMod}, | |
| 194 | } | |
| 195 | if entry.DefaultBranch != "" && entry.HeadSHA != "" { | |
| 196 | urls = append(urls, sitemapURL{ | |
| 197 | Loc: base + root + "/commits/" + url.PathEscape(entry.DefaultBranch), | |
| 198 | LastMod: headMod, | |
| 199 | }) | |
| 200 | } | |
| 201 | ||
| 202 | // Releases first: the detail pages are the ones a search result is most | |
| 203 | // likely to want, so a repository close to the URL limit keeps them. | |
| 204 | tags, err := w.Index.Tags(entry.ID, 0) | |
| 205 | if err != nil { | |
| 206 | w.serverError(rw, r, "sitemap: cannot read releases: "+err.Error()) | |
| 207 | return | |
| 208 | } | |
| 209 | for _, t := range tags { | |
| 210 | urls = append(urls, sitemapURL{ | |
| 211 | Loc: base + root + "/releases/" + url.PathEscape(t.Name), | |
| 212 | LastMod: sitemapUnix(t.When), | |
| 213 | }) | |
| 214 | } | |
| 215 | ||
| 216 | issues, err := w.Index.Issues(entry.ID, index.Filter{}) | |
| 217 | if err != nil { | |
| 218 | w.serverError(rw, r, "sitemap: cannot read issues: "+err.Error()) | |
| 219 | return | |
| 220 | } | |
| 221 | for _, iss := range issues { | |
| 222 | urls = append(urls, sitemapURL{ | |
| 223 | Loc: base + root + "/issues/" + url.PathEscape(iss.ID), | |
| 224 | LastMod: sitemapTime(iss.Updated), | |
| 225 | }) | |
| 226 | } | |
| 227 | ||
| 228 | requests, err := w.Index.Requests(entry.ID, index.RequestFilter{}) | |
| 229 | if err != nil { | |
| 230 | w.serverError(rw, r, "sitemap: cannot read changes: "+err.Error()) | |
| 231 | return | |
| 232 | } | |
| 233 | for _, req := range requests { | |
| 234 | urls = append(urls, sitemapURL{ | |
| 235 | Loc: base + root + "/changes/" + url.PathEscape(req.ID), | |
| 236 | LastMod: sitemapTime(req.Updated), | |
| 237 | }) | |
| 238 | } | |
| 239 | ||
| 240 | if len(urls) > sitemapMaxURLs { | |
| 241 | if w.Logger != nil { | |
| 242 | w.Logger("sitemap: %s has %d URLs; keeping the first %d", entry.Name, len(urls), sitemapMaxURLs) | |
| 243 | } | |
| 244 | urls = urls[:sitemapMaxURLs] | |
| 245 | } | |
| 246 | w.writeSitemap(rw, sitemapURLSet{XMLNS: sitemapNS, URLs: urls}) | |
| 247 | } | |
| 248 | ||
| 249 | // publicRepos returns the repositories an anonymous reader can open. It reads the | |
| 250 | // ACL rather than the request's principal, so the answer does not depend on who | |
| 251 | // asked. | |
| 252 | func (w *Web) publicRepos() ([]index.Entry, error) { | |
| 253 | if w.Index == nil { | |
| 254 | return nil, fmt.Errorf("the index is unavailable") | |
| 255 | } | |
| 256 | entries, err := w.Index.List() | |
| 257 | if err != nil { | |
| 258 | return nil, err | |
| 259 | } | |
| 260 | public := make([]index.Entry, 0, len(entries)) | |
| 261 | for _, e := range entries { | |
| 262 | if w.Repos.Public(e.Name) { | |
| 263 | public = append(public, e) | |
| 264 | } | |
| 265 | } | |
| 266 | return public, nil | |
| 267 | } | |
| 268 | ||
| 269 | // writeSitemap renders a url set or an index. The body is built before the | |
| 270 | // headers are set so an encoding error is a 500 rather than a truncated XML | |
| 271 | // document that a crawler would remember as the sitemap. | |
| 272 | func (w *Web) writeSitemap(rw http.ResponseWriter, v any) { | |
| 273 | body, err := xml.MarshalIndent(v, "", " ") | |
| 274 | if err != nil { | |
| 275 | if w.Logger != nil { | |
| 276 | w.Logger("sitemap: %v", err) | |
| 277 | } | |
| 278 | http.Error(rw, "The sitemap could not be rendered.", http.StatusInternalServerError) | |
| 279 | return | |
| 280 | } | |
| 281 | rw.Header().Set("Content-Type", "application/xml; charset=utf-8") | |
| 282 | rw.Header().Set("Cache-Control", sitemapCacheControl) | |
| 283 | _, _ = rw.Write([]byte(xml.Header)) | |
| 284 | _, _ = rw.Write(body) | |
| 285 | _, _ = rw.Write([]byte("\n")) | |
| 286 | } | |
| 287 | ||
| 288 | // sitemapTime formats a page timestamp. An empty value is omitted by the URL | |
| 289 | // struct rather than written as the zero time, which is not a date a crawler can | |
| 290 | // compare. | |
| 291 | func sitemapTime(t time.Time) string { | |
| 292 | if t.IsZero() { | |
| 293 | return "" | |
| 294 | } | |
| 295 | return t.UTC().Format(time.RFC3339) | |
| 296 | } | |
| 297 | ||
| 298 | func sitemapUnix(sec int64) string { | |
| 299 | if sec <= 0 { | |
| 300 | return "" | |
| 301 | } | |
| 302 | return time.Unix(sec, 0).UTC().Format(time.RFC3339) | |
| 303 | } |