Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Publish a robots.txt and sitemap for public pages

80300457bddaf6b90f472a336893ed5dee3c0a48 · Keenan Nemetz · 1d ago · parent c980fe53 

Publish a robots.txt and sitemap for public pages

A crawler needs to know which paths are worth fetching and where the public pages are. robots.txt allows the public surface, closes the account, administration, write, fragment and Git paths, and points at a sitemap index. The index splits the instance pages from each public repository; a repository sitemap lists its overview, activity, code browser, issues, changes, releases and default-branch commits, with lastmod on detail pages.

Visibility is the anonymous read rule, not the viewer's session: a private repository is absent from the index and its sitemap is a 404 even for an admin. Account profiles stay out of the sitemap and out of robots.txt so their existing noindex request remains visible to crawlers.

server/seo_test.go +170

@@ -0,0 +1,170 @@
1 package server_test
2
3 // robots.txt and the sitemap are the crawl surface: the paths worth fetching,
4 // and the public repositories and pages behind them. These tests pin the two
5 // rules that matter: a private repository never appears in either, and a
6 // repository's own sitemap is the same document for a stranger and for an admin.
7
8 import (
9 "encoding/xml"
10 "net/http"
11 "strings"
12 "testing"
13 )
14
15 func TestRobotsTxtDescribesThePublicSurface(t *testing.T) {
16 t.Parallel()
17 f := newFixture(t)
18
19 resp, body := f.browserGet(t, "/robots.txt", "")
20 if resp.StatusCode != http.StatusOK {
21 t.Fatalf("robots.txt = %d\n%s", resp.StatusCode, body)
22 }
23 if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
24 t.Fatalf("robots.txt content type = %q", ct)
25 }
26 for _, want := range []string{
27 "User-agent: *",
28 "Allow: /",
29 "Disallow: /admin",
30 "Disallow: /account",
31 "Disallow: /r/*/settings",
32 "Disallow: /r/*/issues/new",
33 "Disallow: /git/",
34 "Sitemap: " + f.http.URL + "/sitemap.xml",
35 } {
36 if !strings.Contains(body, want) {
37 t.Fatalf("robots.txt is missing %q:\n%s", want, body)
38 }
39 }
40 // Profiles are noindex pages, not disallowed ones: a crawler has to be able
41 // to fetch them to see the request.
42 for _, unwanted := range []string{"Disallow: /u\n", "Disallow: /u/"} {
43 if strings.Contains(body, unwanted) {
44 t.Fatalf("robots.txt hides a noindex page with %q:\n%s", unwanted, body)
45 }
46 }
47 }
48
49 func TestSitemapListsPublicRepositoriesAndPages(t *testing.T) {
50 t.Parallel()
51 f := newFixture(t)
52 f.publish(f.repo)
53
54 private, err := f.srv.Repos.Create("secret", f.ownerID, "private repo")
55 if err != nil {
56 t.Fatal(err)
57 }
58 if err := f.srv.Index.Upsert(private); err != nil {
59 t.Fatal(err)
60 }
61
62 resp, body := f.browserGet(t, "/sitemap.xml", "")
63 if resp.StatusCode != http.StatusOK {
64 t.Fatalf("sitemap.xml = %d\n%s", resp.StatusCode, body)
65 }
66 if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") {
67 t.Fatalf("sitemap.xml content type = %q", ct)
68 }
69 for _, want := range []string{
70 `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
71 "<loc>" + f.http.URL + "/sitemap-pages.xml</loc>",
72 "<loc>" + f.http.URL + "/r/demo/sitemap.xml</loc>",
73 } {
74 if !strings.Contains(body, want) {
75 t.Fatalf("sitemap index is missing %q:\n%s", want, body)
76 }
77 }
78 if strings.Contains(body, "secret") {
79 t.Fatalf("sitemap index names a private repository:\n%s", body)
80 }
81
82 // The answer belongs to the instance, not the session: an admin asking for
83 // it must not add a private repository to a cacheable document.
84 _, cookie := f.browserSession(t, f.ownerID)
85 if _, signedIn := f.browserGet(t, "/sitemap.xml", cookie); strings.Contains(signedIn, "secret") {
86 t.Fatalf("a signed-in sitemap names a private repository:\n%s", signedIn)
87 }
88 if code, _ := f.get("/r/secret/sitemap.xml"); code != http.StatusNotFound {
89 t.Fatalf("private repository sitemap = %d, want 404", code)
90 }
91 resp, _ = f.browserGet(t, "/r/secret/sitemap.xml", cookie)
92 if resp.StatusCode != http.StatusNotFound {
93 t.Fatalf("signed-in private repository sitemap = %d, want 404", resp.StatusCode)
94 }
95
96 _, body = f.browserGet(t, "/sitemap-pages.xml", "")
97 for _, want := range []string{
98 "<loc>" + f.http.URL + "/</loc>",
99 "<loc>" + f.http.URL + "/activity</loc>",
100 } {
101 if !strings.Contains(body, want) {
102 t.Fatalf("page sitemap is missing %q:\n%s", want, body)
103 }
104 }
105 }
106
107 func TestRepoSitemapListsIssuesChangesAndReleases(t *testing.T) {
108 t.Parallel()
109 f := newFixture(t)
110 f.publish(f.repo)
111
112 work := workRepo(t, map[string]string{"readme.md": "sitemap\n"})
113 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
114 t.Fatalf("main push: %v\n%s", err, errs)
115 }
116 out, _, _ := runGit(t, work, "rev-parse", "HEAD")
117 head := strings.TrimSpace(out)
118 if _, errs, err := runGit(t, work, "tag", "-a", "v1.0.0", "-m", "first release\n"); err != nil {
119 t.Fatalf("tag: %v\n%s", err, errs)
120 }
121 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "refs/tags/v1.0.0:refs/tags/v1.0.0"); err != nil {
122 t.Fatalf("tag push: %v\n%s", err, errs)
123 }
124
125 const issueID = "01M5JR2JK29KKMAA1T9MDNTS01"
126 const changeID = "01M5JR2JK29KKMAA1T9MDNTS02"
127 meta := f.newMetaRepo(t)
128 if err := meta.apply(map[string]string{
129 "issues/" + issueID + "/issue.md": issueFile("Sitemap issue", "keenan", "open", "2026-10-22T21:00:00Z", []string{"web"}, "body\n"),
130 "changes/" + changeID + "/change.md": requestFileTitled("Sitemap change", "main", head, "keenan", "body\n"),
131 }, "sitemap fixtures"); err != nil {
132 t.Fatalf("metadata push: %v", err)
133 }
134
135 resp, body := f.browserGet(t, "/r/"+f.repo+"/sitemap.xml", "")
136 if resp.StatusCode != http.StatusOK {
137 t.Fatalf("repository sitemap = %d\n%s", resp.StatusCode, body)
138 }
139 base := f.http.URL
140 for _, want := range []string{
141 `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
142 "<loc>" + base + "/r/demo</loc>",
143 "<loc>" + base + "/r/demo/activity</loc>",
144 "<loc>" + base + "/r/demo/code</loc>",
145 "<loc>" + base + "/r/demo/issues</loc>",
146 "<loc>" + base + "/r/demo/issues/" + issueID + "</loc>",
147 "<loc>" + base + "/r/demo/changes</loc>",
148 "<loc>" + base + "/r/demo/changes/" + changeID + "</loc>",
149 "<loc>" + base + "/r/demo/releases</loc>",
150 "<loc>" + base + "/r/demo/releases/v1.0.0</loc>",
151 "<loc>" + base + "/r/demo/commits/main</loc>",
152 "<lastmod>",
153 } {
154 if !strings.Contains(body, want) {
155 t.Fatalf("repository sitemap is missing %q:\n%s", want, body)
156 }
157 }
158
159 var set struct {
160 URLs []struct {
161 Loc string `xml:"loc"`
162 } `xml:"url"`
163 }
164 if err := xml.Unmarshal([]byte(body), &set); err != nil {
165 t.Fatalf("repository sitemap is not XML: %v\n%s", err, body)
166 }
167 if len(set.URLs) < 9 {
168 t.Fatalf("repository sitemap has %d URLs, want the repository, list, issue, change and release pages:\n%s", len(set.URLs), body)
169 }
170 }