Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · server/seo_test.go

Line attribution on main · 170 line(s)

View file History
80300457
Keenan Nemetz
23h ago
 1 package server_test
2
3 // robots.txt and the sitemap are the crawl surface: the paths worth fetching,
4 // and the public repositories and pages behind them. These tests pin the two
5 // rules that matter: a private repository never appears in either, and a
6 // repository's own sitemap is the same document for a stranger and for an admin.
7
8 import (
9 "encoding/xml"
10 "net/http"
11 "strings"
12 "testing"
13 )
14
15 func TestRobotsTxtDescribesThePublicSurface(t *testing.T) {
16 t.Parallel()
17 f := newFixture(t)
18
19 resp, body := f.browserGet(t, "/robots.txt", "")
20 if resp.StatusCode != http.StatusOK {
21 t.Fatalf("robots.txt = %d\n%s", resp.StatusCode, body)
22 }
23 if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
24 t.Fatalf("robots.txt content type = %q", ct)
25 }
26 for _, want := range []string{
27 "User-agent: *",
28 "Allow: /",
ed03b3d1
Keenan Nemetz
3h ago
 29 "Disallow: /-/admin",
30 "Disallow: /-/account",
31 "Disallow: /*/settings",
32 "Disallow: /*/issues/new",
33 "Disallow: /*.git/",
80300457
Keenan Nemetz
23h ago
 34 "Sitemap: " + f.http.URL + "/sitemap.xml",
35 } {
36 if !strings.Contains(body, want) {
37 t.Fatalf("robots.txt is missing %q:\n%s", want, body)
38 }
39 }
40 // Profiles are noindex pages, not disallowed ones: a crawler has to be able
41 // to fetch them to see the request.
ed03b3d1
Keenan Nemetz
3h ago
 42 for _, unwanted := range []string{"Disallow: /-/u\n", "Disallow: /-/u/"} {
80300457
Keenan Nemetz
23h ago
 43 if strings.Contains(body, unwanted) {
44 t.Fatalf("robots.txt hides a noindex page with %q:\n%s", unwanted, body)
45 }
46 }
47 }
48
49 func TestSitemapListsPublicRepositoriesAndPages(t *testing.T) {
50 t.Parallel()
51 f := newFixture(t)
52 f.publish(f.repo)
53
54 private, err := f.srv.Repos.Create("secret", f.ownerID, "private repo")
55 if err != nil {
56 t.Fatal(err)
57 }
58 if err := f.srv.Index.Upsert(private); err != nil {
59 t.Fatal(err)
60 }
61
62 resp, body := f.browserGet(t, "/sitemap.xml", "")
63 if resp.StatusCode != http.StatusOK {
64 t.Fatalf("sitemap.xml = %d\n%s", resp.StatusCode, body)
65 }
66 if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") {
67 t.Fatalf("sitemap.xml content type = %q", ct)
68 }
69 for _, want := range []string{
70 `<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
71 "<loc>" + f.http.URL + "/sitemap-pages.xml</loc>",
ed03b3d1
Keenan Nemetz
3h ago
 72 "<loc>" + f.http.URL + "/demo/sitemap.xml</loc>",
80300457
Keenan Nemetz
23h ago
 73 } {
74 if !strings.Contains(body, want) {
75 t.Fatalf("sitemap index is missing %q:\n%s", want, body)
76 }
77 }
78 if strings.Contains(body, "secret") {
79 t.Fatalf("sitemap index names a private repository:\n%s", body)
80 }
81
82 // The answer belongs to the instance, not the session: an admin asking for
83 // it must not add a private repository to a cacheable document.
84 _, cookie := f.browserSession(t, f.ownerID)
85 if _, signedIn := f.browserGet(t, "/sitemap.xml", cookie); strings.Contains(signedIn, "secret") {
86 t.Fatalf("a signed-in sitemap names a private repository:\n%s", signedIn)
87 }
ed03b3d1
Keenan Nemetz
3h ago
 88 if code, _ := f.get("/secret/sitemap.xml"); code != http.StatusNotFound {
80300457
Keenan Nemetz
23h ago
 89 t.Fatalf("private repository sitemap = %d, want 404", code)
90 }
ed03b3d1
Keenan Nemetz
3h ago
 91 resp, _ = f.browserGet(t, "/secret/sitemap.xml", cookie)
80300457
Keenan Nemetz
23h ago
 92 if resp.StatusCode != http.StatusNotFound {
93 t.Fatalf("signed-in private repository sitemap = %d, want 404", resp.StatusCode)
94 }
95
96 _, body = f.browserGet(t, "/sitemap-pages.xml", "")
97 for _, want := range []string{
98 "<loc>" + f.http.URL + "/</loc>",
ed03b3d1
Keenan Nemetz
3h ago
 99 "<loc>" + f.http.URL + "/-/activity</loc>",
80300457
Keenan Nemetz
23h ago
 100 } {
101 if !strings.Contains(body, want) {
102 t.Fatalf("page sitemap is missing %q:\n%s", want, body)
103 }
104 }
105 }
106
107 func TestRepoSitemapListsIssuesChangesAndReleases(t *testing.T) {
108 t.Parallel()
109 f := newFixture(t)
110 f.publish(f.repo)
111
112 work := workRepo(t, map[string]string{"readme.md": "sitemap\n"})
113 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
114 t.Fatalf("main push: %v\n%s", err, errs)
115 }
116 out, _, _ := runGit(t, work, "rev-parse", "HEAD")
117 head := strings.TrimSpace(out)
118 if _, errs, err := runGit(t, work, "tag", "-a", "v1.0.0", "-m", "first release\n"); err != nil {
119 t.Fatalf("tag: %v\n%s", err, errs)
120 }
121 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "refs/tags/v1.0.0:refs/tags/v1.0.0"); err != nil {
122 t.Fatalf("tag push: %v\n%s", err, errs)
123 }
124
125 const issueID = "01M5JR2JK29KKMAA1T9MDNTS01"
126 const changeID = "01M5JR2JK29KKMAA1T9MDNTS02"
127 meta := f.newMetaRepo(t)
128 if err := meta.apply(map[string]string{
129 "issues/" + issueID + "/issue.md": issueFile("Sitemap issue", "keenan", "open", "2026-10-22T21:00:00Z", []string{"web"}, "body\n"),
130 "changes/" + changeID + "/change.md": requestFileTitled("Sitemap change", "main", head, "keenan", "body\n"),
131 }, "sitemap fixtures"); err != nil {
132 t.Fatalf("metadata push: %v", err)
133 }
134
ed03b3d1
Keenan Nemetz
3h ago
 135 resp, body := f.browserGet(t, "/"+f.repo+"/sitemap.xml", "")
80300457
Keenan Nemetz
23h ago
 136 if resp.StatusCode != http.StatusOK {
137 t.Fatalf("repository sitemap = %d\n%s", resp.StatusCode, body)
138 }
139 base := f.http.URL
140 for _, want := range []string{
141 `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">`,
ed03b3d1
Keenan Nemetz
3h ago
 142 "<loc>" + base + "/demo</loc>",
143 "<loc>" + base + "/demo/activity</loc>",
144 "<loc>" + base + "/demo/code</loc>",
145 "<loc>" + base + "/demo/issues</loc>",
146 "<loc>" + base + "/demo/issues/" + issueID + "</loc>",
147 "<loc>" + base + "/demo/changes</loc>",
148 "<loc>" + base + "/demo/changes/" + changeID + "</loc>",
149 "<loc>" + base + "/demo/releases</loc>",
150 "<loc>" + base + "/demo/releases/v1.0.0</loc>",
151 "<loc>" + base + "/demo/commits/main</loc>",
80300457
Keenan Nemetz
23h ago
 152 "<lastmod>",
153 } {
154 if !strings.Contains(body, want) {
155 t.Fatalf("repository sitemap is missing %q:\n%s", want, body)
156 }
157 }
158
159 var set struct {
160 URLs []struct {
161 Loc string `xml:"loc"`
162 } `xml:"url"`
163 }
164 if err := xml.Unmarshal([]byte(body), &set); err != nil {
165 t.Fatalf("repository sitemap is not XML: %v\n%s", err, body)
166 }
167 if len(set.URLs) < 9 {
168 t.Fatalf("repository sitemap has %d URLs, want the repository, list, issue, change and release pages:\n%s", len(set.URLs), body)
169 }
170 }