Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 6 slice 1: the tree is the site

d3296cda0c18f78097e37373822292707113d245 · Keenan Nemetz · 18d ago · parent 64ac5071 

Phase 6 slice 1: the tree is the site

A repository's `pages` branch is served at <repo>.<pages_domain>, straight out
of git. No export step, no build to drift: the push moves the branch and the
next request reads the new tree.

- internal/pages: a pool of `git cat-file --batch` processes per active
  repository (closed after two idle minutes), an LRU keyed by (commit, path),
  and tree parsing in git's binary format with the object-id width detected
  rather than assumed
- ETag is the blob sha, Last-Modified the commit time, Content-Type from the
  extension or a sniff; If-None-Match and If-Modified-Since answer 304
- directories serve index.html and redirect to add the trailing slash; path
  resolution walks trees, so a traversal-shaped request is a missing page
- repository and ACL are resolved once per one-second memo window together with
  the branch tip, which is what makes the read path fast: doing it per request
  spawned three git processes and cost 3.8 ms, against 6.7 µs warm
- sites follow clone visibility, so a private repository answers 404; the repo
  page links to the site when pages_domain is configured

Measured: a site pushed over SSH is live in 1.4 s (the phase criterion is 5 s);
warm requests 6.7 µs, cold 144 µs. Custom domains and their TLS certificates are
the remaining Phase 6 work.

internal/server/pages_test.go +225

@@ -0,0 +1,225 @@
1 package server_test
2
3 // Site hosting end to end: a branch pushed over SSH is served as a website on its
4 // own hostname, with no export step and no restart.
5
6 import (
7 "io"
8 "net/http"
9 "os"
10 "path/filepath"
11 "strings"
12 "testing"
13 "time"
14
15 "github.com/rodent-software/vole/internal/config"
16 )
17
18 const pagesDomain = "pages.test"
19
20 // noRedirect is a client that reports a redirect instead of chasing it. The
21 // default client would drop the site's Host header on the way, which turns a
22 // redirect test into a test of the forge UI.
23 var noRedirect = &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
24 return http.ErrUseLastResponse
25 }}
26
27 func pagesFixture(t *testing.T) *fixture {
28 t.Helper()
29 return newFixtureWith(t, func(c *config.Config) {
30 c.PagesDomain = pagesDomain
31 c.PagesRef = "pages"
32 c.PagesProcs = 2
33 c.PagesCacheBytes = 1 << 20
34 })
35 }
36
37 // siteGet requests a page from a site host while still talking to the test
38 // listener, which is what a real request with a Host header does.
39 func (f *fixture) siteGet(host, path string, headers ...[2]string) (int, string, http.Header) {
40 f.t.Helper()
41 req, err := http.NewRequest(http.MethodGet, f.http.URL+path, nil)
42 if err != nil {
43 f.t.Fatal(err)
44 }
45 req.Host = host
46 for _, h := range headers {
47 req.Header.Set(h[0], h[1])
48 }
49 // No redirect following: a 301 is part of the contract being tested.
50 resp, err := noRedirect.Do(req)
51 if err != nil {
52 f.t.Fatal(err)
53 }
54 defer resp.Body.Close()
55 body, _ := io.ReadAll(resp.Body)
56 return resp.StatusCode, string(body), resp.Header
57 }
58
59 // pushPages commits files to the site branch, the way a user would.
60 func (f *fixture) pushPages(t *testing.T, files map[string]string) {
61 t.Helper()
62 work := workRepo(t, files)
63 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/pages"); err != nil {
64 t.Fatalf("push pages: %v\n%s", err, errs)
65 }
66 }
67
68 func TestSiteIsServedFromThePushedBranch(t *testing.T) {
69 f := pagesFixture(t)
70 f.publish(f.repo)
71 f.pushPages(t, map[string]string{
72 "index.html": "<!doctype html><h1>hello site</h1>\n",
73 "assets/style.css": "body { color: #123 }\n",
74 "docs/index.html": "<p>docs home</p>\n",
75 })
76
77 host := f.repo + "." + pagesDomain
78 code, body, hdr := f.siteGet(host, "/")
79 if code != 200 {
80 t.Fatalf("GET / = %d: %s", code, body)
81 }
82 if !strings.Contains(body, "hello site") {
83 t.Fatalf("index body = %q", body)
84 }
85 if ct := hdr.Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
86 t.Fatalf("Content-Type = %q", ct)
87 }
88 if hdr.Get("ETag") == "" || hdr.Get("Last-Modified") == "" {
89 t.Fatalf("missing validators: ETag=%q Last-Modified=%q", hdr.Get("ETag"), hdr.Get("Last-Modified"))
90 }
91 if code, body, hdr := f.siteGet(host, "/assets/style.css"); code != 200 ||
92 !strings.HasPrefix(hdr.Get("Content-Type"), "text/css") {
93 t.Fatalf("css = %d %q (%q)", code, hdr.Get("Content-Type"), body)
94 }
95 // A directory URL without its slash redirects, so relative links keep working.
96 if code, _, hdr := f.siteGet(host, "/docs"); code != http.StatusMovedPermanently || hdr.Get("Location") != "/docs/" {
97 t.Fatalf("GET /docs = %d %q", code, hdr.Get("Location"))
98 }
99 if code, body, _ := f.siteGet(host, "/docs/"); code != 200 || !strings.Contains(body, "docs home") {
100 t.Fatalf("GET /docs/ = %d %q", code, body)
101 }
102
103 // The forge UI keeps its own host: the site only owns its subdomain.
104 if code, body := f.get("/r/" + f.repo); code != 200 || !strings.Contains(body, f.repo) {
105 t.Fatalf("forge UI on the plain host = %d", code)
106 }
107 }
108
109 // TestSitePushIsLiveWithinFiveSeconds is the phase's first exit criterion, and it
110 // is measured rather than assumed: push, then poll the site.
111 func TestSitePushIsLiveWithinFiveSeconds(t *testing.T) {
112 f := pagesFixture(t)
113 f.publish(f.repo)
114 work := workRepo(t, map[string]string{"index.html": "version one\n"})
115 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/pages"); err != nil {
116 t.Fatalf("push v1: %v\n%s", err, errs)
117 }
118
119 host := f.repo + "." + pagesDomain
120 if code, body, _ := f.siteGet(host, "/"); code != 200 || !strings.Contains(body, "version one") {
121 t.Fatalf("v1 = %d %q", code, body)
122 }
123
124 // Second push from the same working copy: a site branch is a branch, so it
125 // moves by fast-forward as usual.
126 if err := os.WriteFile(filepath.Join(work, "index.html"), []byte("version two\n"), 0o644); err != nil {
127 t.Fatal(err)
128 }
129 if _, errs, err := runGit(t, work, "add", "-A"); err != nil {
130 t.Fatalf("add: %s", errs)
131 }
132 if _, errs, err := runGit(t, work, "commit", "-qm", "v2"); err != nil {
133 t.Fatalf("commit: %s", errs)
134 }
135 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/pages"); err != nil {
136 t.Fatalf("push v2: %v\n%s", err, errs)
137 }
138
139 deadline := time.Now().Add(5 * time.Second)
140 for {
141 code, body, _ := f.siteGet(host, "/")
142 if code == 200 && strings.Contains(body, "version two") {
143 break
144 }
145 if time.Now().After(deadline) {
146 t.Fatalf("the pushed site was not live within 5s; still serving %q", body)
147 }
148 time.Sleep(100 * time.Millisecond)
149 }
150 }
151
152 func TestSiteConditionalRequests(t *testing.T) {
153 f := pagesFixture(t)
154 f.publish(f.repo)
155 f.pushPages(t, map[string]string{"index.html": "conditional\n"})
156 host := f.repo + "." + pagesDomain
157
158 code, _, hdr := f.siteGet(host, "/")
159 if code != 200 {
160 t.Fatalf("first GET = %d", code)
161 }
162 etag := hdr.Get("ETag")
163 code, body, _ := f.siteGet(host, "/", [2]string{"If-None-Match", etag})
164 if code != http.StatusNotModified || body != "" {
165 t.Fatalf("If-None-Match = %d with %d bytes, want an empty 304", code, len(body))
166 }
167 code, _, _ = f.siteGet(host, "/", [2]string{"If-Modified-Since", hdr.Get("Last-Modified")})
168 if code != http.StatusNotModified {
169 t.Fatalf("If-Modified-Since = %d, want 304", code)
170 }
171 }
172
173 func TestSiteIsNotAnEscapeHatch(t *testing.T) {
174 f := pagesFixture(t)
175 f.publish(f.repo)
176 f.pushPages(t, map[string]string{"index.html": "safe\n"})
177 host := f.repo + "." + pagesDomain
178
179 for _, path := range []string{
180 "/../../etc/passwd",
181 "/%2e%2e/%2e%2e/etc/passwd",
182 "/../../../home/keenan/.ssh/id_ed25519",
183 } {
184 if code, body, _ := f.siteGet(host, path); code != http.StatusNotFound {
185 t.Fatalf("GET %q = %d, body %q", path, code, body)
186 }
187 }
188 }
189
190 func TestSiteNeedsAPublicRepositoryAndAReferencedBranch(t *testing.T) {
191 f := pagesFixture(t)
192 // Published, but no pages branch yet.
193 f.publish(f.repo)
194 host := f.repo + "." + pagesDomain
195 if code, _, _ := f.siteGet(host, "/"); code != http.StatusNotFound {
196 t.Fatalf("no pages branch = %d, want 404", code)
197 }
198
199 // A site branch on a private repository is not served: an anonymous reader
200 // has no way to authenticate, so 404 rather than a login that cannot exist.
201 f.pushPages(t, map[string]string{"index.html": "private\n"})
202 m, err := f.srv.Repos.Open(f.repo)
203 if err != nil {
204 t.Fatal(err)
205 }
206 a, err := m.ACL()
207 if err != nil {
208 t.Fatal(err)
209 }
210 a.Default = "none"
211 if err := m.SetACL(a, "test"); err != nil {
212 t.Fatal(err)
213 }
214 // Site visibility is resolved once per memo window (one second), so the
215 // change lands within that window rather than on the very next request.
216 time.Sleep(1100 * time.Millisecond)
217 if code, _, _ := f.siteGet(host, "/"); code != http.StatusNotFound {
218 t.Fatalf("private repo site = %d, want 404", code)
219 }
220
221 // An unknown site name answers the same way.
222 if code, _, _ := f.siteGet("nosuchrepo."+pagesDomain, "/"); code != http.StatusNotFound {
223 t.Fatalf("unknown repo site = %d, want 404", code)
224 }
225 }