Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 6 slice 1: the tree is the site

d3296cda0c18f78097e37373822292707113d245 · Keenan Nemetz · 18d ago · parent 64ac5071 

Phase 6 slice 1: the tree is the site

A repository's `pages` branch is served at <repo>.<pages_domain>, straight out
of git. No export step, no build to drift: the push moves the branch and the
next request reads the new tree.

- internal/pages: a pool of `git cat-file --batch` processes per active
  repository (closed after two idle minutes), an LRU keyed by (commit, path),
  and tree parsing in git's binary format with the object-id width detected
  rather than assumed
- ETag is the blob sha, Last-Modified the commit time, Content-Type from the
  extension or a sniff; If-None-Match and If-Modified-Since answer 304
- directories serve index.html and redirect to add the trailing slash; path
  resolution walks trees, so a traversal-shaped request is a missing page
- repository and ACL are resolved once per one-second memo window together with
  the branch tip, which is what makes the read path fast: doing it per request
  spawned three git processes and cost 3.8 ms, against 6.7 µs warm
- sites follow clone visibility, so a private repository answers 404; the repo
  page links to the site when pages_domain is configured

Measured: a site pushed over SSH is live in 1.4 s (the phase criterion is 5 s);
warm requests 6.7 µs, cold 144 µs. Custom domains and their TLS certificates are
the remaining Phase 6 work.

internal/pages/pages_test.go +391

@@ -0,0 +1,391 @@
1 package pages
2
3 import (
4 "net/http"
5 "net/http/httptest"
6 "strconv"
7 "strings"
8 "testing"
9 "time"
10
11 "github.com/rodent-software/vole/internal/acl"
12 "github.com/rodent-software/vole/internal/gitx"
13 "github.com/rodent-software/vole/internal/repo"
14 )
15
16 const domain = "pages.test"
17
18 type fixture struct {
19 t testing.TB
20 root string
21 repos *repo.Store
22 srv *Server
23 }
24
25 func newFixture(t testing.TB) *fixture {
26 t.Helper()
27 root := t.TempDir()
28 repos := repo.New(root, "/bin/true")
29 m, err := repos.Create("site", "owner", "a site")
30 if err != nil {
31 t.Fatal(err)
32 }
33 a, err := m.ACL()
34 if err != nil {
35 t.Fatal(err)
36 }
37 a.Default = "read"
38 if err := m.SetACL(a, "test"); err != nil {
39 t.Fatal(err)
40 }
41 srv := New(Config{
42 Domain: domain,
43 Ref: "pages",
44 Procs: 2,
45 CacheBytes: 1 << 20,
46 TipTTL: 20 * time.Millisecond,
47 }, repos)
48 t.Cleanup(func() { srv.Close() })
49 return &fixture{t: t, root: root, repos: repos, srv: srv}
50 }
51
52 // publish commits files to the pages branch and returns the tip.
53 func (f *fixture) publish(files map[string][]byte) string {
54 f.t.Helper()
55 m, err := f.repos.Open("site")
56 if err != nil {
57 f.t.Fatal(err)
58 }
59 g := gitx.Open(m.Dir)
60 tree, err := gitx.WriteFlatTree(g, files)
61 if err != nil {
62 f.t.Fatal(err)
63 }
64 old, _ := g.RevParse("refs/heads/pages")
65 commit, err := g.CommitTree(tree, old, "site update")
66 if err != nil {
67 f.t.Fatal(err)
68 }
69 if err := g.UpdateRef("refs/heads/pages", commit); err != nil {
70 f.t.Fatal(err)
71 }
72 return commit
73 }
74
75 func (f *fixture) get(host, path string, headers ...[2]string) *httptest.ResponseRecorder {
76 f.t.Helper()
77 req := httptest.NewRequest(http.MethodGet, path, nil)
78 req.Host = host
79 for _, h := range headers {
80 req.Header.Set(h[0], h[1])
81 }
82 rec := httptest.NewRecorder()
83 f.srv.ServeHTTP(rec, req)
84 return rec
85 }
86
87 func TestHostNamesTheRepository(t *testing.T) {
88 f := newFixture(t)
89 cases := []struct {
90 host string
91 name string
92 ok bool
93 }{
94 {"site." + domain, "site", true},
95 {"site." + domain + ":8080", "site", true},
96 {"SITE." + domain, "site", true},
97 {domain, "", false},
98 {"other.example.com", "", false},
99 {"site.other.test", "", false},
100 {"not a name." + domain, "", false},
101 {"", "", false},
102 }
103 for _, c := range cases {
104 name, ok := f.srv.Host(c.host)
105 if ok != c.ok || name != c.name {
106 t.Errorf("Host(%q) = %q, %v; want %q, %v", c.host, name, ok, c.name, c.ok)
107 }
108 }
109 }
110
111 func TestServesTheBranchAsASite(t *testing.T) {
112 f := newFixture(t)
113 commit := f.publish(map[string][]byte{
114 "index.html": []byte("<!doctype html><title>home</title>\n"),
115 "about.html": []byte("<p>about</p>\n"),
116 "style.css": []byte("body{color:red}\n"),
117 "docs/index.html": []byte("<p>docs</p>\n"),
118 "docs/guide/intro.md": []byte("# intro\n"),
119 })
120
121 rec := f.get("site."+domain, "/")
122 if rec.Code != 200 || !strings.Contains(rec.Body.String(), "home") {
123 t.Fatalf("GET / = %d %q", rec.Code, rec.Body.String())
124 }
125 if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/html") {
126 t.Fatalf("index Content-Type = %q", ct)
127 }
128 // The ETag is the blob sha, which is what makes invalidation exact.
129 g := gitx.Open(mustDir(t, f))
130 entries, err := g.LsTreeR(commit)
131 if err != nil {
132 t.Fatal(err)
133 }
134 var indexSHA string
135 for _, e := range entries {
136 if e.Name == "index.html" {
137 indexSHA = e.SHA
138 }
139 }
140 if indexSHA == "" {
141 t.Fatalf("index.html is not in commit %s", commit)
142 }
143 if got := rec.Header().Get("ETag"); got != `"`+indexSHA+`"` {
144 t.Fatalf("ETag = %q, want the blob sha %s", got, indexSHA)
145 }
146 if rec.Header().Get("Last-Modified") == "" {
147 t.Fatal("no Last-Modified header")
148 }
149 if n := rec.Header().Get("Content-Length"); n != strconv.Itoa(rec.Body.Len()) {
150 t.Fatalf("Content-Length = %q, body is %d bytes", n, rec.Body.Len())
151 }
152
153 if rec := f.get("site."+domain, "/style.css"); rec.Code != 200 ||
154 !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/css") {
155 t.Fatalf("css = %d %q", rec.Code, rec.Header().Get("Content-Type"))
156 }
157 // A directory serves its index, and a directory URL without a slash redirects
158 // so that relative links inside it still resolve.
159 if rec := f.get("site."+domain, "/docs/"); rec.Code != 200 || !strings.Contains(rec.Body.String(), "docs") {
160 t.Fatalf("GET /docs/ = %d %q", rec.Code, rec.Body.String())
161 }
162 rec = f.get("site."+domain, "/docs")
163 if rec.Code != http.StatusMovedPermanently || rec.Header().Get("Location") != "/docs/" {
164 t.Fatalf("GET /docs = %d %q", rec.Code, rec.Header().Get("Location"))
165 }
166 // A path with no extension still gets a type, sniffed from the bytes.
167 if rec := f.get("site."+domain, "/docs/guide/intro.md"); rec.Code != 200 {
168 t.Fatalf("markdown = %d", rec.Code)
169 } else if rec.Header().Get("Content-Type") == "" {
170 t.Fatal("no Content-Type for an extension-less type")
171 }
172 }
173
174 func TestConditionalRequestsDoNotResendTheBody(t *testing.T) {
175 f := newFixture(t)
176 f.publish(map[string][]byte{"index.html": []byte("<h1>hello</h1>\n")})
177
178 first := f.get("site."+domain, "/")
179 if first.Code != 200 {
180 t.Fatalf("first GET = %d", first.Code)
181 }
182 etag := first.Header().Get("ETag")
183 if etag == "" {
184 t.Fatal("no ETag")
185 }
186 second := f.get("site."+domain, "/", [2]string{"If-None-Match", etag})
187 if second.Code != http.StatusNotModified {
188 t.Fatalf("If-None-Match = %d, want 304", second.Code)
189 }
190 if second.Body.Len() != 0 {
191 t.Fatalf("304 carried %d bytes", second.Body.Len())
192 }
193 // A weak or different tag gets the file, not a 304.
194 if rec := f.get("site."+domain, "/", [2]string{"If-None-Match", `"nope"`}); rec.Code != 200 {
195 t.Fatalf("stale If-None-Match = %d, want 200", rec.Code)
196 }
197 // Last-Modified is the fallback for clients that do not send an ETag.
198 if rec := f.get("site."+domain, "/", [2]string{"If-Modified-Since", first.Header().Get("Last-Modified")}); rec.Code != http.StatusNotModified {
199 t.Fatalf("If-Modified-Since = %d, want 304", rec.Code)
200 }
201 if rec := f.get("site."+domain, "/", [2]string{"If-Modified-Since", time.Now().Add(time.Hour).UTC().Format(http.TimeFormat)}); rec.Code != http.StatusNotModified {
202 t.Fatalf("If-Modified-Since in the future = %d, want 304", rec.Code)
203 }
204 }
205
206 func TestPathTraversalIsJustAMissingPage(t *testing.T) {
207 f := newFixture(t)
208 f.publish(map[string][]byte{
209 "index.html": []byte("home\n"),
210 "odd..name.txt": []byte("dots in a name are fine\n"),
211 })
212 bad := []string{
213 "/../../etc/passwd",
214 "/..%2f..%2fetc/passwd",
215 "/docs/../../etc/passwd",
216 "/%2e%2e/%2e%2e/etc/passwd",
217 "/index.html%00.png",
218 "/sub\\..\\..\\index.html",
219 }
220 for _, path := range bad {
221 if rec := f.get("site."+domain, path); rec.Code != http.StatusNotFound {
222 t.Errorf("GET %q = %d, want 404", path, rec.Code)
223 }
224 }
225 // A name containing dots is not a traversal and must still be served.
226 if rec := f.get("site."+domain, "/odd..name.txt"); rec.Code != 200 {
227 t.Fatalf("GET /odd..name.txt = %d, want 200", rec.Code)
228 }
229 // A traversal that stays inside the site resolves to a real page rather than
230 // escaping: path cleaning happens before the tree lookup.
231 if rec := f.get("site."+domain, "/docs/../index.html"); rec.Code != 200 {
232 t.Fatalf("GET /docs/../index.html = %d, want 200 (cleaned to /index.html)", rec.Code)
233 }
234 }
235
236 func TestNoSiteNoPages(t *testing.T) {
237 f := newFixture(t)
238 // No pages branch at all.
239 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusNotFound {
240 t.Fatalf("repo without a pages branch = %d, want 404", rec.Code)
241 }
242 // Unknown repository.
243 f.publish(map[string][]byte{"index.html": []byte("x\n")})
244 if rec := f.get("nosuch."+domain, "/"); rec.Code != http.StatusNotFound {
245 t.Fatalf("unknown repo = %d, want 404", rec.Code)
246 }
247 // A private repository has no site: there is no reader to authenticate.
248 m, err := f.repos.Open("site")
249 if err != nil {
250 t.Fatal(err)
251 }
252 a, err := m.ACL()
253 if err != nil {
254 t.Fatal(err)
255 }
256 a.Default = "none"
257 if err := m.SetACL(a, "test"); err != nil {
258 t.Fatal(err)
259 }
260 // Authorization is resolved once per memo window, so a visibility change
261 // lands within that window rather than on the next request. Waiting here is
262 // the test being honest about the documented behaviour.
263 time.Sleep(40 * time.Millisecond)
264 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusNotFound {
265 t.Fatalf("private repo = %d, want 404", rec.Code)
266 }
267 // A default that grants more than read still lets an anonymous reader read:
268 // the ACL is a role, and write implies read. Nobody can write over HTTP
269 // anyway, and SSH has no use for this default.
270 a.Default = "write"
271 if err := m.SetACL(a, "test"); err != nil {
272 t.Fatal(err)
273 }
274 time.Sleep(40 * time.Millisecond)
275 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusOK {
276 t.Fatalf("write-default repo = %d, want 200", rec.Code)
277 }
278 // ... and the forge UI is untouched by any of this.
279 req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
280 req.Host = "127.0.0.1:8080"
281 rec := httptest.NewRecorder()
282 f.srv.ServeHTTP(rec, req)
283 if rec.Code != http.StatusNotFound {
284 // The pages handler owns only its own host; anything else is not its
285 // business, and the mux answers. 404 here is the pages server saying so.
286 t.Fatalf("non-pages host = %d", rec.Code)
287 }
288 if _, ok := f.srv.Host("127.0.0.1:8080"); ok {
289 t.Fatal("the pages server claimed a host it does not own")
290 }
291 }
292
293 func TestNewCommitIsLiveAfterTheTipTTL(t *testing.T) {
294 f := newFixture(t)
295 f.publish(map[string][]byte{"index.html": []byte("version one\n")})
296 if body := f.get("site."+domain, "/").Body.String(); !strings.Contains(body, "version one") {
297 t.Fatalf("v1 body = %q", body)
298 }
299 f.publish(map[string][]byte{"index.html": []byte("version two\n")})
300 // Inside the TTL the old answer is still served: that is the price of not
301 // asking git on every request, and it is one second in production.
302 time.Sleep(40 * time.Millisecond)
303 if body := f.get("site."+domain, "/").Body.String(); !strings.Contains(body, "version two") {
304 t.Fatalf("after the TTL the site still says %q", body)
305 }
306 }
307
308 func mustDir(t testing.TB, f *fixture) string {
309 t.Helper()
310 m, err := f.repos.Open("site")
311 if err != nil {
312 t.Fatal(err)
313 }
314 return m.Dir
315 }
316
317 func TestCacheEvictsAndDropsCommits(t *testing.T) {
318 c := newLRU(1024) // maxEntry is an eighth of that: 128 bytes
319 for i := 0; i < 12; i++ {
320 c.put(&cacheEntry{key: "k" + strconv.Itoa(i), commit: "c1", body: make([]byte, 100)})
321 }
322 if _, ok := c.get("k0"); ok {
323 t.Fatal("the oldest entry survived an eviction")
324 }
325 if _, ok := c.get("k11"); !ok {
326 t.Fatal("the newest entry was evicted")
327 }
328 if entries, used := c.stats(); used > 1024 {
329 t.Fatalf("cache holds %d bytes, over its %d budget (%d entries)", used, 1024, entries)
330 }
331 c.put(&cacheEntry{key: "big", commit: "c1", body: make([]byte, 4096)})
332 if _, ok := c.get("big"); ok {
333 t.Fatal("an entry larger than maxEntry was cached")
334 }
335 c.setLookup("c1\x00index.html", treeEntry{sha: "x", typ: "blob"})
336 c.setLookup("c2\x00index.html", treeEntry{sha: "y", typ: "blob"})
337 c.dropCommit("c1")
338 if _, ok := c.lookup("c1\x00index.html"); ok {
339 t.Fatal("dropCommit left a lookup behind")
340 }
341 if _, ok := c.lookup("c2\x00index.html"); !ok {
342 t.Fatal("dropCommit removed another commit's lookup")
343 }
344 }
345
346 // The ACL type is used by the fixture through repo.Meta; keep the import honest.
347 var _ = acl.Read
348
349 // Benchmarks for the read path. They are not part of the gate; run them when
350 // changing the cache or the pool:
351 //
352 // go test ./internal/pages/ -bench . -benchtime=1s -run XXX
353 func benchFixture(b *testing.B) *fixture {
354 b.Helper()
355 f := newFixture(b)
356 f.publish(map[string][]byte{
357 "index.html": []byte(strings.Repeat("<p>page body</p>\n", 256)),
358 "docs/index.html": []byte(strings.Repeat("<p>docs</p>\n", 256)),
359 })
360 return f
361 }
362
363 // BenchmarkPageWarm measures a hit in the blob and lookup caches.
364 func BenchmarkPageWarm(b *testing.B) {
365 f := benchFixture(b)
366 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusOK {
367 b.Fatalf("warm-up request = %d", rec.Code)
368 }
369 b.ResetTimer()
370 for i := 0; i < b.N; i++ {
371 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusOK {
372 b.Fatalf("request %d = %d", i, rec.Code)
373 }
374 }
375 }
376
377 // BenchmarkPageCold measures the same request with the cache emptied, so every
378 // op walks the trees through the cat-file pool and reads the blob.
379 func BenchmarkPageCold(b *testing.B) {
380 f := benchFixture(b)
381 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusOK {
382 b.Fatalf("warm-up request = %d", rec.Code)
383 }
384 b.ResetTimer()
385 for i := 0; i < b.N; i++ {
386 f.srv.cache = newLRU(1 << 20)
387 if rec := f.get("site."+domain, "/"); rec.Code != http.StatusOK {
388 b.Fatalf("request %d = %d", i, rec.Code)
389 }
390 }
391 }