Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 6 slice 1: the tree is the site

d3296cda0c18f78097e37373822292707113d245 · Keenan Nemetz · 18d ago · parent 64ac5071 

Phase 6 slice 1: the tree is the site

A repository's `pages` branch is served at <repo>.<pages_domain>, straight out
of git. No export step, no build to drift: the push moves the branch and the
next request reads the new tree.

- internal/pages: a pool of `git cat-file --batch` processes per active
  repository (closed after two idle minutes), an LRU keyed by (commit, path),
  and tree parsing in git's binary format with the object-id width detected
  rather than assumed
- ETag is the blob sha, Last-Modified the commit time, Content-Type from the
  extension or a sniff; If-None-Match and If-Modified-Since answer 304
- directories serve index.html and redirect to add the trailing slash; path
  resolution walks trees, so a traversal-shaped request is a missing page
- repository and ACL are resolved once per one-second memo window together with
  the branch tip, which is what makes the read path fast: doing it per request
  spawned three git processes and cost 3.8 ms, against 6.7 µs warm
- sites follow clone visibility, so a private repository answers 404; the repo
  page links to the site when pages_domain is configured

Measured: a site pushed over SSH is live in 1.4 s (the phase criterion is 5 s);
warm requests 6.7 µs, cold 144 µs. Custom domains and their TLS certificates are
the remaining Phase 6 work.

internal/pages/pages.go +562

@@ -0,0 +1,562 @@
1 // Package pages serves a repository's site straight out of its git tree.
2 //
3 // There is no export step and no build to drift: the site is a branch, a push
4 // moves the tip, and the next request reads the new tree. Two consequences shape
5 // the design, both measured in docs/spikes/tree-serving:
6 //
7 // - Blobs come through a pool of `git cat-file --batch` processes, because one
8 // process serialises every read (12.1k req/s vs 24.8k with four).
9 // - Results are cached by (commit, path), because a warm read is 16× faster
10 // than a cold one — the cache is what makes this feel like a file server.
11 //
12 // Nothing here builds a filesystem path out of a URL: a request is looked up in
13 // the tree it names, so a path shaped like a traversal simply does not resolve.
14 package pages
15
16 import (
17 "encoding/hex"
18 "errors"
19 "fmt"
20 "log/slog"
21 "mime"
22 "net"
23 "net/http"
24 "path"
25 "strconv"
26 "strings"
27 "sync"
28 "time"
29
30 "github.com/rodent-software/vole/internal/acl"
31 "github.com/rodent-software/vole/internal/repo"
32 )
33
34 const (
35 // tipTTL is how long a resolved branch tip is trusted. A push is live within
36 // this window, well inside the phase's five-second criterion, and in exchange
37 // git sees one ref read per second per site instead of one per request.
38 tipTTL = time.Second
39 // poolIdle is how long an unused cat-file pool lives. A forge with a hundred
40 // repositories should not keep four hundred processes because somebody once
41 // looked at a site.
42 poolIdle = 2 * time.Minute
43 )
44
45 type Config struct {
46 // Domain is the suffix a site is served under: <repo>.<domain>.
47 Domain string
48 // Ref is the branch the site is read from.
49 Ref string
50 // Procs is the number of cat-file processes per active repository.
51 Procs int
52 // CacheBytes caps the in-memory blob cache.
53 CacheBytes int64
54 // TipTTL overrides how long a resolved branch tip is trusted. Zero means the
55 // one-second default; tests shorten it rather than sleeping.
56 TipTTL time.Duration
57 Log *slog.Logger
58 }
59
60 type Server struct {
61 cfg Config
62 repos *repo.Store
63
64 mu sync.Mutex
65 pools map[string]*poolRef
66 sites map[string]siteState
67 cache *lruCache
68 stop chan struct{}
69 once sync.Once
70 }
71
72 type poolRef struct {
73 pool *objectPool
74 lastUsed time.Time
75 }
76
77 // siteState is everything a request needs to route and authorize, resolved in one
78 // place and reused for the memo window. Resolving it means git subprocesses —
79 // `Open` reads the default branch, `ACL` reads a ref — and the Phase 6 benchmark
80 // showed three spawns per request costing ~3.8 ms, forty times the cost of serving
81 // the blob itself.
82 type siteState struct {
83 m *repo.Meta
84 public bool
85 sha string // "" means the repository has no site branch
86 mod int64
87 at time.Time
88 }
89
90 func New(cfg Config, repos *repo.Store) *Server {
91 if cfg.Ref == "" {
92 cfg.Ref = "pages"
93 }
94 if cfg.Procs <= 0 {
95 cfg.Procs = 4
96 }
97 if cfg.Log == nil {
98 cfg.Log = slog.Default()
99 }
100 if cfg.TipTTL <= 0 {
101 cfg.TipTTL = tipTTL
102 }
103 s := &Server{
104 cfg: cfg,
105 repos: repos,
106 pools: map[string]*poolRef{},
107 sites: map[string]siteState{},
108 cache: newLRU(cfg.CacheBytes),
109 stop: make(chan struct{}),
110 }
111 go s.janitor()
112 return s
113 }
114
115 // Domain is the suffix sites are served under, for links and for tests.
116 func (s *Server) Domain() string { return s.cfg.Domain }
117
118 // Host reports the repository a request host names. The suffix decides: every
119 // other host belongs to the forge UI, not to a site.
120 func (s *Server) Host(hostport string) (name string, ok bool) {
121 if s.cfg.Domain == "" {
122 return "", false
123 }
124 host := hostport
125 if h, _, err := net.SplitHostPort(hostport); err == nil {
126 host = h
127 }
128 host = strings.ToLower(strings.TrimSuffix(host, "."))
129 suffix := "." + strings.ToLower(s.cfg.Domain)
130 if !strings.HasSuffix(host, suffix) {
131 return "", false
132 }
133 name = strings.TrimSuffix(host, suffix)
134 // Repository names may contain dots, so the whole prefix is the name; it is
135 // validated rather than parsed.
136 if name == "" {
137 return "", false
138 }
139 if err := repo.ValidateName(name); err != nil {
140 return "", false
141 }
142 return name, true
143 }
144
145 func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
146 name, ok := s.Host(r.Host)
147 if !ok {
148 http.NotFound(w, r)
149 return
150 }
151 st, err := s.state(name)
152 if err != nil || !st.public || st.sha == "" {
153 // A site is as public as its clone URL: there is no session with which to
154 // authenticate an anonymous reader, so a private repository has no site,
155 // and a repository without the branch has nothing to serve.
156 http.NotFound(w, r)
157 return
158 }
159 m, commit, mod := st.m, st.sha, st.mod
160
161 rel, ok := cleanPath(r.URL.Path)
162 if !ok {
163 http.NotFound(w, r)
164 return
165 }
166 ent, err := s.lookup(m, commit, rel)
167 if err != nil {
168 http.NotFound(w, r)
169 return
170 }
171 // A directory is a file called index.html, and a directory URL without its
172 // trailing slash would break every relative link inside it.
173 if ent.typ == "tree" {
174 if !strings.HasSuffix(r.URL.Path, "/") {
175 target := r.URL.Path + "/"
176 if r.URL.RawQuery != "" {
177 target += "?" + r.URL.RawQuery
178 }
179 http.Redirect(w, r, target, http.StatusMovedPermanently)
180 return
181 }
182 rel = path.Join(rel, "index.html")
183 ent, err = s.lookup(m, commit, rel)
184 if err != nil {
185 http.NotFound(w, r)
186 return
187 }
188 }
189 if ent.typ != "blob" {
190 http.NotFound(w, r)
191 return
192 }
193
194 page, err := s.file(m, commit, rel, ent.sha, mod)
195 if err != nil {
196 s.cfg.Log.Error("pages: read blob", "repo", m.Name, "path", rel, "err", err)
197 http.Error(w, "vole: cannot read that file", http.StatusInternalServerError)
198 return
199 }
200
201 h := w.Header()
202 etag := `"` + page.sha + `"`
203 h.Set("ETag", etag)
204 h.Set("Last-Modified", time.Unix(page.modUnix, 0).UTC().Format(http.TimeFormat))
205 h.Set("X-Content-Type-Options", "nosniff")
206 // A site changes by pushing, so a cached copy must be revalidated; the
207 // revalidation is a 304 off the blob sha and costs one cached tree walk.
208 h.Set("Cache-Control", "public, max-age=0, must-revalidate")
209
210 if fresh(r, etag, page.modUnix) {
211 w.WriteHeader(http.StatusNotModified)
212 return
213 }
214 h.Set("Content-Type", page.ctype)
215 h.Set("Content-Length", strconv.Itoa(len(page.body)))
216 if r.Method == http.MethodHead {
217 return
218 }
219 if _, err := w.Write(page.body); err != nil {
220 s.cfg.Log.Debug("pages: write interrupted", "repo", m.Name, "path", rel, "err", err)
221 }
222 }
223
224 // fresh implements the conditional request rules a static server is expected to
225 // follow: ETag wins when present, Last-Modified otherwise.
226 func fresh(r *http.Request, etag string, modUnix int64) bool {
227 if inm := r.Header.Get("If-None-Match"); inm != "" {
228 for _, candidate := range strings.Split(inm, ",") {
229 if strings.TrimSpace(candidate) == etag {
230 return true
231 }
232 }
233 return false
234 }
235 if ims := r.Header.Get("If-Modified-Since"); ims != "" {
236 if t, err := http.ParseTime(ims); err == nil {
237 // HTTP dates have one-second resolution, and a commit made in the same
238 // second is not newer than the client's copy.
239 return modUnix <= t.Unix()
240 }
241 }
242 return false
243 }
244
245 // page is one servable file: everything needed to answer, cached or not.
246 type page struct {
247 sha string
248 ctype string
249 body []byte
250 modUnix int64
251 }
252
253 func (s *Server) file(m *repo.Meta, commit, rel, sha string, mod int64) (*page, error) {
254 key := commit + "\x00" + rel
255 if e, ok := s.cache.get(key); ok {
256 return &page{sha: e.sha, ctype: e.ctype, body: e.body, modUnix: e.modUnix}, nil
257 }
258 pool, err := s.pool(m)
259 if err != nil {
260 return nil, err
261 }
262 obj, err := pool.read(sha)
263 if err != nil {
264 return nil, err
265 }
266 if obj.Type != "blob" {
267 return nil, fmt.Errorf("pages: %s is a %s, not a file", rel, obj.Type)
268 }
269 p := &page{sha: obj.SHA, body: obj.Data, modUnix: mod, ctype: contentType(rel, obj.Data)}
270 s.cache.put(&cacheEntry{
271 key: key, commit: commit, sha: p.sha, ctype: p.ctype, body: p.body, modUnix: mod,
272 })
273 return p, nil
274 }
275
276 func contentType(name string, body []byte) string {
277 if ct := mime.TypeByExtension(path.Ext(name)); ct != "" {
278 return ct
279 }
280 return http.DetectContentType(body)
281 }
282
283 // cleanPath turns a URL path into a tree path, or refuses it. It is deliberately
284 // strict: the tree lookup cannot escape anywhere, but a path with dot segments is
285 // never a page anyone meant to publish.
286 func cleanPath(p string) (string, bool) {
287 if strings.ContainsRune(p, 0) || strings.Contains(p, "\\") {
288 return "", false
289 }
290 trimmed := strings.Trim(path.Clean("/"+p), "/")
291 if trimmed == "" {
292 return "", true // site root: index.html
293 }
294 for _, seg := range strings.Split(trimmed, "/") {
295 if seg == "" || seg == "." || seg == ".." {
296 return "", false
297 }
298 }
299 return trimmed, true
300 }
301
302 // treeEntry is one record of a git tree object.
303 type treeEntry struct {
304 name string
305 sha string
306 typ string
307 mode string
308 }
309
310 // parseTree reads a tree object in git's own binary format: "<mode> <name>\0"
311 // followed by the raw object id. `git cat-file --batch` hands back the object,
312 // not `ls-tree` text, so the id is bytes and its width depends on the repository's
313 // hash algorithm — 20 for sha1, 32 for sha256. Guessing wrong would silently
314 // mis-parse the next entry, so the width is detected rather than assumed.
315 func parseTree(data []byte) []treeEntry {
316 for _, width := range []int{20, 32} {
317 if entries, ok := parseTreeWidth(data, width); ok {
318 return entries
319 }
320 }
321 return nil
322 }
323
324 func parseTreeWidth(data []byte, width int) ([]treeEntry, bool) {
325 var out []treeEntry
326 for len(data) > 0 {
327 sp := indexByte(data, ' ')
328 if sp <= 0 {
329 return nil, false
330 }
331 mode := string(data[:sp])
332 rest := data[sp+1:]
333 nul := indexByte(rest, 0)
334 if nul <= 0 || len(rest) < nul+1+width {
335 return nil, false
336 }
337 name := string(rest[:nul])
338 sha := hex.EncodeToString(rest[nul+1 : nul+1+width])
339 out = append(out, treeEntry{name: name, mode: mode, typ: modeType(mode), sha: sha})
340 data = rest[nul+1+width:]
341 }
342 return out, true
343 }
344
345 func indexByte(b []byte, c byte) int {
346 for i := range b {
347 if b[i] == c {
348 return i
349 }
350 }
351 return -1
352 }
353
354 // modeType is how a tree entry's type is written down: the object itself carries
355 // only a mode.
356 func modeType(mode string) string {
357 switch mode {
358 case "40000", "040000":
359 return "tree"
360 case "160000":
361 return "commit"
362 }
363 return "blob"
364 }
365
366 // lookup resolves a path inside a commit. Each level is one tree read, and the
367 // result is cached, so the second request for a page reads no trees at all.
368 func (s *Server) lookup(m *repo.Meta, commit, rel string) (treeEntry, error) {
369 key := commit + "\x00" + rel
370 if e, ok := s.cache.lookup(key); ok {
371 return e, nil
372 }
373 pool, err := s.pool(m)
374 if err != nil {
375 return treeEntry{}, err
376 }
377 obj, err := pool.read(commit)
378 if err != nil {
379 return treeEntry{}, err
380 }
381 if obj.Type != "commit" {
382 return treeEntry{}, fmt.Errorf("pages: %s does not point at a commit", s.cfg.Ref)
383 }
384 root, err := treeOf(obj.Data)
385 if err != nil {
386 return treeEntry{}, err
387 }
388 cur := treeEntry{sha: root, typ: "tree"}
389 if rel == "" {
390 s.cache.setLookup(key, cur)
391 return cur, nil
392 }
393 for _, seg := range strings.Split(rel, "/") {
394 if cur.typ != "tree" {
395 return treeEntry{}, errMissing
396 }
397 obj, err := pool.read(cur.sha)
398 if err != nil {
399 return treeEntry{}, err
400 }
401 var found bool
402 for _, e := range parseTree(obj.Data) {
403 if e.name == seg {
404 cur, found = e, true
405 break
406 }
407 }
408 if !found {
409 return treeEntry{}, errMissing
410 }
411 }
412 s.cache.setLookup(key, cur)
413 return cur, nil
414 }
415
416 // treeOf extracts the root tree id from a commit object.
417 func treeOf(commit []byte) (string, error) {
418 for _, line := range strings.Split(string(commit), "\n") {
419 if line == "" {
420 break
421 }
422 if rest, ok := strings.CutPrefix(line, "tree "); ok {
423 return strings.TrimSpace(rest), nil
424 }
425 }
426 return "", errors.New("pages: commit has no tree")
427 }
428
429 // commitTime extracts the committer timestamp, which is what Last-Modified means
430 // for a static site.
431 func commitTime(commit []byte) int64 {
432 for _, line := range strings.Split(string(commit), "\n") {
433 if line == "" {
434 break
435 }
436 rest, ok := strings.CutPrefix(line, "committer ")
437 if !ok {
438 continue
439 }
440 f := strings.Fields(rest)
441 if len(f) < 2 {
442 return 0
443 }
444 n, err := strconv.ParseInt(f[len(f)-2], 10, 64)
445 if err != nil {
446 return 0
447 }
448 return n
449 }
450 return 0
451 }
452
453 // state resolves a repository for serving, and remembers the answer for the memo
454 // window. It is the only place that pays for git subprocesses on the read path.
455 func (s *Server) state(name string) (siteState, error) {
456 s.mu.Lock()
457 cached, ok := s.sites[name]
458 s.mu.Unlock()
459 if ok && time.Since(cached.at) < s.cfg.TipTTL {
460 return cached, nil
461 }
462
463 m, err := s.repos.Open(name)
464 if err != nil {
465 return siteState{}, err
466 }
467 st := siteState{m: m, at: time.Now()}
468 if a, err := m.ACL(); err == nil {
469 st.public = a.Can("", acl.Read)
470 }
471 if st.public {
472 if sha, mod, err := s.tip(m); err == nil {
473 st.sha, st.mod = sha, mod
474 }
475 }
476
477 s.mu.Lock()
478 prev := s.sites[name]
479 s.sites[name] = st
480 s.mu.Unlock()
481 if prev.sha != "" && prev.sha != st.sha {
482 s.cache.dropCommit(prev.sha)
483 }
484 return st, nil
485 }
486
487 // tip answers which commit the site branch is on. It is called from state(), so
488 // it runs once per memo window, not once per request.
489 func (s *Server) tip(m *repo.Meta) (string, int64, error) {
490 pool, err := s.pool(m)
491 if err != nil {
492 return "", 0, err
493 }
494 obj, err := pool.read("refs/heads/" + s.cfg.Ref)
495 if err != nil {
496 return "", 0, err
497 }
498 if obj.Type != "commit" {
499 return "", 0, fmt.Errorf("pages: %s does not point at a commit", s.cfg.Ref)
500 }
501 return obj.SHA, commitTime(obj.Data), nil
502 }
503
504 func (s *Server) pool(m *repo.Meta) (*objectPool, error) {
505 s.mu.Lock()
506 defer s.mu.Unlock()
507 if pr, ok := s.pools[m.Name]; ok {
508 pr.lastUsed = time.Now()
509 return pr.pool, nil
510 }
511 pool := newObjectPool(m.Dir, s.cfg.Procs)
512 s.pools[m.Name] = &poolRef{pool: pool, lastUsed: time.Now()}
513 return pool, nil
514 }
515
516 // janitor closes pools nobody has touched, so process count tracks sites being
517 // read rather than repositories that exist.
518 func (s *Server) janitor() {
519 t := time.NewTicker(time.Minute)
520 defer t.Stop()
521 for {
522 select {
523 case <-s.stop:
524 return
525 case now := <-t.C:
526 s.mu.Lock()
527 for name, pr := range s.pools {
528 if now.Sub(pr.lastUsed) > poolIdle {
529 pr.pool.Close()
530 delete(s.pools, name)
531 delete(s.sites, name)
532 }
533 }
534 s.mu.Unlock()
535 }
536 }
537 }
538
539 func (s *Server) Close() error {
540 s.once.Do(func() { close(s.stop) })
541 s.mu.Lock()
542 defer s.mu.Unlock()
543 for _, pr := range s.pools {
544 pr.pool.Close()
545 }
546 s.pools = map[string]*poolRef{}
547 return nil
548 }
549
550 // Stats is for the health endpoint and for tests that want to prove the cache did
551 // something.
552 func (s *Server) Stats() map[string]int64 {
553 entries, bytes := s.cache.stats()
554 s.mu.Lock()
555 pools := int64(len(s.pools))
556 s.mu.Unlock()
557 return map[string]int64{
558 "cache_entries": int64(entries),
559 "cache_bytes": bytes,
560 "pools": pools,
561 }
562 }