Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

392 commit(s) on main

Compare History
Merge change request 01M70WES5ZZDJ6RP226WB11BBZ: Let people rename and remove their own passkeys keenan · 2h ago 4b8c9e1d
history blame raw 15.7 KiB
1// Package pages serves a repository's site straight out of its git tree.
2//
3// There is no export step and no build to drift: the site is a branch, a push
4// moves the tip, and the next request reads the new tree. Two measurements shape
5// the design:
6//
7// - Blobs come through a pool of `git cat-file --batch` processes, because one
8// process serialises every read (12.1k req/s vs 24.8k with four).
9// - Results are cached by (commit, path), because a warm read is 16× faster
10// than a cold one — the cache is what makes this feel like a file server.
11//
12// Nothing here builds a filesystem path out of a URL: a request is looked up in
13// the tree it names, so a path shaped like a traversal simply does not resolve.
14package pages
15
16import (
17 "encoding/hex"
18 "errors"
19 "fmt"
20 "log/slog"
21 "mime"
22 "net"
23 "net/http"
24 "path"
25 "strconv"
26 "strings"
27 "sync"
28 "time"
29
30 "rodent.software/vole/forge/access"
31 "rodent.software/vole/forge/repo"
32 "rodent.software/vole/internal/format"
33)
34
35const (
36 // tipTTL is how long a resolved branch tip is trusted. A push is live within
37 // this window, well under five seconds, and in exchange git sees one ref read
38 // per second per site instead of one per request.
39 tipTTL = time.Second
40 // poolIdle is how long an unused cat-file pool lives. A forge with a hundred
41 // repositories should not keep four hundred processes because somebody once
42 // looked at a site.
43 poolIdle = 2 * time.Minute
44 // maxPageBytes is the largest file a site will serve. A site is a website:
45 // a blob past this size is repository content being used as a file host, and
46 // answering 413 costs one file's worth of nothing.
47 maxPageBytes = 25 << 20
48 // maxStructureBytes bounds the commit and tree objects read to resolve a
49 // path. A tree big enough to hit this names more files than a site can have.
50 maxStructureBytes = 64 << 20
51)
52
53// errObjectTooLarge is returned instead of an allocation when an object the site
54// asked for is past its cap.
55var errObjectTooLarge = errors.New("pages: object too large")
56
57type Config struct {
58 // Domain is the suffix a site is served under: <repo>.<domain>.
59 Domain string
60 // Ref is the branch the site is read from.
61 Ref string
62 // Procs is the number of cat-file processes per active repository.
63 Procs int
64 // CacheBytes caps the in-memory blob cache.
65 CacheBytes int64
66 // TipTTL overrides how long a resolved branch tip is trusted. Zero means the
67 // one-second default; tests shorten it rather than sleeping.
68 TipTTL time.Duration
69 Log *slog.Logger
70}
71
72type Server struct {
73 cfg Config
74 repos *repo.Store
75
76 mu sync.Mutex
77 pools map[string]*poolRef
78 sites map[string]siteState
79 cache *lruCache
80 stop chan struct{}
81 once sync.Once
82}
83
84type poolRef struct {
85 pool *objectPool
86 lastUsed time.Time
87}
88
89// siteState is everything a request needs to route and authorize, resolved in one
90// place and reused for the memo window. Resolving it means git subprocesses —
91// `Open` reads the default branch, `ACL` reads a ref — and benchmarking showed
92// three spawns per request costing ~3.8 ms, forty times the cost of serving the
93// blob itself.
94type siteState struct {
95 m *repo.Meta
96 public bool
97 sha string // "" means the repository has no site branch
98 mod int64
99 at time.Time
100}
101
102func New(cfg Config, repos *repo.Store) *Server {
103 if cfg.Ref == "" {
104 cfg.Ref = "pages"
105 }
106 if cfg.Procs <= 0 {
107 cfg.Procs = 4
108 }
109 if cfg.Log == nil {
110 cfg.Log = slog.Default()
111 }
112 if cfg.TipTTL <= 0 {
113 cfg.TipTTL = tipTTL
114 }
115 s := &Server{
116 cfg: cfg,
117 repos: repos,
118 pools: map[string]*poolRef{},
119 sites: map[string]siteState{},
120 cache: newLRU(cfg.CacheBytes),
121 stop: make(chan struct{}),
122 }
123 go s.janitor()
124 return s
125}
126
127// Domain is the suffix sites are served under, for links and for tests.
128func (s *Server) Domain() string { return s.cfg.Domain }
129
130// Host reports the repository a request host names. The suffix decides: every
131// other host belongs to the forge UI, not to a site.
132func (s *Server) Host(hostport string) (name string, ok bool) {
133 if s.cfg.Domain == "" {
134 return "", false
135 }
136 host := hostport
137 if h, _, err := net.SplitHostPort(hostport); err == nil {
138 host = h
139 }
140 host = strings.ToLower(strings.TrimSuffix(host, "."))
141 suffix := "." + strings.ToLower(s.cfg.Domain)
142 if !strings.HasSuffix(host, suffix) {
143 return "", false
144 }
145 name = strings.TrimSuffix(host, suffix)
146 // Repository names may contain dots, so the whole prefix is the name; it is
147 // validated rather than parsed.
148 if name == "" {
149 return "", false
150 }
151 if err := repo.ValidateName(name); err != nil {
152 return "", false
153 }
154 return name, true
155}
156
157func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
158 name, ok := s.Host(r.Host)
159 if !ok {
160 http.NotFound(w, r)
161 return
162 }
163 st, err := s.state(name)
164 if err != nil || !st.public || st.sha == "" {
165 // A site is as public as its clone URL: there is no session with which to
166 // authenticate an anonymous reader, so a private repository has no site,
167 // and a repository without the branch has nothing to serve.
168 http.NotFound(w, r)
169 return
170 }
171 m, commit, mod := st.m, st.sha, st.mod
172
173 rel, ok := cleanPath(r.URL.Path)
174 if !ok {
175 http.NotFound(w, r)
176 return
177 }
178 ent, err := s.lookup(m, commit, rel)
179 if err != nil {
180 http.NotFound(w, r)
181 return
182 }
183 // A directory is a file called index.html, and a directory URL without its
184 // trailing slash would break every relative link inside it.
185 if ent.typ == "tree" {
186 if !strings.HasSuffix(r.URL.Path, "/") {
187 target := r.URL.Path + "/"
188 if r.URL.RawQuery != "" {
189 target += "?" + r.URL.RawQuery
190 }
191 http.Redirect(w, r, target, http.StatusMovedPermanently)
192 return
193 }
194 rel = path.Join(rel, "index.html")
195 ent, err = s.lookup(m, commit, rel)
196 if err != nil {
197 http.NotFound(w, r)
198 return
199 }
200 }
201 if ent.typ != "blob" {
202 http.NotFound(w, r)
203 return
204 }
205
206 page, err := s.file(m, commit, rel, ent.sha, mod)
207 if err != nil {
208 if errors.Is(err, errObjectTooLarge) {
209 // Say why: a site reader whose asset is bigger than the limit needs to
210 // know it was the size and not a broken link.
211 http.Error(w, fmt.Sprintf("vole: this file is larger than the %s a site serves", format.Bytes(maxPageBytes)), http.StatusRequestEntityTooLarge)
212 return
213 }
214 s.cfg.Log.Error("pages: read blob", "repo", m.Name, "path", rel, "err", err)
215 http.Error(w, "vole: cannot read that file", http.StatusInternalServerError)
216 return
217 }
218
219 h := w.Header()
220 etag := `"` + page.sha + `"`
221 h.Set("ETag", etag)
222 h.Set("Last-Modified", time.Unix(page.modUnix, 0).UTC().Format(http.TimeFormat))
223 h.Set("X-Content-Type-Options", "nosniff")
224 // A site changes by pushing, so a cached copy must be revalidated; the
225 // revalidation is a 304 off the blob sha and costs one cached tree walk.
226 h.Set("Cache-Control", "public, max-age=0, must-revalidate")
227
228 if fresh(r, etag, page.modUnix) {
229 w.WriteHeader(http.StatusNotModified)
230 return
231 }
232 h.Set("Content-Type", page.ctype)
233 h.Set("Content-Length", strconv.Itoa(len(page.body)))
234 if r.Method == http.MethodHead {
235 return
236 }
237 if _, err := w.Write(page.body); err != nil {
238 s.cfg.Log.Debug("pages: write interrupted", "repo", m.Name, "path", rel, "err", err)
239 }
240}
241
242// fresh implements the conditional request rules a static server is expected to
243// follow: ETag wins when present, Last-Modified otherwise.
244func fresh(r *http.Request, etag string, modUnix int64) bool {
245 if inm := r.Header.Get("If-None-Match"); inm != "" {
246 for _, candidate := range strings.Split(inm, ",") {
247 if strings.TrimSpace(candidate) == etag {
248 return true
249 }
250 }
251 return false
252 }
253 if ims := r.Header.Get("If-Modified-Since"); ims != "" {
254 if t, err := http.ParseTime(ims); err == nil {
255 // HTTP dates have one-second resolution, and a commit made in the same
256 // second is not newer than the client's copy.
257 return modUnix <= t.Unix()
258 }
259 }
260 return false
261}
262
263// page is one servable file: everything needed to answer, cached or not.
264type page struct {
265 sha string
266 ctype string
267 body []byte
268 modUnix int64
269}
270
271func (s *Server) file(m *repo.Meta, commit, rel, sha string, mod int64) (*page, error) {
272 key := commit + "\x00" + rel
273 if e, ok := s.cache.get(key); ok {
274 return &page{sha: e.sha, ctype: e.ctype, body: e.body, modUnix: e.modUnix}, nil
275 }
276 pool, err := s.pool(m)
277 if err != nil {
278 return nil, err
279 }
280 obj, err := pool.read(sha, maxPageBytes)
281 if err != nil {
282 return nil, err
283 }
284 if obj.Type != "blob" {
285 return nil, fmt.Errorf("pages: %s is a %s, not a file", rel, obj.Type)
286 }
287 p := &page{sha: obj.SHA, body: obj.Data, modUnix: mod, ctype: contentType(rel, obj.Data)}
288 s.cache.put(&cacheEntry{
289 key: key, commit: commit, sha: p.sha, ctype: p.ctype, body: p.body, modUnix: mod,
290 })
291 return p, nil
292}
293
294func contentType(name string, body []byte) string {
295 if ct := mime.TypeByExtension(path.Ext(name)); ct != "" {
296 return ct
297 }
298 return http.DetectContentType(body)
299}
300
301// cleanPath turns a URL path into a tree path, or refuses it. Dot segments are
302// resolved before the check (`../../etc/passwd` becomes `etc/passwd`), and what
303// remains is refused unless every segment is a name. The tree lookup cannot escape
304// anywhere either way, but a caller that builds a link, a cache key, or a log line
305// from the result must not be handed a value with `..` in it.
306func cleanPath(p string) (string, bool) {
307 if strings.ContainsRune(p, 0) || strings.Contains(p, "\\") {
308 return "", false
309 }
310 trimmed := strings.Trim(path.Clean("/"+p), "/")
311 if trimmed == "" {
312 return "", true // site root: index.html
313 }
314 for _, seg := range strings.Split(trimmed, "/") {
315 if seg == "" || seg == "." || seg == ".." {
316 return "", false
317 }
318 }
319 return trimmed, true
320}
321
322// treeEntry is one record of a git tree object.
323type treeEntry struct {
324 name string
325 sha string
326 typ string
327 mode string
328}
329
330// parseTree reads a tree object in git's own binary format: "<mode> <name>\0"
331// followed by the raw object id. `git cat-file --batch` hands back the object,
332// not `ls-tree` text, so the id is bytes and its width depends on the repository's
333// hash algorithm — 20 for sha1, 32 for sha256. Guessing wrong would silently
334// mis-parse the next entry, so the width is detected rather than assumed.
335func parseTree(data []byte) []treeEntry {
336 for _, width := range []int{20, 32} {
337 if entries, ok := parseTreeWidth(data, width); ok {
338 return entries
339 }
340 }
341 return nil
342}
343
344func parseTreeWidth(data []byte, width int) ([]treeEntry, bool) {
345 var out []treeEntry
346 for len(data) > 0 {
347 sp := indexByte(data, ' ')
348 if sp <= 0 {
349 return nil, false
350 }
351 mode := string(data[:sp])
352 rest := data[sp+1:]
353 nul := indexByte(rest, 0)
354 if nul <= 0 || len(rest) < nul+1+width {
355 return nil, false
356 }
357 name := string(rest[:nul])
358 sha := hex.EncodeToString(rest[nul+1 : nul+1+width])
359 out = append(out, treeEntry{name: name, mode: mode, typ: modeType(mode), sha: sha})
360 data = rest[nul+1+width:]
361 }
362 return out, true
363}
364
365func indexByte(b []byte, c byte) int {
366 for i := range b {
367 if b[i] == c {
368 return i
369 }
370 }
371 return -1
372}
373
374// modeType is how a tree entry's type is written down: the object itself carries
375// only a mode.
376func modeType(mode string) string {
377 switch mode {
378 case "40000", "040000":
379 return "tree"
380 case "160000":
381 return "commit"
382 }
383 return "blob"
384}
385
386// lookup resolves a path inside a commit. Each level is one tree read, and the
387// result is cached, so the second request for a page reads no trees at all.
388func (s *Server) lookup(m *repo.Meta, commit, rel string) (treeEntry, error) {
389 key := commit + "\x00" + rel
390 if e, ok := s.cache.lookup(key); ok {
391 return e, nil
392 }
393 pool, err := s.pool(m)
394 if err != nil {
395 return treeEntry{}, err
396 }
397 obj, err := pool.read(commit, maxStructureBytes)
398 if err != nil {
399 return treeEntry{}, err
400 }
401 if obj.Type != "commit" {
402 return treeEntry{}, fmt.Errorf("pages: %s does not point at a commit", s.cfg.Ref)
403 }
404 root, err := treeOf(obj.Data)
405 if err != nil {
406 return treeEntry{}, err
407 }
408 cur := treeEntry{sha: root, typ: "tree"}
409 if rel == "" {
410 s.cache.setLookup(key, cur)
411 return cur, nil
412 }
413 for _, seg := range strings.Split(rel, "/") {
414 if cur.typ != "tree" {
415 return treeEntry{}, errMissing
416 }
417 obj, err := pool.read(cur.sha, maxStructureBytes)
418 if err != nil {
419 return treeEntry{}, err
420 }
421 var found bool
422 for _, e := range parseTree(obj.Data) {
423 if e.name == seg {
424 cur, found = e, true
425 break
426 }
427 }
428 if !found {
429 return treeEntry{}, errMissing
430 }
431 }
432 s.cache.setLookup(key, cur)
433 return cur, nil
434}
435
436// treeOf extracts the root tree id from a commit object.
437func treeOf(commit []byte) (string, error) {
438 for _, line := range strings.Split(string(commit), "\n") {
439 if line == "" {
440 break
441 }
442 if rest, ok := strings.CutPrefix(line, "tree "); ok {
443 return strings.TrimSpace(rest), nil
444 }
445 }
446 return "", errors.New("pages: commit has no tree")
447}
448
449// commitTime extracts the committer timestamp, which is what Last-Modified means
450// for a static site.
451func commitTime(commit []byte) int64 {
452 for _, line := range strings.Split(string(commit), "\n") {
453 if line == "" {
454 break
455 }
456 rest, ok := strings.CutPrefix(line, "committer ")
457 if !ok {
458 continue
459 }
460 f := strings.Fields(rest)
461 if len(f) < 2 {
462 return 0
463 }
464 n, err := strconv.ParseInt(f[len(f)-2], 10, 64)
465 if err != nil {
466 return 0
467 }
468 return n
469 }
470 return 0
471}
472
473// state resolves a repository for serving, and remembers the answer for the memo
474// window. It is the only place that pays for git subprocesses on the read path.
475func (s *Server) state(name string) (siteState, error) {
476 s.mu.Lock()
477 cached, ok := s.sites[name]
478 s.mu.Unlock()
479 if ok && time.Since(cached.at) < s.cfg.TipTTL {
480 return cached, nil
481 }
482
483 m, err := s.repos.Open(name)
484 if err != nil {
485 return siteState{}, err
486 }
487 st := siteState{m: m, at: time.Now()}
488 if a, err := m.ACL(); err == nil {
489 st.public = a.Can("", access.Read)
490 }
491 if st.public {
492 if sha, mod, err := s.tip(m); err == nil {
493 st.sha, st.mod = sha, mod
494 }
495 }
496
497 s.mu.Lock()
498 prev := s.sites[name]
499 s.sites[name] = st
500 s.mu.Unlock()
501 if prev.sha != "" && prev.sha != st.sha {
502 s.cache.dropCommit(prev.sha)
503 }
504 return st, nil
505}
506
507// tip answers which commit the site branch is on. It is called from state(), so
508// it runs once per memo window, not once per request.
509func (s *Server) tip(m *repo.Meta) (string, int64, error) {
510 pool, err := s.pool(m)
511 if err != nil {
512 return "", 0, err
513 }
514 obj, err := pool.read("refs/heads/"+s.cfg.Ref, maxStructureBytes)
515 if err != nil {
516 return "", 0, err
517 }
518 if obj.Type != "commit" {
519 return "", 0, fmt.Errorf("pages: %s does not point at a commit", s.cfg.Ref)
520 }
521 return obj.SHA, commitTime(obj.Data), nil
522}
523
524func (s *Server) pool(m *repo.Meta) (*objectPool, error) {
525 s.mu.Lock()
526 defer s.mu.Unlock()
527 if pr, ok := s.pools[m.Name]; ok {
528 pr.lastUsed = time.Now()
529 return pr.pool, nil
530 }
531 pool := newObjectPool(m.Dir, s.cfg.Procs)
532 s.pools[m.Name] = &poolRef{pool: pool, lastUsed: time.Now()}
533 return pool, nil
534}
535
536// janitor closes pools nobody has touched, so process count tracks sites being
537// read rather than repositories that exist.
538func (s *Server) janitor() {
539 t := time.NewTicker(time.Minute)
540 defer t.Stop()
541 for {
542 select {
543 case <-s.stop:
544 return
545 case now := <-t.C:
546 s.mu.Lock()
547 for name, pr := range s.pools {
548 if now.Sub(pr.lastUsed) > poolIdle {
549 pr.pool.Close()
550 delete(s.pools, name)
551 delete(s.sites, name)
552 }
553 }
554 s.mu.Unlock()
555 }
556 }
557}
558
559func (s *Server) Close() error {
560 s.once.Do(func() { close(s.stop) })
561 s.mu.Lock()
562 defer s.mu.Unlock()
563 for _, pr := range s.pools {
564 pr.pool.Close()
565 }
566 s.pools = map[string]*poolRef{}
567 return nil
568}
569
570// Stats is for the health endpoint and for tests that want to prove the cache did
571// something.
572func (s *Server) Stats() map[string]int64 {
573 entries, bytes := s.cache.stats()
574 s.mu.Lock()
575 pools := int64(len(s.pools))
576 s.mu.Unlock()
577 return map[string]int64{
578 "cache_entries": int64(entries),
579 "cache_bytes": bytes,
580 "pools": pools,
581 }
582}
583