vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound and stream the anonymous read path
4581690d1ae9d87c272428d101a0fbe3538d7f44 · Keenan Nemetz · 13d ago · parent e05f1c48
Bound and stream the anonymous read path Every anonymous read path read a whole git object before deciding what to do with it: the code browser truncated after allocating, the overview rendered the README it had just read in full, and the site pool allocated the entire object for every request. `max_blob_bytes` allows a 100 MiB blob, so one pushed file made every concurrent request for it a 100 MiB allocation, with no account needed to ask for it. Read the header first instead. `core/git` gains `BlobSize`, `BlobSizes` (one `cat-file --batch-check` for a whole tree) and `BlobStream`, which parses the batch header before touching the payload and stops the process when a caller wants only a prefix. Then use it: `/raw` streams with the object's own `Content-Length`, the inline viewer reads a one-MiB prefix and reports the file's real size, the README is capped and named instead of rendered when it is too large, site blobs are capped at 25 MiB with a 413, and the commit and tree objects read to resolve a site path are capped separately. Metadata gets the same treatment on both sides: 4 MiB per file, 2 MiB per body, 20,000 files and 64 MiB for `issues/` and `requests/` together. The loader asks git for every size in one call and refuses the tree before reading it; the push boundary refuses the same tree, so the author learns why instead of finding that their issue list stopped loading. Counting is now per subtree, so the machine written `runs/` tree no longer eats the issue file budget.
server/pages/pages.go +22−4
| @@ -29,6 +29,7 @@ import ( | |||
| 29 | 29 | ||
| 30 | 30 | "github.com/rodent-software/vole/forge/access" | |
| 31 | 31 | "github.com/rodent-software/vole/forge/repo" | |
| 32 | "github.com/rodent-software/vole/internal/format" | ||
| 32 | 33 | ) | |
| 33 | 34 | ||
| 34 | 35 | const ( | |
| @@ -40,8 +41,19 @@ const ( | |||
| 40 | 41 | // repositories should not keep four hundred processes because somebody once | |
| 41 | 42 | // looked at a site. | |
| 42 | 43 | poolIdle = 2 * time.Minute | |
| 44 | // maxPageBytes is the largest file a site will serve. A site is a website: | ||
| 45 | // a blob past this size is repository content being used as a file host, and | ||
| 46 | // answering 413 costs one file's worth of nothing. | ||
| 47 | maxPageBytes = 25 << 20 | ||
| 48 | // maxStructureBytes bounds the commit and tree objects read to resolve a | ||
| 49 | // path. A tree big enough to hit this names more files than a site can have. | ||
| 50 | maxStructureBytes = 64 << 20 | ||
| 43 | 51 | ) | |
| 44 | 52 | ||
| 53 | // errObjectTooLarge is returned instead of an allocation when an object the site | ||
| 54 | // asked for is past its cap. | ||
| 55 | var errObjectTooLarge = errors.New("pages: object too large") | ||
| 56 | |||
| 45 | 57 | type Config struct { | |
| 46 | 58 | // Domain is the suffix a site is served under: <repo>.<domain>. | |
| 47 | 59 | Domain string | |
| @@ -193,6 +205,12 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { | |||
| 193 | 205 | ||
| 194 | 206 | page, err := s.file(m, commit, rel, ent.sha, mod) | |
| 195 | 207 | if err != nil { | |
| 208 | if errors.Is(err, errObjectTooLarge) { | ||
| 209 | // Say why: a site reader whose asset is bigger than the limit needs to | ||
| 210 | // know it was the size and not a broken link. | ||
| 211 | http.Error(w, fmt.Sprintf("vole: this file is larger than the %s a site serves", format.Bytes(maxPageBytes)), http.StatusRequestEntityTooLarge) | ||
| 212 | return | ||
| 213 | } | ||
| 196 | 214 | s.cfg.Log.Error("pages: read blob", "repo", m.Name, "path", rel, "err", err) | |
| 197 | 215 | http.Error(w, "vole: cannot read that file", http.StatusInternalServerError) | |
| 198 | 216 | return | |
| @@ -259,7 +277,7 @@ func (s *Server) file(m *repo.Meta, commit, rel, sha string, mod int64) (*page, | |||
| 259 | 277 | if err != nil { | |
| 260 | 278 | return nil, err | |
| 261 | 279 | } | |
| 262 | obj, err := pool.read(sha) | ||
| 280 | obj, err := pool.read(sha, maxPageBytes) | ||
| 263 | 281 | if err != nil { | |
| 264 | 282 | return nil, err | |
| 265 | 283 | } | |
| @@ -374,7 +392,7 @@ func (s *Server) lookup(m *repo.Meta, commit, rel string) (treeEntry, error) { | |||
| 374 | 392 | if err != nil { | |
| 375 | 393 | return treeEntry{}, err | |
| 376 | 394 | } | |
| 377 | obj, err := pool.read(commit) | ||
| 395 | obj, err := pool.read(commit, maxStructureBytes) | ||
| 378 | 396 | if err != nil { | |
| 379 | 397 | return treeEntry{}, err | |
| 380 | 398 | } | |
| @@ -394,7 +412,7 @@ func (s *Server) lookup(m *repo.Meta, commit, rel string) (treeEntry, error) { | |||
| 394 | 412 | if cur.typ != "tree" { | |
| 395 | 413 | return treeEntry{}, errMissing | |
| 396 | 414 | } | |
| 397 | obj, err := pool.read(cur.sha) | ||
| 415 | obj, err := pool.read(cur.sha, maxStructureBytes) | ||
| 398 | 416 | if err != nil { | |
| 399 | 417 | return treeEntry{}, err | |
| 400 | 418 | } | |
| @@ -491,7 +509,7 @@ func (s *Server) tip(m *repo.Meta) (string, int64, error) { | |||
| 491 | 509 | if err != nil { | |
| 492 | 510 | return "", 0, err | |
| 493 | 511 | } | |
| 494 | obj, err := pool.read("refs/heads/" + s.cfg.Ref) | ||
| 512 | obj, err := pool.read("refs/heads/"+s.cfg.Ref, maxStructureBytes) | ||
| 495 | 513 | if err != nil { | |
| 496 | 514 | return "", 0, err | |
| 497 | 515 | } | |