vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound and stream the anonymous read path
4581690d1ae9d87c272428d101a0fbe3538d7f44 · Keenan Nemetz · 13d ago · parent e05f1c48
Bound and stream the anonymous read path Every anonymous read path read a whole git object before deciding what to do with it: the code browser truncated after allocating, the overview rendered the README it had just read in full, and the site pool allocated the entire object for every request. `max_blob_bytes` allows a 100 MiB blob, so one pushed file made every concurrent request for it a 100 MiB allocation, with no account needed to ask for it. Read the header first instead. `core/git` gains `BlobSize`, `BlobSizes` (one `cat-file --batch-check` for a whole tree) and `BlobStream`, which parses the batch header before touching the payload and stops the process when a caller wants only a prefix. Then use it: `/raw` streams with the object's own `Content-Length`, the inline viewer reads a one-MiB prefix and reports the file's real size, the README is capped and named instead of rendered when it is too large, site blobs are capped at 25 MiB with a 413, and the commit and tree objects read to resolve a site path are capped separately. Metadata gets the same treatment on both sides: 4 MiB per file, 2 MiB per body, 20,000 files and 64 MiB for `issues/` and `requests/` together. The loader asks git for every size in one call and refuses the tree before reading it; the push boundary refuses the same tree, so the author learns why instead of finding that their issue list stopped loading. Counting is now per subtree, so the machine written `runs/` tree no longer eats the issue file budget.
metadata/limits.go +123
| @@ -0,0 +1,123 @@ | |||
| 1 | package metadata | ||
| 2 | |||
| 3 | // Limits on the metadata a request is willing to read. | ||
| 4 | // | ||
| 5 | // The metadata branch is a tree of small markdown files, and every issue page | ||
| 6 | // render loads the whole of issues/ and requests/ to answer. Nothing about the | ||
| 7 | // format needs a large file, so the sizes here are generous rather than tight: | ||
| 8 | // their job is to make the cost of a page render independent of what a | ||
| 9 | // contributor pushed, not to police how long an issue may be. | ||
| 10 | // | ||
| 11 | // Both sides of the boundary use these numbers: the loader refuses a tree that | ||
| 12 | // is past them (so an anonymous read never allocates it), and pre-receive | ||
| 13 | // refuses a push that would create one (so the refusal arrives as a rejected | ||
| 14 | // push with a reason, not as a page that stops working afterwards). | ||
| 15 | |||
| 16 | import ( | ||
| 17 | "errors" | ||
| 18 | "fmt" | ||
| 19 | "strings" | ||
| 20 | |||
| 21 | "github.com/rodent-software/vole/core/git" | ||
| 22 | "github.com/rodent-software/vole/internal/format" | ||
| 23 | ) | ||
| 24 | |||
| 25 | const ( | ||
| 26 | // MaxFileBytes is the largest single issue or comment file. A body is capped | ||
| 27 | // at MaxBodyBytes, so the headroom is frontmatter and room for growth. | ||
| 28 | MaxFileBytes = 4 << 20 | ||
| 29 | // MaxBodyBytes is the largest issue or comment body a writer may submit. It | ||
| 30 | // matches the web form limit, so a body the browser accepts never fails on | ||
| 31 | // the way to the branch. | ||
| 32 | MaxBodyBytes = 2 << 20 | ||
| 33 | // MaxEntries is how many issue and comment files one load will read. | ||
| 34 | MaxEntries = maxIssuesScan | ||
| 35 | // MaxTreeBytes is how much issues/ and requests/ may total. A forge past | ||
| 36 | // this needs pagination, which is a real feature and not a bigger number. | ||
| 37 | MaxTreeBytes = 64 << 20 | ||
| 38 | ) | ||
| 39 | |||
| 40 | // readMetadata loads every file under prefix into a path -> content map, after | ||
| 41 | // checking the sizes of all of them in one git call. An oversized file is | ||
| 42 | // refused before any of it is allocated: that is the difference between a page | ||
| 43 | // that always costs what it usually costs and one that costs whatever the last | ||
| 44 | // push said. | ||
| 45 | func readMetadata(g *git.Repo, commit, prefix string) (map[string]string, error) { | ||
| 46 | blobs, err := g.LsTreeR(commit) | ||
| 47 | if err != nil { | ||
| 48 | return nil, err | ||
| 49 | } | ||
| 50 | paths, shas := metadataPaths(blobs, prefix) | ||
| 51 | if problems := metadataSizeProblems(paths, shas, g); len(problems) > 0 { | ||
| 52 | return nil, errors.New(problems[0]) | ||
| 53 | } | ||
| 54 | contents := make(map[string]string, len(paths)) | ||
| 55 | for i, path := range paths { | ||
| 56 | data, err := g.BlobBySHA(shas[i]) | ||
| 57 | if err != nil { | ||
| 58 | return nil, err | ||
| 59 | } | ||
| 60 | contents[path] = string(data) | ||
| 61 | } | ||
| 62 | return contents, nil | ||
| 63 | } | ||
| 64 | |||
| 65 | // metadataPaths picks the files under prefix, keeping their object ids in step | ||
| 66 | // with their paths so a size can be looked up without a second walk. | ||
| 67 | func metadataPaths(blobs []git.Entry, prefix string) (paths, shas []string) { | ||
| 68 | p := prefix + "/" | ||
| 69 | for _, e := range blobs { | ||
| 70 | if !strings.HasPrefix(e.Name, p) { | ||
| 71 | continue | ||
| 72 | } | ||
| 73 | paths = append(paths, e.Name) | ||
| 74 | shas = append(shas, e.SHA) | ||
| 75 | } | ||
| 76 | return paths, shas | ||
| 77 | } | ||
| 78 | |||
| 79 | // metadataSizeProblems reports every reason the named files are too big to read | ||
| 80 | // as a set: too many of them, one of them past MaxFileBytes, or all of them past | ||
| 81 | // MaxTreeBytes. It asks git for the sizes once and reads nothing. | ||
| 82 | func metadataSizeProblems(paths, shas []string, g *git.Repo) []string { | ||
| 83 | if len(paths) > MaxEntries { | ||
| 84 | return []string{fmt.Sprintf("metadata: refusing to read %d files (limit %d)", len(paths), MaxEntries)} | ||
| 85 | } | ||
| 86 | if len(paths) == 0 { | ||
| 87 | return nil | ||
| 88 | } | ||
| 89 | sizes, err := g.BlobSizes(shas) | ||
| 90 | if err != nil { | ||
| 91 | return []string{fmt.Sprintf("metadata: cannot size the files: %v", err)} | ||
| 92 | } | ||
| 93 | var problems []string | ||
| 94 | var total int64 | ||
| 95 | for i, path := range paths { | ||
| 96 | size, ok := sizes[shas[i]] | ||
| 97 | if !ok { | ||
| 98 | problems = append(problems, fmt.Sprintf("%s: cannot read the file", path)) | ||
| 99 | continue | ||
| 100 | } | ||
| 101 | if size > MaxFileBytes { | ||
| 102 | problems = append(problems, fmt.Sprintf("%s: %s, limit %s per file", | ||
| 103 | path, format.Bytes(size), format.Bytes(MaxFileBytes))) | ||
| 104 | continue | ||
| 105 | } | ||
| 106 | total += size | ||
| 107 | } | ||
| 108 | if total > MaxTreeBytes { | ||
| 109 | problems = append(problems, fmt.Sprintf("metadata: issues and change requests total %s, limit %s", | ||
| 110 | format.Bytes(total), format.Bytes(MaxTreeBytes))) | ||
| 111 | } | ||
| 112 | return problems | ||
| 113 | } | ||
| 114 | |||
| 115 | // bodyProblems checks one prose body against MaxBodyBytes. The web form already | ||
| 116 | // refuses a body this large, so the check here is what keeps the CLI honest and | ||
| 117 | // what makes a hand-written push fail with a readable reason. | ||
| 118 | func bodyProblems(what, body string) []string { | ||
| 119 | if len(body) > MaxBodyBytes { | ||
| 120 | return []string{fmt.Sprintf("%s is %s, limit %s", what, format.Bytes(int64(len(body))), format.Bytes(MaxBodyBytes))} | ||
| 121 | } | ||
| 122 | return nil | ||
| 123 | } | ||