vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound and stream the anonymous read path
4581690d1ae9d87c272428d101a0fbe3538d7f44 · Keenan Nemetz · 13d ago · parent e05f1c48
Bound and stream the anonymous read path Every anonymous read path read a whole git object before deciding what to do with it: the code browser truncated after allocating, the overview rendered the README it had just read in full, and the site pool allocated the entire object for every request. `max_blob_bytes` allows a 100 MiB blob, so one pushed file made every concurrent request for it a 100 MiB allocation, with no account needed to ask for it. Read the header first instead. `core/git` gains `BlobSize`, `BlobSizes` (one `cat-file --batch-check` for a whole tree) and `BlobStream`, which parses the batch header before touching the payload and stops the process when a caller wants only a prefix. Then use it: `/raw` streams with the object's own `Content-Length`, the inline viewer reads a one-MiB prefix and reports the file's real size, the README is capped and named instead of rendered when it is too large, site blobs are capped at 25 MiB with a 413, and the commit and tree objects read to resolve a site path are capped separately. Metadata gets the same treatment on both sides: 4 MiB per file, 2 MiB per body, 20,000 files and 64 MiB for `issues/` and `requests/` together. The loader asks git for every size in one call and refuses the tree before reading it; the push boundary refuses the same tree, so the author learns why instead of finding that their issue list stopped loading. Counting is now per subtree, so the machine written `runs/` tree no longer eats the issue file budget.
server/read_limits_test.go +115
| @@ -0,0 +1,115 @@ | |||
| 1 | // Read-path bounds: a large blob must not cost a request in proportion to its | ||
| 2 | // size, and a metadata file past the cap must be refused at the push boundary. | ||
| 3 | package server_test | ||
| 4 | |||
| 5 | import ( | ||
| 6 | "io" | ||
| 7 | "net/http" | ||
| 8 | "strings" | ||
| 9 | "testing" | ||
| 10 | ) | ||
| 11 | |||
| 12 | // TestLargeBlobIsBounded covers every anonymous way a repository blob reaches a | ||
| 13 | // stranger: the inline viewer, the raw download and the README on the overview. | ||
| 14 | func TestLargeBlobIsBounded(t *testing.T) { | ||
| 15 | t.Parallel() | ||
| 16 | f := newFixture(t) | ||
| 17 | f.publish(f.repo) | ||
| 18 | |||
| 19 | // 8 MiB of text is eight times the inline cap and well under the 100 MiB blob | ||
| 20 | // a push may carry, so it is a file the forge accepts and must still read | ||
| 21 | // cheaply. | ||
| 22 | pattern := "0123456789abcdef" | ||
| 23 | big := strings.Repeat(pattern, (8<<20)/len(pattern)) | ||
| 24 | readme := "# big\n\n" + strings.Repeat("a readme line\n", (2<<20)/14) | ||
| 25 | |||
| 26 | work := workRepo(t, map[string]string{ | ||
| 27 | "big.txt": big, | ||
| 28 | "README.md": readme, | ||
| 29 | "small.txt": "small\n", | ||
| 30 | }) | ||
| 31 | if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil { | ||
| 32 | t.Fatalf("push: %v\n%s", err, errs) | ||
| 33 | } | ||
| 34 | |||
| 35 | // The inline view is a preview: the first MiB, and the file's real size stated | ||
| 36 | // rather than the size of the preview. | ||
| 37 | code, page := f.get("/r/" + f.repo + "/blob/main/big.txt") | ||
| 38 | if code != http.StatusOK { | ||
| 39 | t.Fatalf("blob page = %d", code) | ||
| 40 | } | ||
| 41 | if !strings.Contains(page, "Showing the first 1 MiB") { | ||
| 42 | t.Fatalf("blob page did not say it was truncated") | ||
| 43 | } | ||
| 44 | if !strings.Contains(page, "8.0 MiB") { | ||
| 45 | t.Fatalf("blob page did not state the file's size") | ||
| 46 | } | ||
| 47 | if strings.Contains(page, strings.Repeat(pattern, (3<<20)/len(pattern))) { | ||
| 48 | // 1 MiB of the pattern is expected; 3 MiB would mean the page carried the | ||
| 49 | // blob and the cap only applied after reading all of it. | ||
| 50 | t.Fatalf("blob page carried more than the inline cap") | ||
| 51 | } | ||
| 52 | |||
| 53 | // The raw download carries all of it, streamed, with a length the client can | ||
| 54 | // trust before the first byte arrives. | ||
| 55 | resp, err := http.Get(f.http.URL + "/r/" + f.repo + "/raw/main/big.txt") | ||
| 56 | if err != nil { | ||
| 57 | t.Fatal(err) | ||
| 58 | } | ||
| 59 | defer resp.Body.Close() | ||
| 60 | if resp.StatusCode != http.StatusOK { | ||
| 61 | t.Fatalf("raw = %d", resp.StatusCode) | ||
| 62 | } | ||
| 63 | if resp.ContentLength != int64(len(big)) { | ||
| 64 | t.Fatalf("raw Content-Length = %d, want %d", resp.ContentLength, len(big)) | ||
| 65 | } | ||
| 66 | n, err := io.Copy(io.Discard, resp.Body) | ||
| 67 | if err != nil { | ||
| 68 | t.Fatal(err) | ||
| 69 | } | ||
| 70 | if n != int64(len(big)) { | ||
| 71 | t.Fatalf("raw body = %d bytes, want %d", n, len(big)) | ||
| 72 | } | ||
| 73 | |||
| 74 | // A README past the cap is named and measured instead of rendered: silently | ||
| 75 | // dropping it would look like a bug in the forge. | ||
| 76 | code, overview := f.get("/r/" + f.repo) | ||
| 77 | if code != http.StatusOK { | ||
| 78 | t.Fatalf("overview = %d", code) | ||
| 79 | } | ||
| 80 | if !strings.Contains(overview, "past the 1.0 MiB") { | ||
| 81 | t.Fatalf("overview did not explain the oversized README") | ||
| 82 | } | ||
| 83 | if strings.Contains(overview, "a readme line") { | ||
| 84 | t.Fatal("overview rendered a README past the cap") | ||
| 85 | } | ||
| 86 | |||
| 87 | // Small files are unaffected. | ||
| 88 | if code, body := f.get("/r/" + f.repo + "/blob/main/small.txt"); code != http.StatusOK || !strings.Contains(body, "small") { | ||
| 89 | t.Fatalf("small blob = %d", code) | ||
| 90 | } | ||
| 91 | } | ||
| 92 | |||
| 93 | // TestOversizedMetadataPushIsRejected is the write-side half of the metadata | ||
| 94 | // limit: the push is refused with a reason, so nobody discovers the cap by | ||
| 95 | // breaking their issue list. | ||
| 96 | func TestOversizedMetadataPushIsRejected(t *testing.T) { | ||
| 97 | t.Parallel() | ||
| 98 | f := newFixture(t) | ||
| 99 | f.publish(f.repo) | ||
| 100 | |||
| 101 | huge := strings.Repeat("x", 5<<20) // past MaxFileBytes, under max_blob_bytes | ||
| 102 | work := f.issuesRepo(t, issueFiles("01M5JR2JK29KKMAA1T9MDNTPBZ", "too big", huge)) | ||
| 103 | |||
| 104 | _, errs, err := f.git(work, "push", f.sshURL(f.repo), "HEAD:refs/heads/vole") | ||
| 105 | if err == nil { | ||
| 106 | t.Fatal("a 5 MiB issue file was accepted") | ||
| 107 | } | ||
| 108 | if !strings.Contains(errs, "limit") { | ||
| 109 | t.Fatalf("rejection did not name the limit:\n%s", errs) | ||
| 110 | } | ||
| 111 | if _, _, err := runGit(t, f.cfg.ReposDir()+"/"+f.repo+".git", | ||
| 112 | "rev-parse", "--verify", "--quiet", "refs/heads/vole"); err == nil { | ||
| 113 | t.Fatal("the rejected push created the metadata branch") | ||
| 114 | } | ||
| 115 | } | ||