vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Bound and stream the anonymous read path

4581690d1ae9d87c272428d101a0fbe3538d7f44 · Keenan Nemetz · 13d ago · parent e05f1c48

Bound and stream the anonymous read path

Every anonymous read path read a whole git object before deciding what to do
with it: the code browser truncated after allocating, the overview rendered the
README it had just read in full, and the site pool allocated the entire object
for every request. `max_blob_bytes` allows a 100 MiB blob, so one pushed file
made every concurrent request for it a 100 MiB allocation, with no account
needed to ask for it.

Read the header first instead. `core/git` gains `BlobSize`, `BlobSizes` (one
`cat-file --batch-check` for a whole tree) and `BlobStream`, which parses the
batch header before touching the payload and stops the process when a caller
wants only a prefix.

Then use it: `/raw` streams with the object's own `Content-Length`, the inline
viewer reads a one-MiB prefix and reports the file's real size, the README is
capped and named instead of rendered when it is too large, site blobs are capped
at 25 MiB with a 413, and the commit and tree objects read to resolve a site
path are capped separately.

Metadata gets the same treatment on both sides: 4 MiB per file, 2 MiB per body,
20,000 files and 64 MiB for `issues/` and `requests/` together. The loader asks
git for every size in one call and refuses the tree before reading it; the push
boundary refuses the same tree, so the author learns why instead of finding that
their issue list stopped loading. Counting is now per subtree, so the machine
written `runs/` tree no longer eats the issue file budget.

core/git/blob_test.go +176

@@ -0,0 +1,176 @@
	1	package git
	2
	3	import (
	4	"bytes"
	5	"errors"
	6	"io"
	7	"os/exec"
	8	"path/filepath"
	9	"runtime"
	10	"testing"
	11	)
	12
	13	// blobRepo is a bare repository whose main branch holds the files given. The
	14	// tests here are about how bytes leave git, so they need real objects rather
	15	// than fixtures.
	16	func blobRepo(t testing.T, files map[string][]byte) Repo {
	17	t.Helper()
	18	dir := filepath.Join(t.TempDir(), "r.git")
	19	if out, err := exec.Command("git", "init", "--bare", "-q", dir).CombinedOutput(); err != nil {
	20	t.Fatalf("init: %v: %s", err, out)
	21	}
	22	g := Open(dir)
	23	if _, err := CommitFiles(g, "refs/heads/main", func(dst map[string][]byte) error {
	24	for name, data := range files {
	25	dst[name] = data
	26	}
	27	return nil
	28	}, "init", "test"); err != nil {
	29	t.Fatal(err)
	30	}
	31	return g
	32	}
	33
	34	func TestBlobSizeAndPrefix(t *testing.T) {
	35	body := bytes.Repeat([]byte("0123456789abcdef"), 4096) // 64 KiB
	36	g := blobRepo(t, map[string][]byte{"a.txt": body})
	37
	38	size, err := g.BlobSize("refs/heads/main:a.txt")
	39	if err != nil {
	40	t.Fatal(err)
	41	}
	42	if size != int64(len(body)) {
	43	t.Fatalf("BlobSize = %d, want %d", size, len(body))
	44	}
	45
	46	head, truncated, err := g.BlobPrefixAt("refs/heads/main", "a.txt", 1024)
	47	if err != nil {
	48	t.Fatal(err)
	49	}
	50	if !truncated {
	51	t.Fatal("BlobPrefixAt reported no truncation for a 64 KiB blob read at 1 KiB")
	52	}
	53	if len(head) != 1024 \|\| !bytes.Equal(head, body[:1024]) {
	54	t.Fatalf("prefix = %d bytes, want the first 1024 bytes of the blob", len(head))
	55	}
	56
	57	// A blob shorter than the cap comes back whole and says so.
	58	whole, truncated, err := g.BlobPrefixAt("refs/heads/main", "a.txt", int64(len(body))+10)
	59	if err != nil {
	60	t.Fatal(err)
	61	}
	62	if truncated {
	63	t.Fatal("BlobPrefixAt reported truncation for a cap larger than the blob")
	64	}
	65	if !bytes.Equal(whole, body) {
	66	t.Fatal("short read differs from the blob")
	67	}
	68	}
	69
	70	func TestBlobSizes(t *testing.T) {
	71	g := blobRepo(t, map[string][]byte{
	72	"small": []byte("hi"),
	73	"big": bytes.Repeat([]byte("x"), 5000),
	74	})
	75	tree, err := g.FlatTree("refs/heads/main")
	76	if err != nil {
	77	t.Fatal(err)
	78	}
	79	entries, err := g.LsTreeR("refs/heads/main")
	80	if err != nil {
	81	t.Fatal(err)
	82	}
	83	shas := make([]string, 0, len(entries))
	84	for _, e := range entries {
	85	shas = append(shas, e.SHA)
	86	}
	87	sizes, err := g.BlobSizes(shas)
	88	if err != nil {
	89	t.Fatal(err)
	90	}
	91	if len(sizes) != 2 {
	92	t.Fatalf("BlobSizes returned %d entries, want 2: %v", len(sizes), sizes)
	93	}
	94	for _, e := range entries {
	95	if sizes[e.SHA] != int64(len(tree[e.Name])) {
	96	t.Fatalf("%s: size %d, want %d", e.Name, sizes[e.SHA], len(tree[e.Name]))
	97	}
	98	}
	99	}
	100
	101	// TestStreamingLargeBlobDoesNotAllocate guards the bug this API exists for: a
	102	// reader that copies a big blob to an io.Writer must not hold it. Before
	103	// OpenBlob every read path called BlobAt, which allocated the object.
	104	func TestStreamingLargeBlobDoesNotAllocate(t *testing.T) {
	105	const size = 16 << 20
	106	body := bytes.Repeat([]byte("v"), size)
	107	g := blobRepo(t, map[string][]byte{"huge.bin": body})
	108	body = nil
	109
	110	stream, err := g.OpenBlobAt("refs/heads/main", "huge.bin")
	111	if err != nil {
	112	t.Fatal(err)
	113	}
	114	defer stream.Close()
	115	if stream.Size != size {
	116	t.Fatalf("Size = %d, want %d", stream.Size, size)
	117	}
	118
	119	var before, after runtime.MemStats
	120	runtime.GC()
	121	runtime.ReadMemStats(&before)
	122	var got int64
	123	buf := make([]byte, 32<<10)
	124	for {
	125	n, rerr := stream.Read(buf)
	126	got += int64(n)
	127	if rerr != nil {
	128	if !errors.Is(rerr, io.EOF) {
	129	t.Fatal(rerr)
	130	}
	131	break
	132	}
	133	}
	134	runtime.ReadMemStats(&after)
	135
	136	if got != size {
	137	t.Fatalf("read %d bytes, want %d", got, size)
	138	}
	139	// The copy itself allocates nothing; allow slack for GC bookkeeping.
	140	if alloc := after.TotalAlloc - before.TotalAlloc; alloc > 4<<20 {
	141	t.Fatalf("streaming a %d byte blob allocated %d bytes", size, alloc)
	142	}
	143	}
	144
	145	func TestPrefixReadStopsEarlyAndLeavesRepoUsable(t *testing.T) {
	146	g := blobRepo(t, map[string][]byte{
	147	"huge.bin": bytes.Repeat([]byte("v"), 4<<20),
	148	"tiny.txt": []byte("tiny"),
	149	})
	150	head, truncated, err := g.BlobPrefixAt("refs/heads/main", "huge.bin", 4096)
	151	if err != nil {
	152	t.Fatal(err)
	153	}
	154	if !truncated \|\| len(head) != 4096 {
	155	t.Fatalf("got %d bytes, truncated=%v", len(head), truncated)
	156	}
	157	// The aborted read killed a cat-file process; the next read must not see a
	158	// half-consumed stream.
	159	small, err := g.BlobAt("refs/heads/main", "tiny.txt")
	160	if err != nil {
	161	t.Fatal(err)
	162	}
	163	if string(small) != "tiny" {
	164	t.Fatalf("tiny.txt = %q", small)
	165	}
	166	}
	167
	168	func TestOpenBlobMissing(t *testing.T) {
	169	g := blobRepo(t, map[string][]byte{"a.txt": []byte("a")})
	170	if _, err := g.OpenBlobAt("refs/heads/main", "nope.txt"); err == nil {
	171	t.Fatal("OpenBlobAt accepted a path that does not exist")
	172	}
	173	if _, err := g.OpenBlob(""); err == nil {
	174	t.Fatal("OpenBlob accepted an empty revision")
	175	}
	176	}