vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · metadata/limits.go

Line attribution on main · 135 line(s)

View file History

4581690d Keenan Nemetz 13d ago	1	package metadata
	2
	3	// Limits on the metadata a request is willing to read.
	4	//
	5	// The metadata branch is a tree of small markdown files, and every issue page
b492b557 Keenan Nemetz 3d ago	6	// render loads the whole of issues/ and changes/ to answer. Nothing about the
4581690d Keenan Nemetz 13d ago	7	// format needs a large file, so the sizes here are generous rather than tight:
	8	// their job is to make the cost of a page render independent of what a
	9	// contributor pushed, not to police how long an issue may be.
	10	//
	11	// Both sides of the boundary use these numbers: the loader refuses a tree that
	12	// is past them (so an anonymous read never allocates it), and pre-receive
	13	// refuses a push that would create one (so the refusal arrives as a rejected
	14	// push with a reason, not as a page that stops working afterwards).
	15
	16	import (
	17	"errors"
	18	"fmt"
	19	"strings"
	20
d37d301f Keenan Nemetz 15h ago	21	"rodent.software/vole/core/git"
d37d301f Keenan Nemetz 15h ago	22	"rodent.software/vole/internal/format"
4581690d Keenan Nemetz 13d ago	23	)
	24
	25	const (
	26	// MaxFileBytes is the largest single issue or comment file. A body is capped
	27	// at MaxBodyBytes, so the headroom is frontmatter and room for growth.
	28	MaxFileBytes = 4 << 20
	29	// MaxBodyBytes is the largest issue or comment body a writer may submit. It
	30	// matches the web form limit, so a body the browser accepts never fails on
	31	// the way to the branch.
	32	MaxBodyBytes = 2 << 20
	33	// MaxEntries is how many issue and comment files one load will read.
	34	MaxEntries = maxIssuesScan
b492b557 Keenan Nemetz 3d ago	35	// MaxTreeBytes is how much issues/ and changes/ may total. A forge past
4581690d Keenan Nemetz 13d ago	36	// this needs pagination, which is a real feature and not a bigger number.
	37	MaxTreeBytes = 64 << 20
	38	)
	39
	40	// readMetadata loads every file under prefix into a path -> content map, after
	41	// checking the sizes of all of them in one git call. An oversized file is
	42	// refused before any of it is allocated: that is the difference between a page
	43	// that always costs what it usually costs and one that costs whatever the last
	44	// push said.
	45	func readMetadata(g *git.Repo, commit, prefix string) (map[string]string, error) {
	46	blobs, err := g.LsTreeR(commit)
	47	if err != nil {
	48	return nil, err
	49	}
	50	paths, shas := metadataPaths(blobs, prefix)
	51	if problems := metadataSizeProblems(paths, shas, g); len(problems) > 0 {
	52	return nil, errors.New(problems[0])
	53	}
5cecc618 Keenan Nemetz 10d ago	54	return readBlobs(g, paths, shas)
	55	}
	56
	57	// readBlobs reads the named files in one git call. It is one process whether the
	58	// prefix holds one file or ten thousand: a page that lists a repository's change
	59	// requests must not cost a process per request, and a push validator must not
	60	// cost one per file either.
	61	func readBlobs(g *git.Repo, paths, shas []string) (map[string]string, error) {
	62	blobs, err := g.BlobsBySHA(shas)
	63	if err != nil {
	64	return nil, fmt.Errorf("cannot read the metadata files: %w", err)
	65	}
4581690d Keenan Nemetz 13d ago	66	contents := make(map[string]string, len(paths))
4581690d Keenan Nemetz 13d ago	67	for i, path := range paths {
5cecc618 Keenan Nemetz 10d ago	68	data, ok := blobs[shas[i]]
	69	if !ok {
	70	return nil, fmt.Errorf("metadata: %s was listed but not read", path)
4581690d Keenan Nemetz 13d ago	71	}
	72	contents[path] = string(data)
	73	}
	74	return contents, nil
	75	}
	76
	77	// metadataPaths picks the files under prefix, keeping their object ids in step
	78	// with their paths so a size can be looked up without a second walk.
	79	func metadataPaths(blobs []git.Entry, prefix string) (paths, shas []string) {
	80	p := prefix + "/"
	81	for _, e := range blobs {
	82	if !strings.HasPrefix(e.Name, p) {
	83	continue
	84	}
	85	paths = append(paths, e.Name)
	86	shas = append(shas, e.SHA)
	87	}
	88	return paths, shas
	89	}
	90
	91	// metadataSizeProblems reports every reason the named files are too big to read
	92	// as a set: too many of them, one of them past MaxFileBytes, or all of them past
	93	// MaxTreeBytes. It asks git for the sizes once and reads nothing.
	94	func metadataSizeProblems(paths, shas []string, g *git.Repo) []string {
	95	if len(paths) > MaxEntries {
	96	return []string{fmt.Sprintf("metadata: refusing to read %d files (limit %d)", len(paths), MaxEntries)}
	97	}
	98	if len(paths) == 0 {
	99	return nil
	100	}
	101	sizes, err := g.BlobSizes(shas)
	102	if err != nil {
	103	return []string{fmt.Sprintf("metadata: cannot size the files: %v", err)}
	104	}
	105	var problems []string
	106	var total int64
	107	for i, path := range paths {
	108	size, ok := sizes[shas[i]]
	109	if !ok {
	110	problems = append(problems, fmt.Sprintf("%s: cannot read the file", path))
	111	continue
	112	}
	113	if size > MaxFileBytes {
	114	problems = append(problems, fmt.Sprintf("%s: %s, limit %s per file",
	115	path, format.Bytes(size), format.Bytes(MaxFileBytes)))
	116	continue
	117	}
	118	total += size
	119	}
	120	if total > MaxTreeBytes {
	121	problems = append(problems, fmt.Sprintf("metadata: issues and change requests total %s, limit %s",
	122	format.Bytes(total), format.Bytes(MaxTreeBytes)))
	123	}
	124	return problems
	125	}
	126
	127	// bodyProblems checks one prose body against MaxBodyBytes. The web form already
	128	// refuses a body this large, so the check here is what keeps the CLI honest and
	129	// what makes a hand-written push fail with a readable reason.
	130	func bodyProblems(what, body string) []string {
	131	if len(body) > MaxBodyBytes {
	132	return []string{fmt.Sprintf("%s is %s, limit %s", what, format.Bytes(int64(len(body))), format.Bytes(MaxBodyBytes))}
	133	}
	134	return nil
135	}