vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · server/load_test.go

Line attribution on main · 832 line(s)

View file History

0448a978 Keenan Nemetz 9d ago	1	//go:build load
	2
	3	package server_test
	4
	5	// A load harness for a real, running instance.
	6	//
	7	// It is behind the `load` build tag so `go test ./...` and the verify workflow
	8	// never run it. Everything else follows the integration fixture: a server on a
	9	// loopback port with a temporary data directory, so the numbers describe the
	10	// same code an operator runs, not a mock.
	11	//
1dfa8cbd Keenan Nemetz 9d ago	12	// Run every route:
0448a978 Keenan Nemetz 9d ago	13	//
1dfa8cbd Keenan Nemetz 9d ago	14	// VOLE_LOAD_SCENARIO=all VOLE_LOAD_CONCURRENCY=4 VOLE_LOAD_DURATION=5s \
1dfa8cbd Keenan Nemetz 9d ago	15	// go test -tags load,sqlite_fts5 -run '^TestLoad$' -v ./server
0448a978 Keenan Nemetz 9d ago	16	//
1dfa8cbd Keenan Nemetz 9d ago	17	// Or one route, e.g. clones past the per-client limit:
	18	//
	19	// VOLE_LOAD_SCENARIO=clone VOLE_LOAD_CONCURRENCY=24 VOLE_LOAD_DURATION=10s \
	20	// go test -tags load,sqlite_fts5 -run '^TestLoad$' -v ./server
0448a978 Keenan Nemetz 9d ago	21	//
	22	// The knobs are environment variables rather than flags so the documented
	23	// command has no `-args` in it. See docs/load-testing.md.
	24
	25	import (
1dfa8cbd Keenan Nemetz 9d ago	26	"bytes"
0448a978 Keenan Nemetz 9d ago	27	"fmt"
0448a978 Keenan Nemetz 9d ago	28	"io"
1dfa8cbd Keenan Nemetz 9d ago	29	"math/rand"
0448a978 Keenan Nemetz 9d ago	30	"net/http"
122cbe4f Keenan Nemetz 7d ago	31	"net/url"
0448a978 Keenan Nemetz 9d ago	32	"os"
	33	"os/exec"
	34	"path/filepath"
	35	"runtime"
	36	"sort"
	37	"strconv"
	38	"strings"
	39	"sync"
	40	"testing"
	41	"time"
	42
d37d301f Keenan Nemetz 15h ago	43	"rodent.software/vole/core/git"
	44	"rodent.software/vole/core/lfs"
	45	"rodent.software/vole/forge/config"
	46	"rodent.software/vole/metadata"
0448a978 Keenan Nemetz 9d ago	47	)
0448a978 Keenan Nemetz 9d ago	48
1dfa8cbd Keenan Nemetz 9d ago	49	const (
	50	loadPagesDomain = "pages.test"
	51	// loadBlobLimit is the configured max_blob_bytes, so a file at the limit is
	52	// something a push could actually deliver.
	53	loadBlobLimit = 8 << 20
	54	// loadLargeCommits and loadLargeBlob make the second repository large enough
	55	// that a clone of it is a different measurement from a clone of the fixture.
	56	loadLargeCommits = 3000
	57	loadLargeBlob = 16 << 20
	58	)
	59
	60	// loadSeed is what the scenarios need to name a route.
	61	type loadSeed struct {
	62	repo string
	63	smallRepo string
	64	largeRepo string
	65	issueID string
	66	changeID string
	67	}
	68
0448a978 Keenan Nemetz 9d ago	69	func TestLoad(t *testing.T) {
	70	scenario := envString("VOLE_LOAD_SCENARIO", "clone")
	71	concurrency := envInt("VOLE_LOAD_CONCURRENCY", 4)
	72	duration := envDuration("VOLE_LOAD_DURATION", 10*time.Second)
	73	if concurrency < 1 {
	74	t.Fatalf("VOLE_LOAD_CONCURRENCY=%d, want at least 1", concurrency)
	75	}
	76	if duration <= 0 {
	77	t.Fatalf("VOLE_LOAD_DURATION=%s, want a positive duration", duration)
	78	}
	79
1dfa8cbd Keenan Nemetz 9d ago	80	f := newFixtureWith(t, func(c *config.Config) {
	81	c.PagesDomain = loadPagesDomain
	82	c.PagesRef = "pages"
	83	c.MaxBlobBytes = loadBlobLimit
	84	// The clone scenario gives every worker its own forwarded address, so the
	85	// global git_http_max_concurrent is the limit that refuses, not the
	86	// per-client one. The fixture is loopback, so trusting the header is safe.
	87	c.GitHTTPClientIPHeader = "X-Forwarded-For"
	88	})
	89	seed := seedLoadRepo(t, f)
	90
0448a978 Keenan Nemetz 9d ago	91	// One read per repository builds its index before the run, so the first
0448a978 Keenan Nemetz 9d ago	92	// reader does not pay for the whole metadata branch inside the measurement.
ed03b3d1 Keenan Nemetz 2h ago	93	f.get("/" + f.repo)
ed03b3d1 Keenan Nemetz 2h ago	94	f.get("/" + seed.largeRepo)
1dfa8cbd Keenan Nemetz 9d ago	95	scenarios := loadScenarios(t, f, seed)
	96
	97	if scenario == "all" {
	98	for _, sc := range scenarios {
	99	runScenario(t, f, sc, duration, concurrency)
	100	t.Logf("")
	101	}
	102	return
	103	}
	104	for _, sc := range scenarios {
	105	if sc.name == scenario {
	106	runScenario(t, f, sc, duration, concurrency)
	107	return
	108	}
	109	}
	110	var names []string
	111	for _, sc := range scenarios {
	112	names = append(names, sc.name)
	113	}
	114	t.Fatalf("unknown VOLE_LOAD_SCENARIO %q (have: all, %s)", scenario, strings.Join(names, ", "))
	115	}
	116
	117	type scenario struct {
	118	name string
	119	step loadStep
	120	}
	121
	122	// loadStep is one request. The worker number exists so a step that needs a
	123	// workspace (a clone target) gets its own instead of racing its peers.
	124	type loadStep func(client *http.Client, worker int) (status string, err error)
	125
	126	func runScenario(t testing.T, f fixture, sc scenario, duration time.Duration, concurrency int) {
	127	t.Helper()
	128	client := &http.Client{Timeout: 120 * time.Second}
	129	results := newLoadResults()
	130	peaks := startSampler()
	131	deadline := time.Now().Add(duration)
	132
	133	var wg sync.WaitGroup
	134	for i := 0; i < concurrency; i++ {
	135	wg.Add(1)
	136	go func(worker int) {
	137	defer wg.Done()
	138	for time.Now().Before(deadline) {
	139	start := time.Now()
	140	status, err := sc.step(client, worker)
	141	results.record(time.Since(start), status, err)
	142	}
	143	}(i)
	144	}
	145	wg.Wait()
	146	reportLoad(t, sc.name, duration, concurrency, results, peaks.stop())
	147	}
	148
	149	// loadScenarios names every route #100 asks about. Each is one step a worker
	150	// repeats until the deadline.
	151	func loadScenarios(t testing.T, f fixture, seed loadSeed) []scenario {
	152	t.Helper()
	153	cloneDir := t.TempDir()
ed03b3d1 Keenan Nemetz 2h ago	154	repo := f.http.URL + "/" + seed.repo
1dfa8cbd Keenan Nemetz 9d ago	155	siteURL := f.http.URL + "/"
	156	cloneURL := f.httpURL(seed.smallRepo)
	157	largeCloneURL := f.httpURL(seed.largeRepo)
	158
	159	return []scenario{
	160	{name: "clone", step: cloneStep(cloneURL, cloneDir)},
	161	{name: "clone-large", step: cloneStep(largeCloneURL, cloneDir)},
	162	{name: "blob-small", step: getStep(repo + "/blob/main/small.txt")},
	163	{name: "blob-limit", step: getStep(repo + "/blob/main/limit.bin")},
	164	{name: "raw-small", step: getStep(repo + "/raw/main/small.txt")},
	165	{name: "raw-limit", step: getStep(repo + "/raw/main/limit.bin")},
	166	{name: "raw-range", step: getStepWith(repo+"/raw/main/limit.bin", map[string]string{"Range": "bytes=0-65535"})},
	167	{name: "overview", step: getStep(repo)},
	168	{name: "tree", step: getStep(repo + "/tree/main")},
	169	{name: "commits", step: getStep(repo + "/commits/main")},
	170	{name: "compare", step: getStep(repo + "/compare/main...feature")},
8fb7aabf Keenan Nemetz 3d ago	171	{name: "change", step: getStep(repo + "/changes/" + seed.changeID)},
1dfa8cbd Keenan Nemetz 9d ago	172	{name: "issue", step: getStep(repo + "/issues/" + seed.issueID)},
4d938fe8 Keenan Nemetz 9d ago	173	{name: "activity", step: getStep(repo + "/activity")},
ed03b3d1 Keenan Nemetz 2h ago	174	{name: "instance-activity", step: getStep(f.http.URL + "/-/activity")},
4d938fe8 Keenan Nemetz 9d ago	175	{name: "home", step: getStep(f.http.URL + "/")},
1dfa8cbd Keenan Nemetz 9d ago	176	{name: "search-code", step: getStep(repo + "/search?q=revision")},
ed03b3d1 Keenan Nemetz 2h ago	177	{name: "search-find", step: getStepWith(f.http.URL+"/-/search/palette?q=revision", map[string]string{"HX-Request": "true"})},
1dfa8cbd Keenan Nemetz 9d ago	178	{name: "artifact", step: getStep(repo + "/releases/v1/artifacts/load-artifact.txt")},
	179	{name: "artifact-range", step: getStepWith(repo+"/releases/v1/artifacts/load-artifact.txt", map[string]string{"Range": "bytes=0-1023"})},
	180	{name: "site", step: siteStep(siteURL, seed.repo+"."+loadPagesDomain)},
122cbe4f Keenan Nemetz 7d ago	181	{name: "comment", step: commentStep(t, f, f.writerID, repo+"/issues/"+seed.issueID)},
1dfa8cbd Keenan Nemetz 9d ago	182	}
1dfa8cbd Keenan Nemetz 9d ago	183	}
0448a978 Keenan Nemetz 9d ago	184
1dfa8cbd Keenan Nemetz 9d ago	185	// getStep is one anonymous GET, with the body drained so the server sees the
	186	// whole response.
	187	func getStep(url string) loadStep {
	188	return getStepWith(url, nil)
	189	}
	190
	191	func getStepWith(url string, headers map[string]string) loadStep {
	192	return func(client *http.Client, worker int) (string, error) {
	193	req, err := http.NewRequest(http.MethodGet, url, nil)
	194	if err != nil {
	195	return "error", err
	196	}
	197	for k, v := range headers {
	198	req.Header.Set(k, v)
	199	}
	200	resp, err := client.Do(req)
	201	if err != nil {
	202	return "error", err
	203	}
	204	_, _ = io.Copy(io.Discard, resp.Body)
	205	resp.Body.Close()
	206	return strconv.Itoa(resp.StatusCode), nil
	207	}
	208	}
	209
	210	// siteStep requests a hosted site page while still talking to the test listener,
	211	// which is what a real request with a Host header does.
	212	func siteStep(url, host string) loadStep {
	213	return func(client *http.Client, worker int) (string, error) {
	214	req, err := http.NewRequest(http.MethodGet, url, nil)
	215	if err != nil {
	216	return "error", err
	217	}
	218	req.Host = host
	219	resp, err := client.Do(req)
	220	if err != nil {
	221	return "error", err
	222	}
	223	_, _ = io.Copy(io.Discard, resp.Body)
	224	resp.Body.Close()
	225	return strconv.Itoa(resp.StatusCode), nil
	226	}
	227	}
	228
122cbe4f Keenan Nemetz 7d ago	229	// commentStep is the only scenario that writes. Each request is one comment, and a
	230	// comment is one commit on the metadata branch every write in the instance shares, so
	231	// this is the scenario that measures what concurrency on that branch costs. The
	232	// redirect after a successful post is not followed: the write is the measurement, and
	233	// the page it lands on is the issue scenario.
	234	func commentStep(t testing.T, f fixture, accountID, issuePath string) loadStep {
	235	t.Helper()
	236	sess, err := f.srv.Sessions.Create(accountID, nil, "load")
	237	if err != nil {
	238	t.Fatal(err)
	239	}
	240	cookie := "__Host-vole_session=" + sess.Token
	241	comment := &http.Client{
	242	Timeout: 120 * time.Second,
	243	CheckRedirect: func(http.Request, []http.Request) error { return http.ErrUseLastResponse },
	244	}
	245	return func(client *http.Client, worker int) (string, error) {
	246	form := url.Values{"csrf": {sess.CSRF}, "body": {"load comment"}}
	247	req, err := http.NewRequest(http.MethodPost, issuePath+"/comments", strings.NewReader(form.Encode()))
	248	if err != nil {
	249	return "error", err
	250	}
	251	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	252	req.Header.Set("Cookie", cookie)
	253	resp, err := comment.Do(req)
	254	if err != nil {
	255	return "error", err
	256	}
	257	_, _ = io.Copy(io.Discard, resp.Body)
	258	resp.Body.Close()
	259	return strconv.Itoa(resp.StatusCode), nil
	260	}
	261	}
	262
1dfa8cbd Keenan Nemetz 9d ago	263	// cloneStep probes `info/refs` and then clones. The probe is what a server can
	264	// refuse (503 past the concurrency limit), so it is recorded as the step's
	265	// status; the git process's own failure is recorded separately.
	266	func cloneStep(repoURL, root string) loadStep {
	267	infoRefs := repoURL + "/info/refs?service=git-upload-pack"
	268	return func(client *http.Client, worker int) (string, error) {
	269	// A distinct forwarded address per worker keeps the per-client limit from
	270	// binding before the global one.
	271	forwarded := fmt.Sprintf("10.0.%d.%d", worker/250, worker%250+1)
	272	status, err := probe(client, infoRefs, forwarded)
	273	if err != nil {
	274	return status, err
	275	}
	276	if status != "200" {
	277	return status, nil
	278	}
	279	return status, cloneShallow(repoURL, filepath.Join(root, strconv.Itoa(worker)), forwarded)
	280	}
	281	}
	282
	283	func probe(client *http.Client, url, forwarded string) (string, error) {
	284	req, err := http.NewRequest(http.MethodGet, url, nil)
	285	if err != nil {
	286	return "error", err
	287	}
	288	req.Header.Set("X-Forwarded-For", forwarded)
	289	resp, err := client.Do(req)
	290	if err != nil {
	291	return "error", err
	292	}
	293	_, _ = io.Copy(io.Discard, resp.Body)
	294	resp.Body.Close()
	295	return strconv.Itoa(resp.StatusCode), nil
	296	}
	297
	298	func cloneShallow(url, dir, forwarded string) error {
	299	if err := os.RemoveAll(dir); err != nil {
	300	return err
	301	}
	302	cmd := exec.Command("git", "-c", "http.extraHeader=X-Forwarded-For: "+forwarded,
	303	"clone", "--quiet", "--depth", "1", url, dir)
	304	cmd.Env = append(os.Environ(),
	305	"GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
	306	out, err := cmd.CombinedOutput()
	307	if err != nil {
	308	return fmt.Errorf("clone: %v: %s", err, firstLine(string(out)))
0448a978 Keenan Nemetz 9d ago	309	}
1dfa8cbd Keenan Nemetz 9d ago	310	return nil
0448a978 Keenan Nemetz 9d ago	311	}
0448a978 Keenan Nemetz 9d ago	312
1dfa8cbd Keenan Nemetz 9d ago	313	// seedLoadRepo gives the instance the shape #100 measures: a small repository
	314	// with history and files at both ends of max_blob_bytes, a data-only change with
	315	// a large diff, a release artifact, a hosted site, and a second repository large
	316	// enough that cloning it costs something.
	317	func seedLoadRepo(t testing.T, f fixture) loadSeed {
0448a978 Keenan Nemetz 9d ago	318	t.Helper()
0448a978 Keenan Nemetz 9d ago	319	f.publish(f.repo)
1dfa8cbd Keenan Nemetz 9d ago	320	seed := loadSeed{repo: f.repo, smallRepo: "small", largeRepo: "large"}
0448a978 Keenan Nemetz 9d ago	321
1dfa8cbd Keenan Nemetz 9d ago	322	small := strings.Repeat("small line\n", 100)
	323	medium := strings.Repeat("export const value = 42;\n", 5000)
	324	big := make([]byte, loadBlobLimit)
	325	rand.New(rand.NewSource(42)).Read(big)
0448a978 Keenan Nemetz 9d ago	326	files := map[string]string{
0448a978 Keenan Nemetz 9d ago	327	"README.md": "# load test\n\nA repository with enough shape to measure.\n",
1dfa8cbd Keenan Nemetz 9d ago	328	"small.txt": small,
	329	"medium.js": medium,
	330	"limit.bin": string(big),
0448a978 Keenan Nemetz 9d ago	331	}
	332	work := workRepo(t, files)
	333	for i := 0; i < 300; i++ {
	334	appendLine(t, filepath.Join(work, "service.go"), fmt.Sprintf("// revision %d\n", i))
	335	runGit(t, work, "add", "-A")
	336	runGit(t, work, "commit", "-qm", fmt.Sprintf("revision %d", i))
	337	}
	338	if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
	339	t.Fatalf("push main: %s", errs)
	340	}
	341	if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/feature"); err != nil {
	342	t.Fatalf("push feature: %s", errs)
	343	}
	344
1dfa8cbd Keenan Nemetz 9d ago	345	// A branch whose diff is large enough to be a different page from the small
	346	// one: 4000 changed lines, under the page's diff cap.
	347	runGit(t, work, "checkout", "-q", "-b", "big-change")
	348	appendLine(t, filepath.Join(work, "diff.txt"), strings.Repeat("a changed line for the diff\n", 4000))
	349	runGit(t, work, "add", "-A")
	350	runGit(t, work, "commit", "-qm", "large diff")
	351	if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "big-change:refs/heads/big-change"); err != nil {
	352	t.Fatalf("push big-change: %s", errs)
	353	}
	354	runGit(t, work, "checkout", "-q", "main")
	355
	356	// A hosted site on the pages branch.
	357	pages := workRepo(t, map[string]string{
	358	"index.html": "<!doctype html><h1>load test</h1><link rel=stylesheet href=style.css>\n",
	359	"style.css": strings.Repeat("body { margin: 0 }\n", 100),
	360	"docs/index.html": "<p>docs</p>\n",
	361	})
	362	if _, errs, err := f.git(pages, "push", f.sshURL(f.repo), "main:refs/heads/pages"); err != nil {
	363	t.Fatalf("push pages: %s", errs)
	364	}
	365
0448a978 Keenan Nemetz 9d ago	366	m, err := f.srv.Repos.Open(f.repo)
	367	if err != nil {
	368	t.Fatal(err)
	369	}
	370	g := m.Git()
	371	now := time.Now().UTC()
1dfa8cbd Keenan Nemetz 9d ago	372	head, err := g.RevParse("refs/heads/big-change")
	373	if err != nil \|\| head == "" {
	374	t.Fatalf("resolve big-change: %q, %v", head, err)
	375	}
0448a978 Keenan Nemetz 9d ago	376
	377	iss, err := metadata.NewIssue("Load test issue",
	378	strings.Repeat("A body with enough text to index. ", 40), "keenan",
	379	[]string{"p2", "testing"}, []string{"keenan"}, now)
	380	if err != nil {
	381	t.Fatal(err)
	382	}
	383	if err := metadata.CreateIssue(g, iss, "keenan"); err != nil {
	384	t.Fatal(err)
	385	}
1dfa8cbd Keenan Nemetz 9d ago	386	seed.issueID = iss.ID
0448a978 Keenan Nemetz 9d ago	387	for i := 0; i < 5; i++ {
	388	comment, err := metadata.NewIssueComment(fmt.Sprintf("Comment %d on the issue.\n", i), "keenan", now.Add(time.Duration(i)*time.Minute))
	389	if err != nil {
	390	t.Fatal(err)
	391	}
	392	if err := metadata.AddIssueComment(g, iss.ID, comment, "keenan"); err != nil {
	393	t.Fatal(err)
	394	}
	395	}
	396
1dfa8cbd Keenan Nemetz 9d ago	397	req, err := metadata.NewRequest("Load test change with a large diff",
0448a978 Keenan Nemetz 9d ago	398	strings.Repeat("Why this change. ", 40), "keenan", "main", head, now)
	399	if err != nil {
	400	t.Fatal(err)
	401	}
1dfa8cbd Keenan Nemetz 9d ago	402	req.HeadRef = "big-change"
0448a978 Keenan Nemetz 9d ago	403	req.Closes = []string{iss.ID}
	404	if err := metadata.CreateRequest(g, req, "keenan"); err != nil {
	405	t.Fatal(err)
	406	}
1dfa8cbd Keenan Nemetz 9d ago	407	seed.changeID = req.ID
0448a978 Keenan Nemetz 9d ago	408	for i := 0; i < 5; i++ {
	409	comment, err := metadata.NewRequestComment(fmt.Sprintf("Review note %d.\n", i), "keenan", now.Add(time.Duration(i)*time.Minute))
	410	if err != nil {
	411	t.Fatal(err)
	412	}
	413	if err := metadata.AddRequestComment(g, req.ID, comment, "keenan"); err != nil {
	414	t.Fatal(err)
	415	}
	416	}
	417	f.addWriter(t)
1dfa8cbd Keenan Nemetz 9d ago	418	if _, err := metadata.ApproveRequest(g, req.ID, f.writerID, "main", head, now, f.writerID); err != nil {
0448a978 Keenan Nemetz 9d ago	419	t.Fatal(err)
	420	}
	421
	422	// A release with one artifact in the LFS store, so the routes that download
	423	// one have something real to serve.
1dfa8cbd Keenan Nemetz 9d ago	424	artifact := []byte(strings.Repeat("artifact bytes\n", 10000))
0448a978 Keenan Nemetz 9d ago	425	oid, err := f.srv.LFS.PutBytes(f.repo, artifact)
	426	if err != nil {
	427	t.Fatal(err)
	428	}
	429	if err := g.WriteAnnotatedTag("v1", head, "Load test release\n", "keenan", false); err != nil {
	430	t.Fatal(err)
	431	}
	432	if _, err := git.CommitFiles(g, metadata.Ref, func(files map[string][]byte) error {
	433	files[metadata.ReleasePath("v1", "load-artifact.txt")] =
	434	lfs.Pointer{OID: oid, Size: int64(len(artifact))}.Encode()
	435	return nil
	436	}, "release: v1", "keenan"); err != nil {
	437	t.Fatal(err)
	438	}
1dfa8cbd Keenan Nemetz 9d ago	439
	440	seedSmallRepo(t, f, seed.smallRepo)
	441	seedLargeRepo(t, f, seed.largeRepo)
	442	return seed
0448a978 Keenan Nemetz 9d ago	443	}
0448a978 Keenan Nemetz 9d ago	444
1dfa8cbd Keenan Nemetz 9d ago	445	// seedSmallRepo is the other end of the clone measurement: a repository a clone
	446	// finishes quickly, so the small and large numbers are different in the way the
	447	// route's cost actually differs.
	448	func seedSmallRepo(t testing.T, f fixture, name string) {
0448a978 Keenan Nemetz 9d ago	449	t.Helper()
1dfa8cbd Keenan Nemetz 9d ago	450	m, err := f.srv.Repos.Create(name, f.ownerID, "small load repository")
	451	if err != nil {
	452	t.Fatal(err)
0448a978 Keenan Nemetz 9d ago	453	}
1dfa8cbd Keenan Nemetz 9d ago	454	a, err := m.ACL()
0448a978 Keenan Nemetz 9d ago	455	if err != nil {
1dfa8cbd Keenan Nemetz 9d ago	456	t.Fatal(err)
	457	}
	458	a.Default = "read"
	459	if err := m.SetACL(a, "test"); err != nil {
	460	t.Fatal(err)
	461	}
	462	work := workRepo(t, map[string]string{
	463	"README.md": "# small\n",
	464	"main.go": "package main\n\nfunc main() {}\n",
	465	})
	466	for i := 0; i < 50; i++ {
	467	appendLine(t, filepath.Join(work, "main.go"), fmt.Sprintf("// revision %d\n", i))
	468	runGit(t, work, "add", "-A")
	469	runGit(t, work, "commit", "-qm", fmt.Sprintf("revision %d", i))
	470	}
	471	if _, errs, err := f.git(work, "push", f.sshURL(name), "main:refs/heads/main"); err != nil {
	472	t.Fatalf("push small: %s", errs)
	473	}
	474	if err := f.srv.Index.Upsert(m); err != nil {
	475	t.Fatal(err)
0448a978 Keenan Nemetz 9d ago	476	}
	477	}
	478
1dfa8cbd Keenan Nemetz 9d ago	479	// seedLargeRepo writes a repository straight through fast-import: a few thousand
	480	// commits and one incompressible blob, so its clone is a real transfer and the
	481	// history walk is not trivial.
	482	func seedLargeRepo(t testing.T, f fixture, name string) {
	483	t.Helper()
	484	m, err := f.srv.Repos.Create(name, f.ownerID, "large load repository")
	485	if err != nil {
	486	t.Fatal(err)
0448a978 Keenan Nemetz 9d ago	487	}
1dfa8cbd Keenan Nemetz 9d ago	488	a, err := m.ACL()
0448a978 Keenan Nemetz 9d ago	489	if err != nil {
1dfa8cbd Keenan Nemetz 9d ago	490	t.Fatal(err)
	491	}
	492	a.Default = "read"
	493	if err := m.SetACL(a, "test"); err != nil {
	494	t.Fatal(err)
0448a978 Keenan Nemetz 9d ago	495	}
0448a978 Keenan Nemetz 9d ago	496
1dfa8cbd Keenan Nemetz 9d ago	497	big := make([]byte, loadLargeBlob)
	498	rand.New(rand.NewSource(7)).Read(big)
	499	var stream bytes.Buffer
	500	fmt.Fprintf(&stream, "blob\nmark :1\ndata %d\n%s\n", len(big), big)
	501	readme := "large load repository\n"
	502	fmt.Fprintf(&stream, "blob\nmark :2\ndata %d\n%s\n", len(readme), readme)
	503
	504	base := "package demo\n\n// revision 0\nfunc F() {}\n"
	505	fmt.Fprintf(&stream, "blob\nmark :3\ndata %d\n%s\n", len(base), base)
	506	fmt.Fprintf(&stream, "commit refs/heads/main\n")
	507	fmt.Fprintf(&stream, "committer Load <load@example.com> %d +0000\n", 1700000000)
	508	msg := "initial"
	509	fmt.Fprintf(&stream, "data %d\n%s\n", len(msg), msg)
	510	fmt.Fprintf(&stream, "M 100644 :1 big.bin\n")
	511	fmt.Fprintf(&stream, "M 100644 :2 README.md\n")
	512	fmt.Fprintf(&stream, "M 100644 :3 service.go\n")
	513
	514	for i := 1; i < loadLargeCommits; i++ {
	515	content := fmt.Sprintf("package demo\n\n// revision %d\nfunc F() {}\n", i)
	516	mark := 10 + i
	517	fmt.Fprintf(&stream, "blob\nmark :%d\ndata %d\n%s\n", mark, len(content), content)
	518	fmt.Fprintf(&stream, "commit refs/heads/main\n")
	519	fmt.Fprintf(&stream, "committer Load <load@example.com> %d +0000\n", 1700000000+i)
	520	msg := fmt.Sprintf("revision %d", i)
	521	fmt.Fprintf(&stream, "data %d\n%s\n", len(msg), msg)
	522	fmt.Fprintf(&stream, "M 100644 :%d service.go\n", mark)
	523	}
	524
	525	cmd := exec.Command("git", "fast-import", "--quiet")
	526	cmd.Dir = m.Dir
	527	cmd.Env = append(os.Environ(),
	528	"GIT_DIR="+m.Dir, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
	529	cmd.Stdin = &stream
	530	if out, err := cmd.CombinedOutput(); err != nil {
	531	t.Fatalf("fast-import large repo: %v: %s", err, firstLine(string(out)))
0448a978 Keenan Nemetz 9d ago	532	}
1dfa8cbd Keenan Nemetz 9d ago	533	if err := f.srv.Index.Upsert(m); err != nil {
1dfa8cbd Keenan Nemetz 9d ago	534	t.Fatal(err)
0448a978 Keenan Nemetz 9d ago	535	}
	536	}
	537
	538	// loadResults collects one latency sample per request and classifies the
	539	// outcomes by HTTP status and by failure reason.
	540	type loadResults struct {
	541	mu sync.Mutex
	542	requests int
	543	latency []time.Duration
	544	statuses map[string]int
	545	errors map[string]int
	546	}
	547
	548	func newLoadResults() *loadResults {
	549	return &loadResults{statuses: map[string]int{}, errors: map[string]int{}}
	550	}
	551
	552	func (r *loadResults) record(latency time.Duration, status string, err error) {
	553	r.mu.Lock()
	554	defer r.mu.Unlock()
	555	r.requests++
	556	r.latency = append(r.latency, latency)
	557	r.statuses[status]++
	558	if err != nil {
	559	r.errors[firstLine(err.Error())]++
	560	}
	561	}
	562
	563	func (r *loadResults) summary() (requests int, statuses, errors map[string]int, p50, p90, p99 time.Duration) {
	564	r.mu.Lock()
	565	defer r.mu.Unlock()
	566	requests = r.requests
	567	statuses = map[string]int{}
	568	for k, v := range r.statuses {
	569	statuses[k] = v
	570	}
	571	errors = map[string]int{}
	572	for k, v := range r.errors {
	573	errors[k] = v
	574	}
	575	sorted := append([]time.Duration(nil), r.latency...)
	576	sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
	577	p50 = percentile(sorted, 0.50)
	578	p90 = percentile(sorted, 0.90)
	579	p99 = percentile(sorted, 0.99)
	580	return requests, statuses, errors, p50, p90, p99
	581	}
	582
	583	func percentile(sorted []time.Duration, p float64) time.Duration {
	584	if len(sorted) == 0 {
	585	return 0
	586	}
	587	i := int(p * float64(len(sorted)-1))
	588	if i >= len(sorted) {
	589	i = len(sorted) - 1
	590	}
	591	return sorted[i]
	592	}
	593
	594	// processPeak is the largest sample of what the process held during the run.
	595	type processPeak struct {
	596	RSSBytes int64
	597	FDs int
	598	Goroutines int
599	GitChildren int
600	}
601
602	type sampler struct {
603	stopCh chan struct{}
604	doneCh chan struct{}
605	mu sync.Mutex
606	peak processPeak
607	}
608
609	func startSampler() *sampler {
610	s := &sampler{stopCh: make(chan struct{}), doneCh: make(chan struct{})}
611	go func() {
612	defer close(s.doneCh)
613	ticker := time.NewTicker(200 * time.Millisecond)
614	defer ticker.Stop()
615	for {
616	s.sample()
617	select {
618	case <-s.stopCh:
619	return
620	case <-ticker.C:
621	}
622	}
623	}()
624	return s
625	}
626
627	func (s *sampler) sample() {
628	cur := processPeak{Goroutines: runtime.NumGoroutine()}
629	cur.RSSBytes = procRSS()
630	cur.FDs = countDir("/proc/self/fd")
631	cur.GitChildren = gitChildren()
632	s.mu.Lock()
633	defer s.mu.Unlock()
634	if cur.RSSBytes > s.peak.RSSBytes {
635	s.peak.RSSBytes = cur.RSSBytes
636	}
637	if cur.FDs > s.peak.FDs {
638	s.peak.FDs = cur.FDs
639	}
640	if cur.Goroutines > s.peak.Goroutines {
641	s.peak.Goroutines = cur.Goroutines
642	}
643	if cur.GitChildren > s.peak.GitChildren {
644	s.peak.GitChildren = cur.GitChildren
645	}
646	}
647
648	func (s *sampler) stop() processPeak {
649	close(s.stopCh)
650	<-s.doneCh
651	s.mu.Lock()
652	defer s.mu.Unlock()
653	return s.peak
654	}
655
656	// procRSS reads VmRSS from /proc/self/status. A kernel without procfs (or an
657	// operating system without /proc) reports zero rather than failing the run.
658	func procRSS() int64 {
659	data, err := os.ReadFile("/proc/self/status")
660	if err != nil {
661	return 0
662	}
663	for _, line := range strings.Split(string(data), "\n") {
664	if !strings.HasPrefix(line, "VmRSS:") {
665	continue
666	}
667	fields := strings.Fields(line)
668	if len(fields) < 2 {
669	return 0
670	}
671	kb, err := strconv.ParseInt(fields[1], 10, 64)
672	if err != nil {
673	return 0
674	}
675	return kb * 1024
676	}
677	return 0
678	}
679
680	func countDir(dir string) int {
681	entries, err := os.ReadDir(dir)
682	if err != nil {
683	return 0
684	}
685	return len(entries)
686	}
687
688	// gitChildren counts the git processes the server has running: http-backend and
689	// the pack services it starts. The clone processes the harness itself starts are
690	// children of this process too, so they are excluded by name rather than counted
691	// as server load.
692	func gitChildren() int {
693	self := os.Getpid()
694	entries, err := os.ReadDir("/proc")
695	if err != nil {
696	return 0
697	}
698	n := 0
699	for _, e := range entries {
700	pid, err := strconv.Atoi(e.Name())
701	if err != nil \|\| pid == self {
702	continue
703	}
704	stat, err := os.ReadFile(filepath.Join("/proc", e.Name(), "stat"))
705	if err != nil {
706	continue
707	}
708	// pid (comm) state ppid ...
709	s := string(stat)
710	close := strings.LastIndexByte(s, ')')
711	if close < 0 {
712	continue
713	}
714	fields := strings.Fields(s[close+1:])
715	if len(fields) < 2 {
716	continue
717	}
718	ppid, _ := strconv.Atoi(fields[1])
719	if ppid != self {
720	continue
721	}
722	cmdline, err := os.ReadFile(filepath.Join("/proc", e.Name(), "cmdline"))
723	if err != nil {
724	continue
725	}
726	cmd := strings.ReplaceAll(string(cmdline), "\x00", " ")
727	if strings.Contains(cmd, "http-backend") \|\| strings.Contains(cmd, "upload-pack") \|\|
728	strings.Contains(cmd, "receive-pack") {
729	n++
730	}
731	}
732	return n
733	}
734
735	func reportLoad(t testing.T, scenario string, duration time.Duration, concurrency int, results loadResults, peak processPeak) {
736	t.Helper()
737	requests, statuses, errors, p50, p90, p99 := results.summary()
738	rate := float64(requests) / duration.Seconds()
739	t.Logf("%s: concurrency=%d duration=%s", scenario, concurrency, duration)
740	t.Logf(" requests %d (%.1f/s)", requests, rate)
741	for _, status := range sortedKeys(statuses) {
742	t.Logf(" status %-5s %d", status, statuses[status])
743	}
744	for _, reason := range sortedKeys(errors) {
745	t.Logf(" failure %d %s", errors[reason], reason)
746	}
747	t.Logf(" latency p50=%s p90=%s p99=%s", roundDur(p50), roundDur(p90), roundDur(p99))
748	t.Logf(" peak rss=%s fds=%d goroutines=%d git-children=%d",
749	roundBytes(peak.RSSBytes), peak.FDs, peak.Goroutines, peak.GitChildren)
750	}
751
752	func sortedKeys(m map[string]int) []string {
753	out := make([]string, 0, len(m))
754	for k := range m {
755	out = append(out, k)
756	}
757	sort.Strings(out)
758	return out
759	}
760
761	func roundDur(d time.Duration) time.Duration { return d.Round(time.Millisecond) }
762
763	func roundBytes(n int64) string {
764	const unit = 1024
765	if n < unit {
766	return fmt.Sprintf("%d B", n)
767	}
768	k := float64(n) / unit
769	for _, suffix := range []string{"KiB", "MiB", "GiB"} {
770	if k < unit {
771	return fmt.Sprintf("%.1f %s", k, suffix)
772	}
773	k /= unit
774	}
775	return fmt.Sprintf("%.1f TiB", k)
776	}
777
778	func appendLine(t *testing.T, path, line string) {
779	t.Helper()
780	f, err := os.OpenFile(path, os.O_CREATE\|os.O_WRONLY\|os.O_APPEND, 0o644)
781	if err != nil {
782	t.Fatal(err)
783	}
784	if _, err := f.WriteString(line); err != nil {
785	f.Close()
786	t.Fatal(err)
787	}
788	if err := f.Close(); err != nil {
789	t.Fatal(err)
790	}
791	}
792
1dfa8cbd Keenan Nemetz 9d ago	793	func firstLine(s string) string {
	794	if i := strings.IndexByte(s, '\n'); i >= 0 {
	795	s = s[:i]
	796	}
	797	if len(s) > 120 {
	798	s = s[:120]
	799	}
	800	return s
	801	}
	802
0448a978 Keenan Nemetz 9d ago	803	func envString(name, def string) string {
	804	if v := strings.TrimSpace(os.Getenv(name)); v != "" {
	805	return v
	806	}
	807	return def
	808	}
	809
	810	func envInt(name string, def int) int {
	811	v := strings.TrimSpace(os.Getenv(name))
	812	if v == "" {
	813	return def
	814	}
	815	n, err := strconv.Atoi(v)
	816	if err != nil {
	817	return def
	818	}
	819	return n
	820	}
	821
	822	func envDuration(name string, def time.Duration) time.Duration {
	823	v := strings.TrimSpace(os.Getenv(name))
	824	if v == "" {
	825	return def
	826	}
	827	d, err := time.ParseDuration(v)
	828	if err != nil {
	829	return def
	830	}
	831	return d
	832	}