Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Code

395 commit(s) on main

Compare History
update readme Keenan Nemetz · 29m ago 32290114
history blame raw 24.8 KiB
1//go:build load
2
3package server_test
4
5// A load harness for a real, running instance.
6//
7// It is behind the `load` build tag so `go test ./...` and the verify workflow
8// never run it. Everything else follows the integration fixture: a server on a
9// loopback port with a temporary data directory, so the numbers describe the
10// same code an operator runs, not a mock.
11//
12// Run every route:
13//
14// VOLE_LOAD_SCENARIO=all VOLE_LOAD_CONCURRENCY=4 VOLE_LOAD_DURATION=5s \
15// go test -tags load,sqlite_fts5 -run '^TestLoad$' -v ./server
16//
17// Or one route, e.g. clones past the per-client limit:
18//
19// VOLE_LOAD_SCENARIO=clone VOLE_LOAD_CONCURRENCY=24 VOLE_LOAD_DURATION=10s \
20// go test -tags load,sqlite_fts5 -run '^TestLoad$' -v ./server
21//
22// The knobs are environment variables rather than flags so the documented
23// command has no `-args` in it. See docs/load-testing.md.
24
25import (
26 "bytes"
27 "fmt"
28 "io"
29 "math/rand"
30 "net/http"
31 "net/url"
32 "os"
33 "os/exec"
34 "path/filepath"
35 "runtime"
36 "sort"
37 "strconv"
38 "strings"
39 "sync"
40 "testing"
41 "time"
42
43 "rodent.software/vole/core/git"
44 "rodent.software/vole/core/lfs"
45 "rodent.software/vole/forge/config"
46 "rodent.software/vole/metadata"
47)
48
49const (
50 loadPagesDomain = "pages.test"
51 // loadBlobLimit is the configured max_blob_bytes, so a file at the limit is
52 // something a push could actually deliver.
53 loadBlobLimit = 8 << 20
54 // loadLargeCommits and loadLargeBlob make the second repository large enough
55 // that a clone of it is a different measurement from a clone of the fixture.
56 loadLargeCommits = 3000
57 loadLargeBlob = 16 << 20
58)
59
60// loadSeed is what the scenarios need to name a route.
61type loadSeed struct {
62 repo string
63 smallRepo string
64 largeRepo string
65 issueID string
66 changeID string
67}
68
69func TestLoad(t *testing.T) {
70 scenario := envString("VOLE_LOAD_SCENARIO", "clone")
71 concurrency := envInt("VOLE_LOAD_CONCURRENCY", 4)
72 duration := envDuration("VOLE_LOAD_DURATION", 10*time.Second)
73 if concurrency < 1 {
74 t.Fatalf("VOLE_LOAD_CONCURRENCY=%d, want at least 1", concurrency)
75 }
76 if duration <= 0 {
77 t.Fatalf("VOLE_LOAD_DURATION=%s, want a positive duration", duration)
78 }
79
80 f := newFixtureWith(t, func(c *config.Config) {
81 c.PagesDomain = loadPagesDomain
82 c.PagesRef = "pages"
83 c.MaxBlobBytes = loadBlobLimit
84 // The clone scenario gives every worker its own forwarded address, so the
85 // global git_http_max_concurrent is the limit that refuses, not the
86 // per-client one. The fixture is loopback, so trusting the header is safe.
87 c.GitHTTPClientIPHeader = "X-Forwarded-For"
88 })
89 seed := seedLoadRepo(t, f)
90
91 // One read per repository builds its index before the run, so the first
92 // reader does not pay for the whole metadata branch inside the measurement.
93 f.get("/" + f.repo)
94 f.get("/" + seed.largeRepo)
95 scenarios := loadScenarios(t, f, seed)
96
97 if scenario == "all" {
98 for _, sc := range scenarios {
99 runScenario(t, f, sc, duration, concurrency)
100 t.Logf("")
101 }
102 return
103 }
104 for _, sc := range scenarios {
105 if sc.name == scenario {
106 runScenario(t, f, sc, duration, concurrency)
107 return
108 }
109 }
110 var names []string
111 for _, sc := range scenarios {
112 names = append(names, sc.name)
113 }
114 t.Fatalf("unknown VOLE_LOAD_SCENARIO %q (have: all, %s)", scenario, strings.Join(names, ", "))
115}
116
117type scenario struct {
118 name string
119 step loadStep
120}
121
122// loadStep is one request. The worker number exists so a step that needs a
123// workspace (a clone target) gets its own instead of racing its peers.
124type loadStep func(client *http.Client, worker int) (status string, err error)
125
126func runScenario(t *testing.T, f *fixture, sc scenario, duration time.Duration, concurrency int) {
127 t.Helper()
128 client := &http.Client{Timeout: 120 * time.Second}
129 results := newLoadResults()
130 peaks := startSampler()
131 deadline := time.Now().Add(duration)
132
133 var wg sync.WaitGroup
134 for i := 0; i < concurrency; i++ {
135 wg.Add(1)
136 go func(worker int) {
137 defer wg.Done()
138 for time.Now().Before(deadline) {
139 start := time.Now()
140 status, err := sc.step(client, worker)
141 results.record(time.Since(start), status, err)
142 }
143 }(i)
144 }
145 wg.Wait()
146 reportLoad(t, sc.name, duration, concurrency, results, peaks.stop())
147}
148
149// loadScenarios names every route #100 asks about. Each is one step a worker
150// repeats until the deadline.
151func loadScenarios(t *testing.T, f *fixture, seed loadSeed) []scenario {
152 t.Helper()
153 cloneDir := t.TempDir()
154 repo := f.http.URL + "/" + seed.repo
155 siteURL := f.http.URL + "/"
156 cloneURL := f.httpURL(seed.smallRepo)
157 largeCloneURL := f.httpURL(seed.largeRepo)
158
159 return []scenario{
160 {name: "clone", step: cloneStep(cloneURL, cloneDir)},
161 {name: "clone-large", step: cloneStep(largeCloneURL, cloneDir)},
162 {name: "blob-small", step: getStep(repo + "/blob/main/small.txt")},
163 {name: "blob-limit", step: getStep(repo + "/blob/main/limit.bin")},
164 {name: "raw-small", step: getStep(repo + "/raw/main/small.txt")},
165 {name: "raw-limit", step: getStep(repo + "/raw/main/limit.bin")},
166 {name: "raw-range", step: getStepWith(repo+"/raw/main/limit.bin", map[string]string{"Range": "bytes=0-65535"})},
167 {name: "overview", step: getStep(repo)},
168 {name: "tree", step: getStep(repo + "/tree/main")},
169 {name: "commits", step: getStep(repo + "/commits/main")},
170 {name: "compare", step: getStep(repo + "/compare/main...feature")},
171 {name: "change", step: getStep(repo + "/changes/" + seed.changeID)},
172 {name: "issue", step: getStep(repo + "/issues/" + seed.issueID)},
173 {name: "activity", step: getStep(repo + "/activity")},
174 {name: "instance-activity", step: getStep(f.http.URL + "/-/activity")},
175 {name: "home", step: getStep(f.http.URL + "/")},
176 {name: "search-code", step: getStep(repo + "/search?q=revision")},
177 {name: "search-find", step: getStepWith(f.http.URL+"/-/search/palette?q=revision", map[string]string{"HX-Request": "true"})},
178 {name: "artifact", step: getStep(repo + "/releases/v1/artifacts/load-artifact.txt")},
179 {name: "artifact-range", step: getStepWith(repo+"/releases/v1/artifacts/load-artifact.txt", map[string]string{"Range": "bytes=0-1023"})},
180 {name: "site", step: siteStep(siteURL, seed.repo+"."+loadPagesDomain)},
181 {name: "comment", step: commentStep(t, f, f.writerID, repo+"/issues/"+seed.issueID)},
182 }
183}
184
185// getStep is one anonymous GET, with the body drained so the server sees the
186// whole response.
187func getStep(url string) loadStep {
188 return getStepWith(url, nil)
189}
190
191func getStepWith(url string, headers map[string]string) loadStep {
192 return func(client *http.Client, worker int) (string, error) {
193 req, err := http.NewRequest(http.MethodGet, url, nil)
194 if err != nil {
195 return "error", err
196 }
197 for k, v := range headers {
198 req.Header.Set(k, v)
199 }
200 resp, err := client.Do(req)
201 if err != nil {
202 return "error", err
203 }
204 _, _ = io.Copy(io.Discard, resp.Body)
205 resp.Body.Close()
206 return strconv.Itoa(resp.StatusCode), nil
207 }
208}
209
210// siteStep requests a hosted site page while still talking to the test listener,
211// which is what a real request with a Host header does.
212func siteStep(url, host string) loadStep {
213 return func(client *http.Client, worker int) (string, error) {
214 req, err := http.NewRequest(http.MethodGet, url, nil)
215 if err != nil {
216 return "error", err
217 }
218 req.Host = host
219 resp, err := client.Do(req)
220 if err != nil {
221 return "error", err
222 }
223 _, _ = io.Copy(io.Discard, resp.Body)
224 resp.Body.Close()
225 return strconv.Itoa(resp.StatusCode), nil
226 }
227}
228
229// commentStep is the only scenario that writes. Each request is one comment, and a
230// comment is one commit on the metadata branch every write in the instance shares, so
231// this is the scenario that measures what concurrency on that branch costs. The
232// redirect after a successful post is not followed: the write is the measurement, and
233// the page it lands on is the issue scenario.
234func commentStep(t *testing.T, f *fixture, accountID, issuePath string) loadStep {
235 t.Helper()
236 sess, err := f.srv.Sessions.Create(accountID, nil, "load")
237 if err != nil {
238 t.Fatal(err)
239 }
240 cookie := "__Host-vole_session=" + sess.Token
241 comment := &http.Client{
242 Timeout: 120 * time.Second,
243 CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
244 }
245 return func(client *http.Client, worker int) (string, error) {
246 form := url.Values{"csrf": {sess.CSRF}, "body": {"load comment"}}
247 req, err := http.NewRequest(http.MethodPost, issuePath+"/comments", strings.NewReader(form.Encode()))
248 if err != nil {
249 return "error", err
250 }
251 req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
252 req.Header.Set("Cookie", cookie)
253 resp, err := comment.Do(req)
254 if err != nil {
255 return "error", err
256 }
257 _, _ = io.Copy(io.Discard, resp.Body)
258 resp.Body.Close()
259 return strconv.Itoa(resp.StatusCode), nil
260 }
261}
262
263// cloneStep probes `info/refs` and then clones. The probe is what a server can
264// refuse (503 past the concurrency limit), so it is recorded as the step's
265// status; the git process's own failure is recorded separately.
266func cloneStep(repoURL, root string) loadStep {
267 infoRefs := repoURL + "/info/refs?service=git-upload-pack"
268 return func(client *http.Client, worker int) (string, error) {
269 // A distinct forwarded address per worker keeps the per-client limit from
270 // binding before the global one.
271 forwarded := fmt.Sprintf("10.0.%d.%d", worker/250, worker%250+1)
272 status, err := probe(client, infoRefs, forwarded)
273 if err != nil {
274 return status, err
275 }
276 if status != "200" {
277 return status, nil
278 }
279 return status, cloneShallow(repoURL, filepath.Join(root, strconv.Itoa(worker)), forwarded)
280 }
281}
282
283func probe(client *http.Client, url, forwarded string) (string, error) {
284 req, err := http.NewRequest(http.MethodGet, url, nil)
285 if err != nil {
286 return "error", err
287 }
288 req.Header.Set("X-Forwarded-For", forwarded)
289 resp, err := client.Do(req)
290 if err != nil {
291 return "error", err
292 }
293 _, _ = io.Copy(io.Discard, resp.Body)
294 resp.Body.Close()
295 return strconv.Itoa(resp.StatusCode), nil
296}
297
298func cloneShallow(url, dir, forwarded string) error {
299 if err := os.RemoveAll(dir); err != nil {
300 return err
301 }
302 cmd := exec.Command("git", "-c", "http.extraHeader=X-Forwarded-For: "+forwarded,
303 "clone", "--quiet", "--depth", "1", url, dir)
304 cmd.Env = append(os.Environ(),
305 "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
306 out, err := cmd.CombinedOutput()
307 if err != nil {
308 return fmt.Errorf("clone: %v: %s", err, firstLine(string(out)))
309 }
310 return nil
311}
312
313// seedLoadRepo gives the instance the shape #100 measures: a small repository
314// with history and files at both ends of max_blob_bytes, a data-only change with
315// a large diff, a release artifact, a hosted site, and a second repository large
316// enough that cloning it costs something.
317func seedLoadRepo(t *testing.T, f *fixture) loadSeed {
318 t.Helper()
319 f.publish(f.repo)
320 seed := loadSeed{repo: f.repo, smallRepo: "small", largeRepo: "large"}
321
322 small := strings.Repeat("small line\n", 100)
323 medium := strings.Repeat("export const value = 42;\n", 5000)
324 big := make([]byte, loadBlobLimit)
325 rand.New(rand.NewSource(42)).Read(big)
326 files := map[string]string{
327 "README.md": "# load test\n\nA repository with enough shape to measure.\n",
328 "small.txt": small,
329 "medium.js": medium,
330 "limit.bin": string(big),
331 }
332 work := workRepo(t, files)
333 for i := 0; i < 300; i++ {
334 appendLine(t, filepath.Join(work, "service.go"), fmt.Sprintf("// revision %d\n", i))
335 runGit(t, work, "add", "-A")
336 runGit(t, work, "commit", "-qm", fmt.Sprintf("revision %d", i))
337 }
338 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
339 t.Fatalf("push main: %s", errs)
340 }
341 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/feature"); err != nil {
342 t.Fatalf("push feature: %s", errs)
343 }
344
345 // A branch whose diff is large enough to be a different page from the small
346 // one: 4000 changed lines, under the page's diff cap.
347 runGit(t, work, "checkout", "-q", "-b", "big-change")
348 appendLine(t, filepath.Join(work, "diff.txt"), strings.Repeat("a changed line for the diff\n", 4000))
349 runGit(t, work, "add", "-A")
350 runGit(t, work, "commit", "-qm", "large diff")
351 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "big-change:refs/heads/big-change"); err != nil {
352 t.Fatalf("push big-change: %s", errs)
353 }
354 runGit(t, work, "checkout", "-q", "main")
355
356 // A hosted site on the pages branch.
357 pages := workRepo(t, map[string]string{
358 "index.html": "<!doctype html><h1>load test</h1><link rel=stylesheet href=style.css>\n",
359 "style.css": strings.Repeat("body { margin: 0 }\n", 100),
360 "docs/index.html": "<p>docs</p>\n",
361 })
362 if _, errs, err := f.git(pages, "push", f.sshURL(f.repo), "main:refs/heads/pages"); err != nil {
363 t.Fatalf("push pages: %s", errs)
364 }
365
366 m, err := f.srv.Repos.Open(f.repo)
367 if err != nil {
368 t.Fatal(err)
369 }
370 g := m.Git()
371 now := time.Now().UTC()
372 head, err := g.RevParse("refs/heads/big-change")
373 if err != nil || head == "" {
374 t.Fatalf("resolve big-change: %q, %v", head, err)
375 }
376
377 iss, err := metadata.NewIssue("Load test issue",
378 strings.Repeat("A body with enough text to index. ", 40), "keenan",
379 []string{"p2", "testing"}, []string{"keenan"}, now)
380 if err != nil {
381 t.Fatal(err)
382 }
383 if err := metadata.CreateIssue(g, iss, "keenan"); err != nil {
384 t.Fatal(err)
385 }
386 seed.issueID = iss.ID
387 for i := 0; i < 5; i++ {
388 comment, err := metadata.NewIssueComment(fmt.Sprintf("Comment %d on the issue.\n", i), "keenan", now.Add(time.Duration(i)*time.Minute))
389 if err != nil {
390 t.Fatal(err)
391 }
392 if err := metadata.AddIssueComment(g, iss.ID, comment, "keenan"); err != nil {
393 t.Fatal(err)
394 }
395 }
396
397 req, err := metadata.NewRequest("Load test change with a large diff",
398 strings.Repeat("Why this change. ", 40), "keenan", "main", head, now)
399 if err != nil {
400 t.Fatal(err)
401 }
402 req.HeadRef = "big-change"
403 req.Closes = []string{iss.ID}
404 if err := metadata.CreateRequest(g, req, "keenan"); err != nil {
405 t.Fatal(err)
406 }
407 seed.changeID = req.ID
408 for i := 0; i < 5; i++ {
409 comment, err := metadata.NewRequestComment(fmt.Sprintf("Review note %d.\n", i), "keenan", now.Add(time.Duration(i)*time.Minute))
410 if err != nil {
411 t.Fatal(err)
412 }
413 if err := metadata.AddRequestComment(g, req.ID, comment, "keenan"); err != nil {
414 t.Fatal(err)
415 }
416 }
417 f.addWriter(t)
418 if _, err := metadata.ApproveRequest(g, req.ID, f.writerID, "main", head, now, f.writerID); err != nil {
419 t.Fatal(err)
420 }
421
422 // A release with one artifact in the LFS store, so the routes that download
423 // one have something real to serve.
424 artifact := []byte(strings.Repeat("artifact bytes\n", 10000))
425 oid, err := f.srv.LFS.PutBytes(f.repo, artifact)
426 if err != nil {
427 t.Fatal(err)
428 }
429 if err := g.WriteAnnotatedTag("v1", head, "Load test release\n", "keenan", false); err != nil {
430 t.Fatal(err)
431 }
432 if _, err := git.CommitFiles(g, metadata.Ref, func(files map[string][]byte) error {
433 files[metadata.ReleasePath("v1", "load-artifact.txt")] =
434 lfs.Pointer{OID: oid, Size: int64(len(artifact))}.Encode()
435 return nil
436 }, "release: v1", "keenan"); err != nil {
437 t.Fatal(err)
438 }
439
440 seedSmallRepo(t, f, seed.smallRepo)
441 seedLargeRepo(t, f, seed.largeRepo)
442 return seed
443}
444
445// seedSmallRepo is the other end of the clone measurement: a repository a clone
446// finishes quickly, so the small and large numbers are different in the way the
447// route's cost actually differs.
448func seedSmallRepo(t *testing.T, f *fixture, name string) {
449 t.Helper()
450 m, err := f.srv.Repos.Create(name, f.ownerID, "small load repository")
451 if err != nil {
452 t.Fatal(err)
453 }
454 a, err := m.ACL()
455 if err != nil {
456 t.Fatal(err)
457 }
458 a.Default = "read"
459 if err := m.SetACL(a, "test"); err != nil {
460 t.Fatal(err)
461 }
462 work := workRepo(t, map[string]string{
463 "README.md": "# small\n",
464 "main.go": "package main\n\nfunc main() {}\n",
465 })
466 for i := 0; i < 50; i++ {
467 appendLine(t, filepath.Join(work, "main.go"), fmt.Sprintf("// revision %d\n", i))
468 runGit(t, work, "add", "-A")
469 runGit(t, work, "commit", "-qm", fmt.Sprintf("revision %d", i))
470 }
471 if _, errs, err := f.git(work, "push", f.sshURL(name), "main:refs/heads/main"); err != nil {
472 t.Fatalf("push small: %s", errs)
473 }
474 if err := f.srv.Index.Upsert(m); err != nil {
475 t.Fatal(err)
476 }
477}
478
479// seedLargeRepo writes a repository straight through fast-import: a few thousand
480// commits and one incompressible blob, so its clone is a real transfer and the
481// history walk is not trivial.
482func seedLargeRepo(t *testing.T, f *fixture, name string) {
483 t.Helper()
484 m, err := f.srv.Repos.Create(name, f.ownerID, "large load repository")
485 if err != nil {
486 t.Fatal(err)
487 }
488 a, err := m.ACL()
489 if err != nil {
490 t.Fatal(err)
491 }
492 a.Default = "read"
493 if err := m.SetACL(a, "test"); err != nil {
494 t.Fatal(err)
495 }
496
497 big := make([]byte, loadLargeBlob)
498 rand.New(rand.NewSource(7)).Read(big)
499 var stream bytes.Buffer
500 fmt.Fprintf(&stream, "blob\nmark :1\ndata %d\n%s\n", len(big), big)
501 readme := "large load repository\n"
502 fmt.Fprintf(&stream, "blob\nmark :2\ndata %d\n%s\n", len(readme), readme)
503
504 base := "package demo\n\n// revision 0\nfunc F() {}\n"
505 fmt.Fprintf(&stream, "blob\nmark :3\ndata %d\n%s\n", len(base), base)
506 fmt.Fprintf(&stream, "commit refs/heads/main\n")
507 fmt.Fprintf(&stream, "committer Load <load@example.com> %d +0000\n", 1700000000)
508 msg := "initial"
509 fmt.Fprintf(&stream, "data %d\n%s\n", len(msg), msg)
510 fmt.Fprintf(&stream, "M 100644 :1 big.bin\n")
511 fmt.Fprintf(&stream, "M 100644 :2 README.md\n")
512 fmt.Fprintf(&stream, "M 100644 :3 service.go\n")
513
514 for i := 1; i < loadLargeCommits; i++ {
515 content := fmt.Sprintf("package demo\n\n// revision %d\nfunc F() {}\n", i)
516 mark := 10 + i
517 fmt.Fprintf(&stream, "blob\nmark :%d\ndata %d\n%s\n", mark, len(content), content)
518 fmt.Fprintf(&stream, "commit refs/heads/main\n")
519 fmt.Fprintf(&stream, "committer Load <load@example.com> %d +0000\n", 1700000000+i)
520 msg := fmt.Sprintf("revision %d", i)
521 fmt.Fprintf(&stream, "data %d\n%s\n", len(msg), msg)
522 fmt.Fprintf(&stream, "M 100644 :%d service.go\n", mark)
523 }
524
525 cmd := exec.Command("git", "fast-import", "--quiet")
526 cmd.Dir = m.Dir
527 cmd.Env = append(os.Environ(),
528 "GIT_DIR="+m.Dir, "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
529 cmd.Stdin = &stream
530 if out, err := cmd.CombinedOutput(); err != nil {
531 t.Fatalf("fast-import large repo: %v: %s", err, firstLine(string(out)))
532 }
533 if err := f.srv.Index.Upsert(m); err != nil {
534 t.Fatal(err)
535 }
536}
537
538// loadResults collects one latency sample per request and classifies the
539// outcomes by HTTP status and by failure reason.
540type loadResults struct {
541 mu sync.Mutex
542 requests int
543 latency []time.Duration
544 statuses map[string]int
545 errors map[string]int
546}
547
548func newLoadResults() *loadResults {
549 return &loadResults{statuses: map[string]int{}, errors: map[string]int{}}
550}
551
552func (r *loadResults) record(latency time.Duration, status string, err error) {
553 r.mu.Lock()
554 defer r.mu.Unlock()
555 r.requests++
556 r.latency = append(r.latency, latency)
557 r.statuses[status]++
558 if err != nil {
559 r.errors[firstLine(err.Error())]++
560 }
561}
562
563func (r *loadResults) summary() (requests int, statuses, errors map[string]int, p50, p90, p99 time.Duration) {
564 r.mu.Lock()
565 defer r.mu.Unlock()
566 requests = r.requests
567 statuses = map[string]int{}
568 for k, v := range r.statuses {
569 statuses[k] = v
570 }
571 errors = map[string]int{}
572 for k, v := range r.errors {
573 errors[k] = v
574 }
575 sorted := append([]time.Duration(nil), r.latency...)
576 sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
577 p50 = percentile(sorted, 0.50)
578 p90 = percentile(sorted, 0.90)
579 p99 = percentile(sorted, 0.99)
580 return requests, statuses, errors, p50, p90, p99
581}
582
583func percentile(sorted []time.Duration, p float64) time.Duration {
584 if len(sorted) == 0 {
585 return 0
586 }
587 i := int(p * float64(len(sorted)-1))
588 if i >= len(sorted) {
589 i = len(sorted) - 1
590 }
591 return sorted[i]
592}
593
594// processPeak is the largest sample of what the process held during the run.
595type processPeak struct {
596 RSSBytes int64
597 FDs int
598 Goroutines int
599 GitChildren int
600}
601
602type sampler struct {
603 stopCh chan struct{}
604 doneCh chan struct{}
605 mu sync.Mutex
606 peak processPeak
607}
608
609func startSampler() *sampler {
610 s := &sampler{stopCh: make(chan struct{}), doneCh: make(chan struct{})}
611 go func() {
612 defer close(s.doneCh)
613 ticker := time.NewTicker(200 * time.Millisecond)
614 defer ticker.Stop()
615 for {
616 s.sample()
617 select {
618 case <-s.stopCh:
619 return
620 case <-ticker.C:
621 }
622 }
623 }()
624 return s
625}
626
627func (s *sampler) sample() {
628 cur := processPeak{Goroutines: runtime.NumGoroutine()}
629 cur.RSSBytes = procRSS()
630 cur.FDs = countDir("/proc/self/fd")
631 cur.GitChildren = gitChildren()
632 s.mu.Lock()
633 defer s.mu.Unlock()
634 if cur.RSSBytes > s.peak.RSSBytes {
635 s.peak.RSSBytes = cur.RSSBytes
636 }
637 if cur.FDs > s.peak.FDs {
638 s.peak.FDs = cur.FDs
639 }
640 if cur.Goroutines > s.peak.Goroutines {
641 s.peak.Goroutines = cur.Goroutines
642 }
643 if cur.GitChildren > s.peak.GitChildren {
644 s.peak.GitChildren = cur.GitChildren
645 }
646}
647
648func (s *sampler) stop() processPeak {
649 close(s.stopCh)
650 <-s.doneCh
651 s.mu.Lock()
652 defer s.mu.Unlock()
653 return s.peak
654}
655
656// procRSS reads VmRSS from /proc/self/status. A kernel without procfs (or an
657// operating system without /proc) reports zero rather than failing the run.
658func procRSS() int64 {
659 data, err := os.ReadFile("/proc/self/status")
660 if err != nil {
661 return 0
662 }
663 for _, line := range strings.Split(string(data), "\n") {
664 if !strings.HasPrefix(line, "VmRSS:") {
665 continue
666 }
667 fields := strings.Fields(line)
668 if len(fields) < 2 {
669 return 0
670 }
671 kb, err := strconv.ParseInt(fields[1], 10, 64)
672 if err != nil {
673 return 0
674 }
675 return kb * 1024
676 }
677 return 0
678}
679
680func countDir(dir string) int {
681 entries, err := os.ReadDir(dir)
682 if err != nil {
683 return 0
684 }
685 return len(entries)
686}
687
688// gitChildren counts the git processes the server has running: http-backend and
689// the pack services it starts. The clone processes the harness itself starts are
690// children of this process too, so they are excluded by name rather than counted
691// as server load.
692func gitChildren() int {
693 self := os.Getpid()
694 entries, err := os.ReadDir("/proc")
695 if err != nil {
696 return 0
697 }
698 n := 0
699 for _, e := range entries {
700 pid, err := strconv.Atoi(e.Name())
701 if err != nil || pid == self {
702 continue
703 }
704 stat, err := os.ReadFile(filepath.Join("/proc", e.Name(), "stat"))
705 if err != nil {
706 continue
707 }
708 // pid (comm) state ppid ...
709 s := string(stat)
710 close := strings.LastIndexByte(s, ')')
711 if close < 0 {
712 continue
713 }
714 fields := strings.Fields(s[close+1:])
715 if len(fields) < 2 {
716 continue
717 }
718 ppid, _ := strconv.Atoi(fields[1])
719 if ppid != self {
720 continue
721 }
722 cmdline, err := os.ReadFile(filepath.Join("/proc", e.Name(), "cmdline"))
723 if err != nil {
724 continue
725 }
726 cmd := strings.ReplaceAll(string(cmdline), "\x00", " ")
727 if strings.Contains(cmd, "http-backend") || strings.Contains(cmd, "upload-pack") ||
728 strings.Contains(cmd, "receive-pack") {
729 n++
730 }
731 }
732 return n
733}
734
735func reportLoad(t *testing.T, scenario string, duration time.Duration, concurrency int, results *loadResults, peak processPeak) {
736 t.Helper()
737 requests, statuses, errors, p50, p90, p99 := results.summary()
738 rate := float64(requests) / duration.Seconds()
739 t.Logf("%s: concurrency=%d duration=%s", scenario, concurrency, duration)
740 t.Logf(" requests %d (%.1f/s)", requests, rate)
741 for _, status := range sortedKeys(statuses) {
742 t.Logf(" status %-5s %d", status, statuses[status])
743 }
744 for _, reason := range sortedKeys(errors) {
745 t.Logf(" failure %d %s", errors[reason], reason)
746 }
747 t.Logf(" latency p50=%s p90=%s p99=%s", roundDur(p50), roundDur(p90), roundDur(p99))
748 t.Logf(" peak rss=%s fds=%d goroutines=%d git-children=%d",
749 roundBytes(peak.RSSBytes), peak.FDs, peak.Goroutines, peak.GitChildren)
750}
751
752func sortedKeys(m map[string]int) []string {
753 out := make([]string, 0, len(m))
754 for k := range m {
755 out = append(out, k)
756 }
757 sort.Strings(out)
758 return out
759}
760
761func roundDur(d time.Duration) time.Duration { return d.Round(time.Millisecond) }
762
763func roundBytes(n int64) string {
764 const unit = 1024
765 if n < unit {
766 return fmt.Sprintf("%d B", n)
767 }
768 k := float64(n) / unit
769 for _, suffix := range []string{"KiB", "MiB", "GiB"} {
770 if k < unit {
771 return fmt.Sprintf("%.1f %s", k, suffix)
772 }
773 k /= unit
774 }
775 return fmt.Sprintf("%.1f TiB", k)
776}
777
778func appendLine(t *testing.T, path, line string) {
779 t.Helper()
780 f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
781 if err != nil {
782 t.Fatal(err)
783 }
784 if _, err := f.WriteString(line); err != nil {
785 f.Close()
786 t.Fatal(err)
787 }
788 if err := f.Close(); err != nil {
789 t.Fatal(err)
790 }
791}
792
793func firstLine(s string) string {
794 if i := strings.IndexByte(s, '\n'); i >= 0 {
795 s = s[:i]
796 }
797 if len(s) > 120 {
798 s = s[:120]
799 }
800 return s
801}
802
803func envString(name, def string) string {
804 if v := strings.TrimSpace(os.Getenv(name)); v != "" {
805 return v
806 }
807 return def
808}
809
810func envInt(name string, def int) int {
811 v := strings.TrimSpace(os.Getenv(name))
812 if v == "" {
813 return def
814 }
815 n, err := strconv.Atoi(v)
816 if err != nil {
817 return def
818 }
819 return n
820}
821
822func envDuration(name string, def time.Duration) time.Duration {
823 v := strings.TrimSpace(os.Getenv(name))
824 if v == "" {
825 return def
826 }
827 d, err := time.ParseDuration(v)
828 if err != nil {
829 return def
830 }
831 return d
832}
833