Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.

vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Add a load harness behind the load tag

0448a9783b69753a26bd7478e9eb70c510a8e487 · Keenan Nemetz · 10d ago · parent 14b5c648

Add a load harness behind the load tag

Every limit in the operations guide was chosen from reasoning. The harness
is what replaces that with a number: a real server on a loopback port with
a temporary data directory, seeded with a repository that has history,
files at several sizes, an issue and a change with comments and an
approval, and a release artifact in the LFS store.

One scenario proves it end to end: anonymous shallow clones over HTTP.
Each iteration probes `info/refs` so a refusal is reported as the status
it was refused with, then clones. Workers run until the configured
duration, and the report gives requests, throughput, p50/p90/p99 latency,
statuses, failure reasons, and the peak of RSS, file descriptors,
goroutines and the server's git child processes.

It is behind `//go:build load`, so `go test ./...` and the verify workflow
never run it, and nothing in it ships in the binary. The other read-path
scenarios in #100 add their own step functions to the same runner.

Run it:

  VOLE_LOAD_SCENARIO=clone VOLE_LOAD_CONCURRENCY=8 VOLE_LOAD_DURATION=30s \
    go test -tags load,sqlite_fts5 -run '^TestLoad$' -v ./server

server/load_test.go +526

@@ -0,0 +1,526 @@
1 //go:build load
2
3 package server_test
4
5 // A load harness for a real, running instance.
6 //
7 // It is behind the `load` build tag so `go test ./...` and the verify workflow
8 // never run it. Everything else follows the integration fixture: a server on a
9 // loopback port with a temporary data directory, so the numbers describe the
10 // same code an operator runs, not a mock.
11 //
12 // One scenario is enough to prove the harness end to end: an anonymous clone
13 // over HTTP. The other routes in #100 add their own step functions; the seeding,
14 // sampling, reporting and configuration here are what they share.
15 //
16 // Run it:
17 //
18 // VOLE_LOAD_SCENARIO=clone VOLE_LOAD_CONCURRENCY=8 VOLE_LOAD_DURATION=30s \
19 // go test -tags load -run '^TestLoad$' -v ./server
20 //
21 // The knobs are environment variables rather than flags so the documented
22 // command has no `-args` in it. See docs/load-testing.md.
23
24 import (
25 "fmt"
26 "io"
27 "net/http"
28 "os"
29 "os/exec"
30 "path/filepath"
31 "runtime"
32 "sort"
33 "strconv"
34 "strings"
35 "sync"
36 "testing"
37 "time"
38
39 "github.com/rodent-software/vole/core/git"
40 "github.com/rodent-software/vole/core/lfs"
41 "github.com/rodent-software/vole/metadata"
42 )
43
44 func TestLoad(t *testing.T) {
45 scenario := envString("VOLE_LOAD_SCENARIO", "clone")
46 concurrency := envInt("VOLE_LOAD_CONCURRENCY", 4)
47 duration := envDuration("VOLE_LOAD_DURATION", 10*time.Second)
48 if concurrency < 1 {
49 t.Fatalf("VOLE_LOAD_CONCURRENCY=%d, want at least 1", concurrency)
50 }
51 if duration <= 0 {
52 t.Fatalf("VOLE_LOAD_DURATION=%s, want a positive duration", duration)
53 }
54
55 f := newFixture(t)
56 seedLoadRepo(t, f)
57 // One read per repository builds its index before the run, so the first
58 // reader does not pay for the whole metadata branch inside the measurement.
59 f.get("/r/" + f.repo)
60
61 switch scenario {
62 case "clone":
63 runCloneScenario(t, f, duration, concurrency)
64 default:
65 t.Fatalf("unknown VOLE_LOAD_SCENARIO %q (have: clone)", scenario)
66 }
67 }
68
69 // seedLoadRepo gives the instance a repository with the shape a real one has:
70 // history, files at a few sizes, an issue and a change request with comments and
71 // an approval, and a release artifact.
72 func seedLoadRepo(t *testing.T, f *fixture) {
73 t.Helper()
74 f.publish(f.repo)
75
76 files := map[string]string{
77 "README.md": "# load test\n\nA repository with enough shape to measure.\n",
78 "small.txt": strings.Repeat("small line\n", 100),
79 "medium.js": strings.Repeat("export const value = 42;\n", 5000),
80 "large.bin": strings.Repeat("x", 1<<20),
81 }
82 work := workRepo(t, files)
83 for i := 0; i < 300; i++ {
84 appendLine(t, filepath.Join(work, "service.go"), fmt.Sprintf("// revision %d\n", i))
85 runGit(t, work, "add", "-A")
86 runGit(t, work, "commit", "-qm", fmt.Sprintf("revision %d", i))
87 }
88 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/main"); err != nil {
89 t.Fatalf("push main: %s", errs)
90 }
91 if _, errs, err := f.git(work, "push", f.sshURL(f.repo), "main:refs/heads/feature"); err != nil {
92 t.Fatalf("push feature: %s", errs)
93 }
94
95 m, err := f.srv.Repos.Open(f.repo)
96 if err != nil {
97 t.Fatal(err)
98 }
99 g := m.Git()
100 now := time.Now().UTC()
101
102 iss, err := metadata.NewIssue("Load test issue",
103 strings.Repeat("A body with enough text to index. ", 40), "keenan",
104 []string{"p2", "testing"}, []string{"keenan"}, now)
105 if err != nil {
106 t.Fatal(err)
107 }
108 if err := metadata.CreateIssue(g, iss, "keenan"); err != nil {
109 t.Fatal(err)
110 }
111 for i := 0; i < 5; i++ {
112 comment, err := metadata.NewIssueComment(fmt.Sprintf("Comment %d on the issue.\n", i), "keenan", now.Add(time.Duration(i)*time.Minute))
113 if err != nil {
114 t.Fatal(err)
115 }
116 if err := metadata.AddIssueComment(g, iss.ID, comment, "keenan"); err != nil {
117 t.Fatal(err)
118 }
119 }
120
121 head, err := g.RevParse("refs/heads/feature")
122 if err != nil || head == "" {
123 t.Fatalf("resolve feature: %q, %v", head, err)
124 }
125 req, err := metadata.NewRequest("Load test change",
126 strings.Repeat("Why this change. ", 40), "keenan", "main", head, now)
127 if err != nil {
128 t.Fatal(err)
129 }
130 req.Closes = []string{iss.ID}
131 if err := metadata.CreateRequest(g, req, "keenan"); err != nil {
132 t.Fatal(err)
133 }
134 for i := 0; i < 5; i++ {
135 comment, err := metadata.NewRequestComment(fmt.Sprintf("Review note %d.\n", i), "keenan", now.Add(time.Duration(i)*time.Minute))
136 if err != nil {
137 t.Fatal(err)
138 }
139 if err := metadata.AddRequestComment(g, req.ID, comment, "keenan"); err != nil {
140 t.Fatal(err)
141 }
142 }
143 // A second account approves: an author cannot approve their own change, and
144 // the seeded record should look like one that can actually land.
145 f.addWriter(t)
146 approver := f.writerID
147 if _, err := metadata.ApproveRequest(g, req.ID, approver, "main", head, now, approver); err != nil {
148 t.Fatal(err)
149 }
150
151 // A release with one artifact in the LFS store, so the routes that download
152 // one have something real to serve.
153 artifact := []byte(strings.Repeat("artifact bytes\n", 100))
154 oid, err := f.srv.LFS.PutBytes(f.repo, artifact)
155 if err != nil {
156 t.Fatal(err)
157 }
158 if err := g.WriteAnnotatedTag("v1", head, "Load test release\n", "keenan", false); err != nil {
159 t.Fatal(err)
160 }
161 if _, err := git.CommitFiles(g, metadata.Ref, func(files map[string][]byte) error {
162 files[metadata.ReleasePath("v1", "load-artifact.txt")] =
163 lfs.Pointer{OID: oid, Size: int64(len(artifact))}.Encode()
164 return nil
165 }, "release: v1", "keenan"); err != nil {
166 t.Fatal(err)
167 }
168 }
169
170 // runCloneScenario drives anonymous shallow clones over the HTTP transport. Each
171 // iteration probes `info/refs` first, so a refusal is reported as the status it
172 // was refused with, then clones the repository.
173 func runCloneScenario(t *testing.T, f *fixture, duration time.Duration, concurrency int) {
174 t.Helper()
175 base := f.httpURL(f.repo)
176 infoRefs := base + "/info/refs?service=git-upload-pack"
177 client := &http.Client{Timeout: 60 * time.Second}
178 root := t.TempDir()
179
180 results := newLoadResults()
181 peaks := startSampler()
182 deadline := time.Now().Add(duration)
183
184 var wg sync.WaitGroup
185 for i := 0; i < concurrency; i++ {
186 wg.Add(1)
187 go func(worker int) {
188 defer wg.Done()
189 dir := filepath.Join(root, strconv.Itoa(worker))
190 for time.Now().Before(deadline) {
191 start := time.Now()
192 status, err := probe(client, infoRefs)
193 if err == nil {
194 err = cloneShallow(base, dir)
195 }
196 results.record(time.Since(start), status, err)
197 }
198 }(i)
199 }
200 wg.Wait()
201 sampled := peaks.stop()
202 reportLoad(t, "clone", duration, concurrency, results, sampled)
203 }
204
205 // probe records the status of one HTTP request. It is the part of a clone that a
206 // server can refuse, so a load run reports it separately from the git process's
207 // exit status.
208 func probe(client *http.Client, url string) (string, error) {
209 resp, err := client.Get(url)
210 if err != nil {
211 return "error", err
212 }
213 _, _ = io.Copy(io.Discard, resp.Body)
214 resp.Body.Close()
215 return strconv.Itoa(resp.StatusCode), nil
216 }
217
218 func cloneShallow(url, dir string) error {
219 if err := os.RemoveAll(dir); err != nil {
220 return err
221 }
222 cmd := exec.Command("git", "clone", "--quiet", "--depth", "1", url, dir)
223 cmd.Env = append(os.Environ(),
224 "GIT_TERMINAL_PROMPT=0", "GIT_CONFIG_GLOBAL=/dev/null", "GIT_CONFIG_SYSTEM=/dev/null")
225 out, err := cmd.CombinedOutput()
226 if err != nil {
227 return fmt.Errorf("clone: %v: %s", err, firstLine(string(out)))
228 }
229 return nil
230 }
231
232 func firstLine(s string) string {
233 if i := strings.IndexByte(s, '\n'); i >= 0 {
234 s = s[:i]
235 }
236 if len(s) > 120 {
237 s = s[:120]
238 }
239 return s
240 }
241
242 // loadResults collects one latency sample per request and classifies the
243 // outcomes by HTTP status and by failure reason.
244 type loadResults struct {
245 mu sync.Mutex
246 requests int
247 latency []time.Duration
248 statuses map[string]int
249 errors map[string]int
250 }
251
252 func newLoadResults() *loadResults {
253 return &loadResults{statuses: map[string]int{}, errors: map[string]int{}}
254 }
255
256 func (r *loadResults) record(latency time.Duration, status string, err error) {
257 r.mu.Lock()
258 defer r.mu.Unlock()
259 r.requests++
260 r.latency = append(r.latency, latency)
261 r.statuses[status]++
262 if err != nil {
263 r.errors[firstLine(err.Error())]++
264 }
265 }
266
267 func (r *loadResults) summary() (requests int, statuses, errors map[string]int, p50, p90, p99 time.Duration) {
268 r.mu.Lock()
269 defer r.mu.Unlock()
270 requests = r.requests
271 statuses = map[string]int{}
272 for k, v := range r.statuses {
273 statuses[k] = v
274 }
275 errors = map[string]int{}
276 for k, v := range r.errors {
277 errors[k] = v
278 }
279 sorted := append([]time.Duration(nil), r.latency...)
280 sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
281 p50 = percentile(sorted, 0.50)
282 p90 = percentile(sorted, 0.90)
283 p99 = percentile(sorted, 0.99)
284 return requests, statuses, errors, p50, p90, p99
285 }
286
287 func percentile(sorted []time.Duration, p float64) time.Duration {
288 if len(sorted) == 0 {
289 return 0
290 }
291 i := int(p * float64(len(sorted)-1))
292 if i >= len(sorted) {
293 i = len(sorted) - 1
294 }
295 return sorted[i]
296 }
297
298 // processPeak is the largest sample of what the process held during the run.
299 type processPeak struct {
300 RSSBytes int64
301 FDs int
302 Goroutines int
303 GitChildren int
304 }
305
306 type sampler struct {
307 stopCh chan struct{}
308 doneCh chan struct{}
309 mu sync.Mutex
310 peak processPeak
311 }
312
313 func startSampler() *sampler {
314 s := &sampler{stopCh: make(chan struct{}), doneCh: make(chan struct{})}
315 go func() {
316 defer close(s.doneCh)
317 ticker := time.NewTicker(200 * time.Millisecond)
318 defer ticker.Stop()
319 for {
320 s.sample()
321 select {
322 case <-s.stopCh:
323 return
324 case <-ticker.C:
325 }
326 }
327 }()
328 return s
329 }
330
331 func (s *sampler) sample() {
332 cur := processPeak{Goroutines: runtime.NumGoroutine()}
333 cur.RSSBytes = procRSS()
334 cur.FDs = countDir("/proc/self/fd")
335 cur.GitChildren = gitChildren()
336 s.mu.Lock()
337 defer s.mu.Unlock()
338 if cur.RSSBytes > s.peak.RSSBytes {
339 s.peak.RSSBytes = cur.RSSBytes
340 }
341 if cur.FDs > s.peak.FDs {
342 s.peak.FDs = cur.FDs
343 }
344 if cur.Goroutines > s.peak.Goroutines {
345 s.peak.Goroutines = cur.Goroutines
346 }
347 if cur.GitChildren > s.peak.GitChildren {
348 s.peak.GitChildren = cur.GitChildren
349 }
350 }
351
352 func (s *sampler) stop() processPeak {
353 close(s.stopCh)
354 <-s.doneCh
355 s.mu.Lock()
356 defer s.mu.Unlock()
357 return s.peak
358 }
359
360 // procRSS reads VmRSS from /proc/self/status. A kernel without procfs (or an
361 // operating system without /proc) reports zero rather than failing the run.
362 func procRSS() int64 {
363 data, err := os.ReadFile("/proc/self/status")
364 if err != nil {
365 return 0
366 }
367 for _, line := range strings.Split(string(data), "\n") {
368 if !strings.HasPrefix(line, "VmRSS:") {
369 continue
370 }
371 fields := strings.Fields(line)
372 if len(fields) < 2 {
373 return 0
374 }
375 kb, err := strconv.ParseInt(fields[1], 10, 64)
376 if err != nil {
377 return 0
378 }
379 return kb * 1024
380 }
381 return 0
382 }
383
384 func countDir(dir string) int {
385 entries, err := os.ReadDir(dir)
386 if err != nil {
387 return 0
388 }
389 return len(entries)
390 }
391
392 // gitChildren counts the git processes the server has running: http-backend and
393 // the pack services it starts. The clone processes the harness itself starts are
394 // children of this process too, so they are excluded by name rather than counted
395 // as server load.
396 func gitChildren() int {
397 self := os.Getpid()
398 entries, err := os.ReadDir("/proc")
399 if err != nil {
400 return 0
401 }
402 n := 0
403 for _, e := range entries {
404 pid, err := strconv.Atoi(e.Name())
405 if err != nil || pid == self {
406 continue
407 }
408 stat, err := os.ReadFile(filepath.Join("/proc", e.Name(), "stat"))
409 if err != nil {
410 continue
411 }
412 // pid (comm) state ppid ...
413 s := string(stat)
414 close := strings.LastIndexByte(s, ')')
415 if close < 0 {
416 continue
417 }
418 fields := strings.Fields(s[close+1:])
419 if len(fields) < 2 {
420 continue
421 }
422 ppid, _ := strconv.Atoi(fields[1])
423 if ppid != self {
424 continue
425 }
426 cmdline, err := os.ReadFile(filepath.Join("/proc", e.Name(), "cmdline"))
427 if err != nil {
428 continue
429 }
430 cmd := strings.ReplaceAll(string(cmdline), "\x00", " ")
431 if strings.Contains(cmd, "http-backend") || strings.Contains(cmd, "upload-pack") ||
432 strings.Contains(cmd, "receive-pack") {
433 n++
434 }
435 }
436 return n
437 }
438
439 func reportLoad(t *testing.T, scenario string, duration time.Duration, concurrency int, results *loadResults, peak processPeak) {
440 t.Helper()
441 requests, statuses, errors, p50, p90, p99 := results.summary()
442 rate := float64(requests) / duration.Seconds()
443 t.Logf("%s: concurrency=%d duration=%s", scenario, concurrency, duration)
444 t.Logf(" requests %d (%.1f/s)", requests, rate)
445 for _, status := range sortedKeys(statuses) {
446 t.Logf(" status %-5s %d", status, statuses[status])
447 }
448 for _, reason := range sortedKeys(errors) {
449 t.Logf(" failure %d %s", errors[reason], reason)
450 }
451 t.Logf(" latency p50=%s p90=%s p99=%s", roundDur(p50), roundDur(p90), roundDur(p99))
452 t.Logf(" peak rss=%s fds=%d goroutines=%d git-children=%d",
453 roundBytes(peak.RSSBytes), peak.FDs, peak.Goroutines, peak.GitChildren)
454 }
455
456 func sortedKeys(m map[string]int) []string {
457 out := make([]string, 0, len(m))
458 for k := range m {
459 out = append(out, k)
460 }
461 sort.Strings(out)
462 return out
463 }
464
465 func roundDur(d time.Duration) time.Duration { return d.Round(time.Millisecond) }
466
467 func roundBytes(n int64) string {
468 const unit = 1024
469 if n < unit {
470 return fmt.Sprintf("%d B", n)
471 }
472 k := float64(n) / unit
473 for _, suffix := range []string{"KiB", "MiB", "GiB"} {
474 if k < unit {
475 return fmt.Sprintf("%.1f %s", k, suffix)
476 }
477 k /= unit
478 }
479 return fmt.Sprintf("%.1f TiB", k)
480 }
481
482 func appendLine(t *testing.T, path, line string) {
483 t.Helper()
484 f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
485 if err != nil {
486 t.Fatal(err)
487 }
488 if _, err := f.WriteString(line); err != nil {
489 f.Close()
490 t.Fatal(err)
491 }
492 if err := f.Close(); err != nil {
493 t.Fatal(err)
494 }
495 }
496
497 func envString(name, def string) string {
498 if v := strings.TrimSpace(os.Getenv(name)); v != "" {
499 return v
500 }
501 return def
502 }
503
504 func envInt(name string, def int) int {
505 v := strings.TrimSpace(os.Getenv(name))
506 if v == "" {
507 return def
508 }
509 n, err := strconv.Atoi(v)
510 if err != nil {
511 return def
512 }
513 return n
514 }
515
516 func envDuration(name string, def time.Duration) time.Duration {
517 v := strings.TrimSpace(os.Getenv(name))
518 if v == "" {
519 return def
520 }
521 d, err := time.ParseDuration(v)
522 if err != nil {
523 return def
524 }
525 return d
526 }