vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 0: measure metadata writes, git transport, and tree serving
c5626f1b87084870524ea54987ef47a65ea4740b · Keenan Nemetz · 18d ago · parent 17311880
Phase 0: measure metadata writes, git transport, and tree serving Three throwaway spikes under docs/spikes, each with its own harness, README, and reproduce command. Findings are folded into the roadmap and its decision log. - ref-cas: sharded + batched refs sustain 25,637 entity writes/s at 20k entities with zero lost writes; a single unsharded ref manages 30/s with 22 CAS retries per write and decays quadratically as it fills. Metadata refs must be sharded. Unsharded is now off the table. - smart-http: driving git http-backend as a subprocess is correct (hook stderr reaches the client, aborts leave no ref and no orphaned children) and pushes 150 MiB/s on compressible data, but a 1 GiB push peaks at 1.13 GB of git-child RSS. Phase 1 needs a push size cap and a concurrency semaphore, plus chunked-upload spooling. - tree-serving: sites can be served straight from git trees with no export step - 1.2 ms p50 cold, 0.075 ms warm, 170k req/s behind an 18.5 MiB cache. Harness bugs caught during the work are recorded in the spike READMEs, including one that silently dropped 83% of writes while reporting plausible throughput.
docs/spikes/tree-serving/main.go +493
| @@ -0,0 +1,493 @@ | |||
| 1 | // Spike C: can we serve a repo's static site straight out of git trees, and | ||
| 2 | // how does it compare to serving the same content from a filesystem? | ||
| 3 | // | ||
| 4 | // Builds a 10k-file tree, then serves it four ways: | ||
| 5 | // | ||
| 6 | // tree-nocache-1proc one long-lived `git cat-file --batch`, no cache | ||
| 7 | // tree-nocache-4proc four cat-file processes over the same object store | ||
| 8 | // tree-warmcache-4proc four processes with an in-memory blob cache | ||
| 9 | // filesystem-baseline the same content exported to disk and served | ||
| 10 | package main | ||
| 11 | |||
| 12 | import ( | ||
| 13 | "bufio" | ||
| 14 | "bytes" | ||
| 15 | "encoding/json" | ||
| 16 | "flag" | ||
| 17 | "fmt" | ||
| 18 | "io" | ||
| 19 | "net" | ||
| 20 | "net/http" | ||
| 21 | "os" | ||
| 22 | "os/exec" | ||
| 23 | "path/filepath" | ||
| 24 | "sort" | ||
| 25 | "strconv" | ||
| 26 | "strings" | ||
| 27 | "sync" | ||
| 28 | "time" | ||
| 29 | ) | ||
| 30 | |||
| 31 | // ---------- git plumbing ---------- | ||
| 32 | |||
| 33 | type catFile struct { | ||
| 34 | cmd *exec.Cmd | ||
| 35 | in *bufio.Writer | ||
| 36 | out *bufio.Reader | ||
| 37 | mu sync.Mutex | ||
| 38 | pipe io.WriteCloser | ||
| 39 | calls int64 | ||
| 40 | } | ||
| 41 | |||
| 42 | func newCatFile(dir string) (*catFile, error) { | ||
| 43 | cmd := exec.Command("git", "cat-file", "--batch") | ||
| 44 | cmd.Env = append(os.Environ(), "GIT_DIR="+dir) | ||
| 45 | in, err := cmd.StdinPipe() | ||
| 46 | if err != nil { | ||
| 47 | return nil, err | ||
| 48 | } | ||
| 49 | out, err := cmd.StdoutPipe() | ||
| 50 | if err != nil { | ||
| 51 | return nil, err | ||
| 52 | } | ||
| 53 | var errb bytes.Buffer | ||
| 54 | cmd.Stderr = &errb | ||
| 55 | if err := cmd.Start(); err != nil { | ||
| 56 | return nil, err | ||
| 57 | } | ||
| 58 | return &catFile{cmd: cmd, in: bufio.NewWriterSize(in, 64<<10), out: bufio.NewReaderSize(out, 256<<10), pipe: in}, nil | ||
| 59 | } | ||
| 60 | |||
| 61 | type object struct { | ||
| 62 | oid string | ||
| 63 | typ string | ||
| 64 | data []byte | ||
| 65 | } | ||
| 66 | |||
| 67 | func (c *catFile) get(spec string) (*object, error) { | ||
| 68 | c.mu.Lock() | ||
| 69 | defer c.mu.Unlock() | ||
| 70 | if _, err := c.in.WriteString(spec + "\n"); err != nil { | ||
| 71 | return nil, err | ||
| 72 | } | ||
| 73 | if err := c.in.Flush(); err != nil { | ||
| 74 | return nil, err | ||
| 75 | } | ||
| 76 | header, err := c.out.ReadString('\n') | ||
| 77 | if err != nil { | ||
| 78 | return nil, err | ||
| 79 | } | ||
| 80 | f := strings.Fields(strings.TrimRight(header, "\n")) | ||
| 81 | if len(f) == 2 && f[1] == "missing" { | ||
| 82 | return nil, os.ErrNotExist | ||
| 83 | } | ||
| 84 | if len(f) != 3 { | ||
| 85 | return nil, fmt.Errorf("bad cat-file header %q", header) | ||
| 86 | } | ||
| 87 | size, err := strconv.Atoi(f[2]) | ||
| 88 | if err != nil { | ||
| 89 | return nil, err | ||
| 90 | } | ||
| 91 | buf := make([]byte, size+1) // trailing newline | ||
| 92 | if _, err := io.ReadFull(c.out, buf); err != nil { | ||
| 93 | return nil, err | ||
| 94 | } | ||
| 95 | c.calls++ | ||
| 96 | return &object{oid: f[0], typ: f[1], data: buf[:size]}, nil | ||
| 97 | } | ||
| 98 | |||
| 99 | // pool spreads reads over several cat-file processes. | ||
| 100 | type pool struct { | ||
| 101 | cs []*catFile | ||
| 102 | i int | ||
| 103 | mu sync.Mutex | ||
| 104 | } | ||
| 105 | |||
| 106 | func newPool(dir string, n int) (*pool, error) { | ||
| 107 | p := &pool{} | ||
| 108 | for i := 0; i < n; i++ { | ||
| 109 | c, err := newCatFile(dir) | ||
| 110 | if err != nil { | ||
| 111 | return nil, err | ||
| 112 | } | ||
| 113 | p.cs = append(p.cs, c) | ||
| 114 | } | ||
| 115 | return p, nil | ||
| 116 | } | ||
| 117 | |||
| 118 | func (p *pool) get(spec string) (*object, error) { | ||
| 119 | p.mu.Lock() | ||
| 120 | c := p.cs[p.i%len(p.cs)] | ||
| 121 | p.i++ | ||
| 122 | p.mu.Unlock() | ||
| 123 | return c.get(spec) | ||
| 124 | } | ||
| 125 | |||
| 126 | // ---------- cache ---------- | ||
| 127 | |||
| 128 | type cache struct { | ||
| 129 | mu sync.RWMutex | ||
| 130 | m map[string]*object | ||
| 131 | n int64 | ||
| 132 | } | ||
| 133 | |||
| 134 | func newCache() *cache { return &cache{m: map[string]*object{}} } | ||
| 135 | |||
| 136 | func (c *cache) get(k string) (*object, bool) { | ||
| 137 | c.mu.RLock() | ||
| 138 | o, ok := c.m[k] | ||
| 139 | c.mu.RUnlock() | ||
| 140 | return o, ok | ||
| 141 | } | ||
| 142 | |||
| 143 | func (c *cache) put(k string, o *object) { | ||
| 144 | c.mu.Lock() | ||
| 145 | if _, exists := c.m[k]; !exists { | ||
| 146 | c.m[k] = o | ||
| 147 | c.n += int64(len(o.data)) | ||
| 148 | } | ||
| 149 | c.mu.Unlock() | ||
| 150 | } | ||
| 151 | |||
| 152 | // ---------- server ---------- | ||
| 153 | |||
| 154 | type siteServer struct { | ||
| 155 | p *pool | ||
| 156 | commit string | ||
| 157 | cache *cache | ||
| 158 | useCache bool | ||
| 159 | } | ||
| 160 | |||
| 161 | func contentType(path string) string { | ||
| 162 | switch strings.ToLower(filepath.Ext(path)) { | ||
| 163 | case ".html", ".htm": | ||
| 164 | return "text/html; charset=utf-8" | ||
| 165 | case ".css": | ||
| 166 | return "text/css; charset=utf-8" | ||
| 167 | case ".js": | ||
| 168 | return "application/javascript" | ||
| 169 | case ".json": | ||
| 170 | return "application/json" | ||
| 171 | case ".svg": | ||
| 172 | return "image/svg+xml" | ||
| 173 | case ".png": | ||
| 174 | return "image/png" | ||
| 175 | case ".md": | ||
| 176 | return "text/markdown; charset=utf-8" | ||
| 177 | default: | ||
| 178 | return "application/octet-stream" | ||
| 179 | } | ||
| 180 | } | ||
| 181 | |||
| 182 | func (s *siteServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { | ||
| 183 | p := strings.TrimPrefix(r.URL.Path, "/") | ||
| 184 | p = strings.TrimPrefix(p, "files/") | ||
| 185 | if p == "" { | ||
| 186 | http.NotFound(w, r) | ||
| 187 | return | ||
| 188 | } | ||
| 189 | spec := s.commit + ":" + p | ||
| 190 | var o *object | ||
| 191 | if s.useCache { | ||
| 192 | if hit, ok := s.cache.get(spec); ok { | ||
| 193 | o = hit | ||
| 194 | } | ||
| 195 | } | ||
| 196 | if o == nil { | ||
| 197 | var err error | ||
| 198 | o, err = s.p.get(spec) | ||
| 199 | if err != nil { | ||
| 200 | http.NotFound(w, r) | ||
| 201 | return | ||
| 202 | } | ||
| 203 | if o.typ != "blob" { | ||
| 204 | http.NotFound(w, r) | ||
| 205 | return | ||
| 206 | } | ||
| 207 | if s.useCache { | ||
| 208 | s.cache.put(spec, o) | ||
| 209 | } | ||
| 210 | } | ||
| 211 | h := w.Header() | ||
| 212 | h.Set("ETag", `"`+o.oid+`"`) | ||
| 213 | h.Set("Content-Type", contentType(p)) | ||
| 214 | h.Set("Cache-Control", "public, max-age=60") | ||
| 215 | if match := r.Header.Get("If-None-Match"); match != "" && strings.Contains(match, o.oid) { | ||
| 216 | w.WriteHeader(http.StatusNotModified) | ||
| 217 | return | ||
| 218 | } | ||
| 219 | h.Set("Content-Length", strconv.Itoa(len(o.data))) | ||
| 220 | w.Write(o.data) | ||
| 221 | } | ||
| 222 | |||
| 223 | // ---------- fixture ---------- | ||
| 224 | |||
| 225 | // fileContent keeps the fixture and the byte accounting in sync. | ||
| 226 | func fileContent(i int) string { | ||
| 227 | body := strings.Repeat("<p>hello from a static site</p>\n", 60) // ~1.7 KiB | ||
| 228 | return fmt.Sprintf("<!-- %d -->\n%s%d", i, body, i%7) | ||
| 229 | } | ||
| 230 | |||
| 231 | // buildTree creates `files` blobs named like dir-XX/file-YYY.html in one commit. | ||
| 232 | func buildTree(dir string, files int) (string, error) { | ||
| 233 | os.RemoveAll(dir) | ||
| 234 | if _, err := runCmd("", nil, "git", "init", "--bare", "-q", dir); err != nil { | ||
| 235 | return "", err | ||
| 236 | } | ||
| 237 | var buf bytes.Buffer | ||
| 238 | for i := 0; i < files; i++ { | ||
| 239 | content := fileContent(i) | ||
| 240 | fmt.Fprintf(&buf, "blob\nmark :%d\ndata %d\n%s\n", i+1, len(content), content) | ||
| 241 | } | ||
| 242 | fmt.Fprintf(&buf, "commit refs/heads/main\ncommitter spike <spike@example.com> 1700000000 +0000\ndata 5\ninit\n") | ||
| 243 | for i := 0; i < files; i++ { | ||
| 244 | name := fmt.Sprintf("dir-%02d/file-%03d.html", i%100, i) | ||
| 245 | fmt.Fprintf(&buf, "M 100644 :%d %s\n", i+1, name) | ||
| 246 | } | ||
| 247 | cmd := exec.Command("git", "fast-import", "--quiet") | ||
| 248 | cmd.Env = append(os.Environ(), "GIT_DIR="+dir) | ||
| 249 | cmd.Stdin = &buf | ||
| 250 | var errb bytes.Buffer | ||
| 251 | cmd.Stderr = &errb | ||
| 252 | if err := cmd.Run(); err != nil { | ||
| 253 | return "", fmt.Errorf("fast-import: %w: %s", err, errb.String()) | ||
| 254 | } | ||
| 255 | out, err := runCmd(dir, nil, "git", "rev-parse", "refs/heads/main") | ||
| 256 | if err != nil { | ||
| 257 | return "", err | ||
| 258 | } | ||
| 259 | return strings.TrimSpace(string(out)), nil | ||
| 260 | } | ||
| 261 | |||
| 262 | func runCmd(dir string, env []string, args ...string) (string, error) { | ||
| 263 | cmd := exec.Command(args[0], args[1:]...) | ||
| 264 | cmd.Dir = dir | ||
| 265 | cmd.Env = append(os.Environ(), env...) | ||
| 266 | var out, errb bytes.Buffer | ||
| 267 | cmd.Stdout = &out | ||
| 268 | cmd.Stderr = &errb | ||
| 269 | if err := cmd.Run(); err != nil { | ||
| 270 | return out.String(), fmt.Errorf("%w: %s", err, strings.TrimSpace(errb.String())) | ||
| 271 | } | ||
| 272 | return out.String(), nil | ||
| 273 | } | ||
| 274 | |||
| 275 | // ---------- load ---------- | ||
| 276 | |||
| 277 | type sample struct { | ||
| 278 | name string | ||
| 279 | ok int64 | ||
| 280 | errs int64 | ||
| 281 | latencies []int64 | ||
| 282 | elapsed time.Duration | ||
| 283 | } | ||
| 284 | |||
| 285 | func (s *sample) report() map[string]any { | ||
| 286 | sort.Slice(s.latencies, func(i, j int) bool { return s.latencies[i] < s.latencies[j] }) | ||
| 287 | p := func(q float64) float64 { | ||
| 288 | if len(s.latencies) == 0 { | ||
| 289 | return 0 | ||
| 290 | } | ||
| 291 | i := int(q * float64(len(s.latencies)-1)) | ||
| 292 | return float64(s.latencies[i]) / 1e6 | ||
| 293 | } | ||
| 294 | return map[string]any{ | ||
| 295 | "server": s.name, | ||
| 296 | "requests_ok": s.ok, | ||
| 297 | "errors": s.errs, | ||
| 298 | "elapsed_s": s.elapsed.Seconds(), | ||
| 299 | "rps": float64(s.ok) / s.elapsed.Seconds(), | ||
| 300 | "p50_ms": p(0.50), | ||
| 301 | "p95_ms": p(0.95), | ||
| 302 | "p99_ms": p(0.99), | ||
| 303 | "max_ms": p(1.0), | ||
| 304 | } | ||
| 305 | } | ||
| 306 | |||
| 307 | func drive(handler http.Handler, paths []string, conc, total int) *sample { | ||
| 308 | addr, _ := freePort() | ||
| 309 | ln, err := net.Listen("tcp", addr) | ||
| 310 | if err != nil { | ||
| 311 | panic(err) | ||
| 312 | } | ||
| 313 | srv := &http.Server{Handler: handler, ReadHeaderTimeout: 5 * time.Second} | ||
| 314 | go srv.Serve(ln) | ||
| 315 | defer srv.Close() | ||
| 316 | |||
| 317 | s := &sample{} | ||
| 318 | jobs := make(chan string, 1024) | ||
| 319 | var wg sync.WaitGroup | ||
| 320 | var mu sync.Mutex | ||
| 321 | s.name = "" | ||
| 322 | start := time.Now() | ||
| 323 | tr := &http.Transport{MaxIdleConnsPerHost: conc, DisableKeepAlives: false} | ||
| 324 | client := &http.Client{Transport: tr, Timeout: 10 * time.Second} | ||
| 325 | for w := 0; w < conc; w++ { | ||
| 326 | wg.Add(1) | ||
| 327 | go func() { | ||
| 328 | defer wg.Done() | ||
| 329 | for p := range jobs { | ||
| 330 | t0 := time.Now() | ||
| 331 | resp, err := client.Get("http://" + addr + "/files/" + p) | ||
| 332 | if err != nil { | ||
| 333 | mu.Lock() | ||
| 334 | s.errs++ | ||
| 335 | mu.Unlock() | ||
| 336 | continue | ||
| 337 | } | ||
| 338 | io.Copy(io.Discard, resp.Body) | ||
| 339 | resp.Body.Close() | ||
| 340 | d := time.Since(t0).Nanoseconds() | ||
| 341 | mu.Lock() | ||
| 342 | s.latencies = append(s.latencies, d) | ||
| 343 | if resp.StatusCode == 200 { | ||
| 344 | s.ok++ | ||
| 345 | } else { | ||
| 346 | s.errs++ | ||
| 347 | } | ||
| 348 | mu.Unlock() | ||
| 349 | } | ||
| 350 | }() | ||
| 351 | } | ||
| 352 | // Deliberately random order so we exercise the 10k-file tree, not one dir. | ||
| 353 | step := 7919 // coprime with 10000 | ||
| 354 | for i := 0; i < total; i++ { | ||
| 355 | idx := (i * step) % len(paths) | ||
| 356 | jobs <- paths[idx] | ||
| 357 | } | ||
| 358 | close(jobs) | ||
| 359 | wg.Wait() | ||
| 360 | s.elapsed = time.Since(start) | ||
| 361 | return s | ||
| 362 | } | ||
| 363 | |||
| 364 | func errString(err error) string { | ||
| 365 | if err == nil { | ||
| 366 | return "" | ||
| 367 | } | ||
| 368 | return err.Error() | ||
| 369 | } | ||
| 370 | |||
| 371 | func freePort() (string, error) { | ||
| 372 | l, err := net.Listen("tcp", "127.0.0.1:0") | ||
| 373 | if err != nil { | ||
| 374 | return "", err | ||
| 375 | } | ||
| 376 | defer l.Close() | ||
| 377 | return l.Addr().String(), nil | ||
| 378 | } | ||
| 379 | |||
| 380 | func main() { | ||
| 381 | files := flag.Int("files", 10000, "files in the tree") | ||
| 382 | conc := flag.Int("conc", 16, "concurrent clients") | ||
| 383 | reqs := flag.Int("reqs", 4000, "total requests per configuration") | ||
| 384 | flag.Parse() | ||
| 385 | |||
| 386 | root, err := os.MkdirTemp("", "vole-spike-c-") | ||
| 387 | if err != nil { | ||
| 388 | panic(err) | ||
| 389 | } | ||
| 390 | defer os.RemoveAll(root) | ||
| 391 | |||
| 392 | repo := filepath.Join(root, "site.git") | ||
| 393 | start := time.Now() | ||
| 394 | commit, err := buildTree(repo, *files) | ||
| 395 | if err != nil { | ||
| 396 | panic(err) | ||
| 397 | } | ||
| 398 | built := time.Since(start) | ||
| 399 | |||
| 400 | paths := make([]string, *files) | ||
| 401 | var totalBytes int64 | ||
| 402 | for i := 0; i < *files; i++ { | ||
| 403 | paths[i] = fmt.Sprintf("dir-%02d/file-%03d.html", i%100, i) | ||
| 404 | totalBytes += int64(len(fileContent(i))) | ||
| 405 | } | ||
| 406 | |||
| 407 | results := []map[string]any{{ | ||
| 408 | "scenario": "fixture", | ||
| 409 | "files": *files, | ||
| 410 | "total_mib": float64(totalBytes) / (1 << 20), | ||
| 411 | "build_ms": built.Milliseconds(), | ||
| 412 | "concurrency": *conc, | ||
| 413 | "requests_each": *reqs, | ||
| 414 | }} | ||
| 415 | |||
| 416 | // Correctness check: the bytes served must be the bytes committed. | ||
| 417 | { | ||
| 418 | pv, err := newPool(repo, 1) | ||
| 419 | if err != nil { | ||
| 420 | panic(err) | ||
| 421 | } | ||
| 422 | o, err := pv.get(commit + ":dir-00/file-000.html") | ||
| 423 | ok := err == nil && o != nil && string(o.data) == fileContent(0) | ||
| 424 | n := 0 | ||
| 425 | if o != nil { | ||
| 426 | n = len(o.data) | ||
| 427 | } | ||
| 428 | results = append(results, map[string]any{ | ||
| 429 | "scenario": "verify-tree-read", | ||
| 430 | "bytes_match": ok, | ||
| 431 | "bytes": n, | ||
| 432 | "err": errString(err), | ||
| 433 | }) | ||
| 434 | } | ||
| 435 | |||
| 436 | // 1: one cat-file process, no cache | ||
| 437 | { | ||
| 438 | p1, err := newPool(repo, 1) | ||
| 439 | if err != nil { | ||
| 440 | panic(err) | ||
| 441 | } | ||
| 442 | s := drive(&siteServer{p: p1, commit: commit}, paths, *conc, *reqs) | ||
| 443 | s.name = "tree-nocache-1proc" | ||
| 444 | results = append(results, s.report()) | ||
| 445 | } | ||
| 446 | |||
| 447 | // 2: four cat-file processes, no cache | ||
| 448 | { | ||
| 449 | p4, err := newPool(repo, 4) | ||
| 450 | if err != nil { | ||
| 451 | panic(err) | ||
| 452 | } | ||
| 453 | s := drive(&siteServer{p: p4, commit: commit}, paths, *conc, *reqs) | ||
| 454 | s.name = "tree-nocache-4proc" | ||
| 455 | results = append(results, s.report()) | ||
| 456 | } | ||
| 457 | |||
| 458 | // 3: warm cache (first pass populates, second pass measured) | ||
| 459 | { | ||
| 460 | p4, err := newPool(repo, 4) | ||
| 461 | if err != nil { | ||
| 462 | panic(err) | ||
| 463 | } | ||
| 464 | c := newCache() | ||
| 465 | srv := &siteServer{p: p4, commit: commit, cache: c, useCache: true} | ||
| 466 | _ = drive(srv, paths, 4, *files) // populate | ||
| 467 | s := drive(srv, paths, *conc, *reqs) | ||
| 468 | s.name = "tree-warmcache-4proc" | ||
| 469 | rep := s.report() | ||
| 470 | rep["cache_bytes"] = c.n | ||
| 471 | rep["cache_entries"] = len(c.m) | ||
| 472 | rep["cache_mib"] = float64(c.n) / (1 << 20) | ||
| 473 | results = append(results, rep) | ||
| 474 | } | ||
| 475 | |||
| 476 | // 4: filesystem baseline | ||
| 477 | { | ||
| 478 | exported := filepath.Join(root, "export") | ||
| 479 | if _, err := runCmd("", nil, "sh", "-c", fmt.Sprintf( | ||
| 480 | "mkdir -p %s && git --git-dir=%s archive %s | tar -x -C %s", | ||
| 481 | exported, repo, commit, exported)); err != nil { | ||
| 482 | panic(err) | ||
| 483 | } | ||
| 484 | fs := http.StripPrefix("/files/", http.FileServer(http.Dir(exported))) | ||
| 485 | s := drive(fs, paths, *conc, *reqs) | ||
| 486 | s.name = "filesystem-baseline" | ||
| 487 | results = append(results, s.report()) | ||
| 488 | } | ||
| 489 | |||
| 490 | out, _ := json.MarshalIndent(results, "", " ") | ||
| 491 | fmt.Println(string(out)) | ||
| 492 | os.WriteFile(filepath.Join(root, "results.json"), out, 0o644) | ||
| 493 | } | ||