Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Docs: rewrite the README and organize the guides

520470e19cd906bd6fff987e2d132d793ce56486 · Keenan Nemetz · 16d ago · parent c5902451 

Docs: rewrite the README and organize the guides

Replace the long README with a landing page, split feature details into focused guides, and remove the roadmap and spike notes. Update comments that referenced them.

docs/spikes/archive-streaming/main.go −512

@@ -1,512 +0,0 @@
1 // Spike D — streaming release archives out of git.
2 //
3 // The question: can a release tarball be generated per request from a tag
4 // instead of being stored, and what does that cost in latency, CPU and memory?
5 //
6 // Nothing here ships. It exists to be re-run.
7 //
8 // go run . -build-fixture /tmp/fix -files 20000 -dirs 200 -filekib 1 -binarymb 2
9 // go run . -repo /tmp/fix -rev HEAD
10 // go run . -repo /home/keenan/Projects/nasdf/defradb -rev HEAD
11 package main
12
13 import (
14 "archive/tar"
15 "bytes"
16 "crypto/sha1"
17 "crypto/sha256"
18 "encoding/hex"
19 "flag"
20 "fmt"
21 "hash"
22 "io"
23 "os"
24 "os/exec"
25 "path/filepath"
26 "sort"
27 "strings"
28 "sync"
29 "syscall"
30 "time"
31 )
32
33 type config struct {
34 name string
35 args []string // extra `git -c` arguments
36 format string
37 }
38
39 func configs(withZstd bool) []config {
40 cs := []config{
41 {name: "tar", format: "tar"},
42 {name: "tar.gz (default)", format: "tar.gz"},
43 {name: "tar.gz (level 1)", args: []string{"-c", "core.compression=1"}, format: "tar.gz"},
44 {name: "tar.gz (level 9)", args: []string{"-c", "core.compression=9"}, format: "tar.gz"},
45 }
46 if withZstd {
47 cs = append(cs, config{
48 name: "tar.zst (zstd -3)",
49 args: []string{"-c", "tar.zst.command=zstd -3 -c"},
50 format: "zst",
51 })
52 }
53 return cs
54 }
55
56 type result struct {
57 cfg config
58 bytes int64
59 elapsed time.Duration
60 ttfb time.Duration
61 rssKiB int64
62 sha string
63 }
64
65 func runArchive(repo, rev string, cfg config) (result, error) {
66 r := result{cfg: cfg}
67 args := append([]string{"-C", repo}, cfg.args...)
68 args = append(args, "archive", "--format="+cfg.format, rev)
69 cmd := exec.Command("git", args...)
70 cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
71 stdout, err := cmd.StdoutPipe()
72 if err != nil {
73 return r, err
74 }
75 var stderr bytes.Buffer
76 cmd.Stderr = &stderr
77 start := time.Now()
78 if err := cmd.Start(); err != nil {
79 return r, err
80 }
81 h := sha256.New()
82 tw := &timedWriter{h: h, start: start}
83 if _, err := io.Copy(tw, stdout); err != nil {
84 return r, fmt.Errorf("%s: read: %w (%s)", cfg.name, err, stderr.String())
85 }
86 if err := cmd.Wait(); err != nil {
87 return r, fmt.Errorf("%s: %w: %s", cfg.name, err, stderr.String())
88 }
89 r.elapsed = time.Since(start)
90 r.bytes = tw.n
91 r.ttfb = tw.first
92 r.sha = hex.EncodeToString(h.Sum(nil))
93 if ru, ok := cmd.ProcessState.SysUsage().(*syscall.Rusage); ok {
94 r.rssKiB = ru.Maxrss // KiB on Linux
95 }
96 return r, nil
97 }
98
99 // timedWriter records when the first byte arrived and hashes everything.
100 type timedWriter struct {
101 h hash.Hash
102 start time.Time
103 first time.Duration
104 n int64
105 }
106
107 func (w *timedWriter) Write(p []byte) (int, error) {
108 if w.first == 0 && len(p) > 0 {
109 w.first = time.Since(w.start)
110 }
111 w.h.Write(p)
112 w.n += int64(len(p))
113 return len(p), nil
114 }
115
116 func mib(b int64) float64 { return float64(b) / (1 << 20) }
117
118 func rate(r result) float64 {
119 if r.elapsed == 0 {
120 return 0
121 }
122 return mib(r.bytes) / r.elapsed.Seconds()
123 }
124
125 // ---------------------------------------------------------------------------
126 // verification: the archive must contain the bytes the tree says it contains.
127
128 // treeEntries maps path -> blob sha for every regular file at rev.
129 func treeEntries(repo, rev string) (map[string]string, int64, error) {
130 out, err := exec.Command("git", "-C", repo, "ls-tree", "-r", "-l", rev).Output()
131 if err != nil {
132 return nil, 0, fmt.Errorf("ls-tree: %w", err)
133 }
134 m := map[string]string{}
135 var total int64
136 for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
137 if line == "" {
138 continue
139 }
140 meta, path, ok := strings.Cut(line, "\t")
141 if !ok {
142 continue
143 }
144 f := strings.Fields(meta)
145 if len(f) < 4 || f[1] != "blob" {
146 continue
147 }
148 var size int64
149 fmt.Sscan(f[3], &size)
150 m[path] = f[2]
151 total += size
152 }
153 return m, total, nil
154 }
155
156 func objectFormat(repo string) string {
157 out, err := exec.Command("git", "-C", repo, "rev-parse", "--show-object-format").Output()
158 if err != nil {
159 return "sha1"
160 }
161 return strings.TrimSpace(string(out))
162 }
163
164 type verifyStats struct {
165 Files int
166 Matched int
167 Missing int // in the tar, not in ls-tree
168 Mismatched []string
169 TarBytes int64
170 ContentByte int64
171 }
172
173 // verifyTar parses a plain tar stream and checks every regular file's content
174 // against the blob id ls-tree reports for that path. A tarball that streams
175 // fast but contains the wrong bytes is not an answer to the question.
176 func verifyTar(repo, rev string) (verifyStats, error) {
177 var st verifyStats
178 want, _, err := treeEntries(repo, rev)
179 if err != nil {
180 return st, err
181 }
182 algo := objectFormat(repo)
183 cmd := exec.Command("git", "-C", repo, "archive", "--format=tar", rev)
184 stdout, err := cmd.StdoutPipe()
185 if err != nil {
186 return st, err
187 }
188 cmd.Stderr = os.Stderr
189 if err := cmd.Start(); err != nil {
190 return st, err
191 }
192 cr := &countReader{r: stdout}
193 tr := tar.NewReader(cr)
194 for {
195 hdr, err := tr.Next()
196 if err == io.EOF {
197 break
198 }
199 if err != nil {
200 return st, fmt.Errorf("tar: %w", err)
201 }
202 if hdr.Typeflag != tar.TypeReg {
203 continue
204 }
205 path := strings.TrimPrefix(hdr.Name, "./")
206 var h hash.Hash
207 if algo == "sha256" {
208 h = sha256.New()
209 } else {
210 h = sha1.New()
211 }
212 // A git object id is over "<type> <len>\x00" + content.
213 fmt.Fprintf(h, "blob %d\x00", hdr.Size)
214 n, err := io.Copy(h, tr)
215 if err != nil {
216 return st, fmt.Errorf("tar body %s: %w", path, err)
217 }
218 st.Files++
219 st.ContentByte += n
220 wantOID, ok := want[path]
221 if !ok {
222 st.Missing++
223 continue
224 }
225 if n == hdr.Size && hex.EncodeToString(h.Sum(nil)) == wantOID {
226 st.Matched++
227 } else {
228 st.Mismatched = append(st.Mismatched, path)
229 }
230 }
231 if err := cmd.Wait(); err != nil {
232 return st, err
233 }
234 st.TarBytes = cr.n
235 return st, nil
236 }
237
238 type countReader struct {
239 r io.Reader
240 n int64
241 }
242
243 func (c *countReader) Read(p []byte) (int, error) {
244 n, err := c.r.Read(p)
245 c.n += int64(n)
246 return n, err
247 }
248
249 // ---------------------------------------------------------------------------
250
251 func determinism(repo, rev string, cfg config, runs int) (string, bool, error) {
252 var first string
253 same := true
254 for i := 0; i < runs; i++ {
255 r, err := runArchive(repo, rev, cfg)
256 if err != nil {
257 return "", false, err
258 }
259 if i == 0 {
260 first = r.sha
261 continue
262 }
263 if r.sha != first {
264 same = false
265 }
266 }
267 return first, same, nil
268 }
269
270 // cloneTo makes a fresh clone so archive output can be compared across repos
271 // with identical content but different pack layouts.
272 func cloneTo(repo, dir string) error {
273 if err := os.RemoveAll(dir); err != nil {
274 return err
275 }
276 cmd := exec.Command("git", "clone", "-q", "--no-hardlinks", repo, dir)
277 cmd.Stderr = os.Stderr
278 return cmd.Run()
279 }
280
281 func concurrency(repo, rev string, cfg config, levels []int, runs int) map[int]float64 {
282 out := map[int]float64{}
283 for _, c := range levels {
284 var best float64
285 for i := 0; i < runs; i++ {
286 var wg sync.WaitGroup
287 errs := make([]error, c)
288 start := time.Now()
289 for j := 0; j < c; j++ {
290 wg.Add(1)
291 go func(j int) {
292 defer wg.Done()
293 _, errs[j] = runArchive(repo, rev, cfg)
294 }(j)
295 }
296 wg.Wait()
297 el := time.Since(start)
298 var bad error
299 for _, e := range errs {
300 if e != nil {
301 bad = e
302 }
303 }
304 if bad != nil {
305 fmt.Fprintf(os.Stderr, "concurrency %d: %v\n", c, bad)
306 continue
307 }
308 // One archive's bytes per worker, divided by the wall time all of
309 // them took: aggregate streaming throughput.
310 r, _ := runArchive(repo, rev, cfg)
311 v := mib(r.bytes) * float64(c) / el.Seconds()
312 if v > best {
313 best = v
314 }
315 }
316 out[c] = best
317 }
318 return out
319 }
320
321 // ---------------------------------------------------------------------------
322 // fixture: a repo with many files and some incompressible weight, so the
323 // metadata cost and the byte cost can be told apart.
324
325 func buildFixture(dir string, files, dirs, fileKiB, binaryMiB int) error {
326 if err := os.RemoveAll(dir); err != nil {
327 return err
328 }
329 if err := os.MkdirAll(dir, 0o755); err != nil {
330 return err
331 }
332 run := func(args ...string) error {
333 cmd := exec.Command("git", append([]string{"-C", dir}, args...)...)
334 cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
335 return cmd.Run()
336 }
337 if err := run("init", "-q", "-b", "main"); err != nil {
338 return err
339 }
340 unit := "the quick brown fox jumps over the lazy dog\n"
341 line := strings.Repeat(unit, (fileKiB*1024)/len(unit)+1)[:fileKiB*1024]
342 perDir := files / dirs
343 if perDir == 0 {
344 perDir = 1
345 }
346 start := time.Now()
347 for d := 0; d < dirs; d++ {
348 dd := filepath.Join(dir, fmt.Sprintf("dir-%03d", d))
349 if err := os.MkdirAll(dd, 0o755); err != nil {
350 return err
351 }
352 for f := 0; f < perDir; f++ {
353 body := fmt.Sprintf("file %d/%d\n%s", d, f, line)
354 if err := os.WriteFile(filepath.Join(dd, fmt.Sprintf("file-%05d.txt", f)), []byte(body), 0o644); err != nil {
355 return err
356 }
357 }
358 }
359 if binaryMiB <= 0 {
360 // text-only fixture: compression is the interesting cost
361 } else {
362 bin := filepath.Join(dir, "assets")
363 if err := os.MkdirAll(bin, 0o755); err != nil {
364 return err
365 }
366 buf := make([]byte, binaryMiB<<20)
367 for i := 0; i < len(buf); i += 8 { // cheap non-repeating fill
368 x := uint64(i)*0x9E3779B97F4A7C15 + 12345
369 for j := 0; j < 8 && i+j < len(buf); j++ {
370 buf[i+j] = byte(x >> (8 * j))
371 }
372 }
373 for b := 0; b < 40; b++ {
374 if err := os.WriteFile(filepath.Join(bin, fmt.Sprintf("asset-%02d.bin", b)), buf, 0o644); err != nil {
375 return err
376 }
377 }
378 }
379 fmt.Printf("wrote %d files + 40 binaries in %s\n", dirs*perDir, time.Since(start).Round(time.Millisecond))
380 if err := run("add", "-A"); err != nil {
381 return err
382 }
383 return run("-c", "user.email=spike@example.com", "-c", "user.name=spike",
384 "commit", "-qm", "fixture")
385 }
386
387 func haveZstd() bool {
388 _, err := exec.LookPath("zstd")
389 return err == nil
390 }
391
392 func main() {
393 var (
394 repo = flag.String("repo", "", "repository to archive")
395 rev = flag.String("rev", "HEAD", "revision to archive")
396 runs = flag.Int("runs", 3, "serially timed runs per configuration (min is reported)")
397 levels = flag.String("conc", "1,4,8", "concurrency levels for aggregate throughput")
398 doVerify = flag.Bool("verify", true, "parse the tar and compare to ls-tree")
399 buildDir = flag.String("build-fixture", "", "build a fixture repo at this path and exit")
400 files = flag.Int("files", 20000, "fixture: number of text files")
401 dirs = flag.Int("dirs", 200, "fixture: number of directories")
402 fileKiB = flag.Int("filekib", 1, "fixture: size of each text file in KiB")
403 binaryMiB = flag.Int("binarymb", 2, "fixture: size of each of the 40 binary files in MiB")
404 cloneDir = flag.String("clone-to", "", "if set, also archive from a fresh clone at this path")
405 )
406 flag.Parse()
407
408 if *buildDir != "" {
409 if err := buildFixture(*buildDir, *files, *dirs, *fileKiB, *binaryMiB); err != nil {
410 fmt.Fprintln(os.Stderr, "fixture:", err)
411 os.Exit(1)
412 }
413 fmt.Println("ok")
414 return
415 }
416 if *repo == "" {
417 fmt.Fprintln(os.Stderr, "need -repo or -build-fixture")
418 flag.Usage()
419 os.Exit(2)
420 }
421
422 tree, total, err := treeEntries(*repo, *rev)
423 if err != nil {
424 fmt.Fprintln(os.Stderr, err)
425 os.Exit(1)
426 }
427 fmt.Printf("repo=%s rev=%s files=%d tree=%.1f MiB zstd=%v\n\n", *repo, *rev, len(tree), mib(total), haveZstd())
428
429 if *doVerify {
430 st, err := verifyTar(*repo, *rev)
431 if err != nil {
432 fmt.Fprintln(os.Stderr, "verify:", err)
433 } else {
434 fmt.Printf("verify: %d regular files in tar, %d matched ls-tree, %d not in tree, %d mismatched, tar=%.1f MiB content=%.1f MiB\n\n",
435 st.Files, st.Matched, st.Missing, len(st.Mismatched), mib(st.TarBytes), mib(st.ContentByte))
436 for i, p := range st.Mismatched {
437 if i > 5 {
438 break
439 }
440 fmt.Println(" mismatch:", p)
441 }
442 }
443 }
444
445 fmt.Println("| Configuration | MiB/s | wall s | out MiB | peak RSS MiB | TTFB ms |")
446 fmt.Println("|---|---|---|---|---|---|")
447 for _, cfg := range configs(haveZstd()) {
448 best := result{elapsed: time.Duration(1 << 62)}
449 for i := 0; i < *runs; i++ {
450 r, err := runArchive(*repo, *rev, cfg)
451 if err != nil {
452 fmt.Printf("| %s | error | | | | |\n", cfg.name)
453 fmt.Fprintln(os.Stderr, err)
454 best.elapsed = 0
455 break
456 }
457 if r.elapsed < best.elapsed {
458 best = r
459 }
460 }
461 if best.elapsed == 0 {
462 continue
463 }
464 fmt.Printf("| %s | %.1f | %.3f | %.1f | %.1f | %.1f |\n",
465 cfg.name, rate(best), best.elapsed.Seconds(), mib(best.bytes),
466 float64(best.rssKiB)/1024, float64(best.ttfb.Microseconds())/1000)
467 }
468
469 fmt.Println()
470 var lv []int
471 for _, s := range strings.Split(*levels, ",") {
472 var n int
473 if _, err := fmt.Sscan(s, &n); err == nil && n > 0 {
474 lv = append(lv, n)
475 }
476 }
477 sort.Ints(lv)
478 plain := config{name: "tar", format: "tar"}
479 gz := config{name: "tar.gz", format: "tar.gz"}
480 fmt.Println("| Format | conc 1 | conc 4 | conc 8 | (aggregate MiB/s) |")
481 fmt.Println("|---|---|---|---|---|")
482 for _, cfg := range []config{plain, gz} {
483 res := concurrency(*repo, *rev, cfg, lv, 2)
484 fmt.Printf("| %s |", cfg.name)
485 for _, l := range lv {
486 fmt.Printf(" %.1f |", res[l])
487 }
488 fmt.Println()
489 }
490
491 fmt.Println()
492 for _, cfg := range []config{{name: "tar", format: "tar"}, {name: "tar.gz", format: "tar.gz"}} {
493 sha, same, err := determinism(*repo, *rev, cfg, 3)
494 if err != nil {
495 fmt.Fprintln(os.Stderr, err)
496 continue
497 }
498 fmt.Printf("determinism %-7s same_across_3_runs=%v sha256=%s\n", cfg.name, same, sha[:16])
499 if *cloneDir != "" {
500 if err := cloneTo(*repo, *cloneDir); err != nil {
501 fmt.Fprintln(os.Stderr, "clone:", err)
502 continue
503 }
504 r, err := runArchive(*cloneDir, *rev, cfg)
505 if err != nil {
506 fmt.Fprintln(os.Stderr, err)
507 continue
508 }
509 fmt.Printf("determinism %-7s same_in_fresh_clone=%v\n", cfg.name, r.sha == sha)
510 }
511 }
512 }