vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Spike D: a source archive is a 54 ms stream, not a stored build
ada367f1067bf80ae094f2f0f158f69090b3a741 · Keenan Nemetz · 18d ago · parent 78912785
Spike D: a source archive is a 54 ms stream, not a stored build Answers whether release tarballs can be generated per request from a tag instead of stored. Measured with git 2.53 on three generated fixtures (20k files / 100 MiB with binaries, 40k and 100k file text trees) and one real repository: - a real 11.6 MiB repo archives in 54 ms plain, 221 ms gzipped, first byte at 4.1 ms; the 100 MiB binary fixture takes 0.12 s plain, 1.64 s gzipped - output is byte-identical across runs and across a fresh clone, so an ETag can come from the tag object id and a cache could never serve stale bytes - core.compression is inert for git archive: levels 1, 6 and 9 produce the same sha256 and the same time - zstd via tar.<fmt>.command is 14x faster than gzip at the same output size (0.117 s vs 1.643 s for 80 MiB), for 6x the RSS - gzip barely pays on binaries (119.2 -> 80.4 MiB) and does not parallelise: 8 concurrent gzipped streams reach 273 MiB/s against 5,005 MiB/s plain - peak RSS tracks file count (55 MiB at 100k files), so a per-request archive needs the same concurrency bound the push path has - git archive emits LFS pointers verbatim, so a source tarball of an LFS repo ships pointers unless the server resolves them Every fixture and the real repo are verified by hashing each tar entry as a git object and comparing to ls-tree: 2,566 / 20,040 / 40,000 / 100,000 files matched, none mismatched.
docs/spikes/archive-streaming/main.go +512
| @@ -0,0 +1,512 @@ | |||
| 1 | // Spike D — streaming release archives out of git. | ||
| 2 | // | ||
| 3 | // The question: can a release tarball be generated per request from a tag | ||
| 4 | // instead of being stored, and what does that cost in latency, CPU and memory? | ||
| 5 | // | ||
| 6 | // Nothing here ships. It exists to be re-run. | ||
| 7 | // | ||
| 8 | // go run . -build-fixture /tmp/fix -files 20000 -dirs 200 -filekib 1 -binarymb 2 | ||
| 9 | // go run . -repo /tmp/fix -rev HEAD | ||
| 10 | // go run . -repo /home/keenan/Projects/nasdf/defradb -rev HEAD | ||
| 11 | package main | ||
| 12 | |||
| 13 | import ( | ||
| 14 | "archive/tar" | ||
| 15 | "bytes" | ||
| 16 | "crypto/sha1" | ||
| 17 | "crypto/sha256" | ||
| 18 | "encoding/hex" | ||
| 19 | "flag" | ||
| 20 | "fmt" | ||
| 21 | "hash" | ||
| 22 | "io" | ||
| 23 | "os" | ||
| 24 | "os/exec" | ||
| 25 | "path/filepath" | ||
| 26 | "sort" | ||
| 27 | "strings" | ||
| 28 | "sync" | ||
| 29 | "syscall" | ||
| 30 | "time" | ||
| 31 | ) | ||
| 32 | |||
| 33 | type config struct { | ||
| 34 | name string | ||
| 35 | args []string // extra `git -c` arguments | ||
| 36 | format string | ||
| 37 | } | ||
| 38 | |||
| 39 | func configs(withZstd bool) []config { | ||
| 40 | cs := []config{ | ||
| 41 | {name: "tar", format: "tar"}, | ||
| 42 | {name: "tar.gz (default)", format: "tar.gz"}, | ||
| 43 | {name: "tar.gz (level 1)", args: []string{"-c", "core.compression=1"}, format: "tar.gz"}, | ||
| 44 | {name: "tar.gz (level 9)", args: []string{"-c", "core.compression=9"}, format: "tar.gz"}, | ||
| 45 | } | ||
| 46 | if withZstd { | ||
| 47 | cs = append(cs, config{ | ||
| 48 | name: "tar.zst (zstd -3)", | ||
| 49 | args: []string{"-c", "tar.zst.command=zstd -3 -c"}, | ||
| 50 | format: "zst", | ||
| 51 | }) | ||
| 52 | } | ||
| 53 | return cs | ||
| 54 | } | ||
| 55 | |||
| 56 | type result struct { | ||
| 57 | cfg config | ||
| 58 | bytes int64 | ||
| 59 | elapsed time.Duration | ||
| 60 | ttfb time.Duration | ||
| 61 | rssKiB int64 | ||
| 62 | sha string | ||
| 63 | } | ||
| 64 | |||
| 65 | func runArchive(repo, rev string, cfg config) (result, error) { | ||
| 66 | r := result{cfg: cfg} | ||
| 67 | args := append([]string{"-C", repo}, cfg.args...) | ||
| 68 | args = append(args, "archive", "--format="+cfg.format, rev) | ||
| 69 | cmd := exec.Command("git", args...) | ||
| 70 | cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0") | ||
| 71 | stdout, err := cmd.StdoutPipe() | ||
| 72 | if err != nil { | ||
| 73 | return r, err | ||
| 74 | } | ||
| 75 | var stderr bytes.Buffer | ||
| 76 | cmd.Stderr = &stderr | ||
| 77 | start := time.Now() | ||
| 78 | if err := cmd.Start(); err != nil { | ||
| 79 | return r, err | ||
| 80 | } | ||
| 81 | h := sha256.New() | ||
| 82 | tw := &timedWriter{h: h, start: start} | ||
| 83 | if _, err := io.Copy(tw, stdout); err != nil { | ||
| 84 | return r, fmt.Errorf("%s: read: %w (%s)", cfg.name, err, stderr.String()) | ||
| 85 | } | ||
| 86 | if err := cmd.Wait(); err != nil { | ||
| 87 | return r, fmt.Errorf("%s: %w: %s", cfg.name, err, stderr.String()) | ||
| 88 | } | ||
| 89 | r.elapsed = time.Since(start) | ||
| 90 | r.bytes = tw.n | ||
| 91 | r.ttfb = tw.first | ||
| 92 | r.sha = hex.EncodeToString(h.Sum(nil)) | ||
| 93 | if ru, ok := cmd.ProcessState.SysUsage().(*syscall.Rusage); ok { | ||
| 94 | r.rssKiB = ru.Maxrss // KiB on Linux | ||
| 95 | } | ||
| 96 | return r, nil | ||
| 97 | } | ||
| 98 | |||
| 99 | // timedWriter records when the first byte arrived and hashes everything. | ||
| 100 | type timedWriter struct { | ||
| 101 | h hash.Hash | ||
| 102 | start time.Time | ||
| 103 | first time.Duration | ||
| 104 | n int64 | ||
| 105 | } | ||
| 106 | |||
| 107 | func (w *timedWriter) Write(p []byte) (int, error) { | ||
| 108 | if w.first == 0 && len(p) > 0 { | ||
| 109 | w.first = time.Since(w.start) | ||
| 110 | } | ||
| 111 | w.h.Write(p) | ||
| 112 | w.n += int64(len(p)) | ||
| 113 | return len(p), nil | ||
| 114 | } | ||
| 115 | |||
| 116 | func mib(b int64) float64 { return float64(b) / (1 << 20) } | ||
| 117 | |||
| 118 | func rate(r result) float64 { | ||
| 119 | if r.elapsed == 0 { | ||
| 120 | return 0 | ||
| 121 | } | ||
| 122 | return mib(r.bytes) / r.elapsed.Seconds() | ||
| 123 | } | ||
| 124 | |||
| 125 | // --------------------------------------------------------------------------- | ||
| 126 | // verification: the archive must contain the bytes the tree says it contains. | ||
| 127 | |||
| 128 | // treeEntries maps path -> blob sha for every regular file at rev. | ||
| 129 | func treeEntries(repo, rev string) (map[string]string, int64, error) { | ||
| 130 | out, err := exec.Command("git", "-C", repo, "ls-tree", "-r", "-l", rev).Output() | ||
| 131 | if err != nil { | ||
| 132 | return nil, 0, fmt.Errorf("ls-tree: %w", err) | ||
| 133 | } | ||
| 134 | m := map[string]string{} | ||
| 135 | var total int64 | ||
| 136 | for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { | ||
| 137 | if line == "" { | ||
| 138 | continue | ||
| 139 | } | ||
| 140 | meta, path, ok := strings.Cut(line, "\t") | ||
| 141 | if !ok { | ||
| 142 | continue | ||
| 143 | } | ||
| 144 | f := strings.Fields(meta) | ||
| 145 | if len(f) < 4 || f[1] != "blob" { | ||
| 146 | continue | ||
| 147 | } | ||
| 148 | var size int64 | ||
| 149 | fmt.Sscan(f[3], &size) | ||
| 150 | m[path] = f[2] | ||
| 151 | total += size | ||
| 152 | } | ||
| 153 | return m, total, nil | ||
| 154 | } | ||
| 155 | |||
| 156 | func objectFormat(repo string) string { | ||
| 157 | out, err := exec.Command("git", "-C", repo, "rev-parse", "--show-object-format").Output() | ||
| 158 | if err != nil { | ||
| 159 | return "sha1" | ||
| 160 | } | ||
| 161 | return strings.TrimSpace(string(out)) | ||
| 162 | } | ||
| 163 | |||
| 164 | type verifyStats struct { | ||
| 165 | Files int | ||
| 166 | Matched int | ||
| 167 | Missing int // in the tar, not in ls-tree | ||
| 168 | Mismatched []string | ||
| 169 | TarBytes int64 | ||
| 170 | ContentByte int64 | ||
| 171 | } | ||
| 172 | |||
| 173 | // verifyTar parses a plain tar stream and checks every regular file's content | ||
| 174 | // against the blob id ls-tree reports for that path. A tarball that streams | ||
| 175 | // fast but contains the wrong bytes is not an answer to the question. | ||
| 176 | func verifyTar(repo, rev string) (verifyStats, error) { | ||
| 177 | var st verifyStats | ||
| 178 | want, _, err := treeEntries(repo, rev) | ||
| 179 | if err != nil { | ||
| 180 | return st, err | ||
| 181 | } | ||
| 182 | algo := objectFormat(repo) | ||
| 183 | cmd := exec.Command("git", "-C", repo, "archive", "--format=tar", rev) | ||
| 184 | stdout, err := cmd.StdoutPipe() | ||
| 185 | if err != nil { | ||
| 186 | return st, err | ||
| 187 | } | ||
| 188 | cmd.Stderr = os.Stderr | ||
| 189 | if err := cmd.Start(); err != nil { | ||
| 190 | return st, err | ||
| 191 | } | ||
| 192 | cr := &countReader{r: stdout} | ||
| 193 | tr := tar.NewReader(cr) | ||
| 194 | for { | ||
| 195 | hdr, err := tr.Next() | ||
| 196 | if err == io.EOF { | ||
| 197 | break | ||
| 198 | } | ||
| 199 | if err != nil { | ||
| 200 | return st, fmt.Errorf("tar: %w", err) | ||
| 201 | } | ||
| 202 | if hdr.Typeflag != tar.TypeReg { | ||
| 203 | continue | ||
| 204 | } | ||
| 205 | path := strings.TrimPrefix(hdr.Name, "./") | ||
| 206 | var h hash.Hash | ||
| 207 | if algo == "sha256" { | ||
| 208 | h = sha256.New() | ||
| 209 | } else { | ||
| 210 | h = sha1.New() | ||
| 211 | } | ||
| 212 | // A git object id is over "<type> <len>\x00" + content. | ||
| 213 | fmt.Fprintf(h, "blob %d\x00", hdr.Size) | ||
| 214 | n, err := io.Copy(h, tr) | ||
| 215 | if err != nil { | ||
| 216 | return st, fmt.Errorf("tar body %s: %w", path, err) | ||
| 217 | } | ||
| 218 | st.Files++ | ||
| 219 | st.ContentByte += n | ||
| 220 | wantOID, ok := want[path] | ||
| 221 | if !ok { | ||
| 222 | st.Missing++ | ||
| 223 | continue | ||
| 224 | } | ||
| 225 | if n == hdr.Size && hex.EncodeToString(h.Sum(nil)) == wantOID { | ||
| 226 | st.Matched++ | ||
| 227 | } else { | ||
| 228 | st.Mismatched = append(st.Mismatched, path) | ||
| 229 | } | ||
| 230 | } | ||
| 231 | if err := cmd.Wait(); err != nil { | ||
| 232 | return st, err | ||
| 233 | } | ||
| 234 | st.TarBytes = cr.n | ||
| 235 | return st, nil | ||
| 236 | } | ||
| 237 | |||
| 238 | type countReader struct { | ||
| 239 | r io.Reader | ||
| 240 | n int64 | ||
| 241 | } | ||
| 242 | |||
| 243 | func (c *countReader) Read(p []byte) (int, error) { | ||
| 244 | n, err := c.r.Read(p) | ||
| 245 | c.n += int64(n) | ||
| 246 | return n, err | ||
| 247 | } | ||
| 248 | |||
| 249 | // --------------------------------------------------------------------------- | ||
| 250 | |||
| 251 | func determinism(repo, rev string, cfg config, runs int) (string, bool, error) { | ||
| 252 | var first string | ||
| 253 | same := true | ||
| 254 | for i := 0; i < runs; i++ { | ||
| 255 | r, err := runArchive(repo, rev, cfg) | ||
| 256 | if err != nil { | ||
| 257 | return "", false, err | ||
| 258 | } | ||
| 259 | if i == 0 { | ||
| 260 | first = r.sha | ||
| 261 | continue | ||
| 262 | } | ||
| 263 | if r.sha != first { | ||
| 264 | same = false | ||
| 265 | } | ||
| 266 | } | ||
| 267 | return first, same, nil | ||
| 268 | } | ||
| 269 | |||
| 270 | // cloneTo makes a fresh clone so archive output can be compared across repos | ||
| 271 | // with identical content but different pack layouts. | ||
| 272 | func cloneTo(repo, dir string) error { | ||
| 273 | if err := os.RemoveAll(dir); err != nil { | ||
| 274 | return err | ||
| 275 | } | ||
| 276 | cmd := exec.Command("git", "clone", "-q", "--no-hardlinks", repo, dir) | ||
| 277 | cmd.Stderr = os.Stderr | ||
| 278 | return cmd.Run() | ||
| 279 | } | ||
| 280 | |||
| 281 | func concurrency(repo, rev string, cfg config, levels []int, runs int) map[int]float64 { | ||
| 282 | out := map[int]float64{} | ||
| 283 | for _, c := range levels { | ||
| 284 | var best float64 | ||
| 285 | for i := 0; i < runs; i++ { | ||
| 286 | var wg sync.WaitGroup | ||
| 287 | errs := make([]error, c) | ||
| 288 | start := time.Now() | ||
| 289 | for j := 0; j < c; j++ { | ||
| 290 | wg.Add(1) | ||
| 291 | go func(j int) { | ||
| 292 | defer wg.Done() | ||
| 293 | _, errs[j] = runArchive(repo, rev, cfg) | ||
| 294 | }(j) | ||
| 295 | } | ||
| 296 | wg.Wait() | ||
| 297 | el := time.Since(start) | ||
| 298 | var bad error | ||
| 299 | for _, e := range errs { | ||
| 300 | if e != nil { | ||
| 301 | bad = e | ||
| 302 | } | ||
| 303 | } | ||
| 304 | if bad != nil { | ||
| 305 | fmt.Fprintf(os.Stderr, "concurrency %d: %v\n", c, bad) | ||
| 306 | continue | ||
| 307 | } | ||
| 308 | // One archive's bytes per worker, divided by the wall time all of | ||
| 309 | // them took: aggregate streaming throughput. | ||
| 310 | r, _ := runArchive(repo, rev, cfg) | ||
| 311 | v := mib(r.bytes) * float64(c) / el.Seconds() | ||
| 312 | if v > best { | ||
| 313 | best = v | ||
| 314 | } | ||
| 315 | } | ||
| 316 | out[c] = best | ||
| 317 | } | ||
| 318 | return out | ||
| 319 | } | ||
| 320 | |||
| 321 | // --------------------------------------------------------------------------- | ||
| 322 | // fixture: a repo with many files and some incompressible weight, so the | ||
| 323 | // metadata cost and the byte cost can be told apart. | ||
| 324 | |||
| 325 | func buildFixture(dir string, files, dirs, fileKiB, binaryMiB int) error { | ||
| 326 | if err := os.RemoveAll(dir); err != nil { | ||
| 327 | return err | ||
| 328 | } | ||
| 329 | if err := os.MkdirAll(dir, 0o755); err != nil { | ||
| 330 | return err | ||
| 331 | } | ||
| 332 | run := func(args ...string) error { | ||
| 333 | cmd := exec.Command("git", append([]string{"-C", dir}, args...)...) | ||
| 334 | cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr | ||
| 335 | return cmd.Run() | ||
| 336 | } | ||
| 337 | if err := run("init", "-q", "-b", "main"); err != nil { | ||
| 338 | return err | ||
| 339 | } | ||
| 340 | unit := "the quick brown fox jumps over the lazy dog\n" | ||
| 341 | line := strings.Repeat(unit, (fileKiB*1024)/len(unit)+1)[:fileKiB*1024] | ||
| 342 | perDir := files / dirs | ||
| 343 | if perDir == 0 { | ||
| 344 | perDir = 1 | ||
| 345 | } | ||
| 346 | start := time.Now() | ||
| 347 | for d := 0; d < dirs; d++ { | ||
| 348 | dd := filepath.Join(dir, fmt.Sprintf("dir-%03d", d)) | ||
| 349 | if err := os.MkdirAll(dd, 0o755); err != nil { | ||
| 350 | return err | ||
| 351 | } | ||
| 352 | for f := 0; f < perDir; f++ { | ||
| 353 | body := fmt.Sprintf("file %d/%d\n%s", d, f, line) | ||
| 354 | if err := os.WriteFile(filepath.Join(dd, fmt.Sprintf("file-%05d.txt", f)), []byte(body), 0o644); err != nil { | ||
| 355 | return err | ||
| 356 | } | ||
| 357 | } | ||
| 358 | } | ||
| 359 | if binaryMiB <= 0 { | ||
| 360 | // text-only fixture: compression is the interesting cost | ||
| 361 | } else { | ||
| 362 | bin := filepath.Join(dir, "assets") | ||
| 363 | if err := os.MkdirAll(bin, 0o755); err != nil { | ||
| 364 | return err | ||
| 365 | } | ||
| 366 | buf := make([]byte, binaryMiB<<20) | ||
| 367 | for i := 0; i < len(buf); i += 8 { // cheap non-repeating fill | ||
| 368 | x := uint64(i)*0x9E3779B97F4A7C15 + 12345 | ||
| 369 | for j := 0; j < 8 && i+j < len(buf); j++ { | ||
| 370 | buf[i+j] = byte(x >> (8 * j)) | ||
| 371 | } | ||
| 372 | } | ||
| 373 | for b := 0; b < 40; b++ { | ||
| 374 | if err := os.WriteFile(filepath.Join(bin, fmt.Sprintf("asset-%02d.bin", b)), buf, 0o644); err != nil { | ||
| 375 | return err | ||
| 376 | } | ||
| 377 | } | ||
| 378 | } | ||
| 379 | fmt.Printf("wrote %d files + 40 binaries in %s\n", dirs*perDir, time.Since(start).Round(time.Millisecond)) | ||
| 380 | if err := run("add", "-A"); err != nil { | ||
| 381 | return err | ||
| 382 | } | ||
| 383 | return run("-c", "user.email=spike@example.com", "-c", "user.name=spike", | ||
| 384 | "commit", "-qm", "fixture") | ||
| 385 | } | ||
| 386 | |||
| 387 | func haveZstd() bool { | ||
| 388 | _, err := exec.LookPath("zstd") | ||
| 389 | return err == nil | ||
| 390 | } | ||
| 391 | |||
| 392 | func main() { | ||
| 393 | var ( | ||
| 394 | repo = flag.String("repo", "", "repository to archive") | ||
| 395 | rev = flag.String("rev", "HEAD", "revision to archive") | ||
| 396 | runs = flag.Int("runs", 3, "serially timed runs per configuration (min is reported)") | ||
| 397 | levels = flag.String("conc", "1,4,8", "concurrency levels for aggregate throughput") | ||
| 398 | doVerify = flag.Bool("verify", true, "parse the tar and compare to ls-tree") | ||
| 399 | buildDir = flag.String("build-fixture", "", "build a fixture repo at this path and exit") | ||
| 400 | files = flag.Int("files", 20000, "fixture: number of text files") | ||
| 401 | dirs = flag.Int("dirs", 200, "fixture: number of directories") | ||
| 402 | fileKiB = flag.Int("filekib", 1, "fixture: size of each text file in KiB") | ||
| 403 | binaryMiB = flag.Int("binarymb", 2, "fixture: size of each of the 40 binary files in MiB") | ||
| 404 | cloneDir = flag.String("clone-to", "", "if set, also archive from a fresh clone at this path") | ||
| 405 | ) | ||
| 406 | flag.Parse() | ||
| 407 | |||
| 408 | if *buildDir != "" { | ||
| 409 | if err := buildFixture(*buildDir, *files, *dirs, *fileKiB, *binaryMiB); err != nil { | ||
| 410 | fmt.Fprintln(os.Stderr, "fixture:", err) | ||
| 411 | os.Exit(1) | ||
| 412 | } | ||
| 413 | fmt.Println("ok") | ||
| 414 | return | ||
| 415 | } | ||
| 416 | if *repo == "" { | ||
| 417 | fmt.Fprintln(os.Stderr, "need -repo or -build-fixture") | ||
| 418 | flag.Usage() | ||
| 419 | os.Exit(2) | ||
| 420 | } | ||
| 421 | |||
| 422 | tree, total, err := treeEntries(*repo, *rev) | ||
| 423 | if err != nil { | ||
| 424 | fmt.Fprintln(os.Stderr, err) | ||
| 425 | os.Exit(1) | ||
| 426 | } | ||
| 427 | fmt.Printf("repo=%s rev=%s files=%d tree=%.1f MiB zstd=%v\n\n", *repo, *rev, len(tree), mib(total), haveZstd()) | ||
| 428 | |||
| 429 | if *doVerify { | ||
| 430 | st, err := verifyTar(*repo, *rev) | ||
| 431 | if err != nil { | ||
| 432 | fmt.Fprintln(os.Stderr, "verify:", err) | ||
| 433 | } else { | ||
| 434 | fmt.Printf("verify: %d regular files in tar, %d matched ls-tree, %d not in tree, %d mismatched, tar=%.1f MiB content=%.1f MiB\n\n", | ||
| 435 | st.Files, st.Matched, st.Missing, len(st.Mismatched), mib(st.TarBytes), mib(st.ContentByte)) | ||
| 436 | for i, p := range st.Mismatched { | ||
| 437 | if i > 5 { | ||
| 438 | break | ||
| 439 | } | ||
| 440 | fmt.Println(" mismatch:", p) | ||
| 441 | } | ||
| 442 | } | ||
| 443 | } | ||
| 444 | |||
| 445 | fmt.Println("| Configuration | MiB/s | wall s | out MiB | peak RSS MiB | TTFB ms |") | ||
| 446 | fmt.Println("|---|---|---|---|---|---|") | ||
| 447 | for _, cfg := range configs(haveZstd()) { | ||
| 448 | best := result{elapsed: time.Duration(1 << 62)} | ||
| 449 | for i := 0; i < *runs; i++ { | ||
| 450 | r, err := runArchive(*repo, *rev, cfg) | ||
| 451 | if err != nil { | ||
| 452 | fmt.Printf("| %s | error | | | | |\n", cfg.name) | ||
| 453 | fmt.Fprintln(os.Stderr, err) | ||
| 454 | best.elapsed = 0 | ||
| 455 | break | ||
| 456 | } | ||
| 457 | if r.elapsed < best.elapsed { | ||
| 458 | best = r | ||
| 459 | } | ||
| 460 | } | ||
| 461 | if best.elapsed == 0 { | ||
| 462 | continue | ||
| 463 | } | ||
| 464 | fmt.Printf("| %s | %.1f | %.3f | %.1f | %.1f | %.1f |\n", | ||
| 465 | cfg.name, rate(best), best.elapsed.Seconds(), mib(best.bytes), | ||
| 466 | float64(best.rssKiB)/1024, float64(best.ttfb.Microseconds())/1000) | ||
| 467 | } | ||
| 468 | |||
| 469 | fmt.Println() | ||
| 470 | var lv []int | ||
| 471 | for _, s := range strings.Split(*levels, ",") { | ||
| 472 | var n int | ||
| 473 | if _, err := fmt.Sscan(s, &n); err == nil && n > 0 { | ||
| 474 | lv = append(lv, n) | ||
| 475 | } | ||
| 476 | } | ||
| 477 | sort.Ints(lv) | ||
| 478 | plain := config{name: "tar", format: "tar"} | ||
| 479 | gz := config{name: "tar.gz", format: "tar.gz"} | ||
| 480 | fmt.Println("| Format | conc 1 | conc 4 | conc 8 | (aggregate MiB/s) |") | ||
| 481 | fmt.Println("|---|---|---|---|---|") | ||
| 482 | for _, cfg := range []config{plain, gz} { | ||
| 483 | res := concurrency(*repo, *rev, cfg, lv, 2) | ||
| 484 | fmt.Printf("| %s |", cfg.name) | ||
| 485 | for _, l := range lv { | ||
| 486 | fmt.Printf(" %.1f |", res[l]) | ||
| 487 | } | ||
| 488 | fmt.Println() | ||
| 489 | } | ||
| 490 | |||
| 491 | fmt.Println() | ||
| 492 | for _, cfg := range []config{{name: "tar", format: "tar"}, {name: "tar.gz", format: "tar.gz"}} { | ||
| 493 | sha, same, err := determinism(*repo, *rev, cfg, 3) | ||
| 494 | if err != nil { | ||
| 495 | fmt.Fprintln(os.Stderr, err) | ||
| 496 | continue | ||
| 497 | } | ||
| 498 | fmt.Printf("determinism %-7s same_across_3_runs=%v sha256=%s\n", cfg.name, same, sha[:16]) | ||
| 499 | if *cloneDir != "" { | ||
| 500 | if err := cloneTo(*repo, *cloneDir); err != nil { | ||
| 501 | fmt.Fprintln(os.Stderr, "clone:", err) | ||
| 502 | continue | ||
| 503 | } | ||
| 504 | r, err := runArchive(*cloneDir, *rev, cfg) | ||
| 505 | if err != nil { | ||
| 506 | fmt.Fprintln(os.Stderr, err) | ||
| 507 | continue | ||
| 508 | } | ||
| 509 | fmt.Printf("determinism %-7s same_in_fresh_clone=%v\n", cfg.name, r.sha == sha) | ||
| 510 | } | ||
| 511 | } | ||
| 512 | } | ||