vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Bound and stream the anonymous read path
4581690d1ae9d87c272428d101a0fbe3538d7f44 · Keenan Nemetz · 13d ago · parent e05f1c48
Bound and stream the anonymous read path Every anonymous read path read a whole git object before deciding what to do with it: the code browser truncated after allocating, the overview rendered the README it had just read in full, and the site pool allocated the entire object for every request. `max_blob_bytes` allows a 100 MiB blob, so one pushed file made every concurrent request for it a 100 MiB allocation, with no account needed to ask for it. Read the header first instead. `core/git` gains `BlobSize`, `BlobSizes` (one `cat-file --batch-check` for a whole tree) and `BlobStream`, which parses the batch header before touching the payload and stops the process when a caller wants only a prefix. Then use it: `/raw` streams with the object's own `Content-Length`, the inline viewer reads a one-MiB prefix and reports the file's real size, the README is capped and named instead of rendered when it is too large, site blobs are capped at 25 MiB with a 413, and the commit and tree objects read to resolve a site path are capped separately. Metadata gets the same treatment on both sides: 4 MiB per file, 2 MiB per body, 20,000 files and 64 MiB for `issues/` and `requests/` together. The loader asks git for every size in one call and refuses the tree before reading it; the push boundary refuses the same tree, so the author learns why instead of finding that their issue list stopped loading. Counting is now per subtree, so the machine written `runs/` tree no longer eats the issue file budget.
core/git/git.go +188
| @@ -4,6 +4,7 @@ | |||
| 4 | 4 | package git | |
| 5 | 5 | ||
| 6 | 6 | import ( | |
| 7 | "bufio" | ||
| 7 | 8 | "bytes" | |
| 8 | 9 | "errors" | |
| 9 | 10 | "fmt" | |
| @@ -406,6 +407,193 @@ func (r *Repo) BlobAt(commit, path string) ([]byte, error) { | |||
| 406 | 407 | return out, nil | |
| 407 | 408 | } | |
| 408 | 409 | ||
| 410 | // BlobSize returns the length of the object at rev without reading it. It is | ||
| 411 | // what lets a reader decide to stream a 100 MiB file or to render only its first | ||
| 412 | // page, instead of allocating the whole thing and measuring it afterwards. | ||
| 413 | func (r *Repo) BlobSize(rev string) (int64, error) { | ||
| 414 | out, errs, err := r.run(nil, "cat-file", "-s", rev) | ||
| 415 | if err != nil { | ||
| 416 | return 0, fmt.Errorf("cat-file -s %s: %w: %s", rev, err, errs) | ||
| 417 | } | ||
| 418 | n, err := strconv.ParseInt(strings.TrimSpace(string(out)), 10, 64) | ||
| 419 | if err != nil { | ||
| 420 | return 0, fmt.Errorf("cat-file -s %s: bad size %q", rev, out) | ||
| 421 | } | ||
| 422 | return n, nil | ||
| 423 | } | ||
| 424 | |||
| 425 | // BlobSizes returns the size of every named blob in one git call. A loader that | ||
| 426 | // is about to read a whole tree of files uses it to refuse the oversized ones | ||
| 427 | // before any of them becomes a []byte. | ||
| 428 | func (r *Repo) BlobSizes(shas []string) (map[string]int64, error) { | ||
| 429 | for _, sha := range shas { | ||
| 430 | if strings.ContainsAny(sha, "\n\r\x00\t ") { | ||
| 431 | return nil, fmt.Errorf("cat-file: malformed object id %q", sha) | ||
| 432 | } | ||
| 433 | } | ||
| 434 | return r.blobSizes(shas) | ||
| 435 | } | ||
| 436 | |||
| 437 | // ErrObjectTooLarge reports that a caller asked for an object that exceeds the | ||
| 438 | // cap it named. It is a policy answer, not a broken repository. | ||
| 439 | var ErrObjectTooLarge = errors.New("git: object exceeds size limit") | ||
| 440 | |||
| 441 | // BlobStream is one object read from git as a stream, with its header already | ||
| 442 | // parsed. It exists so a download can be copied to the wire without the object | ||
| 443 | // first becoming a []byte: serving a 100 MiB blob costs one 64 KiB buffer no | ||
| 444 | // matter how many readers ask for it at once. | ||
| 445 | // | ||
| 446 | // A stream holds a git process open, so every one of them must be closed. Close | ||
| 447 | // on a partly read stream kills the process instead of draining the rest: the | ||
| 448 | // caller stopped caring about the bytes. | ||
| 449 | type BlobStream struct { | ||
| 450 | SHA string | ||
| 451 | Type string | ||
| 452 | Size int64 | ||
| 453 | |||
| 454 | br *bufio.Reader | ||
| 455 | cmd *exec.Cmd | ||
| 456 | in io.WriteCloser | ||
| 457 | left int64 | ||
| 458 | done bool | ||
| 459 | closed bool | ||
| 460 | } | ||
| 461 | |||
| 462 | // OpenBlob resolves rev (a revision, a commit:path pair, or an object id) and | ||
| 463 | // returns its header with the reader positioned at the first payload byte. | ||
| 464 | func (r *Repo) OpenBlob(rev string) (*BlobStream, error) { | ||
| 465 | if rev == "" || strings.ContainsAny(rev, "\n\r\x00") { | ||
| 466 | return nil, fmt.Errorf("cat-file: malformed revision %q", rev) | ||
| 467 | } | ||
| 468 | cmd := r.command("cat-file", "--batch") | ||
| 469 | in, err := cmd.StdinPipe() | ||
| 470 | if err != nil { | ||
| 471 | return nil, err | ||
| 472 | } | ||
| 473 | out, err := cmd.StdoutPipe() | ||
| 474 | if err != nil { | ||
| 475 | in.Close() | ||
| 476 | return nil, err | ||
| 477 | } | ||
| 478 | cmd.Stderr = io.Discard | ||
| 479 | if err := cmd.Start(); err != nil { | ||
| 480 | in.Close() | ||
| 481 | return nil, err | ||
| 482 | } | ||
| 483 | kill := func() { | ||
| 484 | in.Close() | ||
| 485 | cmd.Process.Kill() | ||
| 486 | cmd.Wait() | ||
| 487 | } | ||
| 488 | if _, err := io.WriteString(in, rev+"\n"); err != nil { | ||
| 489 | kill() | ||
| 490 | return nil, fmt.Errorf("cat-file %s: %w", rev, err) | ||
| 491 | } | ||
| 492 | br := bufio.NewReaderSize(out, 64<<10) | ||
| 493 | line, err := br.ReadString('\n') | ||
| 494 | if err != nil { | ||
| 495 | kill() | ||
| 496 | return nil, fmt.Errorf("cat-file %s: %w", rev, err) | ||
| 497 | } | ||
| 498 | fields := strings.Fields(strings.TrimRight(line, "\n")) | ||
| 499 | if len(fields) == 2 && fields[1] == "missing" { | ||
| 500 | kill() | ||
| 501 | return nil, fmt.Errorf("cat-file %s: object not found", rev) | ||
| 502 | } | ||
| 503 | if len(fields) != 3 { | ||
| 504 | kill() | ||
| 505 | return nil, fmt.Errorf("cat-file %s: unexpected header %q", rev, line) | ||
| 506 | } | ||
| 507 | size, err := strconv.ParseInt(fields[2], 10, 64) | ||
| 508 | if err != nil || size < 0 { | ||
| 509 | kill() | ||
| 510 | return nil, fmt.Errorf("cat-file %s: bad size in %q", rev, line) | ||
| 511 | } | ||
| 512 | return &BlobStream{SHA: fields[0], Type: fields[1], Size: size, br: br, cmd: cmd, in: in, left: size}, nil | ||
| 513 | } | ||
| 514 | |||
| 515 | // OpenBlobAt is OpenBlob for commit:path. | ||
| 516 | func (r *Repo) OpenBlobAt(commit, path string) (*BlobStream, error) { | ||
| 517 | return r.OpenBlob(commit + ":" + path) | ||
| 518 | } | ||
| 519 | |||
| 520 | // Read implements io.Reader over the object's payload. It stops at the object's | ||
| 521 | // own length; the trailing newline git writes after each object is eaten by the | ||
| 522 | // last read. | ||
| 523 | func (b *BlobStream) Read(p []byte) (int, error) { | ||
| 524 | if b.closed || b.left <= 0 { | ||
| 525 | return 0, io.EOF | ||
| 526 | } | ||
| 527 | if int64(len(p)) > b.left { | ||
| 528 | p = p[:b.left] | ||
| 529 | } | ||
| 530 | n, err := b.br.Read(p) | ||
| 531 | b.left -= int64(n) | ||
| 532 | if b.left == 0 { | ||
| 533 | b.finish() | ||
| 534 | } | ||
| 535 | return n, err | ||
| 536 | } | ||
| 537 | |||
| 538 | // finish eats the object's trailing newline and reaps the process. A failure | ||
| 539 | // here is not the reader's business: the payload is already delivered. | ||
| 540 | func (b *BlobStream) finish() { | ||
| 541 | if b.done { | ||
| 542 | return | ||
| 543 | } | ||
| 544 | b.done = true | ||
| 545 | b.br.Discard(1) | ||
| 546 | b.in.Close() | ||
| 547 | b.cmd.Wait() | ||
| 548 | } | ||
| 549 | |||
| 550 | // Close releases the process. It is safe to call more than once and after a | ||
| 551 | // full read. | ||
| 552 | func (b *BlobStream) Close() error { | ||
| 553 | if b.closed { | ||
| 554 | return nil | ||
| 555 | } | ||
| 556 | b.closed = true | ||
| 557 | if !b.done { | ||
| 558 | b.done = true | ||
| 559 | b.in.Close() | ||
| 560 | if b.cmd.Process != nil { | ||
| 561 | b.cmd.Process.Kill() | ||
| 562 | } | ||
| 563 | b.cmd.Wait() | ||
| 564 | } | ||
| 565 | return nil | ||
| 566 | } | ||
| 567 | |||
| 568 | // ReadPrefix returns at most max bytes of the stream and reports whether more | ||
| 569 | // bytes were left behind. The rest is not read: the process is stopped. | ||
| 570 | func (b *BlobStream) ReadPrefix(max int64) ([]byte, bool, error) { | ||
| 571 | if max < 0 { | ||
| 572 | max = 0 | ||
| 573 | } | ||
| 574 | take := b.left | ||
| 575 | if take > max { | ||
| 576 | take = max | ||
| 577 | } | ||
| 578 | buf := make([]byte, take) | ||
| 579 | if _, err := io.ReadFull(b, buf); err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { | ||
| 580 | return nil, false, err | ||
| 581 | } | ||
| 582 | return buf, b.left > 0, nil | ||
| 583 | } | ||
| 584 | |||
| 585 | // BlobPrefixAt returns at most max bytes of the blob at commit:path and whether | ||
| 586 | // there were more. It is the read for anything that renders a file inline: a | ||
| 587 | // caller that wants a preview never needs more than the preview. | ||
| 588 | func (r *Repo) BlobPrefixAt(commit, path string, max int64) ([]byte, bool, error) { | ||
| 589 | stream, err := r.OpenBlobAt(commit, path) | ||
| 590 | if err != nil { | ||
| 591 | return nil, false, err | ||
| 592 | } | ||
| 593 | defer stream.Close() | ||
| 594 | return stream.ReadPrefix(max) | ||
| 595 | } | ||
| 596 | |||
| 409 | 597 | func (r *Repo) ObjectType(rev string) (string, error) { | |
| 410 | 598 | out, _, err := r.run(nil, "cat-file", "-t", rev) | |
| 411 | 599 | if err != nil { | |