vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Blame · server/pages/catfile.go

Line attribution on main · 172 line(s)

View file History

d3296cda Keenan Nemetz 17d ago	1	package pages
	2
	3	// A pool of `git cat-file --batch` processes: the read path for site blobs.
	4	//
520470e1 Keenan Nemetz 16d ago	5	// Measurements show that one process is not the ceiling — 1 to 4 processes
520470e1 Keenan Nemetz 16d ago	6	// doubled throughput (12.1k to 24.8k req/s) — so the pool exists to
d3296cda Keenan Nemetz 17d ago	7	// let reads run in parallel. Each process is a pipe pair guarded by one mutex;
	8	// a process that dies is restarted on the next read rather than taking the site
	9	// down with it.
	10
	11	import (
	12	"bufio"
	13	"errors"
	14	"fmt"
	15	"io"
	16	"os"
	17	"os/exec"
	18	"strconv"
	19	"strings"
	20	"sync"
	21	)
	22
	23	var errMissing = errors.New("pages: object not found")
	24
	25	type objectPool struct {
	26	dir string
	27	procs []*catFileProc
	28	next int
	29	mu sync.Mutex
	30	}
	31
	32	type catFileProc struct {
	33	dir string
	34	mu sync.Mutex
	35	cmd *exec.Cmd
	36	in io.WriteCloser
	37	out *bufio.Reader
	38	dead bool
	39	}
	40
	41	func newObjectPool(dir string, n int) *objectPool {
	42	if n <= 0 {
	43	n = 1
	44	}
	45	p := &objectPool{dir: dir, procs: make([]*catFileProc, n)}
	46	for i := range p.procs {
	47	p.procs[i] = &catFileProc{dir: dir}
	48	}
	49	return p
	50	}
	51
	52	// object is one thing read out of git: the id git resolved, the type, and bytes.
	53	type object struct {
	54	SHA string
	55	Type string
	56	Data []byte
	57	}
	58
4581690d Keenan Nemetz 13d ago	59	// read resolves rev and returns its content, refusing anything past max bytes.
	60	// rev may be a ref name, which is how the pool answers "what commit is this
	61	// branch on" without a second subprocess. A refused object is never allocated:
	62	// its size is in the batch header, so the answer costs one header line and the
	63	// process is dropped rather than drained.
	64	func (p *objectPool) read(rev string, max int64) (object, error) {
d3296cda Keenan Nemetz 17d ago	65	if strings.ContainsAny(rev, "\n\r") {
	66	return object{}, fmt.Errorf("pages: malformed revision %q", rev)
	67	}
	68	p.mu.Lock()
	69	proc := p.procs[p.next%len(p.procs)]
	70	p.next++
	71	p.mu.Unlock()
4581690d Keenan Nemetz 13d ago	72	return proc.read(rev, max)
d3296cda Keenan Nemetz 17d ago	73	}
	74
	75	func (p *objectPool) Close() {
	76	for _, proc := range p.procs {
	77	proc.close()
	78	}
	79	}
	80
4581690d Keenan Nemetz 13d ago	81	func (proc *catFileProc) read(rev string, max int64) (object, error) {
d3296cda Keenan Nemetz 17d ago	82	proc.mu.Lock()
	83	defer proc.mu.Unlock()
	84	if proc.dead \|\| proc.cmd == nil {
	85	if err := proc.start(); err != nil {
	86	return object{}, err
	87	}
	88	}
	89	if _, err := io.WriteString(proc.in, rev+"\n"); err != nil {
	90	proc.dead = true
	91	return object{}, err
	92	}
	93	line, err := proc.out.ReadString('\n')
	94	if err != nil {
	95	proc.dead = true
	96	return object{}, err
	97	}
	98	fields := strings.Fields(strings.TrimRight(line, "\n"))
	99	if len(fields) == 2 && fields[1] == "missing" {
	100	return object{}, errMissing
	101	}
	102	if len(fields) != 3 {
	103	proc.dead = true
	104	return object{}, fmt.Errorf("pages: unexpected cat-file header %q", line)
	105	}
	106	size, err := strconv.ParseInt(fields[2], 10, 64)
	107	if err != nil \|\| size < 0 {
	108	proc.dead = true
	109	return object{}, fmt.Errorf("pages: bad object size in %q", line)
	110	}
4581690d Keenan Nemetz 13d ago	111	if max > 0 && size > max {
	112	// The object is still pending on the stream. Killing the process is
	113	// cheaper and safer than reading a blob we are not going to keep: the next
	114	// read starts a fresh one.
	115	proc.stop()
	116	return object{}, fmt.Errorf("%w: %s is %d bytes, limit %d", errObjectTooLarge, rev, size, max)
	117	}
d3296cda Keenan Nemetz 17d ago	118	data := make([]byte, size)
	119	if _, err := io.ReadFull(proc.out, data); err != nil {
	120	proc.dead = true
	121	return object{}, err
	122	}
	123	// cat-file terminates each object with one newline.
	124	if _, err := proc.out.Discard(1); err != nil {
	125	proc.dead = true
	126	return object{}, err
	127	}
	128	return object{SHA: fields[0], Type: fields[1], Data: data}, nil
	129	}
	130
	131	func (proc *catFileProc) start() error {
	132	cmd := exec.Command("git", "cat-file", "--batch")
	133	cmd.Dir = proc.dir
	134	cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
	135	in, err := cmd.StdinPipe()
	136	if err != nil {
	137	return err
	138	}
	139	out, err := cmd.StdoutPipe()
	140	if err != nil {
	141	return err
	142	}
	143	cmd.Stderr = io.Discard
	144	if err := cmd.Start(); err != nil {
	145	return err
	146	}
	147	proc.cmd, proc.in, proc.out, proc.dead = cmd, in, bufio.NewReaderSize(out, 64<<10), false
	148	return nil
	149	}
	150
	151	func (proc *catFileProc) close() {
	152	proc.mu.Lock()
	153	defer proc.mu.Unlock()
4581690d Keenan Nemetz 13d ago	154	proc.stop()
	155	}
	156
	157	// stop kills the process behind the stream, leaving it to restart lazily. The
	158	// caller holds proc.mu.
	159	func (proc *catFileProc) stop() {
d3296cda Keenan Nemetz 17d ago	160	if proc.cmd == nil {
4581690d Keenan Nemetz 13d ago	161	proc.dead = true
d3296cda Keenan Nemetz 17d ago	162	return
d3296cda Keenan Nemetz 17d ago	163	}
4581690d Keenan Nemetz 13d ago	164	if proc.in != nil {
	165	proc.in.Close()
	166	}
	167	if proc.cmd.Process != nil {
	168	proc.cmd.Process.Kill()
	169	}
d3296cda Keenan Nemetz 17d ago	170	proc.cmd.Wait()
	171	proc.cmd, proc.in, proc.out, proc.dead = nil, nil, nil, true
	172	}