vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Metadata writers queue on the branch instead of racing for it

122cbe4f62bf87a941538450584ba1b20444c135 · Keenan Nemetz · 7d ago · parent 4739664b

Metadata writers queue on the branch instead of racing for it

Every write to refs/heads/vole read the whole branch, decided, and moved the
ref with a compare-and-swap, retrying up to 64 times with no backoff and no
coordination. Concurrent writers therefore did not queue: they each paid a full
read, lost, and paid it again. Measured on a 2,000-file branch, one write needed
one attempt alone and 21 with 32 writers, and throughput fell as writers were
added because the retries were the load. At 64 simultaneous writers the loop ran
out and the write failed for being unlucky.

The read-modify-write now runs with one writer at a time, holding a flock in the
repository so the SSH hook and the server queue behind each other. The
compare-and-swap stays for writers that go around the lock — a push, a hand-run
git command, another host — and losing one now waits with jittered backoff
before deciding again.

Writers read the branch once. The load and the decision happen inside the commit
instead of before it, which also stops an edit from replacing a change that
landed while the page was open: the decision is made against the tree it writes.

A push only does that work when it has metadata to move. Post-receive asks the
index which open requests follow the branch it moved, and a branch nobody is
proposing from now writes an audit line and no commit.

Measurements, 32 concurrent writers: 21.3 CAS attempts per write before, 1.0
after. Through the web, posting comments at concurrency 32: p99 4.7 s -> 0.95 s,
and throughput no longer falls as workers are added. The load harness gains a
`comment` scenario so the number is visible.

Closes: 01M6D9G8H6E0RCQ3AD5TMAGX34

core/git/trees.go +77−9

@@ -6,7 +6,9 @@ import (
	6	6	"encoding/hex"
	7	7	"fmt"
	8	8	"hash"
		9	"math/rand/v2"
	9	10	"strings"
		11	"time"
	10	12	)
	11	13
	12	14	// FlatTree reads every blob under commit as a path -> content map. Callers mutate
@@ -86,18 +88,57 @@ func WriteFlatTree(g *Repo, files map[string][]byte) (string, error) {
	86	88	}
	87	89
	88	90	// CommitFiles applies mutate to the tree behind ref and moves the ref with a
	89		// compare-and-swap, retrying when another writer wins. It is the single write
	90		// path for metadata refs; a second implementation would eventually disagree
	91		// about ordering or retries, and that shows up as lost data.
		91	// compare-and-swap, attributing the commit to author. It is the single write path
		92	// for metadata refs; a second implementation would eventually disagree about
		93	// ordering or retries, and that shows up as lost data.
		94	//
		95	// CommitFilesBy is the same write for a caller whose author is only known once the
		96	// tree has been read.
		97	func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, message, author string) (string, error) {
		98	return CommitFilesBy(g, ref, func(files map[string][]byte) (string, error) {
		99	return author, mutate(files)
		100	}, message)
		101	}
		102
		103	// CommitFilesBy applies mutate to the tree behind ref and moves the ref with a
		104	// compare-and-swap. mutate returns the name to record as the commit's author along
		105	// with its change, because a caller with no account name of its own — a library, or
		106	// an edit made offline against a clone — has only the author the file already names,
		107	// and that is the one thing a caller would otherwise read the branch twice to learn.
		108	// Both the author and the bytes are worked out again on a retry, which is what makes
		109	// losing a compare-and-swap harmless.
	92	110	//
	93	111	// Only the files that changed are hashed, and only the directories on the way to
	94	112	// them are written again. A comment on a branch with ten thousand unrelated
	95	113	// files therefore costs one blob and one tree per path segment, not one process
	96	114	// per file in the repository.
	97		func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, message, author string) (string, error) {
		115	//
		116	// Writers of the same ref take turns: the whole read-modify-write runs under
		117	// lockWrites, so a write that arrives while another is in flight waits instead of
		118	// reading a tree it is about to lose. The compare-and-swap and its retry are kept
		119	// for the writers that went around the lock — a push that moved the ref, a
		120	// hand-run git command, a second host — which is why losing one is still safe and
		121	// still finishes.
		122	//
		123	// mutate must be a decision about the tree it is handed, not a replay of one read
		124	// earlier, and it must not call CommitFiles: one writer holds the lock at a time.
		125	func CommitFilesBy(g *Repo, ref string, mutate func(map[string][]byte) (string, error), message string) (string, error) {
		126	unlock, err := g.lockWrites(ref)
		127	if err != nil {
		128	return "", err
		129	}
		130	defer unlock()
		131
	98	132	const maxAttempts = 64
	99		var lastErr error
	100		for attempt := 0; attempt < maxAttempts; attempt++ {
		133	for attempt := 1; attempt <= maxAttempts; attempt++ {
		134	if attempt > 1 {
		135	// The writer that beat us did not hold the lock, so waiting is the only
		136	// way to let it finish writing its objects. Jittered so the writers that
		137	// lost the same race do not line up again.
		138	if err := g.pause(backoff(attempt)); err != nil {
		139	return "", err
		140	}
		141	}
	101	142	old, err := g.RevParseRaw(ref)
	102	143	if err != nil {
	103	144	return "", err
@@ -106,7 +147,8 @@ func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, mess
	106	147	if err != nil {
	107	148	return "", err
	108	149	}
	109		if err := mutate(files); err != nil {
		150	author, err := mutate(files)
		151	if err != nil {
	110	152	return "", err
	111	153	}
	112	154	tree, err := writeChangedTree(g, entries, files)
@@ -124,9 +166,35 @@ func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, mess
	124	166	if ok {
	125	167	return commit, nil
	126	168	}
	127		lastErr = fmt.Errorf("%s: lost %d ref race(s)", ref, attempt+1)
	128	169	}
	129		return "", lastErr
		170	return "", fmt.Errorf("%s: gave up after %d attempts because another writer kept moving the branch; try again",
		171	ref, maxAttempts)
		172	}
		173
		174	// backoff is how long to wait after losing attempt n: it doubles from a
		175	// millisecond to a quarter of a second and is jittered by up to half its length,
		176	// so writers that were refused together do not all come back together.
		177	func backoff(attempt int) time.Duration {
		178	d := time.Millisecond << uint(min(attempt-2, 8))
		179	half := d / 2
		180	return half + time.Duration(rand.Int64N(int64(half)+1))
		181	}
		182
		183	// pause waits between attempts without ignoring a cancelled request: a page that
		184	// has gone away should not keep a writer waiting for a ref.
		185	func (r *Repo) pause(d time.Duration) error {
		186	if r.ctx == nil {
		187	time.Sleep(d)
		188	return nil
		189	}
		190	t := time.NewTimer(d)
		191	defer t.Stop()
		192	select {
		193	case <-t.C:
		194	return nil
		195	case <-r.ctx.Done():
		196	return r.ctx.Err()
		197	}
	130	198	}
	131	199
	132	200	// writeChangedTree builds the tree for files, reusing the blob and subtree ids