vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Metadata writers queue on the branch instead of racing for it
122cbe4f62bf87a941538450584ba1b20444c135 · Keenan Nemetz · 7d ago · parent 4739664b
Metadata writers queue on the branch instead of racing for it Every write to refs/heads/vole read the whole branch, decided, and moved the ref with a compare-and-swap, retrying up to 64 times with no backoff and no coordination. Concurrent writers therefore did not queue: they each paid a full read, lost, and paid it again. Measured on a 2,000-file branch, one write needed one attempt alone and 21 with 32 writers, and throughput fell as writers were added because the retries were the load. At 64 simultaneous writers the loop ran out and the write failed for being unlucky. The read-modify-write now runs with one writer at a time, holding a flock in the repository so the SSH hook and the server queue behind each other. The compare-and-swap stays for writers that go around the lock — a push, a hand-run git command, another host — and losing one now waits with jittered backoff before deciding again. Writers read the branch once. The load and the decision happen inside the commit instead of before it, which also stops an edit from replacing a change that landed while the page was open: the decision is made against the tree it writes. A push only does that work when it has metadata to move. Post-receive asks the index which open requests follow the branch it moved, and a branch nobody is proposing from now writes an audit line and no commit. Measurements, 32 concurrent writers: 21.3 CAS attempts per write before, 1.0 after. Through the web, posting comments at concurrency 32: p99 4.7 s -> 0.95 s, and throughput no longer falls as workers are added. The load harness gains a `comment` scenario so the number is visible. Closes: 01M6D9G8H6E0RCQ3AD5TMAGX34
core/git/lock_test.go +196
| @@ -0,0 +1,196 @@ | |||
| 1 | package git | ||
| 2 | |||
| 3 | // Writers of one ref take turns. These tests hold the two things that makes true: | ||
| 4 | // concurrent writes all land, and they land without each writer re-reading the | ||
| 5 | // tree it is about to lose. The attempt count is read from GIT_TRACE, the same | ||
| 6 | // way TestCommitFilesStaysProportionalToTheChange counts processes: every | ||
| 7 | // `update-ref` is one attempt, won or lost. | ||
| 8 | |||
| 9 | import ( | ||
| 10 | "bytes" | ||
| 11 | "fmt" | ||
| 12 | "os" | ||
| 13 | "path/filepath" | ||
| 14 | "strings" | ||
| 15 | "sync" | ||
| 16 | "testing" | ||
| 17 | ) | ||
| 18 | |||
| 19 | func lockRepo(t *testing.T) (*Repo, string) { | ||
| 20 | t.Helper() | ||
| 21 | dir := filepath.Join(t.TempDir(), "r.git") | ||
| 22 | if err := InitBare(dir, "main"); err != nil { | ||
| 23 | t.Fatal(err) | ||
| 24 | } | ||
| 25 | g := Open(dir) | ||
| 26 | ref := "refs/heads/main" | ||
| 27 | if _, err := CommitFiles(g, ref, func(files map[string][]byte) error { | ||
| 28 | // Enough unrelated content that a full read is worth losing, which is what | ||
| 29 | // the retry used to pay for. | ||
| 30 | for i := 0; i < 200; i++ { | ||
| 31 | files[fmt.Sprintf("issues/%03d/issue.md", i)] = []byte("seed\n") | ||
| 32 | } | ||
| 33 | return nil | ||
| 34 | }, "seed", "test"); err != nil { | ||
| 35 | t.Fatal(err) | ||
| 36 | } | ||
| 37 | return g, ref | ||
| 38 | } | ||
| 39 | |||
| 40 | // TestCommitFilesSerializesWriters is the regression test for the retry storm: with | ||
| 41 | // writers taking turns, every write needs exactly one attempt, and every write is in | ||
| 42 | // the tree afterwards. | ||
| 43 | func TestCommitFilesSerializesWriters(t *testing.T) { | ||
| 44 | g, ref := lockRepo(t) | ||
| 45 | const writers, each = 8, 4 | ||
| 46 | const total = writers * each | ||
| 47 | |||
| 48 | trace := filepath.Join(t.TempDir(), "git.trace") | ||
| 49 | traced := *g | ||
| 50 | traced.ExtraEnv = append(traced.ExtraEnv, "GIT_TRACE="+trace) | ||
| 51 | |||
| 52 | var wg sync.WaitGroup | ||
| 53 | errs := make(chan error, total) | ||
| 54 | for w := 0; w < writers; w++ { | ||
| 55 | wg.Add(1) | ||
| 56 | go func(w int) { | ||
| 57 | defer wg.Done() | ||
| 58 | for i := 0; i < each; i++ { | ||
| 59 | path := fmt.Sprintf("issues/w%02d-%d/issue.md", w, i) | ||
| 60 | if _, err := CommitFiles(&traced, ref, func(files map[string][]byte) error { | ||
| 61 | files[path] = []byte(fmt.Sprintf("w%d i%d\n", w, i)) | ||
| 62 | return nil | ||
| 63 | }, "write", "test"); err != nil { | ||
| 64 | errs <- err | ||
| 65 | return | ||
| 66 | } | ||
| 67 | } | ||
| 68 | }(w) | ||
| 69 | } | ||
| 70 | wg.Wait() | ||
| 71 | close(errs) | ||
| 72 | for err := range errs { | ||
| 73 | t.Fatal(err) | ||
| 74 | } | ||
| 75 | |||
| 76 | data, err := os.ReadFile(trace) | ||
| 77 | if err != nil { | ||
| 78 | t.Fatal(err) | ||
| 79 | } | ||
| 80 | attempts := bytes.Count(data, []byte("update-ref")) | ||
| 81 | if attempts > total { | ||
| 82 | t.Fatalf("%d writes from %d goroutines took %d attempts: writers are not taking turns", | ||
| 83 | total, writers, attempts) | ||
| 84 | } | ||
| 85 | |||
| 86 | // Nothing was lost on the way. | ||
| 87 | files, err := g.FlatTree(ref) | ||
| 88 | if err != nil { | ||
| 89 | t.Fatal(err) | ||
| 90 | } | ||
| 91 | for w := 0; w < writers; w++ { | ||
| 92 | for i := 0; i < each; i++ { | ||
| 93 | path := fmt.Sprintf("issues/w%02d-%d/issue.md", w, i) | ||
| 94 | want := fmt.Sprintf("w%d i%d\n", w, i) | ||
| 95 | if got := string(files[path]); got != want { | ||
| 96 | t.Fatalf("%s = %q, want %q", path, got, want) | ||
| 97 | } | ||
| 98 | } | ||
| 99 | } | ||
| 100 | } | ||
| 101 | |||
| 102 | // A write that fails still gives the lock back, or the branch would be stuck for | ||
| 103 | // every writer after it. | ||
| 104 | func TestCommitFilesReleasesTheLockAfterAFailure(t *testing.T) { | ||
| 105 | g, ref := lockRepo(t) | ||
| 106 | _, err := CommitFiles(g, ref, func(files map[string][]byte) error { | ||
| 107 | return fmt.Errorf("nothing to do") | ||
| 108 | }, "failed", "test") | ||
| 109 | if err == nil { | ||
| 110 | t.Fatal("a failing mutation was reported as a commit") | ||
| 111 | } | ||
| 112 | if _, err := CommitFiles(g, ref, func(files map[string][]byte) error { | ||
| 113 | files["issues/after/issue.md"] = []byte("after\n") | ||
| 114 | return nil | ||
| 115 | }, "after", "test"); err != nil { | ||
| 116 | t.Fatalf("the write after a failure waited forever or failed: %v", err) | ||
| 117 | } | ||
| 118 | } | ||
| 119 | |||
| 120 | // Two refs are two queues: a slow write on one must not hold up the other, which is | ||
| 121 | // what makes the lock per-ref rather than per-repository. | ||
| 122 | func TestCommitFilesLocksEachRefSeparately(t *testing.T) { | ||
| 123 | g, ref := lockRepo(t) | ||
| 124 | other := "refs/heads/pages" | ||
| 125 | started := make(chan struct{}) | ||
| 126 | release := make(chan struct{}) | ||
| 127 | done := make(chan error, 1) | ||
| 128 | go func() { | ||
| 129 | _, err := CommitFiles(g, ref, func(files map[string][]byte) error { | ||
| 130 | close(started) | ||
| 131 | <-release | ||
| 132 | files["issues/slow/issue.md"] = []byte("slow\n") | ||
| 133 | return nil | ||
| 134 | }, "slow", "test") | ||
| 135 | done <- err | ||
| 136 | }() | ||
| 137 | <-started | ||
| 138 | if _, err := CommitFiles(g, other, func(files map[string][]byte) error { | ||
| 139 | files["index.html"] = []byte("hello\n") | ||
| 140 | return nil | ||
| 141 | }, "pages", "test"); err != nil { | ||
| 142 | t.Fatalf("a write to another ref was held up: %v", err) | ||
| 143 | } | ||
| 144 | close(release) | ||
| 145 | if err := <-done; err != nil { | ||
| 146 | t.Fatal(err) | ||
| 147 | } | ||
| 148 | } | ||
| 149 | |||
| 150 | // The retry loop is still there for a writer that went around the lock, and still | ||
| 151 | // finishes. A ref moved by hand between the read and the compare-and-swap is the | ||
| 152 | // only way to reach it once writers take turns. | ||
| 153 | func TestCommitFilesStillRetriesARefMovedByHand(t *testing.T) { | ||
| 154 | g, ref := lockRepo(t) | ||
| 155 | before, err := g.RevParse(ref) | ||
| 156 | if err != nil { | ||
| 157 | t.Fatal(err) | ||
| 158 | } | ||
| 159 | // A commit that does not move the tree, so a write that rebases onto it is | ||
| 160 | // still the write the caller asked for. | ||
| 161 | out, err := g.Run(nil, "rev-parse", "--verify", before+"^{tree}") | ||
| 162 | if err != nil { | ||
| 163 | t.Fatal(err) | ||
| 164 | } | ||
| 165 | tree := strings.TrimSpace(string(out)) | ||
| 166 | side, err := g.CommitTree(tree, before, "hand") | ||
| 167 | if err != nil { | ||
| 168 | t.Fatal(err) | ||
| 169 | } | ||
| 170 | moved := false | ||
| 171 | _, err = CommitFiles(g, ref, func(files map[string][]byte) error { | ||
| 172 | if !moved { | ||
| 173 | // Stand in for `git update-ref` run by someone else: the lock cannot see | ||
| 174 | // it, so the first compare-and-swap must lose. | ||
| 175 | moved = true | ||
| 176 | if err := g.UpdateRef(ref, side); err != nil { | ||
| 177 | return err | ||
| 178 | } | ||
| 179 | } | ||
| 180 | files["issues/racing/issue.md"] = []byte("racing\n") | ||
| 181 | return nil | ||
| 182 | }, "write", "test") | ||
| 183 | if err != nil { | ||
| 184 | t.Fatalf("a write that lost one compare-and-swap gave up: %v", err) | ||
| 185 | } | ||
| 186 | files, err := g.FlatTree(ref) | ||
| 187 | if err != nil { | ||
| 188 | t.Fatal(err) | ||
| 189 | } | ||
| 190 | if string(files["issues/racing/issue.md"]) != "racing\n" { | ||
| 191 | t.Fatal("the write that lost the race is not in the tree") | ||
| 192 | } | ||
| 193 | if head, err := g.RevParse(ref); err != nil || head == side { | ||
| 194 | t.Fatalf("the branch is still where the hand edit left it: %s, %v", head, err) | ||
| 195 | } | ||
| 196 | } | ||