vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Write only the part of a metadata tree that changed

e4be104b377627630497d737667c2854a9aa4310 · Keenan Nemetz · 11d ago · parent db5e4382

Write only the part of a metadata tree that changed

Filing an issue rewrote the whole vole branch one file at a time: FlatTree
started a git cat-file per blob, and WriteFlatTree hashed every blob and
rebuilt every directory, so a write grew with the repository rather than the
change. At 959 files a create took about 12 seconds, and every CI run added
seven files to that cost.

Read the tree with one cat-file --batch and an ls-tree that includes the tree
ids. Hash content in memory to find the paths that changed, then write only
those blobs and the directories above them, reusing the old blob and subtree
ids everywhere else. A one-file write on a 2,000-file tree now takes about
44 ms; a test counts the git processes so a full rewrite cannot come back.

core/git/trees_test.go +278

@@ -0,0 +1,278 @@
	1	package git
	2
	3	import (
	4	"bytes"
	5	"fmt"
	6	"os"
	7	"path/filepath"
	8	"strings"
	9	"testing"
	10	)
	11
	12	// TestBlobIDMatchesGit pins the one piece of object formatting this package does
	13	// itself. It only decides whether a blob needs writing; git still writes it.
	14	func TestBlobIDMatchesGit(t *testing.T) {
	15	g := blobRepo(t, nil)
	16	for _, content := range [][]byte{
	17	nil,
	18	{},
	19	[]byte("hello\n"),
	20	{0, 1, 2, 255, '\n', 0},
	21	bytes.Repeat([]byte("0123456789abcdef"), 4096),
	22	} {
	23	want, err := g.HashObject(content)
	24	if err != nil {
	25	t.Fatal(err)
	26	}
	27	if got := blobID(content, false); got != want {
	28	t.Fatalf("blobID(%d bytes) = %s, want %s", len(content), got, want)
	29	}
	30	}
	31	}
	32
	33	// TestBlobsBySHABinary pins the batch framing: content may contain newlines and
	34	// NULs, an empty blob still has a header, and asking for the same object twice
	35	// answers twice.
	36	func TestBlobsBySHABinary(t *testing.T) {
	37	files := map[string][]byte{
	38	"empty": {},
	39	"nul": {0, '\n', 0, 255},
	40	"newlines": []byte("\n\n\n"),
	41	"text": []byte("no trailing newline"),
	42	}
	43	g := blobRepo(t, files)
	44	head, err := g.RevParse("refs/heads/main")
	45	if err != nil {
	46	t.Fatal(err)
	47	}
	48	entries, err := g.LsTree(head)
	49	if err != nil {
	50	t.Fatal(err)
	51	}
	52	shas := make([]string, 0, len(entries))
	53	for _, e := range entries {
	54	shas = append(shas, e.SHA)
	55	}
	56	blobs, err := g.BlobsBySHA(append(shas, shas...))
	57	if err != nil {
	58	t.Fatal(err)
	59	}
	60	if len(blobs) != len(entries) {
	61	t.Fatalf("BlobsBySHA returned %d objects, want %d", len(blobs), len(entries))
	62	}
	63	for _, e := range entries {
	64	if !bytes.Equal(blobs[e.SHA], files[e.Name]) {
	65	t.Fatalf("%s = %q, want %q", e.Name, blobs[e.SHA], files[e.Name])
	66	}
	67	}
	68	}
	69
	70	func TestUsesSHA256(t *testing.T) {
	71	if usesSHA256(nil) {
	72	t.Fatal("an empty tree claimed SHA-256")
	73	}
	74	if usesSHA256(map[string]Entry{"a": {SHA: strings.Repeat("a", 40)}}) {
	75	t.Fatal("a 40-character id claimed SHA-256")
	76	}
	77	if !usesSHA256(map[string]Entry{"a": {SHA: strings.Repeat("a", 64)}}) {
	78	t.Fatal("a 64-character id did not claim SHA-256")
	79	}
	80	}
	81
	82	// TestCommitFilesIncrementalEdits checks that a change, an add, and a delete
	83	// produce the tree the caller asked for while leaving unrelated subtrees alone.
	84	func TestCommitFilesIncrementalEdits(t *testing.T) {
	85	dir := filepath.Join(t.TempDir(), "r.git")
	86	if err := InitBare(dir, "main"); err != nil {
	87	t.Fatal(err)
	88	}
	89	g := Open(dir)
	90	ref := "refs/heads/main"
	91	if _, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	92	files["top.txt"] = []byte("top\n")
	93	files["stable/one.txt"] = []byte("one\n")
	94	files["dir/keep.txt"] = []byte("keep\n")
	95	files["dir/change.txt"] = []byte("before\n")
	96	files["dir/sub/gone.txt"] = []byte("gone\n")
	97	return nil
	98	}, "seed", "test"); err != nil {
	99	t.Fatal(err)
	100	}
	101	before, err := g.RevParse(ref)
	102	if err != nil {
	103	t.Fatal(err)
	104	}
	105	beforeTree := treeID(t, g, before)
	106	beforeRoot, err := g.LsTree(before)
	107	if err != nil {
	108	t.Fatal(err)
	109	}
	110
	111	after, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	112	files["dir/change.txt"] = []byte("after\n")
	113	files["dir/new.txt"] = []byte("new\n")
	114	delete(files, "dir/sub/gone.txt")
	115	return nil
	116	}, "edit", "test")
	117	if err != nil {
	118	t.Fatal(err)
	119	}
	120	afterRoot, err := g.LsTree(after)
	121	if err != nil {
	122	t.Fatal(err)
	123	}
	124	root := func(entries []Entry) map[string]Entry {
	125	out := map[string]Entry{}
	126	for _, e := range entries {
	127	out[e.Name] = e
	128	}
	129	return out
	130	}
	131	if got, want := root(afterRoot)["stable"].SHA, root(beforeRoot)["stable"].SHA; got != want {
	132	t.Fatalf("unchanged subtree was rewritten: %s, want %s", got, want)
	133	}
	134	if root(afterRoot)["dir"].SHA == root(beforeRoot)["dir"].SHA {
	135	t.Fatal("the changed subtree kept its old id")
	136	}
	137
	138	files, err := g.FlatTree(after)
	139	if err != nil {
	140	t.Fatal(err)
	141	}
	142	want := map[string]string{
	143	"top.txt": "top\n",
	144	"stable/one.txt": "one\n",
	145	"dir/keep.txt": "keep\n",
	146	"dir/change.txt": "after\n",
	147	"dir/new.txt": "new\n",
	148	}
	149	if len(files) != len(want) {
	150	t.Fatalf("tree has %d files, want %d: %v", len(files), len(want), files)
	151	}
	152	for path, content := range want {
	153	if got := string(files[path]); got != content {
	154	t.Fatalf("%s = %q, want %q", path, got, content)
	155	}
	156	}
	157	if _, ok := files["dir/sub/gone.txt"]; ok {
	158	t.Fatal("the deleted file is still in the tree")
	159	}
	160	afterTree := treeID(t, g, after)
	161	if afterTree == beforeTree {
	162	t.Fatal("the root tree did not change")
	163	}
	164	}
	165
	166	// TestCommitFilesReusesTreeWhenNothingChanged guards the reuse path: a mutator
	167	// that writes the same bytes must not move the tree.
	168	func TestCommitFilesReusesTreeWhenNothingChanged(t *testing.T) {
	169	dir := filepath.Join(t.TempDir(), "r.git")
	170	if err := InitBare(dir, "main"); err != nil {
	171	t.Fatal(err)
	172	}
	173	g := Open(dir)
	174	ref := "refs/heads/main"
	175	if _, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	176	files["a.txt"] = []byte("same\n")
	177	files["dir/b.txt"] = []byte("same\n")
	178	return nil
	179	}, "seed", "test"); err != nil {
	180	t.Fatal(err)
	181	}
	182	before := treeID(t, g, ref)
	183	commit, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	184	files["a.txt"] = []byte("same\n")
	185	files["dir/b.txt"] = []byte("same\n")
	186	return nil
	187	}, "no-op", "test")
	188	if err != nil {
	189	t.Fatal(err)
	190	}
	191	after := treeID(t, g, commit)
	192	if after != before {
	193	t.Fatalf("tree moved from %s to %s without a content change", before, after)
	194	}
	195	}
	196
	197	// TestCommitFilesStaysProportionalToTheChange is the regression test for the
	198	// thousands of git processes a one-file write used to start. GIT_TRACE counts
	199	// every child git, and a changed file at depth two needs a bounded handful.
	200	func TestCommitFilesStaysProportionalToTheChange(t *testing.T) {
	201	dir := filepath.Join(t.TempDir(), "r.git")
	202	if err := InitBare(dir, "main"); err != nil {
	203	t.Fatal(err)
	204	}
	205	g := Open(dir)
	206	ref := "refs/heads/main"
	207	seed := map[string][]byte{}
	208	for i := 0; i < 200; i++ {
	209	seed[fmt.Sprintf("runs/r%03d/1/record.md", i)] = []byte("record\n")
	210	}
	211	seed["issues/change/issue.md"] = []byte("before\n")
	212	if _, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	213	for path, content := range seed {
	214	files[path] = content
	215	}
	216	return nil
	217	}, "seed", "test"); err != nil {
	218	t.Fatal(err)
	219	}
	220
	221	trace := filepath.Join(t.TempDir(), "git.trace")
	222	traced := *g
	223	traced.ExtraEnv = append(traced.ExtraEnv, "GIT_TRACE="+trace)
	224	if _, err := CommitFiles(&traced, ref, func(files map[string][]byte) error {
	225	files["issues/change/issue.md"] = []byte("after\n")
	226	return nil
	227	}, "change", "test"); err != nil {
	228	t.Fatal(err)
	229	}
	230	data, err := os.ReadFile(trace)
	231	if err != nil {
	232	t.Fatal(err)
	233	}
	234	// rev-parse, ls-tree, one cat-file --batch, one hash-object, three mktree
	235	// (issues/change, issues, root), commit-tree, update-ref. A full rewrite
	236	// starts one git per file and per directory: over 400 here.
	237	const limit = 20
	238	if starts := bytes.Count(data, []byte("built-in: git ")); starts > limit {
	239	t.Fatalf("one changed file started %d git processes (limit %d):\n%s", starts, limit, data)
	240	}
	241	}
	242
	243	func treeID(t testing.T, g Repo, rev string) string {
	244	t.Helper()
	245	out, err := g.Run(nil, "rev-parse", "--verify", rev+"^{tree}")
	246	if err != nil {
	247	t.Fatal(err)
	248	}
	249	return strings.TrimSpace(string(out))
	250	}
	251
	252	func BenchmarkCommitFilesOneFile(b *testing.B) {
	253	dir := filepath.Join(b.TempDir(), "r.git")
	254	if err := InitBare(dir, "main"); err != nil {
	255	b.Fatal(err)
	256	}
	257	g := Open(dir)
	258	ref := "refs/heads/main"
	259	const unrelated = 2000
	260	if _, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	261	for i := 0; i < unrelated; i++ {
	262	files[fmt.Sprintf("runs/r%04d/1/record.md", i)] = []byte("record\n")
	263	}
	264	return nil
	265	}, "seed", "bench"); err != nil {
	266	b.Fatal(err)
	267	}
	268	b.ResetTimer()
	269	for i := 0; i < b.N; i++ {
	270	content := []byte(fmt.Sprintf("change %d\n", i))
	271	if _, err := CommitFiles(g, ref, func(files map[string][]byte) error {
	272	files["issues/one/issue.md"] = content
	273	return nil
	274	}, "change", "bench"); err != nil {
	275	b.Fatal(err)
	276	}
	277	}
	278	}