vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Write only the part of a metadata tree that changed

e4be104b377627630497d737667c2854a9aa4310 · Keenan Nemetz · 11d ago · parent db5e4382

Write only the part of a metadata tree that changed

Filing an issue rewrote the whole vole branch one file at a time: FlatTree
started a git cat-file per blob, and WriteFlatTree hashed every blob and
rebuilt every directory, so a write grew with the repository rather than the
change. At 959 files a create took about 12 seconds, and every CI run added
seven files to that cost.

Read the tree with one cat-file --batch and an ls-tree that includes the tree
ids. Hash content in memory to find the paths that changed, then write only
those blobs and the directories above them, reusing the old blob and subtree
ids everywhere else. A one-file write on a 2,000-file tree now takes about
44 ms; a test counts the git processes so a full rewrite cannot come back.

core/git/trees.go +189−15

@@ -1,36 +1,62 @@
	1	1	package git
	2	2
	3	3	import (
		4	"crypto/sha1"
		5	"crypto/sha256"
		6	"encoding/hex"
	4	7	"fmt"
		8	"hash"
	5	9	"strings"
	6	10	)
	7	11
	8	12	// FlatTree reads every blob under commit as a path -> content map. Callers mutate
	9		// the map and hand it back to WriteFlatTree, which is the simplest way to express
		13	// the map and hand it back to CommitFiles, which is the simplest way to express
	10	14	// "update one file in a tree" without a worktree.
	11	15	func (r *Repo) FlatTree(commit string) (map[string][]byte, error) {
	12		out := map[string][]byte{}
		16	files, _, err := r.flatTree(commit)
		17	return files, err
		18	}
		19
		20	// flatTree reads the blobs and every entry, including directories, keyed by
		21	// path. A writer that starts from an existing commit needs the directory ids as
		22	// well as the content, because it reuses both for the parts it did not change.
		23	func (r *Repo) flatTree(commit string) (map[string][]byte, map[string]Entry, error) {
		24	files := map[string][]byte{}
		25	entries := map[string]Entry{}
	13	26	if commit == "" {
	14		return out, nil
		27	return files, entries, nil
	15	28	}
	16		entries, err := r.LsTreeR(commit)
		29	list, err := r.LsTreeRT(commit)
	17	30	if err != nil {
	18		return nil, err
		31	return nil, nil, err
	19	32	}
	20		for _, e := range entries {
		33	shas := make([]string, 0, len(list))
		34	for _, e := range list {
		35	entries[e.Name] = e
		36	if e.Type == "blob" {
		37	shas = append(shas, e.SHA)
		38	}
		39	}
		40	blobs, err := r.BlobsBySHA(shas)
		41	if err != nil {
		42	return nil, nil, err
		43	}
		44	for _, e := range list {
	21	45	if e.Type != "blob" {
	22		continue // submodules and symlink targets are not ours to rewrite
		46	continue
	23	47	}
	24		data, err := r.BlobBySHA(e.SHA)
	25		if err != nil {
	26		return nil, err
		48	data, ok := blobs[e.SHA]
		49	if !ok {
		50	return nil, nil, fmt.Errorf("flat tree: %s has no content for %s", e.Name, e.SHA)
	27	51	}
	28		out[e.Name] = data
		52	files[e.Name] = data
	29	53	}
	30		return out, nil
		54	return files, entries, nil
	31	55	}
	32	56
	33		// WriteFlatTree writes a nested tree from flat paths and returns its id.
		57	// WriteFlatTree writes a nested tree from flat paths and returns its id. It
		58	// writes every blob it is given, so it is for a tree that does not exist yet;
		59	// CommitFiles reuses the parts of an existing tree that did not change.
	34	60	func WriteFlatTree(g *Repo, files map[string][]byte) (string, error) {
	35	61	var entries []Entry
	36	62	dirs := map[string]map[string][]byte{}
@@ -63,6 +89,11 @@ func WriteFlatTree(g *Repo, files map[string][]byte) (string, error) {
	63	89	// compare-and-swap, retrying when another writer wins. It is the single write
	64	90	// path for metadata refs; a second implementation would eventually disagree
	65	91	// about ordering or retries, and that shows up as lost data.
		92	//
		93	// Only the files that changed are hashed, and only the directories on the way to
		94	// them are written again. A comment on a branch with ten thousand unrelated
		95	// files therefore costs one blob and one tree per path segment, not one process
		96	// per file in the repository.
	66	97	func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, message, author string) (string, error) {
	67	98	const maxAttempts = 64
	68	99	var lastErr error
@@ -71,14 +102,14 @@ func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, mess
	71	102	if err != nil {
	72	103	return "", err
	73	104	}
	74		files, err := g.FlatTree(old)
		105	files, entries, err := g.flatTree(old)
	75	106	if err != nil {
	76	107	return "", err
	77	108	}
	78	109	if err := mutate(files); err != nil {
	79	110	return "", err
	80	111	}
	81		tree, err := WriteFlatTree(g, files)
		112	tree, err := writeChangedTree(g, entries, files)
	82	113	if err != nil {
	83	114	return "", err
	84	115	}
@@ -97,3 +128,146 @@ func CommitFiles(g *Repo, ref string, mutate func(map[string][]byte) error, mess
	97	128	}
	98	129	return "", lastErr
	99	130	}
		131
		132	// writeChangedTree builds the tree for files, reusing the blob and subtree ids
		133	// in old for everything that still says the same thing.
		134	func writeChangedTree(g *Repo, old map[string]Entry, files map[string][]byte) (string, error) {
		135	sha256IDs := usesSHA256(old)
		136	changed := changedPaths(old, files, sha256IDs)
		137	return writeTree(g, buildTree(files), "", old, changed, changedDirectories(changed))
		138	}
		139
		140	// changedPaths reports the files a write adds, removes, or changes. Content is
		141	// hashed in memory rather than by a git process per file: the cost is the bytes
		142	// on the branch, while the number of files no longer decides it.
		143	func changedPaths(old map[string]Entry, files map[string][]byte, sha256IDs bool) map[string]bool {
		144	changed := make(map[string]bool)
		145	for path, content := range files {
		146	e, ok := old[path]
		147	if !ok \|\| e.Type != "blob" \|\| e.SHA != blobID(content, sha256IDs) {
		148	changed[path] = true
		149	}
		150	}
		151	for path, e := range old {
		152	if e.Type != "blob" {
		153	continue
		154	}
		155	if _, ok := files[path]; !ok {
		156	changed[path] = true
		157	}
		158	}
		159	return changed
		160	}
		161
		162	// blobID is the id git would give content as a blob, without writing it. It only
		163	// decides whether a path needs a new blob; every changed blob is still written by
		164	// git. Vole creates SHA-1 repositories, but a tree whose ids are 64 characters
		165	// says the repository is SHA-256 and is answered in kind.
		166	func blobID(content []byte, sha256IDs bool) string {
		167	var h hash.Hash
		168	if sha256IDs {
		169	h = sha256.New()
		170	} else {
		171	h = sha1.New()
		172	}
		173	fmt.Fprintf(h, "blob %d\x00", len(content))
		174	h.Write(content)
		175	return hex.EncodeToString(h.Sum(nil))
		176	}
		177
		178	// usesSHA256 reads the object format from an existing tree id. An empty tree has
		179	// nothing to say, and a repository created by Vole is SHA-1.
		180	func usesSHA256(entries map[string]Entry) bool {
		181	for _, e := range entries {
		182	return len(e.SHA) == sha256.Size*2
		183	}
		184	return false
		185	}
		186
		187	// treeNode is a directory while a flat path map is being turned back into trees.
		188	type treeNode struct {
		189	files map[string][]byte
		190	dirs map[string]*treeNode
		191	}
		192
		193	func buildTree(files map[string][]byte) *treeNode {
		194	root := &treeNode{files: map[string][]byte{}, dirs: map[string]*treeNode{}}
		195	for path, content := range files {
		196	parts := strings.Split(path, "/")
		197	node := root
		198	for _, part := range parts[:len(parts)-1] {
		199	next := node.dirs[part]
		200	if next == nil {
		201	next = &treeNode{files: map[string][]byte{}, dirs: map[string]*treeNode{}}
		202	node.dirs[part] = next
		203	}
		204	node = next
		205	}
		206	node.files[parts[len(parts)-1]] = content
		207	}
		208	return root
		209	}
		210
		211	// changedDirectories names every directory above a changed file, so the writer
		212	// knows which subtrees have to be rebuilt.
		213	func changedDirectories(changed map[string]bool) map[string]bool {
		214	dirs := map[string]bool{}
		215	for path := range changed {
		216	for {
		217	i := strings.LastIndexByte(path, '/')
		218	if i < 0 {
		219	break
		220	}
		221	path = path[:i]
		222	dirs[path] = true
		223	}
		224	}
		225	return dirs
		226	}
		227
		228	func writeTree(g Repo, node treeNode, prefix string, old map[string]Entry, changed, dirs map[string]bool) (string, error) {
		229	entries := make([]Entry, 0, len(node.files)+len(node.dirs))
		230	for name, content := range node.files {
		231	path := joinPath(prefix, name)
		232	mode := "100644"
		233	previous, had := old[path]
		234	if had && previous.Type == "blob" {
		235	mode = previous.Mode
		236	}
		237	if !changed[path] && had && previous.Type == "blob" {
		238	entries = append(entries, Entry{Mode: mode, Type: "blob", SHA: previous.SHA, Name: name})
		239	continue
		240	}
		241	blob, err := g.HashObject(content)
		242	if err != nil {
		243	return "", err
		244	}
		245	entries = append(entries, Entry{Mode: mode, Type: "blob", SHA: blob, Name: name})
		246	}
		247	for name, child := range node.dirs {
		248	path := joinPath(prefix, name)
		249	if !dirs[path] {
		250	if previous, ok := old[path]; ok && previous.Type == "tree" {
		251	entries = append(entries, Entry{Mode: previous.Mode, Type: "tree", SHA: previous.SHA, Name: name})
		252	continue
		253	}
		254	}
		255	sha, err := writeTree(g, child, path, old, changed, dirs)
		256	if err != nil {
		257	return "", err
		258	}
		259	mode := "040000"
		260	if previous, ok := old[path]; ok && previous.Type == "tree" {
		261	mode = previous.Mode
		262	}
		263	entries = append(entries, Entry{Mode: mode, Type: "tree", SHA: sha, Name: name})
		264	}
		265	return g.MkTree(entries)
		266	}
		267
		268	func joinPath(prefix, name string) string {
		269	if prefix == "" {
		270	return name
		271	}
		272	return prefix + "/" + name
		273	}