vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Write only the part of a metadata tree that changed
e4be104b377627630497d737667c2854a9aa4310 · Keenan Nemetz · 11d ago · parent db5e4382
Write only the part of a metadata tree that changed Filing an issue rewrote the whole vole branch one file at a time: FlatTree started a git cat-file per blob, and WriteFlatTree hashed every blob and rebuilt every directory, so a write grew with the repository rather than the change. At 959 files a create took about 12 seconds, and every CI run added seven files to that cost. Read the tree with one cat-file --batch and an ls-tree that includes the tree ids. Hash content in memory to find the paths that changed, then write only those blobs and the directories above them, reusing the old blob and subtree ids everywhere else. A one-file write on a 2,000-file tree now takes about 44 ms; a test counts the git processes so a full rewrite cannot come back.
core/git/git.go +75−22
| @@ -195,33 +195,33 @@ func (r *Repo) Exists() bool { | |||
| 195 | 195 | ||
| 196 | 196 | // LsTree lists the entries at treeish (a commit, a tree, or commit:path). | |
| 197 | 197 | func (r *Repo) LsTree(treeish string) ([]Entry, error) { | |
| 198 | out, errs, err := r.run(nil, "ls-tree", "-z", treeish) | ||
| 199 | if err != nil { | ||
| 200 | return nil, fmt.Errorf("ls-tree %q: %w: %s", treeish, err, errs) | ||
| 201 | } | ||
| 202 | var es []Entry | ||
| 203 | for _, rec := range bytes.Split(out, []byte{0}) { | ||
| 204 | if len(rec) == 0 { | ||
| 205 | continue | ||
| 206 | } | ||
| 207 | tab := bytes.IndexByte(rec, '\t') | ||
| 208 | if tab < 0 { | ||
| 209 | return nil, fmt.Errorf("bad ls-tree record %q", rec) | ||
| 210 | } | ||
| 211 | f := strings.Fields(string(rec[:tab])) | ||
| 212 | if len(f) != 3 { | ||
| 213 | return nil, fmt.Errorf("bad ls-tree fields %q", rec) | ||
| 214 | } | ||
| 215 | es = append(es, Entry{Mode: f[0], Type: f[1], SHA: f[2], Name: string(rec[tab+1:])}) | ||
| 216 | } | ||
| 217 | return es, nil | ||
| 198 | return r.lsTree(false, false, treeish) | ||
| 218 | 199 | } | |
| 219 | 200 | ||
| 220 | 201 | // LsTreeR lists every blob under treeish recursively, with full paths. | |
| 221 | 202 | func (r *Repo) LsTreeR(treeish string) ([]Entry, error) { | |
| 222 | out, errs, err := r.run(nil, "ls-tree", "-r", "-z", treeish) | ||
| 203 | return r.lsTree(true, false, treeish) | ||
| 204 | } | ||
| 205 | |||
| 206 | // LsTreeRT lists every entry under treeish recursively, including the trees | ||
| 207 | // themselves, with full paths. A writer uses it to find the subtree ids it can | ||
| 208 | // reuse; a reader that only wants files uses LsTreeR. | ||
| 209 | func (r *Repo) LsTreeRT(treeish string) ([]Entry, error) { | ||
| 210 | return r.lsTree(true, true, treeish) | ||
| 211 | } | ||
| 212 | |||
| 213 | func (r *Repo) lsTree(recursive, withTrees bool, treeish string) ([]Entry, error) { | ||
| 214 | args := []string{"ls-tree"} | ||
| 215 | if recursive { | ||
| 216 | args = append(args, "-r") | ||
| 217 | } | ||
| 218 | if withTrees { | ||
| 219 | args = append(args, "-t") | ||
| 220 | } | ||
| 221 | args = append(args, "-z", treeish) | ||
| 222 | out, errs, err := r.run(nil, args...) | ||
| 223 | 223 | if err != nil { | |
| 224 | return nil, fmt.Errorf("ls-tree -r %q: %w: %s", treeish, err, errs) | ||
| 224 | return nil, fmt.Errorf("ls-tree %q: %w: %s", treeish, err, errs) | ||
| 225 | 225 | } | |
| 226 | 226 | var es []Entry | |
| 227 | 227 | for _, rec := range bytes.Split(out, []byte{0}) { | |
| @@ -250,6 +250,59 @@ func (r *Repo) BlobBySHA(sha string) ([]byte, error) { | |||
| 250 | 250 | return out, nil | |
| 251 | 251 | } | |
| 252 | 252 | ||
| 253 | // BlobsBySHA returns the contents of every named blob in one git call. A caller | ||
| 254 | // that is about to read a whole tree uses it so the cost is one process, not one | ||
| 255 | // per file. The result is keyed by the object id git reports, which is the id | ||
| 256 | // that was asked for. | ||
| 257 | func (r *Repo) BlobsBySHA(shas []string) (map[string][]byte, error) { | ||
| 258 | if len(shas) == 0 { | ||
| 259 | return map[string][]byte{}, nil | ||
| 260 | } | ||
| 261 | var in bytes.Buffer | ||
| 262 | for _, sha := range shas { | ||
| 263 | if strings.ContainsAny(sha, "\n\r\x00\t ") { | ||
| 264 | return nil, fmt.Errorf("cat-file: malformed object id %q", sha) | ||
| 265 | } | ||
| 266 | in.WriteString(sha + "\n") | ||
| 267 | } | ||
| 268 | out, errs, err := r.run(in.Bytes(), "cat-file", "--batch") | ||
| 269 | if err != nil { | ||
| 270 | return nil, fmt.Errorf("cat-file --batch: %w: %s", err, errs) | ||
| 271 | } | ||
| 272 | blobs := make(map[string][]byte, len(shas)) | ||
| 273 | // The contents are sliced out of out rather than copied, so the map shares | ||
| 274 | // one buffer with the read: a large tree costs one copy, not two. | ||
| 275 | rest := out | ||
| 276 | for len(rest) > 0 { | ||
| 277 | i := bytes.IndexByte(rest, '\n') | ||
| 278 | if i < 0 { | ||
| 279 | return nil, fmt.Errorf("cat-file: unterminated object header") | ||
| 280 | } | ||
| 281 | header := string(rest[:i]) | ||
| 282 | rest = rest[i+1:] | ||
| 283 | fields := strings.Fields(header) | ||
| 284 | if len(fields) == 2 && fields[1] == "missing" { | ||
| 285 | return nil, fmt.Errorf("cat-file: %s is missing", fields[0]) | ||
| 286 | } | ||
| 287 | if len(fields) != 3 { | ||
| 288 | return nil, fmt.Errorf("cat-file: bad object header %q", header) | ||
| 289 | } | ||
| 290 | size, err := strconv.ParseInt(fields[2], 10, 64) | ||
| 291 | if err != nil || size < 0 || int64(len(rest)) < size+1 { | ||
| 292 | return nil, fmt.Errorf("cat-file: bad object size %q", fields[2]) | ||
| 293 | } | ||
| 294 | data := rest[:size] | ||
| 295 | // Every object is followed by one newline. It is framing, not content: a | ||
| 296 | // blob that does not end in a newline still gets one here. | ||
| 297 | if rest[size] != '\n' { | ||
| 298 | return nil, fmt.Errorf("cat-file: object %s is not newline-terminated", fields[0]) | ||
| 299 | } | ||
| 300 | blobs[fields[0]] = data | ||
| 301 | rest = rest[size+1:] | ||
| 302 | } | ||
| 303 | return blobs, nil | ||
| 304 | } | ||
| 305 | |||
| 253 | 306 | // SetConfig writes a repo-local git config value. | |
| 254 | 307 | func (r *Repo) SetConfig(key, value string) error { | |
| 255 | 308 | _, err := r.Run(nil, "config", key, value) | |