vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 5: LFS objects, over SSH for writes and the proxy for reads
64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf
Phase 5: LFS objects, over SSH for writes and the proxy for reads Large files leave the git object database and land in one content-addressed store per instance. Reads are anonymous like clone; writes ride the SSH key that already authenticates the push, so no token exists anywhere. - internal/lfs: sha256-addressed store that verifies content and size before an object becomes visible; parser for the pointer format a client writes - HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid> with the object id as ETag; upload over HTTP is refused and names SSH - SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so `git lfs push` needs nothing but the key at the handshake - pre-receive: a pointer whose object is missing, or whose size disagrees with the stored object, fails the push that introduced it; a binary over lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run before the protected-branch gate, because an admin's approval bypass is about review, not about whether the commit can be checked out - runs: a finished log becomes a content-addressed object in the same write that records the status, served at /r/<repo>/runs/<id>/log - vole gc [--dry-run] [--grace]: marks pointer targets from every ref in every repository plus archived logs, sweeps the rest. Mark-and-sweep rather than reference counts: a counter that is one too low deletes an artifact, a walk over the refs cannot be wrong about what is reachable
internal/hook/lfs.go +141
| @@ -0,0 +1,141 @@ | |||
| 1 | package hook | ||
| 2 | |||
| 3 | // LFS policy at the gate. | ||
| 4 | // | ||
| 5 | // Two different rules, because they answer two different mistakes: | ||
| 6 | // | ||
| 7 | // - A pointer whose object is missing is a broken repository. The checkout | ||
| 8 | // cannot be completed by anyone, so the push is refused, before the ref moves. | ||
| 9 | // - A large binary committed straight into git is a repository that will get | ||
| 10 | // slow, but it is not broken. The push is accepted and the user is told how | ||
| 11 | // to move it to LFS. | ||
| 12 | |||
| 13 | import ( | ||
| 14 | "fmt" | ||
| 15 | "io" | ||
| 16 | "sort" | ||
| 17 | "strings" | ||
| 18 | |||
| 19 | "github.com/rodent-software/vole/internal/gitx" | ||
| 20 | "github.com/rodent-software/vole/internal/lfs" | ||
| 21 | ) | ||
| 22 | |||
| 23 | // lfsFindings scans the blobs of the tree a pushed ref will serve. problems | ||
| 24 | // reject the push, warnings are printed and the push proceeds. | ||
| 25 | func lfsFindings(g *gitx.Repo, new string, store *lfs.Store, warnBytes int64) (problems, warnings []string, err error) { | ||
| 26 | if store == nil { | ||
| 27 | return nil, nil, fmt.Errorf("no LFS store configured") | ||
| 28 | } | ||
| 29 | blobs, err := g.TreeBlobs(new) | ||
| 30 | if err != nil { | ||
| 31 | return nil, nil, err | ||
| 32 | } | ||
| 33 | shas := make([]string, 0, len(blobs)) | ||
| 34 | for sha := range blobs { | ||
| 35 | shas = append(shas, sha) | ||
| 36 | } | ||
| 37 | sort.Strings(shas) // stable messages, so a test and a human see the same order | ||
| 38 | |||
| 39 | for _, sha := range shas { | ||
| 40 | b := blobs[sha] | ||
| 41 | if b.Size <= lfs.MaxPointerBytes { | ||
| 42 | head, err := g.BlobPrefix(sha, int(lfs.MaxPointerBytes)) | ||
| 43 | if err != nil { | ||
| 44 | return nil, nil, err | ||
| 45 | } | ||
| 46 | p, ok := lfs.ParsePointer(head) | ||
| 47 | if !ok { | ||
| 48 | continue | ||
| 49 | } | ||
| 50 | problems = append(problems, pointerProblem(store, p, b.Path)...) | ||
| 51 | continue | ||
| 52 | } | ||
| 53 | if warnBytes <= 0 || b.Size <= warnBytes { | ||
| 54 | continue | ||
| 55 | } | ||
| 56 | head, err := g.BlobPrefix(sha, 8000) | ||
| 57 | if err != nil { | ||
| 58 | return nil, nil, err | ||
| 59 | } | ||
| 60 | if !looksBinary(head) { | ||
| 61 | continue | ||
| 62 | } | ||
| 63 | warnings = append(warnings, fmt.Sprintf( | ||
| 64 | "%s is %s and stored in git. Track it with LFS instead:\n"+ | ||
| 65 | " git lfs track %q && git add .gitattributes %s\n"+ | ||
| 66 | " (a file already in history needs `git lfs migrate import --include=%q`)", | ||
| 67 | b.Path, humanSize(b.Size), trackPattern(b.Path), b.Path, trackPattern(b.Path))) | ||
| 68 | } | ||
| 69 | return problems, warnings, nil | ||
| 70 | } | ||
| 71 | |||
| 72 | func pointerProblem(store *lfs.Store, p lfs.Pointer, path string) []string { | ||
| 73 | size, err := store.Stat(p.OID) | ||
| 74 | if err != nil { | ||
| 75 | return []string{fmt.Sprintf( | ||
| 76 | "%s points at LFS object %s (%s), which is not on this server. "+ | ||
| 77 | "Upload it first: git lfs push origin <ref>", path, oidShort(p.OID), humanSize(p.Size))} | ||
| 78 | } | ||
| 79 | if p.Size > 0 && size != p.Size { | ||
| 80 | return []string{fmt.Sprintf( | ||
| 81 | "%s points at LFS object %s and says it is %d bytes, but the stored object is %d", | ||
| 82 | path, oidShort(p.OID), p.Size, size)} | ||
| 83 | } | ||
| 84 | return nil | ||
| 85 | } | ||
| 86 | |||
| 87 | // oidShort is longer than short(): when the message tells someone which object to | ||
| 88 | // upload, twelve hex characters are enough to find it in `git lfs ls-files` | ||
| 89 | // without being a wall of hex. | ||
| 90 | func oidShort(oid string) string { | ||
| 91 | if len(oid) > 12 { | ||
| 92 | return oid[:12] | ||
| 93 | } | ||
| 94 | return oid | ||
| 95 | } | ||
| 96 | |||
| 97 | // looksBinary is git's own heuristic, narrowed to "contains a NUL in the first | ||
| 98 | // 8 KiB". A file that could be a pointer was already handled: only blobs larger | ||
| 99 | // than the pointer cap reach here. | ||
| 100 | func looksBinary(head []byte) bool { | ||
| 101 | for _, c := range head { | ||
| 102 | if c == 0 { | ||
| 103 | return true | ||
| 104 | } | ||
| 105 | } | ||
| 106 | return false | ||
| 107 | } | ||
| 108 | |||
| 109 | // trackPattern suggests the narrowest useful gitattributes pattern: the | ||
| 110 | // extension when there is one, otherwise the path itself. | ||
| 111 | func trackPattern(path string) string { | ||
| 112 | base := path | ||
| 113 | if i := strings.LastIndexByte(base, '/'); i >= 0 { | ||
| 114 | base = base[i+1:] | ||
| 115 | } | ||
| 116 | if i := strings.LastIndexByte(base, '.'); i > 0 && i < len(base)-1 { | ||
| 117 | return "*" + base[i:] | ||
| 118 | } | ||
| 119 | return path | ||
| 120 | } | ||
| 121 | |||
| 122 | func humanSize(n int64) string { | ||
| 123 | const unit = 1024 | ||
| 124 | if n < unit { | ||
| 125 | return fmt.Sprintf("%d B", n) | ||
| 126 | } | ||
| 127 | div, exp := int64(unit), 0 | ||
| 128 | for m := n / unit; m >= unit; m /= unit { | ||
| 129 | div *= unit | ||
| 130 | exp++ | ||
| 131 | } | ||
| 132 | return fmt.Sprintf("%.1f %ciB", float64(n)/float64(div), "KMGTPE"[exp]) | ||
| 133 | } | ||
| 134 | |||
| 135 | // printWarnings writes findings the push accepts anyway. They go to the same | ||
| 136 | // stream as rejections, which git forwards to the client as remote: lines. | ||
| 137 | func printWarnings(w io.Writer, warnings []string) { | ||
| 138 | for _, warn := range warnings { | ||
| 139 | fmt.Fprintf(w, "vole: warning: %s\n", strings.TrimSpace(warn)) | ||
| 140 | } | ||
| 141 | } | ||