vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 17d ago · parent 35eaeccf

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/gitx/gitx.go +150−14

@@ -5,7 +5,9 @@ package gitx
	5	5
	6	6	import (
	7	7	"bytes"
		8	"errors"
	8	9	"fmt"
		10	"io"
	9	11	"os"
	10	12	"os/exec"
	11	13	"sort"
@@ -42,14 +44,7 @@ func (r *Repo) run(stdin []byte, args ...string) ([]byte, string, error) {
	42	44	// carry an author without mutating process state. os/exec keeps the last value
	43	45	// for a duplicated key, which is what makes the override reliable.
	44	46	func (r *Repo) runEnv(extra []string, stdin []byte, args ...string) ([]byte, string, error) {
	45		cmd := exec.Command("git", args...)
	46		cmd.Env = append(scrubbedEnv(),
	47		"GIT_DIR="+r.Dir,
	48		"GIT_TERMINAL_PROMPT=0",
	49		"GIT_AUTHOR_NAME=vole", "GIT_AUTHOR_EMAIL=vole@localhost",
	50		"GIT_COMMITTER_NAME=vole", "GIT_COMMITTER_EMAIL=vole@localhost",
	51		)
	52		cmd.Env = append(cmd.Env, r.ExtraEnv...)
		47	cmd := r.command(args...)
	53	48	cmd.Env = append(cmd.Env, extra...)
	54	49	if stdin != nil {
	55	50	cmd.Stdin = bytes.NewReader(stdin)
@@ -70,6 +65,50 @@ func (r *Repo) Run(stdin []byte, args ...string) ([]byte, error) {
	70	65	return out, nil
	71	66	}
	72	67
		68	// command builds a git invocation with the environment every call shares.
		69	func (r Repo) command(args ...string) exec.Cmd {
		70	cmd := exec.Command("git", args...)
		71	cmd.Env = append(scrubbedEnv(),
		72	"GIT_DIR="+r.Dir,
		73	"GIT_TERMINAL_PROMPT=0",
		74	"GIT_AUTHOR_NAME=vole", "GIT_AUTHOR_EMAIL=vole@localhost",
		75	"GIT_COMMITTER_NAME=vole", "GIT_COMMITTER_EMAIL=vole@localhost",
		76	)
		77	cmd.Env = append(cmd.Env, r.ExtraEnv...)
		78	return cmd
		79	}
		80
		81	// runHead returns at most n bytes of a git command's stdout. `cat-file blob`
		82	// writes the whole object, and the caller only wants to sniff the first bytes of
		83	// a file that is about to be accepted, so the process is stopped once the
		84	// sample is read rather than after a needless 100 MiB copy.
		85	func (r *Repo) runHead(n int, args ...string) ([]byte, string, error) {
		86	cmd := r.command(args...)
		87	var errb bytes.Buffer
		88	cmd.Stderr = &errb
		89	stdout, err := cmd.StdoutPipe()
		90	if err != nil {
		91	return nil, "", err
		92	}
		93	if err := cmd.Start(); err != nil {
		94	return nil, "", err
		95	}
		96	buf := make([]byte, n)
		97	got, rerr := io.ReadFull(stdout, buf)
		98	if rerr != nil && !errors.Is(rerr, io.ErrUnexpectedEOF) && !errors.Is(rerr, io.EOF) {
		99	cmd.Process.Kill()
		100	cmd.Wait()
		101	return nil, strings.TrimSpace(errb.String()), rerr
		102	}
		103	// Short blob: the process has already finished. Otherwise stop it; git's
		104	// exit status for a closed pipe is not a failure of this call.
		105	if got == n {
		106	cmd.Process.Kill()
		107	}
		108	cmd.Wait()
		109	return buf[:got], strings.TrimSpace(errb.String()), nil
		110	}
		111
	73	112	// InitBare creates a bare repo at dir with the given default branch.
	74	113	func InitBare(dir, branch string) error {
	75	114	if err := os.MkdirAll(dir, 0o755); err != nil {
@@ -534,9 +573,33 @@ func (r *Repo) CountCommits(rev string) (int, error) {
	534	573	// old. Used by the pre-receive hook to enforce object size limits without
	535	574	// walking the entire history.
	536	575	func (r *Repo) NewBlobSizes(old, new string) (map[string]int64, error) {
		576	blobs, err := r.NewBlobs(old, new)
		577	if err != nil {
		578	return nil, err
		579	}
		580	out := make(map[string]int64, len(blobs))
		581	for sha, b := range blobs {
		582	out[sha] = b.Size
		583	}
		584	return out, nil
		585	}
		586
		587	// Blob is one object a push introduces, with the path git reported for it.
		588	// An object reachable from two paths is reported once, under one of them: the
		589	// path is for the message, not for authorization or naming anything durable.
		590	type Blob struct {
		591	SHA string
		592	Path string
		593	Size int64
		594	}
		595
		596	// NewBlobs lists the blobs a push adds, with sizes. rev-list --objects is the
		597	// only git command here that reports path and object id together, so it is the
		598	// one used for both the size check and LFS pointer scanning.
		599	func (r *Repo) NewBlobs(old, new string) (map[string]Blob, error) {
	537	600	args := []string{"rev-list", "--objects", new}
	538	601	// A brand-new ref reports the zero SHA as its old value; passing that to
	539		// --not would make git error out and silently skip the size check.
		602	// --not would make git error out and silently skip the check.
	540	603	if old != "" && old != ZeroSHA {
	541	604	args = append(args, "--not", old)
	542	605	}
@@ -544,17 +607,79 @@ func (r *Repo) NewBlobSizes(old, new string) (map[string]int64, error) {
	544	607	if err != nil {
	545	608	return nil, fmt.Errorf("rev-list: %w: %s", err, errs)
	546	609	}
		610	paths := map[string]string{}
	547	611	var shas []string
	548	612	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
	549	613	if line == "" {
	550	614	continue
	551	615	}
	552		sha := line
	553		if i := strings.IndexByte(line, ' '); i > 0 {
	554		sha = line[:i]
		616	sha, path, _ := strings.Cut(line, " ")
		617	shas = append(shas, sha)
		618	paths[sha] = path
		619	}
		620	if len(shas) == 0 {
		621	return map[string]Blob{}, nil
		622	}
		623	sizes, err := r.blobSizes(shas)
		624	if err != nil {
		625	return nil, err
		626	}
		627	blobs := make(map[string]Blob, len(sizes))
		628	for sha, n := range sizes {
		629	blobs[sha] = Blob{SHA: sha, Path: paths[sha], Size: n}
		630	}
		631	return blobs, nil
		632	}
		633
		634	// TreeBlobs lists every blob in a commit's tree with its size. This is what the
		635	// LFS check scans: what a ref serves after the push, not every object the push
		636	// happens to carry. A commit that deletes a broken pointer must stay pushable,
		637	// or there is no way out of the mistake.
		638	func (r *Repo) TreeBlobs(treeish string) (map[string]Blob, error) {
		639	entries, err := r.LsTreeR(treeish)
		640	if err != nil {
		641	return nil, err
		642	}
		643	paths := map[string]string{}
		644	var shas []string
		645	for _, e := range entries {
		646	if e.Type != "blob" {
		647	continue
	555	648	}
		649	paths[e.SHA] = e.Name
		650	shas = append(shas, e.SHA)
		651	}
		652	sizes, err := r.blobSizes(shas)
		653	if err != nil {
		654	return nil, err
		655	}
		656	blobs := make(map[string]Blob, len(sizes))
		657	for sha, n := range sizes {
		658	blobs[sha] = Blob{SHA: sha, Path: paths[sha], Size: n}
		659	}
		660	return blobs, nil
		661	}
		662
		663	// AllBlobs lists every blob reachable from any ref, with its size. It is the
		664	// mark phase of garbage collection, so it walks history rather than tips: an
		665	// old commit still serves its files.
		666	func (r *Repo) AllBlobs() (map[string]int64, error) {
		667	out, errs, err := r.run(nil, "rev-list", "--objects", "--all")
		668	if err != nil {
		669	return nil, fmt.Errorf("rev-list --all: %w: %s", err, errs)
		670	}
		671	var shas []string
		672	for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
		673	if line == "" {
		674	continue
		675	}
		676	sha, _, _ := strings.Cut(line, " ")
	556	677	shas = append(shas, sha)
	557	678	}
		679	return r.blobSizes(shas)
		680	}
		681
		682	func (r *Repo) blobSizes(shas []string) (map[string]int64, error) {
	558	683	if len(shas) == 0 {
	559	684	return map[string]int64{}, nil
	560	685	}
@@ -562,9 +687,9 @@ func (r *Repo) NewBlobSizes(old, new string) (map[string]int64, error) {
	562	687	for _, s := range shas {
	563	688	in.WriteString(s + "\n")
	564	689	}
	565		bo, errs2, err := r.run(in.Bytes(), "cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)")
		690	bo, errs, err := r.run(in.Bytes(), "cat-file", "--batch-check=%(objectname) %(objecttype) %(objectsize)")
	566	691	if err != nil {
	567		return nil, fmt.Errorf("cat-file --batch-check: %w: %s", err, errs2)
		692	return nil, fmt.Errorf("cat-file --batch-check: %w: %s", err, errs)
	568	693	}
	569	694	sizes := map[string]int64{}
	570	695	for _, line := range strings.Split(strings.TrimSpace(string(bo)), "\n") {
@@ -580,3 +705,14 @@ func (r *Repo) NewBlobSizes(old, new string) (map[string]int64, error) {
	580	705	}
	581	706	return sizes, nil
	582	707	}
		708
		709	// BlobPrefix reads at most n bytes from a blob. Sniffing for a pointer or for
		710	// binary content should not cost a 100 MiB read of a blob that is about to be
		711	// accepted anyway.
		712	func (r *Repo) BlobPrefix(sha string, n int) ([]byte, error) {
		713	out, errs, err := r.runHead(n, "cat-file", "blob", sha)
		714	if err != nil {
		715	return nil, fmt.Errorf("cat-file blob %s: %w: %s", sha, err, errs)
		716	}
		717	return out, nil
		718	}