vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/lfs/lfs_test.go +163

@@ -0,0 +1,163 @@
	1	package lfs
	2
	3	import (
	4	"bytes"
	5	"crypto/sha256"
	6	"encoding/hex"
	7	"errors"
	8	"io"
	9	"os"
	10	"path/filepath"
	11	"strings"
	12	"testing"
	13	)
	14
	15	func oidOf(b []byte) string {
	16	sum := sha256.Sum256(b)
	17	return hex.EncodeToString(sum[:])
	18	}
	19
	20	func TestPointerRoundTrip(t *testing.T) {
	21	data := bytes.Repeat([]byte("x"), 1234)
	22	p := Pointer{OID: oidOf(data), Size: int64(len(data))}
	23	got, ok := ParsePointer(p.Encode())
	24	if !ok {
	25	t.Fatalf("could not parse what Encode wrote: %q", p.Encode())
	26	}
	27	if got != p {
	28	t.Fatalf("round trip changed the pointer: %+v != %+v", got, p)
	29	}
	30	}
	31
	32	func TestPointerIsStrictAboutWhatItNames(t *testing.T) {
	33	good := string(Pointer{OID: strings.Repeat("a", 64), Size: 5}.Encode())
	34	cases := map[string]string{
	35	"wrong version": strings.Replace(good, PointerVersion, "https://example.com/v1", 1),
	36	"sha1 oid": strings.Replace(good, "sha256:", "sha1:", 1),
	37	"short oid": strings.Replace(good, strings.Repeat("a", 64), "abc", 1),
	38	"negative size": strings.Replace(good, "size 5", "size -5", 1),
	39	"missing size": strings.Replace(good, "size 5\n", "", 1),
	40	"not a pointer": "title: a note, not an LFS pointer\n",
	41	"empty": "",
	42	"binary garbage": "\x00\x01\x02",
	43	}
	44	for name, text := range cases {
	45	if _, ok := ParsePointer([]byte(text)); ok {
	46	t.Errorf("%s: parsed as a pointer, should not have", name)
	47	}
	48	}
	49	// Unknown keys are fine: git-lfs writes ext- lines for its extensions.
	50	withExt := good + "ext-0-foo sha256:" + strings.Repeat("b", 64) + "\n"
	51	if _, ok := ParsePointer([]byte(withExt)); !ok {
	52	t.Errorf("an unknown key should not make a pointer unreadable")
	53	}
	54	}
	55
	56	func TestStorePutOpenDedupe(t *testing.T) {
	57	s := New(t.TempDir(), 1<<20)
	58	data := []byte("some bytes worth storing")
	59	oid := oidOf(data)
	60
	61	n, created, err := s.Put(oid, int64(len(data)), bytes.NewReader(data))
	62	if err != nil \|\| n != int64(len(data)) \|\| !created {
	63	t.Fatalf("Put = %d, %v, %v; want %d, true, nil", n, created, err, len(data))
	64	}
	65	if size, err := s.Stat(oid); err != nil \|\| size != int64(len(data)) {
	66	t.Fatalf("Stat = %d, %v", size, err)
	67	}
	68	f, size, err := s.Open(oid)
	69	if err != nil {
	70	t.Fatal(err)
	71	}
	72	got, _ := io.ReadAll(f)
	73	f.Close()
	74	if !bytes.Equal(got, data) \|\| size != int64(len(data)) {
	75	t.Fatalf("Open returned %q (%d bytes)", got, size)
	76	}
	77	// The same content twice is one object.
	78	if _, created, err := s.Put(oid, int64(len(data)), bytes.NewReader(data)); err != nil \|\| created {
	79	t.Fatalf("second Put created=%v err=%v; want deduplicated", created, err)
	80	}
	81
	82	objects, bytesUsed, err := s.Usage()
	83	if err != nil \|\| objects != 1 \|\| bytesUsed != int64(len(data)) {
	84	t.Fatalf("Usage = %d, %d, %v", objects, bytesUsed, err)
	85	}
	86	if err := s.Remove(oid); err != nil {
	87	t.Fatal(err)
	88	}
	89	if s.Has(oid) {
	90	t.Fatal("object still present after Remove")
	91	}
	92	if err := s.Remove(oid); err != nil {
	93	t.Fatalf("Remove of a missing object should be quiet: %v", err)
	94	}
	95	}
	96
	97	func TestStoreRefusesWhatDoesNotMatchItsKey(t *testing.T) {
	98	s := New(t.TempDir(), 1<<20)
	99	data := []byte("real content")
	100	oid := oidOf([]byte("different content"))
	101
	102	if _, _, err := s.Put(oid, int64(len(data)), bytes.NewReader(data)); !errors.Is(err, ErrMismatch) {
	103	t.Fatalf("Put of mismatched content = %v; want ErrMismatch", err)
	104	}
	105	if s.Has(oid) {
	106	t.Fatal("mismatched content was stored under the wrong id")
	107	}
	108
	109	tight := New(t.TempDir(), 1024)
	110	big := bytes.Repeat([]byte("x"), 2048)
	111	if _, _, err := tight.Put(oidOf(big), 0, bytes.NewReader(big)); !errors.Is(err, ErrTooLarge) {
	112	t.Fatalf("oversized Put = %v; want ErrTooLarge", err)
	113	}
	114
	115	small := []byte("declared size lies")
	116	if _, _, err := s.Put(oidOf(small), 9999, bytes.NewReader(small)); !errors.Is(err, ErrMismatch) {
	117	t.Fatalf("Put with a lying size = %v; want ErrMismatch", err)
	118	}
	119
	120	if _, err := s.Stat("not-an-oid"); !errors.Is(err, ErrBadOID) {
	121	t.Fatalf("Stat of a malformed id = %v", err)
	122	}
	123	if _, err := s.Stat(strings.Repeat("f", 64)); !errors.Is(err, ErrNotFound) {
	124	t.Fatalf("Stat of an absent object = %v", err)
	125	}
	126	}
	127
	128	func TestStoreWalkFindsShardedObjects(t *testing.T) {
	129	root := t.TempDir()
	130	s := New(root, 1<<20)
	131	var want []string
	132	for i := 0; i < 3; i++ {
	133	data := []byte(strings.Repeat("ab", i+1))
	134	oid := oidOf(data)
	135	if _, _, err := s.Put(oid, int64(len(data)), bytes.NewReader(data)); err != nil {
	136	t.Fatal(err)
	137	}
	138	want = append(want, oid)
	139	}
	140	var seen []string
	141	if err := s.Walk(func(oid string, size int64) error {
	142	seen = append(seen, oid)
	143	return nil
	144	}); err != nil {
	145	t.Fatal(err)
	146	}
	147	if len(seen) != len(want) {
	148	t.Fatalf("Walk saw %d objects, want %d (%v)", len(seen), len(want), seen)
	149	}
	150	for _, oid := range want {
	151	if !s.Has(oid) {
	152	t.Fatalf("Walk did not see %s, which is at %s", oid, s.ObjectPath(oid))
	153	}
	154	}
	155	// The path really is where the file is: a sweep that cannot stat objects
	156	// silently deletes nothing and reports success.
	157	if _, err := os.Stat(s.ObjectPath(want[0])); err != nil {
	158	t.Fatalf("ObjectPath does not exist: %v", err)
	159	}
	160	if filepath.Base(s.ObjectPath(want[0])) != want[0] {
	161	t.Fatalf("ObjectPath basename %q is not the object id", filepath.Base(s.ObjectPath(want[0])))
	162	}
	163	}