vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/server/lfs.go +162

@@ -0,0 +1,162 @@
	1	package server
	2
	3	// LFS over the read path. Like everything else on HTTP, downloads are anonymous
	4	// and gated only by the repository's default role: a private repository answers
	5	// 404, not 403. Uploads are refused here on purpose — the write transport is
	6	// SSH, where the key at the handshake is the identity that owns the bytes.
	7
	8	import (
	9	"encoding/json"
	10	"errors"
	11	"io"
	12	"net/http"
	13	"strconv"
	14	"strings"
	15
	16	"github.com/rodent-software/vole/internal/acl"
	17	"github.com/rodent-software/vole/internal/lfs"
	18	"github.com/rodent-software/vole/internal/repo"
	19	)
	20
	21	const (
	22	batchBodyLimit = 4 << 20 // a batch of 4 MiB of oids is already absurd
	23	objectShard = "/info/lfs/"
	24	)
	25
	26	// lfsRepo resolves the repository named in an LFS URL and applies the read
	27	// rule. It writes the response itself when the repository is not readable.
	28	func (s Server) lfsRepo(w http.ResponseWriter, r http.Request) (*repo.Meta, bool) {
	29	name := strings.TrimSuffix(r.PathValue("name"), ".git")
	30	m, err := s.Repos.Open(name)
	31	if err != nil {
	32	http.NotFound(w, r)
	33	return nil, false
	34	}
	35	a, err := m.ACL()
	36	if err != nil {
	37	s.Log.Error("read acl", "repo", name, "err", err)
	38	http.Error(w, "repository unavailable", http.StatusInternalServerError)
	39	return nil, false
	40	}
	41	if !a.Can("", acl.Read) {
	42	http.NotFound(w, r)
	43	return nil, false
	44	}
	45	return m, true
	46	}
	47
	48	// LFSBatch implements POST /git/<name>.git/info/lfs/objects/batch.
	49	func (s Server) LFSBatch(w http.ResponseWriter, r http.Request) {
	50	m, ok := s.lfsRepo(w, r)
	51	if !ok {
	52	return
	53	}
	54	if ct := r.Header.Get("Content-Type"); !strings.HasPrefix(ct, lfs.MediaType) {
	55	lfs.WriteError(w, http.StatusUnsupportedMediaType,
	56	"vole: expected Content-Type %s, got %q", lfs.MediaType, ct)
	57	return
	58	}
	59	var req lfs.BatchRequest
	60	dec := json.NewDecoder(io.LimitReader(r.Body, batchBodyLimit))
	61	if err := dec.Decode(&req); err != nil {
	62	lfs.WriteError(w, http.StatusBadRequest, "vole: malformed batch request: %v", err)
	63	return
	64	}
	65	switch req.Operation {
	66	case "download":
	67	case "upload":
	68	lfs.WriteError(w, http.StatusForbidden,
	69	"vole: LFS uploads are not accepted over HTTP. Push over SSH — `git push` on an ssh remote sends the objects through the same key that pushes the pointer files.")
	70	return
	71	default:
	72	lfs.WriteError(w, http.StatusBadRequest, "vole: unsupported operation %q", req.Operation)
	73	return
	74	}
	75	if len(req.Objects) == 0 {
	76	lfs.WriteError(w, http.StatusBadRequest, "vole: batch request lists no objects")
	77	return
	78	}
	79	if req.HashAlgo != "" && req.HashAlgo != "sha256" {
	80	lfs.WriteError(w, http.StatusBadRequest, "vole: only sha256 object ids are supported, not %q", req.HashAlgo)
	81	return
	82	}
	83
	84	base := s.lfsObjectBase(r, m.Name)
	85	out := lfs.BatchResponse{Transfer: "basic", HashAlgo: "sha256"}
	86	for _, o := range req.Objects {
	87	item := lfs.ObjectResponse{OID: o.OID, Size: o.Size}
	88	switch {
	89	case !lfs.ValidOID(o.OID):
	90	item.Error = &lfs.ObjectError{Code: 422, Message: "malformed object id"}
	91	default:
	92	if size, err := s.LFS.Stat(o.OID); err == nil {
	93	item.Size = size
	94	item.Actions = map[string]lfs.Action{
	95	"download": {Href: base + "/" + o.OID},
	96	}
	97	} else if errors.Is(err, lfs.ErrNotFound) {
	98	item.Error = &lfs.ObjectError{Code: 404, Message: "object not found on this server"}
	99	} else {
	100	s.Log.Error("lfs stat", "repo", m.Name, "oid", o.OID, "err", err)
	101	item.Error = &lfs.ObjectError{Code: 500, Message: "store unavailable"}
	102	}
	103	}
	104	out.Objects = append(out.Objects, item)
	105	}
	106
	107	w.Header().Set("Content-Type", lfs.MediaType)
	108	if err := json.NewEncoder(w).Encode(out); err != nil {
	109	s.Log.Error("lfs batch response", "repo", m.Name, "err", err)
	110	}
	111	}
	112
	113	// LFSObject implements GET /git/<name>.git/info/lfs/objects/<oid>. The object id
	114	// is its own ETag, so a client that already has the bytes never asks twice.
	115	func (s Server) LFSObject(w http.ResponseWriter, r http.Request) {
	116	m, ok := s.lfsRepo(w, r)
	117	if !ok {
	118	return
	119	}
	120	oid := r.PathValue("oid")
	121	if !lfs.ValidOID(oid) {
	122	http.Error(w, "vole: malformed object id", http.StatusBadRequest)
	123	return
	124	}
	125	f, size, err := s.LFS.Open(oid)
	126	if err != nil {
	127	if errors.Is(err, lfs.ErrNotFound) {
	128	http.Error(w, "vole: object not found", http.StatusNotFound)
	129	return
	130	}
	131	s.Log.Error("lfs open", "repo", m.Name, "oid", oid, "err", err)
	132	http.Error(w, "vole: store unavailable", http.StatusInternalServerError)
	133	return
	134	}
	135	defer f.Close()
	136
	137	w.Header().Set("Content-Type", "application/octet-stream")
	138	w.Header().Set("ETag", `"`+oid+`"`)
	139	w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
	140	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
	141	if r.Method == http.MethodHead {
	142	return
	143	}
	144	if _, err := io.Copy(w, f); err != nil {
	145	// The client hung up; nothing to report to it.
	146	s.Log.Debug("lfs download interrupted", "repo", m.Name, "oid", oid, "err", err)
	147	}
	148	}
	149
	150	// lfsObjectBase is where download hrefs point. It is derived from the request
	151	// rather than configured, so a clone through a proxy hands back a URL on the
	152	// same host the client already uses.
	153	func (s Server) lfsObjectBase(r http.Request, name string) string {
	154	scheme := "http"
	155	if r.TLS != nil {
	156	scheme = "https"
	157	}
	158	if p := r.Header.Get("X-Forwarded-Proto"); p == "https" \|\| p == "http" {
	159	scheme = p
	160	}
	161	return scheme + "://" + r.Host + "/git/" + name + ".git" + objectShard + "objects"
	162	}