Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.

vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 5: LFS objects, over SSH for writes and the proxy for reads

64ac5071a173e4e9ce8bc8d7170f9b0411bee52a · Keenan Nemetz · 18d ago · parent 35eaeccf

Phase 5: LFS objects, over SSH for writes and the proxy for reads

Large files leave the git object database and land in one content-addressed
store per instance. Reads are anonymous like clone; writes ride the SSH key
that already authenticates the push, so no token exists anywhere.

- internal/lfs: sha256-addressed store that verifies content and size before
  an object becomes visible; parser for the pointer format a client writes
- HTTP: POST .../info/lfs/objects/batch for downloads, GET .../objects/<oid>
  with the object id as ETag; upload over HTTP is refused and names SSH
- SSH: git-lfs-transfer speaking the pure SSH protocol (pkt-line), so
  `git lfs push` needs nothing but the key at the handshake
- pre-receive: a pointer whose object is missing, or whose size disagrees
  with the stored object, fails the push that introduced it; a binary over
  lfs_warn_bytes is accepted with a `git lfs track` suggestion. Checks run
  before the protected-branch gate, because an admin's approval bypass is
  about review, not about whether the commit can be checked out
- runs: a finished log becomes a content-addressed object in the same write
  that records the status, served at /r/<repo>/runs/<id>/log
- vole gc [--dry-run] [--grace]: marks pointer targets from every ref in
  every repository plus archived logs, sweeps the rest. Mark-and-sweep
  rather than reference counts: a counter that is one too low deletes an
  artifact, a walk over the refs cannot be wrong about what is reachable

internal/lfs/transfer.go +511

@@ -0,0 +1,511 @@
1 package lfs
2
3 // The pure SSH transfer protocol: what a git-lfs client speaks when it runs
4 // `ssh host git-lfs-transfer <repo> upload|download`. Objects move inside the
5 // SSH session that carries the push, so there is no token and no second
6 // endpoint to authenticate: the key that pushed the pointer is the key that
7 // uploaded the bytes it names.
8 //
9 // The wire format is git's pkt-line framing with a JSON-free command set; see
10 // git-lfs docs/proposals/ssh_adapter.md. Only what a client needs to move
11 // objects is implemented: version, batch, put-object, get-object,
12 // verify-object, quit. Locking is not supported, so `locking` is never
13 // advertised and a client will not offer it.
14
15 import (
16 "bufio"
17 "errors"
18 "fmt"
19 "io"
20 "log/slog"
21 "strconv"
22 "strings"
23 )
24
25 const (
26 pktFlush = 0
27 pktDelim = 1
28 pktEnd = 2
29 pktErr = 3
30 // maxPkt is the largest packet payload: the 4-byte length is hex and must
31 // fit in 16 bits, so a whole packet is 65520 bytes including the header.
32 maxPkt = 65516
33 )
34
35 var errProtocol = errors.New("lfs: protocol error")
36
37 // Transfer serves one LFS session. CanRead and CanWrite come from the repo's
38 // ACL, resolved by the SSH server from the account that owns the key.
39 type Transfer struct {
40 Store *Store
41 Repo string
42 AccountID string
43 CanRead bool
44 CanWrite bool
45 Log *slog.Logger
46 }
47
48 // Serve runs the session until the client quits or the connection ends. op is
49 // the operation the client asked for ("upload" or "download").
50 func (t *Transfer) Serve(op string, r io.Reader, w io.Writer) error {
51 if op != "upload" && op != "download" {
52 return fmt.Errorf("%w: unsupported operation %q", errProtocol, op)
53 }
54 pr := newPktReader(r)
55 pw := newPktWriter(w)
56
57 // Capabilities first: the client reads them before it says anything.
58 if err := pw.pkt("version=1"); err != nil {
59 return err
60 }
61 if err := pw.flush(); err != nil {
62 return err
63 }
64
65 for {
66 head, kind, err := pr.next()
67 if err != nil {
68 return err
69 }
70 if kind == pktFlush {
71 continue
72 }
73 line := strings.TrimSpace(string(head))
74 switch {
75 case line == "version 1":
76 if err := pr.skipArgs(); err != nil {
77 return err
78 }
79 if err := pw.statusLines(200, nil, nil); err != nil {
80 return err
81 }
82 case line == "quit":
83 if err := pr.skipArgs(); err != nil {
84 return err
85 }
86 return pw.status(200)
87 case line == "batch":
88 if err := t.batch(op, pr, pw); err != nil {
89 return err
90 }
91 case strings.HasPrefix(line, "put-object "):
92 if err := t.put(op, strings.TrimSpace(strings.TrimPrefix(line, "put-object ")), pr, pw); err != nil {
93 return err
94 }
95 case strings.HasPrefix(line, "get-object "):
96 if err := t.get(strings.TrimSpace(strings.TrimPrefix(line, "get-object ")), pr, pw); err != nil {
97 return err
98 }
99 case strings.HasPrefix(line, "verify-object "):
100 if err := t.verify(strings.TrimSpace(strings.TrimPrefix(line, "verify-object ")), pr, pw); err != nil {
101 return err
102 }
103 default:
104 // Unknown commands must still consume their arguments, or the next
105 // command is read from the middle of this one.
106 if err := pr.skipArgs(); err != nil {
107 return err
108 }
109 if err := pw.statusLines(400, nil, []string{"unsupported command: " + line}); err != nil {
110 return err
111 }
112 }
113 }
114 }
115
116 // batch answers which objects the client still has to send (upload) or may
117 // fetch (download). Existing objects are answered `noop`, which is how the
118 // client learns it can skip them.
119 func (t *Transfer) batch(op string, pr *pktReader, pw *pktWriter) error {
120 args, oidLines, err := pr.readBatch()
121 if err != nil {
122 return err
123 }
124 _ = args // transfer=, hash-algo=, refname=: nothing here changes the answer
125 if op == "upload" && !t.CanWrite {
126 return pw.statusLines(403, []string{t.denied()}, []string{t.denied()})
127 }
128 if op == "download" && !t.CanRead {
129 return pw.statusLines(403, []string{t.denied()}, []string{t.denied()})
130 }
131
132 var out []string
133 for _, l := range oidLines {
134 f := strings.Fields(l)
135 if len(f) < 2 || !ValidOID(f[0]) {
136 return pw.statusLines(400, nil, []string{"malformed object line: " + l})
137 }
138 oid := f[0]
139 size, _ := strconv.ParseInt(f[1], 10, 64)
140 switch op {
141 case "upload":
142 if t.Store.Has(oid) {
143 out = append(out, oid+" "+strconv.FormatInt(size, 10)+" noop")
144 } else {
145 out = append(out, oid+" "+strconv.FormatInt(size, 10)+" upload")
146 }
147 case "download":
148 // An object we do not have gets no action; the client then reports
149 // the missing object by oid, which is the only thing it can say.
150 if n, err := t.Store.Stat(oid); err == nil {
151 out = append(out, oid+" "+strconv.FormatInt(n, 10)+" download")
152 }
153 }
154 }
155 return pw.statusLines(200, []string{"hash-algo=sha256"}, out)
156 }
157
158 // put receives one object. The bytes are hashed while they stream to disk, so a
159 // client that sends the wrong content gets an error instead of an object filed
160 // under a key that lies about it.
161 func (t *Transfer) put(op, oid string, pr *pktReader, pw *pktWriter) error {
162 if op != "upload" {
163 if err := pr.skipMessage(); err != nil {
164 return err
165 }
166 return pw.statusLines(403, nil, []string{"uploads are not accepted in a download session"})
167 }
168 args, data, stop, err := pr.readData()
169 if err != nil {
170 return err
171 }
172 defer stop()
173 if !t.CanWrite {
174 if _, err := io.Copy(io.Discard, data); err != nil {
175 return err
176 }
177 return pw.statusLines(403, []string{t.denied()}, []string{t.denied()})
178 }
179 var size int64
180 for _, a := range args {
181 if v, ok := strings.CutPrefix(a, "size="); ok {
182 size, _ = strconv.ParseInt(v, 10, 64)
183 }
184 }
185 n, created, err := t.Store.Put(oid, size, data)
186 // Drain whatever the client still has queued so the session stays in sync.
187 if _, derr := io.Copy(io.Discard, data); derr != nil && err == nil {
188 err = derr
189 }
190 if err != nil {
191 msg := err.Error()
192 code := 422
193 if errors.Is(err, ErrTooLarge) {
194 code = 413
195 }
196 if t.Log != nil {
197 t.Log.Warn("lfs upload refused", "repo", t.Repo, "oid", oid, "err", err)
198 }
199 return pw.statusLines(code, nil, []string{msg})
200 }
201 if t.Log != nil {
202 t.Log.Info("lfs upload", "repo", t.Repo, "oid", oid, "bytes", n, "created", created, "user", t.AccountID)
203 }
204 return pw.statusLines(200, nil, nil)
205 }
206
207 // get sends one object. `size=` is mandatory on a successful response; the
208 // client uses it to size its temp file and rejects the object without it.
209 func (t *Transfer) get(oid string, pr *pktReader, pw *pktWriter) error {
210 if err := pr.skipArgs(); err != nil {
211 return err
212 }
213 if !t.CanRead {
214 return pw.statusLines(403, []string{t.denied()}, []string{t.denied()})
215 }
216 f, size, err := t.Store.Open(oid)
217 if err != nil {
218 code := 404
219 if !errors.Is(err, ErrNotFound) {
220 code = 500
221 }
222 return pw.statusLines(code, nil, []string{err.Error()})
223 }
224 defer f.Close()
225 if err := pw.statusData(200, []string{"size=" + strconv.FormatInt(size, 10)}); err != nil {
226 return err
227 }
228 buf := make([]byte, 32<<10)
229 if _, err := io.CopyBuffer(pw, f, buf); err != nil {
230 return err
231 }
232 return pw.flush()
233 }
234
235 // verify confirms an upload landed and is the size the client said. A client
236 // that gets a non-2xx here reports the upload as failed.
237 func (t *Transfer) verify(oid string, pr *pktReader, pw *pktWriter) error {
238 args, err := pr.readUntilFlush()
239 if err != nil {
240 return err
241 }
242 if !t.CanWrite {
243 return pw.statusLines(403, nil, []string{t.denied()})
244 }
245 want := int64(-1)
246 for _, a := range args {
247 if v, ok := strings.CutPrefix(a, "size="); ok {
248 want, _ = strconv.ParseInt(v, 10, 64)
249 }
250 }
251 n, err := t.Store.Stat(oid)
252 if err != nil {
253 return pw.statusLines(404, nil, []string{err.Error()})
254 }
255 if want >= 0 && n != want {
256 return pw.statusLines(422, nil, []string{
257 fmt.Sprintf("object %s is %d bytes, client said %d", oid, n, want)})
258 }
259 return pw.statusLines(200, nil, nil)
260 }
261
262 func (t *Transfer) denied() string {
263 return "your account does not have write access to " + t.Repo
264 }
265
266 // --- pkt-line -------------------------------------------------------------
267
268 type pktReader struct {
269 r *bufio.Reader
270 }
271
272 func newPktReader(r io.Reader) *pktReader { return &pktReader{r: bufio.NewReaderSize(r, 64<<10)} }
273
274 // next returns one packet. kind is pktFlush, pktDelim, pktEnd or -1 for data.
275 func (p *pktReader) next() ([]byte, int, error) {
276 var hdr [4]byte
277 if _, err := io.ReadFull(p.r, hdr[:]); err != nil {
278 return nil, -1, err
279 }
280 n, err := strconv.ParseUint(string(hdr[:]), 16, 16)
281 if err != nil {
282 return nil, -1, fmt.Errorf("%w: bad packet header %q", errProtocol, hdr)
283 }
284 switch n {
285 case pktFlush, pktDelim, pktEnd, pktErr:
286 return nil, int(n), nil
287 }
288 if n < 4 {
289 return nil, -1, fmt.Errorf("%w: short packet %d", errProtocol, n)
290 }
291 data := make([]byte, n-4)
292 if _, err := io.ReadFull(p.r, data); err != nil {
293 return nil, -1, err
294 }
295 return data, -1, nil
296 }
297
298 // readBatch reads the argument packets, the delimiter, and then the object
299 // lines of a batch request.
300 func (p *pktReader) readBatch() (args, lines []string, err error) {
301 args, err = p.readUntilDelim()
302 if err != nil {
303 return nil, nil, err
304 }
305 for {
306 data, kind, err := p.next()
307 if err != nil {
308 return nil, nil, err
309 }
310 if kind == pktFlush {
311 return args, lines, nil
312 }
313 if kind != -1 {
314 continue
315 }
316 lines = append(lines, strings.TrimRight(string(data), "\n"))
317 }
318 }
319
320 // readData reads the argument packets of a put-object, then hands back a reader
321 // over the binary packets that follow. stop must be called: it consumes the
322 // rest of the message so the stream does not desynchronise.
323 func (p *pktReader) readData() (args []string, data io.Reader, stop func(), err error) {
324 args, err = p.readUntilDelim()
325 if err != nil {
326 return nil, nil, nil, err
327 }
328 dr := &dataStream{p: p}
329 return args, dr, dr.stop, nil
330 }
331
332 // skipArgs consumes packets until a flush, discarding them.
333 func (p *pktReader) skipArgs() error {
334 _, err := p.readUntilFlush()
335 return err
336 }
337
338 // skipMessage consumes packets until a flush, including packets after a
339 // delimiter.
340 func (p *pktReader) skipMessage() error {
341 for {
342 _, kind, err := p.next()
343 if err != nil {
344 return err
345 }
346 if kind == pktFlush {
347 return nil
348 }
349 }
350 }
351
352 func (p *pktReader) readUntilDelim() ([]string, error) {
353 var out []string
354 for {
355 data, kind, err := p.next()
356 if err != nil {
357 return nil, err
358 }
359 switch kind {
360 case pktFlush:
361 return out, nil
362 case pktDelim:
363 return out, nil
364 case -1:
365 out = append(out, strings.TrimRight(string(data), "\n"))
366 }
367 }
368 }
369
370 func (p *pktReader) readUntilFlush() ([]string, error) {
371 var out []string
372 for {
373 data, kind, err := p.next()
374 if err != nil {
375 return nil, err
376 }
377 if kind == pktFlush {
378 return out, nil
379 }
380 if kind == -1 {
381 out = append(out, strings.TrimRight(string(data), "\n"))
382 }
383 }
384 }
385
386 // dataStream presents the binary part of a put-object as an io.Reader. Once a
387 // flush is seen it reports EOF; further reads keep returning EOF, and stop is
388 // idempotent so a caller can always defer it.
389 type dataStream struct {
390 p *pktReader
391 buf []byte
392 done bool
393 stopped bool
394 }
395
396 func (d *dataStream) Read(b []byte) (int, error) {
397 for len(d.buf) == 0 {
398 if d.done {
399 return 0, io.EOF
400 }
401 data, kind, err := d.p.next()
402 if err != nil {
403 return 0, err
404 }
405 if kind == pktFlush {
406 d.done = true
407 return 0, io.EOF
408 }
409 if kind != -1 {
410 continue
411 }
412 d.buf = data
413 }
414 n := copy(b, d.buf)
415 d.buf = d.buf[n:]
416 return n, nil
417 }
418
419 func (d *dataStream) stop() {
420 d.stopped = true
421 io.Copy(io.Discard, d)
422 }
423
424 type pktWriter struct {
425 w *bufio.Writer
426 }
427
428 func newPktWriter(w io.Writer) *pktWriter { return &pktWriter{w: bufio.NewWriterSize(w, 64<<10)} }
429
430 // pkt writes one text packet. git's framing includes the trailing newline
431 // inside the length, and the client strips it.
432 func (p *pktWriter) pkt(s string) error {
433 b := []byte(s)
434 if len(b) > maxPkt {
435 return fmt.Errorf("%w: packet of %d bytes", errProtocol, len(b))
436 }
437 if _, err := fmt.Fprintf(p.w, "%04x%s\n", len(b)+5, b); err != nil {
438 return err
439 }
440 return nil
441 }
442
443 func (p *pktWriter) delim() error { _, err := p.w.WriteString("0001"); return err }
444 func (p *pktWriter) flush() error {
445 if _, err := p.w.WriteString("0000"); err != nil {
446 return err
447 }
448 return p.w.Flush()
449 }
450
451 func (p *pktWriter) status(code int) error {
452 if err := p.pkt(fmt.Sprintf("status %d", code)); err != nil {
453 return err
454 }
455 return p.flush()
456 }
457
458 func (p *pktWriter) statusLines(code int, args, lines []string) error {
459 if err := p.pkt(fmt.Sprintf("status %d", code)); err != nil {
460 return err
461 }
462 for _, a := range args {
463 if err := p.pkt(a); err != nil {
464 return err
465 }
466 }
467 if err := p.delim(); err != nil {
468 return err
469 }
470 for _, l := range lines {
471 if err := p.pkt(l); err != nil {
472 return err
473 }
474 }
475 return p.flush()
476 }
477
478 // statusData writes the status and arguments, then the delimiter. The caller
479 // writes the payload through pktWriter's Write and ends with flush, so a large
480 // object is never held in memory.
481 func (p *pktWriter) statusData(code int, args []string) error {
482 if err := p.pkt(fmt.Sprintf("status %d", code)); err != nil {
483 return err
484 }
485 for _, a := range args {
486 if err := p.pkt(a); err != nil {
487 return err
488 }
489 }
490 return p.delim()
491 }
492
493 // Write frames b as binary packets of at most maxPkt bytes.
494 func (p *pktWriter) Write(b []byte) (int, error) {
495 written := 0
496 for len(b) > 0 {
497 n := len(b)
498 if n > maxPkt {
499 n = maxPkt
500 }
501 if _, err := fmt.Fprintf(p.w, "%04x", n+4); err != nil {
502 return written, err
503 }
504 if _, err := p.w.Write(b[:n]); err != nil {
505 return written, err
506 }
507 b = b[n:]
508 written += n
509 }
510 return written, nil
511 }