vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
refactor
c5902451905acdaa478f8476844ccd4a510d9ca9 · Keenan Nemetz · 17d ago · parent ec2aba4b
client/transfer.go +366
| @@ -0,0 +1,366 @@ | |||
| 1 | package client | ||
| 2 | |||
| 3 | import ( | ||
| 4 | "bufio" | ||
| 5 | "crypto/sha256" | ||
| 6 | "encoding/hex" | ||
| 7 | "errors" | ||
| 8 | "fmt" | ||
| 9 | "io" | ||
| 10 | "os" | ||
| 11 | "os/exec" | ||
| 12 | "slices" | ||
| 13 | "strconv" | ||
| 14 | "strings" | ||
| 15 | |||
| 16 | "github.com/rodent-software/vole/core/lfs" | ||
| 17 | "github.com/rodent-software/vole/internal/format" | ||
| 18 | ) | ||
| 19 | |||
| 20 | // Transfer is one SSH object-transfer session (`git-lfs-transfer`). | ||
| 21 | type Transfer struct { | ||
| 22 | p *pktConn | ||
| 23 | cmd *exec.Cmd | ||
| 24 | stdin io.Closer | ||
| 25 | // closed guards Close so a deferred Close after an error is harmless. | ||
| 26 | closed bool | ||
| 27 | } | ||
| 28 | |||
| 29 | // DialTransfer starts `ssh host git-lfs-transfer <repo> <op>` and negotiates the | ||
| 30 | // protocol version. remote is the URL of the git remote, so the upload cannot | ||
| 31 | // land on a different instance than the push. | ||
| 32 | func DialTransfer(remote, op string) (*Transfer, error) { | ||
| 33 | host, port, path, err := ParseSSHRemote(remote) | ||
| 34 | if err != nil { | ||
| 35 | return nil, err | ||
| 36 | } | ||
| 37 | cmd := SSHCommand(host, port, "git-lfs-transfer", path, op) | ||
| 38 | cmd.Stderr = os.Stderr | ||
| 39 | stdin, err := cmd.StdinPipe() | ||
| 40 | if err != nil { | ||
| 41 | return nil, err | ||
| 42 | } | ||
| 43 | stdout, err := cmd.StdoutPipe() | ||
| 44 | if err != nil { | ||
| 45 | return nil, err | ||
| 46 | } | ||
| 47 | if err := cmd.Start(); err != nil { | ||
| 48 | return nil, fmt.Errorf("start ssh: %w", err) | ||
| 49 | } | ||
| 50 | t := &Transfer{p: newPktConn(stdout, stdin), cmd: cmd, stdin: stdin} | ||
| 51 | if err := t.hello(); err != nil { | ||
| 52 | t.Close() | ||
| 53 | return nil, err | ||
| 54 | } | ||
| 55 | return t, nil | ||
| 56 | } | ||
| 57 | |||
| 58 | // hello reads the capability advertisement and negotiates the version. It is a | ||
| 59 | // separate step from starting ssh so a test can drive it over a pipe. | ||
| 60 | func (t *Transfer) hello() error { | ||
| 61 | // The server advertises version=1 before the client says anything. | ||
| 62 | var caps []string | ||
| 63 | for { | ||
| 64 | data, kind, err := t.p.next() | ||
| 65 | if err != nil { | ||
| 66 | return fmt.Errorf("read capabilities: %w", err) | ||
| 67 | } | ||
| 68 | if kind == kindFlush { | ||
| 69 | break | ||
| 70 | } | ||
| 71 | if kind == kindData { | ||
| 72 | caps = append(caps, strings.TrimRight(string(data), "\n")) | ||
| 73 | } | ||
| 74 | } | ||
| 75 | if !slices.Contains(caps, "version=1") { | ||
| 76 | return fmt.Errorf("this instance does not speak LFS transfer v1 (advertised %v)", caps) | ||
| 77 | } | ||
| 78 | if err := t.p.pkt("version 1"); err != nil { | ||
| 79 | return err | ||
| 80 | } | ||
| 81 | if err := t.p.flush(); err != nil { | ||
| 82 | return err | ||
| 83 | } | ||
| 84 | _, err := t.p.expect() | ||
| 85 | return err | ||
| 86 | } | ||
| 87 | |||
| 88 | // Upload sends one file and returns the pointer that names it. The object is | ||
| 89 | // hashed locally first, so a file that changes while it is being sent cannot be | ||
| 90 | // filed under a key that lies about its content — the store checks the same sum | ||
| 91 | // on arrival and would refuse it anyway. | ||
| 92 | func (t *Transfer) Upload(path string) (lfs.Pointer, error) { | ||
| 93 | f, err := os.Open(path) | ||
| 94 | if err != nil { | ||
| 95 | return lfs.Pointer{}, err | ||
| 96 | } | ||
| 97 | defer f.Close() | ||
| 98 | st, err := f.Stat() | ||
| 99 | if err != nil { | ||
| 100 | return lfs.Pointer{}, err | ||
| 101 | } | ||
| 102 | if st.IsDir() { | ||
| 103 | return lfs.Pointer{}, fmt.Errorf("%s is a directory; attach files", path) | ||
| 104 | } | ||
| 105 | h := sha256.New() | ||
| 106 | if _, err := io.Copy(h, f); err != nil { | ||
| 107 | return lfs.Pointer{}, err | ||
| 108 | } | ||
| 109 | oid := hex.EncodeToString(h.Sum(nil)) | ||
| 110 | size := st.Size() | ||
| 111 | ptr := lfs.Pointer{OID: oid, Size: size} | ||
| 112 | |||
| 113 | action, err := t.batch(oid, size) | ||
| 114 | if err != nil { | ||
| 115 | return lfs.Pointer{}, err | ||
| 116 | } | ||
| 117 | if action == "noop" { | ||
| 118 | // Content-addressed: the object is already there, so the same bytes | ||
| 119 | // uploaded twice cost one copy and no transfer. | ||
| 120 | return ptr, nil | ||
| 121 | } | ||
| 122 | if action != "upload" { | ||
| 123 | return lfs.Pointer{}, fmt.Errorf("the server asked for %q on an upload session", action) | ||
| 124 | } | ||
| 125 | if _, err := f.Seek(0, io.SeekStart); err != nil { | ||
| 126 | return lfs.Pointer{}, err | ||
| 127 | } | ||
| 128 | if err := t.put(oid, size, f); err != nil { | ||
| 129 | return lfs.Pointer{}, err | ||
| 130 | } | ||
| 131 | if err := t.verify(oid, size); err != nil { | ||
| 132 | return lfs.Pointer{}, err | ||
| 133 | } | ||
| 134 | return ptr, nil | ||
| 135 | } | ||
| 136 | |||
| 137 | func (t *Transfer) batch(oid string, size int64) (string, error) { | ||
| 138 | for _, s := range []string{"batch", "transfer=ssh", "hash-algo=sha256"} { | ||
| 139 | if err := t.p.pkt(s); err != nil { | ||
| 140 | return "", err | ||
| 141 | } | ||
| 142 | } | ||
| 143 | if err := t.p.delim(); err != nil { | ||
| 144 | return "", err | ||
| 145 | } | ||
| 146 | if err := t.p.pkt(fmt.Sprintf("%s %d", oid, size)); err != nil { | ||
| 147 | return "", err | ||
| 148 | } | ||
| 149 | if err := t.p.flush(); err != nil { | ||
| 150 | return "", err | ||
| 151 | } | ||
| 152 | lines, err := t.p.expect() | ||
| 153 | if err != nil { | ||
| 154 | return "", err | ||
| 155 | } | ||
| 156 | for _, l := range lines { | ||
| 157 | f := strings.Fields(l) | ||
| 158 | if len(f) >= 3 && f[0] == oid { | ||
| 159 | return f[2], nil | ||
| 160 | } | ||
| 161 | } | ||
| 162 | return "", fmt.Errorf("the server did not answer for object %s", format.OID(oid)) | ||
| 163 | } | ||
| 164 | |||
| 165 | func (t *Transfer) put(oid string, size int64, r io.Reader) error { | ||
| 166 | if err := t.p.pkt("put-object " + oid); err != nil { | ||
| 167 | return err | ||
| 168 | } | ||
| 169 | if err := t.p.pkt("size=" + strconv.FormatInt(size, 10)); err != nil { | ||
| 170 | return err | ||
| 171 | } | ||
| 172 | if err := t.p.delim(); err != nil { | ||
| 173 | return err | ||
| 174 | } | ||
| 175 | buf := make([]byte, 32<<10) | ||
| 176 | var sent int64 | ||
| 177 | for { | ||
| 178 | n, err := r.Read(buf) | ||
| 179 | if n > 0 { | ||
| 180 | if werr := t.p.data(buf[:n]); werr != nil { | ||
| 181 | return werr | ||
| 182 | } | ||
| 183 | sent += int64(n) | ||
| 184 | } | ||
| 185 | if errors.Is(err, io.EOF) { | ||
| 186 | break | ||
| 187 | } | ||
| 188 | if err != nil { | ||
| 189 | return err | ||
| 190 | } | ||
| 191 | } | ||
| 192 | if sent != size { | ||
| 193 | // The server would reject the size anyway; failing here names the file. | ||
| 194 | return fmt.Errorf("read %d bytes, wanted %d", sent, size) | ||
| 195 | } | ||
| 196 | if err := t.p.flush(); err != nil { | ||
| 197 | return err | ||
| 198 | } | ||
| 199 | _, err := t.p.expect() | ||
| 200 | return err | ||
| 201 | } | ||
| 202 | |||
| 203 | func (t *Transfer) verify(oid string, size int64) error { | ||
| 204 | if err := t.p.pkt("verify-object " + oid); err != nil { | ||
| 205 | return err | ||
| 206 | } | ||
| 207 | if err := t.p.pkt("size=" + strconv.FormatInt(size, 10)); err != nil { | ||
| 208 | return err | ||
| 209 | } | ||
| 210 | if err := t.p.flush(); err != nil { | ||
| 211 | return err | ||
| 212 | } | ||
| 213 | _, err := t.p.expect() | ||
| 214 | return err | ||
| 215 | } | ||
| 216 | |||
| 217 | // Close ends the session and waits for ssh to exit. | ||
| 218 | func (t *Transfer) Close() error { | ||
| 219 | if t == nil || t.closed { | ||
| 220 | return nil | ||
| 221 | } | ||
| 222 | t.closed = true | ||
| 223 | _ = t.p.pkt("quit") | ||
| 224 | _ = t.p.flush() | ||
| 225 | _, _, _ = t.p.readStatus() | ||
| 226 | // Closing the pipe is the EOF the server session ends on; signalling the | ||
| 227 | // process instead would race the last packets out of the door. | ||
| 228 | if t.stdin != nil { | ||
| 229 | _ = t.stdin.Close() | ||
| 230 | } | ||
| 231 | if t.cmd == nil { // a test drives the protocol without spawning ssh | ||
| 232 | return nil | ||
| 233 | } | ||
| 234 | return t.cmd.Wait() | ||
| 235 | } | ||
| 236 | |||
| 237 | // ---------- protocol ---------- | ||
| 238 | // | ||
| 239 | // git's pkt-line framing over the SSH session's stdin/stdout. The server side is | ||
| 240 | // lfs.Transfer; the framing here mirrors it, and a test drives this client against | ||
| 241 | // the real handler rather than a mock, so a change to one that does not match the | ||
| 242 | // other fails immediately. | ||
| 243 | |||
| 244 | const ( | ||
| 245 | kindData = -1 | ||
| 246 | kindFlush = 0 | ||
| 247 | kindDelim = 1 | ||
| 248 | pktMax = 65516 | ||
| 249 | ) | ||
| 250 | |||
| 251 | type pktConn struct { | ||
| 252 | in *bufio.Writer | ||
| 253 | out *bufio.Reader | ||
| 254 | } | ||
| 255 | |||
| 256 | func newPktConn(r io.Reader, w io.Writer) *pktConn { | ||
| 257 | return &pktConn{in: bufio.NewWriterSize(w, 64<<10), out: bufio.NewReaderSize(r, 64<<10)} | ||
| 258 | } | ||
| 259 | |||
| 260 | func (p *pktConn) pkt(s string) error { | ||
| 261 | if len(s) > pktMax { | ||
| 262 | return fmt.Errorf("lfs: packet of %d bytes", len(s)) | ||
| 263 | } | ||
| 264 | if _, err := fmt.Fprintf(p.in, "%04x%s\n", len(s)+5, s); err != nil { | ||
| 265 | return err | ||
| 266 | } | ||
| 267 | return nil | ||
| 268 | } | ||
| 269 | |||
| 270 | // data frames binary bytes. A packet's length covers only its payload, so the | ||
| 271 | // newline pkt() adds for text must not appear here. | ||
| 272 | func (p *pktConn) data(b []byte) error { | ||
| 273 | for len(b) > 0 { | ||
| 274 | n := len(b) | ||
| 275 | if n > pktMax { | ||
| 276 | n = pktMax | ||
| 277 | } | ||
| 278 | if _, err := fmt.Fprintf(p.in, "%04x", n+4); err != nil { | ||
| 279 | return err | ||
| 280 | } | ||
| 281 | if _, err := p.in.Write(b[:n]); err != nil { | ||
| 282 | return err | ||
| 283 | } | ||
| 284 | b = b[n:] | ||
| 285 | } | ||
| 286 | return nil | ||
| 287 | } | ||
| 288 | |||
| 289 | func (p *pktConn) delim() error { _, err := p.in.WriteString("0001"); return err } | ||
| 290 | |||
| 291 | func (p *pktConn) flush() error { | ||
| 292 | if _, err := p.in.WriteString("0000"); err != nil { | ||
| 293 | return err | ||
| 294 | } | ||
| 295 | return p.in.Flush() | ||
| 296 | } | ||
| 297 | |||
| 298 | func (p *pktConn) next() ([]byte, int, error) { | ||
| 299 | var hdr [4]byte | ||
| 300 | if _, err := io.ReadFull(p.out, hdr[:]); err != nil { | ||
| 301 | return nil, 0, err | ||
| 302 | } | ||
| 303 | n, err := strconv.ParseUint(string(hdr[:]), 16, 16) | ||
| 304 | if err != nil { | ||
| 305 | return nil, 0, fmt.Errorf("lfs: bad packet header %q", hdr) | ||
| 306 | } | ||
| 307 | if n <= 3 { | ||
| 308 | return nil, int(n), nil | ||
| 309 | } | ||
| 310 | buf := make([]byte, n-4) | ||
| 311 | if _, err := io.ReadFull(p.out, buf); err != nil { | ||
| 312 | return nil, 0, err | ||
| 313 | } | ||
| 314 | return buf, kindData, nil | ||
| 315 | } | ||
| 316 | |||
| 317 | // readStatus reads one status frame: the code, then the lines after the | ||
| 318 | // delimiter. The object lines of a batch answer arrive as those lines. | ||
| 319 | func (p *pktConn) readStatus() (int, []string, error) { | ||
| 320 | data, _, err := p.next() | ||
| 321 | if err != nil { | ||
| 322 | return 0, nil, err | ||
| 323 | } | ||
| 324 | line := strings.TrimRight(string(data), "\n") | ||
| 325 | s, ok := strings.CutPrefix(line, "status ") | ||
| 326 | if !ok { | ||
| 327 | return 0, nil, fmt.Errorf("lfs: expected a status line, got %q", line) | ||
| 328 | } | ||
| 329 | code, err := strconv.Atoi(s) | ||
| 330 | if err != nil { | ||
| 331 | return 0, nil, fmt.Errorf("lfs: bad status line %q", line) | ||
| 332 | } | ||
| 333 | var lines []string | ||
| 334 | inLines := false | ||
| 335 | for { | ||
| 336 | data, kind, err := p.next() | ||
| 337 | if err != nil { | ||
| 338 | return code, nil, err | ||
| 339 | } | ||
| 340 | switch kind { | ||
| 341 | case kindFlush: | ||
| 342 | return code, lines, nil | ||
| 343 | case kindDelim: | ||
| 344 | inLines = true | ||
| 345 | case kindData: | ||
| 346 | if inLines { | ||
| 347 | lines = append(lines, strings.TrimRight(string(data), "\n")) | ||
| 348 | } | ||
| 349 | } | ||
| 350 | } | ||
| 351 | } | ||
| 352 | |||
| 353 | func (p *pktConn) expect() ([]string, error) { | ||
| 354 | code, lines, err := p.readStatus() | ||
| 355 | if err != nil { | ||
| 356 | return nil, err | ||
| 357 | } | ||
| 358 | if code/100 != 2 { | ||
| 359 | msg := strings.Join(lines, "; ") | ||
| 360 | if msg == "" { | ||
| 361 | msg = "no detail" | ||
| 362 | } | ||
| 363 | return nil, fmt.Errorf("the server refused (%d): %s", code, msg) | ||
| 364 | } | ||
| 365 | return lines, nil | ||
| 366 | } | ||