Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Bound the anonymous Git HTTP read path

86991e7020a418a08a5e3a36ce8f296561392e3b · Keenan Nemetz · 13d ago · parent e05f1c48 

Bound the anonymous Git HTTP read path

The transport that needs no account had no bound on anything: the request body
was capped only at "not negative", the backend ran without a request context or
deadline, the path check accepted any suffix after `git-upload-pack` and handed
it to the CGI as `PATH_INFO`, and the server set no `ReadTimeout` and no
`MaxHeaderBytes`.

Bound what the read side can spend, which is every dimension available to a
fetch and none of the ones a push would need: a 2 MiB body cap with a required
`Content-Length`, exactly two endpoints with one method and one service each so
the CGI never sees an unvalidated path, `exec.CommandContext` plus `WaitDelay`
plus an explicit kill so a disconnect or an expired budget stops git instead of
leaving it running, and a concurrency semaphore with a per-client bound that
answers 503 with Retry-After rather than queueing.

The client address is the connection's own unless the operator names a proxy
header, because behind a proxy the connection address is the proxy. Both limits
and the header name are configuration, `git_http` turns the transport off
entirely, and `status` lists them so an operator can see what is in force.

`WriteTimeout` is deliberately not set: clone, archives, LFS objects and site
files are streamed, so those flows are bounded where they are produced.

server/githttp/githttp.go +200−21

@@ -1,43 +1,90 @@
1 1 // Package githttp serves anonymous, read-only git over smart HTTP.
2 2 //
3 3 // Writes live on SSH, so this package has no push path at all: no chunked
4 // spooling, no push semaphore, no byte cap. Those existed only for POST bodies
5 // that carried packs. What remains is upload-pack behind a visibility check.
6 // Private repositories answer 404 rather than 403, because a 403 confirms that
7 // the repository exists.
4 // spooling, no push semaphore. What remains is upload-pack behind a visibility
5 // check. Private repositories answer 404 rather than 403, because a 403 confirms
6 // that the repository exists.
7 //
8 // Read-only is not the same as cheap. This handler is reachable without an
9 // account, so every way it can spend a resource is bounded: the request body, the
10 // number of clones at once, the rate from one address, the time a backend may
11 // run, and the shape of the path it will accept. A read path can be bounded
12 // tightly in a way a push path cannot — a fetch negotiation is a few hundred
13 // kilobytes at most — so these caps are small and absolute rather than
14 // best-effort.
8 15 package githttp
9 16
10 17 import (
11 18 "bufio"
12 19 "bytes"
20 "context"
21 "errors"
13 22 "fmt"
14 23 "io"
15 24 "log/slog"
25 "net"
16 26 "net/http"
17 27 "os"
18 28 "os/exec"
19 29 "strconv"
20 30 "strings"
31 "sync"
21 32 "sync/atomic"
33 "time"
22 34
23 35 "github.com/rodent-software/vole/forge/access"
24 36 "github.com/rodent-software/vole/forge/repo"
25 37 )
26 38
39 const (
40 // maxUploadPackBody is the largest POST body a fetch needs. `want` and `have`
41 // lines are tens of bytes each and protocol v2 negotiation is a single round
42 // trip, so this is generous by two orders of magnitude — and, unlike a push,
43 // it can be a hard cap rather than a spooling threshold.
44 maxUploadPackBody = 2 << 20
45 // defaultConcurrent and defaultPerClient bound the load one instance accepts
46 // from the anonymous read path. They are defaults, not policy: an operator
47 // with a proxy in front raises them.
48 defaultConcurrent = 16
49 defaultPerClient = 8
50 // defaultRequestTimeout is how long one clone may hold a backend process. A
51 // large clone over a slow link is minutes; a stalled one is hours, and that is
52 // what this stops.
53 defaultRequestTimeout = 10 * time.Minute
54 )
55
27 56 type Config struct {
28 57 // Root is the directory containing <name>.git bare repos.
29 58 Root string
30 59 // Open resolves a repo by name.
31 60 Open func(name string) (*repo.Meta, error)
32 61 Log *slog.Logger
62 // MaxConcurrent bounds simultaneous reads on the transport.
63 MaxConcurrent int
64 // MaxPerClient bounds simultaneous reads from one address. Zero disables it.
65 MaxPerClient int
66 // ClientIPHeader names the header a trusted proxy sets with the client
67 // address. Empty trusts only the connection's own address, which is correct
68 // when nothing sits in front of Vole and wrong behind a proxy: every visitor
69 // would then share one identity and one per-client budget.
70 ClientIPHeader string
71 // RequestTimeout bounds one read, process included.
72 RequestTimeout time.Duration
33 73 }
34 74
35 75 type Proxy struct {
36 cfg Config
76 cfg Config
77 sem chan struct{}
78
79 mu sync.Mutex
80 perClient map[string]int
81
37 82 stats struct {
38 reads atomic.Int64
39 hidden atomic.Int64
40 pushes atomic.Int64
83 reads atomic.Int64
84 hidden atomic.Int64
85 pushes atomic.Int64
86 busy atomic.Int64
87 timedOut atomic.Int64
41 88 }
42 89 }
43 90
@@ -45,7 +92,16 @@ func New(cfg Config) *Proxy {
45 92 if cfg.Log == nil {
46 93 cfg.Log = slog.Default()
47 94 }
48 return &Proxy{cfg: cfg}
95 if cfg.MaxConcurrent <= 0 {
96 cfg.MaxConcurrent = defaultConcurrent
97 }
98 if cfg.MaxPerClient <= 0 {
99 cfg.MaxPerClient = defaultPerClient
100 }
101 if cfg.RequestTimeout <= 0 {
102 cfg.RequestTimeout = defaultRequestTimeout
103 }
104 return &Proxy{cfg: cfg, sem: make(chan struct{}, cfg.MaxConcurrent), perClient: map[string]int{}}
49 105 }
50 106
51 107 func (p *Proxy) Stats() map[string]int64 {
@@ -53,6 +109,8 @@ func (p *Proxy) Stats() map[string]int64 {
53 109 "reads": p.stats.reads.Load(),
54 110 "hidden": p.stats.hidden.Load(),
55 111 "push_attempts": p.stats.pushes.Load(),
112 "refused_busy": p.stats.busy.Load(),
113 "timed_out": p.stats.timedOut.Load(),
56 114 }
57 115 }
58 116
@@ -125,31 +183,124 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
125 183 http.Error(w, "vole: pushes are not accepted over HTTP; use an SSH remote", http.StatusForbidden)
126 184 return
127 185 }
128 if rest != "info/refs" && !strings.HasPrefix(rest, "git-upload-pack") {
186 // Two endpoints exist, and each has exactly one method. Accepting the prefixes
187 // instead let anything shaped like a path through to the CGI, where containment
188 // depended on git's own handling of PATH_INFO.
189 switch {
190 case rest == "info/refs" && r.Method == http.MethodGet:
191 if svc := parseQuery(r.URL.RawQuery)["service"]; svc != "git-upload-pack" {
192 http.NotFound(w, r)
193 return
194 }
195 case rest == "git-upload-pack" && r.Method == http.MethodPost:
196 default:
129 197 http.NotFound(w, r)
130 198 return
131 199 }
132 if r.Method != http.MethodGet && r.Method != http.MethodPost {
133 http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
134 return
200 if r.Method == http.MethodPost {
201 // CGI cannot express "length unknown", and an anonymous body with no length
202 // is a body with no bound.
203 if r.ContentLength < 0 {
204 http.Error(w, "vole: chunked request bodies are not supported on the read path",
205 http.StatusLengthRequired)
206 return
207 }
208 if r.ContentLength == 0 || r.ContentLength > maxUploadPackBody {
209 http.Error(w, "vole: request body is not a fetch negotiation", http.StatusRequestEntityTooLarge)
210 return
211 }
135 212 }
136 // upload-pack request bodies are small (wants/haves). CGI cannot express
137 // "length unknown", so refuse rather than buffer.
138 if r.Method == http.MethodPost && r.ContentLength < 0 {
139 http.Error(w, "vole: chunked request bodies are not supported on the read path",
140 http.StatusLengthRequired)
213
214 // Acquire after the checks that can reject the request outright: a 404 must
215 // not cost a slot a real clone could be using.
216 ip := clientIP(r, p.cfg.ClientIPHeader)
217 if !p.acquire(ip) {
218 p.stats.busy.Add(1)
219 w.Header().Set("Retry-After", "1")
220 http.Error(w, "vole: too many concurrent clones; retry shortly", http.StatusServiceUnavailable)
141 221 return
142 222 }
223 defer p.release(ip)
143 224
144 225 if err := p.backend(w, r, m, rest); err != nil {
226 if errors.Is(err, context.DeadlineExceeded) {
227 p.stats.timedOut.Add(1)
228 return // the client already has an answer; nothing left to send
229 }
145 230 p.cfg.Log.Error("git backend", "repo", name, "err", err)
146 231 return
147 232 }
148 233 p.stats.reads.Add(1)
149 234 }
150 235
236 // acquire takes a transport slot and a per-client slot, or reports that the
237 // request was refused. It never blocks: a queue of waiting clones is another way
238 // to spend memory without bound, and a 503 with Retry-After is honest about it.
239 func (p *Proxy) acquire(ip string) bool {
240 select {
241 case p.sem <- struct{}{}:
242 default:
243 return false
244 }
245 if p.cfg.MaxPerClient <= 0 {
246 return true
247 }
248 p.mu.Lock()
249 n := p.perClient[ip]
250 p.mu.Unlock()
251 if n >= p.cfg.MaxPerClient {
252 <-p.sem
253 return false
254 }
255 p.mu.Lock()
256 p.perClient[ip] = n + 1
257 p.mu.Unlock()
258 return true
259 }
260
261 func (p *Proxy) release(ip string) {
262 if p.cfg.MaxPerClient > 0 {
263 p.mu.Lock()
264 if p.perClient[ip] <= 1 {
265 delete(p.perClient, ip)
266 } else {
267 p.perClient[ip]--
268 }
269 p.mu.Unlock()
270 }
271 <-p.sem
272 }
273
274 // clientIP is who a limit applies to. The connection's own address is the only
275 // one that cannot be forged; a proxy header is used only when the operator named
276 // one, and then the last entry wins because that is the one the nearest proxy
277 // appended.
278 func clientIP(r *http.Request, header string) string {
279 if header != "" {
280 if v := r.Header.Get(header); v != "" {
281 if i := strings.LastIndexByte(v, ','); i >= 0 {
282 v = v[i+1:]
283 }
284 if v = strings.TrimSpace(v); v != "" {
285 return v
286 }
287 }
288 }
289 if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil {
290 return host
291 }
292 return r.RemoteAddr
293 }
294
151 295 func (p *Proxy) backend(w http.ResponseWriter, r *http.Request, m *repo.Meta, rest string) error {
152 cmd := exec.Command("git", "http-backend")
296 // The subprocess belongs to the request: a client that disconnects, or one that
297 // keeps a clone open past its budget, must not leave git running. WaitDelay
298 // bounds the reap once the context is done and the pipes are closed, so a
299 // backend that ignores stdin's closing cannot hold the goroutine either.
300 ctx, cancel := context.WithTimeout(r.Context(), p.cfg.RequestTimeout)
301 defer cancel()
302 cmd := exec.CommandContext(ctx, "git", "http-backend")
303 cmd.WaitDelay = 5 * time.Second
153 304 cmd.Env = append(os.Environ(), p.cgiEnv(r, m, rest)...)
154 305 stdin, err := cmd.StdinPipe()
155 306 if err != nil {
@@ -167,10 +318,27 @@ func (p *Proxy) backend(w http.ResponseWriter, r *http.Request, m *repo.Meta, re
167 318
168 319 go func() {
169 320 defer stdin.Close()
170 if r.Body != nil {
171 io.Copy(stdin, r.Body)
321 if r.Body == nil {
322 return
323 }
324 // A body that trips the cap, or a client that goes away mid-upload, ends
325 // the backend instead of being read to the end.
326 if _, err := io.Copy(stdin, r.Body); err != nil {
327 cancel()
172 328 }
173 329 }()
330 // Closing the request body is what unblocks that copy when the request ends:
331 // a Read on a body whose client has gone otherwise waits for the connection's
332 // own timeout, holding the backend's stdin open and the reap with it.
333 copyDone := make(chan struct{})
334 go func() {
335 select {
336 case <-ctx.Done():
337 r.Body.Close()
338 case <-copyDone:
339 }
340 }()
341 defer close(copyDone)
174 342
175 343 br := bufio.NewReaderSize(stdout, 256<<10)
176 344 status := http.StatusOK
@@ -178,7 +346,12 @@ func (p *Proxy) backend(w http.ResponseWriter, r *http.Request, m *repo.Meta, re
178 346 for {
179 347 line, err := br.ReadString('\n')
180 348 if err != nil {
349 cancel()
181 350 cmd.Wait()
351 if errors.Is(ctx.Err(), context.DeadlineExceeded) {
352 http.Error(w, "vole: the fetch exceeded its time budget", http.StatusGatewayTimeout)
353 return context.DeadlineExceeded
354 }
182 355 msg := strings.TrimSpace(stderr.String())
183 356 if msg == "" {
184 357 msg = err.Error()
@@ -222,6 +395,9 @@ func (p *Proxy) backend(w http.ResponseWriter, r *http.Request, m *repo.Meta, re
222 395 n, rerr := br.Read(buf)
223 396 if n > 0 {
224 397 if _, werr := w.Write(buf[:n]); werr != nil {
398 // The reader is gone. Stop git rather than let it finish writing a
399 // pack nobody will receive.
400 cancel()
225 401 break
226 402 }
227 403 if flusher != nil {
@@ -234,6 +410,9 @@ func (p *Proxy) backend(w http.ResponseWriter, r *http.Request, m *repo.Meta, re
234 410 }
235 411 // A non-zero exit here is git refusing a read; the client has the reason.
236 412 cmd.Wait()
413 if errors.Is(ctx.Err(), context.DeadlineExceeded) {
414 return context.DeadlineExceeded
415 }
237 416 return nil
238 417 }
239 418