Sign in

Search issue titles and bodies, change requests and comments. Private repositories you cannot read are not included.
vole
Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Phase 1: a working git remote with access control

653e0cfc227c992e5b26ea10057aa0b424b6ce4f · Keenan Nemetz · 18d ago · parent c5626f1b 

Phase 1: a working git remote with access control

Vole is now a functioning forge foundation: one CGO-free binary that serves
real git over smart HTTP and enforces per-repo policy in pre-receive.

What is in:

- internal/transport drives `git http-backend` as a CGI subprocess. Chunked
  uploads are spooled to learn CONTENT_LENGTH (git's default client config
  produces them), spool files are cleaned on error and abort, and a janitor
  sweeps anything a crash left behind. Pushes are capped by size and
  serialised behind a semaphore, because Phase 0 measured 1.13 GB of git RSS
  for a single 1 GiB push.
- internal/acl stores roles on refs/vole/acl inside each repository, so
  access rules travel with a clone. One evaluation function is shared by the
  hook and the web layer.
- internal/hook enforces policy in pre-receive: authentication, write role,
  protected refs, per-blob size limits, and a refusal to accept pushes to
  refs/vole/* - without that last check a writer could push their own
  acl.json and grant themselves admin.
- internal/auth holds users, personal access tokens and web sessions in
  SQLite. Passwords use PBKDF2-SHA256; tokens are 256-bit random values and
  are stored as SHA-256 hashes, since there is no low-entropy secret to grind.
- internal/index is the derived read model, rebuildable from disk and refs.
- internal/repo owns the bare-repo store and the metadata ref CAS helper.
- internal/web is the browse surface: repo list, tree, blob, raw, history,
  login, and per-repo settings for access rules, rename and delete.
- internal/signing creates the instance ed25519 key at first boot for the
  Phase 7 work.

Every Phase 1 exit criterion has a test that would fail if it regressed:
clone/push against a real listener, read-only and anonymous rejection with the
ref proven unmoved, rebuild-from-refs compared against the actual acl ref, a
spool file proven absent after an aborted chunked upload, and two pushes
completing through a single-slot semaphore. scripts/ci.sh runs gofmt, vet,
build, a markdown link check and the race-enabled suite.

Verified by hand as well: a clean `vole init` -> user -> token -> push ->
clone cycle, web login and browse, and a 256 MiB chunked push killed
mid-flight leaving no ref, no spool file and no orphaned git processes.

internal/server/integration_test.go +571

@@ -0,0 +1,571 @@
1 // End-to-end tests for Phase 1. These drive the real git binary against a real
2 // HTTP listener, because the thing being verified is the exit criteria: clone,
3 // push, refusal, and rebuild-from-refs.
4 package server_test
5
6 import (
7 "bytes"
8 "fmt"
9 "io"
10 "log/slog"
11 "net/http"
12 "net/http/cookiejar"
13 "net/http/httptest"
14 "net/url"
15 "os"
16 "os/exec"
17 "path/filepath"
18 "strings"
19 "sync"
20 "testing"
21 "time"
22
23 "github.com/rodent-software/vole/internal/config"
24 "github.com/rodent-software/vole/internal/index"
25 "github.com/rodent-software/vole/internal/server"
26 )
27
28 var binPath string
29
30 func TestMain(m *testing.M) {
31 dir, err := os.MkdirTemp("", "vole-bin-")
32 if err != nil {
33 fmt.Fprintln(os.Stderr, err)
34 os.Exit(1)
35 }
36 binPath = filepath.Join(dir, "vole")
37 build := exec.Command("go", "build", "-o", binPath, "github.com/rodent-software/vole/cmd/vole")
38 build.Stdout, build.Stderr = os.Stdout, os.Stderr
39 if err := build.Run(); err != nil {
40 fmt.Fprintln(os.Stderr, "build vole:", err)
41 os.Exit(1)
42 }
43 code := m.Run()
44 os.RemoveAll(dir)
45 os.Exit(code)
46 }
47
48 // ---------- harness ----------
49
50 type fixture struct {
51 t *testing.T
52 cfg *config.Config
53 srv *server.Server
54 ts *httptest.Server
55 repo string
56 owner string
57 ownerTok string
58 reader string
59 readerTok string
60 writer string
61 writerTok string
62 closed bool
63 }
64
65 func newFixture(t *testing.T, mutate func(*config.Config)) *fixture {
66 t.Helper()
67 cfg, err := config.Load("")
68 if err != nil {
69 t.Fatal(err)
70 }
71 cfg.DataDir = t.TempDir()
72 cfg.SessionTTLDur = time.Hour
73 cfg.QueueTimeoutDur = 30 * time.Second
74 if mutate != nil {
75 mutate(cfg)
76 }
77 log := slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelError}))
78 srv, err := server.New(cfg, binPath, log)
79 if err != nil {
80 t.Fatal(err)
81 }
82 f := &fixture{
83 t: t, cfg: cfg, srv: srv,
84 ts: httptest.NewServer(srv.Handler()),
85 repo: "demo",
86 owner: "owner", reader: "reader", writer: "writer",
87 }
88 t.Cleanup(f.Close)
89
90 f.ownerTok = f.addUser(f.owner, "adminpw-123")
91 f.readerTok = f.addUser(f.reader, "readerpw-123")
92 f.writerTok = f.addUser(f.writer, "writerpw-123")
93
94 m, err := srv.Repos.Create(f.repo, f.owner, "demo repo")
95 if err != nil {
96 t.Fatal(err)
97 }
98 if err := srv.Index.Upsert(m); err != nil {
99 t.Fatal(err)
100 }
101 a, err := m.ACL()
102 if err != nil {
103 t.Fatal(err)
104 }
105 _ = a.SetEntry(f.reader, "read")
106 _ = a.SetEntry(f.writer, "write")
107 if err := m.SetACL(a, "test"); err != nil {
108 t.Fatal(err)
109 }
110 if err := srv.Index.Upsert(m); err != nil {
111 t.Fatal(err)
112 }
113 return f
114 }
115
116 func (f *fixture) addUser(name, pw string) string {
117 f.t.Helper()
118 u, err := f.srv.Auth.CreateUser(name, pw)
119 if err != nil {
120 f.t.Fatal(err)
121 }
122 tok, _, err := f.srv.Auth.CreateToken(u.ID, "test", "write", time.Hour)
123 if err != nil {
124 f.t.Fatal(err)
125 }
126 return tok
127 }
128
129 func (f *fixture) Close() {
130 if f.closed {
131 return
132 }
133 f.closed = true
134 f.ts.Close()
135 f.srv.Close()
136 }
137
138 func (f *fixture) urlAs(tok string) string {
139 u, _ := url.Parse(f.ts.URL)
140 u.User = url.UserPassword("x", tok)
141 u.Path = "/git/" + f.repo + ".git"
142 return u.String()
143 }
144
145 func gitRun(t *testing.T, dir string, args ...string) (string, string, error) {
146 t.Helper()
147 cmd := exec.Command("git", args...)
148 cmd.Dir = dir
149 cmd.Env = append(os.Environ(),
150 "GIT_TERMINAL_PROMPT=0",
151 "GIT_ASKPASS=/bin/true",
152 "GIT_CONFIG_GLOBAL=/dev/null",
153 "GIT_CONFIG_SYSTEM=/dev/null",
154 "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com",
155 "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com",
156 )
157 var out, errb bytes.Buffer
158 cmd.Stdout, cmd.Stderr = &out, &errb
159 err := cmd.Run()
160 return out.String(), errb.String(), err
161 }
162
163 // workRepo creates a local repo with one commit and returns its path.
164 func workRepo(t *testing.T, files map[string]string) string {
165 t.Helper()
166 dir := t.TempDir()
167 if _, _, err := gitRun(t, dir, "init", "-q", "-b", "main"); err != nil {
168 t.Fatal(err)
169 }
170 for name, content := range files {
171 p := filepath.Join(dir, name)
172 if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
173 t.Fatal(err)
174 }
175 if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
176 t.Fatal(err)
177 }
178 }
179 if _, _, err := gitRun(t, dir, "add", "-A"); err != nil {
180 t.Fatal(err)
181 }
182 if _, errs, err := gitRun(t, dir, "commit", "-q", "-m", "initial"); err != nil {
183 t.Fatalf("commit: %v: %s", err, errs)
184 }
185 return dir
186 }
187
188 func (f *fixture) push(t *testing.T, work, refspec, tok string, extra ...string) (string, string, error) {
189 t.Helper()
190 args := append([]string{"push"}, extra...)
191 args = append(args, f.urlAs(tok), refspec)
192 return gitRun(t, work, args...)
193 }
194
195 func (f *fixture) serverRef(ref string) string {
196 out, _, err := gitRun(f.t, filepath.Join(f.cfg.ReposDir(), f.repo+".git"), "rev-parse", "--verify", "--quiet", ref)
197 if err != nil {
198 return ""
199 }
200 return strings.TrimSpace(out)
201 }
202
203 // ---------- tests ----------
204
205 func TestPushThenClone(t *testing.T) {
206 f := newFixture(t, nil)
207 work := workRepo(t, map[string]string{"readme.md": "# hello\n", "docs/a.txt": "a\n"})
208
209 if _, errs, err := f.push(t, work, "main:refs/heads/main", f.ownerTok); err != nil {
210 t.Fatalf("push failed: %v\n%s", err, errs)
211 }
212 if f.serverRef("refs/heads/main") == "" {
213 t.Fatal("server has no refs/heads/main after a successful push")
214 }
215
216 // A read-only user must be able to clone.
217 dst := filepath.Join(t.TempDir(), "clone")
218 if _, errs, err := gitRun(t, "", "clone", "-q", f.urlAs(f.readerTok), dst); err != nil {
219 t.Fatalf("clone as reader failed: %v\n%s", err, errs)
220 }
221 got, err := os.ReadFile(filepath.Join(dst, "readme.md"))
222 if err != nil {
223 t.Fatal(err)
224 }
225 if string(got) != "# hello\n" {
226 t.Fatalf("cloned content = %q", got)
227 }
228 }
229
230 func TestReadOnlyPushRejectedAndRefUnmoved(t *testing.T) {
231 f := newFixture(t, nil)
232 work := workRepo(t, map[string]string{"readme.md": "v1\n"})
233 if _, errs, err := f.push(t, work, "main:refs/heads/main", f.ownerTok); err != nil {
234 t.Fatalf("owner push failed: %s", errs)
235 }
236 before := f.serverRef("refs/heads/main")
237
238 if err := os.WriteFile(filepath.Join(work, "readme.md"), []byte("v2\n"), 0o644); err != nil {
239 t.Fatal(err)
240 }
241 if _, _, err := gitRun(t, work, "commit", "-aqm", "v2"); err != nil {
242 t.Fatal(err)
243 }
244 _, errs, err := f.push(t, work, "main:refs/heads/main", f.readerTok)
245 if err == nil {
246 t.Fatal("read-only push succeeded")
247 }
248 if !strings.Contains(errs, "does not have write access") {
249 t.Fatalf("rejection reason not shown to the client:\n%s", errs)
250 }
251 if after := f.serverRef("refs/heads/main"); after != before {
252 t.Fatalf("ref moved on a rejected push: %s -> %s", before, after)
253 }
254 }
255
256 func TestAnonymousPushRejected(t *testing.T) {
257 f := newFixture(t, nil)
258 work := workRepo(t, map[string]string{"a.txt": "a\n"})
259 u, _ := url.Parse(f.ts.URL)
260 u.Path = "/git/" + f.repo + ".git"
261 _, _, err := gitRun(t, work, "push", u.String(), "main:refs/heads/main")
262 if err == nil {
263 t.Fatal("anonymous push succeeded")
264 }
265 if f.serverRef("refs/heads/main") != "" {
266 t.Fatal("anonymous push created a ref")
267 }
268 }
269
270 func TestMetadataRefPushRejected(t *testing.T) {
271 f := newFixture(t, nil)
272 work := workRepo(t, map[string]string{"acl.json": `{"version":1,"entries":{"attacker":"admin"}}`})
273 _, errs, err := f.push(t, work, "main:refs/vole/acl", f.ownerTok, "--force")
274 if err == nil {
275 t.Fatal("owner was allowed to overwrite the ACL ref")
276 }
277 if !strings.Contains(errs, "server-owned") {
278 t.Fatalf("client not told why:\n%s", errs)
279 }
280 if f.serverRef("refs/vole/acl") == "" {
281 t.Fatal("the original ACL ref disappeared")
282 }
283 }
284
285 func TestProtectedBranchRejectsWriter(t *testing.T) {
286 f := newFixture(t, nil)
287 work := workRepo(t, map[string]string{"a.txt": "a\n"})
288 if _, errs, err := f.push(t, work, "main:refs/heads/main", f.ownerTok); err != nil {
289 t.Fatalf("owner push: %s", errs)
290 }
291
292 m, err := f.srv.Repos.Open(f.repo)
293 if err != nil {
294 t.Fatal(err)
295 }
296 a, _ := m.ACL()
297 a.Protected = []string{"refs/heads/main"}
298 if err := m.SetACL(a, "test"); err != nil {
299 t.Fatal(err)
300 }
301
302 late := workRepo(t, map[string]string{"b.txt": "b\n"})
303 _, errs, err := f.push(t, late, "main:refs/heads/main", f.writerTok, "--force")
304 if err == nil {
305 t.Fatal("writer pushed a protected branch")
306 }
307 if !strings.Contains(errs, "protected") {
308 t.Fatalf("rejection reason not shown:\n%s", errs)
309 }
310 }
311
312 // TestIndexRebuildReproducesState is the exit criterion that proves the index is
313 // derived: delete it, rebuild from disk and refs, compare.
314 func TestIndexRebuildReproducesState(t *testing.T) {
315 f := newFixture(t, nil)
316 work := workRepo(t, map[string]string{"readme.md": "hello\n"})
317 if _, errs, err := f.push(t, work, "main:refs/heads/main", f.ownerTok); err != nil {
318 t.Fatalf("push: %s", errs)
319 }
320 before, err := f.srv.Index.List()
321 if err != nil {
322 t.Fatal(err)
323 }
324 if len(before) != 1 {
325 t.Fatalf("want 1 indexed repo, got %d", len(before))
326 }
327
328 // Mutate the ACL through git only, so a rebuild must read the ref to see it.
329 m, err := f.srv.Repos.Open(f.repo)
330 if err != nil {
331 t.Fatal(err)
332 }
333 a, _ := m.ACL()
334 if err := a.SetEntry("newcomer", "read"); err != nil {
335 t.Fatal(err)
336 }
337 if err := m.SetACL(a, "test"); err != nil {
338 t.Fatal(err)
339 }
340
341 f.Close()
342 for _, p := range []string{f.cfg.IndexDB(), f.cfg.IndexDB() + "-wal", f.cfg.IndexDB() + "-shm"} {
343 if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
344 t.Fatal(err)
345 }
346 }
347
348 cfgPath := filepath.Join(t.TempDir(), "vole.toml")
349 if err := os.WriteFile(cfgPath, []byte(fmt.Sprintf("data_dir = %q\n", f.cfg.DataDir)), 0o644); err != nil {
350 t.Fatal(err)
351 }
352 out, errs, err := execBinary(binPath, "index", "rebuild", "-config", cfgPath)
353 if err != nil {
354 t.Fatalf("rebuild: %v\n%s\n%s", err, out, errs)
355 }
356 if !strings.Contains(out, "1 repo") {
357 t.Fatalf("unexpected rebuild output: %s", out)
358 }
359
360 ix, err := index.Open(f.cfg.IndexDB())
361 if err != nil {
362 t.Fatal(err)
363 }
364 defer ix.Close()
365 after, err := ix.List()
366 if err != nil {
367 t.Fatal(err)
368 }
369 if len(after) != 1 {
370 t.Fatalf("want 1 repo after rebuild, got %d", len(after))
371 }
372 if after[0].Name != before[0].Name || after[0].ID != before[0].ID {
373 t.Fatalf("identity changed across rebuild: %+v vs %+v", before[0], after[0])
374 }
375 if after[0].ACLPrincipals != before[0].ACLPrincipals+1 {
376 t.Fatalf("rebuild did not pick up the new ACL entry: %d -> %d",
377 before[0].ACLPrincipals, after[0].ACLPrincipals)
378 }
379 if after[0].ACLSHA == before[0].ACLSHA {
380 t.Fatal("ACL sha unchanged: rebuild is not reading the ref")
381 }
382 refSHA := f.serverRef("refs/vole/acl")
383 if after[0].ACLSHA != refSHA {
384 t.Fatalf("indexed acl sha %s != ref %s", after[0].ACLSHA, refSHA)
385 }
386 }
387
388 func TestPushOverSizeCapRejected(t *testing.T) {
389 f := newFixture(t, func(c *config.Config) { c.MaxPushBytes = 4096 })
390 work := workRepo(t, map[string]string{"big.bin": incompressible(64 << 10)})
391 _, _, err := f.push(t, work, "main:refs/heads/main", f.ownerTok)
392 if err == nil {
393 t.Fatal("push over the size cap succeeded")
394 }
395 if f.serverRef("refs/heads/main") != "" {
396 t.Fatal("oversized push created a ref")
397 }
398 }
399
400 // TestChunkedPushSucceedsAndLeavesNoSpool forces the chunked-upload path with a
401 // tiny postBuffer, then checks the spool directory is clean.
402 func TestChunkedPushSucceedsAndLeavesNoSpool(t *testing.T) {
403 f := newFixture(t, nil)
404 work := workRepo(t, map[string]string{"data.txt": strings.Repeat("chunk me\n", 40000)})
405 _, errs, err := gitRun(t, work, "-c", "http.postBuffer=1", "push", f.urlAs(f.ownerTok), "main:refs/heads/main")
406 if err != nil {
407 t.Fatalf("chunked push failed: %v\n%s", err, errs)
408 }
409 if f.serverRef("refs/heads/main") == "" {
410 t.Fatal("chunked push did not land")
411 }
412 ents, err := os.ReadDir(f.cfg.SpoolDir())
413 if err != nil && !os.IsNotExist(err) {
414 t.Fatal(err)
415 }
416 for _, e := range ents {
417 if strings.HasPrefix(e.Name(), "vole-spool-") {
418 t.Fatalf("spool file left behind: %s", e.Name())
419 }
420 }
421 }
422
423 // TestConcurrentPushesQueueRatherThanFail runs two pushes against a proxy with a
424 // single push slot. Both must complete: the second waits, it does not 503.
425 func TestConcurrentPushesQueueRatherThanFail(t *testing.T) {
426 f := newFixture(t, func(c *config.Config) { c.PushConcurrency = 1 })
427 w1 := workRepo(t, map[string]string{"one.txt": strings.Repeat("1", 50000)})
428 w2 := workRepo(t, map[string]string{"two.txt": strings.Repeat("2", 50000)})
429
430 var wg sync.WaitGroup
431 errs := make([]error, 2)
432 msgs := make([]string, 2)
433 for i, job := range []struct {
434 work string
435 ref string
436 }{{w1, "main:refs/heads/one"}, {w2, "main:refs/heads/two"}} {
437 wg.Add(1)
438 go func(i int, work, ref string) {
439 defer wg.Done()
440 _, m, err := f.push(t, work, ref, f.ownerTok)
441 errs[i], msgs[i] = err, m
442 }(i, job.work, job.ref)
443 }
444 wg.Wait()
445 for i, err := range errs {
446 if err != nil {
447 t.Fatalf("concurrent push %d failed: %v\n%s", i, err, msgs[i])
448 }
449 }
450 if f.serverRef("refs/heads/one") == "" || f.serverRef("refs/heads/two") == "" {
451 t.Fatal("a queued push did not land")
452 }
453 }
454
455 func TestWebLoginAndBrowse(t *testing.T) {
456 f := newFixture(t, nil)
457 work := workRepo(t, map[string]string{"readme.md": "# hi from vole\n"})
458 if _, errs, err := f.push(t, work, "main:refs/heads/main", f.ownerTok); err != nil {
459 t.Fatalf("push: %s", errs)
460 }
461
462 jar, _ := cookiejar.New(nil)
463 client := &http.Client{
464 Jar: jar,
465 CheckRedirect: func(req *http.Request, via []*http.Request) error {
466 return http.ErrUseLastResponse
467 },
468 }
469 resp, err := client.Get(f.ts.URL + "/")
470 if err != nil {
471 t.Fatal(err)
472 }
473 resp.Body.Close()
474 if resp.StatusCode != http.StatusSeeOther {
475 t.Fatalf("unauthenticated index: status %d, want 303 to login", resp.StatusCode)
476 }
477
478 form := url.Values{"username": {f.owner}, "password": {"adminpw-123"}}
479 resp, err = client.PostForm(f.ts.URL+"/login", form)
480 if err != nil {
481 t.Fatal(err)
482 }
483 resp.Body.Close()
484 if resp.StatusCode != http.StatusSeeOther {
485 t.Fatalf("login: status %d", resp.StatusCode)
486 }
487 if len(jar.Cookies(mustURL(t, f.ts.URL))) == 0 {
488 t.Fatal("no session cookie was set")
489 }
490
491 for _, c := range []struct{ path, want string }{
492 {"/", "demo"},
493 {"/r/demo", "Clone"},
494 {"/r/demo/blob/main/readme.md", "hi from vole"},
495 {"/r/demo/commits/main", "initial"},
496 {"/r/demo/raw/main/readme.md", "# hi from vole"},
497 } {
498 resp, err := client.Get(f.ts.URL + c.path)
499 if err != nil {
500 t.Fatal(err)
501 }
502 body, _ := io.ReadAll(resp.Body)
503 resp.Body.Close()
504 if resp.StatusCode != http.StatusOK {
505 t.Fatalf("%s: status %d", c.path, resp.StatusCode)
506 }
507 if !strings.Contains(string(body), c.want) {
508 t.Fatalf("%s: missing %q in body:\n%s", c.path, c.want, body)
509 }
510 }
511 }
512
513 func TestWebDeniesReadForStranger(t *testing.T) {
514 f := newFixture(t, nil)
515 strangerTok := f.addUser("stranger", "strangerpw-123")
516 _ = strangerTok
517
518 jar, _ := cookiejar.New(nil)
519 client := &http.Client{Jar: jar, CheckRedirect: func(*http.Request, []*http.Request) error {
520 return http.ErrUseLastResponse
521 }}
522 form := url.Values{"username": {"stranger"}, "password": {"strangerpw-123"}}
523 resp, err := client.PostForm(f.ts.URL+"/login", form)
524 if err != nil {
525 t.Fatal(err)
526 }
527 resp.Body.Close()
528
529 resp, err = client.Get(f.ts.URL + "/r/demo")
530 if err != nil {
531 t.Fatal(err)
532 }
533 body, _ := io.ReadAll(resp.Body)
534 resp.Body.Close()
535 if resp.StatusCode != http.StatusForbidden {
536 t.Fatalf("stranger got status %d, want 403", resp.StatusCode)
537 }
538 if !strings.Contains(string(body), "read access") {
539 t.Fatalf("unclear denial: %s", body)
540 }
541 }
542
543 // execBinary runs the vole CLI (not git).
544 func execBinary(bin string, args ...string) (string, string, error) {
545 cmd := exec.Command(bin, args...)
546 var out, errb bytes.Buffer
547 cmd.Stdout, cmd.Stderr = &out, &errb
548 err := cmd.Run()
549 return out.String(), errb.String(), err
550 }
551
552 // incompressible produces bytes that git cannot shrink, so size limits are
553 // actually exercised.
554 func incompressible(n int) string {
555 b := make([]byte, n)
556 state := uint32(0x9e3779b9)
557 for i := range b {
558 state = state*1664525 + 1013904223
559 b[i] = byte(state >> 24)
560 }
561 return string(b)
562 }
563
564 func mustURL(t *testing.T, raw string) *url.URL {
565 t.Helper()
566 u, err := url.Parse(raw)
567 if err != nil {
568 t.Fatal(err)
569 }
570 return u
571 }