vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Stream run logs over the runner's SSH channel

5ece70a50ebda392626b92bdf29d28bc9b444954 · Keenan Nemetz · 11d ago · parent fdf2340f

Stream run logs over the runner's SSH channel

The runner wrote the log to a temp file and uploaded it as one LFS object
when the job ended, so a running job had no log to read and a log was stored
like a shared artifact even though it belongs to one attempt.

The runner sends 128 KiB chunks over the same runner channel that holds the
lease. The server appends them to data/logs/<repo>/<run>/<attempt>.log and
records the size while the run is still running. The finish record carries
the final size; the server checks the file length, syncs it, and stores the
result. A partial or failed stream leaves no log on the record.

Logs are no longer LFS objects: they are not shared, not content-addressed,
and the web serves the file directly, with range and step slices. GC no
longer treats them as LFS roots.

ci/logs/logs_test.go +97

@@ -0,0 +1,97 @@
	1	package logs
	2
	3	import (
	4	"errors"
	5	"io"
	6	"os"
	7	"path/filepath"
	8	"testing"
	9	"time"
	10
	11	"github.com/rodent-software/vole/core/id"
	12	)
	13
	14	func store(t testing.T) Store {
	15	t.Helper()
	16	return New(filepath.Join(t.TempDir(), "logs"))
	17	}
	18
	19	func runID(t *testing.T) string {
	20	t.Helper()
	21	id, err := id.New(time.Now().UTC())
	22	if err != nil {
	23	t.Fatal(err)
	24	}
	25	return id
	26	}
	27
	28	func TestAppendReadAndRemove(t *testing.T) {
	29	t.Parallel()
	30	s := store(t)
	31	run := runID(t)
	32	if n, err := s.Append("demo", run, 1, 0, []byte("hello")); err != nil \|\| n != 5 {
	33	t.Fatalf("append = %d, %v", n, err)
	34	}
	35	if n, err := s.Append("demo", run, 1, 5, []byte(" world")); err != nil \|\| n != 11 {
	36	t.Fatalf("append = %d, %v", n, err)
	37	}
	38	if n, err := s.Size("demo", run, 1); err != nil \|\| n != 11 {
	39	t.Fatalf("size = %d, %v", n, err)
	40	}
	41	f, info, err := s.Open("demo", run, 1)
	42	if err != nil {
	43	t.Fatal(err)
	44	}
	45	defer f.Close()
	46	if info.Size() != 11 {
	47	t.Fatalf("info size = %d", info.Size())
	48	}
	49	data, err := io.ReadAll(f)
	50	if err != nil \|\| string(data) != "hello world" {
	51	t.Fatalf("read = %q, %v", data, err)
	52	}
	53	if err := s.Sync("demo", run, 1); err != nil {
	54	t.Fatal(err)
	55	}
	56	if err := s.Remove("demo", run, 1); err != nil {
	57	t.Fatal(err)
	58	}
	59	if n, err := s.Size("demo", run, 1); err != nil \|\| n != 0 {
	60	t.Fatalf("size after remove = %d, %v", n, err)
	61	}
	62	if _, err := os.Stat(filepath.Join(s.root, "demo", run, "1.log")); !errors.Is(err, os.ErrNotExist) {
	63	t.Fatalf("log still exists: %v", err)
	64	}
	65	}
	66
	67	func TestAppendRefusesAGapAndAnOversizedLog(t *testing.T) {
	68	t.Parallel()
	69	s := store(t)
	70	run := runID(t)
	71	if _, err := s.Append("demo", run, 1, 4, []byte("x")); !errors.Is(err, ErrOffset) {
	72	t.Fatalf("gap = %v, want ErrOffset", err)
	73	}
	74	if _, err := s.Append("demo", run, 1, s.max, []byte("x")); !errors.Is(err, ErrTooLarge) {
	75	t.Fatalf("past cap = %v, want ErrTooLarge", err)
	76	}
	77	}
	78
	79	func TestPathRefusesBadComponents(t *testing.T) {
	80	t.Parallel()
	81	s := store(t)
	82	run := runID(t)
	83	for _, tc := range []struct {
	84	repo string
	85	run string
	86	attempt int
	87	}{
	88	{"../escape", run, 1},
	89	{"demo", "../escape", 1},
	90	{"demo", run, 0},
	91	} {
	92	if _, err := s.Append(tc.repo, tc.run, tc.attempt, 0, []byte("x")); err == nil {
	93	t.Fatalf("accepted repo=%q run=%q attempt=%d", tc.repo, tc.run, tc.attempt)
	94	}
	95	}
	96
	97	}