vole

Clone

Read only, no credentials:

https://rodent.software/git/vole.git

Write over SSH with a registered key:

git clone ssh://git@0.0.0.0:2222/vole.git

A free and open-source forge that lives in your Git repo.

Stream run logs over the runner's SSH channel

5ece70a50ebda392626b92bdf29d28bc9b444954 · Keenan Nemetz · 10d ago · parent fdf2340f

Stream run logs over the runner's SSH channel

The runner wrote the log to a temp file and uploaded it as one LFS object
when the job ended, so a running job had no log to read and a log was stored
like a shared artifact even though it belongs to one attempt.

The runner sends 128 KiB chunks over the same runner channel that holds the
lease. The server appends them to data/logs/<repo>/<run>/<attempt>.log and
records the size while the run is still running. The finish record carries
the final size; the server checks the file length, syncs it, and stores the
result. A partial or failed stream leaves no log on the record.

Logs are no longer LFS objects: they are not shared, not content-addressed,
and the web serves the file directly, with range and step slices. GC no
longer treats them as LFS roots.

ci/agent/agent.go +31−40

@@ -4,9 +4,9 @@
	4	4	// with, leases one job at a time, and gives the job a disposable checkout. Steps
	5	5	// run in a container built from the image the workflow names, or from a pinned
	6	6	// .vole/images definition; the runner's key and trusted Git cache stay outside.
	7		// When the job ends the runner uploads the log and artifacts, sends the attempt to
	8		// the server with its finish message, and attaches any release pointers. This
	9		// package is the agent; argument parsing and the CLI's messages stay in cmd/vole.
		7	// The runner streams the log while the job runs, uploads artifacts, and sends the
		8	// attempt to the server with its finish message. This package is the agent;
		9	// argument parsing and the CLI's messages stay in cmd/vole.
	10	10	package agent
	11	11
	12	12	import (
@@ -33,7 +33,6 @@ import (
	33	33	"github.com/rodent-software/vole/ci/container"
	34	34	"github.com/rodent-software/vole/ci/runs"
	35	35	"github.com/rodent-software/vole/client"
	36		"github.com/rodent-software/vole/core/lfs"
	37	36	"github.com/rodent-software/vole/forge/access"
	38	37	"github.com/rodent-software/vole/internal/format"
	39	38	"github.com/rodent-software/vole/metadata"
@@ -102,6 +101,14 @@ type runnerArtifact struct {
	102	101	Size int64 `json:"size"`
	103	102	}
	104	103
		104	// runnerLogChunk is one piece of an attempt's log. Offset is where it starts in
		105	// the attempt's file, so the server can refuse a gap or a duplicate.
		106	type runnerLogChunk struct {
		107	Attempt int `json:"attempt"`
		108	Offset int64 `json:"offset"`
		109	Data []byte `json:"data"`
		110	}
		111
	105	112	// runnerRequest is the wire. A finish carries the attempt's record: the runner
	106	113	// no longer publishes it to Git, and this channel is what the server checks and
	107	114	// stores.
@@ -110,6 +117,8 @@ type runnerRequest struct {
	110	117	Run string `json:"run"`
	111	118	// Record is the attempt's result, sent with finish.
	112	119	Record *runs.Record `json:"record,omitempty"`
		120	// Log is one chunk of the attempt's log, sent while the job runs.
		121	Log *runnerLogChunk `json:"log,omitempty"`
	113	122	// Repos is the --repos filter, sent with every lease. The server intersects it
	114	123	// with what the account's allowlists grant, so it can only narrow.
	115	124	Repos []string `json:"repos,omitempty"`
@@ -135,6 +144,8 @@ type runnerReply struct {
	135	144	// "ping" with these so a --repos filter can be checked against them at startup:
	136	145	// a filter that matches nothing would otherwise look exactly like a quiet instance.
	137	146	Repos []string `json:"repos,omitempty"`
		147	// LogSize is how much of the log the server has after a log chunk.
		148	LogSize int64 `json:"log_size,omitempty"`
	138	149	}
	139	150
	140	151	// Options configure one runner process.
@@ -334,6 +345,8 @@ type runnerSession struct {
	334	345	enc *json.Encoder
	335	346	dec *json.Decoder
	336	347	mu sync.Mutex
		348	// stream is the log streamer for the job this session is running.
		349	stream *logStream
	337	350	// name is the account the server resolved from the connection. A record
	338	351	// names it, and the gate refuses a record whose signer is not the pusher.
	339	352	name string
@@ -470,6 +483,11 @@ func (s runnerSession) run(remote, cache string, job runnerJob, lease time.Dur
	470	483	return
	471	484	}
	472	485	defer log.Close()
		486	s.stream = s.startLogStream(job, log)
		487	defer func() {
		488	s.stream.stop()
		489	s.stream = nil
		490	}()
	473	491	co, err := prepareWorktree(remote, cache, job.Repo, job.SHA, job.ID, job.Attempt)
	474	492	if err != nil {
	475	493	// No checkout means no branch to commit a record to, so there is nothing to
@@ -625,13 +643,14 @@ func (s runnerSession) cacheFor(job runnerJob) []container.Mount {
	625	643	// and the failure is reported on stderr rather than swallowed. A record the server
	626	644	// does not accept is not reported at all.
	627	645	func (s runnerSession) finish(remote string, co checkout, job runnerJob, log *spool, exit int, steps []runs.StepResult, arts, releaseArts []runnerArtifact, box runnerBox) {
	628		oid, size, err := uploadLog(remote, job.Repo, log)
	629		if err != nil {
	630		fmt.Fprintf(os.Stderr, "vole runner: could not upload the log for %s: %v\n", job.ID, err)
		646	if err := log.seal(); err != nil {
		647	fmt.Fprintf(os.Stderr, "vole runner: could not seal the log for %s: %v\n", job.ID, err)
	631	648	}
	632		if oid == "" {
	633		// The step ranges describe bytes in a log this attempt does not have. Keep
	634		// the timings, which are still true, and drop the claims about the log.
		649	streamErr := s.stream.drain()
		650	if streamErr != nil {
		651	fmt.Fprintf(os.Stderr, "vole runner: log stream for %s failed: %v\n", job.ID, streamErr)
		652	// The step ranges describe bytes the server does not have. Keep the timings,
		653	// which are still true, and drop the claims about the log.
	635	654	for i := range steps {
	636	655	steps[i].LogOffset, steps[i].LogSize = 0, 0
	637	656	}
@@ -652,8 +671,8 @@ func (s runnerSession) finish(remote string, co checkout, job runnerJob, log *
	652	671	}
	653	672	rec.Image, rec.ImageFile, rec.ImageSHA = job.Image, job.ImageFile, job.ImageSHA
	654	673	rec.Runtime = box.Runtime
	655		if oid != "" {
	656		rec.Log = &lfs.Pointer{OID: oid, Size: size}
		674	if streamErr == nil {
		675	rec.LogSize = log.Len()
	657	676	}
	658	677
	659	678	// Keep the lease through the report: the record is checked against it, and a
@@ -746,34 +765,6 @@ func checkReleaseJob(job *runnerJob) error {
	746	765	return nil
	747	766	}
	748	767
	749		// uploadLog puts the attempt's log in the instance's object store over the same
	750		// SSH transfer the artifacts use, and returns the id the store computed. The spool
	751		// is already a file, so it is sent as it is: no copy through memory and no second
	752		// pass to hash it. An empty log uploads nothing and reports no object, so a run
	753		// with no output has no log rather than an empty file.
	754		func uploadLog(remote, repo string, log *spool) (oid string, size int64, err error) {
	755		if log == nil \|\| log.Len() == 0 {
	756		return "", 0, nil
	757		}
	758		if err := log.seal(); err != nil {
	759		return "", 0, err
	760		}
	761		url, err := client.RepoURL(remote, repo)
	762		if err != nil {
	763		return "", 0, err
	764		}
	765		tx, err := client.DialTransfer(url, "upload")
	766		if err != nil {
	767		return "", 0, err
	768		}
	769		defer tx.Close()
	770		ptr, err := tx.Upload(log.Path())
	771		if err != nil {
	772		return "", 0, err
	773		}
	774		return ptr.OID, ptr.Size, nil
	775		}
	776
	777	768	// publishArtifacts expands the job's declared patterns in the checkout and uploads
	778	769	// what they match through the same SSH object transfer git-lfs uses. The bytes go
	779	770	// in before the finish, so the server can refuse a claim it cannot serve.