vole

Clone

Read only, no credentials:

https://rodent.software/vole.git

Write over SSH with a registered key:

git clone ssh://git@rodent.software:2222/vole.git

A free and open-source forge that lives in your Git repo.

Mask secret values in the log as it is written

e82bbe7ddc584dc54ee57c7ce3b97f933c9e9886 · Keenan Nemetz · 7d ago · parent 4ac42680

Mask secret values in the log as it is written

A step that prints a secret by accident used to put it in the attempt log, and
the log is streamed to the instance and kept. The runner now replaces every
value it was given with *** before the bytes reach the log file, so the copy
the server stores never holds one and a value split across two writes is still
caught: the mask holds back the last longest-1 bytes and the runner flushes at
the end of each step.

ci/redact is the one implementation, so a local `vole workflow run` masks its
terminal output the same way. Its own error text goes through the same mask,
because "the runtime message does not contain a value" should be true rather
than assumed.

It is best effort, and the docs say where it stops: a re-encoded value does not
match, a value shorter than four bytes is left alone because masking every
occurrence of it would rewrite the log, and an artifact or a site file is
published exactly as the step wrote it.

Slice 3 of #154.

ci/redact/redact.go +180

@@ -0,0 +1,180 @@
	1	// Package redact keeps a job's secret values out of the text it produces.
	2	//
	3	// The attempt log is written on the runner and streamed to the instance, so the
	4	// runner is the last place a value can be removed before it becomes part of a
	5	// record somebody reads. A local run has the same problem with a terminal and a
	6	// scrollback buffer. This package is the one implementation both use: it replaces
	7	// each delivered value with Placeholder as the bytes go by, and it holds back the
	8	// last few bytes of a write so a value split across two writes is still caught.
	9	//
	10	// It is not a guarantee. A value that has been re-encoded — base64, hex, a URL
	11	// escape, one piece of a connection string — no longer matches the bytes that were
	12	// delivered, and no amount of scanning finds it. A value shorter than MinBytes is
	13	// not masked at all: replacing every occurrence of "1" or "yes" would rewrite the
	14	// output and protect nothing that was worth calling a secret.
	15	package redact
	16
	17	import (
	18	"bytes"
	19	"io"
	20	"sort"
	21	"sync"
	22	)
	23
	24	const (
	25	// Placeholder is what a masked value becomes. It is short on purpose: a masked
	26	// log should still read as a log.
	27	Placeholder = "***"
	28	// MinBytes is the shortest value worth masking. A three-byte string has 32
	29	// possibilities per byte and is a flag, not a credential.
	30	MinBytes = 4
	31	)
	32
	33	// Mask is a set of values to remove from a stream of bytes.
	34	//
	35	// A Mask is not safe for concurrent use. A stream that is written from more than
	36	// one goroutine uses a Writer, which serialises writes and owns its Mask.
	37	type Mask struct {
	38	patterns [][]byte
	39	longest int
	40	pending []byte
	41	}
	42
	43	// New builds a mask from the values a job was given. Short values are dropped, and
	44	// the longest match wins where one value is a prefix of another, so the tail of a
	45	// longer value is not left behind.
	46	func New(values []string) *Mask {
	47	m := &Mask{}
	48	seen := make(map[string]bool, len(values))
	49	for _, value := range values {
	50	if len(value) < MinBytes \|\| seen[value] {
	51	continue
	52	}
	53	seen[value] = true
	54	m.patterns = append(m.patterns, []byte(value))
	55	}
	56	sort.Slice(m.patterns, func(i, j int) bool { return len(m.patterns[i]) > len(m.patterns[j]) })
	57	for _, pattern := range m.patterns {
	58	m.longest = max(m.longest, len(pattern))
	59	}
	60	return m
	61	}
	62
	63	// Empty reports whether there is nothing to mask, so a caller can skip the work.
	64	func (m *Mask) Empty() bool { return m == nil \|\| len(m.patterns) == 0 }
	65
	66	// Write takes one write's bytes and returns the prefix that can be emitted now,
	67	// already masked. The last longest-1 bytes are held back, because a value may
	68	// continue in the next write; Flush emits them.
	69	func (m *Mask) Write(p []byte) []byte {
	70	if m.Empty() {
	71	return p
	72	}
	73	m.pending = append(m.pending, p...)
	74	keep := m.longest - 1
	75	if len(m.pending) <= keep {
	76	return nil
	77	}
	78	cut := len(m.pending) - keep
	79	out, taken := m.redactTo(m.pending, cut)
	80	// A match may have started before the cut and ended after it; it was emitted
	81	// whole, so it is consumed whole.
	82	m.pending = append(m.pending[:0], m.pending[taken:]...)
	83	return out
	84	}
	85
	86	// Flush returns the held-back bytes, masked, and clears them. It is called when a
	87	// step ends and when the log is sealed, so the tail is never lost.
	88	func (m *Mask) Flush() []byte {
	89	if m.Empty() \|\| len(m.pending) == 0 {
	90	return nil
	91	}
	92	out, _ := m.redactTo(m.pending, len(m.pending))
	93	m.pending = nil
	94	return out
	95	}
	96
	97	// String masks one complete message, such as a runner's own error text, without
	98	// touching the carry of the stream. It is for callers that have a whole string and
	99	// not a stream.
	100	func (m *Mask) String(s string) string {
	101	if m.Empty() \|\| s == "" {
	102	return s
	103	}
	104	buf := []byte(s)
	105	out, _ := m.redactTo(buf, len(buf))
	106	return string(out)
	107	}
	108
	109	// redactTo masks occurrences in buf that start before cut and reports how many
	110	// input bytes it consumed, which can run past cut when a match spans it.
	111	func (m *Mask) redactTo(buf []byte, cut int) (out []byte, taken int) {
	112	out = make([]byte, 0, cut)
	113	i := 0
	114	for i < cut {
	115	if n := m.matchAt(buf, i); n > 0 {
	116	out = append(out, Placeholder...)
	117	i += n
	118	continue
	119	}
	120	out = append(out, buf[i])
	121	i++
	122	}
	123	return out, i
	124	}
	125
	126	// matchAt is the length of the longest value that starts at i, or zero.
	127	func (m *Mask) matchAt(buf []byte, i int) int {
	128	for _, pattern := range m.patterns {
	129	if len(pattern) <= len(buf)-i && bytes.Equal(buf[i:i+len(pattern)], pattern) {
	130	return len(pattern)
	131	}
	132	}
	133	return 0
	134	}
	135
	136	// Writer masks a stream and appends it to another writer. A command's stdout and
	137	// stderr are copied by two goroutines, so one Writer may be written from both; the
	138	// mutex makes that a serialised stream, which is what the log is anyway.
	139	type Writer struct {
	140	mu sync.Mutex
	141	m *Mask
	142	out io.Writer
	143	}
	144
	145	// NewWriter wraps out and masks these values.
	146	func NewWriter(out io.Writer, values []string) *Writer {
	147	return &Writer{m: New(values), out: out}
	148	}
	149
	150	// Write masks as much of p as can be emitted and reports the whole write as taken,
	151	// so a masked value never stops a program with a short write.
	152	func (w *Writer) Write(p []byte) (int, error) {
	153	w.mu.Lock()
	154	defer w.mu.Unlock()
	155	if !w.m.Empty() {
	156	if err := w.write(w.m.Write(p)); err != nil {
	157	return 0, err
	158	}
	159	return len(p), nil
	160	}
	161	return w.out.Write(p)
	162	}
	163
	164	// Flush writes whatever the mask held back. A caller calls it when the stream ends.
	165	func (w *Writer) Flush() error {
	166	w.mu.Lock()
	167	defer w.mu.Unlock()
	168	if w.m.Empty() {
	169	return nil
	170	}
	171	return w.write(w.m.Flush())
	172	}
	173
	174	func (w *Writer) write(b []byte) error {
	175	if len(b) == 0 {
	176	return nil
	177	}
	178	_, err := w.out.Write(b)
	179	return err
	180	}