vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Mask secret values in the log as it is written
e82bbe7ddc584dc54ee57c7ce3b97f933c9e9886 · Keenan Nemetz · 7d ago · parent 4ac42680
Mask secret values in the log as it is written A step that prints a secret by accident used to put it in the attempt log, and the log is streamed to the instance and kept. The runner now replaces every value it was given with *** before the bytes reach the log file, so the copy the server stores never holds one and a value split across two writes is still caught: the mask holds back the last longest-1 bytes and the runner flushes at the end of each step. ci/redact is the one implementation, so a local `vole workflow run` masks its terminal output the same way. Its own error text goes through the same mask, because "the runtime message does not contain a value" should be true rather than assumed. It is best effort, and the docs say where it stops: a re-encoded value does not match, a value shorter than four bytes is left alone because masking every occurrence of it would rewrite the log, and an artifact or a site file is published exactly as the step wrote it. Slice 3 of #154.
ci/agent/spool.go +56−1
| @@ -14,6 +14,7 @@ import ( | |||
| 14 | 14 | "sync" | |
| 15 | 15 | ||
| 16 | 16 | "github.com/rodent-software/vole/ci/logs" | |
| 17 | "github.com/rodent-software/vole/ci/redact" | ||
| 17 | 18 | ) | |
| 18 | 19 | ||
| 19 | 20 | const ( | |
| @@ -48,6 +49,10 @@ type spool struct { | |||
| 48 | 49 | n int64 | |
| 49 | 50 | limit int64 | |
| 50 | 51 | cut bool | |
| 52 | // mask removes the job's secret values as the log is written, so the file the | ||
| 53 | // streamer sends and the instance stores never holds one. It is set before the | ||
| 54 | // first step and before anything a step could print. | ||
| 55 | mask *redact.Mask | ||
| 51 | 56 | } | |
| 52 | 57 | ||
| 53 | 58 | // newSpool opens a log file in the runner's temporary directory. A runner that | |
| @@ -61,20 +66,36 @@ func newSpool(limit int64) (*spool, error) { | |||
| 61 | 66 | return &spool{f: f, limit: limit}, nil | |
| 62 | 67 | } | |
| 63 | 68 | ||
| 69 | // SetMask makes the log redact the job's secret values from everything written | ||
| 70 | // after this call. It is called before the first step, so a step cannot print one. | ||
| 71 | func (s *spool) SetMask(mask *redact.Mask) { | ||
| 72 | s.mu.Lock() | ||
| 73 | defer s.mu.Unlock() | ||
| 74 | s.mask = mask | ||
| 75 | } | ||
| 76 | |||
| 64 | 77 | // Write keeps at most limit bytes and reports the whole write as taken, so a | |
| 65 | 78 | // chatty process is not killed by a short write; the excess is dropped and noted. | |
| 79 | // A value is masked before the cap is counted, so the limit bounds what a reader | ||
| 80 | // sees rather than what the step printed. | ||
| 66 | 81 | func (s *spool) Write(p []byte) (int, error) { | |
| 67 | 82 | s.mu.Lock() | |
| 68 | 83 | defer s.mu.Unlock() | |
| 69 | 84 | if s.f == nil { | |
| 70 | 85 | return len(p), nil | |
| 71 | 86 | } | |
| 87 | chunk := p | ||
| 88 | if s.mask != nil { | ||
| 89 | chunk = s.mask.Write(p) | ||
| 90 | } | ||
| 91 | if len(chunk) == 0 { | ||
| 92 | return len(p), nil | ||
| 93 | } | ||
| 72 | 94 | room := s.limit - s.n | |
| 73 | 95 | if room <= 0 { | |
| 74 | 96 | s.cut = true | |
| 75 | 97 | return len(p), nil | |
| 76 | 98 | } | |
| 77 | chunk := p | ||
| 78 | 99 | if int64(len(chunk)) > room { | |
| 79 | 100 | chunk = chunk[:room] | |
| 80 | 101 | s.cut = true | |
| @@ -105,6 +126,30 @@ func (s *spool) Path() string { | |||
| 105 | 126 | return s.f.Name() | |
| 106 | 127 | } | |
| 107 | 128 | ||
| 129 | // Flush writes whatever the mask held back. The runner calls it at the end of each | ||
| 130 | // step, so a step's byte range ends where its own output ended rather than carrying | ||
| 131 | // a few held-back bytes into the next step's range. | ||
| 132 | func (s *spool) Flush() { | ||
| 133 | s.mu.Lock() | ||
| 134 | defer s.mu.Unlock() | ||
| 135 | if s.mask == nil || s.f == nil { | ||
| 136 | return | ||
| 137 | } | ||
| 138 | tail := s.mask.Flush() | ||
| 139 | room := s.limit - s.n | ||
| 140 | if room <= 0 || len(tail) == 0 { | ||
| 141 | return | ||
| 142 | } | ||
| 143 | if int64(len(tail)) > room { | ||
| 144 | tail = tail[:room] | ||
| 145 | } | ||
| 146 | // The file's own error surfaces at seal, which is where a caller can act on it; | ||
| 147 | // this tail is at most one value's worth of bytes. | ||
| 148 | if _, err := s.f.Write(tail); err == nil { | ||
| 149 | s.n += int64(len(tail)) | ||
| 150 | } | ||
| 151 | } | ||
| 152 | |||
| 108 | 153 | // seal finishes the log: a truncated log gets the note explaining why it ends, | |
| 109 | 154 | // and the file is synced so the upload reads what was written. | |
| 110 | 155 | func (s *spool) seal() error { | |
| @@ -113,6 +158,16 @@ func (s *spool) seal() error { | |||
| 113 | 158 | if s.f == nil { | |
| 114 | 159 | return nil | |
| 115 | 160 | } | |
| 161 | // The bytes the mask held back are written before the note, so a value split | ||
| 162 | // across the last two writes is still masked rather than lost. | ||
| 163 | if s.mask != nil { | ||
| 164 | if tail := s.mask.Flush(); len(tail) > 0 { | ||
| 165 | if _, err := s.f.Write(tail); err != nil { | ||
| 166 | return err | ||
| 167 | } | ||
| 168 | s.n += int64(len(tail)) | ||
| 169 | } | ||
| 170 | } | ||
| 116 | 171 | if s.cut { | |
| 117 | 172 | note := fmt.Sprintf(truncationNote, sizeLabel(s.limit)) | |
| 118 | 173 | if _, err := s.f.WriteString(note); err != nil { | |