vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Phase 4 slice 1: workflow format, validated at the push that introduces it
8fe238d48181201272a44d9ba7b8d97342afa4ca · Keenan Nemetz · 17d ago · parent 1e955bfb
Phase 4 slice 1: workflow format, validated at the push that introduces it
.vole/workflows/*.yaml is parsed in pre-receive for any push that touches it, so a
broken workflow fails the push that added it, naming the file and every problem in
it. The alternative — accepting the file and discovering months later that it
never ran — is the failure this exists to prevent.
version: 1
on: {push: {branches: [main, "release/*"]}} # globs, default: every branch
jobs: {test: {steps: [{run: go test ./..., timeout: 20m}]}}
Strict by construction:
- Unknown keys are rejected rather than ignored, so `runs-on:` or a typo'd `setps:`
cannot silently do nothing.
- Every problem is reported in one rejection, not the first one only: a person
fixes the file once instead of once per push.
- Job names, step bodies, timeouts (default 10m, cap 1h) and the directory layout
are all checked, including files that are not .yaml/.yml and nested directories.
- The validator is in internal/workflow and `vole workflow check` runs the same
code, so client and server cannot disagree about what is valid.
Validation is scoped by a diff: a push that does not touch the workflow directory
costs one `git diff` and nothing else.
Adds gopkg.in/yaml.v3 (already in the module cache). Nothing executes yet — the
queue, runner and status reporting are the next slice, and the README says so
plainly rather than implying CI runs.
Verified on a live instance: pushing a workflow with a step that runs nothing and a
9h timeout is rejected with both problems named; the corrected file pushes and
`vole workflow list` reports `ci.yaml on main,release/* test(2 steps)`.
internal/workflow/workflow.go +306
| @@ -0,0 +1,306 @@ | |||
| 1 | // Package workflow validates Vole's CI workflow files. | ||
| 2 | // | ||
| 3 | // .vole/workflows/<name>.yaml | ||
| 4 | // | ||
| 5 | // The format is deliberately small: jobs of shell steps, filtered by branch. It | ||
| 6 | // is validated when it is pushed, so a broken workflow fails the push that | ||
| 7 | // introduced it instead of silently never running — which is the failure mode | ||
| 8 | // that wastes the most of a person's afternoon. | ||
| 9 | package workflow | ||
| 10 | |||
| 11 | import ( | ||
| 12 | "fmt" | ||
| 13 | "os" | ||
| 14 | "path" | ||
| 15 | "path/filepath" | ||
| 16 | "regexp" | ||
| 17 | "sort" | ||
| 18 | "strings" | ||
| 19 | "time" | ||
| 20 | |||
| 21 | "github.com/rodent-software/vole/internal/gitx" | ||
| 22 | "gopkg.in/yaml.v3" | ||
| 23 | ) | ||
| 24 | |||
| 25 | const ( | ||
| 26 | // Dir is where workflows live, relative to the repository root. | ||
| 27 | Dir = ".vole/workflows" | ||
| 28 | |||
| 29 | MaxFileBytes = 256 << 10 | ||
| 30 | MaxStepBytes = 64 << 10 | ||
| 31 | MaxJobs = 50 | ||
| 32 | MaxSteps = 100 | ||
| 33 | // DefaultTimeout applies to a step that does not set one. A hung step must not | ||
| 34 | // hold a runner forever. | ||
| 35 | DefaultTimeout = 10 * time.Minute | ||
| 36 | MaxTimeout = time.Hour | ||
| 37 | // Version is the only schema this binary understands. | ||
| 38 | Version = 1 | ||
| 39 | ) | ||
| 40 | |||
| 41 | var ( | ||
| 42 | jobIDRE = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,31}$`) | ||
| 43 | branchGlob = regexp.MustCompile(`^[A-Za-z0-9*?\[\]!._/-]+$`) | ||
| 44 | ) | ||
| 45 | |||
| 46 | type Step struct { | ||
| 47 | Name string | ||
| 48 | Run string | ||
| 49 | Timeout time.Duration | ||
| 50 | } | ||
| 51 | |||
| 52 | type Job struct { | ||
| 53 | ID string | ||
| 54 | Steps []Step | ||
| 55 | } | ||
| 56 | |||
| 57 | type Workflow struct { | ||
| 58 | Path string | ||
| 59 | Version int | ||
| 60 | Branches []string | ||
| 61 | Jobs []Job | ||
| 62 | } | ||
| 63 | |||
| 64 | // Applies reports whether this workflow should run for a push to branch. | ||
| 65 | func (w *Workflow) Applies(branch string) bool { | ||
| 66 | for _, pattern := range w.Branches { | ||
| 67 | if ok, err := path.Match(pattern, branch); err == nil && ok { | ||
| 68 | return true | ||
| 69 | } | ||
| 70 | } | ||
| 71 | return false | ||
| 72 | } | ||
| 73 | |||
| 74 | type rawStep struct { | ||
| 75 | Name string `yaml:"name"` | ||
| 76 | Run string `yaml:"run"` | ||
| 77 | Timeout string `yaml:"timeout"` | ||
| 78 | } | ||
| 79 | |||
| 80 | type rawJob struct { | ||
| 81 | Steps []rawStep `yaml:"steps"` | ||
| 82 | } | ||
| 83 | |||
| 84 | type rawWorkflow struct { | ||
| 85 | Version int `yaml:"version"` | ||
| 86 | On struct { | ||
| 87 | Push struct { | ||
| 88 | Branches []string `yaml:"branches"` | ||
| 89 | } `yaml:"push"` | ||
| 90 | } `yaml:"on"` | ||
| 91 | Jobs map[string]rawJob `yaml:"jobs"` | ||
| 92 | } | ||
| 93 | |||
| 94 | // Parse validates one workflow file. Every problem is reported, not just the first. | ||
| 95 | func Parse(file string, content string) (*Workflow, []string) { | ||
| 96 | var problems []string | ||
| 97 | bad := func(format string, args ...any) { | ||
| 98 | problems = append(problems, fmt.Sprintf("%s: %s", file, fmt.Sprintf(format, args...))) | ||
| 99 | } | ||
| 100 | |||
| 101 | if len(content) > MaxFileBytes { | ||
| 102 | bad("file is %d bytes, limit %d", len(content), MaxFileBytes) | ||
| 103 | return nil, problems | ||
| 104 | } | ||
| 105 | |||
| 106 | var rw rawWorkflow | ||
| 107 | dec := yaml.NewDecoder(strings.NewReader(content)) | ||
| 108 | dec.KnownFields(true) | ||
| 109 | if err := dec.Decode(&rw); err != nil { | ||
| 110 | if err.Error() == "EOF" { | ||
| 111 | bad("file is empty") | ||
| 112 | return nil, problems | ||
| 113 | } | ||
| 114 | bad("%s", tidyYAMLError(err)) | ||
| 115 | return nil, problems | ||
| 116 | } | ||
| 117 | |||
| 118 | if rw.Version == 0 { | ||
| 119 | bad("version is required (currently %d)", Version) | ||
| 120 | } else if rw.Version != Version { | ||
| 121 | bad("version %d is not supported; this server runs version %d", rw.Version, Version) | ||
| 122 | } | ||
| 123 | |||
| 124 | wf := &Workflow{Path: file, Version: rw.Version} | ||
| 125 | wf.Branches = rw.On.Push.Branches | ||
| 126 | if len(wf.Branches) == 0 { | ||
| 127 | wf.Branches = []string{"*"} | ||
| 128 | } | ||
| 129 | for _, b := range wf.Branches { | ||
| 130 | if strings.TrimSpace(b) == "" || !branchGlob.MatchString(b) || strings.Contains(b, "..") { | ||
| 131 | bad("on.push.branches: %q is not a branch pattern", b) | ||
| 132 | } | ||
| 133 | } | ||
| 134 | |||
| 135 | if len(rw.Jobs) == 0 { | ||
| 136 | bad("at least one job is required") | ||
| 137 | } | ||
| 138 | if len(rw.Jobs) > MaxJobs { | ||
| 139 | bad("%d jobs, limit %d", len(rw.Jobs), MaxJobs) | ||
| 140 | } | ||
| 141 | |||
| 142 | ids := make([]string, 0, len(rw.Jobs)) | ||
| 143 | for id := range rw.Jobs { | ||
| 144 | ids = append(ids, id) | ||
| 145 | } | ||
| 146 | sort.Strings(ids) | ||
| 147 | for _, id := range ids { | ||
| 148 | jr := rw.Jobs[id] | ||
| 149 | if !jobIDRE.MatchString(id) { | ||
| 150 | bad("jobs.%s: job names are 1-32 characters, lowercase letters, digits, - and _, starting with a letter", id) | ||
| 151 | } | ||
| 152 | if len(jr.Steps) == 0 { | ||
| 153 | bad("jobs.%s: at least one step is required", id) | ||
| 154 | continue | ||
| 155 | } | ||
| 156 | if len(jr.Steps) > MaxSteps { | ||
| 157 | bad("jobs.%s: %d steps, limit %d", id, len(jr.Steps), MaxSteps) | ||
| 158 | } | ||
| 159 | job := Job{ID: id} | ||
| 160 | for i, sr := range jr.Steps { | ||
| 161 | where := fmt.Sprintf("jobs.%s.steps[%d]", id, i) | ||
| 162 | run := strings.TrimSpace(sr.Run) | ||
| 163 | if run == "" { | ||
| 164 | bad("%s: run is required (this runs a shell command)", where) | ||
| 165 | continue | ||
| 166 | } | ||
| 167 | if len(sr.Run) > MaxStepBytes { | ||
| 168 | bad("%s: run is %d bytes, limit %d", where, len(sr.Run), MaxStepBytes) | ||
| 169 | } | ||
| 170 | step := Step{Name: strings.TrimSpace(sr.Name), Run: run, Timeout: DefaultTimeout} | ||
| 171 | if sr.Timeout != "" { | ||
| 172 | d, err := time.ParseDuration(sr.Timeout) | ||
| 173 | switch { | ||
| 174 | case err != nil: | ||
| 175 | bad("%s.timeout: %q is not a duration (try 90s, 10m)", where, sr.Timeout) | ||
| 176 | case d <= 0: | ||
| 177 | bad("%s.timeout: must be positive", where) | ||
| 178 | case d > MaxTimeout: | ||
| 179 | bad("%s.timeout: %s exceeds the %s limit", where, d, MaxTimeout) | ||
| 180 | default: | ||
| 181 | step.Timeout = d | ||
| 182 | } | ||
| 183 | } | ||
| 184 | if step.Name == "" { | ||
| 185 | step.Name = firstLine(run) | ||
| 186 | } | ||
| 187 | job.Steps = append(job.Steps, step) | ||
| 188 | } | ||
| 189 | wf.Jobs = append(wf.Jobs, job) | ||
| 190 | } | ||
| 191 | if len(problems) > 0 { | ||
| 192 | return nil, problems | ||
| 193 | } | ||
| 194 | return wf, nil | ||
| 195 | } | ||
| 196 | |||
| 197 | // Check validates workflow files given as path to contents, e.g. a pushed tree. | ||
| 198 | func Check(files map[string]string) ([]*Workflow, []string) { | ||
| 199 | var out []*Workflow | ||
| 200 | var problems []string | ||
| 201 | |||
| 202 | paths := make([]string, 0, len(files)) | ||
| 203 | for p := range files { | ||
| 204 | paths = append(paths, p) | ||
| 205 | } | ||
| 206 | sort.Strings(paths) | ||
| 207 | |||
| 208 | for _, p := range paths { | ||
| 209 | if !strings.HasPrefix(p, Dir+"/") { | ||
| 210 | continue | ||
| 211 | } | ||
| 212 | rel := strings.TrimPrefix(p, Dir+"/") | ||
| 213 | if strings.Contains(rel, "/") { | ||
| 214 | problems = append(problems, fmt.Sprintf( | ||
| 215 | "%s: workflows live directly in %s, not in subdirectories", p, Dir)) | ||
| 216 | continue | ||
| 217 | } | ||
| 218 | if !strings.HasSuffix(rel, ".yaml") && !strings.HasSuffix(rel, ".yml") { | ||
| 219 | problems = append(problems, fmt.Sprintf( | ||
| 220 | "%s: a workflow file must end in .yaml or .yml", p)) | ||
| 221 | continue | ||
| 222 | } | ||
| 223 | wf, probs := Parse(p, files[p]) | ||
| 224 | problems = append(problems, probs...) | ||
| 225 | if wf != nil { | ||
| 226 | out = append(out, wf) | ||
| 227 | } | ||
| 228 | } | ||
| 229 | return out, problems | ||
| 230 | } | ||
| 231 | |||
| 232 | // tidyYAMLError drops the Go type name yaml.v3 leaks into its messages, which | ||
| 233 | // means nothing to someone editing a workflow. | ||
| 234 | func tidyYAMLError(err error) string { | ||
| 235 | msg := err.Error() | ||
| 236 | msg = strings.ReplaceAll(msg, "yaml: unmarshal errors:\n ", "") | ||
| 237 | for _, suffix := range []string{ | ||
| 238 | " not found in type workflow.rawJob", | ||
| 239 | " not found in type workflow.rawStep", | ||
| 240 | " not found in type workflow.rawWorkflow", | ||
| 241 | " not found in type workflow.raw", | ||
| 242 | } { | ||
| 243 | msg = strings.ReplaceAll(msg, suffix, " is not a known key here") | ||
| 244 | } | ||
| 245 | msg = strings.ReplaceAll(msg, "yaml: ", "") | ||
| 246 | return strings.TrimSpace(msg) | ||
| 247 | } | ||
| 248 | |||
| 249 | func firstLine(s string) string { | ||
| 250 | if i := strings.IndexByte(s, '\n'); i >= 0 { | ||
| 251 | s = s[:i] | ||
| 252 | } | ||
| 253 | s = strings.TrimSpace(s) | ||
| 254 | if len(s) > 60 { | ||
| 255 | s = s[:57] + "..." | ||
| 256 | } | ||
| 257 | return s | ||
| 258 | } | ||
| 259 | |||
| 260 | // ValidateTree checks the workflows in a git tree. Called from pre-receive, so a | ||
| 261 | // bad workflow fails the push that introduced it. | ||
| 262 | func ValidateTree(g *gitx.Repo, commit string) []string { | ||
| 263 | if commit == "" || commit == gitx.ZeroSHA { | ||
| 264 | return nil | ||
| 265 | } | ||
| 266 | blobs, err := g.LsTreeR(commit) | ||
| 267 | if err != nil { | ||
| 268 | return []string{fmt.Sprintf("cannot read %s: %v", Dir, err)} | ||
| 269 | } | ||
| 270 | files := map[string]string{} | ||
| 271 | for _, e := range blobs { | ||
| 272 | if !strings.HasPrefix(e.Name, Dir+"/") { | ||
| 273 | continue | ||
| 274 | } | ||
| 275 | data, err := g.BlobBySHA(e.SHA) | ||
| 276 | if err != nil { | ||
| 277 | return []string{fmt.Sprintf("cannot read %s: %v", e.Name, err)} | ||
| 278 | } | ||
| 279 | files[e.Name] = string(data) | ||
| 280 | } | ||
| 281 | _, problems := Check(files) | ||
| 282 | return problems | ||
| 283 | } | ||
| 284 | |||
| 285 | // CheckDir validates the workflows in a working tree, for `vole workflow check`. | ||
| 286 | func CheckDir(root string) ([]*Workflow, []string, error) { | ||
| 287 | pattern := filepath.Join(root, filepath.FromSlash(Dir), "*.y*ml") | ||
| 288 | paths, err := filepath.Glob(pattern) | ||
| 289 | if err != nil { | ||
| 290 | return nil, nil, err | ||
| 291 | } | ||
| 292 | files := map[string]string{} | ||
| 293 | for _, p := range paths { | ||
| 294 | data, err := os.ReadFile(p) | ||
| 295 | if err != nil { | ||
| 296 | return nil, nil, err | ||
| 297 | } | ||
| 298 | rel, err := filepath.Rel(root, p) | ||
| 299 | if err != nil { | ||
| 300 | rel = p | ||
| 301 | } | ||
| 302 | files[filepath.ToSlash(rel)] = string(data) | ||
| 303 | } | ||
| 304 | wfs, problems := Check(files) | ||
| 305 | return wfs, problems, nil | ||
| 306 | } | ||