vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
3b39a84c
Keenan Nemetz
14d ago
|
1 | // Package container runs a job's steps in the OCI container its image names. |
| 2 | // | |
| 3 | // Two callers share it. The runner uses it as the boundary between untrusted | |
| 4 | // repository code and the machine that executes it: one bind mount, no | |
| 5 | // capabilities, no new privileges, and no network unless the operator asked for | |
| 6 | // it. `vole workflow run` uses it so a local run is the run CI performs — same | |
| 7 | // image, same environment, same paths — rather than whatever happens to be | |
| 8 | // installed on the developer's laptop. | |
| 9 | // | |
| 10 | // What protects the host is the mount set and the credential boundary, not the | |
| 11 | // writability of the container's own filesystem: the task usually runs as an | |
| 12 | // unprivileged uid, and everything outside the image and the job tree simply is | |
| 13 | // not reachable. The container's filesystem is left writable because images set up | |
| 14 | // world-writable toolchain directories (Go's /go is one) for exactly this case, and | |
| 15 | // because the whole layer is discarded with the container. It is immutable in the | |
| 16 | // one configuration where that matters — a non-root job cannot write the parts of | |
| 17 | // the image that DAC protects. RunsAsRoot reports the exception. | |
| 18 | // | |
| 19 | // The flag list below is that boundary in one place. Keeping it here is what stops | |
| 20 | // a local run and a pushed run from quietly disagreeing about how a step executes. | |
| 21 | package container | |
| 22 | ||
| 23 | import ( | |
|
f28097af
Keenan Nemetz
1h ago
|
24 | "bytes" |
|
3b39a84c
Keenan Nemetz
14d ago
|
25 | "context" |
|
c1929d2f
Keenan Nemetz
5d ago
|
26 | "encoding/json" |
|
3b39a84c
Keenan Nemetz
14d ago
|
27 | "fmt" |
| 28 | "io" | |
| 29 | "os" | |
| 30 | "os/exec" | |
| 31 | "path/filepath" | |
| 32 | "sort" | |
| 33 | "strconv" | |
| 34 | "strings" | |
|
c1929d2f
Keenan Nemetz
5d ago
|
35 | "sync" |
|
3b39a84c
Keenan Nemetz
14d ago
|
36 | "time" |
| 37 | ||
|
d37d301f
Keenan Nemetz
15h ago
|
38 | "rodent.software/vole/ci" |
|
3b39a84c
Keenan Nemetz
14d ago
|
39 | ) |
| 40 | ||
|
c1929d2f
Keenan Nemetz
5d ago
|
41 | // MaxServiceLogBytes bounds one service's output in an attempt log. A database |
| 42 | // that fails to start says why at the end; the tail is what the reader needs, | |
| 43 | // and the cap keeps three services from crowding out the steps. | |
| 44 | const MaxServiceLogBytes = 256 << 10 | |
| 45 | ||
|
4e3759da
Keenan Nemetz
1h ago
|
46 | // DefaultPIDs bounds a container's processes when the operator does not. A build |
| 47 | // that needs more than this many threads or children is misconfigured; the point of | |
| 48 | // the default is that a fork bomb costs the job, not the machine. | |
| 49 | const DefaultPIDs = 2048 | |
| 50 | ||
| 51 | // DefaultTmpfs is the size of the job's /tmp when --container-tmpfs is not given. It | |
| 52 | // is a tmpfs, so it is memory, and an unbounded one is a job that can fill the | |
| 53 | // machine's RAM without writing a single byte to the workspace. | |
| 54 | const DefaultTmpfs = "1g" | |
| 55 | ||
|
3b39a84c
Keenan Nemetz
14d ago
|
56 | // KeepAlive is PID 1's command. Steps run with exec, so the container has to stay |
| 57 | // up between them. `/bin/sh` is also what the steps need, which keeps the | |
| 58 | // requirement to one the image must satisfy anyway. | |
| 59 | const KeepAlive = `trap 'exit 0' TERM INT; while :; do sleep 3600; done` | |
| 60 | ||
| 61 | // Config is the runtime and isolation settings for one container. It is validated | |
| 62 | // once, before any work starts, so a missing runtime or an invalid option is an | |
| 63 | // error at startup rather than a job that fails mysteriously later. | |
| 64 | type Config struct { | |
| 65 | Runtime string | |
| 66 | Network string | |
| 67 | Pull string | |
| 68 | User string | |
| 69 | Userns string | |
| 70 | Memory string | |
| 71 | CPUs string | |
| 72 | PIDs int | |
|
4e3759da
Keenan Nemetz
1h ago
|
73 | // Tmpfs is the size of the job's /tmp, which is a tmpfs in RAM: a compile that |
| 74 | // writes its output there and then runs it must not be able to take the machine | |
| 75 | // with it. It cannot exceed Memory when both are set, because tmpfs pages are | |
| 76 | // charged to the container's own cgroup. | |
| 77 | Tmpfs string | |
|
3b39a84c
Keenan Nemetz
14d ago
|
78 | } |
| 79 | ||
| 80 | // Validate fills in the defaults and refuses settings the runtime would reject. | |
| 81 | func Validate(cfg Config) (Config, error) { | |
| 82 | if cfg.Runtime == "" { | |
| 83 | cfg.Runtime = "docker" | |
| 84 | } | |
| 85 | switch cfg.Runtime { | |
| 86 | case "docker", "podman", "nerdctl": | |
| 87 | default: | |
| 88 | return Config{}, fmt.Errorf("--container-runtime %q is not supported; use docker, podman, or nerdctl", cfg.Runtime) | |
| 89 | } | |
| 90 | if _, err := exec.LookPath(cfg.Runtime); err != nil { | |
| 91 | return Config{}, fmt.Errorf("--container-runtime %s is not on PATH: %w", cfg.Runtime, err) | |
| 92 | } | |
| 93 | if cfg.Network == "" { | |
| 94 | cfg.Network = "none" | |
| 95 | } | |
| 96 | switch cfg.Network { | |
| 97 | case "none", "bridge": | |
| 98 | default: | |
| 99 | return Config{}, fmt.Errorf("--container-network %q is not supported; use none or bridge", cfg.Network) | |
| 100 | } | |
| 101 | if cfg.Pull == "" { | |
| 102 | cfg.Pull = "missing" | |
| 103 | } | |
| 104 | switch cfg.Pull { | |
| 105 | case "missing", "always", "never": | |
| 106 | default: | |
| 107 | return Config{}, fmt.Errorf("--container-pull %q is not supported; use missing, always, or never", cfg.Pull) | |
| 108 | } | |
| 109 | defaultUser := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) | |
| 110 | if cfg.User == "" { | |
| 111 | cfg.User = defaultUser | |
| 112 | } | |
| 113 | if strings.ContainsAny(cfg.User, " \t\r\n") { | |
| 114 | return Config{}, fmt.Errorf("--container-user %q is not a user", cfg.User) | |
| 115 | } | |
| 116 | if strings.ContainsAny(cfg.Userns, " \t\r\n") { | |
| 117 | return Config{}, fmt.Errorf("--container-userns %q is not a user namespace mode", cfg.Userns) | |
| 118 | } | |
| 119 | if cfg.Userns == "" && cfg.Runtime == "podman" && os.Geteuid() != 0 && cfg.User == defaultUser { | |
| 120 | // Rootless Podman maps container root to the caller but maps other UIDs | |
| 121 | // through subuids. keep-id makes the numeric --container-user line up with | |
| 122 | // the host user, so the bind-mounted job tree is writable. A caller that | |
| 123 | // intentionally overrides --container-user may need a different mode. | |
| 124 | cfg.Userns = "keep-id" | |
| 125 | } | |
| 126 | if cfg.PIDs < 0 { | |
| 127 | return Config{}, fmt.Errorf("--container-pids must not be negative") | |
| 128 | } | |
|
4e3759da
Keenan Nemetz
1h ago
|
129 | // Zero means the default, not "no limit": a job that can fork without a bound is |
| 130 | // a fork bomb away from taking the machine, and a bound nobody set is the easiest | |
| 131 | // one to be missing. An operator who wants a different number sets it. | |
| 132 | if cfg.PIDs == 0 { | |
| 133 | cfg.PIDs = DefaultPIDs | |
| 134 | } | |
| 135 | if cfg.Tmpfs == "" { | |
| 136 | cfg.Tmpfs = DefaultTmpfs | |
| 137 | } | |
| 138 | if strings.ContainsAny(cfg.Tmpfs, " \t\r\n") { | |
| 139 | return Config{}, fmt.Errorf("--container-tmpfs %q is not a size", cfg.Tmpfs) | |
| 140 | } | |
|
3b39a84c
Keenan Nemetz
14d ago
|
141 | return cfg, nil |
| 142 | } | |
| 143 | ||
|
f28097af
Keenan Nemetz
1h ago
|
144 | // ProbeTimeout bounds the startup probe. It is generous because the first `info` |
| 145 | // on a machine that has never run a container initializes the runtime's storage; | |
| 146 | // on a runner that has run a job it answers in milliseconds. | |
| 147 | const ProbeTimeout = 60 * time.Second | |
| 148 | ||
| 149 | // Probe proves the runtime can create a container, not just that its binary is on | |
| 150 | // PATH, so a runner that cannot run a job refuses to serve instead of taking one | |
| 151 | // and failing it. | |
| 152 | // | |
| 153 | // A missing OCI runtime, an unwritable run root, a storage driver the kernel will | |
| 154 | // not load, and an unset or missing XDG_RUNTIME_DIR all pass the LookPath check in | |
| 155 | // Validate and none of them can start a container. They are the states that make a | |
| 156 | // runtime exit 125 on the first job, and every one of them makes `info` fail. It is | |
| 157 | // cheap, it pulls no image, and it is the same query whether a job is run here or | |
| 158 | // on a runner. | |
| 159 | // | |
| 160 | // The error names the runtime and repeats its own last word, because "exited 125" | |
| 161 | // is what left an operator reading a job log to find out that the machine, not the | |
| 162 | // repository, was broken. | |
| 163 | func Probe(ctx context.Context, cfg Config) error { | |
| 164 | ctx, cancel := context.WithTimeout(ctx, ProbeTimeout) | |
| 165 | defer cancel() | |
| 166 | // Both streams, one buffer: docker reports a daemon it cannot reach on stderr | |
| 167 | // and podman writes a warning there before the error, while the report on | |
| 168 | // success goes to stdout and is not read. | |
| 169 | cmd := exec.CommandContext(ctx, cfg.Runtime, "info") | |
| 170 | var out bytes.Buffer | |
| 171 | cmd.Stdout = &out | |
| 172 | cmd.Stderr = &out | |
| 173 | err := cmd.Run() | |
| 174 | if err == nil { | |
| 175 | return nil | |
| 176 | } | |
| 177 | reason := lastLine(out.String()) | |
| 178 | if ctx.Err() == context.DeadlineExceeded { | |
| 179 | reason = fmt.Sprintf("%s info did not finish within %s", cfg.Runtime, ProbeTimeout) | |
| 180 | } | |
| 181 | if reason == "" { | |
| 182 | reason = err.Error() | |
| 183 | } | |
| 184 | return fmt.Errorf("cannot run a container with %s: %s", cfg.Runtime, reason) | |
| 185 | } | |
| 186 | ||
| 187 | // lastLine is the runtime's own verdict: podman ends with "Error: default OCI | |
| 188 | // runtime \"crun\" not found" after any number of warnings above it, and docker | |
| 189 | // says one line. The warnings are not the cause, so they are not repeated. | |
| 190 | func lastLine(out string) string { | |
| 191 | lines := strings.Split(out, "\n") | |
| 192 | for i := len(lines) - 1; i >= 0; i-- { | |
| 193 | if line := strings.TrimSpace(lines[i]); line != "" { | |
| 194 | return line | |
| 195 | } | |
| 196 | } | |
| 197 | return "" | |
| 198 | } | |
| 199 | ||
|
3b39a84c
Keenan Nemetz
14d ago
|
200 | // RunsAsRoot reports whether the task will run as uid 0 in its container. That is |
| 201 | // the one case where the image's own files are fully writable for the length of the | |
| 202 | // job, so a caller should say so: an ordinary user is protected by the image's | |
| 203 | // permissions, root is not. | |
| 204 | func RunsAsRoot(cfg Config) bool { | |
| 205 | user := cfg.User | |
| 206 | if i := strings.IndexByte(user, ':'); i >= 0 { | |
| 207 | user = user[:i] | |
| 208 | } | |
| 209 | return user == "0" || user == "root" | |
| 210 | } | |
| 211 | ||
| 212 | // Mount is one extra host directory made visible in the container. It exists for | |
| 213 | // the cache, which is a property of the runner rather than of the job: the job sees | |
| 214 | // exactly one directory and cannot reach the namespaces around it. | |
| 215 | type Mount struct { | |
| 216 | Host string | |
| 217 | Path string | |
| 218 | } | |
| 219 | ||
| 220 | // Spec is everything about one container that is not a runtime setting. | |
| 221 | type Spec struct { | |
| 222 | Image string | |
| 223 | Name string | |
| 224 | Work string // the job tree, bind-mounted read-write at /workspace | |
| 225 | Labels map[string]string | |
| 226 | // Mounts are extra directories to make visible. They are read-write and owned | |
| 227 | // by the runner's user, so a job can fill one; the caller that creates them is | |
| 228 | // the caller that has to bound them. | |
| 229 | Mounts []Mount | |
| 230 | // Env is the job's environment on top of the fixed set below. Names here cannot | |
| 231 | // collide with the reserves: the workflow validator refuses those. | |
| 232 | Env map[string]string | |
|
4ac42680
Keenan Nemetz
7d ago
|
233 | // SecretsFile, when set, is the path of a file inside the container holding |
| 234 | // `export NAME='value'` lines. Every step sources it before its own script, so a | |
| 235 | // secret reaches the step's shell and never an argv: the runtime's own argument | |
| 236 | // list is visible to every process on the host. It is a value the caller writes | |
| 237 | // into the job tree, so the caller owns what is in it. | |
| 238 | SecretsFile string | |
|
3b39a84c
Keenan Nemetz
14d ago
|
239 | } |
| 240 | ||
|
6bfc4294
Keenan Nemetz
12d ago
|
241 | // ValidatePaths refuses a spec whose paths cannot be passed as `--volume` |
| 242 | // arguments. The separator is a colon, so a path containing one would be read as | |
| 243 | // an extra field; a NUL or a newline would end or split the argument the runtime | |
| 244 | // sees. The runner builds these paths itself, so this is the check that keeps that | |
| 245 | // assumption true rather than the check that discovers it. | |
| 246 | func ValidatePaths(s Spec) error { | |
| 247 | paths := []string{s.Work} | |
| 248 | for _, m := range s.Mounts { | |
| 249 | paths = append(paths, m.Host, m.Path) | |
| 250 | } | |
|
4ac42680
Keenan Nemetz
7d ago
|
251 | if s.SecretsFile != "" { |
| 252 | if !strings.HasPrefix(s.SecretsFile, "/") { | |
| 253 | return fmt.Errorf("the secrets file %q is not a path in the container", s.SecretsFile) | |
| 254 | } | |
| 255 | paths = append(paths, s.SecretsFile) | |
| 256 | } | |
|
6bfc4294
Keenan Nemetz
12d ago
|
257 | for _, p := range paths { |
| 258 | if p == "" { | |
| 259 | return fmt.Errorf("container path is empty") | |
| 260 | } | |
| 261 | if strings.ContainsAny(p, ":\x00\r\n") { | |
| 262 | return fmt.Errorf("container path %q contains a character a volume argument cannot carry", p) | |
| 263 | } | |
| 264 | } | |
| 265 | return nil | |
| 266 | } | |
| 267 | ||
|
4e3759da
Keenan Nemetz
1h ago
|
268 | // tmpfsArg is the /tmp mount, with its own size. exec is explicit because the |
| 269 | // runtime mounts a tmpfs noexec by default; compilers write their output here and | |
| 270 | // then run it, so noexec breaks every Go workflow. | |
| 271 | func tmpfsArg(cfg Config) string { | |
| 272 | size := cfg.Tmpfs | |
| 273 | if size == "" { | |
| 274 | // Validate fills this in; a Config built by hand still gets a mounted /tmp with | |
| 275 | // a size rather than one that is unbounded, because "no size" is the failure | |
| 276 | // mode this exists to prevent. | |
| 277 | size = DefaultTmpfs | |
| 278 | } | |
| 279 | return "/tmp:rw,nosuid,nodev,exec,mode=1777,size=" + size | |
| 280 | } | |
| 281 | ||
|
3b39a84c
Keenan Nemetz
14d ago
|
282 | // RunArgs is the argument list for `docker run`. It is pure so it can be checked |
| 283 | // without a daemon. | |
| 284 | func RunArgs(cfg Config, s Spec) []string { | |
| 285 | args := []string{ | |
| 286 | "run", "-d", "--rm", | |
| 287 | "--name", s.Name, | |
| 288 | } | |
| 289 | keys := make([]string, 0, len(s.Labels)) | |
| 290 | for k := range s.Labels { | |
| 291 | keys = append(keys, k) | |
| 292 | } | |
| 293 | sort.Strings(keys) | |
| 294 | for _, k := range keys { | |
| 295 | args = append(args, "--label", k+"="+s.Labels[k]) | |
| 296 | } | |
| 297 | args = append(args, | |
| 298 | "--workdir", "/workspace", | |
| 299 | "--user", cfg.User, | |
| 300 | ) | |
| 301 | if cfg.Userns != "" { | |
| 302 | args = append(args, "--userns", cfg.Userns) | |
| 303 | } | |
| 304 | for _, kv := range []string{ | |
| 305 | "CI=true", | |
| 306 | "VOLE_CI=1", | |
| 307 | "GIT_TERMINAL_PROMPT=0", | |
| 308 | "HOME=/tmp", | |
| 309 | "TMPDIR=/tmp", | |
| 310 | } { | |
| 311 | args = append(args, "--env", kv) | |
| 312 | } | |
| 313 | envNames := make([]string, 0, len(s.Env)) | |
| 314 | for name := range s.Env { | |
| 315 | envNames = append(envNames, name) | |
| 316 | } | |
| 317 | sort.Strings(envNames) | |
| 318 | for _, name := range envNames { | |
| 319 | args = append(args, "--env", name+"="+s.Env[name]) | |
| 320 | } | |
| 321 | args = append(args, "--volume", s.Work+":/workspace:rw") | |
| 322 | for _, m := range s.Mounts { | |
| 323 | args = append(args, "--volume", m.Host+":"+m.Path+":rw") | |
| 324 | } | |
| 325 | args = append(args, | |
|
4e3759da
Keenan Nemetz
1h ago
|
326 | "--tmpfs", tmpfsArg(cfg), |
|
3b39a84c
Keenan Nemetz
14d ago
|
327 | "--cap-drop", "ALL", |
| 328 | "--security-opt", "no-new-privileges", | |
| 329 | "--init", | |
| 330 | "--stop-timeout", "5", | |
| 331 | "--network", cfg.Network, | |
| 332 | "--pull", cfg.Pull, | |
| 333 | ) | |
| 334 | if cfg.PIDs > 0 { | |
| 335 | args = append(args, "--pids-limit", strconv.Itoa(cfg.PIDs)) | |
| 336 | } | |
| 337 | if cfg.Memory != "" { | |
| 338 | args = append(args, "--memory", cfg.Memory) | |
| 339 | } | |
| 340 | if cfg.CPUs != "" { | |
| 341 | args = append(args, "--cpus", cfg.CPUs) | |
| 342 | } | |
| 343 | return append(args, | |
| 344 | "--entrypoint", "/bin/sh", | |
| 345 | s.Image, | |
| 346 | "-c", KeepAlive, | |
| 347 | ) | |
| 348 | } | |
| 349 | ||
|
4ac42680
Keenan Nemetz
7d ago
|
350 | // SecretsEnv renders the values a job runs with as the file a step sources. Each |
| 351 | // line exports one variable, and the value is POSIX single-quoted, so a quote | |
| 352 | // becomes '\” and a newline stays a newline; nothing is interpolated and no value | |
| 353 | // is part of any command line. Names are sorted so the file is the same bytes for | |
| 354 | // the same values. | |
| 355 | func SecretsEnv(values map[string]string) ([]byte, error) { | |
| 356 | names := make([]string, 0, len(values)) | |
| 357 | for name := range values { | |
| 358 | names = append(names, name) | |
| 359 | } | |
| 360 | sort.Strings(names) | |
| 361 | var b strings.Builder | |
| 362 | for _, name := range names { | |
| 363 | value := values[name] | |
| 364 | if strings.ContainsRune(value, 0) { | |
| 365 | // A NUL would end the string the shell reads and turn the rest into a | |
| 366 | // command, so it cannot be written even though the store refuses it. | |
| 367 | return nil, fmt.Errorf("secret %s contains a NUL byte", name) | |
| 368 | } | |
| 369 | b.WriteString("export " + name + "='") | |
| 370 | b.WriteString(strings.ReplaceAll(value, "'", `'\''`)) | |
| 371 | b.WriteString("'\n") | |
| 372 | } | |
| 373 | return []byte(b.String()), nil | |
| 374 | } | |
| 375 | ||
|
3b39a84c
Keenan Nemetz
14d ago
|
376 | // Container is one running container. |
| 377 | type Container struct { | |
|
4ac42680
Keenan Nemetz
7d ago
|
378 | cfg Config |
| 379 | name string | |
| 380 | work string | |
| 381 | secrets string | |
|
3b39a84c
Keenan Nemetz
14d ago
|
382 | } |
| 383 | ||
| 384 | // Start creates the container and waits for it to be running. A runtime that | |
| 385 | // cannot start it is reported with the runtime's own message: a pull failure and a | |
| 386 | // bad flag look the same from here, and the message is what tells them apart. | |
| 387 | func Start(cfg Config, s Spec) (*Container, error) { | |
|
6bfc4294
Keenan Nemetz
12d ago
|
388 | if err := ValidatePaths(s); err != nil { |
| 389 | return nil, err | |
| 390 | } | |
|
4ac42680
Keenan Nemetz
7d ago
|
391 | c := &Container{cfg: cfg, name: s.Name, work: s.Work, secrets: s.SecretsFile} |
|
3b39a84c
Keenan Nemetz
14d ago
|
392 | cmd := exec.Command(cfg.Runtime, RunArgs(cfg, s)...) |
| 393 | var stderr strings.Builder | |
| 394 | cmd.Stderr = &stderr | |
| 395 | if err := cmd.Run(); err != nil { | |
| 396 | _ = exec.Command(cfg.Runtime, "rm", "-f", s.Name).Run() | |
| 397 | return nil, fmt.Errorf("%s could not start %s: %w: %s", | |
| 398 | cfg.Runtime, s.Image, err, strings.TrimSpace(stderr.String())) | |
| 399 | } | |
| 400 | return c, nil | |
| 401 | } | |
| 402 | ||
| 403 | // Exec runs one step in the existing container. Each step is a separate exec, | |
| 404 | // which is what gives it its own log, timeout and exit status while keeping the | |
| 405 | // filesystem and background processes shared with the steps around it. | |
| 406 | func (c *Container) Exec(ctx context.Context, script string) *exec.Cmd { | |
|
4ac42680
Keenan Nemetz
7d ago
|
407 | // A step's script is prefixed with the secrets file rather than handed the |
| 408 | // values, so each step's shell exports them and no value appears in the | |
| 409 | // runtime's argument list. A step that shadows one of its own is the step's | |
| 410 | // business: the file is its own environment. | |
| 411 | if c.secrets != "" { | |
| 412 | script = ". " + c.secrets + "\n" + script | |
| 413 | } | |
|
3b39a84c
Keenan Nemetz
14d ago
|
414 | cmd := exec.CommandContext(ctx, c.cfg.Runtime, "exec", |
| 415 | "-w", "/workspace", c.name, "/bin/sh", "-c", script) | |
| 416 | // The runtime ignores the client's working directory, but keeping it at the job | |
| 417 | // tree lets a test runtime execute the step in the same place without pretending | |
| 418 | // to be a container. | |
| 419 | cmd.Dir = c.work | |
| 420 | return cmd | |
| 421 | } | |
| 422 | ||
| 423 | // Stop kills and removes the container. `rm -f` is one command that also handles a | |
| 424 | // container that has already exited. | |
| 425 | func (c *Container) Stop() { | |
| 426 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 427 | defer cancel() | |
| 428 | _ = exec.CommandContext(ctx, c.cfg.Runtime, "rm", "-f", c.name).Run() | |
| 429 | } | |
| 430 | ||
| 431 | // ImageTag names the image built from a definition. The digest is what the image | |
| 432 | // means, so a changed definition builds a different tag and an unchanged one is | |
| 433 | // reused across jobs and across runs. | |
| 434 | func ImageTag(imageFile, digest string) string { | |
| 435 | sha := digest | |
| 436 | if len(sha) > 16 { | |
| 437 | sha = sha[:16] | |
| 438 | } | |
| 439 | return "vole/image-file:" + imageFile + "-" + sha | |
| 440 | } | |
| 441 | ||
| 442 | // Exists reports whether an image is present locally. | |
| 443 | func Exists(runtime, tag string) bool { | |
| 444 | return exec.Command(runtime, "image", "inspect", tag).Run() == nil | |
| 445 | } | |
| 446 | ||
| 447 | // Build renders an image definition and builds it. It is for callers with nothing | |
| 448 | // else to keep alive; a runner builds through its own command loop so a long build | |
| 449 | // keeps its lease. | |
| 450 | func Build(ctx context.Context, cfg Config, definition, tag string, out io.Writer) error { | |
| 451 | dockerfile, err := ci.RenderImage(definition) | |
| 452 | if err != nil { | |
| 453 | return err | |
| 454 | } | |
| 455 | dir, err := os.MkdirTemp("", "vole-image-*") | |
| 456 | if err != nil { | |
| 457 | return err | |
| 458 | } | |
| 459 | defer os.RemoveAll(dir) | |
| 460 | if err := os.WriteFile(filepath.Join(dir, "Dockerfile"), []byte(dockerfile), 0o600); err != nil { | |
| 461 | return err | |
| 462 | } | |
| 463 | args := []string{"build", "-t", tag, "-f", filepath.Join(dir, "Dockerfile"), dir} | |
| 464 | switch { | |
| 465 | case cfg.Pull == "always" && cfg.Runtime == "podman": | |
| 466 | args = append(args, "--pull=always") | |
| 467 | case cfg.Pull == "always": | |
| 468 | args = append(args, "--pull") | |
| 469 | } | |
| 470 | cmd := exec.CommandContext(ctx, cfg.Runtime, args...) | |
| 471 | cmd.Stdout, cmd.Stderr = out, out | |
| 472 | if err := cmd.Run(); err != nil { | |
| 473 | return fmt.Errorf("building image %s: %w", tag, err) | |
| 474 | } | |
| 475 | if !Exists(cfg.Runtime, tag) { | |
| 476 | return fmt.Errorf("building image %s produced no image", tag) | |
| 477 | } | |
| 478 | return nil | |
| 479 | } | |
|
c1929d2f
Keenan Nemetz
5d ago
|
480 | |
| 481 | // ServiceSpec is one server container started beside a job: a database, a queue, | |
| 482 | // or another process the steps reach by name. It has no job tree and no secrets | |
| 483 | // file, and it runs the image's own command rather than a keepalive shell. | |
| 484 | type ServiceSpec struct { | |
| 485 | // Name is the container's name on the runtime. It is unique to this attempt. | |
| 486 | Name string | |
| 487 | // Alias is the name the job's network resolves to this container. It is the | |
| 488 | // service's declared name, so a step connects to `db:5432`. | |
| 489 | Alias string | |
| 490 | Image string | |
| 491 | Env map[string]string | |
| 492 | Labels map[string]string | |
| 493 | } | |
| 494 | ||
| 495 | // ServiceRunArgs is the argument list for a service container. It is the same | |
| 496 | // isolation as RunArgs where that makes sense: the job's user, capabilities | |
| 497 | // dropped, no-new-privileges, a tmpfs /tmp, the runtime's resource limits, and the | |
| 498 | // network the job was given. What is absent is the point: no workspace mount, no | |
| 499 | // cache mount, no secrets file, and no entrypoint override. | |
| 500 | func ServiceRunArgs(cfg Config, s ServiceSpec) []string { | |
| 501 | args := []string{ | |
| 502 | "run", "-d", | |
| 503 | "--name", s.Name, | |
| 504 | } | |
| 505 | keys := make([]string, 0, len(s.Labels)) | |
| 506 | for k := range s.Labels { | |
| 507 | keys = append(keys, k) | |
| 508 | } | |
| 509 | sort.Strings(keys) | |
| 510 | for _, k := range keys { | |
| 511 | args = append(args, "--label", k+"="+s.Labels[k]) | |
| 512 | } | |
| 513 | args = append(args, "--user", cfg.User) | |
| 514 | if cfg.Userns != "" { | |
| 515 | args = append(args, "--userns", cfg.Userns) | |
| 516 | } | |
| 517 | if s.Alias != "" { | |
| 518 | args = append(args, "--network-alias", s.Alias) | |
| 519 | } | |
| 520 | names := make([]string, 0, len(s.Env)) | |
| 521 | for name := range s.Env { | |
| 522 | names = append(names, name) | |
| 523 | } | |
| 524 | sort.Strings(names) | |
| 525 | for _, name := range names { | |
| 526 | args = append(args, "--env", name+"="+s.Env[name]) | |
| 527 | } | |
| 528 | args = append(args, | |
|
4e3759da
Keenan Nemetz
1h ago
|
529 | "--tmpfs", tmpfsArg(cfg), |
|
c1929d2f
Keenan Nemetz
5d ago
|
530 | "--cap-drop", "ALL", |
| 531 | "--security-opt", "no-new-privileges", | |
| 532 | "--init", | |
| 533 | "--stop-timeout", "5", | |
| 534 | "--network", cfg.Network, | |
| 535 | "--pull", cfg.Pull, | |
| 536 | ) | |
| 537 | if cfg.PIDs > 0 { | |
| 538 | args = append(args, "--pids-limit", strconv.Itoa(cfg.PIDs)) | |
| 539 | } | |
| 540 | if cfg.Memory != "" { | |
| 541 | args = append(args, "--memory", cfg.Memory) | |
| 542 | } | |
| 543 | if cfg.CPUs != "" { | |
| 544 | args = append(args, "--cpus", cfg.CPUs) | |
| 545 | } | |
| 546 | return append(args, s.Image) | |
| 547 | } | |
| 548 | ||
| 549 | // StartService creates a service container and waits for the runtime to report it | |
| 550 | // started. The container is not removed with --rm: the runner reads its log before | |
| 551 | // removing it, and a killed runner's leftovers are found by their labels. | |
| 552 | func StartService(cfg Config, s ServiceSpec) (*Container, error) { | |
| 553 | c := &Container{cfg: cfg, name: s.Name} | |
| 554 | cmd := exec.Command(cfg.Runtime, ServiceRunArgs(cfg, s)...) | |
| 555 | var stderr strings.Builder | |
| 556 | cmd.Stderr = &stderr | |
| 557 | if err := cmd.Run(); err != nil { | |
| 558 | _ = exec.Command(cfg.Runtime, "rm", "-f", s.Name).Run() | |
| 559 | return nil, fmt.Errorf("%s could not start service %s: %w: %s", | |
| 560 | cfg.Runtime, s.Alias, err, strings.TrimSpace(stderr.String())) | |
| 561 | } | |
| 562 | return c, nil | |
| 563 | } | |
| 564 | ||
| 565 | // Running reports whether the runtime still has the container up. A service that | |
| 566 | // exited before the steps begin is a failed job, not a connection error later. | |
| 567 | func (c *Container) Running(ctx context.Context) (bool, error) { | |
| 568 | out, err := exec.CommandContext(ctx, c.cfg.Runtime, "inspect", | |
| 569 | "--format", "{{.State.Running}}", c.name).Output() | |
| 570 | if err != nil { | |
| 571 | return false, fmt.Errorf("%s inspect %s: %w", c.cfg.Runtime, c.name, err) | |
| 572 | } | |
| 573 | switch strings.TrimSpace(string(out)) { | |
| 574 | case "true", "running": | |
| 575 | return true, nil | |
| 576 | } | |
| 577 | return false, nil | |
| 578 | } | |
| 579 | ||
| 580 | // Logs reads the container's output, keeping the last max bytes. A service that | |
| 581 | // failed usually says why at the end, so the tail is the useful part; max also | |
| 582 | // bounds what one service can add to an attempt log. A non-zero runtime exit with | |
| 583 | // output is not an error here: the output is what the caller wanted. | |
| 584 | func (c *Container) Logs(ctx context.Context, max int64) ([]byte, error) { | |
| 585 | if max <= 0 { | |
| 586 | max = 256 << 10 | |
| 587 | } | |
| 588 | tail := &tailBuffer{max: max} | |
| 589 | cmd := exec.CommandContext(ctx, c.cfg.Runtime, "logs", c.name) | |
| 590 | cmd.Stdout = tail | |
| 591 | cmd.Stderr = tail | |
| 592 | err := cmd.Run() | |
| 593 | data := tail.Bytes() | |
| 594 | if err != nil && len(data) == 0 { | |
| 595 | return nil, fmt.Errorf("%s logs %s: %w", c.cfg.Runtime, c.name, err) | |
| 596 | } | |
| 597 | return data, nil | |
| 598 | } | |
| 599 | ||
| 600 | // tailBuffer keeps the last max bytes written to it. The runtime's stdout and | |
| 601 | // stderr are copied by two goroutines, so the writes are serialized here; without | |
| 602 | // the lock the two streams would race and corrupt the tail. | |
| 603 | type tailBuffer struct { | |
| 604 | mu sync.Mutex | |
| 605 | max int64 | |
| 606 | data []byte | |
| 607 | } | |
| 608 | ||
| 609 | func (b *tailBuffer) Write(p []byte) (int, error) { | |
| 610 | b.mu.Lock() | |
| 611 | defer b.mu.Unlock() | |
| 612 | if int64(len(p)) >= b.max { | |
| 613 | b.data = append(b.data[:0], p[int64(len(p))-b.max:]...) | |
| 614 | return len(p), nil | |
| 615 | } | |
| 616 | b.data = append(b.data, p...) | |
| 617 | if int64(len(b.data)) > b.max { | |
| 618 | b.data = append(b.data[:0], b.data[int64(len(b.data))-b.max:]...) | |
| 619 | } | |
| 620 | return len(p), nil | |
| 621 | } | |
| 622 | ||
| 623 | func (b *tailBuffer) Bytes() []byte { | |
| 624 | b.mu.Lock() | |
| 625 | defer b.mu.Unlock() | |
| 626 | return append([]byte(nil), b.data...) | |
| 627 | } | |
| 628 | ||
| 629 | // Network is one user-defined network created for an attempt. The job and its | |
| 630 | // services are its only members, so service names resolve inside it and nowhere | |
| 631 | // else. | |
| 632 | type Network struct { | |
| 633 | cfg Config | |
| 634 | name string | |
| 635 | } | |
| 636 | ||
| 637 | // CreateNetwork makes the per-attempt network. The labels are the same runner and | |
| 638 | // run labels a container carries, so a killed runner's network is found the same | |
| 639 | // way its containers are. | |
| 640 | func CreateNetwork(ctx context.Context, cfg Config, name string, labels map[string]string) (*Network, error) { | |
| 641 | args := []string{"network", "create"} | |
| 642 | keys := make([]string, 0, len(labels)) | |
| 643 | for k := range labels { | |
| 644 | keys = append(keys, k) | |
| 645 | } | |
| 646 | sort.Strings(keys) | |
| 647 | for _, k := range keys { | |
| 648 | args = append(args, "--label", k+"="+labels[k]) | |
| 649 | } | |
| 650 | args = append(args, name) | |
| 651 | cmd := exec.CommandContext(ctx, cfg.Runtime, args...) | |
| 652 | var stderr strings.Builder | |
| 653 | cmd.Stderr = &stderr | |
| 654 | if err := cmd.Run(); err != nil { | |
| 655 | return nil, fmt.Errorf("%s could not create network %s: %w: %s", | |
| 656 | cfg.Runtime, name, err, strings.TrimSpace(stderr.String())) | |
| 657 | } | |
| 658 | return &Network{cfg: cfg, name: name}, nil | |
| 659 | } | |
| 660 | ||
| 661 | // Name is what the job container and its services pass as their network. | |
| 662 | func (n *Network) Name() string { return n.name } | |
| 663 | ||
| 664 | // Remove deletes the network. It is best effort on a path that is already failing: | |
| 665 | // a network with a container still attached cannot be removed, and the container | |
| 666 | // cleanup is what runs first. | |
| 667 | func (n *Network) Remove(ctx context.Context) error { | |
| 668 | return RemoveNetwork(ctx, n.cfg, n.name) | |
| 669 | } | |
| 670 | ||
| 671 | // RemoveNetwork deletes a network by name. Cleanup after a killed runner knows the | |
| 672 | // name from the label listing but has no Network value, so the name is the handle. | |
| 673 | func RemoveNetwork(ctx context.Context, cfg Config, name string) error { | |
| 674 | return exec.CommandContext(ctx, cfg.Runtime, "network", "rm", name).Run() | |
| 675 | } | |
| 676 | ||
| 677 | // ListNetworks returns the networks carrying one label. It is a list rather than a | |
| 678 | // count so the caller can inspect each one's owner before removing it. | |
| 679 | func ListNetworks(ctx context.Context, cfg Config, label string) ([]string, error) { | |
| 680 | out, err := exec.CommandContext(ctx, cfg.Runtime, "network", "ls", | |
| 681 | "--filter", "label="+label, "--format", "{{.Name}}").Output() | |
| 682 | if err != nil { | |
| 683 | return nil, err | |
| 684 | } | |
| 685 | var names []string | |
| 686 | for _, line := range strings.Split(string(out), "\n") { | |
| 687 | if name := strings.TrimSpace(line); name != "" { | |
| 688 | names = append(names, name) | |
| 689 | } | |
| 690 | } | |
| 691 | return names, nil | |
| 692 | } | |
| 693 | ||
| 694 | // NetworkLabels reads a network's labels, which say which runner and process | |
| 695 | // created it. Parsing JSON rather than a template keeps the result the same across | |
| 696 | // the three supported runtimes. | |
| 697 | func NetworkLabels(ctx context.Context, cfg Config, name string) (map[string]string, error) { | |
| 698 | out, err := exec.CommandContext(ctx, cfg.Runtime, "network", "inspect", | |
| 699 | "--format", "{{json .Labels}}", name).Output() | |
| 700 | if err != nil { | |
| 701 | return nil, err | |
| 702 | } | |
| 703 | var labels map[string]string | |
| 704 | if err := json.Unmarshal(out, &labels); err != nil { | |
| 705 | return nil, err | |
| 706 | } | |
| 707 | return labels, nil | |
| 708 | } |