vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Run CI jobs in a container chosen by the runner
7da9111ecabb236d48fc48592cd3d1d45f9afe78 · Keenan Nemetz · 15d ago · parent 474a812d
Run CI jobs in a container chosen by the runner Each lease gets a disposable checkout and, when --image is set, its own container from that image. The runner keeps the SSH key and trusted Git cache outside the job, so workflow code cannot reach either; older caches are rebuilt because a previous job may have written to them. The record names the image and runtime, and the queue mirrors them for the run page. Without --image the runner still executes on the host and says so.
ci/agent/sandbox.go +261
| @@ -0,0 +1,261 @@ | |||
| 1 | package agent | ||
| 2 | |||
| 3 | // Container execution for one job. | ||
| 4 | // | ||
| 5 | // The runner process stays on the host: it holds the SSH key and the trusted Git | ||
| 6 | // cache, leases work, publishes the record, and uploads artifacts. The workflow | ||
| 7 | // steps run in a container started from the image the operator chose with --image. | ||
| 8 | // One container per job is the isolation boundary: a job cannot see the runner's | ||
| 9 | // key, and the next job starts with a clean container and a fresh job tree. | ||
| 10 | // | ||
| 11 | // The image is deliberately a runner option rather than a workflow field. The | ||
| 12 | // workflow asks for a machine with runs-on; the operator decides what that machine | ||
| 13 | // runs, and the workflow schema does not grow a package manager. | ||
| 14 | |||
| 15 | import ( | ||
| 16 | "context" | ||
| 17 | "crypto/sha256" | ||
| 18 | "errors" | ||
| 19 | "fmt" | ||
| 20 | "os" | ||
| 21 | "os/exec" | ||
| 22 | "strconv" | ||
| 23 | "strings" | ||
| 24 | "syscall" | ||
| 25 | "time" | ||
| 26 | ) | ||
| 27 | |||
| 28 | // keepAlive is PID 1's command. Vole runs each step with exec, so the container | ||
| 29 | // has to stay up between steps. `/bin/sh` is also what the steps need, which keeps | ||
| 30 | // the requirement to one the image must satisfy anyway. | ||
| 31 | const keepAlive = `trap 'exit 0' TERM INT; while :; do sleep 3600; done` | ||
| 32 | |||
| 33 | type containerConfig struct { | ||
| 34 | runtime string | ||
| 35 | image string | ||
| 36 | network string | ||
| 37 | pull string | ||
| 38 | user string | ||
| 39 | userns string | ||
| 40 | memory string | ||
| 41 | cpus string | ||
| 42 | pids int | ||
| 43 | // id is stable for one runner cache, so a stale container from a previous | ||
| 44 | // process using the same --dir can be found at startup. pid says whether that | ||
| 45 | // process is still alive: two runners may share a cache by mistake, and one | ||
| 46 | // must not remove the other's container. | ||
| 47 | id string | ||
| 48 | pid int | ||
| 49 | } | ||
| 50 | |||
| 51 | func (c containerConfig) enabled() bool { return c.image != "" } | ||
| 52 | |||
| 53 | // containerOptions validates the runner's container settings and fills in the | ||
| 54 | // defaults. It runs before the runner connects, so a bad image reference or a | ||
| 55 | // missing runtime is a startup error rather than a job that fails mysteriously | ||
| 56 | // after it has been leased. | ||
| 57 | func containerOptions(opts Options, cache string) (containerConfig, error) { | ||
| 58 | if strings.TrimSpace(opts.Image) == "" { | ||
| 59 | if strings.TrimSpace(opts.ContainerRuntime) != "" || | ||
| 60 | strings.TrimSpace(opts.ContainerNetwork) != "" || | ||
| 61 | strings.TrimSpace(opts.ContainerPull) != "" || | ||
| 62 | strings.TrimSpace(opts.ContainerUser) != "" || | ||
| 63 | strings.TrimSpace(opts.ContainerUserns) != "" || | ||
| 64 | strings.TrimSpace(opts.ContainerMemory) != "" || | ||
| 65 | strings.TrimSpace(opts.ContainerCPUs) != "" || | ||
| 66 | opts.ContainerPIDs != 0 { | ||
| 67 | return containerConfig{}, fmt.Errorf("container options need --image") | ||
| 68 | } | ||
| 69 | return containerConfig{}, nil | ||
| 70 | } | ||
| 71 | cfg := containerConfig{ | ||
| 72 | runtime: strings.TrimSpace(opts.ContainerRuntime), | ||
| 73 | image: strings.TrimSpace(opts.Image), | ||
| 74 | network: strings.TrimSpace(opts.ContainerNetwork), | ||
| 75 | pull: strings.TrimSpace(opts.ContainerPull), | ||
| 76 | user: strings.TrimSpace(opts.ContainerUser), | ||
| 77 | userns: strings.TrimSpace(opts.ContainerUserns), | ||
| 78 | memory: strings.TrimSpace(opts.ContainerMemory), | ||
| 79 | cpus: strings.TrimSpace(opts.ContainerCPUs), | ||
| 80 | pids: opts.ContainerPIDs, | ||
| 81 | } | ||
| 82 | if cfg.runtime == "" { | ||
| 83 | cfg.runtime = "docker" | ||
| 84 | } | ||
| 85 | switch cfg.runtime { | ||
| 86 | case "docker", "podman", "nerdctl": | ||
| 87 | default: | ||
| 88 | return containerConfig{}, fmt.Errorf("--container-runtime %q is not supported; use docker, podman, or nerdctl", cfg.runtime) | ||
| 89 | } | ||
| 90 | if _, err := exec.LookPath(cfg.runtime); err != nil { | ||
| 91 | return containerConfig{}, fmt.Errorf("--container-runtime %s is not on PATH: %w", cfg.runtime, err) | ||
| 92 | } | ||
| 93 | // The image is passed as its own argument, never through a shell, but a leading | ||
| 94 | // dash would still be read as an option by the container CLI. | ||
| 95 | if strings.HasPrefix(cfg.image, "-") || strings.ContainsAny(cfg.image, " \t\r\n") { | ||
| 96 | return containerConfig{}, fmt.Errorf("--image %q is not an image reference", cfg.image) | ||
| 97 | } | ||
| 98 | if cfg.network == "" { | ||
| 99 | cfg.network = "none" | ||
| 100 | } | ||
| 101 | switch cfg.network { | ||
| 102 | case "none", "bridge": | ||
| 103 | default: | ||
| 104 | return containerConfig{}, fmt.Errorf("--container-network %q is not supported; use none or bridge", cfg.network) | ||
| 105 | } | ||
| 106 | if cfg.pull == "" { | ||
| 107 | cfg.pull = "missing" | ||
| 108 | } | ||
| 109 | switch cfg.pull { | ||
| 110 | case "missing", "always", "never": | ||
| 111 | default: | ||
| 112 | return containerConfig{}, fmt.Errorf("--container-pull %q is not supported; use missing, always, or never", cfg.pull) | ||
| 113 | } | ||
| 114 | defaultUser := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) | ||
| 115 | if cfg.user == "" { | ||
| 116 | cfg.user = defaultUser | ||
| 117 | } | ||
| 118 | if strings.ContainsAny(cfg.user, " \t\r\n") { | ||
| 119 | return containerConfig{}, fmt.Errorf("--container-user %q is not a user", cfg.user) | ||
| 120 | } | ||
| 121 | if strings.ContainsAny(cfg.userns, " \t\r\n") { | ||
| 122 | return containerConfig{}, fmt.Errorf("--container-userns %q is not a user namespace mode", cfg.userns) | ||
| 123 | } | ||
| 124 | if cfg.userns == "" && cfg.runtime == "podman" && os.Geteuid() != 0 && cfg.user == defaultUser { | ||
| 125 | // Rootless Podman maps container root to the runner user but maps other | ||
| 126 | // UIDs through subuids. keep-id makes the numeric --container-user line up | ||
| 127 | // with the host user, so the bind-mounted job tree is writable. A user that | ||
| 128 | // intentionally overrides --container-user may need a different mode. | ||
| 129 | cfg.userns = "keep-id" | ||
| 130 | } | ||
| 131 | if cfg.pids < 0 { | ||
| 132 | return containerConfig{}, fmt.Errorf("--container-pids must not be negative") | ||
| 133 | } | ||
| 134 | // One stable id per cache directory: the cache belongs to a runner, and the id | ||
| 135 | // lets a restarted runner find containers a killed predecessor left behind. | ||
| 136 | sum := sha256.Sum256([]byte(cache + "\x00" + opts.Remote)) | ||
| 137 | cfg.id = fmt.Sprintf("%x", sum[:6]) | ||
| 138 | cfg.pid = os.Getpid() | ||
| 139 | return cfg, nil | ||
| 140 | } | ||
| 141 | |||
| 142 | // cleanupStaleContainers removes containers from an earlier process that used this | ||
| 143 | // runner's cache. Normally the executor removes its own container, but a killed | ||
| 144 | // runner leaves the keepalive process behind. | ||
| 145 | func cleanupStaleContainers(cfg containerConfig) { | ||
| 146 | if !cfg.enabled() { | ||
| 147 | return | ||
| 148 | } | ||
| 149 | out, err := exec.Command(cfg.runtime, "ps", "-a", | ||
| 150 | "--filter", "label=vole.runner.id="+cfg.id, | ||
| 151 | "--format", `{{.ID}} {{.Label "vole.runner.pid"}}`).Output() | ||
| 152 | if err != nil { | ||
| 153 | return // the first job will report a runtime problem with more context | ||
| 154 | } | ||
| 155 | var stale []string | ||
| 156 | for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { | ||
| 157 | fields := strings.Fields(line) | ||
| 158 | if len(fields) == 0 { | ||
| 159 | continue | ||
| 160 | } | ||
| 161 | pid, _ := strconv.Atoi(fields[len(fields)-1]) | ||
| 162 | if pid > 0 && processAlive(pid) { | ||
| 163 | continue // another runner using the same cache is still working | ||
| 164 | } | ||
| 165 | stale = append(stale, fields[0]) | ||
| 166 | } | ||
| 167 | if len(stale) == 0 { | ||
| 168 | return | ||
| 169 | } | ||
| 170 | args := append([]string{"rm", "-f"}, stale...) | ||
| 171 | _ = exec.Command(cfg.runtime, args...).Run() | ||
| 172 | } | ||
| 173 | |||
| 174 | // containerRunArgs is pure so the argument list can be checked without a daemon. | ||
| 175 | // The defaults are deliberately closed: no host mounts beyond the job tree, no | ||
| 176 | // capabilities, no writable root, and no network unless the operator asked for it. | ||
| 177 | func containerRunArgs(cfg containerConfig, name, work, runID string) []string { | ||
| 178 | args := []string{ | ||
| 179 | "run", "-d", "--rm", | ||
| 180 | "--name", name, | ||
| 181 | "--label", "vole.runner=1", | ||
| 182 | "--label", "vole.runner.id=" + cfg.id, | ||
| 183 | "--label", "vole.runner.pid=" + strconv.Itoa(cfg.pid), | ||
| 184 | "--label", "vole.run=" + runID, | ||
| 185 | "--workdir", "/workspace", | ||
| 186 | "--user", cfg.user, | ||
| 187 | } | ||
| 188 | if cfg.userns != "" { | ||
| 189 | args = append(args, "--userns", cfg.userns) | ||
| 190 | } | ||
| 191 | args = append(args, | ||
| 192 | "--env", "CI=true", | ||
| 193 | "--env", "VOLE_CI=1", | ||
| 194 | "--env", "GIT_TERMINAL_PROMPT=0", | ||
| 195 | "--env", "HOME=/tmp", | ||
| 196 | "--env", "TMPDIR=/tmp", | ||
| 197 | "--volume", work+":/workspace:rw", | ||
| 198 | "--tmpfs", "/tmp:rw,nosuid,nodev,mode=1777", | ||
| 199 | "--read-only", | ||
| 200 | "--cap-drop", "ALL", | ||
| 201 | "--security-opt", "no-new-privileges", | ||
| 202 | "--init", | ||
| 203 | "--stop-timeout", "5", | ||
| 204 | "--network", cfg.network, | ||
| 205 | "--pull", cfg.pull, | ||
| 206 | ) | ||
| 207 | if cfg.pids > 0 { | ||
| 208 | args = append(args, "--pids-limit", strconv.Itoa(cfg.pids)) | ||
| 209 | } | ||
| 210 | if cfg.memory != "" { | ||
| 211 | args = append(args, "--memory", cfg.memory) | ||
| 212 | } | ||
| 213 | if cfg.cpus != "" { | ||
| 214 | args = append(args, "--cpus", cfg.cpus) | ||
| 215 | } | ||
| 216 | return append(args, | ||
| 217 | "--entrypoint", "/bin/sh", | ||
| 218 | cfg.image, | ||
| 219 | "-c", keepAlive, | ||
| 220 | ) | ||
| 221 | } | ||
| 222 | |||
| 223 | func processAlive(pid int) bool { | ||
| 224 | err := syscall.Kill(pid, 0) | ||
| 225 | return err == nil || errors.Is(err, syscall.EPERM) | ||
| 226 | } | ||
| 227 | |||
| 228 | type container struct { | ||
| 229 | cfg containerConfig | ||
| 230 | name string | ||
| 231 | } | ||
| 232 | |||
| 233 | func startContainer(cfg containerConfig, job *runnerJob, work string) (*container, error) { | ||
| 234 | name := fmt.Sprintf("vole-%s-%s-%d", cfg.id, strings.ToLower(job.ID), job.Attempt) | ||
| 235 | c := &container{cfg: cfg, name: name} | ||
| 236 | cmd := exec.Command(cfg.runtime, containerRunArgs(cfg, name, work, job.ID)...) | ||
| 237 | var stderr strings.Builder | ||
| 238 | cmd.Stderr = &stderr | ||
| 239 | if err := cmd.Run(); err != nil { | ||
| 240 | _ = exec.Command(cfg.runtime, "rm", "-f", name).Run() | ||
| 241 | return nil, fmt.Errorf("%s could not start %s: %w: %s", | ||
| 242 | cfg.runtime, cfg.image, err, strings.TrimSpace(stderr.String())) | ||
| 243 | } | ||
| 244 | return c, nil | ||
| 245 | } | ||
| 246 | |||
| 247 | // execCommand runs one step in the existing container. Each step is a separate | ||
| 248 | // exec, which is what gives it its own log, timeout and exit status while keeping | ||
| 249 | // the filesystem and background processes shared with the steps around it. | ||
| 250 | func (c *container) execCommand(ctx context.Context, step runnerStep) *exec.Cmd { | ||
| 251 | return exec.CommandContext(ctx, c.cfg.runtime, "exec", | ||
| 252 | "-w", "/workspace", c.name, "/bin/sh", "-c", step.Run) | ||
| 253 | } | ||
| 254 | |||
| 255 | // stop kills and removes the container. `rm -f` is one command that also handles a | ||
| 256 | // container that has already exited. | ||
| 257 | func (c *container) stop() { | ||
| 258 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| 259 | defer cancel() | ||
| 260 | _ = exec.CommandContext(ctx, c.cfg.runtime, "rm", "-f", c.name).Run() | ||
| 261 | } | ||