vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Add packaging, a CI workflow, and containerised local runs
3b39a84c913cabe15bfa6a60dd176ff2dd28d241 · Keenan Nemetz · 15d ago · parent 30a72490
Add packaging, a CI workflow, and containerised local runs A v* tag now builds a .deb and a .rpm for amd64 and arm64 with nfpm and attaches them, with checksums, to the release. The package ships the systemd unit, a config at /etc/vole/vole.toml, and a postinstall that creates the service account and prepares the instance that config names; `vole admin init --config` is what makes that possible, and the binary's version is stamped from the tag instead of the source tree. scripts/ci.sh becomes .vole/workflows/ci.yaml, so `vole workflow run ci` executes exactly what a push runs, and the Markdown link check moves to scripts/check-links.py. Local runs now always execute in the job image and share ci/container with the runner; --no-container is gone, because a run in a different environment proves nothing about CI. The runner mounts a per-repository namespace at /cache and sets no variable for it: a job opts in through its image's env or the new job-level env, which is pinned with the queue and delivered with the lease. Whole namespaces are evicted least-recently-written first so a module cache is never half-deleted, and contributor branches get their own namespace so an untrusted tip cannot seed the cache a protected branch builds into. Running the suite in that container found four bugs worth naming: /tmp was mounted noexec, which broke `go test` and `go run`; the read-only root is gone, because images make toolchain directories world-writable deliberately and the mount set is what protects the host; OpenSSH needs a passwd entry for the job's uid; and a lost metadata push reported as "cannot lock ref" was not classified as a race, so concurrent CLI writes gave up on the first collision. That retry now backs off exponentially with jitter.
ci/agent/sandbox.go +45−175
| @@ -11,38 +11,26 @@ package agent | |||
| 11 | 11 | // The image is part of the workflow, so a job carries the environment it needs. | |
| 12 | 12 | // A custom .vole/images definition is built by the runner before the container | |
| 13 | 13 | // starts; a registry reference is used as written. | |
| 14 | // | ||
| 15 | // The container itself lives in ci/container, which `vole workflow run` shares, so | ||
| 16 | // a local run and a pushed run cannot disagree about how a step executes. | ||
| 14 | 17 | ||
| 15 | 18 | import ( | |
| 16 | "context" | ||
| 17 | 19 | "crypto/sha256" | |
| 18 | "errors" | ||
| 19 | 20 | "fmt" | |
| 20 | 21 | "os" | |
| 21 | 22 | "os/exec" | |
| 22 | 23 | "strconv" | |
| 23 | 24 | "strings" | |
| 24 | 25 | "syscall" | |
| 25 | "time" | ||
| 26 | ) | ||
| 27 | 26 | ||
| 28 | // keepAlive is PID 1's command. Vole runs each step with exec, so the container | ||
| 29 | // has to stay up between steps. `/bin/sh` is also what the steps need, which keeps | ||
| 30 | // the requirement to one the image must satisfy anyway. | ||
| 31 | const keepAlive = `trap 'exit 0' TERM INT; while :; do sleep 3600; done` | ||
| 27 | "github.com/rodent-software/vole/ci/container" | ||
| 28 | ) | ||
| 32 | 29 | ||
| 33 | type containerConfig struct { | ||
| 34 | runtime string | ||
| 35 | network string | ||
| 36 | pull string | ||
| 37 | user string | ||
| 38 | userns string | ||
| 39 | memory string | ||
| 40 | cpus string | ||
| 41 | pids int | ||
| 42 | // id is stable for one runner cache, so a stale container from a previous | ||
| 43 | // process using the same --dir can be found at startup. pid says whether that | ||
| 44 | // process is still alive: two runners may share a cache by mistake, and one | ||
| 45 | // must not remove the other's container. | ||
| 30 | // runnerBox is the validated container settings plus the identity this runner | ||
| 31 | // process uses to find containers a killed predecessor left behind. | ||
| 32 | type runnerBox struct { | ||
| 33 | container.Config | ||
| 46 | 34 | id string | |
| 47 | 35 | pid int | |
| 48 | 36 | } | |
| @@ -51,77 +39,49 @@ type containerConfig struct { | |||
| 51 | 39 | // defaults. It runs before the runner connects, so a missing runtime or an invalid | |
| 52 | 40 | // option is a startup error rather than a job that fails mysteriously after it has | |
| 53 | 41 | // been leased. | |
| 54 | func containerOptions(opts Options, cache string) (containerConfig, error) { | ||
| 55 | cfg := containerConfig{ | ||
| 56 | runtime: strings.TrimSpace(opts.ContainerRuntime), | ||
| 57 | network: strings.TrimSpace(opts.ContainerNetwork), | ||
| 58 | pull: strings.TrimSpace(opts.ContainerPull), | ||
| 59 | user: strings.TrimSpace(opts.ContainerUser), | ||
| 60 | userns: strings.TrimSpace(opts.ContainerUserns), | ||
| 61 | memory: strings.TrimSpace(opts.ContainerMemory), | ||
| 62 | cpus: strings.TrimSpace(opts.ContainerCPUs), | ||
| 63 | pids: opts.ContainerPIDs, | ||
| 64 | } | ||
| 65 | if cfg.runtime == "" { | ||
| 66 | cfg.runtime = "docker" | ||
| 67 | } | ||
| 68 | switch cfg.runtime { | ||
| 69 | case "docker", "podman", "nerdctl": | ||
| 70 | default: | ||
| 71 | return containerConfig{}, fmt.Errorf("--container-runtime %q is not supported; use docker, podman, or nerdctl", cfg.runtime) | ||
| 72 | } | ||
| 73 | if _, err := exec.LookPath(cfg.runtime); err != nil { | ||
| 74 | return containerConfig{}, fmt.Errorf("--container-runtime %s is not on PATH: %w", cfg.runtime, err) | ||
| 75 | } | ||
| 76 | if cfg.network == "" { | ||
| 77 | cfg.network = "none" | ||
| 78 | } | ||
| 79 | switch cfg.network { | ||
| 80 | case "none", "bridge": | ||
| 81 | default: | ||
| 82 | return containerConfig{}, fmt.Errorf("--container-network %q is not supported; use none or bridge", cfg.network) | ||
| 83 | } | ||
| 84 | if cfg.pull == "" { | ||
| 85 | cfg.pull = "missing" | ||
| 86 | } | ||
| 87 | switch cfg.pull { | ||
| 88 | case "missing", "always", "never": | ||
| 89 | default: | ||
| 90 | return containerConfig{}, fmt.Errorf("--container-pull %q is not supported; use missing, always, or never", cfg.pull) | ||
| 91 | } | ||
| 92 | defaultUser := fmt.Sprintf("%d:%d", os.Getuid(), os.Getgid()) | ||
| 93 | if cfg.user == "" { | ||
| 94 | cfg.user = defaultUser | ||
| 95 | } | ||
| 96 | if strings.ContainsAny(cfg.user, " \t\r\n") { | ||
| 97 | return containerConfig{}, fmt.Errorf("--container-user %q is not a user", cfg.user) | ||
| 98 | } | ||
| 99 | if strings.ContainsAny(cfg.userns, " \t\r\n") { | ||
| 100 | return containerConfig{}, fmt.Errorf("--container-userns %q is not a user namespace mode", cfg.userns) | ||
| 101 | } | ||
| 102 | if cfg.userns == "" && cfg.runtime == "podman" && os.Geteuid() != 0 && cfg.user == defaultUser { | ||
| 103 | // Rootless Podman maps container root to the runner user but maps other | ||
| 104 | // UIDs through subuids. keep-id makes the numeric --container-user line up | ||
| 105 | // with the host user, so the bind-mounted job tree is writable. A user that | ||
| 106 | // intentionally overrides --container-user may need a different mode. | ||
| 107 | cfg.userns = "keep-id" | ||
| 108 | } | ||
| 109 | if cfg.pids < 0 { | ||
| 110 | return containerConfig{}, fmt.Errorf("--container-pids must not be negative") | ||
| 42 | func containerOptions(opts Options, cache string) (runnerBox, error) { | ||
| 43 | cfg, err := container.Validate(container.Config{ | ||
| 44 | Runtime: strings.TrimSpace(opts.ContainerRuntime), | ||
| 45 | Network: strings.TrimSpace(opts.ContainerNetwork), | ||
| 46 | Pull: strings.TrimSpace(opts.ContainerPull), | ||
| 47 | User: strings.TrimSpace(opts.ContainerUser), | ||
| 48 | Userns: strings.TrimSpace(opts.ContainerUserns), | ||
| 49 | Memory: strings.TrimSpace(opts.ContainerMemory), | ||
| 50 | CPUs: strings.TrimSpace(opts.ContainerCPUs), | ||
| 51 | PIDs: opts.ContainerPIDs, | ||
| 52 | }) | ||
| 53 | if err != nil { | ||
| 54 | return runnerBox{}, err | ||
| 111 | 55 | } | |
| 112 | 56 | // One stable id per cache directory: the cache belongs to a runner, and the id | |
| 113 | 57 | // lets a restarted runner find containers a killed predecessor left behind. | |
| 114 | 58 | sum := sha256.Sum256([]byte(cache + "\x00" + opts.Remote)) | |
| 115 | cfg.id = fmt.Sprintf("%x", sum[:6]) | ||
| 116 | cfg.pid = os.Getpid() | ||
| 117 | return cfg, nil | ||
| 59 | return runnerBox{ | ||
| 60 | Config: cfg, | ||
| 61 | id: fmt.Sprintf("%x", sum[:6]), | ||
| 62 | pid: os.Getpid(), | ||
| 63 | }, nil | ||
| 64 | } | ||
| 65 | |||
| 66 | // labels are how a container says which runner and run it belongs to. | ||
| 67 | func (b runnerBox) labels(job *runnerJob) map[string]string { | ||
| 68 | return map[string]string{ | ||
| 69 | "vole.runner": "1", | ||
| 70 | "vole.runner.id": b.id, | ||
| 71 | "vole.runner.pid": strconv.Itoa(b.pid), | ||
| 72 | "vole.run": job.ID, | ||
| 73 | } | ||
| 74 | } | ||
| 75 | |||
| 76 | func containerName(box runnerBox, job *runnerJob) string { | ||
| 77 | return fmt.Sprintf("vole-%s-%s-%d", box.id, strings.ToLower(job.ID), job.Attempt) | ||
| 118 | 78 | } | |
| 119 | 79 | ||
| 120 | 80 | // cleanupStaleContainers removes containers from an earlier process that used this | |
| 121 | 81 | // runner's cache. Normally the executor removes its own container, but a killed | |
| 122 | 82 | // runner leaves the keepalive process behind. | |
| 123 | func cleanupStaleContainers(cfg containerConfig) { | ||
| 124 | out, err := exec.Command(cfg.runtime, "ps", "-a", | ||
| 83 | func cleanupStaleContainers(cfg runnerBox) { | ||
| 84 | out, err := exec.Command(cfg.Runtime, "ps", "-a", | ||
| 125 | 85 | "--filter", "label=vole.runner.id="+cfg.id, | |
| 126 | 86 | "--format", `{{.ID}} {{.Label "vole.runner.pid"}}`).Output() | |
| 127 | 87 | if err != nil { | |
| @@ -143,100 +103,10 @@ func cleanupStaleContainers(cfg containerConfig) { | |||
| 143 | 103 | return | |
| 144 | 104 | } | |
| 145 | 105 | args := append([]string{"rm", "-f"}, stale...) | |
| 146 | _ = exec.Command(cfg.runtime, args...).Run() | ||
| 147 | } | ||
| 148 | |||
| 149 | // containerRunArgs is pure so the argument list can be checked without a daemon. | ||
| 150 | // The defaults are deliberately closed: no host mounts beyond the job tree, no | ||
| 151 | // capabilities, no writable root, and no network unless the operator asked for it. | ||
| 152 | func containerRunArgs(cfg containerConfig, image, name, work, runID string) []string { | ||
| 153 | args := []string{ | ||
| 154 | "run", "-d", "--rm", | ||
| 155 | "--name", name, | ||
| 156 | "--label", "vole.runner=1", | ||
| 157 | "--label", "vole.runner.id=" + cfg.id, | ||
| 158 | "--label", "vole.runner.pid=" + strconv.Itoa(cfg.pid), | ||
| 159 | "--label", "vole.run=" + runID, | ||
| 160 | "--workdir", "/workspace", | ||
| 161 | "--user", cfg.user, | ||
| 162 | } | ||
| 163 | if cfg.userns != "" { | ||
| 164 | args = append(args, "--userns", cfg.userns) | ||
| 165 | } | ||
| 166 | args = append(args, | ||
| 167 | "--env", "CI=true", | ||
| 168 | "--env", "VOLE_CI=1", | ||
| 169 | "--env", "GIT_TERMINAL_PROMPT=0", | ||
| 170 | "--env", "HOME=/tmp", | ||
| 171 | "--env", "TMPDIR=/tmp", | ||
| 172 | "--volume", work+":/workspace:rw", | ||
| 173 | "--tmpfs", "/tmp:rw,nosuid,nodev,mode=1777", | ||
| 174 | "--read-only", | ||
| 175 | "--cap-drop", "ALL", | ||
| 176 | "--security-opt", "no-new-privileges", | ||
| 177 | "--init", | ||
| 178 | "--stop-timeout", "5", | ||
| 179 | "--network", cfg.network, | ||
| 180 | "--pull", cfg.pull, | ||
| 181 | ) | ||
| 182 | if cfg.pids > 0 { | ||
| 183 | args = append(args, "--pids-limit", strconv.Itoa(cfg.pids)) | ||
| 184 | } | ||
| 185 | if cfg.memory != "" { | ||
| 186 | args = append(args, "--memory", cfg.memory) | ||
| 187 | } | ||
| 188 | if cfg.cpus != "" { | ||
| 189 | args = append(args, "--cpus", cfg.cpus) | ||
| 190 | } | ||
| 191 | return append(args, | ||
| 192 | "--entrypoint", "/bin/sh", | ||
| 193 | image, | ||
| 194 | "-c", keepAlive, | ||
| 195 | ) | ||
| 106 | _ = exec.Command(cfg.Runtime, args...).Run() | ||
| 196 | 107 | } | |
| 197 | 108 | ||
| 198 | 109 | func processAlive(pid int) bool { | |
| 199 | 110 | err := syscall.Kill(pid, 0) | |
| 200 | return err == nil || errors.Is(err, syscall.EPERM) | ||
| 201 | } | ||
| 202 | |||
| 203 | type container struct { | ||
| 204 | cfg containerConfig | ||
| 205 | name string | ||
| 206 | work string | ||
| 207 | } | ||
| 208 | |||
| 209 | func startContainer(cfg containerConfig, image string, job *runnerJob, work string) (*container, error) { | ||
| 210 | name := fmt.Sprintf("vole-%s-%s-%d", cfg.id, strings.ToLower(job.ID), job.Attempt) | ||
| 211 | c := &container{cfg: cfg, name: name, work: work} | ||
| 212 | cmd := exec.Command(cfg.runtime, containerRunArgs(cfg, image, name, work, job.ID)...) | ||
| 213 | var stderr strings.Builder | ||
| 214 | cmd.Stderr = &stderr | ||
| 215 | if err := cmd.Run(); err != nil { | ||
| 216 | _ = exec.Command(cfg.runtime, "rm", "-f", name).Run() | ||
| 217 | return nil, fmt.Errorf("%s could not start %s: %w: %s", | ||
| 218 | cfg.runtime, image, err, strings.TrimSpace(stderr.String())) | ||
| 219 | } | ||
| 220 | return c, nil | ||
| 221 | } | ||
| 222 | |||
| 223 | // execCommand runs one step in the existing container. Each step is a separate | ||
| 224 | // exec, which is what gives it its own log, timeout and exit status while keeping | ||
| 225 | // the filesystem and background processes shared with the steps around it. | ||
| 226 | func (c *container) execCommand(ctx context.Context, step runnerStep) *exec.Cmd { | ||
| 227 | cmd := exec.CommandContext(ctx, c.cfg.runtime, "exec", | ||
| 228 | "-w", "/workspace", c.name, "/bin/sh", "-c", step.Run) | ||
| 229 | // The runtime ignores the client's working directory, but keeping it at the job | ||
| 230 | // tree lets a test runtime execute the step in the same place without pretending | ||
| 231 | // to be a container. | ||
| 232 | cmd.Dir = c.work | ||
| 233 | return cmd | ||
| 234 | } | ||
| 235 | |||
| 236 | // stop kills and removes the container. `rm -f` is one command that also handles a | ||
| 237 | // container that has already exited. | ||
| 238 | func (c *container) stop() { | ||
| 239 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| 240 | defer cancel() | ||
| 241 | _ = exec.CommandContext(ctx, c.cfg.runtime, "rm", "-f", c.name).Run() | ||
| 111 | return err == nil || err == syscall.EPERM | ||
| 242 | 112 | } | |