vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
|
7da9111e
Keenan Nemetz
15d ago
|
1 | package agent |
| 2 | ||
| 3 | // Container execution for one job. | |
| 4 | // | |
| 5 | // The runner process stays on the host: it holds the SSH key and the trusted Git | |
| 6 | // cache, leases work, publishes the record, and uploads artifacts. The workflow | |
|
70b70264
Keenan Nemetz
15d ago
|
7 | // steps run in a container built from the image the job names. One container per |
| 8 | // job is the isolation boundary: a job cannot see the runner's key, and the next | |
| 9 | // job starts with a clean container and a fresh job tree. | |
|
7da9111e
Keenan Nemetz
15d ago
|
10 | // |
|
70b70264
Keenan Nemetz
15d ago
|
11 | // The image is part of the workflow, so a job carries the environment it needs. |
| 12 | // A custom .vole/images definition is built by the runner before the container | |
| 13 | // starts; a registry reference is used as written. | |
|
3b39a84c
Keenan Nemetz
14d ago
|
14 | // |
| 15 | // The container itself lives in ci/container, which `vole workflow run` shares, so | |
| 16 | // a local run and a pushed run cannot disagree about how a step executes. | |
|
7da9111e
Keenan Nemetz
15d ago
|
17 | |
| 18 | import ( | |
|
c1929d2f
Keenan Nemetz
5d ago
|
19 | "context" |
|
7da9111e
Keenan Nemetz
15d ago
|
20 | "crypto/sha256" |
| 21 | "fmt" | |
|
c1929d2f
Keenan Nemetz
5d ago
|
22 | "io" |
|
7da9111e
Keenan Nemetz
15d ago
|
23 | "os" |
| 24 | "os/exec" | |
| 25 | "strconv" | |
| 26 | "strings" | |
| 27 | "syscall" | |
|
c1929d2f
Keenan Nemetz
5d ago
|
28 | "time" |
|
7da9111e
Keenan Nemetz
15d ago
|
29 | |
|
d37d301f
Keenan Nemetz
17h ago
|
30 | "rodent.software/vole/ci" |
| 31 | "rodent.software/vole/ci/container" | |
|
3b39a84c
Keenan Nemetz
14d ago
|
32 | ) |
|
7da9111e
Keenan Nemetz
15d ago
|
33 | |
|
3b39a84c
Keenan Nemetz
14d ago
|
34 | // runnerBox is the validated container settings plus the identity this runner |
|
77056ba8
Keenan Nemetz
2d ago
|
35 | // process uses to find containers a killed predecessor left behind, and the builds |
| 36 | // it has in flight. | |
|
3b39a84c
Keenan Nemetz
14d ago
|
37 | type runnerBox struct { |
| 38 | container.Config | |
|
7da9111e
Keenan Nemetz
15d ago
|
39 | id string |
| 40 | pid int | |
|
77056ba8
Keenan Nemetz
2d ago
|
41 | // builds is shared by every slot in the process, so two jobs that name the same |
| 42 | // image-file build it once. Nil means the caller never validated these options, | |
| 43 | // which only a test does. | |
| 44 | builds *imageBuilds | |
|
7da9111e
Keenan Nemetz
15d ago
|
45 | } |
| 46 | ||
| 47 | // containerOptions validates the runner's container settings and fills in the | |
|
70b70264
Keenan Nemetz
15d ago
|
48 | // defaults. It runs before the runner connects, so a missing runtime or an invalid |
| 49 | // option is a startup error rather than a job that fails mysteriously after it has | |
| 50 | // been leased. | |
|
3b39a84c
Keenan Nemetz
14d ago
|
51 | func containerOptions(opts Options, cache string) (runnerBox, error) { |
| 52 | cfg, err := container.Validate(container.Config{ | |
| 53 | Runtime: strings.TrimSpace(opts.ContainerRuntime), | |
| 54 | Network: strings.TrimSpace(opts.ContainerNetwork), | |
| 55 | Pull: strings.TrimSpace(opts.ContainerPull), | |
| 56 | User: strings.TrimSpace(opts.ContainerUser), | |
| 57 | Userns: strings.TrimSpace(opts.ContainerUserns), | |
| 58 | Memory: strings.TrimSpace(opts.ContainerMemory), | |
| 59 | CPUs: strings.TrimSpace(opts.ContainerCPUs), | |
| 60 | PIDs: opts.ContainerPIDs, | |
|
4e3759da
Keenan Nemetz
2h ago
|
61 | Tmpfs: strings.TrimSpace(opts.ContainerTmpfs), |
|
3b39a84c
Keenan Nemetz
14d ago
|
62 | }) |
| 63 | if err != nil { | |
| 64 | return runnerBox{}, err | |
|
7da9111e
Keenan Nemetz
15d ago
|
65 | } |
| 66 | // One stable id per cache directory: the cache belongs to a runner, and the id | |
| 67 | // lets a restarted runner find containers a killed predecessor left behind. | |
| 68 | sum := sha256.Sum256([]byte(cache + "\x00" + opts.Remote)) | |
|
3b39a84c
Keenan Nemetz
14d ago
|
69 | return runnerBox{ |
| 70 | Config: cfg, | |
| 71 | id: fmt.Sprintf("%x", sum[:6]), | |
| 72 | pid: os.Getpid(), | |
|
77056ba8
Keenan Nemetz
2d ago
|
73 | builds: &imageBuilds{}, |
|
3b39a84c
Keenan Nemetz
14d ago
|
74 | }, nil |
| 75 | } | |
| 76 | ||
|
d7958024
Keenan Nemetz
6d ago
|
77 | // networkFor is the container settings for one job. A job that names a network gets |
| 78 | // exactly that: `network: none` withholds the network even on a runner that could | |
| 79 | // give one, and `network: bridge` is refused here on a runner that cannot, because | |
| 80 | // running it without the network it asked for would fail later and less clearly than | |
| 81 | // saying so at the start. A job that names nothing gets the runner's own default, | |
| 82 | // which is what every workflow meant before the field existed. | |
| 83 | func (b runnerBox) networkFor(job *runnerJob) (container.Config, error) { | |
| 84 | cfg := b.Config | |
|
c1929d2f
Keenan Nemetz
5d ago
|
85 | // A service is reached over the job's network. A job that declares one without |
| 86 | // asking for bridge would otherwise get a network its workflow never requested, | |
| 87 | // or wait forever on a runner whose default is none, so it is refused here as | |
| 88 | // well as at push time. | |
| 89 | if len(job.Services) > 0 && job.Network != ci.NetworkBridge { | |
| 90 | return cfg, fmt.Errorf("job %s declares services but not network: bridge", job.ID) | |
| 91 | } | |
|
d7958024
Keenan Nemetz
6d ago
|
92 | if job.Network == "" { |
| 93 | return cfg, nil | |
| 94 | } | |
| 95 | if err := ci.ValidNetwork(job.Network); err != nil { | |
| 96 | return cfg, fmt.Errorf("job %s: %w", job.ID, err) | |
| 97 | } | |
| 98 | if job.Network == ci.NetworkBridge && cfg.Network != ci.NetworkBridge { | |
| 99 | return cfg, fmt.Errorf("job %s asks for network: bridge but this runner was started with "+ | |
| 100 | "--container-network %s, so it cannot provide one", job.ID, cfg.Network) | |
| 101 | } | |
| 102 | cfg.Network = job.Network | |
| 103 | return cfg, nil | |
| 104 | } | |
| 105 | ||
|
3b39a84c
Keenan Nemetz
14d ago
|
106 | // labels are how a container says which runner and run it belongs to. |
| 107 | func (b runnerBox) labels(job *runnerJob) map[string]string { | |
| 108 | return map[string]string{ | |
| 109 | "vole.runner": "1", | |
| 110 | "vole.runner.id": b.id, | |
| 111 | "vole.runner.pid": strconv.Itoa(b.pid), | |
| 112 | "vole.run": job.ID, | |
| 113 | } | |
| 114 | } | |
| 115 | ||
| 116 | func containerName(box runnerBox, job *runnerJob) string { | |
| 117 | return fmt.Sprintf("vole-%s-%s-%d", box.id, strings.ToLower(job.ID), job.Attempt) | |
|
7da9111e
Keenan Nemetz
15d ago
|
118 | } |
| 119 | ||
| 120 | // cleanupStaleContainers removes containers from an earlier process that used this | |
| 121 | // runner's cache. Normally the executor removes its own container, but a killed | |
| 122 | // runner leaves the keepalive process behind. | |
|
3b39a84c
Keenan Nemetz
14d ago
|
123 | func cleanupStaleContainers(cfg runnerBox) { |
| 124 | out, err := exec.Command(cfg.Runtime, "ps", "-a", | |
|
7da9111e
Keenan Nemetz
15d ago
|
125 | "--filter", "label=vole.runner.id="+cfg.id, |
| 126 | "--format", `{{.ID}} {{.Label "vole.runner.pid"}}`).Output() | |
| 127 | if err != nil { | |
| 128 | return // the first job will report a runtime problem with more context | |
| 129 | } | |
| 130 | var stale []string | |
| 131 | for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { | |
| 132 | fields := strings.Fields(line) | |
| 133 | if len(fields) == 0 { | |
| 134 | continue | |
| 135 | } | |
| 136 | pid, _ := strconv.Atoi(fields[len(fields)-1]) | |
| 137 | if pid > 0 && processAlive(pid) { | |
| 138 | continue // another runner using the same cache is still working | |
| 139 | } | |
| 140 | stale = append(stale, fields[0]) | |
| 141 | } | |
| 142 | if len(stale) == 0 { | |
| 143 | return | |
| 144 | } | |
| 145 | args := append([]string{"rm", "-f"}, stale...) | |
|
3b39a84c
Keenan Nemetz
14d ago
|
146 | _ = exec.Command(cfg.Runtime, args...).Run() |
|
7da9111e
Keenan Nemetz
15d ago
|
147 | } |
| 148 | ||
| 149 | func processAlive(pid int) bool { | |
| 150 | err := syscall.Kill(pid, 0) | |
|
3b39a84c
Keenan Nemetz
14d ago
|
151 | return err == nil || err == syscall.EPERM |
|
7da9111e
Keenan Nemetz
15d ago
|
152 | } |
|
c1929d2f
Keenan Nemetz
5d ago
|
153 | |
| 154 | // jobSandbox is the job container, the services started beside it, and the network | |
| 155 | // they share. Stop is idempotent: the failure paths stop early so the service logs | |
| 156 | // can go into the attempt, and the deferred call still runs. | |
| 157 | type jobSandbox struct { | |
| 158 | job *container.Container | |
| 159 | services []*container.Container | |
| 160 | serviceIDs []string | |
| 161 | network *container.Network | |
| 162 | } | |
| 163 | ||
| 164 | func (s *jobSandbox) Stop() { s.StopWithLogs(nil) } | |
| 165 | ||
| 166 | // StopJob stops the job container and leaves the services and network for the | |
| 167 | // caller's final Stop. The job is stopped before artifacts and a site are read so a | |
| 168 | // background step cannot change the tree while it is being published; a service | |
| 169 | // does not touch that tree, and keeping it up means a publish failure can still | |
| 170 | // attach its output. | |
| 171 | func (s *jobSandbox) StopJob() { | |
| 172 | if s.job != nil { | |
| 173 | s.job.Stop() | |
| 174 | s.job = nil | |
| 175 | } | |
| 176 | } | |
| 177 | ||
| 178 | // StopWithLogs stops the job, then the services, then removes the network. When w | |
| 179 | // is not nil, each service's last output is written under its own heading first: | |
| 180 | // a job that failed after a database refused to start should not make the reader | |
| 181 | // find the database's log in a second place. | |
| 182 | func (s *jobSandbox) StopWithLogs(w io.Writer) { | |
| 183 | s.StopJob() | |
| 184 | if w != nil { | |
| 185 | for i, svc := range s.services { | |
| 186 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) | |
| 187 | data, err := svc.Logs(ctx, container.MaxServiceLogBytes) | |
| 188 | cancel() | |
| 189 | if err != nil || len(data) == 0 { | |
| 190 | continue | |
| 191 | } | |
| 192 | fmt.Fprintf(w, "\n=== service %s ===\n", s.serviceIDs[i]) | |
| 193 | _, _ = w.Write(data) | |
| 194 | if data[len(data)-1] != '\n' { | |
| 195 | fmt.Fprintln(w) | |
| 196 | } | |
| 197 | } | |
| 198 | } | |
| 199 | for _, svc := range s.services { | |
| 200 | svc.Stop() | |
| 201 | } | |
| 202 | s.services, s.serviceIDs = nil, nil | |
| 203 | if s.network != nil { | |
| 204 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | |
| 205 | _ = s.network.Remove(ctx) | |
| 206 | cancel() | |
| 207 | s.network = nil | |
| 208 | } | |
| 209 | } | |
| 210 | ||
| 211 | // serviceNetworkName is the per-attempt network the job and its services join. | |
| 212 | // The attempt is part of the name because two attempts of the same run must not | |
| 213 | // share a network. | |
| 214 | func serviceNetworkName(box runnerBox, job *runnerJob) string { | |
| 215 | return fmt.Sprintf("vole-%s-%s-%d-net", box.id, strings.ToLower(job.ID), job.Attempt) | |
| 216 | } | |
| 217 | ||
| 218 | func serviceContainerName(box runnerBox, job *runnerJob, id string) string { | |
| 219 | return fmt.Sprintf("vole-%s-%s-%d-svc-%s", box.id, strings.ToLower(job.ID), job.Attempt, id) | |
| 220 | } | |
| 221 | ||
| 222 | // cleanupStaleNetworks removes networks a killed runner left behind. It runs after | |
| 223 | // cleanupStaleContainers, because a network with a member cannot be removed. | |
| 224 | func cleanupStaleNetworks(cfg runnerBox) { | |
| 225 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) | |
| 226 | defer cancel() | |
| 227 | names, err := container.ListNetworks(ctx, cfg.Config, "vole.runner.id="+cfg.id) | |
| 228 | if err != nil { | |
| 229 | return // the first job will report a runtime problem with more context | |
| 230 | } | |
| 231 | for _, name := range names { | |
| 232 | labels, err := container.NetworkLabels(ctx, cfg.Config, name) | |
| 233 | if err != nil { | |
| 234 | continue | |
| 235 | } | |
| 236 | pid, _ := strconv.Atoi(labels["vole.runner.pid"]) | |
| 237 | if pid > 0 && processAlive(pid) { | |
| 238 | continue // another runner using the same cache is still working | |
| 239 | } | |
| 240 | _ = container.RemoveNetwork(ctx, cfg.Config, name) | |
| 241 | } | |
| 242 | } |