vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Let a job start service containers beside it
c1929d2fceb5e9e65204d5eef907d0781b9a5391 · Keenan Nemetz · 5d ago · parent 1d06b008
ci/agent/sandbox.go +100
| @@ -16,13 +16,16 @@ package agent | |||
| 16 | 16 | // a local run and a pushed run cannot disagree about how a step executes. | |
| 17 | 17 | ||
| 18 | 18 | import ( | |
| 19 | "context" | ||
| 19 | 20 | "crypto/sha256" | |
| 20 | 21 | "fmt" | |
| 22 | "io" | ||
| 21 | 23 | "os" | |
| 22 | 24 | "os/exec" | |
| 23 | 25 | "strconv" | |
| 24 | 26 | "strings" | |
| 25 | 27 | "syscall" | |
| 28 | "time" | ||
| 26 | 29 | ||
| 27 | 30 | "github.com/rodent-software/vole/ci" | |
| 28 | 31 | "github.com/rodent-software/vole/ci/container" | |
| @@ -72,6 +75,13 @@ func containerOptions(opts Options, cache string) (runnerBox, error) { | |||
| 72 | 75 | // which is what every workflow meant before the field existed. | |
| 73 | 76 | func (b runnerBox) networkFor(job *runnerJob) (container.Config, error) { | |
| 74 | 77 | cfg := b.Config | |
| 78 | // A service is reached over the job's network. A job that declares one without | ||
| 79 | // asking for bridge would otherwise get a network its workflow never requested, | ||
| 80 | // or wait forever on a runner whose default is none, so it is refused here as | ||
| 81 | // well as at push time. | ||
| 82 | if len(job.Services) > 0 && job.Network != ci.NetworkBridge { | ||
| 83 | return cfg, fmt.Errorf("job %s declares services but not network: bridge", job.ID) | ||
| 84 | } | ||
| 75 | 85 | if job.Network == "" { | |
| 76 | 86 | return cfg, nil | |
| 77 | 87 | } | |
| @@ -133,3 +143,93 @@ func processAlive(pid int) bool { | |||
| 133 | 143 | err := syscall.Kill(pid, 0) | |
| 134 | 144 | return err == nil || err == syscall.EPERM | |
| 135 | 145 | } | |
| 146 | |||
| 147 | // jobSandbox is the job container, the services started beside it, and the network | ||
| 148 | // they share. Stop is idempotent: the failure paths stop early so the service logs | ||
| 149 | // can go into the attempt, and the deferred call still runs. | ||
| 150 | type jobSandbox struct { | ||
| 151 | job *container.Container | ||
| 152 | services []*container.Container | ||
| 153 | serviceIDs []string | ||
| 154 | network *container.Network | ||
| 155 | } | ||
| 156 | |||
| 157 | func (s *jobSandbox) Stop() { s.StopWithLogs(nil) } | ||
| 158 | |||
| 159 | // StopJob stops the job container and leaves the services and network for the | ||
| 160 | // caller's final Stop. The job is stopped before artifacts and a site are read so a | ||
| 161 | // background step cannot change the tree while it is being published; a service | ||
| 162 | // does not touch that tree, and keeping it up means a publish failure can still | ||
| 163 | // attach its output. | ||
| 164 | func (s *jobSandbox) StopJob() { | ||
| 165 | if s.job != nil { | ||
| 166 | s.job.Stop() | ||
| 167 | s.job = nil | ||
| 168 | } | ||
| 169 | } | ||
| 170 | |||
| 171 | // StopWithLogs stops the job, then the services, then removes the network. When w | ||
| 172 | // is not nil, each service's last output is written under its own heading first: | ||
| 173 | // a job that failed after a database refused to start should not make the reader | ||
| 174 | // find the database's log in a second place. | ||
| 175 | func (s *jobSandbox) StopWithLogs(w io.Writer) { | ||
| 176 | s.StopJob() | ||
| 177 | if w != nil { | ||
| 178 | for i, svc := range s.services { | ||
| 179 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) | ||
| 180 | data, err := svc.Logs(ctx, container.MaxServiceLogBytes) | ||
| 181 | cancel() | ||
| 182 | if err != nil || len(data) == 0 { | ||
| 183 | continue | ||
| 184 | } | ||
| 185 | fmt.Fprintf(w, "\n=== service %s ===\n", s.serviceIDs[i]) | ||
| 186 | _, _ = w.Write(data) | ||
| 187 | if data[len(data)-1] != '\n' { | ||
| 188 | fmt.Fprintln(w) | ||
| 189 | } | ||
| 190 | } | ||
| 191 | } | ||
| 192 | for _, svc := range s.services { | ||
| 193 | svc.Stop() | ||
| 194 | } | ||
| 195 | s.services, s.serviceIDs = nil, nil | ||
| 196 | if s.network != nil { | ||
| 197 | ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) | ||
| 198 | _ = s.network.Remove(ctx) | ||
| 199 | cancel() | ||
| 200 | s.network = nil | ||
| 201 | } | ||
| 202 | } | ||
| 203 | |||
| 204 | // serviceNetworkName is the per-attempt network the job and its services join. | ||
| 205 | // The attempt is part of the name because two attempts of the same run must not | ||
| 206 | // share a network. | ||
| 207 | func serviceNetworkName(box runnerBox, job *runnerJob) string { | ||
| 208 | return fmt.Sprintf("vole-%s-%s-%d-net", box.id, strings.ToLower(job.ID), job.Attempt) | ||
| 209 | } | ||
| 210 | |||
| 211 | func serviceContainerName(box runnerBox, job *runnerJob, id string) string { | ||
| 212 | return fmt.Sprintf("vole-%s-%s-%d-svc-%s", box.id, strings.ToLower(job.ID), job.Attempt, id) | ||
| 213 | } | ||
| 214 | |||
| 215 | // cleanupStaleNetworks removes networks a killed runner left behind. It runs after | ||
| 216 | // cleanupStaleContainers, because a network with a member cannot be removed. | ||
| 217 | func cleanupStaleNetworks(cfg runnerBox) { | ||
| 218 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) | ||
| 219 | defer cancel() | ||
| 220 | names, err := container.ListNetworks(ctx, cfg.Config, "vole.runner.id="+cfg.id) | ||
| 221 | if err != nil { | ||
| 222 | return // the first job will report a runtime problem with more context | ||
| 223 | } | ||
| 224 | for _, name := range names { | ||
| 225 | labels, err := container.NetworkLabels(ctx, cfg.Config, name) | ||
| 226 | if err != nil { | ||
| 227 | continue | ||
| 228 | } | ||
| 229 | pid, _ := strconv.Atoi(labels["vole.runner.pid"]) | ||
| 230 | if pid > 0 && processAlive(pid) { | ||
| 231 | continue // another runner using the same cache is still working | ||
| 232 | } | ||
| 233 | _ = container.RemoveNetwork(ctx, cfg.Config, name) | ||
| 234 | } | ||
| 235 | } | ||