vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Let a job start service containers beside it
c1929d2fceb5e9e65204d5eef907d0781b9a5391 · Keenan Nemetz · 5d ago · parent 1d06b008
ci/container/container.go +236
| @@ -22,6 +22,7 @@ package container | |||
| 22 | 22 | ||
| 23 | 23 | import ( | |
| 24 | 24 | "context" | |
| 25 | "encoding/json" | ||
| 25 | 26 | "fmt" | |
| 26 | 27 | "io" | |
| 27 | 28 | "os" | |
| @@ -30,11 +31,17 @@ import ( | |||
| 30 | 31 | "sort" | |
| 31 | 32 | "strconv" | |
| 32 | 33 | "strings" | |
| 34 | "sync" | ||
| 33 | 35 | "time" | |
| 34 | 36 | ||
| 35 | 37 | "github.com/rodent-software/vole/ci" | |
| 36 | 38 | ) | |
| 37 | 39 | ||
| 40 | // MaxServiceLogBytes bounds one service's output in an attempt log. A database | ||
| 41 | // that fails to start says why at the end; the tail is what the reader needs, | ||
| 42 | // and the cap keeps three services from crowding out the steps. | ||
| 43 | const MaxServiceLogBytes = 256 << 10 | ||
| 44 | |||
| 38 | 45 | // KeepAlive is PID 1's command. Steps run with exec, so the container has to stay | |
| 39 | 46 | // up between them. `/bin/sh` is also what the steps need, which keeps the | |
| 40 | 47 | // requirement to one the image must satisfy anyway. | |
| @@ -377,3 +384,232 @@ func Build(ctx context.Context, cfg Config, definition, tag string, out io.Write | |||
| 377 | 384 | } | |
| 378 | 385 | return nil | |
| 379 | 386 | } | |
| 387 | |||
| 388 | // ServiceSpec is one server container started beside a job: a database, a queue, | ||
| 389 | // or another process the steps reach by name. It has no job tree and no secrets | ||
| 390 | // file, and it runs the image's own command rather than a keepalive shell. | ||
| 391 | type ServiceSpec struct { | ||
| 392 | // Name is the container's name on the runtime. It is unique to this attempt. | ||
| 393 | Name string | ||
| 394 | // Alias is the name the job's network resolves to this container. It is the | ||
| 395 | // service's declared name, so a step connects to `db:5432`. | ||
| 396 | Alias string | ||
| 397 | Image string | ||
| 398 | Env map[string]string | ||
| 399 | Labels map[string]string | ||
| 400 | } | ||
| 401 | |||
| 402 | // ServiceRunArgs is the argument list for a service container. It is the same | ||
| 403 | // isolation as RunArgs where that makes sense: the job's user, capabilities | ||
| 404 | // dropped, no-new-privileges, a tmpfs /tmp, the runtime's resource limits, and the | ||
| 405 | // network the job was given. What is absent is the point: no workspace mount, no | ||
| 406 | // cache mount, no secrets file, and no entrypoint override. | ||
| 407 | func ServiceRunArgs(cfg Config, s ServiceSpec) []string { | ||
| 408 | args := []string{ | ||
| 409 | "run", "-d", | ||
| 410 | "--name", s.Name, | ||
| 411 | } | ||
| 412 | keys := make([]string, 0, len(s.Labels)) | ||
| 413 | for k := range s.Labels { | ||
| 414 | keys = append(keys, k) | ||
| 415 | } | ||
| 416 | sort.Strings(keys) | ||
| 417 | for _, k := range keys { | ||
| 418 | args = append(args, "--label", k+"="+s.Labels[k]) | ||
| 419 | } | ||
| 420 | args = append(args, "--user", cfg.User) | ||
| 421 | if cfg.Userns != "" { | ||
| 422 | args = append(args, "--userns", cfg.Userns) | ||
| 423 | } | ||
| 424 | if s.Alias != "" { | ||
| 425 | args = append(args, "--network-alias", s.Alias) | ||
| 426 | } | ||
| 427 | names := make([]string, 0, len(s.Env)) | ||
| 428 | for name := range s.Env { | ||
| 429 | names = append(names, name) | ||
| 430 | } | ||
| 431 | sort.Strings(names) | ||
| 432 | for _, name := range names { | ||
| 433 | args = append(args, "--env", name+"="+s.Env[name]) | ||
| 434 | } | ||
| 435 | args = append(args, | ||
| 436 | "--tmpfs", "/tmp:rw,nosuid,nodev,exec,mode=1777", | ||
| 437 | "--cap-drop", "ALL", | ||
| 438 | "--security-opt", "no-new-privileges", | ||
| 439 | "--init", | ||
| 440 | "--stop-timeout", "5", | ||
| 441 | "--network", cfg.Network, | ||
| 442 | "--pull", cfg.Pull, | ||
| 443 | ) | ||
| 444 | if cfg.PIDs > 0 { | ||
| 445 | args = append(args, "--pids-limit", strconv.Itoa(cfg.PIDs)) | ||
| 446 | } | ||
| 447 | if cfg.Memory != "" { | ||
| 448 | args = append(args, "--memory", cfg.Memory) | ||
| 449 | } | ||
| 450 | if cfg.CPUs != "" { | ||
| 451 | args = append(args, "--cpus", cfg.CPUs) | ||
| 452 | } | ||
| 453 | return append(args, s.Image) | ||
| 454 | } | ||
| 455 | |||
| 456 | // StartService creates a service container and waits for the runtime to report it | ||
| 457 | // started. The container is not removed with --rm: the runner reads its log before | ||
| 458 | // removing it, and a killed runner's leftovers are found by their labels. | ||
| 459 | func StartService(cfg Config, s ServiceSpec) (*Container, error) { | ||
| 460 | c := &Container{cfg: cfg, name: s.Name} | ||
| 461 | cmd := exec.Command(cfg.Runtime, ServiceRunArgs(cfg, s)...) | ||
| 462 | var stderr strings.Builder | ||
| 463 | cmd.Stderr = &stderr | ||
| 464 | if err := cmd.Run(); err != nil { | ||
| 465 | _ = exec.Command(cfg.Runtime, "rm", "-f", s.Name).Run() | ||
| 466 | return nil, fmt.Errorf("%s could not start service %s: %w: %s", | ||
| 467 | cfg.Runtime, s.Alias, err, strings.TrimSpace(stderr.String())) | ||
| 468 | } | ||
| 469 | return c, nil | ||
| 470 | } | ||
| 471 | |||
| 472 | // Running reports whether the runtime still has the container up. A service that | ||
| 473 | // exited before the steps begin is a failed job, not a connection error later. | ||
| 474 | func (c *Container) Running(ctx context.Context) (bool, error) { | ||
| 475 | out, err := exec.CommandContext(ctx, c.cfg.Runtime, "inspect", | ||
| 476 | "--format", "{{.State.Running}}", c.name).Output() | ||
| 477 | if err != nil { | ||
| 478 | return false, fmt.Errorf("%s inspect %s: %w", c.cfg.Runtime, c.name, err) | ||
| 479 | } | ||
| 480 | switch strings.TrimSpace(string(out)) { | ||
| 481 | case "true", "running": | ||
| 482 | return true, nil | ||
| 483 | } | ||
| 484 | return false, nil | ||
| 485 | } | ||
| 486 | |||
| 487 | // Logs reads the container's output, keeping the last max bytes. A service that | ||
| 488 | // failed usually says why at the end, so the tail is the useful part; max also | ||
| 489 | // bounds what one service can add to an attempt log. A non-zero runtime exit with | ||
| 490 | // output is not an error here: the output is what the caller wanted. | ||
| 491 | func (c *Container) Logs(ctx context.Context, max int64) ([]byte, error) { | ||
| 492 | if max <= 0 { | ||
| 493 | max = 256 << 10 | ||
| 494 | } | ||
| 495 | tail := &tailBuffer{max: max} | ||
| 496 | cmd := exec.CommandContext(ctx, c.cfg.Runtime, "logs", c.name) | ||
| 497 | cmd.Stdout = tail | ||
| 498 | cmd.Stderr = tail | ||
| 499 | err := cmd.Run() | ||
| 500 | data := tail.Bytes() | ||
| 501 | if err != nil && len(data) == 0 { | ||
| 502 | return nil, fmt.Errorf("%s logs %s: %w", c.cfg.Runtime, c.name, err) | ||
| 503 | } | ||
| 504 | return data, nil | ||
| 505 | } | ||
| 506 | |||
| 507 | // tailBuffer keeps the last max bytes written to it. The runtime's stdout and | ||
| 508 | // stderr are copied by two goroutines, so the writes are serialized here; without | ||
| 509 | // the lock the two streams would race and corrupt the tail. | ||
| 510 | type tailBuffer struct { | ||
| 511 | mu sync.Mutex | ||
| 512 | max int64 | ||
| 513 | data []byte | ||
| 514 | } | ||
| 515 | |||
| 516 | func (b *tailBuffer) Write(p []byte) (int, error) { | ||
| 517 | b.mu.Lock() | ||
| 518 | defer b.mu.Unlock() | ||
| 519 | if int64(len(p)) >= b.max { | ||
| 520 | b.data = append(b.data[:0], p[int64(len(p))-b.max:]...) | ||
| 521 | return len(p), nil | ||
| 522 | } | ||
| 523 | b.data = append(b.data, p...) | ||
| 524 | if int64(len(b.data)) > b.max { | ||
| 525 | b.data = append(b.data[:0], b.data[int64(len(b.data))-b.max:]...) | ||
| 526 | } | ||
| 527 | return len(p), nil | ||
| 528 | } | ||
| 529 | |||
| 530 | func (b *tailBuffer) Bytes() []byte { | ||
| 531 | b.mu.Lock() | ||
| 532 | defer b.mu.Unlock() | ||
| 533 | return append([]byte(nil), b.data...) | ||
| 534 | } | ||
| 535 | |||
| 536 | // Network is one user-defined network created for an attempt. The job and its | ||
| 537 | // services are its only members, so service names resolve inside it and nowhere | ||
| 538 | // else. | ||
| 539 | type Network struct { | ||
| 540 | cfg Config | ||
| 541 | name string | ||
| 542 | } | ||
| 543 | |||
| 544 | // CreateNetwork makes the per-attempt network. The labels are the same runner and | ||
| 545 | // run labels a container carries, so a killed runner's network is found the same | ||
| 546 | // way its containers are. | ||
| 547 | func CreateNetwork(ctx context.Context, cfg Config, name string, labels map[string]string) (*Network, error) { | ||
| 548 | args := []string{"network", "create"} | ||
| 549 | keys := make([]string, 0, len(labels)) | ||
| 550 | for k := range labels { | ||
| 551 | keys = append(keys, k) | ||
| 552 | } | ||
| 553 | sort.Strings(keys) | ||
| 554 | for _, k := range keys { | ||
| 555 | args = append(args, "--label", k+"="+labels[k]) | ||
| 556 | } | ||
| 557 | args = append(args, name) | ||
| 558 | cmd := exec.CommandContext(ctx, cfg.Runtime, args...) | ||
| 559 | var stderr strings.Builder | ||
| 560 | cmd.Stderr = &stderr | ||
| 561 | if err := cmd.Run(); err != nil { | ||
| 562 | return nil, fmt.Errorf("%s could not create network %s: %w: %s", | ||
| 563 | cfg.Runtime, name, err, strings.TrimSpace(stderr.String())) | ||
| 564 | } | ||
| 565 | return &Network{cfg: cfg, name: name}, nil | ||
| 566 | } | ||
| 567 | |||
| 568 | // Name is what the job container and its services pass as their network. | ||
| 569 | func (n *Network) Name() string { return n.name } | ||
| 570 | |||
| 571 | // Remove deletes the network. It is best effort on a path that is already failing: | ||
| 572 | // a network with a container still attached cannot be removed, and the container | ||
| 573 | // cleanup is what runs first. | ||
| 574 | func (n *Network) Remove(ctx context.Context) error { | ||
| 575 | return RemoveNetwork(ctx, n.cfg, n.name) | ||
| 576 | } | ||
| 577 | |||
| 578 | // RemoveNetwork deletes a network by name. Cleanup after a killed runner knows the | ||
| 579 | // name from the label listing but has no Network value, so the name is the handle. | ||
| 580 | func RemoveNetwork(ctx context.Context, cfg Config, name string) error { | ||
| 581 | return exec.CommandContext(ctx, cfg.Runtime, "network", "rm", name).Run() | ||
| 582 | } | ||
| 583 | |||
| 584 | // ListNetworks returns the networks carrying one label. It is a list rather than a | ||
| 585 | // count so the caller can inspect each one's owner before removing it. | ||
| 586 | func ListNetworks(ctx context.Context, cfg Config, label string) ([]string, error) { | ||
| 587 | out, err := exec.CommandContext(ctx, cfg.Runtime, "network", "ls", | ||
| 588 | "--filter", "label="+label, "--format", "{{.Name}}").Output() | ||
| 589 | if err != nil { | ||
| 590 | return nil, err | ||
| 591 | } | ||
| 592 | var names []string | ||
| 593 | for _, line := range strings.Split(string(out), "\n") { | ||
| 594 | if name := strings.TrimSpace(line); name != "" { | ||
| 595 | names = append(names, name) | ||
| 596 | } | ||
| 597 | } | ||
| 598 | return names, nil | ||
| 599 | } | ||
| 600 | |||
| 601 | // NetworkLabels reads a network's labels, which say which runner and process | ||
| 602 | // created it. Parsing JSON rather than a template keeps the result the same across | ||
| 603 | // the three supported runtimes. | ||
| 604 | func NetworkLabels(ctx context.Context, cfg Config, name string) (map[string]string, error) { | ||
| 605 | out, err := exec.CommandContext(ctx, cfg.Runtime, "network", "inspect", | ||
| 606 | "--format", "{{json .Labels}}", name).Output() | ||
| 607 | if err != nil { | ||
| 608 | return nil, err | ||
| 609 | } | ||
| 610 | var labels map[string]string | ||
| 611 | if err := json.Unmarshal(out, &labels); err != nil { | ||
| 612 | return nil, err | ||
| 613 | } | ||
| 614 | return labels, nil | ||
| 615 | } | ||