vole
Clone
Read only, no credentials:
https://rodent.software/vole.git
Write over SSH with a registered key:
git clone ssh://git@rodent.software:2222/vole.git
A free and open-source forge that lives in your Git repo.
Let a job budget its whole run
e7aa0785cb69ace8c1338cc0b924113ea52a7abb · Keenan Nemetz · 6d ago · parent 2bc0ae74
Let a job budget its whole run A step was capped at an hour and a job was not capped at all, so a job of several slow steps held a runner for as long as they took, with no number saying what the job was allowed to spend. And one long step — a session, a soak test, a slow suite — could not be expressed, because the only clock was per step and its ceiling was an hour. A job may now declare `timeout: 2h`: a wall clock its steps share, starting with the first step. A step that declares its own timeout is cut short when the budget ends first, and a step that declares none runs with whatever is left, so one long step needs only the job's number. A budget spent before a step starts stops the job there instead of running the step on its own cap. The budget is optional, pinned with the run, and spent by a local run the same way, so nothing changes for a workflow that does not declare one. The step ceiling rises from one hour to four to make a single long step possible; the default stays ten minutes. The image build is not charged to the budget — it has its own cap, so a slow pull cannot eat the time the steps were given. A step timeout longer than its job's budget is refused at push time: the budget would end first, so the step could never use what it asked for. Closes #161. # Conflicts: # ci/agent/agent.go # cmd/vole/workflowrun.go # docs/reference/workflow-reference.md
ci/agent/agent.go +28−8
| @@ -81,6 +81,10 @@ type runnerJob struct { | |||
| 81 | 81 | // workflow did not say. The runner applies it to the container it starts, so the | |
| 82 | 82 | // job's request decides and the machine's default is what an unspecified job gets. | |
| 83 | 83 | Network string `json:"network,omitempty"` | |
| 84 | // Timeout is the job's budget as a duration string, or empty when it has none. | ||
| 85 | // The runner starts the clock with the first step and cuts every later step short | ||
| 86 | // when the budget runs out. | ||
| 87 | Timeout string `json:"timeout,omitempty"` | ||
| 84 | 88 | // Image is a registry reference or "file:<name>". For the latter, ImageDefinition | |
| 85 | 89 | // is the pinned recipe and ImageSHA its digest; the runner builds it locally. | |
| 86 | 90 | Image string `json:"image"` | |
| @@ -669,13 +673,29 @@ func (s *runnerSession) run(remote, cache string, job *runnerJob, lease time.Dur | |||
| 669 | 673 | }() | |
| 670 | 674 | ||
| 671 | 675 | steps := make([]runs.StepResult, 0, len(job.Steps)) | |
| 676 | // The job's budget starts with its first step. An image build has its own cap, so a | ||
| 677 | // slow pull must not eat the time the steps were given. | ||
| 678 | var deadline time.Time | ||
| 679 | if d, err := time.ParseDuration(job.Timeout); err == nil && d > 0 { | ||
| 680 | deadline = time.Now().Add(d) | ||
| 681 | } | ||
| 672 | 682 | for _, step := range job.Steps { | |
| 673 | 683 | // The step's section of the log starts at its header, so a reader can open | |
| 674 | 684 | // one step without holding the image build that ran before it. | |
| 675 | 685 | logStart := log.Len() | |
| 676 | 686 | started := time.Now().UTC() | |
| 677 | 687 | fmt.Fprintf(log, "\n--- %s\n", step.Name) | |
| 678 | code, lost := s.execStep(job.ID, sandbox, step, lease, log) | ||
| 688 | own, _ := time.ParseDuration(step.Timeout) | ||
| 689 | limit := ci.StepLimit(ci.BudgetLeft(deadline, time.Now()), own) | ||
| 690 | code, lost := 0, false | ||
| 691 | if limit <= 0 { | ||
| 692 | // The budget is spent, so the step must not start. Reporting it as a failed | ||
| 693 | // step keeps the record the same shape as any other failure. | ||
| 694 | fmt.Fprintf(log, "\nthe job's %s budget ran out before this step\n", job.Timeout) | ||
| 695 | code = 1 | ||
| 696 | } else { | ||
| 697 | code, lost = s.execStep(job.ID, sandbox, step, limit, lease, log) | ||
| 698 | } | ||
| 679 | 699 | // The mask holds back a few bytes so a value split across writes is caught; | |
| 680 | 700 | // flushing here keeps this step's byte range its own. | |
| 681 | 701 | log.Flush() | |
| @@ -1085,12 +1105,10 @@ func collectArtifactFiles(root string, patterns []string) ([]artifactFile, error | |||
| 1085 | 1105 | ||
| 1086 | 1106 | // execStep runs one shell step, heartbeating while it works. It returns the exit | |
| 1087 | 1107 | // code, the output produced, and whether the lease was lost. | |
| 1088 | func (s *runnerSession) execStep(runID string, sandbox *container.Container, step runnerStep, lease time.Duration, out io.Writer) (int, bool) { | ||
| 1089 | timeout := 10 * time.Minute | ||
| 1090 | if d, err := time.ParseDuration(step.Timeout); err == nil && d > 0 { | ||
| 1091 | timeout = d | ||
| 1092 | } | ||
| 1093 | ctx, cancel := context.WithTimeout(context.Background(), timeout) | ||
| 1108 | func (s *runnerSession) execStep(runID string, sandbox *container.Container, step runnerStep, limit time.Duration, lease time.Duration, out io.Writer) (int, bool) { | ||
| 1109 | // limit is already resolved: the step's own timeout, or what is left of the job's | ||
| 1110 | // budget. The caller never passes zero. | ||
| 1111 | ctx, cancel := context.WithTimeout(context.Background(), limit) | ||
| 1094 | 1112 | defer cancel() | |
| 1095 | 1113 | ||
| 1096 | 1114 | cmd := sandbox.Exec(ctx, step.Run) | |
| @@ -1098,7 +1116,9 @@ func (s *runnerSession) execStep(runID string, sandbox *container.Container, ste | |||
| 1098 | 1116 | if ctx.Err() == context.DeadlineExceeded { | |
| 1099 | 1117 | // The heartbeat will have extended the lease this far; report the timeout as | |
| 1100 | 1118 | // a failure rather than leaving the job hanging. | |
| 1101 | fmt.Fprintf(out, "\nstep timed out after %s\n", timeout) | ||
| 1119 | // What is left of a budget is rarely a round number; a reader wants the promise, | ||
| 1120 | // not the microseconds the bookkeeping cost. | ||
| 1121 | fmt.Fprintf(out, "\nstep timed out after %s\n", limit.Round(time.Millisecond)) | ||
| 1102 | 1122 | return 1, lost | |
| 1103 | 1123 | } | |
| 1104 | 1124 | return code, lost | |