vole
Clone
Read only, no credentials:
https://rodent.software/git/vole.git
Write over SSH with a registered key:
git clone ssh://git@0.0.0.0:2222/vole.git
A free and open-source forge that lives in your Git repo.
Keep a finished CI attempt and drain before shutting down
73755a56550dc6819b47469fbb08e85e7c7c5d04 · Keenan Nemetz · 9d ago · parent 1265ea55
Keep a finished CI attempt and drain before shutting down A job whose lease expired was requeued even when its last step had already succeeded. The queue cleared the owner and the runner refused to report, so the same work ran again as attempt 2 for no failing step. Keep the owner on a requeued row and accept a late record and log until another runner leases the job. The runner now sends a late record when the heartbeat says the lease is gone, and the server accepts it only for the attempt that is still current. A clean server stop also drained nothing: closing SSH killed the runner and threw the job away. Stop leasing new work on shutdown and wait up to a minute for the jobs already running to report.
server/server.go +42
| @@ -213,6 +213,45 @@ func (s *Server) Handler() http.Handler { | |||
| 213 | 213 | }) | |
| 214 | 214 | } | |
| 215 | 215 | ||
| 216 | // drainGrace is how long a clean shutdown lets an already-leased job finish before | ||
| 217 | // closing the listeners. It is shorter than systemd's default stop timeout, which | ||
| 218 | // leaves room for the HTTP server's own shutdown. Work that does not fit in the | ||
| 219 | // window is still lost on a restart; a result that arrives inside it is not. | ||
| 220 | const drainGrace = time.Minute | ||
| 221 | |||
| 222 | // drain stops new leases and waits for the jobs already running to report. It is | ||
| 223 | // best effort: a runner that died, or a build longer than the grace, is left for | ||
| 224 | // the next process, which can requeue it from the durable lease. | ||
| 225 | func (s *Server) drain() { | ||
| 226 | if s.Runs == nil { | ||
| 227 | return | ||
| 228 | } | ||
| 229 | s.Runs.Drain() | ||
| 230 | n, err := s.Runs.Running() | ||
| 231 | if err != nil { | ||
| 232 | s.Log.Warn("drain: cannot count running jobs", "err", err) | ||
| 233 | return | ||
| 234 | } | ||
| 235 | if n == 0 { | ||
| 236 | return | ||
| 237 | } | ||
| 238 | s.Log.Info("draining running jobs", "jobs", n, "grace", drainGrace) | ||
| 239 | deadline := time.Now().Add(drainGrace) | ||
| 240 | for time.Now().Before(deadline) { | ||
| 241 | time.Sleep(250 * time.Millisecond) | ||
| 242 | n, err = s.Runs.Running() | ||
| 243 | if err != nil { | ||
| 244 | s.Log.Warn("drain: cannot count running jobs", "err", err) | ||
| 245 | return | ||
| 246 | } | ||
| 247 | if n == 0 { | ||
| 248 | s.Log.Info("running jobs finished; shutting down") | ||
| 249 | return | ||
| 250 | } | ||
| 251 | } | ||
| 252 | s.Log.Warn("drain grace expired with jobs still running", "jobs", n) | ||
| 253 | } | ||
| 254 | |||
| 216 | 255 | // Run starts the listeners and blocks until ctx is cancelled. With automatic | |
| 217 | 256 | // HTTPS, the HTTP address only answers ACME challenges and redirects, and the | |
| 218 | 257 | // forge is served over TLS on HTTPSAddr. | |
| @@ -291,6 +330,9 @@ func (s *Server) Run(ctx context.Context) error { | |||
| 291 | 330 | ||
| 292 | 331 | select { | |
| 293 | 332 | case <-ctx.Done(): | |
| 333 | // Let the runners finish what they already hold before the sockets go away. | ||
| 334 | // A deploy that cuts a job in its last step costs a whole second attempt. | ||
| 335 | s.drain() | ||
| 294 | 336 | sh, cancel := context.WithTimeout(context.Background(), 20*time.Second) | |
| 295 | 337 | defer cancel() | |
| 296 | 338 | s.SSH.Close() | |