feat: add Runner.ShutdownTimeout config option

This controls the amount of time the runner will wait for running jobs
to finish before cancelling them. Defaults to 0s in order to maintain
backwards compatibility with previous behavior.

internal/app/cmd/daemon.go

@@ -125,9 +125,16 @@ func runDaemon(ctx context.Context, configFile *string) func(cmd *cobra.Command,
 		go poller.Poll()
 
 		<-ctx.Done()
-		log.Infof("runner: %s gracefully shutting down", resp.Msg.Runner.Name)
+		log.Infof("runner: %s shutdown initiated, waiting %s for running jobs to complete before shutting down", resp.Msg.Runner.Name, cfg.Runner.ShutdownTimeout)
 
-		return poller.Shutdown(context.Background())
+		ctx, cancel := context.WithTimeout(context.Background(), cfg.Runner.ShutdownTimeout)
+		defer cancel()
+
+		err = poller.Shutdown(ctx)
+		if err != nil {
+			log.Warnf("runner: %s cancelled in progress jobs during shutdown", resp.Msg.Runner.Name)
+		}
+		return nil
 	}
 }
 

internal/app/poll/poller.go

@@ -74,14 +74,27 @@ func (p *Poller) Shutdown(ctx context.Context) error {
 	p.shutdownPolling()
 
 	select {
-	// gracefully shutdown
+	// graceful shutdown completed succesfully
 	case <-p.done:
 		return nil
 
-	// Our timeout for shutting down ran out
+	// our timeout for shutting down ran out
 	case <-ctx.Done():
+		// when both the timeout fires and the graceful shutdown
+		// completed succsfully, this branch of the select may
+		// fire. Do a non-blocking check here against the graceful
+		// shutdown status to avoid sending an error if we don't need to.
+		_, ok := <-p.done
+		if !ok {
+			return nil
+		}
+
 		// force a shutdown of all running jobs
 		p.shutdownJobs()
+
+		// wait for running jobs to report their status to Gitea
+		_, _ = <-p.done
+
 		return ctx.Err()
 	}
 }