feat: add startup janitor for stale bind-workdir task workspaces (#870)

- Add idle-time cleanup for stale bind-workdir task directories instead of cleaning them on the task execution path.
- Make cleanup behavior configurable with `runner.startup_cleanup_age` as the stale-age threshold (default: `24h`) and `runner.idle_cleanup_interval` as the idle cleanup cadence (default: `10m`).
- Restrict cleanup scope to numeric task directory names only, to avoid touching operator-managed folders.
- Document the cleanup settings in `config.example.yaml` and `README.md`.
- Add tests for stale-directory cleanup, idle cleanup throttling, and config default/override parsing.

## Why

When a runner or host crashes, normal per-task cleanup may not run, leaving stale task directories under the bind-workdir root. Running this cleanup only while the runner is idle recovers that disk space without adding overhead to active job execution.

If you want, I can also tighten the wording around `startup_cleanup_age`, since the key name now reads a bit misleadingly relative to the actual behavior.

---------

Co-authored-by: silverwind <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/870
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

internal/pkg/config/config.example.yaml

@@ -40,6 +40,12 @@ runner:
   # The runner uses exponential backoff when idle, increasing the interval up to this maximum.
   # Set to 0 or same as fetch_interval to disable backoff.
   fetch_interval_max: 5s
+  # While idle, remove stale bind-workdir task directories older than this duration.
+  # Setting either workdir_cleanup_age or idle_cleanup_interval to 0 (or any
+  # non-positive value) disables workdir cleanup entirely.
+  workdir_cleanup_age: 24h
+  # Cadence for the idle stale bind-workdir cleanup pass.
+  idle_cleanup_interval: 10m
   # The base interval for periodic log flush to the Gitea instance.
   # Logs may be sent earlier if the buffer reaches log_report_batch_size
   # or if log_report_max_latency expires after the first buffered row.
@@ -107,6 +113,7 @@ container:
   # If the path starts with '/', the '/' will be trimmed.
   # For example, if the parent directory is /path/to/my/dir, workdir_parent should be path/to/my/dir
   # If it's empty, /workspace will be used.
+  # Purely numeric subdirectories under this path are reserved for task workspaces and may be removed by idle cleanup.
   workdir_parent:
   # Volumes (including bind mounts) can be mounted to containers. Glob syntax is supported, see https://github.com/gobwas/glob
   # You can specify multiple volumes. If the sequence is empty, no volumes can be mounted.

internal/pkg/config/config.go

@@ -33,6 +33,8 @@ type Runner struct {
 	FetchTimeout        time.Duration     `yaml:"fetch_timeout"`          // FetchTimeout specifies the timeout duration for fetching resources.
 	FetchInterval       time.Duration     `yaml:"fetch_interval"`         // FetchInterval specifies the interval duration for fetching resources.
 	FetchIntervalMax    time.Duration     `yaml:"fetch_interval_max"`     // FetchIntervalMax specifies the maximum backoff interval when idle.
+	WorkdirCleanupAge   time.Duration     `yaml:"workdir_cleanup_age"`    // WorkdirCleanupAge removes stale bind-workdir task directories older than this duration during idle cleanup.
+	IdleCleanupInterval time.Duration     `yaml:"idle_cleanup_interval"`  // IdleCleanupInterval runs stale bind-workdir cleanup periodically while the runner is idle. Set to 0 to disable cleanup cadence.
 	LogReportInterval   time.Duration     `yaml:"log_report_interval"`    // LogReportInterval specifies the base interval for periodic log flush.
 	LogReportMaxLatency time.Duration     `yaml:"log_report_max_latency"` // LogReportMaxLatency specifies the max time a log row can wait before being sent.
 	LogReportBatchSize  int               `yaml:"log_report_batch_size"`  // LogReportBatchSize triggers immediate log flush when buffer reaches this size.
@@ -92,6 +94,7 @@ type Config struct {
 // If file is not empty, it will be used to load the configuration.
 func LoadDefault(file string) (*Config, error) {
 	cfg := &Config{}
+	definedRunnerKeys := map[string]bool{}
 	if file != "" {
 		content, err := os.ReadFile(file)
 		if err != nil {
@@ -100,6 +103,10 @@ func LoadDefault(file string) (*Config, error) {
 		if err := yaml.Unmarshal(content, cfg); err != nil {
 			return nil, fmt.Errorf("parse config file %q: %w", file, err)
 		}
+		definedRunnerKeys, err = definedRunnerConfigKeys(content)
+		if err != nil {
+			return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err)
+		}
 	}
 	compatibleWithOldEnvs(file != "", cfg)
 
@@ -157,6 +164,12 @@ func LoadDefault(file string) (*Config, error) {
 	if cfg.Runner.FetchIntervalMax <= 0 {
 		cfg.Runner.FetchIntervalMax = 5 * time.Second
 	}
+	if cfg.Runner.WorkdirCleanupAge == 0 && !definedRunnerKeys["workdir_cleanup_age"] {
+		cfg.Runner.WorkdirCleanupAge = 24 * time.Hour
+	}
+	if cfg.Runner.IdleCleanupInterval == 0 && !definedRunnerKeys["idle_cleanup_interval"] {
+		cfg.Runner.IdleCleanupInterval = 10 * time.Minute
+	}
 	if cfg.Runner.LogReportInterval <= 0 {
 		cfg.Runner.LogReportInterval = 5 * time.Second
 	}
@@ -199,3 +212,30 @@ func LoadDefault(file string) (*Config, error) {
 
 	return cfg, nil
 }
+
+func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
+	var root yaml.Node
+	if err := yaml.Unmarshal(content, &root); err != nil {
+		return nil, err
+	}
+
+	defined := map[string]bool{}
+	if len(root.Content) == 0 {
+		return defined, nil
+	}
+
+	doc := root.Content[0]
+	for i := 0; i+1 < len(doc.Content); i += 2 {
+		key := doc.Content[i]
+		value := doc.Content[i+1]
+		if key.Value != "runner" || value.Kind != yaml.MappingNode {
+			continue
+		}
+		for j := 0; j+1 < len(value.Content); j += 2 {
+			defined[value.Content[j].Value] = true
+		}
+		break
+	}
+
+	return defined, nil
+}

internal/pkg/config/config_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -39,3 +40,80 @@ cache:
 	_, err := LoadDefault(path)
 	require.NoError(t, err)
 }
+
+func TestLoadDefault_DefaultsWorkdirCleanupAge(t *testing.T) {
+	cfg, err := LoadDefault("")
+	require.NoError(t, err)
+	assert.Equal(t, 24*time.Hour, cfg.Runner.WorkdirCleanupAge)
+	assert.Equal(t, 10*time.Minute, cfg.Runner.IdleCleanupInterval)
+}
+
+func TestLoadDefault_UsesConfiguredWorkdirCleanupAge(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`
+runner:
+  workdir_cleanup_age: 2h30m
+`), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, 150*time.Minute, cfg.Runner.WorkdirCleanupAge)
+}
+
+func TestLoadDefault_UsesConfiguredIdleCleanupInterval(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`
+runner:
+  idle_cleanup_interval: 45m
+`), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, 45*time.Minute, cfg.Runner.IdleCleanupInterval)
+}
+
+func TestLoadDefault_AllowsDisablingWorkdirCleanup(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`
+runner:
+  workdir_cleanup_age: 0s
+  idle_cleanup_interval: 0s
+`), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, time.Duration(0), cfg.Runner.WorkdirCleanupAge)
+	assert.Equal(t, time.Duration(0), cfg.Runner.IdleCleanupInterval)
+}
+
+func TestLoadDefault_AllowsNegativeWorkdirCleanupValues(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte(`
+runner:
+  workdir_cleanup_age: -1s
+  idle_cleanup_interval: -1s
+`), 0o600))
+
+	cfg, err := LoadDefault(path)
+	require.NoError(t, err)
+	assert.Equal(t, -1*time.Second, cfg.Runner.WorkdirCleanupAge)
+	assert.Equal(t, -1*time.Second, cfg.Runner.IdleCleanupInterval)
+}
+
+// TestLoadDefault_MalformedYAMLReturnsParseError pins the error surfaced for
+// invalid YAML to the canonical "parse config file" message rather than the
+// "for defaults metadata" variant — i.e. the main yaml.Unmarshal runs first.
+func TestLoadDefault_MalformedYAMLReturnsParseError(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "config.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("runner:\n  capacity: [unterminated\n"), 0o600))
+
+	_, err := LoadDefault(path)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "parse config file")
+	assert.NotContains(t, err.Error(), "defaults metadata")
+}