mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-08-06 00:44:22 +02:00
fix: stop leaking per-job docker networks (#1124)
Leaked per-job networks each hold a subnet of the daemon's address pool and nothing reclaimed them, so a host eventually fails every job with `all predefined address pools have been fully subnetted`. This is what CI hit in https://gitea.com/gitea/runner/actions/runs/742177. - teardown no longer loses a container, and its network with it: a failed `ContainerRemove` was reported as success, a container whose id was never learned was skipped, and the daemon's own `AutoRemove` teardown was raced - the idle cleanup reclaims what teardown cannot: networks carry `com.gitea.runner.uuid`, so a runner only touches its own, and a cutoff keeps a job starting during the pass out of scope - pull failures reported mid-stream were discarded, surfacing later as a confusing `No such image`; they now propagate, and fall back to a local copy instead of failing the job - `NetworkCreate` retries pool exhaustion briefly, then says which knobs to turn - digest-pinned images are not re-pulled, and removal kills first so it never waits out Podman's stop timeout (measured 10.1s → 0.09s per container) --------- Co-authored-by: bircni <bircni@icloud.com> Reviewed-on: https://gitea.com/gitea/runner/pulls/1124 Reviewed-by: bircni <bircni@icloud.com>
This commit is contained in:
@@ -297,3 +297,37 @@ func TestRunnerOnIdleSkipsWhenAlreadyCancelled(t *testing.T) {
|
||||
|
||||
assert.DirExists(t, stale)
|
||||
}
|
||||
|
||||
// The idle sweep reclaims the docker networks of jobs this runner did not live to tear down,
|
||||
// and stays out of the way of runners that share the daemon but not the registration.
|
||||
func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
||||
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
cfg := &config.Config{
|
||||
Container: config.Container{RequireDocker: true},
|
||||
Runner: config.Runner{
|
||||
WorkdirCleanupAge: 24 * time.Hour,
|
||||
IdleCleanupInterval: time.Minute,
|
||||
},
|
||||
}
|
||||
|
||||
var swept []string
|
||||
var sweptCutoff time.Time
|
||||
origRemoveOrphanNetworks := removeOrphanNetworks
|
||||
removeOrphanNetworks = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
swept = append(swept, runnerUUID)
|
||||
sweptCutoff = createdBefore
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeOrphanNetworks = origRemoveOrphanNetworks })
|
||||
|
||||
r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
|
||||
r.OnIdle(context.Background())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
// a network of a job starting during the pass is younger than this and so out of scope
|
||||
assert.Equal(t, now.Add(-24*time.Hour), sweptCutoff)
|
||||
|
||||
// a host-only runner has no daemon to sweep
|
||||
hostOnly := &Runner{uuid: "runner-2", cfg: &config.Config{Runner: cfg.Runner}, now: func() time.Time { return now }}
|
||||
hostOnly.OnIdle(context.Background())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user