fix: continue docker cleanup after a failed step

Teardown was built from `Then`, which stops on the previous step's error and
on a cancelled context. A volume that is still in use, or a flaky daemon, then
orphaned everything downstream: the service containers and the job network.

Replace the combinator chain with straight-line best-effort cleanup that runs
every step and joins the errors. Service containers are now removed before the
volumes, since a service can hold them via `--volumes-from`, and the network
stays last.

Co-Authored-By: Claude (Opus 4.8) <noreply@anthropic.com>
This commit is contained in:
silverwind
2026-07-24 18:26:59 +02:00
parent 5a98252565
commit 6f06ca5cde
2 changed files with 56 additions and 30 deletions

View File

@@ -7,6 +7,7 @@ package runner
import (
"bytes"
"context"
"errors"
"fmt"
"os"
"runtime"
@@ -364,7 +365,7 @@ func TestRunContextValidVolumes(t *testing.T) {
assert.Len(t, rc.validVolumes(), len(got), "repeated calls must be stable, not accumulate")
}
func TestCleanupJobContainerCleansServicesWithoutJobContainer(t *testing.T) {
func TestCleanupJobResourcesCleansServicesWithoutJobContainer(t *testing.T) {
service := &containerMock{}
service.On("Remove").Return(func(context.Context) error { return nil }).Once()
service.On("Close").Return(func(context.Context) error { return nil }).Once()
@@ -374,11 +375,36 @@ func TestCleanupJobContainerCleansServicesWithoutJobContainer(t *testing.T) {
ServiceContainers: []container.ExecutionsEnvironment{service},
}
err := rc.cleanupJobContainer(log.StandardLogger(), "external-network", false)(context.Background())
err := rc.cleanupJobResources("external-network", false)(context.Background())
require.NoError(t, err)
service.AssertExpectations(t)
}
// cleanup used to bail out on a previous step's error and on a cancelled context
func TestCleanupJobResourcesContinuesAfterFailure(t *testing.T) {
t.Setenv("DOCKER_HOST", "unix:///nonexistent.sock")
jobContainer := &containerMock{}
jobContainer.On("Remove").Return(func(context.Context) error { return errors.New("removal failed") }).Once()
service := &containerMock{}
service.On("Remove").Return(func(context.Context) error { return nil }).Once()
service.On("Close").Return(func(context.Context) error { return nil }).Once()
rc := &RunContext{
Name: "job",
Config: &Config{},
Run: &model.Run{Workflow: &model.Workflow{Name: "wf"}, JobID: "job"},
JobContainer: jobContainer,
ServiceContainers: []container.ExecutionsEnvironment{service},
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
require.Error(t, rc.cleanupJobResources("job-network", true)(ctx))
jobContainer.AssertExpectations(t)
service.AssertExpectations(t)
}
// TestInterpolateOutputsIsPerMatrixCombo guards the matrix-output fix: combinations share one
// *model.Job, so each must interpolate from its own pristine snapshot. Otherwise the first
// combo's resolved value freezes the shared template and later combos can't resolve their own.