mirror of
https://gitea.com/gitea/act_runner.git
synced 2026-08-06 00:44:22 +02:00
Compare commits
9 Commits
v2.3.0
...
2398d4a527
| Author | SHA1 | Date | |
|---|---|---|---|
|
2398d4a527 | ||
|
b7a3bf98bc | ||
|
da4037899a | ||
|
|
e4fe49dba4 | ||
|
41c72216bf | ||
|
|
3f7fd16ea1 | ||
|
|
0192861155 | ||
|
|
e6c7ba3a15 | ||
|
|
61f0cfa951 |
@@ -33,6 +33,8 @@ jobs:
|
||||
done
|
||||
- name: lint
|
||||
run: make lint
|
||||
- name: checks
|
||||
run: make checks
|
||||
- name: build
|
||||
run: make build
|
||||
- name: test
|
||||
|
||||
@@ -11,6 +11,7 @@ linters:
|
||||
- dupl
|
||||
- errcheck
|
||||
- forbidigo
|
||||
- forcetypeassert
|
||||
- gocheckcompilerdirectives
|
||||
- gocritic
|
||||
- goheader
|
||||
@@ -102,6 +103,9 @@ linters:
|
||||
- linters:
|
||||
- forbidigo
|
||||
path: cmd
|
||||
- linters:
|
||||
- forcetypeassert
|
||||
path: _test\.go
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
|
||||
21
Makefile
21
Makefile
@@ -4,9 +4,9 @@ DIST_DIRS := $(DIST)/binaries $(DIST)/release
|
||||
GO ?= go
|
||||
SHASUM ?= shasum -a 256
|
||||
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
|
||||
XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest
|
||||
XGO_PACKAGE ?= src.techknowlogick.com/xgo@v1.9.0 # renovate: datasource=go
|
||||
XGO_VERSION := go-1.26.x
|
||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.10
|
||||
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/gxz@v0.5.15 # renovate: datasource=go
|
||||
|
||||
LINUX_ARCHS ?= linux/amd64,linux/arm64
|
||||
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
|
||||
@@ -18,8 +18,10 @@ DOCKER_TAG ?= nightly
|
||||
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
|
||||
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
|
||||
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.3.0
|
||||
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 # renovate: datasource=go
|
||||
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.3.0 # renovate: datasource=go
|
||||
|
||||
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
|
||||
|
||||
STATIC ?=
|
||||
EXTLDFLAGS ?=
|
||||
@@ -95,7 +97,7 @@ go-check:
|
||||
|
||||
.PHONY: fmt-check
|
||||
fmt-check: fmt
|
||||
@diff=$$(git diff --color=always); \
|
||||
@diff=$$(git diff --color=always -- '*.go'); \
|
||||
if [ -n "$$diff" ]; then \
|
||||
echo "Please run 'make fmt' and commit the result:"; \
|
||||
printf "%s" "$${diff}"; \
|
||||
@@ -110,6 +112,9 @@ deps-tools: ## install tool dependencies
|
||||
$(GO) install $(GOVULNCHECK_PACKAGE) & \
|
||||
wait
|
||||
|
||||
.PHONY: checks
|
||||
checks: tidy-check fmt-check security-check ## run the non-lint source checks
|
||||
|
||||
.PHONY: lint
|
||||
lint: lint-go lint-go-windows ## lint everything
|
||||
|
||||
@@ -131,7 +136,7 @@ lint-pr-title: ## lint PR title against Conventional Commits (set PR_TITLE=...)
|
||||
@node ./tools/lint-pr-title.ts
|
||||
|
||||
.PHONY: security-check
|
||||
security-check: deps-tools
|
||||
security-check:
|
||||
GOEXPERIMENT= $(GO) run $(GOVULNCHECK_PACKAGE) -show color ./... || true
|
||||
|
||||
.PHONY: tidy
|
||||
@@ -148,8 +153,8 @@ tidy-check: tidy
|
||||
fi
|
||||
|
||||
.PHONY: test
|
||||
test: fmt-check security-check ## test everything (integration tests self-skip without docker/network)
|
||||
@$(GO) test -race -timeout 20m -v -cover -coverprofile coverage.txt ./... && echo "\n==>\033[32m Ok\033[m\n" || exit 1
|
||||
test: ## test everything (integration tests self-skip without docker/network)
|
||||
@$(GO) test $(GOTEST_FLAGS) -cover -coverprofile coverage.txt ./... && echo "\n==>\033[32m Ok\033[m\n" || exit 1
|
||||
|
||||
.PHONY: coverage-report
|
||||
coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
|
||||
|
||||
46
README.md
46
README.md
@@ -209,6 +209,35 @@ Whenever the resulting labels differ from the ones in the registration file, the
|
||||
|
||||
> **Note:** A runner that only exposes `host` labels still needs access to a Docker daemon (e.g. a mounted `/var/run/docker.sock`) whenever a job uses a `docker://` action or a service container. `host` labels only change where the job's own steps run; container-based steps and actions are still executed with Docker.
|
||||
|
||||
#### Proxy
|
||||
|
||||
Set these variables in the runner's environment, with systemd `Environment=`, `docker run -e`, or Kubernetes `env:`:
|
||||
|
||||
```sh
|
||||
http_proxy=http://proxy.example:3128
|
||||
https_proxy=http://proxy.example:3128
|
||||
no_proxy=gitea.internal,.example.local
|
||||
```
|
||||
|
||||
The runner uses them for its own requests and gives them to every job, in lower and upper case.
|
||||
|
||||
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
|
||||
|
||||
- the cache server
|
||||
- `localhost`, `127.0.0.1` and `::1`
|
||||
- the job's service containers
|
||||
- the Docker daemon, when it is reached over `tcp://`
|
||||
|
||||
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
|
||||
|
||||
To change a value for one job, set it in a step's `env:` or in the job's `container.env`. Setting it at workflow or job level has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
|
||||
|
||||
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
|
||||
|
||||
Dockerfile actions are built with these variables as build arguments, so their `RUN` steps can reach the network.
|
||||
|
||||
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
|
||||
|
||||
#### Caching (`actions/cache`)
|
||||
|
||||
Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default.
|
||||
@@ -254,11 +283,12 @@ Besides `GITEA_INSTANCE_URL` and `GITEA_RUNNER_REGISTRATION_TOKEN`, the image en
|
||||
|
||||
For a fuller container-oriented walkthrough, see [examples/docker](examples/docker/README.md).
|
||||
|
||||
When `container.bind_workdir` is enabled, stale task workspace directories can be cleaned while the runner is idle:
|
||||
- directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable)
|
||||
While the runner is idle it cleans up after earlier jobs:
|
||||
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
|
||||
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
|
||||
- cleanup assumes `container.workdir_parent` is not shared across multiple runners
|
||||
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
|
||||
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
|
||||
|
||||
#### Post-task script (`runner.post_task_script`)
|
||||
|
||||
@@ -270,6 +300,16 @@ On Windows, use `.exe`, `.bat`, or `.cmd` paths; **PowerShell (`.ps1`) is not su
|
||||
|
||||
See **[docs/post-task-script.md](docs/post-task-script.md)** for lifecycle details, environment variables, timeout interaction, and platform notes.
|
||||
|
||||
#### Job hooks (`runner.hooks.job_started`, `runner.hooks.job_completed`)
|
||||
|
||||
Optional scripts that run **inside the job environment** (the job container, or the host in host mode), before the job's first step and after its last one. They are the equivalent of GitHub's `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`, which are read when the settings are unset.
|
||||
|
||||
Because they run where the steps run and see the job's environment, they are the place for per-job setup no workflow should have to carry: registry logins, mirror configuration, or masking runner-wide secrets with `::add-mask::`. Their output is part of the job log and is scanned for workflow commands, and they can export to the job through `$GITHUB_ENV` and `$GITHUB_PATH`.
|
||||
|
||||
Both hooks are synchronous and block the job while they run. Either one exiting non-zero fails the job, and there is no per-hook timeout.
|
||||
|
||||
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
|
||||
|
||||
### Example Deployments
|
||||
|
||||
Check out the [examples](examples) directory for sample deployment types.
|
||||
|
||||
@@ -70,6 +70,7 @@ type Handler struct {
|
||||
storage *Storage
|
||||
router *httprouter.Router
|
||||
listener net.Listener
|
||||
port int
|
||||
server *http.Server
|
||||
logger logrus.FieldLogger
|
||||
|
||||
@@ -177,6 +178,12 @@ func StartHandler(dir, outboundIP string, port uint16, internalSecret string, lo
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
addr, ok := listener.Addr().(*net.TCPAddr)
|
||||
if !ok {
|
||||
listener.Close()
|
||||
return nil, fmt.Errorf("cache server listens on %T, want a TCP address", listener.Addr())
|
||||
}
|
||||
h.port = addr.Port
|
||||
server := &http.Server{
|
||||
ReadHeaderTimeout: 2 * time.Second,
|
||||
Handler: router,
|
||||
@@ -194,9 +201,7 @@ func StartHandler(dir, outboundIP string, port uint16, internalSecret string, lo
|
||||
|
||||
func (h *Handler) ExternalURL() string {
|
||||
// TODO: make the external url configurable if necessary
|
||||
return fmt.Sprintf("http://%s:%d",
|
||||
h.outboundIP,
|
||||
h.listener.Addr().(*net.TCPAddr).Port)
|
||||
return fmt.Sprintf("http://%s:%d", h.outboundIP, h.port)
|
||||
}
|
||||
|
||||
// RegisterJob makes token a valid bearer credential for cache requests from
|
||||
|
||||
@@ -445,13 +445,6 @@ func TestHandler(t *testing.T) {
|
||||
require.Equal(t, 404, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("get with not exist id", func(t *testing.T) {
|
||||
resp, err := testClient.Get(signArtifactURL(handler, 100))
|
||||
require.NoError(t, err)
|
||||
defer resp.Body.Close()
|
||||
require.Equal(t, 404, resp.StatusCode)
|
||||
})
|
||||
|
||||
t.Run("get with multiple keys", func(t *testing.T) {
|
||||
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
|
||||
key := strings.ToLower(t.Name())
|
||||
@@ -469,7 +462,8 @@ func TestHandler(t *testing.T) {
|
||||
_, err := rand.Read(contents[i])
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
||||
time.Sleep(time.Second) // ensure CreatedAt of caches are different
|
||||
// ensure CreatedAt of caches are different, in upload order
|
||||
backdateCache(t, handler, keys[i], time.Duration(len(contents)-i)*time.Second)
|
||||
}
|
||||
|
||||
reqKeys := strings.Join([]string{
|
||||
@@ -554,7 +548,8 @@ func TestHandler(t *testing.T) {
|
||||
_, err := rand.Read(contents[i])
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
||||
time.Sleep(time.Second) // ensure CreatedAt of caches are different
|
||||
// ensure CreatedAt of caches are different, in upload order
|
||||
backdateCache(t, handler, keys[i], time.Duration(len(contents)-i)*time.Second)
|
||||
}
|
||||
|
||||
reqKeys := strings.Join([]string{
|
||||
@@ -607,7 +602,8 @@ func TestHandler(t *testing.T) {
|
||||
_, err := rand.Read(contents[i])
|
||||
require.NoError(t, err)
|
||||
uploadCacheNormally(t, base, keys[i], version, contents[i])
|
||||
time.Sleep(time.Second) // ensure CreatedAt of caches are different
|
||||
// ensure CreatedAt of caches are different, in upload order
|
||||
backdateCache(t, handler, keys[i], time.Duration(len(contents)-i)*time.Second)
|
||||
}
|
||||
|
||||
reqKeys := strings.Join([]string{
|
||||
@@ -646,6 +642,20 @@ func TestHandler(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// backdateCache rewrites a cache's CreatedAt. It has one-second resolution, so age-ordering
|
||||
// tests set it directly instead of sleeping a second between uploads.
|
||||
func backdateCache(t *testing.T, handler *Handler, key string, age time.Duration) {
|
||||
db, err := handler.openDB()
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
var caches []*Cache
|
||||
require.NoError(t, db.Find(&caches, bolthold.Where("Key").Eq(key)))
|
||||
require.Len(t, caches, 1)
|
||||
caches[0].CreatedAt = time.Now().Add(-age).Unix()
|
||||
require.NoError(t, db.Update(caches[0].ID, caches[0]))
|
||||
}
|
||||
|
||||
func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { //nolint:unparam // pre-existing issue from nektos/act
|
||||
var id uint64
|
||||
{
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// Simple fast test that verifies max-parallel: 2 limits concurrency
|
||||
func TestMaxParallel2Quick(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
var currentRunning atomic.Int32
|
||||
var maxSimultaneous atomic.Int32
|
||||
|
||||
executors := make([]Executor, 4)
|
||||
for i := range 4 {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
current := currentRunning.Add(1)
|
||||
|
||||
// Update max if needed
|
||||
for {
|
||||
maxValue := maxSimultaneous.Load()
|
||||
if current <= maxValue || maxSimultaneous.CompareAndSwap(maxValue, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
currentRunning.Add(-1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
err := NewParallelExecutor(2, executors...)(ctx)
|
||||
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.LessOrEqual(t, maxSimultaneous.Load(), int32(2),
|
||||
"Should not exceed max-parallel: 2")
|
||||
}
|
||||
|
||||
// Test that verifies max-parallel: 1 enforces sequential execution
|
||||
func TestMaxParallel1Sequential(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
var currentRunning atomic.Int32
|
||||
var maxSimultaneous atomic.Int32
|
||||
var executionOrder []int
|
||||
var orderMutex sync.Mutex
|
||||
|
||||
executors := make([]Executor, 5)
|
||||
for i := range 5 {
|
||||
taskID := i
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
current := currentRunning.Add(1)
|
||||
|
||||
// Track execution order
|
||||
orderMutex.Lock()
|
||||
executionOrder = append(executionOrder, taskID)
|
||||
orderMutex.Unlock()
|
||||
|
||||
// Update max if needed
|
||||
for {
|
||||
maxValue := maxSimultaneous.Load()
|
||||
if current <= maxValue || maxSimultaneous.CompareAndSwap(maxValue, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
currentRunning.Add(-1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
err := NewParallelExecutor(1, executors...)(ctx)
|
||||
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, int32(1), maxSimultaneous.Load(),
|
||||
"max-parallel: 1 should only run 1 task at a time")
|
||||
assert.Len(t, executionOrder, 5, "All 5 tasks should have executed")
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestMaxParallelJobExecution tests actual job execution with max-parallel
|
||||
func TestMaxParallelJobExecution(t *testing.T) {
|
||||
t.Run("MaxParallel=1 Sequential", func(t *testing.T) {
|
||||
var currentRunning atomic.Int32
|
||||
var maxConcurrent int32
|
||||
var executionOrder []int
|
||||
var mu sync.Mutex
|
||||
|
||||
executors := make([]Executor, 5)
|
||||
for i := range 5 {
|
||||
taskID := i
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
current := currentRunning.Add(1)
|
||||
|
||||
// Track max concurrent
|
||||
for {
|
||||
maxValue := atomic.LoadInt32(&maxConcurrent)
|
||||
if current <= maxValue || atomic.CompareAndSwapInt32(&maxConcurrent, maxValue, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
executionOrder = append(executionOrder, taskID)
|
||||
mu.Unlock()
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
currentRunning.Add(-1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := NewParallelExecutor(1, executors...)(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
assert.Equal(t, int32(1), maxConcurrent, "Should never exceed 1 concurrent execution")
|
||||
assert.Len(t, executionOrder, 5, "All tasks should execute")
|
||||
})
|
||||
|
||||
t.Run("MaxParallel=3 Limited", func(t *testing.T) {
|
||||
var currentRunning atomic.Int32
|
||||
var maxConcurrent int32
|
||||
|
||||
executors := make([]Executor, 10)
|
||||
for i := range 10 {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
current := currentRunning.Add(1)
|
||||
|
||||
for {
|
||||
maxValue := atomic.LoadInt32(&maxConcurrent)
|
||||
if current <= maxValue || atomic.CompareAndSwapInt32(&maxConcurrent, maxValue, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
currentRunning.Add(-1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := NewParallelExecutor(3, executors...)(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
assert.LessOrEqual(t, int(maxConcurrent), 3, "Should never exceed 3 concurrent executions")
|
||||
assert.GreaterOrEqual(t, int(maxConcurrent), 1, "Should have at least 1 concurrent execution")
|
||||
})
|
||||
|
||||
t.Run("MaxParallel=0 Uses1Worker", func(t *testing.T) {
|
||||
var maxConcurrent int32
|
||||
var currentRunning atomic.Int32
|
||||
|
||||
executors := make([]Executor, 5)
|
||||
for i := range 5 {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
current := currentRunning.Add(1)
|
||||
|
||||
for {
|
||||
maxValue := atomic.LoadInt32(&maxConcurrent)
|
||||
if current <= maxValue || atomic.CompareAndSwapInt32(&maxConcurrent, maxValue, current) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
currentRunning.Add(-1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// When maxParallel is 0 or negative, it defaults to 1
|
||||
err := NewParallelExecutor(0, executors...)(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
assert.Equal(t, int32(1), maxConcurrent, "Should use 1 worker when max-parallel is 0")
|
||||
})
|
||||
}
|
||||
|
||||
// TestMaxParallelWithErrors tests error handling with max-parallel
|
||||
func TestMaxParallelWithErrors(t *testing.T) {
|
||||
t.Run("OneTaskFailsOthersContinue", func(t *testing.T) {
|
||||
var successCount int32
|
||||
|
||||
executors := make([]Executor, 5)
|
||||
for i := range 5 {
|
||||
taskID := i
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
if taskID == 2 {
|
||||
return assert.AnError
|
||||
}
|
||||
atomic.AddInt32(&successCount, 1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := NewParallelExecutor(2, executors...)(ctx)
|
||||
|
||||
// Should return the error from task 2
|
||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
// Other tasks should still execute
|
||||
assert.Equal(t, int32(4), successCount, "4 tasks should succeed")
|
||||
})
|
||||
|
||||
t.Run("ContextCancellation", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
var startedCount int32
|
||||
executors := make([]Executor, 10)
|
||||
for i := range 10 {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
atomic.AddInt32(&startedCount, 1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Cancel after a short delay
|
||||
go func() {
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
cancel()
|
||||
}()
|
||||
|
||||
err := NewParallelExecutor(3, executors...)(ctx)
|
||||
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.ErrorIs(t, err, context.Canceled) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
// Not all tasks should start due to cancellation (but timing may vary)
|
||||
// Just verify cancellation occurred
|
||||
t.Logf("Started %d tasks before cancellation", startedCount)
|
||||
})
|
||||
}
|
||||
|
||||
// TestMaxParallelResourceSharing tests resource sharing scenarios
|
||||
func TestMaxParallelResourceSharing(t *testing.T) {
|
||||
t.Run("SharedResourceWithMutex", func(t *testing.T) {
|
||||
var sharedCounter int
|
||||
var mu sync.Mutex
|
||||
|
||||
executors := make([]Executor, 100)
|
||||
for i := range 100 {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
mu.Lock()
|
||||
sharedCounter++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := NewParallelExecutor(10, executors...)(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
assert.Equal(t, 100, sharedCounter, "All tasks should increment counter")
|
||||
})
|
||||
|
||||
t.Run("ChannelCommunication", func(t *testing.T) {
|
||||
resultChan := make(chan int, 50)
|
||||
|
||||
executors := make([]Executor, 50)
|
||||
for i := range 50 {
|
||||
taskID := i
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
resultChan <- taskID
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := NewParallelExecutor(5, executors...)(ctx)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
close(resultChan)
|
||||
|
||||
results := make(map[int]bool)
|
||||
for result := range resultChan {
|
||||
results[result] = true
|
||||
}
|
||||
|
||||
assert.Len(t, results, 50, "All task IDs should be received")
|
||||
})
|
||||
}
|
||||
@@ -9,9 +9,9 @@ import (
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -82,44 +82,45 @@ func TestNewConditionalExecutor(t *testing.T) {
|
||||
assert.Equal(1, falseCount)
|
||||
}
|
||||
|
||||
func TestNewParallelExecutor(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies
|
||||
// block until wantActive are in flight so the peak is exact without sleeping, and later copies
|
||||
// find the gate already open so the last one still finishes with no partner left.
|
||||
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
|
||||
var counted, active, peak atomic.Int32
|
||||
var once sync.Once
|
||||
reached := make(chan struct{})
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
var count, activeCount, maxCount atomic.Int32
|
||||
emptyWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
|
||||
count.Add(1)
|
||||
|
||||
active := activeCount.Add(1)
|
||||
return func(ctx context.Context) error {
|
||||
counted.Add(1)
|
||||
running := active.Add(1)
|
||||
for {
|
||||
m := maxCount.Load()
|
||||
if active <= m || maxCount.CompareAndSwap(m, active) {
|
||||
seen := peak.Load()
|
||||
if running <= seen || peak.CompareAndSwap(seen, running) {
|
||||
break
|
||||
}
|
||||
}
|
||||
time.Sleep(2 * time.Second)
|
||||
activeCount.Add(-1)
|
||||
|
||||
if running >= wantActive {
|
||||
once.Do(func() { close(reached) })
|
||||
}
|
||||
<-reached
|
||||
active.Add(-1)
|
||||
return nil
|
||||
})
|
||||
}, &counted, &peak
|
||||
}
|
||||
|
||||
err := NewParallelExecutor(2, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx)
|
||||
func TestNewParallelExecutor(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
assert.Equal(int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(int32(2), maxCount.Load(), "should run at most 2 executors in parallel")
|
||||
assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
exec, count, maxActive := concurrencyProbe(2)
|
||||
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx))
|
||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
|
||||
|
||||
// Reset to test running the executor with 0 parallelism
|
||||
count.Store(0)
|
||||
activeCount.Store(0)
|
||||
maxCount.Store(0)
|
||||
|
||||
errSingle := NewParallelExecutor(0, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx)
|
||||
|
||||
assert.Equal(int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(int32(1), maxCount.Load(), "should run at most 1 executors in parallel")
|
||||
assert.NoError(errSingle)
|
||||
// parallelism below 1 falls back to a single worker
|
||||
exec, count, maxActive = concurrencyProbe(1)
|
||||
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx))
|
||||
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors")
|
||||
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorEmpty(t *testing.T) {
|
||||
@@ -173,6 +174,23 @@ func TestNewParallelExecutorCanceled(t *testing.T) {
|
||||
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
}
|
||||
|
||||
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
|
||||
var successCount atomic.Int32
|
||||
executors := make([]Executor, 5)
|
||||
for i := range executors {
|
||||
executors[i] = func(ctx context.Context) error {
|
||||
if i == 2 {
|
||||
return errors.New("fake error")
|
||||
}
|
||||
successCount.Add(1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
|
||||
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
|
||||
}
|
||||
|
||||
func TestExecutorConditionalsAndFinally(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
var calls []string
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
@@ -32,7 +33,7 @@ var (
|
||||
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
|
||||
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
|
||||
|
||||
cloneLocks sync.Map // key: clone target directory; value: *sync.Mutex
|
||||
cloneLocks lock.Keyed[string] // key: clone target directory
|
||||
|
||||
ErrShortRef = errors.New("short SHA references are not supported")
|
||||
ErrNoRepo = errors.New("unable to find git repo")
|
||||
@@ -43,10 +44,7 @@ var (
|
||||
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
|
||||
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
|
||||
func AcquireCloneLock(dir string) func() {
|
||||
v, _ := cloneLocks.LoadOrStore(dir, &sync.Mutex{})
|
||||
mu := v.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
return cloneLocks.Lock(dir)
|
||||
}
|
||||
|
||||
type Error struct {
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -610,12 +609,4 @@ func TestAcquireCloneLock(t *testing.T) {
|
||||
t.Fatal("acquire on a different directory must not block")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("same directory reuses the same mutex", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
|
||||
v1, _ := cloneLocks.LoadOrStore(dir, &sync.Mutex{})
|
||||
v2, _ := cloneLocks.LoadOrStore(dir, &sync.Mutex{})
|
||||
require.Same(t, v1, v2)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -19,7 +19,9 @@ func GetOutboundIP() net.IP {
|
||||
conn, err := net.Dial("udp", "8.8.8.8:80")
|
||||
if err == nil {
|
||||
defer conn.Close()
|
||||
return conn.LocalAddr().(*net.UDPAddr).IP
|
||||
if addr, ok := conn.LocalAddr().(*net.UDPAddr); ok {
|
||||
return addr.IP
|
||||
}
|
||||
}
|
||||
|
||||
// So the machine cannot access the internet. Pick an IP address from network interfaces.
|
||||
|
||||
@@ -82,12 +82,14 @@ type NewDockerBuildExecutorInput struct {
|
||||
BuildContext io.Reader
|
||||
ImageTag string
|
||||
Platform string
|
||||
BuildArgs map[string]*string
|
||||
}
|
||||
|
||||
// NewDockerNetworkCreateExecutorInput the input for the NewDockerNetworkCreateExecutor function
|
||||
type NewDockerNetworkCreateExecutorInput struct {
|
||||
EnableIPv4 *bool
|
||||
EnableIPv6 *bool
|
||||
RunnerUUID string
|
||||
}
|
||||
|
||||
// NewDockerPullExecutorInput the input for the NewDockerPullExecutor function
|
||||
|
||||
@@ -49,6 +49,7 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
|
||||
Remove: true,
|
||||
AuthConfigs: LoadDockerAuthConfigs(ctx),
|
||||
Dockerfile: input.Dockerfile,
|
||||
BuildArgs: input.BuildArgs,
|
||||
}
|
||||
platform, err := parsePlatform(input.Platform)
|
||||
if err != nil {
|
||||
|
||||
@@ -51,7 +51,8 @@ func TestCreateFlagsValidate(t *testing.T) {
|
||||
|
||||
func TestNewContainerAppliesCreateFlags(t *testing.T) {
|
||||
input := &NewContainerInput{Platform: "linux/amd64", Options: "--platform linux/arm64 --pull never"}
|
||||
cr := NewContainer(input).(*containerReference)
|
||||
cr, ok := NewContainer(input).(*containerReference)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "linux/arm64", input.Platform)
|
||||
assert.Equal(t, pullPolicyNever, cr.pullPolicy)
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser,
|
||||
|
||||
if msg.ErrorDetail.Message != "" {
|
||||
writeLog(logger, isError, "%s", msg.ErrorDetail.Message)
|
||||
return errors.New(msg.Error)
|
||||
return errors.New(msg.ErrorDetail.Message)
|
||||
}
|
||||
|
||||
if msg.Status != "" {
|
||||
|
||||
@@ -8,12 +8,69 @@ package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
"github.com/moby/moby/client"
|
||||
)
|
||||
|
||||
const (
|
||||
networkCreateAttempts = 3
|
||||
networkCreateRetryDelay = time.Second
|
||||
|
||||
// marks the networks a runner creates for its jobs, so it can tell its own leftovers from
|
||||
// those of another runner sharing the daemon
|
||||
runnerUUIDLabel = "com.gitea.runner.uuid"
|
||||
)
|
||||
|
||||
// RemoveOrphanNetworks removes the networks this runner created for jobs whose teardown did
|
||||
// not get to them: the runner died with the job, the teardown timed out, or the network still
|
||||
// had an endpoint on it at the time. Each one holds a subnet of the daemon's address pool
|
||||
// until it is removed. Networks created after createdBefore are left alone, so a job starting
|
||||
// while this runs cannot lose the network it has created but not yet attached a container to.
|
||||
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to the docker daemon: %w", err)
|
||||
}
|
||||
defer cli.Close()
|
||||
|
||||
return removeOrphanNetworks(ctx, cli, runnerUUID, createdBefore)
|
||||
}
|
||||
|
||||
func removeOrphanNetworks(ctx context.Context, cli client.APIClient, runnerUUID string, createdBefore time.Time) error {
|
||||
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{
|
||||
Filters: make(client.Filters).Add("label", runnerUUIDLabel+"="+runnerUUID),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var errs []error
|
||||
for _, n := range networks.Items {
|
||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to inspect network %s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
// the emptiness check, not the label, is what keeps a live job of another process
|
||||
// sharing this registration safe
|
||||
if len(result.Network.Containers) != 0 || result.Network.Created.After(createdBefore) {
|
||||
continue
|
||||
}
|
||||
if _, err := cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", n.Name, err))
|
||||
continue
|
||||
}
|
||||
common.Logger(ctx).Infof("removed docker network %s left behind by an earlier job", n.Name)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
cli, err := GetDockerClient(ctx)
|
||||
@@ -36,18 +93,45 @@ func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExec
|
||||
}
|
||||
}
|
||||
|
||||
_, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
for i := range networkCreateAttempts {
|
||||
if i > 0 {
|
||||
common.Logger(ctx).Infof("Waiting for a free docker address pool to create network %s", name)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(time.Duration(i) * networkCreateRetryDelay):
|
||||
}
|
||||
}
|
||||
if _, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
|
||||
Driver: "bridge",
|
||||
Scope: "local",
|
||||
EnableIPv4: opts.EnableIPv4,
|
||||
EnableIPv6: opts.EnableIPv6,
|
||||
Labels: runnerLabels(opts.RunnerUUID),
|
||||
}); err == nil {
|
||||
return nil
|
||||
}
|
||||
if !isAddressPoolExhausted(err) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("docker has no address pool left for this job's network, lower runner.capacity or widen default-address-pools in the docker daemon config: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runnerLabels(runnerUUID string) map[string]string {
|
||||
if runnerUUID == "" {
|
||||
return nil
|
||||
}
|
||||
return map[string]string{runnerUUIDLabel: runnerUUID}
|
||||
}
|
||||
|
||||
// The daemon reports this as a plain invalid-parameter error, the same kind it uses for every
|
||||
// malformed request, so the message is the only discriminator.
|
||||
func isAddressPoolExhausted(err error) bool {
|
||||
msg := err.Error()
|
||||
return strings.Contains(msg, "all predefined address pools have been fully subnetted") ||
|
||||
strings.Contains(msg, "could not find an available, non-overlapping IPv4 address pool among the defaults") // docker 24 and older
|
||||
}
|
||||
|
||||
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
@@ -66,6 +150,7 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
}
|
||||
// For Gitea, reduce log noise
|
||||
// common.Logger(ctx).Debugf("%v", networks)
|
||||
var errs []error
|
||||
for _, n := range networks.Items {
|
||||
if n.Name == name {
|
||||
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
|
||||
@@ -73,16 +158,17 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(result.Network.Containers) == 0 {
|
||||
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
common.Logger(ctx).Debugf("%v", err)
|
||||
}
|
||||
} else {
|
||||
common.Logger(ctx).Debugf("Refusing to remove network %v because it still has active endpoints", name)
|
||||
// it holds a subnet out of the daemon's pool until something reclaims it
|
||||
if len(result.Network.Containers) != 0 {
|
||||
common.Logger(ctx).Warnf("Refusing to remove network %s because it still has active endpoints, the idle cleanup reclaims it once they are gone", name)
|
||||
continue
|
||||
}
|
||||
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to remove network %s: %w", name, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
}
|
||||
|
||||
50
act/container/docker_network_test.go
Normal file
50
act/container/docker_network_test.go
Normal file
@@ -0,0 +1,50 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/api/types/network"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestIsAddressPoolExhausted(t *testing.T) {
|
||||
assert.True(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("Error response from daemon: all predefined address pools have been fully subnetted")))
|
||||
assert.True(t, isAddressPoolExhausted(errors.New("could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network")))
|
||||
assert.False(t, isAddressPoolExhausted(cerrdefs.ErrInvalidArgument.WithMessage("invalid subnet 10.0.0.0/8: it overlaps with an existing network")))
|
||||
}
|
||||
|
||||
// Of this runner's networks, only the ones nothing is attached to and old enough to predate
|
||||
// any job now starting are the runner's to reclaim. An unexpected NetworkRemove fails the
|
||||
// test on its own, since testify has no expectation to match it against.
|
||||
func TestRemoveOrphanNetworks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cutoff := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
client := &mockDockerClient{}
|
||||
client.On("NetworkList", ctx, mobyclient.NetworkListOptions{
|
||||
Filters: make(mobyclient.Filters).Add("label", runnerUUIDLabel+"=runner-1"),
|
||||
}).Return(mobyclient.NetworkListResult{Items: []network.Summary{
|
||||
{Network: network.Network{ID: "orphan"}},
|
||||
{Network: network.Network{ID: "busy"}},
|
||||
{Network: network.Network{ID: "starting"}},
|
||||
}}, nil)
|
||||
client.On("NetworkInspect", ctx, "orphan", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{}, nil)
|
||||
client.On("NetworkInspect", ctx, "busy", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Containers: map[string]network.EndpointResource{"c": {}}}}, nil)
|
||||
client.On("NetworkInspect", ctx, "starting", mobyclient.NetworkInspectOptions{}).
|
||||
Return(mobyclient.NetworkInspectResult{Network: network.Inspect{Network: network.Network{Created: cutoff.Add(time.Second)}}}, nil)
|
||||
client.On("NetworkRemove", ctx, "orphan", mobyclient.NetworkRemoveOptions{}).
|
||||
Return(mobyclient.NetworkRemoveResult{}, nil)
|
||||
|
||||
require.NoError(t, removeOrphanNetworks(ctx, client, "runner-1", cutoff))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
@@ -30,23 +30,19 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return nil
|
||||
}
|
||||
|
||||
pull := input.ForcePull
|
||||
if !pull {
|
||||
// skip the pull when the image is already here: either none was forced, or a digest
|
||||
// pins the content so a forced pull could only fetch the same bytes again
|
||||
if !input.ForcePull || isPinnedImage(input.Image) {
|
||||
imageExists, err := ImageExistsLocally(ctx, input.Image, input.Platform)
|
||||
logger.Debugf("Image exists? %v", imageExists)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to determine if image already exists for image '%s' (%s): %w", input.Image, input.Platform, err)
|
||||
}
|
||||
|
||||
if !imageExists {
|
||||
pull = true
|
||||
if imageExists {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if !pull {
|
||||
return nil
|
||||
}
|
||||
|
||||
imageRef := cleanImage(ctx, input.Image)
|
||||
logger.Debugf("pulling image '%v' (%s)", imageRef, input.Platform)
|
||||
|
||||
@@ -61,22 +57,32 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
|
||||
return err
|
||||
}
|
||||
|
||||
reader, err := cli.ImagePull(ctx, imageRef, imagePullOptions)
|
||||
|
||||
_ = logDockerResponse(logger, reader, err != nil)
|
||||
if err != nil {
|
||||
if imagePullOptions.RegistryAuth != "" && strings.Contains(err.Error(), "unauthorized") {
|
||||
logger.Errorf("pulling image '%v' (%s) failed with credentials %s retrying without them, please check for stale docker config files", imageRef, input.Platform, err.Error())
|
||||
imagePullOptions.RegistryAuth = ""
|
||||
reader, err = cli.ImagePull(ctx, imageRef, imagePullOptions)
|
||||
|
||||
_ = logDockerResponse(logger, reader, err != nil)
|
||||
}
|
||||
// the daemon reports a failure that happens after the first progress line in the
|
||||
// stream rather than on the call itself, so both have to be checked
|
||||
pullOnce := func(opts client.ImagePullOptions) error {
|
||||
reader, err := cli.ImagePull(ctx, imageRef, opts)
|
||||
streamErr := logDockerResponse(logger, reader, err != nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
||||
return err
|
||||
}
|
||||
return streamErr
|
||||
}
|
||||
return nil
|
||||
|
||||
err = pullOnce(imagePullOptions)
|
||||
if err != nil && imagePullOptions.RegistryAuth != "" && strings.Contains(err.Error(), "unauthorized") {
|
||||
logger.Errorf("pulling image '%v' (%s) failed with credentials %s retrying without them, please check for stale docker config files", imageRef, input.Platform, err.Error())
|
||||
imagePullOptions.RegistryAuth = ""
|
||||
err = pullOnce(imagePullOptions)
|
||||
}
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
// a registry that is down should not fail a job whose image is already here
|
||||
if exists, existsErr := ImageExistsLocally(ctx, input.Image, input.Platform); existsErr == nil && exists {
|
||||
logger.Warnf("could not update image '%s' (%s), continuing with the local copy: %v", imageRef, input.Platform, err)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,6 +128,15 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput)
|
||||
return imagePullOptions, nil
|
||||
}
|
||||
|
||||
func isPinnedImage(image string) bool {
|
||||
ref, err := reference.ParseAnyReference(image)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
_, pinned := ref.(reference.Canonical)
|
||||
return pinned
|
||||
}
|
||||
|
||||
func cleanImage(ctx context.Context, imageName string) string {
|
||||
ref, err := reference.ParseAnyReference(imageName)
|
||||
if err != nil {
|
||||
|
||||
@@ -6,11 +6,15 @@ package container
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/docker/cli/cli/config"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
assert "github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -65,3 +69,21 @@ func TestGetImagePullOptions(t *testing.T) {
|
||||
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
|
||||
}
|
||||
|
||||
// A digest-pinned image is immutable, so its local copy is always current.
|
||||
func TestIsPinnedImage(t *testing.T) {
|
||||
assert.True(t, isPinnedImage("alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"))
|
||||
assert.False(t, isPinnedImage("alpine:latest"))
|
||||
}
|
||||
|
||||
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
|
||||
// whichever of the two shapes the daemon used.
|
||||
func TestLogDockerResponseError(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
streamErr := func(line string) error {
|
||||
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
|
||||
}
|
||||
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
|
||||
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
|
||||
require.NoError(t, streamErr(`{"status":"Downloading"}`))
|
||||
}
|
||||
|
||||
@@ -386,32 +386,71 @@ func (cr *containerReference) find() common.Executor {
|
||||
}
|
||||
}
|
||||
|
||||
// isContainerGone reports whether a failed remove still left the container gone (NotFound or Conflict).
|
||||
func isContainerGone(err error) bool {
|
||||
return cerrdefs.IsNotFound(err) || cerrdefs.IsConflict(err)
|
||||
}
|
||||
|
||||
func (cr *containerReference) remove() common.Executor {
|
||||
return func(ctx context.Context) error {
|
||||
if cr.id == "" {
|
||||
idOrName := cr.id
|
||||
if idOrName == "" && cr.input != nil {
|
||||
idOrName = cr.input.Name
|
||||
}
|
||||
if idOrName == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
logger := common.Logger(ctx)
|
||||
_, err := cr.cli.ContainerRemove(ctx, cr.id, client.ContainerRemoveOptions{
|
||||
// Kill first so removal never waits out a daemon's stop timeout: Docker kills outright
|
||||
// on a forced remove, Podman sends SIGTERM and waits. Only worth it for a container
|
||||
// this started, and removal can still deal with one it could not kill.
|
||||
if cr.id != "" {
|
||||
_, err := cr.cli.ContainerKill(ctx, cr.id, client.ContainerKillOptions{Signal: "SIGKILL"})
|
||||
if err != nil && !cerrdefs.IsConflict(err) && !cerrdefs.IsNotFound(err) {
|
||||
logger.Debugf("Container %s could not be killed: %v", cr.id, err)
|
||||
}
|
||||
}
|
||||
_, err := cr.cli.ContainerRemove(ctx, idOrName, client.ContainerRemoveOptions{
|
||||
RemoveVolumes: true,
|
||||
Force: true,
|
||||
})
|
||||
if err != nil && !isContainerGone(err) {
|
||||
logger.Error(fmt.Errorf("failed to remove container: %w", err))
|
||||
switch {
|
||||
case cerrdefs.IsConflict(err):
|
||||
// the daemon's own AutoRemove teardown is running, and it releases the volume
|
||||
// references and the network endpoint only once it finishes
|
||||
cr.waitForRemoval(ctx, idOrName)
|
||||
case err != nil && !cerrdefs.IsNotFound(err):
|
||||
logger.Error(fmt.Errorf("failed to remove container %s: %w", idOrName, err))
|
||||
return nil // keep the id, the container is still there for a later Remove()
|
||||
}
|
||||
|
||||
logger.Debugf("Removed container: %v", cr.id)
|
||||
logger.Debugf("Removed container: %v", idOrName)
|
||||
cr.id = ""
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName string) {
|
||||
// per container, against the one minute the post-job executor allows for the whole
|
||||
// cleanup, so a job with several services can spend most of that budget here
|
||||
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
waitResult := cr.cli.ContainerWait(ctx, idOrName, client.ContainerWaitOptions{
|
||||
Condition: container.WaitConditionRemoved,
|
||||
})
|
||||
select {
|
||||
case <-waitResult.Result:
|
||||
case <-waitResult.Error:
|
||||
case <-ctx.Done():
|
||||
// the client delivers the result over an unbuffered channel, so leave a receiver
|
||||
// behind or its goroutine parks on the send for the lifetime of the process
|
||||
go func() {
|
||||
select {
|
||||
case <-waitResult.Result:
|
||||
case <-waitResult.Error:
|
||||
}
|
||||
}()
|
||||
common.Logger(ctx).Warnf("Timed out waiting for the daemon to remove container %s, its volumes and network may be left behind", idOrName)
|
||||
}
|
||||
}
|
||||
|
||||
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) {
|
||||
logger := common.Logger(ctx)
|
||||
input := cr.input
|
||||
@@ -471,8 +510,7 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
|
||||
|
||||
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
|
||||
|
||||
hostConfig.Binds = append(hostConfig.Binds, containerConfig.HostConfig.Binds...)
|
||||
hostConfig.Mounts = append(hostConfig.Mounts, containerConfig.HostConfig.Mounts...)
|
||||
overlayVolumes(hostConfig, containerConfig.HostConfig)
|
||||
binds := hostConfig.Binds
|
||||
mounts := hostConfig.Mounts
|
||||
networkMode := hostConfig.NetworkMode
|
||||
@@ -1108,6 +1146,34 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
|
||||
return config, hostConfig
|
||||
}
|
||||
|
||||
// bindTarget returns the container path a bind mounts onto, empty if it cannot be parsed.
|
||||
func bindTarget(bind string) string {
|
||||
parsed, err := loader.ParseVolume(bind)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return parsed.Target
|
||||
}
|
||||
|
||||
// overlayVolumes appends src's volumes to dst, dropping the dst ones they mount over. Docker
|
||||
// rejects two mounts on one target, so the volumes declared last have to win.
|
||||
func overlayVolumes(dst, src *container.HostConfig) {
|
||||
claimed := map[string]bool{}
|
||||
for _, bind := range src.Binds {
|
||||
if target := bindTarget(bind); target != "" {
|
||||
claimed[target] = true
|
||||
}
|
||||
}
|
||||
for _, mt := range src.Mounts {
|
||||
claimed[mt.Target] = true
|
||||
}
|
||||
|
||||
dst.Binds = append(slices.DeleteFunc(slices.Clone(dst.Binds),
|
||||
func(bind string) bool { return claimed[bindTarget(bind)] }), src.Binds...)
|
||||
dst.Mounts = append(slices.DeleteFunc(slices.Clone(dst.Mounts),
|
||||
func(mt mount.Mount) bool { return claimed[mt.Target] }), src.Mounts...)
|
||||
}
|
||||
|
||||
type validVolumeMatcher struct {
|
||||
allowAll bool
|
||||
named []glob.Glob
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
cerrdefs "github.com/containerd/errdefs"
|
||||
"github.com/moby/moby/api/pkg/stdcopy"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
mobyclient "github.com/moby/moby/client"
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -121,6 +122,26 @@ func (m *mockDockerClient) ContainerRemove(ctx context.Context, id string, opts
|
||||
return args.Get(0).(mobyclient.ContainerRemoveResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) ContainerKill(ctx context.Context, id string, opts mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.ContainerKillResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.NetworkListOptions) (mobyclient.NetworkListResult, error) {
|
||||
args := m.Called(ctx, opts)
|
||||
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
|
||||
}
|
||||
|
||||
func (m *mockDockerClient) NetworkRemove(ctx context.Context, id string, opts mobyclient.NetworkRemoveOptions) (mobyclient.NetworkRemoveResult, error) {
|
||||
args := m.Called(ctx, id, opts)
|
||||
return args.Get(0).(mobyclient.NetworkRemoveResult), args.Error(1)
|
||||
}
|
||||
|
||||
type endlessReader struct {
|
||||
io.Reader
|
||||
}
|
||||
@@ -390,29 +411,39 @@ func TestDockerCopyTarStreamErrorInMkdir(t *testing.T) {
|
||||
// be logged as one.
|
||||
func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
||||
removeOpts := mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}
|
||||
killOpts := mobyclient.ContainerKillOptions{Signal: "SIGKILL"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
err error
|
||||
wantLogs bool
|
||||
name string
|
||||
err error
|
||||
wantWait bool
|
||||
wantFailure bool
|
||||
}{
|
||||
{name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress")},
|
||||
{name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress"), wantWait: true},
|
||||
{name: "already removed", err: cerrdefs.ErrNotFound.WithMessage("No such container: abc")},
|
||||
{name: "removed cleanly", err: nil},
|
||||
{name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantLogs: true},
|
||||
{name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantFailure: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
logger, hook := test.NewNullLogger()
|
||||
ctx := common.WithLogger(context.Background(), logger)
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
|
||||
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
|
||||
if tc.wantWait {
|
||||
removed := make(chan container.WaitResponse, 1)
|
||||
removed <- container.WaitResponse{}
|
||||
client.On("ContainerWait", mock.Anything, "abc", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionRemoved}).
|
||||
Return(mobyclient.ContainerWaitResult{Result: removed})
|
||||
}
|
||||
cr := &containerReference{id: "abc", cli: client}
|
||||
|
||||
require.NoError(t, cr.remove()(ctx))
|
||||
assert.Empty(t, cr.id)
|
||||
|
||||
if tc.wantLogs {
|
||||
// a failure keeps the id, so a later Remove() can retry it
|
||||
if tc.wantFailure {
|
||||
assert.Equal(t, "abc", cr.id)
|
||||
assert.Len(t, hook.AllEntries(), 1)
|
||||
} else {
|
||||
assert.Empty(t, cr.id)
|
||||
assert.Empty(t, hook.AllEntries())
|
||||
}
|
||||
client.AssertExpectations(t)
|
||||
@@ -420,6 +451,20 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A container whose id was never learned, because find() could not reach the daemon or
|
||||
// create() lost its reply, must still be removed rather than leaking with its network. It
|
||||
// was never started here, so it is not worth a kill of its own.
|
||||
func TestRemoveWithoutIDUsesName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
client := &mockDockerClient{}
|
||||
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
|
||||
Return(mobyclient.ContainerRemoveResult{}, nil)
|
||||
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
|
||||
|
||||
require.NoError(t, cr.remove()(ctx))
|
||||
client.AssertExpectations(t)
|
||||
}
|
||||
|
||||
// find() must drop a stale cached id so later Copy/Exec don't hit the
|
||||
// daemon with a torn-down container.
|
||||
func TestFindRevalidatesStaleID(t *testing.T) {
|
||||
@@ -660,3 +705,22 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
|
||||
})
|
||||
assert.Empty(t, hostConf.Binds)
|
||||
}
|
||||
|
||||
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
|
||||
logger, _ := test.NewNullLogger()
|
||||
ctx := common.WithLogger(context.Background(), logger)
|
||||
cr := &containerReference{
|
||||
input: &NewContainerInput{
|
||||
NetworkMode: "bridge",
|
||||
Options: "--volume /host/tools:/opt/hostedtoolcache",
|
||||
},
|
||||
}
|
||||
|
||||
_, hostConf, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
|
||||
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
|
||||
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
|
||||
assert.Empty(t, hostConf.Mounts)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ package container
|
||||
import (
|
||||
"context"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
|
||||
@@ -72,3 +73,7 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -330,6 +330,10 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
|
||||
} else {
|
||||
wd = e.Path
|
||||
}
|
||||
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
|
||||
// does in waitForCommand, so the final line of a command's output is not lost.
|
||||
defer common.FlushWriter(e.StdOut)
|
||||
|
||||
f, err := lookupPathHost(command[0], env, e.StdOut)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -66,12 +66,15 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
|
||||
return strings.Join(paths, ":")
|
||||
}
|
||||
|
||||
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
|
||||
const DefaultToolCache = "/opt/hostedtoolcache"
|
||||
|
||||
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
|
||||
return map[string]any{
|
||||
"os": "Linux",
|
||||
"arch": RunnerArch(ctx),
|
||||
"temp": "/tmp",
|
||||
"tool_cache": "/opt/hostedtoolcache",
|
||||
"tool_cache": DefaultToolCache,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.com/gitea/runner/act/model"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLiterals(t *testing.T) {
|
||||
@@ -523,7 +524,9 @@ func TestOperatorsBooleanEvaluation(t *testing.T) {
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
if expected, ok := tt.expected.(float64); ok && math.IsNaN(expected) {
|
||||
assert.True(t, math.IsNaN(output.(float64)))
|
||||
number, ok := output.(float64)
|
||||
require.True(t, ok, "want a number, got %T", output)
|
||||
assert.True(t, math.IsNaN(number))
|
||||
} else {
|
||||
assert.Equal(t, tt.expected, output)
|
||||
}
|
||||
|
||||
@@ -86,11 +86,12 @@ func (w *Workflow) OnSchedule() []string {
|
||||
case []any:
|
||||
allSchedules := []string{}
|
||||
for _, v := range val {
|
||||
for k, cron := range v.(map[string]any) {
|
||||
if k != "cron" {
|
||||
continue
|
||||
}
|
||||
allSchedules = append(allSchedules, cron.(string))
|
||||
entry, ok := v.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if cron, ok := entry["cron"].(string); ok {
|
||||
allSchedules = append(allSchedules, cron)
|
||||
}
|
||||
}
|
||||
return allSchedules
|
||||
@@ -443,9 +444,9 @@ func normalizeMatrixValue(key string, val any) ([]any, error) {
|
||||
// Scalar values are wrapped into single-element arrays automatically.
|
||||
// Template expressions are resolved by EvaluateYamlNode before this method is
|
||||
// called; if unresolved, the literal string is wrapped as a one-element fallback.
|
||||
func (j *Job) Matrix() map[string][]any {
|
||||
func (j *Job) Matrix() (map[string][]any, error) {
|
||||
if j.Strategy == nil || j.Strategy.RawMatrix.Kind != yaml.MappingNode {
|
||||
return nil
|
||||
return map[string][]any{}, nil
|
||||
}
|
||||
|
||||
// Decode to flexible map first so that scalar values don't cause a type error.
|
||||
@@ -455,9 +456,9 @@ func (j *Job) Matrix() map[string][]any {
|
||||
// Fall back to the strict array-only format for backward compatibility.
|
||||
var val map[string][]any
|
||||
if !decodeNode(j.Strategy.RawMatrix, &val) {
|
||||
return nil
|
||||
return map[string][]any{}, nil
|
||||
}
|
||||
return val
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// Convert flexible format to expected format with validation
|
||||
@@ -465,12 +466,11 @@ func (j *Job) Matrix() map[string][]any {
|
||||
for k, v := range flexVal {
|
||||
normalized, err := normalizeMatrixValue(k, v)
|
||||
if err != nil {
|
||||
log.Errorf("matrix validation error: %v", err)
|
||||
return nil
|
||||
return nil, err
|
||||
}
|
||||
val[k] = normalized
|
||||
}
|
||||
return val
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// GetMatrixes returns the matrix cross product
|
||||
@@ -482,38 +482,38 @@ func (j *Job) GetMatrixes() ([]map[string]any, error) {
|
||||
j.Strategy.FailFast = j.Strategy.GetFailFast()
|
||||
j.Strategy.MaxParallel = j.Strategy.GetMaxParallel()
|
||||
|
||||
if m := j.Matrix(); m != nil {
|
||||
m, err := j.Matrix()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(m) > 0 {
|
||||
includes := make([]map[string]any, 0)
|
||||
extraIncludes := make([]map[string]any, 0)
|
||||
addInclude := func(raw any) error {
|
||||
include, ok := raw.(map[string]any)
|
||||
if !ok {
|
||||
return fmt.Errorf("the workflow is not valid. Matrix include %v is not a map of matrix keys to values", raw)
|
||||
}
|
||||
for k := range include {
|
||||
if _, ok := m[k]; ok {
|
||||
includes = append(includes, include)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
extraIncludes = append(extraIncludes, include)
|
||||
return nil
|
||||
}
|
||||
for _, v := range m["include"] {
|
||||
switch t := v.(type) {
|
||||
case []any:
|
||||
for _, i := range t {
|
||||
i := i.(map[string]any)
|
||||
extraInclude := true
|
||||
for k := range i {
|
||||
if _, ok := m[k]; ok {
|
||||
includes = append(includes, i)
|
||||
extraInclude = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if extraInclude {
|
||||
extraIncludes = append(extraIncludes, i)
|
||||
if err := addInclude(i); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
case any:
|
||||
v := v.(map[string]any)
|
||||
extraInclude := true
|
||||
for k := range v {
|
||||
if _, ok := m[k]; ok {
|
||||
includes = append(includes, v)
|
||||
extraInclude = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if extraInclude {
|
||||
extraIncludes = append(extraIncludes, v)
|
||||
if err := addInclude(t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -521,10 +521,13 @@ func (j *Job) GetMatrixes() ([]map[string]any, error) {
|
||||
|
||||
excludes := make([]map[string]any, 0)
|
||||
for _, e := range m["exclude"] {
|
||||
e := e.(map[string]any)
|
||||
for k := range e {
|
||||
exclude, ok := e.(map[string]any)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the workflow is not valid. Matrix exclude %v is not a map of matrix keys to values", e)
|
||||
}
|
||||
for k := range exclude {
|
||||
if _, ok := m[k]; ok {
|
||||
excludes = append(excludes, e)
|
||||
excludes = append(excludes, exclude)
|
||||
} else {
|
||||
// We fail completely here because that's what GitHub does for non-existing matrix keys, fail on exclude, silent skip on include
|
||||
return nil, fmt.Errorf("the workflow is not valid. Matrix exclude key %q does not match any key within the matrix", k)
|
||||
|
||||
@@ -667,7 +667,9 @@ func TestReadWorkflow_Strategy(t *testing.T) {
|
||||
matrixes, err := job.GetMatrixes()
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, matrixes, []map[string]any{{}}) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, job.Matrix(), map[string][]any(nil))
|
||||
matrix, err := job.Matrix()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, matrix)
|
||||
assert.Equal(t, job.Strategy.MaxParallel, 2) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, job.Strategy.FailFast, true) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
@@ -675,7 +677,9 @@ func TestReadWorkflow_Strategy(t *testing.T) {
|
||||
matrixes, err = job.GetMatrixes()
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, matrixes, []map[string]any{{}}) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, job.Matrix(), map[string][]any(nil))
|
||||
matrix, err = job.Matrix()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, matrix)
|
||||
assert.Equal(t, job.Strategy.MaxParallel, 4) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, job.Strategy.FailFast, false) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
@@ -683,7 +687,9 @@ func TestReadWorkflow_Strategy(t *testing.T) {
|
||||
matrixes, err = job.GetMatrixes()
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, matrixes, []map[string]any{{}}) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, job.Matrix(), map[string][]any(nil))
|
||||
matrix, err = job.Matrix()
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, matrix)
|
||||
assert.Equal(t, job.Strategy.MaxParallel, 2) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.Equal(t, job.Strategy.FailFast, false) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
|
||||
@@ -700,7 +706,9 @@ func TestReadWorkflow_Strategy(t *testing.T) {
|
||||
{"datacenter": "site-b", "node-version": "12.x", "site": "dev"},
|
||||
},
|
||||
)
|
||||
assert.Equal(t, job.Matrix(), //nolint:testifylint // pre-existing issue from nektos/act
|
||||
matrix, err = job.Matrix()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, matrix, //nolint:testifylint // pre-existing issue from nektos/act
|
||||
map[string][]any{
|
||||
"datacenter": {"site-c", "site-d"},
|
||||
"exclude": {
|
||||
@@ -1092,13 +1100,15 @@ jobs:
|
||||
t.Fatal("job not found")
|
||||
}
|
||||
|
||||
matrix := job.Matrix()
|
||||
matrix, err := job.Matrix()
|
||||
|
||||
if tt.wantErr {
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, matrix, "matrix should be nil on error")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
if tt.wantLen == 0 {
|
||||
assert.Nil(t, matrix, "matrix should be nil for jobs without strategy")
|
||||
assert.Empty(t, matrix, "no matrix for jobs without strategy")
|
||||
} else {
|
||||
assert.NotNil(t, matrix, "matrix should not be nil")
|
||||
assert.Len(t, matrix, tt.wantLen, "matrix should have expected number of keys")
|
||||
@@ -1130,11 +1140,9 @@ func TestJobMatrixValidation(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
// Attempt to get matrix
|
||||
matrix := job.Matrix()
|
||||
|
||||
// Should return nil due to validation error
|
||||
assert.Nil(t, matrix, "matrix with nested map should return nil")
|
||||
matrix, err := job.Matrix()
|
||||
require.ErrorContains(t, err, `matrix key "config" has invalid nested object value`)
|
||||
assert.Nil(t, matrix)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -129,6 +129,16 @@ func readActionImpl(ctx context.Context, step *model.Step, actionDir, actionPath
|
||||
return action, err
|
||||
}
|
||||
|
||||
// cachedActionTar returns the action's tree from the action cache, which only a remote action
|
||||
// has an entry in.
|
||||
func cachedActionTar(ctx context.Context, step actionStep, name, includePrefix string) (io.ReadCloser, error) {
|
||||
remote, ok := step.(*stepActionRemote)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("action %q is a remote action but runs as %T", name, step)
|
||||
}
|
||||
return step.getRunContext().Config.ActionCache.GetTarArchive(ctx, remote.cacheDir, remote.resolvedSha, includePrefix)
|
||||
}
|
||||
|
||||
func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actionPath, containerActionDir string) error {
|
||||
logger := common.Logger(ctx)
|
||||
rc := step.getRunContext()
|
||||
@@ -147,8 +157,7 @@ func maybeCopyToActionDir(ctx context.Context, step actionStep, actionDir, actio
|
||||
}
|
||||
|
||||
if rc.Config != nil && rc.Config.ActionCache != nil {
|
||||
raction := step.(*stepActionRemote)
|
||||
ta, err := rc.Config.ActionCache.GetTarArchive(ctx, raction.cacheDir, raction.resolvedSha, "")
|
||||
ta, err := cachedActionTar(ctx, step, stepModel.Uses, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -351,8 +360,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
}
|
||||
defer buildContext.Close()
|
||||
} else if rc.Config.ActionCache != nil {
|
||||
rstep := step.(*stepActionRemote)
|
||||
buildContext, err = rc.Config.ActionCache.GetTarArchive(ctx, rstep.cacheDir, rstep.resolvedSha, contextDir)
|
||||
buildContext, err = cachedActionTar(ctx, step, actionName, contextDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -364,6 +372,7 @@ func execAsDocker(ctx context.Context, step actionStep, actionName, actionDir, b
|
||||
ImageTag: image,
|
||||
BuildContext: buildContext,
|
||||
Platform: rc.Config.ContainerArchitecture,
|
||||
BuildArgs: rc.proxyBuildArgs(),
|
||||
})
|
||||
if buildContext == nil {
|
||||
// Held across the whole build: the daemon drains contextDir lazily.
|
||||
@@ -472,10 +481,7 @@ func newStepContainer(ctx context.Context, step step, image string, cmd, entrypo
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/opt/hostedtoolcache"))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux"))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx)))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp"))
|
||||
envList = append(envList, rc.runnerEnv(ctx)...)
|
||||
|
||||
binds, mounts := rc.GetBindsAndMounts()
|
||||
networkMode := "container:" + rc.jobContainerName()
|
||||
|
||||
@@ -5,11 +5,9 @@ package runner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
@@ -42,18 +40,6 @@ func requireDocker(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// requireNetwork skips the test unless github.com is reachable. A few tests exercise behaviour
|
||||
// that inherently needs the network (force-pulling an image, resolving a remote short-sha ref);
|
||||
// gating lets the rest of the suite run offline without these failing.
|
||||
func requireNetwork(t *testing.T) {
|
||||
t.Helper()
|
||||
conn, err := net.DialTimeout("tcp", "github.com:443", 3*time.Second)
|
||||
if err != nil {
|
||||
t.Skipf("skipping: network unavailable: %v", err)
|
||||
}
|
||||
_ = conn.Close()
|
||||
}
|
||||
|
||||
// requireHostTools skips the test unless every named executable is on PATH. Used by the
|
||||
// self-hosted (host environment) suite, which runs steps directly on the host.
|
||||
func requireHostTools(t *testing.T, tools ...string) {
|
||||
|
||||
@@ -226,11 +226,16 @@ func newJobExecutor(info jobInfo, sf stepFactory, rc *RunContext) common.Executo
|
||||
}
|
||||
}
|
||||
|
||||
// The setup section of the job log: download the actions, run the pre steps, then name the job.
|
||||
// The setup section of the job log. The started hook goes first, so what it sets up is
|
||||
// in place for the first action download and the first step.
|
||||
preSteps = append(preSteps, rc.runJobStartedHook)
|
||||
preSteps = append(preSteps, printPrepareActions(rc, preparers))
|
||||
preSteps = append(preSteps, stepPreSteps...)
|
||||
preSteps = append(preSteps, printCompleteJobName(rc))
|
||||
|
||||
// Ahead of the teardown below, while the job environment is still up.
|
||||
postExecutor = postExecutor.Finally(rc.runJobCompletedHook)
|
||||
|
||||
postExecutor = postExecutor.Finally(func(ctx context.Context) error {
|
||||
jobError := common.JobError(ctx)
|
||||
var err error
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
)
|
||||
|
||||
func TestJobExecutor(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Dryrun only checks syntax/planning; all cases resolve locally, so this runs offline.
|
||||
tables := []TestJobFileInfo{
|
||||
{workdir, "uses-and-run-in-one-step", "push", "Invalid run/uses syntax for job:test step:Test", platforms, secrets},
|
||||
@@ -46,6 +47,7 @@ func TestJobExecutor(t *testing.T) {
|
||||
ctx := common.WithDryrun(context.Background(), true)
|
||||
for _, table := range tables {
|
||||
t.Run(table.workflowPath, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
table.runTest(ctx, t, &Config{})
|
||||
})
|
||||
}
|
||||
|
||||
115
act/runner/job_hooks.go
Normal file
115
act/runner/job_hooks.go
Normal file
@@ -0,0 +1,115 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package runner
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
)
|
||||
|
||||
// GitHub's job-hook variables, read as a fallback when the settings are unset.
|
||||
const (
|
||||
jobStartedHookEnv = "ACTIONS_RUNNER_HOOK_JOB_STARTED"
|
||||
jobCompletedHookEnv = "ACTIONS_RUNNER_HOOK_JOB_COMPLETED"
|
||||
)
|
||||
|
||||
// Kept apart from the per-step file-command files, which are truncated on every step.
|
||||
const (
|
||||
hookEnvFileCommand = "workflow/hook-envs.txt"
|
||||
hookPathFileCommand = "workflow/hook-path.txt"
|
||||
)
|
||||
|
||||
func (rc *RunContext) runJobStartedHook(ctx context.Context) error {
|
||||
return rc.runJobHook(ctx, cmp.Or(rc.Config.JobStartedHook, rc.Config.Env[jobStartedHookEnv]), "job started")
|
||||
}
|
||||
|
||||
func (rc *RunContext) runJobCompletedHook(ctx context.Context) error {
|
||||
return rc.runJobHook(ctx, cmp.Or(rc.Config.JobCompletedHook, rc.Config.Env[jobCompletedHookEnv]), "job completed")
|
||||
}
|
||||
|
||||
// runJobHook runs one hook in the job environment. Either hook failing fails the job, as
|
||||
// on GitHub, where the operator is responsible for the hook's own resilience.
|
||||
func (rc *RunContext) runJobHook(ctx context.Context, hookPath, name string) error {
|
||||
if hookPath == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
cmd, shell := hookCommand(hookPath)
|
||||
rawLogger := common.Logger(ctx).WithField(rawOutputField, true)
|
||||
defer rawLogger.Infof("::endgroup::")
|
||||
rawLogger.Infof("::group::Run '%s'", escapeCommandData(hookPath))
|
||||
rawLogger.Infof("A %s hook has been configured by the runner administrator", name)
|
||||
if shell != "" {
|
||||
rawLogger.Infof("shell: %s", shell)
|
||||
}
|
||||
|
||||
env := maps.Clone(rc.GetEnv())
|
||||
if jobContainer := rc.Run.Job().Container(); jobContainer != nil {
|
||||
maps.Copy(env, jobContainer.Env)
|
||||
}
|
||||
rc.withGithubEnv(ctx, rc.getGithubContext(ctx), env)
|
||||
rc.ApplyExtraPath(ctx, &env)
|
||||
|
||||
err := rc.setupHookFileCommands(ctx, env)
|
||||
if err == nil {
|
||||
err = rc.JobContainer.Exec(cmd, env, "", "")(ctx)
|
||||
}
|
||||
// Processed even on failure, so a hook that exports what it managed to set up before
|
||||
// failing still hands it to the job.
|
||||
err = cmp.Or(err, rc.processHookFileCommands(ctx))
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = fmt.Errorf("the %s hook %q failed: %w", name, hookPath, err)
|
||||
// Flip the job status the way a failing pre step does, so success()-default main steps
|
||||
// skip and the task is reported failed.
|
||||
reportStepError(ctx, rc, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// setupHookFileCommands points the hook at its GITHUB_ENV and GITHUB_PATH files, so it can
|
||||
// export to the job's steps, and truncates them so the second hook does not re-read what
|
||||
// the first one wrote.
|
||||
func (rc *RunContext) setupHookFileCommands(ctx context.Context, env map[string]string) error {
|
||||
actPath := rc.JobContainer.GetActPath()
|
||||
env["GITHUB_ENV"] = path.Join(actPath, hookEnvFileCommand)
|
||||
env["GITHUB_PATH"] = path.Join(actPath, hookPathFileCommand)
|
||||
env["GITEA_ENV"] = env["GITHUB_ENV"]
|
||||
env["GITEA_PATH"] = env["GITHUB_PATH"]
|
||||
|
||||
return rc.JobContainer.Copy(actPath,
|
||||
&container.FileEntry{Name: hookEnvFileCommand, Mode: 0o666},
|
||||
&container.FileEntry{Name: hookPathFileCommand, Mode: 0o666},
|
||||
)(ctx)
|
||||
}
|
||||
|
||||
func (rc *RunContext) processHookFileCommands(ctx context.Context) error {
|
||||
if err := processRunnerEnvFileCommand(ctx, hookEnvFileCommand, rc, rc.setEnv); err != nil {
|
||||
return err
|
||||
}
|
||||
return rc.UpdateExtraPath(ctx, path.Join(rc.JobContainer.GetActPath(), hookPathFileCommand))
|
||||
}
|
||||
|
||||
// hookCommand mirrors actions/runner, which deliberately does not apply the shell flags it
|
||||
// gives `run:` steps — a hook sets its own. See docs/adrs/1751-runner-job-hooks.md there.
|
||||
// The second return value is how the invocation is shown in the log, empty when the file is
|
||||
// executed directly.
|
||||
func hookCommand(hookPath string) (cmd []string, shell string) {
|
||||
switch strings.ToLower(path.Ext(hookPath)) {
|
||||
case ".sh":
|
||||
return []string{"bash", "-e", hookPath}, "bash -e {0}"
|
||||
case ".ps1":
|
||||
return []string{"pwsh", "-command", ". '" + hookPath + "'"}, `pwsh -command ". '{0}'"`
|
||||
default:
|
||||
return []string{hookPath}, ""
|
||||
}
|
||||
}
|
||||
162
act/runner/job_hooks_test.go
Normal file
162
act/runner/job_hooks_test.go
Normal file
@@ -0,0 +1,162 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package runner
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"maps"
|
||||
"testing"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/model"
|
||||
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// hookContainer records the command a hook was run with and answers with what the hook
|
||||
// wrote to its GITHUB_ENV and GITHUB_PATH files.
|
||||
type hookContainer struct {
|
||||
fakeContainer
|
||||
cmd []string
|
||||
env map[string]string
|
||||
err error
|
||||
envFile map[string]string
|
||||
pathTar []byte
|
||||
}
|
||||
|
||||
func (c *hookContainer) ToContainerPath(path string) string { return path }
|
||||
func (c *hookContainer) IsEnvironmentCaseInsensitive() bool { return false }
|
||||
|
||||
func (c *hookContainer) GetRunnerContext(context.Context) map[string]any {
|
||||
return map[string]any{"os": "Linux"}
|
||||
}
|
||||
|
||||
func (c *hookContainer) Exec(command []string, env map[string]string, _, _ string) common.Executor {
|
||||
return func(context.Context) error {
|
||||
c.cmd, c.env = command, env
|
||||
return c.err
|
||||
}
|
||||
}
|
||||
|
||||
func (c *hookContainer) UpdateFromEnv(_ string, env *map[string]string) common.Executor {
|
||||
return func(context.Context) error {
|
||||
maps.Copy(*env, c.envFile)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (c *hookContainer) GetContainerArchive(context.Context, string) (io.ReadCloser, error) {
|
||||
return io.NopCloser(bytes.NewReader(c.pathTar)), nil
|
||||
}
|
||||
|
||||
// newHookRunContext returns a RunContext and the context to run a hook with, whose logger is
|
||||
// silenced so the hook's job-log output does not reach the test output.
|
||||
func newHookRunContext(jobContainer *hookContainer, config *Config) (*RunContext, context.Context) {
|
||||
// Env is left nil so that it is built from the config, as it is for a real job.
|
||||
rc := &RunContext{
|
||||
Config: config,
|
||||
Run: &model.Run{JobID: "job", Workflow: &model.Workflow{Jobs: map[string]*model.Job{"job": {}}}},
|
||||
JobContainer: jobContainer,
|
||||
}
|
||||
logger, _ := test.NewNullLogger()
|
||||
ctx := common.WithJobErrorContainer(common.WithLogger(context.Background(), logger.WithField("test", true)))
|
||||
rc.ExprEval = rc.NewExpressionEvaluator(ctx)
|
||||
return rc, ctx
|
||||
}
|
||||
|
||||
func TestRunJobHook(t *testing.T) {
|
||||
t.Run("runs the hook with the job environment", func(t *testing.T) {
|
||||
jobContainer := &hookContainer{}
|
||||
rc, ctx := newHookRunContext(jobContainer, &Config{
|
||||
JobStartedHook: "/hooks/started.sh",
|
||||
Env: map[string]string{"A_VAR": "value", jobStartedHookEnv: "/from/env.sh"},
|
||||
})
|
||||
|
||||
require.NoError(t, rc.runJobStartedHook(ctx))
|
||||
|
||||
// The setting wins over the environment variable.
|
||||
assert.Equal(t, []string{"bash", "-e", "/hooks/started.sh"}, jobContainer.cmd)
|
||||
assert.Equal(t, "value", jobContainer.env["A_VAR"])
|
||||
// The github environment is there too, so a hook can tell which job it runs for.
|
||||
assert.Equal(t, "job", jobContainer.env["GITHUB_JOB"])
|
||||
assert.Equal(t, "/var/run/act/workflow/hook-envs.txt", jobContainer.env["GITHUB_ENV"])
|
||||
assert.Equal(t, "/var/run/act/workflow/hook-path.txt", jobContainer.env["GITHUB_PATH"])
|
||||
})
|
||||
|
||||
// Each hook reads its own variable, so a swapped constant cannot pass.
|
||||
t.Run("falls back to the GitHub environment variables", func(t *testing.T) {
|
||||
for name, hook := range map[string]struct {
|
||||
env string
|
||||
run func(*RunContext, context.Context) error
|
||||
}{
|
||||
"started": {jobStartedHookEnv, (*RunContext).runJobStartedHook},
|
||||
"completed": {jobCompletedHookEnv, (*RunContext).runJobCompletedHook},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
jobContainer := &hookContainer{}
|
||||
rc, ctx := newHookRunContext(jobContainer, &Config{Env: map[string]string{hook.env: "/from/env.sh"}})
|
||||
|
||||
require.NoError(t, hook.run(rc, ctx))
|
||||
assert.Equal(t, []string{"bash", "-e", "/from/env.sh"}, jobContainer.cmd)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("exports what the hook wrote to GITHUB_ENV and GITHUB_PATH", func(t *testing.T) {
|
||||
jobContainer := &hookContainer{
|
||||
envFile: map[string]string{"FROM_HOOK": "1"},
|
||||
pathTar: tarArchive(t, tarEntry{name: "hook-path.txt", body: "/opt/tool/bin\n"}),
|
||||
}
|
||||
rc, ctx := newHookRunContext(jobContainer, &Config{JobStartedHook: "/hooks/started.sh"})
|
||||
|
||||
require.NoError(t, rc.runJobStartedHook(ctx))
|
||||
|
||||
assert.Equal(t, "1", rc.Env["FROM_HOOK"])
|
||||
assert.Equal(t, []string{"/opt/tool/bin"}, rc.ExtraPath)
|
||||
})
|
||||
|
||||
t.Run("a failing hook fails the job", func(t *testing.T) {
|
||||
rc, ctx := newHookRunContext(&hookContainer{err: errors.New("boom")}, &Config{JobStartedHook: "/hooks/started.sh"})
|
||||
|
||||
err := rc.runJobStartedHook(ctx)
|
||||
|
||||
require.ErrorContains(t, err, `the job started hook "/hooks/started.sh" failed`)
|
||||
require.ErrorContains(t, err, "boom")
|
||||
// The failure has to flip the job status, or success()-default steps would still
|
||||
// run and the task would be reported successful despite the missing setup.
|
||||
assert.Equal(t, "failure", rc.getJobContext().Status)
|
||||
require.ErrorContains(t, common.JobError(ctx), "boom")
|
||||
})
|
||||
|
||||
t.Run("is a no-op without a hook", func(t *testing.T) {
|
||||
jobContainer := &hookContainer{}
|
||||
rc, ctx := newHookRunContext(jobContainer, &Config{})
|
||||
|
||||
require.NoError(t, rc.runJobStartedHook(ctx))
|
||||
require.NoError(t, rc.runJobCompletedHook(ctx))
|
||||
assert.Nil(t, jobContainer.cmd)
|
||||
})
|
||||
}
|
||||
|
||||
// actions/runner deliberately runs a hook without the flags it gives `run:` steps, and an
|
||||
// executable without a known extension speaks for itself through its shebang.
|
||||
func TestHookCommand(t *testing.T) {
|
||||
for hookPath, want := range map[string]struct {
|
||||
cmd []string
|
||||
shell string
|
||||
}{
|
||||
"/hooks/started.sh": {[]string{"bash", "-e", "/hooks/started.sh"}, "bash -e {0}"},
|
||||
"/hooks/started.PS1": {[]string{"pwsh", "-command", ". '/hooks/started.PS1'"}, `pwsh -command ". '{0}'"`},
|
||||
"/hooks/started": {[]string{"/hooks/started"}, ""},
|
||||
} {
|
||||
cmd, shell := hookCommand(hookPath)
|
||||
assert.Equal(t, want.cmd, cmd, hookPath)
|
||||
assert.Equal(t, want.shell, shell, hookPath)
|
||||
}
|
||||
}
|
||||
@@ -22,15 +22,17 @@ import (
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
"gitea.com/gitea/runner/act/exprparser"
|
||||
"gitea.com/gitea/runner/act/model"
|
||||
"gitea.com/gitea/runner/internal/pkg/lock"
|
||||
|
||||
"github.com/docker/cli/cli/compose/loader"
|
||||
"github.com/docker/go-connections/nat"
|
||||
"github.com/moby/moby/api/types/mount"
|
||||
"github.com/opencontainers/selinux/go-selinux"
|
||||
)
|
||||
|
||||
@@ -204,52 +206,93 @@ func (rc *RunContext) validVolumes() []string {
|
||||
getDockerDaemonSocketMountPath(rc.containerDaemonSocket()))
|
||||
}
|
||||
|
||||
// toolCache returns the tool cache path the job sees, relocatable through RUNNER_TOOL_CACHE.
|
||||
func (rc *RunContext) toolCache(fallback string) string {
|
||||
if path := rc.GetEnv()["RUNNER_TOOL_CACHE"]; path != "" {
|
||||
return path
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// runnerEnv returns a container's RUNNER_* variables, derived from the values runner.tool_cache
|
||||
// and friends report so the two cannot drift apart.
|
||||
func (rc *RunContext) runnerEnv(ctx context.Context) []string {
|
||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||
runnerContext := ext.GetRunnerContext(ctx)
|
||||
runnerContext["tool_cache"] = rc.toolCache(container.DefaultToolCache)
|
||||
|
||||
env := make([]string, 0, len(runnerContext))
|
||||
for key, value := range runnerContext {
|
||||
env = append(env, fmt.Sprintf("RUNNER_%s=%s", strings.ToUpper(key), value))
|
||||
}
|
||||
slices.Sort(env)
|
||||
return env
|
||||
}
|
||||
|
||||
// splitVolumes routes volume specs into binds and a source:target mount map, and returns the
|
||||
// container paths they mount onto. Only a plain source:target volume fits the map, everything
|
||||
// else (anonymous volumes, host binds, mount options) stays a bind.
|
||||
func splitVolumes(specs []string) ([]string, map[string]string, map[string]bool) {
|
||||
binds := []string{}
|
||||
mounts := map[string]string{}
|
||||
targets := map[string]bool{}
|
||||
|
||||
for _, spec := range specs {
|
||||
parsed, err := loader.ParseVolume(spec)
|
||||
if err != nil {
|
||||
binds = append(binds, spec) // let Docker report the malformed spec
|
||||
continue
|
||||
}
|
||||
targets[parsed.Target] = true
|
||||
if parsed.Type == string(mount.TypeVolume) && parsed.Source != "" && !parsed.ReadOnly {
|
||||
mounts[parsed.Source] = parsed.Target
|
||||
} else {
|
||||
binds = append(binds, spec)
|
||||
}
|
||||
}
|
||||
return binds, mounts, targets
|
||||
}
|
||||
|
||||
// Returns the binds and mounts for the container, resolving paths as appopriate
|
||||
func (rc *RunContext) GetBindsAndMounts() ([]string, map[string]string) {
|
||||
name := rc.jobContainerName()
|
||||
|
||||
binds := []string{}
|
||||
if daemonSocket := rc.containerDaemonSocket(); daemonSocket != "-" {
|
||||
daemonPath := getDockerDaemonSocketMountPath(daemonSocket)
|
||||
binds = append(binds, fmt.Sprintf("%s:%s", daemonPath, "/var/run/docker.sock"))
|
||||
}
|
||||
|
||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||
|
||||
mounts := map[string]string{
|
||||
"act-toolcache": "/opt/hostedtoolcache",
|
||||
name + "-env": ext.GetActPath(),
|
||||
}
|
||||
|
||||
var volumes []string
|
||||
if job := rc.Run.Job(); job != nil {
|
||||
if container := job.Container(); container != nil {
|
||||
for _, v := range container.Volumes {
|
||||
if rc.ExprEval != nil {
|
||||
v = rc.ExprEval.Interpolate(context.Background(), v)
|
||||
}
|
||||
if !strings.Contains(v, ":") || filepath.IsAbs(v) {
|
||||
// Bind anonymous volume or host file.
|
||||
binds = append(binds, v)
|
||||
} else {
|
||||
// Mount existing volume.
|
||||
paths := strings.SplitN(v, ":", 2)
|
||||
mounts[paths[0]] = paths[1]
|
||||
}
|
||||
volumes = append(volumes, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
// the runner's own mounts below yield to the targets the job claims
|
||||
binds, mounts, claimed := splitVolumes(volumes)
|
||||
|
||||
if rc.Config.BindWorkdir {
|
||||
bindModifiers := ""
|
||||
if runtime.GOOS == "darwin" {
|
||||
bindModifiers = ":delegated"
|
||||
if daemonSocket := rc.containerDaemonSocket(); daemonSocket != "-" && !claimed["/var/run/docker.sock"] {
|
||||
binds = append(binds, getDockerDaemonSocketMountPath(daemonSocket)+":/var/run/docker.sock")
|
||||
}
|
||||
if toolCache := rc.toolCache(container.DefaultToolCache); !claimed[toolCache] {
|
||||
mounts["act-toolcache"] = toolCache
|
||||
}
|
||||
mounts[name+"-env"] = ext.GetActPath() // runner-internal, never overridable
|
||||
|
||||
if workdir := ext.ToContainerPath(rc.Config.Workdir); !claimed[workdir] {
|
||||
if rc.Config.BindWorkdir {
|
||||
bindModifiers := ""
|
||||
if runtime.GOOS == "darwin" {
|
||||
bindModifiers = ":delegated"
|
||||
}
|
||||
if selinux.GetEnabled() {
|
||||
bindModifiers = ":z"
|
||||
}
|
||||
binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, workdir, bindModifiers))
|
||||
} else {
|
||||
mounts[name] = workdir
|
||||
}
|
||||
if selinux.GetEnabled() {
|
||||
bindModifiers = ":z"
|
||||
}
|
||||
binds = append(binds, fmt.Sprintf("%s:%s%s", rc.Config.Workdir, ext.ToContainerPath(rc.Config.Workdir), bindModifiers))
|
||||
} else {
|
||||
mounts[name] = ext.ToContainerPath(rc.Config.Workdir)
|
||||
}
|
||||
|
||||
return binds, mounts
|
||||
@@ -283,7 +326,10 @@ func (rc *RunContext) startHostEnvironment() common.Executor {
|
||||
if err := os.MkdirAll(runnerTmp, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
toolCache := filepath.Join(cacheDir, "tool_cache")
|
||||
toolCache := rc.toolCache(filepath.Join(cacheDir, "tool_cache"))
|
||||
if err := os.MkdirAll(toolCache, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
rc.JobContainer = &container.HostEnvironment{
|
||||
Path: path,
|
||||
TmpDir: runnerTmp,
|
||||
@@ -368,10 +414,7 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
|
||||
envList := make([]string, 0)
|
||||
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/opt/hostedtoolcache"))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux"))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx)))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp"))
|
||||
envList = append(envList, rc.runnerEnv(ctx)...)
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "LANG", "C.UTF-8")) // Use same locale as GitHub Actions
|
||||
|
||||
ext := container.LinuxContainerEnvironmentExtensions{}
|
||||
@@ -392,7 +435,9 @@ func (rc *RunContext) startJobContainer() common.Executor {
|
||||
continue
|
||||
}
|
||||
// interpolate env
|
||||
interpolatedEnvs := make(map[string]string, len(spec.Env))
|
||||
interpolatedEnvs := make(map[string]string, len(spec.Env)+len(rc.Config.ProxyEnv))
|
||||
// a service reaches the internet the way the job does; its own env still wins
|
||||
maps0.Copy(interpolatedEnvs, rc.Config.ProxyEnv)
|
||||
for k, v := range spec.Env {
|
||||
interpolatedEnvs[k] = rc.ExprEval.Interpolate(ctx, v)
|
||||
}
|
||||
@@ -672,13 +717,10 @@ func (rc *RunContext) ActionCacheDir() string {
|
||||
// jobMutexes serializes per-job result/output aggregation across the matrix combinations that
|
||||
// share one *model.Job and run in parallel. Keyed by the shared *model.Job (mirrors the
|
||||
// per-directory AcquireCloneLock pattern).
|
||||
var jobMutexes sync.Map // key: *model.Job; value: *sync.Mutex
|
||||
var jobMutexes lock.Keyed[*model.Job]
|
||||
|
||||
func lockJob(job *model.Job) func() {
|
||||
v, _ := jobMutexes.LoadOrStore(job, &sync.Mutex{})
|
||||
mu := v.(*sync.Mutex)
|
||||
mu.Lock()
|
||||
return mu.Unlock
|
||||
return jobMutexes.Lock(job)
|
||||
}
|
||||
|
||||
func (rc *RunContext) interpolateOutputs() common.Executor {
|
||||
@@ -929,6 +971,20 @@ func (rc *RunContext) isEnabled(ctx context.Context) (bool, error) {
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// proxyBuildArgs returns the job's proxy variables as docker build args. The docker CLI
|
||||
// pre-populates these from its own client configuration, but act builds through the API,
|
||||
// so without them a Dockerfile action's RUN steps have no network behind a proxy.
|
||||
func (rc *RunContext) proxyBuildArgs() map[string]*string {
|
||||
if len(rc.Config.ProxyEnv) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := make(map[string]*string, len(rc.Config.ProxyEnv))
|
||||
for name, value := range rc.Config.ProxyEnv {
|
||||
args[name] = &value
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func mergeMaps(maps ...map[string]string) map[string]string {
|
||||
rtnMap := make(map[string]string)
|
||||
for _, m := range maps {
|
||||
@@ -992,6 +1048,8 @@ func (rc *RunContext) getRunnerContext(ctx context.Context) map[string]any {
|
||||
runnerContext := map[string]any{}
|
||||
if rc.JobContainer != nil {
|
||||
maps0.Copy(runnerContext, rc.JobContainer.GetRunnerContext(ctx))
|
||||
defaultToolCache, _ := runnerContext["tool_cache"].(string)
|
||||
runnerContext["tool_cache"] = rc.toolCache(defaultToolCache)
|
||||
}
|
||||
runnerContext["name"] = rc.Config.RunnerName
|
||||
runnerContext["environment"] = "self-hosted"
|
||||
@@ -1363,24 +1421,9 @@ func (rc *RunContext) handleServiceCredentials(ctx context.Context, creds map[st
|
||||
|
||||
// GetServiceBindsAndMounts returns the binds and mounts for the service container, resolving paths as appopriate
|
||||
func (rc *RunContext) GetServiceBindsAndMounts(svcVolumes []string) ([]string, map[string]string) {
|
||||
binds := []string{}
|
||||
if daemonSocket := rc.containerDaemonSocket(); daemonSocket != "-" {
|
||||
daemonPath := getDockerDaemonSocketMountPath(daemonSocket)
|
||||
binds = append(binds, fmt.Sprintf("%s:%s", daemonPath, "/var/run/docker.sock"))
|
||||
binds, mounts, claimed := splitVolumes(svcVolumes)
|
||||
if daemonSocket := rc.containerDaemonSocket(); daemonSocket != "-" && !claimed["/var/run/docker.sock"] {
|
||||
binds = append(binds, getDockerDaemonSocketMountPath(daemonSocket)+":/var/run/docker.sock")
|
||||
}
|
||||
|
||||
mounts := map[string]string{}
|
||||
|
||||
for _, v := range svcVolumes {
|
||||
if !strings.Contains(v, ":") || filepath.IsAbs(v) {
|
||||
// Bind anonymous volume or host file.
|
||||
binds = append(binds, v)
|
||||
} else {
|
||||
// Mount existing volume.
|
||||
paths := strings.SplitN(v, ":", 2)
|
||||
mounts[paths[0]] = paths[1]
|
||||
}
|
||||
}
|
||||
|
||||
return binds, mounts
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.com/gitea/runner/act/exprparser"
|
||||
"gitea.com/gitea/runner/act/model"
|
||||
|
||||
"github.com/docker/cli/cli/compose/loader"
|
||||
log "github.com/sirupsen/logrus"
|
||||
assert "github.com/stretchr/testify/assert"
|
||||
require "github.com/stretchr/testify/require"
|
||||
@@ -291,6 +292,83 @@ jobs:
|
||||
require.Equal(t, [2]string{"", ""}, credentials["redis:latest"])
|
||||
}
|
||||
|
||||
// A service container reaches the internet the same way the job does, so it inherits the
|
||||
// job's proxy; a service that sets the variable itself keeps its own value.
|
||||
func TestStartJobContainerGivesServicesTheJobProxy(t *testing.T) {
|
||||
workflow, err := model.ReadWorkflow(strings.NewReader(`
|
||||
name: test
|
||||
on: push
|
||||
jobs:
|
||||
job:
|
||||
runs-on: ubuntu-latest
|
||||
container:
|
||||
image: registry.example/job:latest
|
||||
services:
|
||||
redis:
|
||||
image: redis:latest
|
||||
db:
|
||||
image: postgres:latest
|
||||
env:
|
||||
no_proxy: db-only.example
|
||||
steps: []
|
||||
`))
|
||||
require.NoError(t, err)
|
||||
|
||||
var inputs []*container.NewContainerInput
|
||||
origNewContainer := newContainer
|
||||
newContainer = func(input *container.NewContainerInput) container.ExecutionsEnvironment {
|
||||
inputs = append(inputs, input)
|
||||
return fakeContainer{}
|
||||
}
|
||||
t.Cleanup(func() { newContainer = origNewContainer })
|
||||
|
||||
rc := &RunContext{
|
||||
Name: "test",
|
||||
Config: &Config{
|
||||
Workdir: "/tmp",
|
||||
ContainerNetworkMode: "host",
|
||||
ReuseContainers: true,
|
||||
Env: map[string]string{},
|
||||
ProxyEnv: map[string]string{"http_proxy": "http://proxy:3128", "no_proxy": "internal.example"},
|
||||
Secrets: map[string]string{},
|
||||
},
|
||||
Env: map[string]string{},
|
||||
Run: &model.Run{
|
||||
JobID: "job",
|
||||
Workflow: workflow,
|
||||
},
|
||||
}
|
||||
rc.ExprEval = rc.NewExpressionEvaluator(t.Context())
|
||||
|
||||
require.NoError(t, rc.startJobContainer()(t.Context()))
|
||||
|
||||
env := map[string][]string{}
|
||||
for _, in := range inputs {
|
||||
env[in.Image] = in.Env
|
||||
}
|
||||
|
||||
require.Contains(t, env["redis:latest"], "http_proxy=http://proxy:3128")
|
||||
require.Contains(t, env["redis:latest"], "no_proxy=internal.example")
|
||||
// the service's own env wins over what the runner injected, without dropping the rest
|
||||
require.Contains(t, env["postgres:latest"], "no_proxy=db-only.example")
|
||||
require.NotContains(t, env["postgres:latest"], "no_proxy=internal.example")
|
||||
require.Contains(t, env["postgres:latest"], "http_proxy=http://proxy:3128")
|
||||
}
|
||||
|
||||
// act builds Dockerfile actions through the API, which does not pre-populate the proxy
|
||||
// build args the docker CLI would, so the RUN steps would have no network behind a proxy.
|
||||
func TestProxyBuildArgs(t *testing.T) {
|
||||
rc := &RunContext{Config: &Config{ProxyEnv: map[string]string{"http_proxy": "http://proxy:3128"}}}
|
||||
|
||||
args := rc.proxyBuildArgs()
|
||||
|
||||
require.Len(t, args, 1)
|
||||
require.Equal(t, "http://proxy:3128", *args["http_proxy"])
|
||||
|
||||
// a job without a proxy builds exactly as it does today
|
||||
require.Nil(t, (&RunContext{Config: &Config{}}).proxyBuildArgs())
|
||||
}
|
||||
|
||||
func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
rctemplate := &RunContext{
|
||||
Name: "TestRCName",
|
||||
@@ -363,6 +441,10 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
{"BindAnonymousVolume", []string{"/volume"}, "/volume", map[string]string{}},
|
||||
{"BindHostFile", []string{"/path/to/file/on/host:/volume"}, "/path/to/file/on/host:/volume", map[string]string{}},
|
||||
{"MountExistingVolume", []string{"volume-id:/volume"}, "", map[string]string{"volume-id": "/volume"}},
|
||||
{"MountExistingVolumeReadOnly", []string{"volume-id:/volume:ro"}, "volume-id:/volume:ro", map[string]string{}},
|
||||
{"BindRelativeHostPath", []string{"./relative:/volume"}, "./relative:/volume", map[string]string{}},
|
||||
{"OverridesToolCache", []string{"/host/tools:/opt/hostedtoolcache"}, "/host/tools:/opt/hostedtoolcache", map[string]string{}},
|
||||
{"OverridesDockerSocket", []string{"/host/docker.sock:/var/run/docker.sock"}, "/host/docker.sock:/var/run/docker.sock", map[string]string{}},
|
||||
}
|
||||
|
||||
t.Run("InterpolatedContainerVolumes", func(t *testing.T) {
|
||||
@@ -418,15 +500,37 @@ func TestRunContext_GetBindsAndMounts(t *testing.T) {
|
||||
rc.Run.JobID = "job1"
|
||||
rc.Run.Workflow.Jobs = map[string]*model.Job{"job1": job}
|
||||
|
||||
gotbind, gotmount := rc.GetBindsAndMounts()
|
||||
jobBinds, jobMounts := rc.GetBindsAndMounts()
|
||||
svcBinds, svcMounts := rc.GetServiceBindsAndMounts(testcase.volumes)
|
||||
// job and service containers classify volumes alike, only their own mounts differ
|
||||
for _, got := range []struct {
|
||||
binds []string
|
||||
mounts map[string]string
|
||||
}{{jobBinds, jobMounts}, {svcBinds, svcMounts}} {
|
||||
gotbind, gotmount := got.binds, got.mounts
|
||||
|
||||
if len(testcase.wantbind) > 0 {
|
||||
assert.Contains(t, gotbind, testcase.wantbind)
|
||||
}
|
||||
if len(testcase.wantbind) > 0 {
|
||||
assert.Contains(t, gotbind, testcase.wantbind)
|
||||
}
|
||||
|
||||
for k, v := range testcase.wantmount {
|
||||
assert.Contains(t, gotmount, k)
|
||||
assert.Equal(t, gotmount[k], v)
|
||||
for k, v := range testcase.wantmount {
|
||||
assert.Contains(t, gotmount, k)
|
||||
assert.Equal(t, gotmount[k], v)
|
||||
}
|
||||
|
||||
// Docker rejects a container with two mounts on one target, so the job's own
|
||||
// volumes must displace the runner's rather than pile up next to them.
|
||||
targets := map[string]bool{}
|
||||
for _, bind := range gotbind {
|
||||
parsed, err := loader.ParseVolume(bind)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, targets, parsed.Target, "%s mounts an already mounted target", bind)
|
||||
targets[parsed.Target] = true
|
||||
}
|
||||
for source, target := range gotmount {
|
||||
assert.NotContains(t, targets, target, "%s mounts an already mounted target", source)
|
||||
targets[target] = true
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -73,6 +73,7 @@ type Config struct {
|
||||
ContainerNetworkMode docker_container.NetworkMode // the network mode of job containers (the value of --network)
|
||||
ContainerNetworkCreateOptions container.NewDockerNetworkCreateExecutorInput // the default network create options
|
||||
ActionCache ActionCache // Use a custom ActionCache Implementation
|
||||
ProxyEnv map[string]string // the proxy variables the job runs with, also given to service containers and image builds
|
||||
|
||||
PresetGitHubContext *model.GithubContext // the preset github context, overrides some fields like DefaultBranch, Env, Secrets etc.
|
||||
EventJSON string // the content of JSON file to use for event.json in containers, overrides EventPath
|
||||
@@ -93,6 +94,8 @@ type Config struct {
|
||||
MaxParallel int // max parallel jobs to run across all workflows (0 = no limit, uses CPU count)
|
||||
AllocatePTY bool // allocate a pseudo-TTY for each step's process
|
||||
RunnerName string // name this runner registered with, reported as `runner.name`, defaults to the hostname
|
||||
JobStartedHook string // script run inside the job environment before the job's first step; ACTIONS_RUNNER_HOOK_JOB_STARTED is read from Env when empty
|
||||
JobCompletedHook string // script run inside the job environment after the job's last step; ACTIONS_RUNNER_HOOK_JOB_COMPLETED is read from Env when empty
|
||||
}
|
||||
|
||||
// RunnerDebug reports whether debug logging is on, exposed as `runner.debug` and
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package runner
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// TestMaxParallelConfig tests that MaxParallel config is properly set
|
||||
func TestMaxParallelConfig(t *testing.T) {
|
||||
t.Run("MaxParallel set to 2", func(t *testing.T) {
|
||||
config := &Config{
|
||||
Workdir: "testdata",
|
||||
MaxParallel: 2,
|
||||
}
|
||||
|
||||
runner, err := New(config)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.NotNil(t, runner)
|
||||
|
||||
// Verify config is properly stored
|
||||
runnerImpl, ok := runner.(*runnerImpl)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 2, runnerImpl.config.MaxParallel)
|
||||
})
|
||||
|
||||
t.Run("MaxParallel set to 0 (no limit)", func(t *testing.T) {
|
||||
config := &Config{
|
||||
Workdir: "testdata",
|
||||
MaxParallel: 0,
|
||||
}
|
||||
|
||||
runner, err := New(config)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.NotNil(t, runner)
|
||||
|
||||
runnerImpl, ok := runner.(*runnerImpl)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 0, runnerImpl.config.MaxParallel)
|
||||
})
|
||||
|
||||
t.Run("MaxParallel not set (defaults to 0)", func(t *testing.T) {
|
||||
config := &Config{
|
||||
Workdir: "testdata",
|
||||
}
|
||||
|
||||
runner, err := New(config)
|
||||
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
assert.NotNil(t, runner)
|
||||
|
||||
runnerImpl, ok := runner.(*runnerImpl)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, 0, runnerImpl.config.MaxParallel)
|
||||
})
|
||||
}
|
||||
|
||||
// TestMaxParallelConcurrencyTracking tests that max-parallel actually limits concurrent execution
|
||||
func TestMaxParallelConcurrencyTracking(t *testing.T) {
|
||||
// This is a unit test for the parallel executor logic
|
||||
// We test that when MaxParallel is set, it limits the number of workers
|
||||
|
||||
var mu sync.Mutex
|
||||
var maxConcurrent int
|
||||
var currentConcurrent int
|
||||
|
||||
// Create a function that tracks concurrent execution
|
||||
trackingFunc := func() {
|
||||
mu.Lock()
|
||||
currentConcurrent++
|
||||
if currentConcurrent > maxConcurrent {
|
||||
maxConcurrent = currentConcurrent
|
||||
}
|
||||
mu.Unlock()
|
||||
|
||||
// Simulate work
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
mu.Lock()
|
||||
currentConcurrent--
|
||||
mu.Unlock()
|
||||
}
|
||||
|
||||
// Run multiple tasks with limited parallelism
|
||||
maxConcurrent = 0
|
||||
currentConcurrent = 0
|
||||
|
||||
// This simulates what NewParallelExecutor does with a semaphore
|
||||
var wg sync.WaitGroup
|
||||
semaphore := make(chan struct{}, 2) // Limit to 2 concurrent
|
||||
|
||||
for range 6 {
|
||||
wg.Go(func() {
|
||||
semaphore <- struct{}{} // Acquire
|
||||
defer func() { <-semaphore }() // Release
|
||||
trackingFunc()
|
||||
})
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
// With a semaphore of 2, max concurrent should be <= 2
|
||||
assert.LessOrEqual(t, maxConcurrent, 2, "Maximum concurrent executions should not exceed limit")
|
||||
assert.GreaterOrEqual(t, maxConcurrent, 1, "Should have at least 1 concurrent execution")
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -163,6 +164,12 @@ func TestGraphEvent(t *testing.T) {
|
||||
assert.Empty(t, plan.Stages)
|
||||
}
|
||||
|
||||
// these two build the same action Dockerfiles into one image tag, so they cannot overlap
|
||||
var sharedImageWorkflows = []string{"local-action-dockerfile", "local-action-via-composite-dockerfile"}
|
||||
|
||||
// bounds concurrent plans: each job holds a network, and the daemon's address pool is finite
|
||||
var planSlots = make(chan struct{}, 4)
|
||||
|
||||
type TestJobFileInfo struct {
|
||||
workdir string
|
||||
workflowPath string
|
||||
@@ -182,12 +189,19 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config
|
||||
|
||||
fullWorkflowPath := filepath.Join(workdir, j.workflowPath)
|
||||
runnerConfig := &Config{
|
||||
Workdir: workdir,
|
||||
BindWorkdir: false,
|
||||
EventName: j.eventName,
|
||||
EventPath: cfg.EventPath,
|
||||
Platforms: j.platforms,
|
||||
ReuseContainers: false,
|
||||
Workdir: workdir,
|
||||
BindWorkdir: false,
|
||||
EventName: j.eventName,
|
||||
EventPath: cfg.EventPath,
|
||||
Platforms: j.platforms,
|
||||
// fixtures reuse workflow and job names, so parallel tests would collide without this
|
||||
ContainerNamePrefix: strings.ReplaceAll(t.Name(), "/", "-"),
|
||||
ReuseContainers: false,
|
||||
// as the shipped runner does, else a fixture asserting a job failure keeps its
|
||||
// container, and its network, on the daemon forever
|
||||
AutoRemove: true,
|
||||
// 0 would run jobs runtime.NumCPU()-wide, making the network peak machine-dependent
|
||||
MaxParallel: 2,
|
||||
ForceRebuild: true,
|
||||
Env: cfg.Env,
|
||||
Secrets: cfg.Secrets,
|
||||
@@ -210,7 +224,11 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config
|
||||
plan, err := planner.PlanEvent(j.eventName)
|
||||
assert.True(t, (err == nil) != (plan == nil), "PlanEvent should return either a plan or an error") //nolint:testifylint // pre-existing issue from nektos/act
|
||||
if err == nil && plan != nil {
|
||||
err = runner.NewPlanExecutor(plan)(ctx)
|
||||
err = func() error {
|
||||
planSlots <- struct{}{}
|
||||
defer func() { <-planSlots }()
|
||||
return runner.NewPlanExecutor(plan)(ctx)
|
||||
}()
|
||||
if j.errorMessage == "" {
|
||||
assert.NoError(t, err, fullWorkflowPath) //nolint:testifylint // pre-existing issue from nektos/act
|
||||
} else {
|
||||
@@ -227,6 +245,7 @@ type TestConfig struct {
|
||||
|
||||
func TestRunEvent(t *testing.T) {
|
||||
requireDocker(t)
|
||||
t.Parallel()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -315,6 +334,9 @@ func TestRunEvent(t *testing.T) {
|
||||
// host /proc bind mounts are Linux-Docker-only
|
||||
requireLinuxDocker(t)
|
||||
}
|
||||
if !slices.Contains(sharedImageWorkflows, table.workflowPath) {
|
||||
t.Parallel()
|
||||
}
|
||||
|
||||
config := &Config{
|
||||
Secrets: table.secrets,
|
||||
@@ -445,6 +467,7 @@ func TestRunEventHostEnvironment(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDryrunEvent(t *testing.T) {
|
||||
t.Parallel()
|
||||
// Dryrun plans without containers or network (shells and local actions only).
|
||||
ctx := common.WithDryrun(context.Background(), true)
|
||||
|
||||
@@ -464,6 +487,7 @@ func TestDryrunEvent(t *testing.T) {
|
||||
|
||||
for _, table := range tables {
|
||||
t.Run(table.workflowPath, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
table.runTest(ctx, t, &Config{})
|
||||
})
|
||||
}
|
||||
@@ -474,33 +498,11 @@ func TestDryrunEvent(t *testing.T) {
|
||||
// workflow's outputs via `needs`).
|
||||
func TestReusableWorkflowCaller(t *testing.T) {
|
||||
requireDocker(t)
|
||||
t.Parallel()
|
||||
table := TestJobFileInfo{workdir, "uses-workflow", "push", "", platforms, map[string]string{"secret": "keep_it_private"}}
|
||||
table.runTest(context.Background(), t, &Config{Secrets: table.secrets})
|
||||
}
|
||||
|
||||
func TestDockerActionForcePullForceRebuild(t *testing.T) {
|
||||
requireDocker(t)
|
||||
requireNetwork(t) // force-pulls a docker action image
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
config := &Config{
|
||||
ForcePull: true,
|
||||
ForceRebuild: true,
|
||||
}
|
||||
|
||||
tables := []TestJobFileInfo{
|
||||
{workdir, "local-action-dockerfile", "push", "", platforms, secrets},
|
||||
{workdir, "local-action-via-composite-dockerfile", "push", "", platforms, secrets},
|
||||
}
|
||||
|
||||
for _, table := range tables {
|
||||
t.Run(table.workflowPath, func(t *testing.T) {
|
||||
table.runTest(ctx, t, config)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type maskJobLoggerFactory struct {
|
||||
Output bytes.Buffer
|
||||
}
|
||||
@@ -513,6 +515,7 @@ func (f *maskJobLoggerFactory) WithJobLogger() *log.Logger {
|
||||
}
|
||||
|
||||
func TestMaskValues(t *testing.T) {
|
||||
t.Parallel()
|
||||
assertNoSecret := func(text, secret string) { //nolint:unparam // pre-existing issue from nektos/act
|
||||
found := strings.Contains(text, "composite secret")
|
||||
if found {
|
||||
@@ -543,6 +546,7 @@ func TestMaskValues(t *testing.T) {
|
||||
|
||||
func TestRunEventSecrets(t *testing.T) {
|
||||
requireDocker(t)
|
||||
t.Parallel()
|
||||
workflowPath := "secrets"
|
||||
|
||||
tjfi := TestJobFileInfo{
|
||||
@@ -598,6 +602,7 @@ func TestRunWithService(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunActionInputs(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireDocker(t)
|
||||
workflowPath := "input-from-cli"
|
||||
|
||||
@@ -617,6 +622,7 @@ func TestRunActionInputs(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunEventPullRequest(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireDocker(t)
|
||||
|
||||
workflowPath := "pull-request"
|
||||
@@ -633,6 +639,7 @@ func TestRunEventPullRequest(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunMatrixWithUserDefinedInclusions(t *testing.T) {
|
||||
t.Parallel()
|
||||
requireDocker(t)
|
||||
workflowPath := "matrix-with-user-inclusions"
|
||||
|
||||
|
||||
@@ -138,9 +138,10 @@ func (sar *stepActionRemote) prepareActionExecutor() common.Executor {
|
||||
})
|
||||
var ntErr common.Executor
|
||||
if err := gitClone(ctx); err != nil {
|
||||
if errors.Is(err, git.ErrShortRef) {
|
||||
var refErr *git.Error
|
||||
if errors.As(err, &refErr) && errors.Is(err, git.ErrShortRef) {
|
||||
return fmt.Errorf("Unable to resolve action `%s`, the provided ref `%s` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `%s` instead",
|
||||
sar.Step.Uses, sar.remoteAction.Ref, err.(*git.Error).Commit())
|
||||
sar.Step.Uses, sar.remoteAction.Ref, refErr.Commit())
|
||||
} else if errors.Is(err, gogit.ErrForceNeeded) { // TODO: figure out if it will be easy to shadow/alias go-git err's
|
||||
ntErr = common.NewInfoExecutor("Non-terminating error while running 'git clone': %v", err)
|
||||
} else {
|
||||
|
||||
@@ -110,10 +110,7 @@ func (sd *stepDocker) newStepContainer(ctx context.Context, image string, cmd, e
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", k, v))
|
||||
}
|
||||
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TOOL_CACHE", "/opt/hostedtoolcache"))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_OS", "Linux"))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_ARCH", container.RunnerArch(ctx)))
|
||||
envList = append(envList, fmt.Sprintf("%s=%s", "RUNNER_TEMP", "/tmp"))
|
||||
envList = append(envList, rc.runnerEnv(ctx)...)
|
||||
|
||||
binds, mounts := rc.GetBindsAndMounts()
|
||||
networkMode := "container:" + rc.jobContainerName()
|
||||
|
||||
70
docs/job-hooks.md
Normal file
70
docs/job-hooks.md
Normal file
@@ -0,0 +1,70 @@
|
||||
# Job hooks
|
||||
|
||||
Job hooks are operator-provided scripts that run **inside the job environment**, before the job's first step and after its last one. They are the equivalent of GitHub's [job hooks](https://docs.github.com/en/actions/how-tos/manage-runners/self-hosted-runners/run-scripts) and are configured under `runner.hooks` in the runner YAML config (see [config.example.yaml](../internal/pkg/config/config.example.yaml)):
|
||||
|
||||
```yaml
|
||||
runner:
|
||||
hooks:
|
||||
job_started: /hooks/started.sh
|
||||
job_completed: /hooks/completed.sh
|
||||
```
|
||||
|
||||
| Setting | Runs |
|
||||
| --- | --- |
|
||||
| `runner.hooks.job_started` | Before the job's first step, before any action is downloaded |
|
||||
| `runner.hooks.job_completed` | After the job's last post step, while the job environment is still up |
|
||||
|
||||
`ACTIONS_RUNNER_HOOK_JOB_STARTED` and `ACTIONS_RUNNER_HOOK_JOB_COMPLETED` are read from the runner's environment (`runner.envs`, `runner.env_file`) when the settings are unset, so a configuration carried over from actions/runner keeps working. The settings take precedence. A workflow cannot point the runner at a different hook: the variables are only read from the runner's own environment, never from the job's.
|
||||
|
||||
Both hooks are **synchronous** and block the job while they run, and a non-zero exit from either one fails the job. There is no `continue-on-error` and no per-hook timeout — the job's own `runner.timeout` is the only bound. The operator is responsible for the hook's resilience; run anything long in the background from within the hook.
|
||||
|
||||
## Where they run
|
||||
|
||||
The hooks run in the same place as the job's steps: inside the job container, or on the host in host mode. The paths are resolved *there*, so the script has to exist in the job image or on the host — a path that only exists on the runner host is not visible to a containerized job. For host-wide cleanup that runs after the job environment is gone, use the [post-task script](post-task-script.md) instead.
|
||||
|
||||
> This is a deliberate difference from actions/runner, which runs its job hooks on the host, outside any container the job declares. Running them where the steps run is what lets a hook prepare the environment the steps actually see.
|
||||
|
||||
The script is run according to its extension:
|
||||
|
||||
| Extension | Command |
|
||||
| --- | --- |
|
||||
| `.sh` | `bash -e <path>` |
|
||||
| `.ps1` | `pwsh -command . '<path>'` |
|
||||
| anything else | the file itself, which needs its own shebang and executable bit |
|
||||
|
||||
As on GitHub, the shell flags applied to `run:` steps are **not** applied to a hook — set `pipefail` or anything else you want inside the script.
|
||||
|
||||
### Docker-in-Docker and Docker-out-of-Docker
|
||||
|
||||
The hook is executed and its files are exchanged over the Docker API, addressed by container ID, so no path is translated between the runner and the daemon. Both setups work unchanged, but they differ in where the hook file has to be:
|
||||
|
||||
- **DinD** — the daemon has its own filesystem. Bake the hook into the job image; a path from the runner's filesystem is not visible to it.
|
||||
- **DooD** — the job container is created by the host's daemon, so a bind mount in `container.options` is resolved against the **host**, not against the runner container. Either bake the hook into the job image, or mount a host directory and add it to `container.valid_volumes`.
|
||||
|
||||
A hook path that does not exist inside the job environment fails the job with `No such file or directory`, naming the path.
|
||||
|
||||
## Environment
|
||||
|
||||
A hook sees the job's environment: the workflow, job and `container:` `env:`, the runner's `envs`, and the `GITHUB_*` context variables, with the same masking applied to its output as to a step's. The step-specific ones (`GITHUB_ACTION`, `GITHUB_OUTPUT`, `GITHUB_STATE`) are not set — a hook is not a step, so `::save-state::` and `::set-output::` have nowhere to go.
|
||||
|
||||
Its stdout is part of the job log, inside a collapsible group, and is scanned for workflow commands. `::add-mask::` registers a value to be masked for the rest of the job, `::set-env::` and `::add-path::` apply to the steps that follow.
|
||||
|
||||
`$GITHUB_ENV` and `$GITHUB_PATH` point at files that are read back after the hook exits, so the file-command form works too:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
echo "REGISTRY_TOKEN=$(fetch-token)" >> "$GITHUB_ENV"
|
||||
echo "/opt/tooling/bin" >> "$GITHUB_PATH"
|
||||
```
|
||||
|
||||
Both files are the hook's own, separate from the per-step ones, so nothing a hook writes is truncated by the first step.
|
||||
|
||||
## Recommendations
|
||||
|
||||
- Keep hooks **fast** and return the right exit code: they are on the critical path of every job, and nothing bounds them.
|
||||
- Use **idempotent** operations, and expect `job_completed` to run after success, failure, and cancellation alike.
|
||||
- Mask anything secret the hook prints or exports with `::add-mask::`.
|
||||
|
||||
## See also
|
||||
|
||||
- [Post-task script](post-task-script.md) — host-side cleanup after the job environment is torn down.
|
||||
@@ -150,6 +150,7 @@ powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File "%~dp0po
|
||||
|
||||
## See also
|
||||
|
||||
- [Job hooks](job-hooks.md) — scripts running inside the job environment, around its steps
|
||||
- [Configuration](../README.md#configuration) — generating and loading `config.yaml`
|
||||
- [config.example.yaml](../internal/pkg/config/config.example.yaml) — all runner options
|
||||
- Bind-workdir idle cleanup (`runner.workdir_cleanup_age`) — separate from this hook; runs only when the runner is idle
|
||||
|
||||
2
go.mod
2
go.mod
@@ -38,6 +38,7 @@ require (
|
||||
github.com/timshannon/bolthold v0.0.0-20240314194003-30aac6950928
|
||||
go.etcd.io/bbolt v1.5.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/term v0.45.0
|
||||
golang.org/x/text v0.40.0
|
||||
@@ -105,7 +106,6 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.44.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/crypto v0.53.0 // indirect
|
||||
golang.org/x/net v0.56.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
|
||||
@@ -64,6 +64,14 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
log.Warn("no labels configured, runner may not be able to pick up jobs")
|
||||
}
|
||||
|
||||
// Before the first Docker API call: the standard library resolves the proxy
|
||||
// environment once. Ungated because host labels still reach the daemon.
|
||||
if dockerSocketPath, err := getDockerSocketPath(cfg.Container.DockerHost); err == nil {
|
||||
run.BypassProxyForDockerHost(dockerSocketPath)
|
||||
} else {
|
||||
log.Debugf("cannot resolve the docker socket path, so Docker API calls are not exempted from the proxy: %v", err)
|
||||
}
|
||||
|
||||
if ls.RequireDocker() || cfg.Container.RequireDocker {
|
||||
// Wait for dockerd be ready
|
||||
if timeout := cfg.Container.DockerTimeout; timeout > 0 {
|
||||
@@ -101,6 +109,7 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
}
|
||||
// if dockerSocketPath passes the check, override DOCKER_HOST with dockerSocketPath
|
||||
os.Setenv("DOCKER_HOST", dockerSocketPath)
|
||||
run.WarnIfDaemonHasNoProxy(ctx)
|
||||
// empty cfg.Container.DockerHost means runner need to find an available docker host automatically
|
||||
// and assign the path to cfg.Container.DockerHost
|
||||
if cfg.Container.DockerHost == "" {
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"gitea.com/gitea/runner/act/common"
|
||||
"gitea.com/gitea/runner/act/model"
|
||||
"gitea.com/gitea/runner/act/runner"
|
||||
"gitea.com/gitea/runner/internal/app/run"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/moby/moby/api/types/container"
|
||||
@@ -415,6 +416,11 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
|
||||
}
|
||||
handler.RegisterJob(actionsRuntimeToken, "__local/__exec")
|
||||
|
||||
// no service aliases: exec builds one config for the whole plan
|
||||
run.BypassProxyForDockerHost(os.Getenv("DOCKER_HOST"))
|
||||
proxyEnv := run.JobProxyEnv(env, env["ACTIONS_CACHE_URL"], nil)
|
||||
maps.Copy(env, proxyEnv)
|
||||
|
||||
// run the plan
|
||||
config := &runner.Config{
|
||||
Workdir: execArgs.Workdir(),
|
||||
@@ -425,6 +431,7 @@ func runExec(ctx context.Context, execArgs *executeArgs) func(cmd *cobra.Command
|
||||
LogOutput: true,
|
||||
JSONLogger: execArgs.jsonLogger,
|
||||
Env: env,
|
||||
ProxyEnv: proxyEnv,
|
||||
Vars: execArgs.LoadVars(),
|
||||
Secrets: execArgs.LoadSecrets(),
|
||||
InsecureSecrets: execArgs.insecureSecrets,
|
||||
|
||||
163
internal/app/run/proxy.go
Normal file
163
internal/app/run/proxy.go
Normal file
@@ -0,0 +1,163 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package run
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.com/gitea/runner/act/container"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/net/http/httpproxy"
|
||||
)
|
||||
|
||||
// proxyFromEnv returns the runner's own proxy configuration, or nil when it has none.
|
||||
func proxyFromEnv() *httpproxy.Config {
|
||||
cfg := httpproxy.FromEnvironment()
|
||||
if cfg.HTTPProxy == "" && cfg.HTTPSProxy == "" {
|
||||
return nil
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// JobProxyEnv returns the proxy variables a job runs with, given what runner.envs already
|
||||
// put in jobEnvs. Gitea is deliberately not made direct, so it stays reachable however the
|
||||
// runner reaches it.
|
||||
func JobProxyEnv(jobEnvs map[string]string, cacheURL string, serviceNames []string) map[string]string {
|
||||
cfg := proxyFromEnv()
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Go bypasses loopback on its own, curl and most other tools in a job do not.
|
||||
direct := append([]string{"localhost", "127.0.0.1", "::1"}, serviceNames...)
|
||||
direct = append(direct, hostOf(cacheURL))
|
||||
|
||||
proxyEnv := map[string]string{}
|
||||
setPair := func(lower, upper, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
// Either spelling in runner.envs takes over both, so the pair cannot disagree.
|
||||
if existing, ok := jobEnvs[lower]; ok {
|
||||
value = existing
|
||||
} else if existing, ok := jobEnvs[upper]; ok {
|
||||
value = existing
|
||||
}
|
||||
proxyEnv[lower], proxyEnv[upper] = value, value
|
||||
}
|
||||
setPair("http_proxy", "HTTP_PROXY", cfg.HTTPProxy)
|
||||
setPair("https_proxy", "HTTPS_PROXY", cfg.HTTPSProxy)
|
||||
|
||||
// no_proxy is merged rather than replaced: the hosts above are structural, and an
|
||||
// operator cannot list the cache server's startup-assigned address in advance.
|
||||
noProxy := appendNoProxy(cfg.NoProxy, direct...)
|
||||
for _, name := range []string{"no_proxy", "NO_PROXY"} {
|
||||
if existing, ok := jobEnvs[name]; ok {
|
||||
noProxy = appendNoProxy(existing, strings.Split(noProxy, ",")...)
|
||||
break
|
||||
}
|
||||
}
|
||||
proxyEnv["no_proxy"], proxyEnv["NO_PROXY"] = noProxy, noProxy
|
||||
return proxyEnv
|
||||
}
|
||||
|
||||
// BypassProxyForDockerHost keeps the runner's Docker API traffic off the proxy: the docker
|
||||
// client proxies every transport that is not a unix socket or a named pipe, so a tcp://
|
||||
// daemon would be reached through a proxy that cannot route to it.
|
||||
//
|
||||
// It must run before the first Docker API call, because the standard library resolves the
|
||||
// proxy environment once.
|
||||
func BypassProxyForDockerHost(dockerHost string) {
|
||||
cfg := proxyFromEnv()
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
host := hostOf(dockerHost)
|
||||
if host == "" {
|
||||
// A unix socket or named pipe is never proxied.
|
||||
return
|
||||
}
|
||||
|
||||
noProxy := appendNoProxy(cfg.NoProxy, host)
|
||||
for _, name := range []string{"no_proxy", "NO_PROXY"} {
|
||||
if err := os.Setenv(name, noProxy); err != nil {
|
||||
log.Warnf("cannot set %s for the runner process: %v", name, err)
|
||||
}
|
||||
}
|
||||
log.Debugf("docker host %s is reached directly, no_proxy is now %q", host, noProxy)
|
||||
}
|
||||
|
||||
// WarnIfDaemonHasNoProxy points at the one part the runner cannot set: the docker daemon
|
||||
// pulls the images, and a daemon in its own container needs its own proxy.
|
||||
func WarnIfDaemonHasNoProxy(ctx context.Context) {
|
||||
if proxyFromEnv() == nil {
|
||||
return
|
||||
}
|
||||
info, err := container.GetHostInfo(ctx)
|
||||
if err != nil {
|
||||
log.Debugf("cannot read the docker daemon's proxy configuration: %v", err)
|
||||
return
|
||||
}
|
||||
if info.HTTPProxy == "" && info.HTTPSProxy == "" {
|
||||
log.Warn("the runner has a proxy but the docker daemon reports none, so image pulls will not use it: https://docs.docker.com/engine/daemon/proxy/")
|
||||
}
|
||||
}
|
||||
|
||||
// proxyPasswords returns the passwords embedded in the runner's proxy URLs, to mask before
|
||||
// a job echoes its environment.
|
||||
func proxyPasswords() []string {
|
||||
cfg := proxyFromEnv()
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var passwords []string
|
||||
for _, raw := range []string{cfg.HTTPProxy, cfg.HTTPSProxy} {
|
||||
parsed, err := url.Parse(raw)
|
||||
if err != nil || parsed.User == nil {
|
||||
continue
|
||||
}
|
||||
if password, ok := parsed.User.Password(); ok && password != "" && !slices.Contains(passwords, password) {
|
||||
passwords = append(passwords, password)
|
||||
}
|
||||
}
|
||||
return passwords
|
||||
}
|
||||
|
||||
// appendNoProxy adds hosts to a no_proxy list, keeping the operator's entries and adding
|
||||
// none twice.
|
||||
func appendNoProxy(noProxy string, hosts ...string) string {
|
||||
entries := []string{}
|
||||
for entry := range strings.SplitSeq(noProxy, ",") {
|
||||
if entry = strings.TrimSpace(entry); entry != "" {
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
}
|
||||
|
||||
for _, host := range hosts {
|
||||
if host == "" || slices.Contains(entries, host) {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, host)
|
||||
}
|
||||
return strings.Join(entries, ",")
|
||||
}
|
||||
|
||||
// hostOf returns the host of a URL without its port, the form a no_proxy entry takes. It is
|
||||
// empty for anything without a network host, such as a unix socket.
|
||||
func hostOf(raw string) string {
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
parsed, err := url.Parse(strings.TrimSuffix(raw, "/"))
|
||||
if err != nil || parsed.Hostname() == "" {
|
||||
return ""
|
||||
}
|
||||
return parsed.Hostname()
|
||||
}
|
||||
192
internal/app/run/proxy_test.go
Normal file
192
internal/app/run/proxy_test.go
Normal file
@@ -0,0 +1,192 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package run
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
// clearProxyEnv starts a test from a runner that has no proxy, whatever the developer's own
|
||||
// environment looks like.
|
||||
func clearProxyEnv(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, name := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "no_proxy", "NO_PROXY"} {
|
||||
t.Setenv(name, "")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJobProxyEnv(t *testing.T) {
|
||||
const loopback = "localhost,127.0.0.1,::1"
|
||||
const proxy = "http://proxy:3128"
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
runner map[string]string // the runner process environment
|
||||
envs map[string]string // what runner.envs already put in the job
|
||||
cacheURL string
|
||||
services []string
|
||||
want map[string]string
|
||||
}{
|
||||
{
|
||||
// The guarantee that makes this safe to ship: a runner without a proxy gives its
|
||||
// jobs nothing at all.
|
||||
name: "runner has no proxy",
|
||||
cacheURL: "http://cache.local:8088/",
|
||||
},
|
||||
{
|
||||
name: "a lone no_proxy is not a proxy",
|
||||
runner: map[string]string{"no_proxy": "example.com"},
|
||||
},
|
||||
{
|
||||
name: "both spellings of every variable",
|
||||
runner: map[string]string{"http_proxy": proxy, "https_proxy": proxy},
|
||||
want: map[string]string{
|
||||
"http_proxy": proxy, "HTTP_PROXY": proxy,
|
||||
"https_proxy": proxy, "HTTPS_PROXY": proxy,
|
||||
"no_proxy": loopback, "NO_PROXY": loopback,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "a variable the runner does not have is omitted",
|
||||
runner: map[string]string{"https_proxy": proxy},
|
||||
want: map[string]string{
|
||||
"https_proxy": proxy, "HTTPS_PROXY": proxy,
|
||||
"no_proxy": loopback, "NO_PROXY": loopback,
|
||||
},
|
||||
},
|
||||
{
|
||||
// The cache server and the service containers are on the local network; Gitea is
|
||||
// not added and keeps being reached through the proxy.
|
||||
name: "hosts the job must reach directly",
|
||||
runner: map[string]string{"http_proxy": proxy, "no_proxy": "internal.example"},
|
||||
cacheURL: "http://192.168.1.10:34567/",
|
||||
services: []string{"postgres", "redis"},
|
||||
want: map[string]string{
|
||||
"http_proxy": proxy, "HTTP_PROXY": proxy,
|
||||
"no_proxy": "internal.example," + loopback + ",postgres,redis,192.168.1.10",
|
||||
"NO_PROXY": "internal.example," + loopback + ",postgres,redis,192.168.1.10",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runner.envs wins, for both spellings",
|
||||
runner: map[string]string{"http_proxy": "http://from-env:3128"},
|
||||
envs: map[string]string{"http_proxy": "http://from-config:3128"},
|
||||
want: map[string]string{
|
||||
"http_proxy": "http://from-config:3128", "HTTP_PROXY": "http://from-config:3128",
|
||||
"no_proxy": loopback, "NO_PROXY": loopback,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runner.envs wins through the upper case spelling too",
|
||||
runner: map[string]string{"http_proxy": "http://from-env:3128"},
|
||||
envs: map[string]string{"HTTP_PROXY": "http://from-config:3128"},
|
||||
want: map[string]string{
|
||||
"http_proxy": "http://from-config:3128", "HTTP_PROXY": "http://from-config:3128",
|
||||
"no_proxy": loopback, "NO_PROXY": loopback,
|
||||
},
|
||||
},
|
||||
{
|
||||
// A runner.envs no_proxy adds to the hosts that must stay direct rather than
|
||||
// replacing them, which would send cache traffic through the proxy.
|
||||
name: "runner.envs no_proxy is merged, not substituted",
|
||||
runner: map[string]string{"http_proxy": proxy},
|
||||
envs: map[string]string{"no_proxy": "operator.example"},
|
||||
cacheURL: "http://192.168.1.10:34567/",
|
||||
want: map[string]string{
|
||||
"http_proxy": proxy, "HTTP_PROXY": proxy,
|
||||
"no_proxy": "operator.example," + loopback + ",192.168.1.10",
|
||||
"NO_PROXY": "operator.example," + loopback + ",192.168.1.10",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "runner.envs NO_PROXY is merged through the upper case spelling too",
|
||||
runner: map[string]string{"http_proxy": proxy},
|
||||
envs: map[string]string{"NO_PROXY": "operator.example"},
|
||||
cacheURL: "http://192.168.1.10:34567/",
|
||||
want: map[string]string{
|
||||
"http_proxy": proxy, "HTTP_PROXY": proxy,
|
||||
"no_proxy": "operator.example," + loopback + ",192.168.1.10",
|
||||
"NO_PROXY": "operator.example," + loopback + ",192.168.1.10",
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
clearProxyEnv(t)
|
||||
for name, value := range tc.runner {
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
assert.Equal(t, tc.want, JobProxyEnv(tc.envs, tc.cacheURL, tc.services))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// docker-in-docker over tcp: the docker client would otherwise send API calls to a proxy
|
||||
// that cannot route to the daemon.
|
||||
func TestBypassProxyForDockerHost(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
httpProxy string
|
||||
dockerHost string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "tcp daemon is added",
|
||||
httpProxy: "http://proxy:3128",
|
||||
dockerHost: "tcp://docker:2375",
|
||||
want: "internal.example,docker",
|
||||
},
|
||||
{
|
||||
name: "unix socket is left alone",
|
||||
httpProxy: "http://proxy:3128",
|
||||
dockerHost: "unix:///var/run/docker.sock",
|
||||
want: "internal.example",
|
||||
},
|
||||
{
|
||||
name: "nothing happens without a proxy",
|
||||
dockerHost: "tcp://docker:2375",
|
||||
want: "internal.example",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
clearProxyEnv(t)
|
||||
t.Setenv("no_proxy", "internal.example")
|
||||
if tc.httpProxy != "" {
|
||||
t.Setenv("http_proxy", tc.httpProxy)
|
||||
}
|
||||
|
||||
BypassProxyForDockerHost(tc.dockerHost)
|
||||
|
||||
assert.Equal(t, tc.want, os.Getenv("no_proxy"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyPasswords(t *testing.T) {
|
||||
clearProxyEnv(t)
|
||||
t.Setenv("http_proxy", "http://user:hunter2@proxy:3128")
|
||||
t.Setenv("https_proxy", "http://user:s3cret@proxy:3128")
|
||||
assert.Equal(t, []string{"hunter2", "s3cret"}, proxyPasswords())
|
||||
|
||||
t.Setenv("https_proxy", "http://proxy:3128")
|
||||
assert.Equal(t, []string{"hunter2"}, proxyPasswords())
|
||||
}
|
||||
|
||||
func TestAppendNoProxy(t *testing.T) {
|
||||
assert.Equal(t, "a.example,b.example", appendNoProxy(" a.example , b.example "))
|
||||
// a host already listed is not repeated
|
||||
assert.Equal(t, "cache.local", appendNoProxy("cache.local", "cache.local"))
|
||||
assert.Empty(t, appendNoProxy("", "", ""))
|
||||
}
|
||||
|
||||
func TestHostOf(t *testing.T) {
|
||||
assert.Equal(t, "cache.local", hostOf("http://cache.local:8088/"))
|
||||
assert.Equal(t, "192.168.1.10", hostOf("http://192.168.1.10:34567"))
|
||||
// nothing to bypass for a socket path or an unparseable URL
|
||||
assert.Empty(t, hostOf("unix:///var/run/docker.sock"))
|
||||
assert.Empty(t, hostOf("cache.local:8088"))
|
||||
assert.Empty(t, hostOf("://nope"))
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -53,6 +54,7 @@ func RunnerCapabilities() []string {
|
||||
// Runner runs the pipeline.
|
||||
type Runner struct {
|
||||
name string
|
||||
uuid string
|
||||
|
||||
cfg *config.Config
|
||||
|
||||
@@ -118,6 +120,7 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
|
||||
runner := &Runner{
|
||||
name: reg.Name,
|
||||
uuid: reg.UUID,
|
||||
cfg: cfg,
|
||||
client: cli,
|
||||
labels: ls,
|
||||
@@ -129,6 +132,9 @@ func NewRunner(cfg *config.Config, reg *config.Registration, cli client.Client)
|
||||
return runner
|
||||
}
|
||||
|
||||
// removeOrphanNetworks is a variable so tests can substitute one that needs no Docker daemon.
|
||||
var removeOrphanNetworks = container.RemoveOrphanNetworks
|
||||
|
||||
// OnIdle performs lightweight maintenance during polling idle windows.
|
||||
// It runs synchronously on the poller goroutine; shouldRunIdleCleanup
|
||||
// throttles invocations to runner.idle_cleanup_interval so the impact on
|
||||
@@ -150,6 +156,21 @@ func (r *Runner) OnIdle(ctx context.Context) {
|
||||
if hostRoot := filepath.FromSlash(r.cfg.Host.WorkdirParent); hostRoot != "" {
|
||||
r.cleanupStaleDirs(ctx, hostRoot, isHostScratchDir)
|
||||
}
|
||||
r.cleanupOrphanNetworks(ctx)
|
||||
}
|
||||
|
||||
// cleanupOrphanNetworks reclaims the per-job networks of jobs this runner did not live to
|
||||
// tear down. A labelled network with no containers on it is finished with, and as for the
|
||||
// directories above, a task beginning during the pass is safe because the cutoff keeps a
|
||||
// network it has created but not yet attached a container to out of scope.
|
||||
func (r *Runner) cleanupOrphanNetworks(ctx context.Context) {
|
||||
if r.uuid == "" || !r.labels.RequireDocker() && !r.cfg.Container.RequireDocker {
|
||||
return
|
||||
}
|
||||
cutoff := r.now().Add(-r.cfg.Runner.WorkdirCleanupAge)
|
||||
if err := removeOrphanNetworks(ctx, r.uuid, cutoff); err != nil {
|
||||
log.Warnf("failed to clean up networks left behind by earlier jobs: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Runner) shouldRunIdleCleanup() bool {
|
||||
@@ -268,7 +289,8 @@ func (r *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, r.cfg.Runner.Timeout)
|
||||
defer cancel()
|
||||
reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg)
|
||||
// A proxy URL may carry credentials, and every job is given it; keep them out of the log.
|
||||
reporter := report.NewReporter(ctx, cancel, r.client, task, r.cfg, proxyPasswords()...)
|
||||
var runErr error
|
||||
defer func() {
|
||||
r.runningCount.Add(-1)
|
||||
@@ -341,6 +363,11 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
taskContext := task.Context.Fields
|
||||
envs := r.cloneEnvs()
|
||||
|
||||
// Added per task because this job's service containers must be reached directly, and
|
||||
// act reaches them by their workflow key.
|
||||
proxyEnv := JobProxyEnv(envs, envs["ACTIONS_CACHE_URL"], slices.Sorted(maps.Keys(job.Services)))
|
||||
maps.Copy(envs, proxyEnv)
|
||||
|
||||
if r.capabilities != "" {
|
||||
envs["GITEA_ACTIONS_CAPABILITIES"] = r.capabilities
|
||||
}
|
||||
@@ -450,6 +477,7 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
LogOutput: true,
|
||||
JSONLogger: false,
|
||||
Env: envs,
|
||||
ProxyEnv: proxyEnv,
|
||||
Secrets: task.Secrets,
|
||||
GitHubInstance: strings.TrimSuffix(r.client.Address(), "/"),
|
||||
AutoRemove: true,
|
||||
@@ -464,6 +492,9 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
ContainerNetworkCreateOptions: container.NewDockerNetworkCreateExecutorInput{
|
||||
EnableIPv4: r.cfg.Container.NetworkCreateOptions.EnableIPv4,
|
||||
EnableIPv6: r.cfg.Container.NetworkCreateOptions.EnableIPv6,
|
||||
// so a network this job leaks, if the runner dies before its teardown, can be
|
||||
// told apart from one belonging to another runner on the same daemon
|
||||
RunnerUUID: r.uuid,
|
||||
},
|
||||
ContainerOptions: r.cfg.Container.Options,
|
||||
ContainerDaemonSocket: r.cfg.Container.DockerHost,
|
||||
@@ -471,6 +502,8 @@ func (r *Runner) run(ctx context.Context, task *runnerv1.Task, reporter *report.
|
||||
DefaultActionInstance: r.getDefaultActionsURL(task),
|
||||
DefaultActionInstanceIsSelfHosted: r.isSelfHostedActionsURL(task),
|
||||
PlatformPicker: r.labels.PickPlatform,
|
||||
JobStartedHook: r.cfg.Runner.Hooks.JobStarted,
|
||||
JobCompletedHook: r.cfg.Runner.Hooks.JobCompleted,
|
||||
Vars: task.Vars,
|
||||
ValidVolumes: r.cfg.Container.ValidVolumes,
|
||||
InsecureSkipTLS: r.cfg.Runner.Insecure,
|
||||
|
||||
@@ -297,3 +297,37 @@ func TestRunnerOnIdleSkipsWhenAlreadyCancelled(t *testing.T) {
|
||||
|
||||
assert.DirExists(t, stale)
|
||||
}
|
||||
|
||||
// The idle sweep reclaims the docker networks of jobs this runner did not live to tear down,
|
||||
// and stays out of the way of runners that share the daemon but not the registration.
|
||||
func TestRunnerOnIdleRemovesOrphanNetworks(t *testing.T) {
|
||||
now := time.Date(2026, time.April, 29, 20, 0, 0, 0, time.UTC)
|
||||
cfg := &config.Config{
|
||||
Container: config.Container{RequireDocker: true},
|
||||
Runner: config.Runner{
|
||||
WorkdirCleanupAge: 24 * time.Hour,
|
||||
IdleCleanupInterval: time.Minute,
|
||||
},
|
||||
}
|
||||
|
||||
var swept []string
|
||||
var sweptCutoff time.Time
|
||||
origRemoveOrphanNetworks := removeOrphanNetworks
|
||||
removeOrphanNetworks = func(_ context.Context, runnerUUID string, createdBefore time.Time) error {
|
||||
swept = append(swept, runnerUUID)
|
||||
sweptCutoff = createdBefore
|
||||
return nil
|
||||
}
|
||||
t.Cleanup(func() { removeOrphanNetworks = origRemoveOrphanNetworks })
|
||||
|
||||
r := &Runner{uuid: "runner-1", cfg: cfg, now: func() time.Time { return now }}
|
||||
r.OnIdle(context.Background())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
// a network of a job starting during the pass is younger than this and so out of scope
|
||||
assert.Equal(t, now.Add(-24*time.Hour), sweptCutoff)
|
||||
|
||||
// a host-only runner has no daemon to sweep
|
||||
hostOnly := &Runner{uuid: "runner-2", cfg: &config.Config{Runner: cfg.Runner}, now: func() time.Time { return now }}
|
||||
hostOnly.OnIdle(context.Background())
|
||||
assert.Equal(t, []string{"runner-1"}, swept)
|
||||
}
|
||||
|
||||
@@ -93,6 +93,24 @@ func TestNewRunnerInitializesLabelsAndEnvironment(t *testing.T) {
|
||||
require.Nil(t, r.cacheHandler)
|
||||
}
|
||||
|
||||
// Proxy variables are assembled per task, because a job's service containers have to be
|
||||
// reached directly and they are only known once the workflow is parsed.
|
||||
func TestNewRunnerLeavesProxyToTheTask(t *testing.T) {
|
||||
clearProxyEnv(t)
|
||||
t.Setenv("http_proxy", "http://proxy:3128")
|
||||
|
||||
cfg := &config.Config{}
|
||||
cfg.Cache.ExternalServer = "http://cache.local:8088/"
|
||||
reg := &config.Registration{Name: "runner"}
|
||||
cli := clientmocks.NewClient(t)
|
||||
cli.On("Address").Return("https://gitea.example/").Maybe()
|
||||
|
||||
r := NewRunner(cfg, reg, cli)
|
||||
|
||||
require.NotContains(t, r.envs, "http_proxy")
|
||||
require.NotContains(t, r.envs, "no_proxy")
|
||||
}
|
||||
|
||||
func taskWithDefaultActionsURL(url string) *runnerv1.Task {
|
||||
return &runnerv1.Task{
|
||||
Context: &structpb.Struct{
|
||||
|
||||
@@ -15,6 +15,9 @@ runner:
|
||||
# Where to store the registration result.
|
||||
file: .runner
|
||||
# Execute how many tasks concurrently at the same time.
|
||||
# With `container.network` empty, each concurrent docker job takes a subnet from the
|
||||
# daemon's address pool, so a high capacity can exhaust it. See `default-address-pools`
|
||||
# in the docker daemon config.
|
||||
capacity: 1
|
||||
# Extra environment variables to run jobs.
|
||||
envs:
|
||||
@@ -43,9 +46,12 @@ runner:
|
||||
# While idle, remove stale bind-workdir task directories and orphaned host-mode
|
||||
# scratch directories (left behind when a host cleanup delete stalls) older than
|
||||
# this duration. Setting either workdir_cleanup_age or idle_cleanup_interval to 0
|
||||
# (or any non-positive value) disables stale-directory cleanup entirely.
|
||||
# (or any non-positive value) disables stale-directory cleanup entirely, along with
|
||||
# the docker network cleanup below.
|
||||
workdir_cleanup_age: 24h
|
||||
# Cadence for the idle stale-directory cleanup pass.
|
||||
# Cadence for the idle cleanup pass. Besides the directories above, on runners that use
|
||||
# docker it removes the per-job networks of jobs this runner did not live to tear down,
|
||||
# which would otherwise hold a subnet of the daemon address pool until the host is rebuilt.
|
||||
idle_cleanup_interval: 10m
|
||||
# The base interval for periodic log flush to the Gitea instance.
|
||||
# Logs may be sent earlier if the buffer reaches log_report_batch_size
|
||||
@@ -104,6 +110,14 @@ runner:
|
||||
post_task_script: ''
|
||||
# Hard limit on post_task_script runtime. Default if omitted: 5m.
|
||||
post_task_script_timeout: 5m
|
||||
# Scripts run inside the job environment before the job's first step and after its last
|
||||
# one, the equivalent of GitHub's ACTIONS_RUNNER_HOOK_JOB_STARTED and
|
||||
# ACTIONS_RUNNER_HOOK_JOB_COMPLETED, which are read when these are unset. The paths are
|
||||
# resolved inside the job environment. Either one failing fails the job.
|
||||
# Full guide: docs/job-hooks.md
|
||||
hooks:
|
||||
job_started: ''
|
||||
job_completed: ''
|
||||
|
||||
cache:
|
||||
# Enable the built-in cache server (used by actions/cache and similar actions).
|
||||
@@ -153,13 +167,19 @@ container:
|
||||
# network_create_options only apply when `network` is left empty and the runner
|
||||
# auto-creates a per-job network that does not already exist. They have no effect
|
||||
# when a custom `network` name is set, because that network is used as-is and never
|
||||
# created by the runner. Omit the entire block to use Docker's defaults.
|
||||
# created by the runner. Omit the entire block to use Docker's defaults. An auto-created
|
||||
# network is labelled com.gitea.runner.uuid=<this runner's uuid>, which is how the idle
|
||||
# cleanup tells its own leftovers apart from those of other runners on the same daemon.
|
||||
network_create_options:
|
||||
enable_ipv4: true # Omit to use Docker's default (IPv4 enabled). Set false to disable IPv4.
|
||||
enable_ipv6: false # Omit to use Docker's default (IPv6 disabled). Enabling it requires dockerd started with --ipv6.
|
||||
# Whether to use privileged mode or not when launching task containers (privileged mode is required for Docker-in-Docker).
|
||||
privileged: false
|
||||
# Any other options to be used when the container is started (e.g., --add-host=my.gitea.url:host-gateway).
|
||||
# Any other options to be used when the container is started, for example:
|
||||
# options: --add-host=my.gitea.url:host-gateway
|
||||
# A volume declared here replaces the one the runner mounts on the same container path, so the
|
||||
# tool cache can be kept on the host. Its source must also be allowed by valid_volumes below:
|
||||
# options: --volume /host/toolcache:/opt/hostedtoolcache
|
||||
options:
|
||||
# The parent directory of a job's working directory.
|
||||
# NOTE: There is no need to add the first '/' of the path as runner will add it automatically.
|
||||
@@ -185,6 +205,9 @@ container:
|
||||
docker_host: ""
|
||||
# Pull docker image(s) even if already present.
|
||||
# Defaults to false when the key is omitted.
|
||||
# Two exceptions: an image pinned by digest (image@sha256:...) cannot change, so it is never
|
||||
# re-pulled, and a pull that fails while a copy is already on the host does not fail the job,
|
||||
# which runs on that copy with a warning in its log.
|
||||
force_pull: false
|
||||
# Rebuild docker image(s) even if already present
|
||||
force_rebuild: false
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
@@ -41,7 +42,7 @@ type Runner struct {
|
||||
FetchInterval time.Duration `yaml:"fetch_interval"` // FetchInterval specifies the interval duration for fetching resources.
|
||||
FetchIntervalMax time.Duration `yaml:"fetch_interval_max"` // FetchIntervalMax specifies the maximum backoff interval when idle.
|
||||
WorkdirCleanupAge time.Duration `yaml:"workdir_cleanup_age"` // WorkdirCleanupAge removes stale bind-workdir task directories and orphaned host-mode scratch dirs older than this duration during idle cleanup.
|
||||
IdleCleanupInterval time.Duration `yaml:"idle_cleanup_interval"` // IdleCleanupInterval runs stale-directory cleanup periodically while the runner is idle. Set to 0 to disable cleanup cadence.
|
||||
IdleCleanupInterval time.Duration `yaml:"idle_cleanup_interval"` // IdleCleanupInterval runs the idle cleanup (stale directories and orphaned docker networks) periodically while the runner is idle. Set to 0 to disable cleanup cadence.
|
||||
LogReportInterval time.Duration `yaml:"log_report_interval"` // LogReportInterval specifies the base interval for periodic log flush.
|
||||
LogReportMaxLatency time.Duration `yaml:"log_report_max_latency"` // LogReportMaxLatency specifies the max time a log row can wait before being sent.
|
||||
LogReportBatchSize int `yaml:"log_report_batch_size"` // LogReportBatchSize triggers immediate log flush when buffer reaches this size.
|
||||
@@ -54,6 +55,13 @@ type Runner struct {
|
||||
AllocatePTY bool `yaml:"allocate_pty"` // AllocatePTY allocates a pseudo-TTY for each step's process. Default is false, matching GitHub's actions/runner. Enable only for jobs that need an interactive terminal; tools like docker build emit redrawing progress frames into the captured log when a TTY is present. Applies to both host and docker backends.
|
||||
PostTaskScript string `yaml:"post_task_script"` // PostTaskScript is the path to an executable script run on the host after each task's cleanup completes. Empty disables the hook. On Windows use .exe/.bat/.cmd; PowerShell (.ps1) is not supported yet as the configured path.
|
||||
PostTaskScriptTimeout time.Duration `yaml:"post_task_script_timeout"` // PostTaskScriptTimeout caps how long the post-task script may run. Default is 5m when post_task_script is set.
|
||||
Hooks RunnerHooks `yaml:"hooks"` // Hooks are scripts run inside the job environment around the job's steps.
|
||||
}
|
||||
|
||||
// RunnerHooks represents the scripts run inside the job environment around the job's steps.
|
||||
type RunnerHooks struct {
|
||||
JobStarted string `yaml:"job_started"` // JobStarted is the path of a script run before the job's first step. Falls back to ACTIONS_RUNNER_HOOK_JOB_STARTED; a failure fails the job.
|
||||
JobCompleted string `yaml:"job_completed"` // JobCompleted is the path of a script run after the job's last step, while the job environment is still up. Falls back to ACTIONS_RUNNER_HOOK_JOB_COMPLETED; a failure fails the job.
|
||||
}
|
||||
|
||||
// Cache represents the configuration for caching.
|
||||
@@ -78,7 +86,7 @@ type Container struct {
|
||||
WorkdirParent string `yaml:"workdir_parent"` // WorkdirParent specifies the parent directory for the container's working directory.
|
||||
ValidVolumes []string `yaml:"valid_volumes"` // ValidVolumes specifies the volumes (including bind mounts) can be mounted to containers.
|
||||
DockerHost string `yaml:"docker_host"` // DockerHost specifies the Docker host. It overrides the value specified in environment variable DOCKER_HOST.
|
||||
ForcePull bool `yaml:"force_pull"` // Pull docker image(s) even if already present
|
||||
ForcePull bool `yaml:"force_pull"` // Pull docker image(s) even if already present, except digest-pinned ones. A pull that fails while a local copy exists is a warning, not a job failure.
|
||||
ForceRebuild bool `yaml:"force_rebuild"` // Rebuild docker image(s) even if already present
|
||||
RequireDocker bool `yaml:"require_docker"` // Always require a reachable docker daemon, even if not required by runner
|
||||
DockerTimeout time.Duration `yaml:"docker_timeout"` // Timeout to wait for the docker daemon to be reachable, if docker is required by require_docker or runner
|
||||
@@ -136,6 +144,7 @@ func LoadDefault(file string) (*Config, error) {
|
||||
if err := yaml.Unmarshal(content, cfg); err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q: %w", file, err)
|
||||
}
|
||||
warnUnknownKeys(file, content)
|
||||
definedRunnerKeys, err = definedRunnerConfigKeys(content)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse config file %q for defaults metadata: %w", file, err)
|
||||
@@ -281,6 +290,21 @@ func LoadDefault(file string) (*Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// warnUnknownKeys reports keys the config does not define, which are otherwise ignored
|
||||
// without a trace. It only warns, so a config carrying keys from another runner version
|
||||
// still loads.
|
||||
func warnUnknownKeys(file string, content []byte) {
|
||||
decoder := yaml.NewDecoder(bytes.NewReader(content))
|
||||
decoder.KnownFields(true)
|
||||
|
||||
var typeErr *yaml.TypeError
|
||||
if err := decoder.Decode(&Config{}); errors.As(err, &typeErr) {
|
||||
for _, message := range typeErr.Errors {
|
||||
log.Warnf("config file %q: %s, it will be ignored", file, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func definedRunnerConfigKeys(content []byte) (map[string]bool, error) {
|
||||
var root yaml.Node
|
||||
if err := yaml.Unmarshal(content, &root); err != nil {
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sirupsen/logrus/hooks/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -139,9 +140,6 @@ runner:
|
||||
assert.Equal(t, -1*time.Second, cfg.Runner.IdleCleanupInterval)
|
||||
}
|
||||
|
||||
// TestLoadDefault_MalformedYAMLReturnsParseError pins the error surfaced for
|
||||
// invalid YAML to the canonical "parse config file" message rather than the
|
||||
// "for defaults metadata" variant — i.e. the main yaml.Unmarshal runs first.
|
||||
func TestLoadDefault_LoadsPostTaskScript(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
@@ -170,6 +168,25 @@ runner:
|
||||
assert.Equal(t, 5*time.Minute, cfg.Runner.PostTaskScriptTimeout)
|
||||
}
|
||||
|
||||
func TestLoadDefault_LoadsJobHooks(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte(`
|
||||
runner:
|
||||
hooks:
|
||||
job_started: /hooks/started.sh
|
||||
job_completed: /hooks/completed.sh
|
||||
`), 0o600))
|
||||
|
||||
cfg, err := LoadDefault(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "/hooks/started.sh", cfg.Runner.Hooks.JobStarted)
|
||||
assert.Equal(t, "/hooks/completed.sh", cfg.Runner.Hooks.JobCompleted)
|
||||
}
|
||||
|
||||
// TestLoadDefault_MalformedYAMLReturnsParseError pins the error surfaced for
|
||||
// invalid YAML to the canonical "parse config file" message rather than the
|
||||
// "for defaults metadata" variant — i.e. the main yaml.Unmarshal runs first.
|
||||
func TestLoadDefault_MalformedYAMLReturnsParseError(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
@@ -181,6 +198,21 @@ func TestLoadDefault_MalformedYAMLReturnsParseError(t *testing.T) {
|
||||
assert.NotContains(t, err.Error(), "defaults metadata")
|
||||
}
|
||||
|
||||
func TestLoadDefault_WarnsOnUnknownKeysButStillLoads(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "config.yaml")
|
||||
require.NoError(t, os.WriteFile(path, []byte("container:\n volumes:\n - /host:/ctr\n privileged: true\n"), 0o600))
|
||||
|
||||
hook := test.NewGlobal()
|
||||
defer hook.Reset()
|
||||
|
||||
cfg, err := LoadDefault(path)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, cfg.Container.Privileged)
|
||||
require.Len(t, hook.Entries, 1)
|
||||
assert.Contains(t, hook.LastEntry().Message, "field volumes not found")
|
||||
}
|
||||
|
||||
func TestContainerNetworkCreateOptions(t *testing.T) {
|
||||
// Verify that the enable_ipv4/enable_ipv6 YAML keys unmarshal into the *bool fields,
|
||||
// distinguishing an explicit true/false from an omitted key (nil). A nil here is
|
||||
|
||||
38
internal/pkg/lock/keyed.go
Normal file
38
internal/pkg/lock/keyed.go
Normal file
@@ -0,0 +1,38 @@
|
||||
// Copyright 2026 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package lock
|
||||
|
||||
import "sync"
|
||||
|
||||
// Keyed serializes the work done under one key while letting different keys run in
|
||||
// parallel. Its zero value is ready to use, and it keeps one mutex per key it has seen.
|
||||
type Keyed[K comparable] struct {
|
||||
mu sync.Mutex
|
||||
mutexes map[K]*sync.Mutex
|
||||
}
|
||||
|
||||
// Lock locks the mutex for key and returns its unlock function.
|
||||
func (kl *Keyed[K]) Lock(key K) func() {
|
||||
kl.mu.Lock()
|
||||
if kl.mutexes == nil {
|
||||
kl.mutexes = map[K]*sync.Mutex{}
|
||||
}
|
||||
lock := kl.mutexes[key]
|
||||
if lock == nil {
|
||||
lock = &sync.Mutex{}
|
||||
kl.mutexes[key] = lock
|
||||
}
|
||||
kl.mu.Unlock()
|
||||
|
||||
lock.Lock()
|
||||
return lock.Unlock
|
||||
}
|
||||
|
||||
// Delete drops the mutex for key. Holders of it keep working, they just no longer share it
|
||||
// with a later Lock of the same key.
|
||||
func (kl *Keyed[K]) Delete(key K) {
|
||||
kl.mu.Lock()
|
||||
delete(kl.mutexes, key)
|
||||
kl.mu.Unlock()
|
||||
}
|
||||
@@ -32,6 +32,12 @@ const (
|
||||
maxOutputValueLen = 1024 * 1024 // 1 MiB
|
||||
)
|
||||
|
||||
// jobOutput is a job output on its way to the server, sent once the server has acknowledged it.
|
||||
type jobOutput struct {
|
||||
value string
|
||||
sent bool
|
||||
}
|
||||
|
||||
type Reporter struct {
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
@@ -53,7 +59,8 @@ type Reporter struct {
|
||||
state *runnerv1.TaskState
|
||||
stateChanged bool
|
||||
stateMu sync.RWMutex
|
||||
outputs sync.Map
|
||||
outputsMu sync.Mutex
|
||||
outputs map[string]jobOutput
|
||||
daemon chan struct{}
|
||||
heartbeatStop chan struct{}
|
||||
heartbeatStopOnce sync.Once
|
||||
@@ -79,8 +86,13 @@ type Reporter struct {
|
||||
stopCommandEndToken string
|
||||
}
|
||||
|
||||
func NewReporter(ctx context.Context, cancel context.CancelFunc, client client.Client, task *runnerv1.Task, cfg *config.Config) *Reporter {
|
||||
// extraMasks are values known before the job starts that are not among its secrets, such as
|
||||
// the password in the runner's proxy URL.
|
||||
func NewReporter(ctx context.Context, cancel context.CancelFunc, client client.Client, task *runnerv1.Task, cfg *config.Config, extraMasks ...string) *Reporter {
|
||||
var oldnew []string
|
||||
for _, v := range extraMasks {
|
||||
oldnew = runner.AppendSecretMasker(oldnew, v)
|
||||
}
|
||||
if v := task.Context.Fields["token"].GetStringValue(); v != "" {
|
||||
oldnew = runner.AppendSecretMasker(oldnew, v)
|
||||
}
|
||||
@@ -394,7 +406,12 @@ func (r *Reporter) logf(format string, a ...any) {
|
||||
func (r *Reporter) SetOutputs(outputs map[string]string) {
|
||||
r.stateMu.Lock()
|
||||
defer r.stateMu.Unlock()
|
||||
r.outputsMu.Lock()
|
||||
defer r.outputsMu.Unlock()
|
||||
|
||||
if r.outputs == nil {
|
||||
r.outputs = map[string]jobOutput{}
|
||||
}
|
||||
for k, v := range outputs {
|
||||
if l := len(k); l > maxOutputKeyLen {
|
||||
log.Warnf("ignore output %q because the key is too long: %d > %d", k, l, maxOutputKeyLen)
|
||||
@@ -406,10 +423,9 @@ func (r *Reporter) SetOutputs(outputs map[string]string) {
|
||||
r.logf("ignore output %q because the value is too long: %d > %d", k, l, maxOutputValueLen)
|
||||
continue
|
||||
}
|
||||
if _, ok := r.outputs.Load(k); ok {
|
||||
continue
|
||||
if _, ok := r.outputs[k]; !ok {
|
||||
r.outputs[k] = jobOutput{value: v}
|
||||
}
|
||||
r.outputs.Store(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -574,14 +590,14 @@ func (r *Reporter) ReportState(reportResult bool) error {
|
||||
r.clientM.Lock()
|
||||
defer r.clientM.Unlock()
|
||||
|
||||
// Build the outputs map first (single Range pass instead of two).
|
||||
outputs := make(map[string]string)
|
||||
r.outputs.Range(func(k, v any) bool {
|
||||
if val, ok := v.(string); ok {
|
||||
outputs[k.(string)] = val
|
||||
r.outputsMu.Lock()
|
||||
for key, out := range r.outputs {
|
||||
if !out.sent {
|
||||
outputs[key] = out.value
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
r.outputsMu.Unlock()
|
||||
|
||||
// Consume stateChanged atomically with the snapshot; restored on error
|
||||
// below so a concurrent Fire() during UpdateTask isn't silently lost.
|
||||
@@ -594,7 +610,8 @@ func (r *Reporter) ReportState(reportResult bool) error {
|
||||
r.stateMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
state := proto.Clone(r.state).(*runnerv1.TaskState)
|
||||
state := &runnerv1.TaskState{}
|
||||
proto.Merge(state, r.state)
|
||||
r.stateChanged = false
|
||||
r.stateMu.Unlock()
|
||||
|
||||
@@ -622,21 +639,23 @@ func (r *Reporter) ReportState(reportResult bool) error {
|
||||
metrics.ReportStateTotal.WithLabelValues(metrics.LabelResultSuccess).Inc()
|
||||
r.lastReportedAtNanos.Store(time.Now().UnixNano())
|
||||
|
||||
var noSent []string
|
||||
r.outputsMu.Lock()
|
||||
for _, k := range resp.Msg.SentOutputs {
|
||||
r.outputs.Store(k, struct{}{})
|
||||
if _, ok := r.outputs[k]; ok {
|
||||
r.outputs[k] = jobOutput{sent: true}
|
||||
}
|
||||
}
|
||||
for key, out := range r.outputs {
|
||||
if !out.sent {
|
||||
noSent = append(noSent, key)
|
||||
}
|
||||
}
|
||||
r.outputsMu.Unlock()
|
||||
|
||||
if resp.Msg.State != nil && resp.Msg.State.Result == runnerv1.Result_RESULT_CANCELLED {
|
||||
r.cancel()
|
||||
}
|
||||
|
||||
var noSent []string
|
||||
r.outputs.Range(func(k, v any) bool {
|
||||
if _, ok := v.(string); ok {
|
||||
noSent = append(noSent, k.(string))
|
||||
}
|
||||
return true
|
||||
})
|
||||
if len(noSent) > 0 {
|
||||
return fmt.Errorf("there are still outputs that have not been sent: %v", noSent)
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -1011,33 +1013,59 @@ func TestReporter_SetOutputs(t *testing.T) {
|
||||
r := &Reporter{state: &runnerv1.TaskState{}}
|
||||
|
||||
r.SetOutputs(map[string]string{"foo": "bar"})
|
||||
got, ok := r.outputs.Load("foo")
|
||||
got, ok := r.outputs["foo"]
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, "bar", got)
|
||||
assert.Equal(t, "bar", got.value)
|
||||
|
||||
// first value wins: a later write to the same key is ignored
|
||||
r.SetOutputs(map[string]string{"foo": "baz"})
|
||||
got, _ = r.outputs.Load("foo")
|
||||
assert.Equal(t, "bar", got)
|
||||
assert.Equal(t, "bar", r.outputs["foo"].value)
|
||||
|
||||
// keys longer than maxOutputKeyLen are dropped
|
||||
longKey := strings.Repeat("k", maxOutputKeyLen+1)
|
||||
r.SetOutputs(map[string]string{longKey: "v"})
|
||||
_, ok = r.outputs.Load(longKey)
|
||||
_, ok = r.outputs[longKey]
|
||||
assert.False(t, ok)
|
||||
|
||||
// values longer than maxOutputValueLen are dropped
|
||||
longValue := strings.Repeat("v", maxOutputValueLen+1)
|
||||
r.SetOutputs(map[string]string{"big": longValue})
|
||||
_, ok = r.outputs.Load("big")
|
||||
_, ok = r.outputs["big"]
|
||||
assert.False(t, ok)
|
||||
|
||||
// a value at exactly the limit is still stored
|
||||
maxValue := strings.Repeat("v", maxOutputValueLen)
|
||||
r.SetOutputs(map[string]string{"atlimit": maxValue})
|
||||
got, ok = r.outputs.Load("atlimit")
|
||||
got, ok = r.outputs["atlimit"]
|
||||
require.True(t, ok)
|
||||
assert.Len(t, got, maxOutputValueLen)
|
||||
assert.Len(t, got.value, maxOutputValueLen)
|
||||
}
|
||||
|
||||
// An output the server acknowledged is not reported again.
|
||||
func TestReporter_OutputsSentOnce(t *testing.T) {
|
||||
client := mocks.NewClient(t)
|
||||
var reported []map[string]string
|
||||
client.On("UpdateTask", mock.Anything, mock.Anything).Return(
|
||||
func(_ context.Context, req *connect_go.Request[runnerv1.UpdateTaskRequest]) (*connect_go.Response[runnerv1.UpdateTaskResponse], error) {
|
||||
reported = append(reported, req.Msg.Outputs)
|
||||
return connect_go.NewResponse(&runnerv1.UpdateTaskResponse{SentOutputs: slices.Collect(maps.Keys(req.Msg.Outputs))}), nil
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
taskCtx, err := structpb.NewStruct(map[string]any{})
|
||||
require.NoError(t, err)
|
||||
cfg, _ := config.LoadDefault("")
|
||||
r := NewReporter(ctx, cancel, client, &runnerv1.Task{Context: taskCtx}, cfg)
|
||||
|
||||
r.SetOutputs(map[string]string{"foo": "bar"})
|
||||
require.NoError(t, r.ReportState(false))
|
||||
assert.True(t, r.outputs["foo"].sent)
|
||||
|
||||
require.NoError(t, r.ReportState(true))
|
||||
require.Len(t, reported, 2)
|
||||
assert.Equal(t, map[string]string{"foo": "bar"}, reported[0])
|
||||
assert.Empty(t, reported[1])
|
||||
}
|
||||
|
||||
func TestReporter_EffectiveCloseTimeout(t *testing.T) {
|
||||
|
||||
Reference in New Issue
Block a user