Compare commits

..
Author SHA1 Message Date
silverwind 76a7e06e33 Merge branch 'main' into fix/659-cleanup-orphan-docker-resources 2026-07-24 16:30:23 +00:00
silverwindandClaude 6f06ca5cde fix: continue docker cleanup after a failed step
Teardown was built from `Then`, which stops on the previous step's error and
on a cancelled context. A volume that is still in use, or a flaky daemon, then
orphaned everything downstream: the service containers and the job network.

Replace the combinator chain with straight-line best-effort cleanup that runs
every step and joins the errors. Service containers are now removed before the
volumes, since a service can hold them via `--volumes-from`, and the network
stays last.

Co-Authored-By: Claude (Opus 4.8) <[email protected]>
2026-07-24 18:26:59 +02:00
bircniandGPT-5 Codex 5a98252565 fix: clean service containers after failed job setup
Ensure service containers are still stopped when job setup fails before the job container has been assigned, so runner-created resources do not linger after failed starts.

Fixes: https://gitea.com/gitea/runner/issues/659

Co-Authored-By: GPT-5 Codex <[email protected]>
2026-07-03 21:54:46 +02:00
353 changed files with 7356 additions and 8730 deletions
+2 -2
View File
@@ -18,8 +18,8 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 5 timeout-minutes: 5
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@v7
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - uses: actions/setup-node@v7
with: with:
node-version: 24 node-version: 24
- run: make lint-pr-title - run: make lint-pr-title
+8 -25
View File
@@ -17,27 +17,14 @@ jobs:
goreleaser: goreleaser:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@v7
with: with:
fetch-depth: 0 fetch-depth: 0
# Custom publishers (the R2 mirror below) run as the very last - uses: actions/setup-go@v7
# step of goreleaser's publish pipeline, after the Gitea release
# has already been created and every artifact already uploaded
# to S3. Fail here instead, before anything is built or
# published, if the R2 secrets are missing.
- name: check R2 configuration
run: sh scripts/upload-r2.sh --check-config
env:
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with: with:
go-version-file: "go.mod" go-version-file: "go.mod"
check-latest: true
- name: goreleaser - name: goreleaser
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7 uses: goreleaser/goreleaser-action@v7
with: with:
distribution: goreleaser-pro distribution: goreleaser-pro
args: release --nightly args: release --nightly
@@ -48,10 +35,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
S3_REGION: ${{ secrets.AWS_REGION }} S3_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET: ${{ secrets.AWS_BUCKET }} S3_BUCKET: ${{ secrets.AWS_BUCKET }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
GORELEASER_FORCE_TOKEN: "gitea" GORELEASER_FORCE_TOKEN: "gitea"
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -75,18 +58,18 @@ jobs:
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 uses: actions/checkout@v7
with: with:
fetch-depth: 0 # all history for all branches and tags fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 uses: docker/setup-qemu-action@v4
- name: Set up Docker BuildX - name: Set up Docker BuildX
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 uses: docker/setup-buildx-action@v4
- name: Login to DockerHub - name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 uses: docker/login-action@v4
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
@@ -100,7 +83,7 @@ jobs:
echo REPO_VERSION=$(git describe --tags --always | sed 's/-/+/' | sed 's/^v//') >> $GITHUB_OUTPUT echo REPO_VERSION=$(git describe --tags --always | sed 's/-/+/' | sed 's/^v//') >> $GITHUB_OUTPUT
- name: Build and push - name: Build and push
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 uses: docker/build-push-action@v7
with: with:
context: . context: .
file: ./Dockerfile file: ./Dockerfile
+10 -27
View File
@@ -9,34 +9,21 @@ jobs:
goreleaser: goreleaser:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@v7
with: with:
fetch-depth: 0 # all history for all branches and tags fetch-depth: 0 # all history for all branches and tags
# Custom publishers (the R2 mirror below) run as the very last - uses: actions/setup-go@v7
# step of goreleaser's publish pipeline, after the Gitea release
# has already been created and every artifact already uploaded
# to S3. Fail here instead, before anything is built or
# published, if the R2 secrets are missing.
- name: check R2 configuration
run: sh scripts/upload-r2.sh --check-config
env:
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with: with:
go-version-file: "go.mod" go-version-file: "go.mod"
check-latest: true
- name: Import GPG key - name: Import GPG key
id: import_gpg id: import_gpg
uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 uses: crazy-max/ghaction-import-gpg@v7
with: with:
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
passphrase: ${{ secrets.PASSPHRASE }} passphrase: ${{ secrets.PASSPHRASE }}
fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0 fingerprint: CC64B1DB67ABBEECAB24B6455FC346329753F4B0
- name: goreleaser - name: goreleaser
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7 uses: goreleaser/goreleaser-action@v7
with: with:
distribution: goreleaser-pro distribution: goreleaser-pro
args: release args: release
@@ -47,10 +34,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
S3_REGION: ${{ secrets.AWS_REGION }} S3_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET: ${{ secrets.AWS_BUCKET }} S3_BUCKET: ${{ secrets.AWS_BUCKET }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
GORELEASER_FORCE_TOKEN: "gitea" GORELEASER_FORCE_TOKEN: "gitea"
GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITEA_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
@@ -78,25 +61,25 @@ jobs:
DOCKER_LATEST: latest DOCKER_LATEST: latest
steps: steps:
- name: Checkout - name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 uses: actions/checkout@v7
with: with:
fetch-depth: 0 # all history for all branches and tags fetch-depth: 0 # all history for all branches and tags
- name: Set up QEMU - name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 uses: docker/setup-qemu-action@v4
- name: Set up Docker BuildX - name: Set up Docker BuildX
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4 uses: docker/setup-buildx-action@v4
- name: Login to DockerHub - name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 uses: docker/login-action@v4
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
- name: "Docker meta" - name: "Docker meta"
id: docker_meta id: docker_meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6 uses: docker/metadata-action@v6
with: with:
images: | images: |
${{ env.DOCKER_ORG }}/runner ${{ env.DOCKER_ORG }}/runner
@@ -109,7 +92,7 @@ jobs:
suffix=${{ matrix.variant.tag_suffix }},onlatest=true suffix=${{ matrix.variant.tag_suffix }},onlatest=true
- name: Build and push - name: Build and push
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 uses: docker/build-push-action@v7
with: with:
context: . context: .
file: ./Dockerfile file: ./Dockerfile
+3 -6
View File
@@ -17,14 +17,13 @@ jobs:
# to ~/.docker with the stale credentials. # to ~/.docker with the stale credentials.
DOCKER_CONFIG: /tmp/docker-noauth DOCKER_CONFIG: /tmp/docker-noauth
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@v7
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 - uses: actions/setup-go@v7
with: with:
go-version-file: 'go.mod' go-version-file: 'go.mod'
check-latest: true
- name: prepare anonymous docker config - name: prepare anonymous docker config
run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json" run: mkdir -p "$DOCKER_CONFIG" && echo '{}' > "$DOCKER_CONFIG/config.json"
# Pre-pull internal/act/runner's two largest base images so a slow pull can't dominate `make test`; # Pre-pull act/runner's two largest base images so a slow pull can't dominate `make test`;
# the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host # the rest (alpine/ubuntu) pull on demand, absorbed by the make-test -timeout. The host
# daemon retains them between runs, so this is usually a fast manifest re-check. # daemon retains them between runs, so this is usually a fast manifest re-check.
- name: pre-pull test images - name: pre-pull test images
@@ -34,8 +33,6 @@ jobs:
done done
- name: lint - name: lint
run: make lint run: make lint
- name: checks
run: make checks
- name: build - name: build
run: make build run: make build
- name: test - name: test
+1 -2
View File
@@ -1,8 +1,7 @@
/gitea-runner /gitea-runner
.env .env
!/internal/act/runner/testdata/secrets/.env !/act/runner/testdata/secrets/.env
.runner .runner
.runner.lock
coverage.txt coverage.txt
.tmp/ .tmp/
/config.yaml /config.yaml
+4 -4
View File
@@ -11,7 +11,6 @@ linters:
- dupl - dupl
- errcheck - errcheck
- forbidigo - forbidigo
- forcetypeassert
- gocheckcompilerdirectives - gocheckcompilerdirectives
- gocritic - gocritic
- goheader - goheader
@@ -37,8 +36,12 @@ linters:
rules: rules:
main: main:
deny: deny:
- pkg: io/ioutil
desc: use os or io instead
- pkg: golang.org/x/exp - pkg: golang.org/x/exp
desc: it's experimental and unreliable desc: it's experimental and unreliable
- pkg: github.com/pkg/errors
desc: use builtin errors package instead
nolintlint: nolintlint:
allow-unused: false allow-unused: false
require-explanation: true require-explanation: true
@@ -99,9 +102,6 @@ linters:
- linters: - linters:
- forbidigo - forbidigo
path: cmd path: cmd
- linters:
- forcetypeassert
path: _test\.go
issues: issues:
max-issues-per-linter: 0 max-issues-per-linter: 0
max-same-issues: 0 max-same-issues: 0
-31
View File
@@ -93,37 +93,6 @@ blobs:
- glob: ./**.xz - glob: ./**.xz
- glob: ./**.sha256 - glob: ./**.sha256
# Mirrors the S3 `blobs:` upload above into Cloudflare R2 during the
# parallel S3+R2 period (S3 will be removed once migration completes).
# A second `blobs:` entry is impossible here since the blob pipe
# authenticates from the global AWS_* env with no per-entry
# credentials; `publishers:` supports per-entry `env:` instead, so
# it's used to invoke scripts/upload-r2.sh once per artifact. Custom
# publishers inherit almost nothing from the environment, hence the
# explicit R2_* forwarding below.
#
# This publisher fires 109 times for 73 distinct keys because
# goreleaser's release pipe already registers `release.extra_files`
# as UploadableFile artifacts, and `internal/exec`'s filterArtifacts
# appends this block's own extra_files with no de-duplication. It
# can't be globbed away, since gobwas/glob (via goreleaser/fileglob)
# has no substring-exclusion matcher. It's harmless: PUT is
# idempotent, and the `./**.xz` glob below is kept deliberately so
# this publisher declares its own complete file set rather than
# implicitly depending on the `release:` block's globs.
publishers:
- name: cloudflare-r2
checksum: true
extra_files:
- glob: ./**.xz
- glob: ./**.sha256
cmd: sh scripts/upload-r2.sh {{ abs .ArtifactPath }} gitea-runner/{{ .Version }}/{{ .ArtifactName }}
env:
- R2_ENDPOINT={{ index .Env "R2_ENDPOINT" }}
- R2_BUCKET={{ index .Env "R2_BUCKET" }}
- R2_ACCESS_KEY_ID={{ index .Env "R2_ACCESS_KEY_ID" }}
- R2_SECRET_ACCESS_KEY={{ index .Env "R2_SECRET_ACCESS_KEY" }}
archives: archives:
- format: binary - format: binary
name_template: "{{ .Binary }}" name_template: "{{ .Binary }}"
+7 -16
View File
@@ -1,19 +1,10 @@
- Never assume, verify before claiming
- Use `make help` to find available development targets - Use `make help` to find available development targets
- PR descriptions: minimal, only what and why, no task lists or file listings - Run `make fmt` to format `.go` files, and run `make lint-go` to lint them
- Reference issues and PRs by full URL, not by number - Run `make tidy` after any `go.mod` changes
- Use Conventional Commits for commit messages and PR titles, plus the `enhance` type for user-facing enhancements - Run single go unit tests with `go test -run '^TestName$' ./modulepath/`
- Add an `Assisted-by: AGENT_NAME:MODEL_VERSION` trailer to commit messages, never `Co-Authored-By` or `Signed-off-by`
- Attribute agent authorship on one trailing line in issue and pull request comments, never as a PR description section
- Never force-push, amend, or squash unless asked. Use new commits and normal push for pull request updates
- Comments: write almost none, short and preferably same-line, explaining why for a future reader. Never narrate code, the change or the prompt. Preserve existing ones that still apply
- Add the current year into the copyright header of new `.go` files - Add the current year into the copyright header of new `.go` files
- Read `DEVELOPMENT.md` for internals and conventions
- Ensure no trailing whitespace in edited files - Ensure no trailing whitespace in edited files
- Run `make fmt` after `.go` edits, `make tidy` after `go.mod` edits, and `make checks` for the non-lint source checks - Never force-push, amend, or squash unless asked. Use new commits and normal push for pull request updates
- Lint what changed with `make lint-go`, and `make lint-go-windows` for Windows and platform-split files - Preserve existing code comments, do not remove or rewrite comments that are still relevant
- Fix the cause rather than disabling a linter or weakening a test. Where unavoidable, use the narrowest scope with a trailing comment giving the reason - Include authorship attribution in issue and pull request comments
- Run single go tests with `go test -run '^TestName$' ./modulepath/`. `make test` self-skips the integration tests without docker or network, `make test-dind` runs the daemon-facing tests against the built dind image - Add `Co-Authored-By` lines to all commits, indicating name and model used
- Write the fewest, fastest tests covering the behavior, extending an existing one where possible. Prefer unit tests where logic is testable in isolation
- Wait on a deterministic condition rather than `sleep`
- Update the files under `docs/` when behavior documented there changes
-32
View File
@@ -1,32 +0,0 @@
# Development
## Job log line format
Gitea stores one log row per line and its web UI decodes the payload, so getting the encoding
wrong never fails a test here, it only shows up in the browser.
**A row cannot contain a real newline.** `FormatLog` rewrites `\n` to a literal backslash-n and
truncates at 64 KiB on a byte boundary.
**The payload of a line starting with a recognised prefix is decoded**, with the escape set
depending on the prefix:
| prefix | decodes |
| --- | --- |
| `##[error]` `##[warning]` `##[notice]` `##[debug]` `##[group]` `##[endgroup]` `##[add-matcher]` | `%25` `%0D` `%0A` `%3B` `%5D` |
| `::error::` `::warning::` `::notice::` `::debug::` (with or without ` key=value` properties), `::group::` `::endgroup::` `::add-matcher::` | `%25` `%0D` `%0A` |
| `##[command]` `[command]`, or no recognised prefix | nothing |
### Rules
- **Emitting a command line?** Escape the payload with `runner.EscapeCommandData`. One escaper
covers both forms: it escapes `%` first, so a literal `%3B` becomes `%253B` that the extra
`##[…]` rules cannot match, and a raw `;` or `]` is never decoded. It is also what makes
multi-line work, `\n` becomes `%0A` and the UI turns it back into a line break.
- **Forwarding a command from step output?** Leave the payload alone, it arrived escaped and is
decoded once. Decoding here double-decodes and destroys multi-line.
- **No prefix?** Do not escape, and split multi-line values into one row each.
- **Interpolating a secret?** Masking runs after escaping, so `AppendSecretMasker` registers the
encoded forms too.
- Command *properties* also escape `%3A` and `%2C`, which the UI never decodes, so the reporter
decodes exactly those two when folding a location into an annotation.
+8 -18
View File
@@ -4,9 +4,9 @@ DIST_DIRS := $(DIST)/binaries $(DIST)/release
GO ?= go GO ?= go
SHASUM ?= shasum -a 256 SHASUM ?= shasum -a 256
HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" ) HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
XGO_PACKAGE ?= src.techknowlogick.com/xgo@v1.9.0 # renovate: datasource=go XGO_PACKAGE ?= src.techknowlogick.com/xgo@latest
XGO_VERSION := go-1.26.x XGO_VERSION := go-1.26.x
GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected]6 # renovate: datasource=go GXZ_PACKAGE ?= github.com/ulikunitz/xz/cmd/[email protected]0
LINUX_ARCHS ?= linux/amd64,linux/arm64 LINUX_ARCHS ?= linux/amd64,linux/arm64
DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64 DARWIN_ARCHS ?= darwin-12/amd64,darwin-12/arm64
@@ -18,10 +18,8 @@ DOCKER_TAG ?= nightly
DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG) DOCKER_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)
DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless DOCKER_ROOTLESS_REF := $(DOCKER_IMAGE):$(DOCKER_TAG)-dind-rootless
GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected] # renovate: datasource=go GOLANGCI_LINT_PACKAGE ?= github.com/golangci/golangci-lint/v2/cmd/[email protected]
GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.6.0 # renovate: datasource=go GOVULNCHECK_PACKAGE ?= golang.org/x/vuln/cmd/govulncheck@v1.3.0
GOTEST_FLAGS ?= -race -timeout 20m -parallel 8
STATIC ?= STATIC ?=
EXTLDFLAGS ?= EXTLDFLAGS ?=
@@ -97,7 +95,7 @@ go-check:
.PHONY: fmt-check .PHONY: fmt-check
fmt-check: fmt fmt-check: fmt
@diff=$$(git diff --color=always -- '*.go'); \ @diff=$$(git diff --color=always); \
if [ -n "$$diff" ]; then \ if [ -n "$$diff" ]; then \
echo "Please run 'make fmt' and commit the result:"; \ echo "Please run 'make fmt' and commit the result:"; \
printf "%s" "$${diff}"; \ printf "%s" "$${diff}"; \
@@ -112,9 +110,6 @@ deps-tools: ## install tool dependencies
$(GO) install $(GOVULNCHECK_PACKAGE) & \ $(GO) install $(GOVULNCHECK_PACKAGE) & \
wait wait
.PHONY: checks
checks: tidy-check fmt-check security-check ## run the non-lint source checks
.PHONY: lint .PHONY: lint
lint: lint-go lint-go-windows ## lint everything lint: lint-go lint-go-windows ## lint everything
@@ -136,17 +131,12 @@ lint-pr-title: ## lint PR title against Conventional Commits (set PR_TITLE=...)
@node ./tools/lint-pr-title.ts @node ./tools/lint-pr-title.ts
.PHONY: security-check .PHONY: security-check
security-check: security-check: deps-tools
GOEXPERIMENT= $(GO) run $(GOVULNCHECK_PACKAGE) -show color ./... || true GOEXPERIMENT= $(GO) run $(GOVULNCHECK_PACKAGE) -show color ./... || true
.PHONY: tidy .PHONY: tidy
tidy: ## run go mod tidy tidy: ## run go mod tidy
$(eval GO_TOOLCHAIN := $(shell grep -Eo '^toolchain\s+go[0-9.]+' go.mod | cut -d' ' -f2))
$(GO) mod tidy $(GO) mod tidy
@# workaround https://github.com/golang/go/issues/75331: restore toolchain if tidy dropped it
@if [ -n "$(GO_TOOLCHAIN)" ] && ! grep -qE '^toolchain\s' go.mod; then \
$(GO) mod edit -toolchain=$(GO_TOOLCHAIN); \
fi
.PHONY: tidy-check .PHONY: tidy-check
tidy-check: tidy tidy-check: tidy
@@ -158,8 +148,8 @@ tidy-check: tidy
fi fi
.PHONY: test .PHONY: test
test: ## test everything (integration tests self-skip without docker/network) test: fmt-check security-check ## test everything (integration tests self-skip without docker/network)
@$(GO) test $(GOTEST_FLAGS) -cover -coverprofile coverage.txt ./... && echo "\n==>\033[32m Ok\033[m\n" || exit 1 @$(GO) test -race -timeout 20m -v -cover -coverprofile coverage.txt ./... && echo "\n==>\033[32m Ok\033[m\n" || exit 1
.PHONY: coverage-report .PHONY: coverage-report
coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md coverage-report: ## turn coverage.txt from `make test` into .tmp/coverage.md
+13 -95
View File
@@ -129,38 +129,27 @@ Same idea as `dind`, but built on `docker:dind-rootless` so the bundled daemon a
### Configuration ### Configuration
The runner reads a YAML file. Without one, every option keeps its default. The runner is configured with a YAML file. Generate a starting point (this matches what ships in the tree):
```bash ```bash
./gitea-runner config init # write config.yaml, with no option set ./gitea-runner generate-config > config.yaml
./gitea-runner config generate | less # read what the options do
./gitea-runner -c config.yaml daemon # -c also works on register and cache-server
``` ```
`config generate` prints [config.example.yaml](internal/pkg/config/config.example.yaml). Every value in it is commented out, so copy the lines you want to change into your own file and uncomment them. Pass it with `-c` / `--config` on any command that loads configuration (`register`, `daemon`, `cache-server`):
#### Editing a config file
`config` edits a file in place, which is handy in provisioning scripts:
```bash ```bash
./gitea-runner config set runner.capacity 4 ./gitea-runner -c config.yaml register
./gitea-runner config set runner.timeout 90m # written as 1h30m0s ./gitea-runner -c config.yaml daemon
./gitea-runner config set runner.envs.MY_VAR value ./gitea-runner -c config.yaml cache-server
./gitea-runner config add runner.labels 'ubuntu:docker://node:22'
./gitea-runner config remove runner.labels 'ubuntu:docker://node:22'
./gitea-runner config get runner.labels
``` ```
A key is its dotted YAML path. An unknown key, a value of the wrong type, or `add`/`remove` on anything but a list is refused before the file is touched. `set` replaces a whole list when you give it several values. Every option is described in [config.example.yaml](internal/pkg/config/config.example.yaml) (the same content `generate-config` prints).
An edit keeps the comments and the key order of the file. Indentation becomes two spaces, and a blank line between two values is dropped. #### Without a config file
`config get`, `set`, `add` and `remove` use `config.yaml` (or `config.yml`) from the working directory, then from the directory of the binary, and print their choice to stderr. `config init` writes `config.yaml` in the working directory, and refuses to overwrite an existing config without `--force`. Pass `-c` for another path. If you omit `-c`, built-in defaults apply (same as an empty YAML document).
#### Environment variables Earlier releases let a small set of environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the default config. Those overrides have been removed — use a YAML config file for all settings instead. For the Docker images, the entrypoint still understands a separate set of variables (such as `RUNNER_STATE_FILE`); see [scripts/run.sh](scripts/run.sh) and the container documentation below.
Earlier releases let a few environment variables (`GITEA_DEBUG`, `GITEA_TRACE`, `GITEA_RUNNER_CAPACITY`, `GITEA_RUNNER_FILE`, `GITEA_RUNNER_ENVIRON`, `GITEA_RUNNER_ENV_FILE`) override parts of the config. They are gone, use the YAML file for all settings. The Docker images still read their own variables, such as `RUNNER_STATE_FILE`, see [scripts/run.sh](scripts/run.sh) and the container documentation below.
### Labels ### Labels
@@ -220,70 +209,10 @@ Whenever the resulting labels differ from the ones in the registration file, the
> **Note:** A runner that only exposes `host` labels still needs access to a Docker daemon (e.g. a mounted `/var/run/docker.sock`) whenever a job uses a `docker://` action or a service container. `host` labels only change where the job's own steps run; container-based steps and actions are still executed with Docker. > **Note:** A runner that only exposes `host` labels still needs access to a Docker daemon (e.g. a mounted `/var/run/docker.sock`) whenever a job uses a `docker://` action or a service container. `host` labels only change where the job's own steps run; container-based steps and actions are still executed with Docker.
#### Service containers
A job's `services` are started before its steps run. When a service's image or its `options` declare a healthcheck, the runner waits for it to report healthy, so a workflow does not have to poll for its own services:
```yaml
services:
postgres:
image: postgres:17
options: >-
--health-cmd pg_isready
--health-interval 5s
--health-retries 10
```
A service that reports unhealthy fails the job right away, with its container log. One that never becomes healthy fails it after `container.service_ready_timeout` (default `5m`, negative disables the wait). A service that exits without declaring a healthcheck only gets its log and a warning.
A job in a container reaches a service by its id on the job network, on the port the service listens on, for example `psql -h postgres -p 5432`. The started containers also fill the `job` context: `job.container.{id,network}` and `job.services.<id>.{id,network,ports}`, where `ports` maps a container port to the host port Docker published it on, for the services that publish one.
Unlike GitHub, a job whose steps run on the host (a `host` label without `container:`) starts no service containers, so `job.services` and `job.container` stay empty. Give such a job a `container:` when it needs services.
#### Proxy
Set these variables in the runner's environment, with systemd `Environment=`, `docker run -e`, or Kubernetes `env:`:
```sh
http_proxy=http://proxy.example:3128
https_proxy=http://proxy.example:3128
no_proxy=gitea.internal,.example.local
```
The runner uses them for its own requests and gives them to every job, in lower and upper case.
These hosts are added to `no_proxy` for jobs, so they are always reached directly:
- the cache server
- `localhost`, `127.0.0.1` and `::1`
- the job's service containers
- the Docker daemon, when it is reached over `tcp://`
Gitea is not added. Add it to `no_proxy` yourself if it should be reached directly.
To change a value for one job, set it in a step's `env:` or in the job's `container.env`. Setting it at workflow or job level has no effect. To change it for the whole runner, set it in `runner.envs`. A `no_proxy` set there is added to the list above instead of replacing it.
Images are pulled by the Docker daemon, which needs its own proxy setting. In the `dind` images the daemon runs in the same container and reads the variables above. For any other daemon, see [the Docker documentation](https://docs.docker.com/engine/daemon/proxy/). The runner logs a warning at startup if it has a proxy and the daemon does not.
Dockerfile actions are built with these variables as build arguments, so their `RUN` steps can reach the network.
A password in a proxy URL is hidden in job logs. Any step can still read it, because the step is given the proxy URL in its environment.
#### Caching (`actions/cache`) #### Caching (`actions/cache`)
Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default. Each runner starts its own cache server automatically. Cache entries are local to that runner — runners do not share a cache by default.
**Cache service v2**
`actions/cache@v4.2` and later can use the *cache service v2* API. The runner serves it from the same store as v1, on by default, and it works with `external_server`. Turn it off with:
```yaml
cache:
v2: false
```
Those actions refuse any host they do not take for GitHub. Rather than misreport the server URL, the runner edits that check out of the action's own bundle and keeps the untouched copy beside it; a bundle it does not recognise is left alone and keeps to v1. The same edit lets the stock `actions/upload-artifact` and `actions/download-artifact` work from `v4.4.0` on, without the `gitea-upload-artifact` fork.
**Shared cache across multiple runners** **Shared cache across multiple runners**
Run one dedicated `gitea-runner cache-server` that all runners point at. Run one dedicated `gitea-runner cache-server` that all runners point at.
@@ -325,12 +254,11 @@ Besides `GITEA_INSTANCE_URL` and `GITEA_RUNNER_REGISTRATION_TOKEN`, the image en
For a fuller container-oriented walkthrough, see [examples/docker](examples/docker/README.md). For a fuller container-oriented walkthrough, see [examples/docker](examples/docker/README.md).
While the runner is idle it cleans up after earlier jobs: When `container.bind_workdir` is enabled, stale task workspace directories can be cleaned while the runner is idle:
- when `container.bind_workdir` is enabled, stale task workspace directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable) - directories older than `runner.workdir_cleanup_age` are removed (default: `24h`; set `0` to disable)
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable)
- only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed - only purely numeric subdirectories under `container.workdir_parent` are treated as task workspaces and may be removed
- cleanup assumes `container.workdir_parent` is not shared across multiple runners - cleanup assumes `container.workdir_parent` is not shared across multiple runners
- on runners that use docker, per-job networks left behind by jobs the runner did not live to tear down are removed, identified by the `com.gitea.runner.uuid` label carrying this runner's uuid
- cleanup runs every `runner.idle_cleanup_interval` (default: `10m`; set `0` to disable), and setting either knob to `0` disables all of the above
#### Post-task script (`runner.post_task_script`) #### Post-task script (`runner.post_task_script`)
@@ -342,16 +270,6 @@ On Windows, use `.exe`, `.bat`, or `.cmd` paths; **PowerShell (`.ps1`) is not su
See **[docs/post-task-script.md](docs/post-task-script.md)** for lifecycle details, environment variables, timeout interaction, and platform notes. See **[docs/post-task-script.md](docs/post-task-script.md)** for lifecycle details, environment variables, timeout interaction, and platform notes.
#### Job hooks (`runner.hooks.job_started`, `runner.hooks.job_completed`)
Optional scripts that run **inside the job environment** (the job container, or the host in host mode), before the job's first step and after its last one. They are the equivalent of GitHub's `ACTIONS_RUNNER_HOOK_JOB_STARTED` / `ACTIONS_RUNNER_HOOK_JOB_COMPLETED`, which are read when the settings are unset.
Because they run where the steps run and see the job's environment, they are the place for per-job setup no workflow should have to carry: registry logins, mirror configuration, or masking runner-wide secrets with `::add-mask::`. Their output is part of the job log and is scanned for workflow commands, and they can export to the job through `$GITHUB_ENV` and `$GITHUB_PATH`.
Both hooks are synchronous and block the job while they run. Either one exiting non-zero fails the job, and there is no per-hook timeout.
See **[docs/job-hooks.md](docs/job-hooks.md)** for the execution order, environment, and platform notes.
### Example Deployments ### Example Deployments
Check out the [examples](examples) directory for sample deployment types. Check out the [examples](examples) directory for sample deployment types.
@@ -26,7 +26,7 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
@@ -52,13 +52,7 @@ type credKey struct{}
// poison another repo's cache, even from inside a container that reaches the // poison another repo's cache, even from inside a container that reaches the
// cache server over the docker bridge network. // cache server over the docker bridge network.
type JobCredential struct { type JobCredential struct {
Repo string `json:"repo"` Repo string
// Results is the instance whose artifact service this server forwards for the job, and
// InsecureTLS how the runner reaches it; see results.go. The tags are the wire format a
// remote runner registers with.
Results string `json:"results"`
InsecureTLS bool `json:"insecure_tls"`
} }
// credEntry holds a registered job's credential along with an active // credEntry holds a registered job's credential along with an active
@@ -76,7 +70,6 @@ type Handler struct {
storage *Storage storage *Storage
router *httprouter.Router router *httprouter.Router
listener net.Listener listener net.Listener
port int
server *http.Server server *http.Server
logger logrus.FieldLogger logger logrus.FieldLogger
@@ -164,14 +157,12 @@ func StartHandler(dir, outboundIP string, port uint16, internalSecret string, lo
router.POST(apiPath+"/clean", h.bearerAuth(h.clean)) router.POST(apiPath+"/clean", h.bearerAuth(h.clean))
// Artifact GET is signed via query-string HMAC because @actions/cache // Artifact GET is signed via query-string HMAC because @actions/cache
// does not attach Authorization when downloading archiveLocation. // does not attach Authorization when downloading archiveLocation.
router.GET(apiPath+"/artifacts/:id", h.signedAuth("", h.get)) router.GET(apiPath+"/artifacts/:id", h.signedURLAuth(h.get))
// Control-plane: a remote runner registers/revokes per-job tokens so the // Control-plane: a remote runner registers/revokes per-job tokens so the
// cache API can authenticate them. Always wired so the routes exist; the // cache API can authenticate them. Always wired so the routes exist; the
// handlers themselves 401 when internalSecret is unset. // handlers themselves 401 when internalSecret is unset.
router.POST(internalPath+"/register", h.internalAuth(h.internalRegister)) router.POST(internalPath+"/register", h.internalAuth(h.internalRegister))
router.POST(internalPath+"/revoke", h.internalAuth(h.internalRevoke)) router.POST(internalPath+"/revoke", h.internalAuth(h.internalRevoke))
h.registerV2Routes(router)
router.NotFound = http.HandlerFunc(h.forwardOrNotFound)
h.router = router h.router = router
@@ -186,12 +177,6 @@ func StartHandler(dir, outboundIP string, port uint16, internalSecret string, lo
if err != nil { if err != nil {
return nil, err return nil, err
} }
addr, ok := listener.Addr().(*net.TCPAddr)
if !ok {
listener.Close()
return nil, fmt.Errorf("cache server listens on %T, want a TCP address", listener.Addr())
}
h.port = addr.Port
server := &http.Server{ server := &http.Server{
ReadHeaderTimeout: 2 * time.Second, ReadHeaderTimeout: 2 * time.Second,
Handler: router, Handler: router,
@@ -209,7 +194,9 @@ func StartHandler(dir, outboundIP string, port uint16, internalSecret string, lo
func (h *Handler) ExternalURL() string { func (h *Handler) ExternalURL() string {
// TODO: make the external url configurable if necessary // TODO: make the external url configurable if necessary
return fmt.Sprintf("http://%s:%d", h.outboundIP, h.port) return fmt.Sprintf("http://%s:%d",
h.outboundIP,
h.listener.Addr().(*net.TCPAddr).Port)
} }
// RegisterJob makes token a valid bearer credential for cache requests from // RegisterJob makes token a valid bearer credential for cache requests from
@@ -218,11 +205,10 @@ func (h *Handler) ExternalURL() string {
// is only accepted while the job is running. // is only accepted while the job is running.
// //
// Registrations are reference-counted: if a token is already registered, the // Registrations are reference-counted: if a token is already registered, the
// credential it was registered with is kept and the refcount is incremented. // existing repo is kept and the refcount is incremented. The entry is
// The entry is removed only when every revoker returned by RegisterJob has // removed only when every revoker returned by RegisterJob has been called.
// been called.
// This keeps a stray re-registration from silently revoking a live job. // This keeps a stray re-registration from silently revoking a live job.
func (h *Handler) RegisterJob(token string, cred JobCredential) func() { func (h *Handler) RegisterJob(token, repo string) func() {
if h == nil || token == "" { if h == nil || token == "" {
return func() {} return func() {}
} }
@@ -231,7 +217,7 @@ func (h *Handler) RegisterJob(token string, cred JobCredential) func() {
existing.refs++ existing.refs++
} else { } else {
h.creds[token] = &credEntry{ h.creds[token] = &credEntry{
cred: cred, cred: JobCredential{Repo: repo},
refs: 1, refs: 1,
} }
} }
@@ -348,7 +334,7 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Para
} }
defer db.Close() defer db.Close()
cache, err := h.lookupCache(db, cred.Repo, keys, version) cache, err := findCache(db, cred.Repo, keys, version)
if err != nil { if err != nil {
h.responseJSON(w, r, 500, err) h.responseJSON(w, r, 500, err)
return return
@@ -357,6 +343,15 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Para
h.responseJSON(w, r, 204) h.responseJSON(w, r, 204)
return return
} }
if ok, err := h.storage.Exist(cache.ID); err != nil {
h.responseJSON(w, r, 500, err)
return
} else if !ok {
_ = db.Delete(cache.ID, cache)
h.responseJSON(w, r, 204)
return
}
h.responseJSON(w, r, 200, map[string]any{ h.responseJSON(w, r, 200, map[string]any{
"result": "hit", "result": "hit",
"archiveLocation": h.signedArtifactURL(cache.ID, time.Now().Add(artifactURLTTL)), "archiveLocation": h.signedArtifactURL(cache.ID, time.Now().Add(artifactURLTTL)),
@@ -364,25 +359,6 @@ func (h *Handler) find(w http.ResponseWriter, r *http.Request, _ httprouter.Para
}) })
} }
// lookupCache returns the entry to restore for these keys, or (nil, nil) when there is none:
// either nothing matched, or the match had lost its blob to a prune, in which case the dangling
// entry is dropped on the way out.
func (h *Handler) lookupCache(db *bolthold.Store, repo string, keys []string, version string) (*Cache, error) {
cache, err := findCache(db, repo, keys, version)
if err != nil || cache == nil {
return nil, err
}
ok, err := h.storage.Exist(cache.ID)
if err != nil {
return nil, err
}
if !ok {
_ = db.Delete(cache.ID, cache)
return nil, nil //nolint:nilnil // absence is not an error here
}
return cache, nil
}
// POST /_apis/artifactcache/caches // POST /_apis/artifactcache/caches
func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { func (h *Handler) reserve(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
cred := credFromContext(r.Context()) cred := credFromContext(r.Context())
@@ -457,7 +433,7 @@ func (h *Handler) upload(w http.ResponseWriter, r *http.Request, params httprout
h.responseJSON(w, r, 500, err) h.responseJSON(w, r, 500, err)
return return
} }
_ = h.touchCache(uint64(id), false) h.useCache(id)
h.responseJSON(w, r, 200) h.responseJSON(w, r, 200)
} }
@@ -498,7 +474,23 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout
db.Close() db.Close()
if err := h.commitCache(cache); err != nil { size, err := h.storage.Commit(cache.ID, cache.Size)
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
// write real size back to cache, it may be different from the current value when the request doesn't specify it.
cache.Size = size
db, err = h.openDB()
if err != nil {
h.responseJSON(w, r, 500, err)
return
}
defer db.Close()
cache.Complete = true
if err := db.Update(cache.ID, cache); err != nil {
h.responseJSON(w, r, 500, err) h.responseJSON(w, r, 500, err)
return return
} }
@@ -506,28 +498,8 @@ func (h *Handler) commit(w http.ResponseWriter, r *http.Request, params httprout
h.responseJSON(w, r, 200) h.responseJSON(w, r, 200)
} }
// commitCache assembles the uploaded parts and marks the entry complete. The caller must
// have closed its store first: Commit concatenates the whole archive and would otherwise
// hold bolt's exclusive file lock for the duration.
func (h *Handler) commitCache(cache *Cache) error {
written, err := h.storage.Commit(cache.ID, cache.Size)
if err != nil {
return err
}
// write real size back to cache, it may be different from the current value when the request doesn't specify it.
cache.Size = written
cache.Complete = true
db, err := h.openDB()
if err != nil {
return err
}
defer db.Close()
return db.Update(cache.ID, cache)
}
// GET /_apis/artifactcache/artifacts/:id // GET /_apis/artifactcache/artifacts/:id
// Authenticated via signed URL (see signedAuth), not bearer, because the // Authenticated via signed URL (see signedURLAuth), not bearer, because the
// @actions/cache toolkit downloads archiveLocation without Authorization. // @actions/cache toolkit downloads archiveLocation without Authorization.
// Repository scoping is already enforced at find() time; the signature binds // Repository scoping is already enforced at find() time; the signature binds
// the URL to the specific cache ID and an expiry. // the URL to the specific cache ID and an expiry.
@@ -537,7 +509,7 @@ func (h *Handler) get(w http.ResponseWriter, r *http.Request, params httprouter.
h.responseJSON(w, r, 400, err) h.responseJSON(w, r, 400, err)
return return
} }
_ = h.touchCache(uint64(id), false) h.useCache(id)
h.storage.Serve(w, r, uint64(id)) h.storage.Serve(w, r, uint64(id))
} }
@@ -571,9 +543,7 @@ func (h *Handler) bearerAuth(handler httprouter.Handle) httprouter.Handle {
} }
} }
// signedAuth authenticates a signed URL. purpose separates the flavours of URL the func (h *Handler) signedURLAuth(handler httprouter.Handle) httprouter.Handle {
// handler hands out, so one cannot be replayed as another; see computeSignature.
func (h *Handler) signedAuth(purpose string, handler httprouter.Handle) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) { return func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
h.logger.Debugf("%s %s", r.Method, r.URL.Path) h.logger.Debugf("%s %s", r.Method, r.URL.Path)
id, err := strconv.ParseInt(params.ByName("id"), 10, 64) id, err := strconv.ParseInt(params.ByName("id"), 10, 64)
@@ -596,7 +566,7 @@ func (h *Handler) signedAuth(purpose string, handler httprouter.Handle) httprout
h.responseJSON(w, r, http.StatusUnauthorized, errors.New("signature expired")) h.responseJSON(w, r, http.StatusUnauthorized, errors.New("signature expired"))
return return
} }
expected := h.computeSignature(purpose, id, exp) expected := h.computeSignature(id, exp)
if !hmac.Equal([]byte(sig), []byte(expected)) { if !hmac.Equal([]byte(sig), []byte(expected)) {
h.responseJSON(w, r, http.StatusUnauthorized, errors.New("bad signature")) h.responseJSON(w, r, http.StatusUnauthorized, errors.New("bad signature"))
return return
@@ -627,7 +597,7 @@ func (h *Handler) internalAuth(handler httprouter.Handle) httprouter.Handle {
type internalRegisterBody struct { type internalRegisterBody struct {
Token string `json:"token"` Token string `json:"token"`
JobCredential Repo string `json:"repo"`
} }
type internalRevokeBody struct { type internalRevokeBody struct {
@@ -635,15 +605,6 @@ type internalRevokeBody struct {
} }
// POST /_internal/register // POST /_internal/register
// ResultsURL is what a job registered with cred should be given as ACTIONS_RESULTS_URL, or "" when
// the credential names no instance to forward the artifact half to.
func (h *Handler) ResultsURL(cred JobCredential) string {
if h == nil || cred.Results == "" {
return ""
}
return h.ExternalURL()
}
func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
var body internalRegisterBody var body internalRegisterBody
if err := json.NewDecoder(r.Body).Decode(&body); err != nil { if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
@@ -654,9 +615,8 @@ func (h *Handler) internalRegister(w http.ResponseWriter, r *http.Request, _ htt
h.responseJSON(w, r, http.StatusBadRequest, errors.New("token is required")) h.responseJSON(w, r, http.StatusBadRequest, errors.New("token is required"))
return return
} }
h.RegisterJob(body.Token, body.JobCredential) h.RegisterJob(body.Token, body.Repo)
// A server too old to forward answers without this, which is how the caller knows. h.responseJSON(w, r, http.StatusOK)
h.responseJSON(w, r, http.StatusOK, map[string]any{"results_url": h.ResultsURL(body.JobCredential)})
} }
// POST /_internal/revoke // POST /_internal/revoke
@@ -690,26 +650,19 @@ func credFromContext(ctx context.Context) JobCredential {
return JobCredential{} return JobCredential{}
} }
// computeSignature signs a URL for one cache entry and expiry. purpose is mixed into the func (h *Handler) computeSignature(cacheID, exp int64) string {
// message so a URL handed out for writing an entry cannot be replayed to read one, and the
// other way round. Downloads use the empty purpose, the message v1 has always signed.
func (h *Handler) computeSignature(purpose string, cacheID, exp int64) string {
mac := hmac.New(sha256.New, h.secret) mac := hmac.New(sha256.New, h.secret)
fmt.Fprintf(mac, "%s%d:%d", purpose, cacheID, exp) fmt.Fprintf(mac, "%d:%d", cacheID, exp)
return hex.EncodeToString(mac.Sum(nil)) return hex.EncodeToString(mac.Sum(nil))
} }
// signedURL builds a URL under path that signedAuth accepts for the same purpose. func (h *Handler) signedArtifactURL(cacheID uint64, exp time.Time) string {
func (h *Handler) signedURL(path, purpose string, cacheID uint64, exp time.Time) string {
expUnix := exp.Unix() expUnix := exp.Unix()
sig := h.computeSignature(int64(cacheID), expUnix)
q := url.Values{} q := url.Values{}
q.Set("exp", strconv.FormatInt(expUnix, 10)) q.Set("exp", strconv.FormatInt(expUnix, 10))
q.Set("sig", h.computeSignature(purpose, int64(cacheID), expUnix)) q.Set("sig", sig)
return fmt.Sprintf("%s%s/%d?%s", h.ExternalURL(), path, cacheID, q.Encode()) return fmt.Sprintf("%s%s/artifacts/%d?%s", h.ExternalURL(), apiPath, cacheID, q.Encode())
}
func (h *Handler) signedArtifactURL(cacheID uint64, exp time.Time) string {
return h.signedURL(apiPath+"/artifacts", "", cacheID, exp)
} }
// if not found, return (nil, nil) instead of an error. // if not found, return (nil, nil) instead of an error.
@@ -717,12 +670,16 @@ func findCache(db *bolthold.Store, repo string, keys []string, version string) (
cache := &Cache{} cache := &Cache{}
for _, prefix := range keys { for _, prefix := range keys {
// if a key in the list matches exactly, don't return partial matches // if a key in the list matches exactly, don't return partial matches
exact, err := findExactCache(db, repo, prefix, version, true) if err := db.FindOne(cache,
bolthold.Where("Repo").Eq(repo).
And("Key").Eq(prefix).
And("Version").Eq(version).
And("Complete").Eq(true).
SortBy("CreatedAt").Reverse()); err == nil || !errors.Is(err, bolthold.ErrNotFound) {
if err != nil { if err != nil {
return nil, err return nil, fmt.Errorf("find cache: %w", err)
} }
if exact != nil { return cache, nil
return exact, nil
} }
prefixPattern := "^" + regexp.QuoteMeta(prefix) prefixPattern := "^" + regexp.QuoteMeta(prefix)
re, err := regexp.Compile(prefixPattern) re, err := regexp.Compile(prefixPattern)
@@ -745,34 +702,6 @@ func findCache(db *bolthold.Store, repo string, keys []string, version string) (
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
} }
// findExactCache returns the entry for exactly this key and version, or (nil, nil) if there is
// none. Unlike findCache it never falls back to a prefix (restore-key) match, which is what both
// its callers need: a new key that is only a prefix of an existing key is not the same entry.
//
// A completed entry is the one to restore, sorted by when it was written. An incomplete one is a
// reservation being uploaded to, sorted by when it was last written to, because the upload route
// touches UsedAt on every part.
func findExactCache(db *bolthold.Store, repo, key, version string, complete bool) (*Cache, error) {
sortBy := "UsedAt"
if complete {
sortBy = "CreatedAt"
}
cache := &Cache{}
err := db.FindOne(cache,
bolthold.Where("Repo").Eq(repo).
And("Key").Eq(key).
And("Version").Eq(version).
And("Complete").Eq(complete).
SortBy(sortBy).Reverse())
if errors.Is(err, bolthold.ErrNotFound) {
return nil, nil //nolint:nilnil // absence is not an error here
}
if err != nil {
return nil, fmt.Errorf("find cache: %w", err)
}
return cache, nil
}
func insertCache(db *bolthold.Store, cache *Cache) error { func insertCache(db *bolthold.Store, cache *Cache) error {
if err := db.Insert(bolthold.NextSequence(), cache); err != nil { if err := db.Insert(bolthold.NextSequence(), cache); err != nil {
return fmt.Errorf("insert cache: %w", err) return fmt.Errorf("insert cache: %w", err)
@@ -784,30 +713,18 @@ func insertCache(db *bolthold.Store, cache *Cache) error {
return nil return nil
} }
// touchCache stamps UsedAt so gcCache does not reap an entry mid-upload. With requireIncomplete func (h *Handler) useCache(id int64) {
// it also refuses an entry that is already complete, which is what the v2 blob route needs: its
// upload URL outlives the finalize call, and overwriting a finished entry would leave the blob
// other jobs restore no longer matching its recorded size. An entry missing from the store is
// accepted, since the signature proves the id was handed out.
func (h *Handler) touchCache(id uint64, requireIncomplete bool) error {
db, err := h.openDB() db, err := h.openDB()
if err != nil { if err != nil {
return err return
} }
defer db.Close() defer db.Close()
cache := &Cache{} cache := &Cache{}
if err := db.Get(id, cache); err != nil { if err := db.Get(id, cache); err != nil {
if errors.Is(err, bolthold.ErrNotFound) { return
return nil
}
return err
}
if requireIncomplete && cache.Complete {
return fmt.Errorf("cache %d: already complete", id)
} }
cache.UsedAt = time.Now().Unix() cache.UsedAt = time.Now().Unix()
return db.Update(cache.ID, cache) _ = db.Update(cache.ID, cache)
} }
const ( const (
@@ -52,7 +52,7 @@ func TestHandler(t *testing.T) {
dir := filepath.Join(t.TempDir(), "artifactcache") dir := filepath.Join(t.TempDir(), "artifactcache")
handler, err := StartHandler(dir, "", 0, "", nil) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
handler.RegisterJob(testToken, JobCredential{Repo: testRepo}) handler.RegisterJob(testToken, testRepo)
base := fmt.Sprintf("%s%s", handler.ExternalURL(), apiPath) base := fmt.Sprintf("%s%s", handler.ExternalURL(), apiPath)
@@ -445,6 +445,13 @@ func TestHandler(t *testing.T) {
require.Equal(t, 404, resp.StatusCode) require.Equal(t, 404, resp.StatusCode)
}) })
t.Run("get with not exist id", func(t *testing.T) {
resp, err := testClient.Get(signArtifactURL(handler, 100))
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, 404, resp.StatusCode)
})
t.Run("get with multiple keys", func(t *testing.T) { t.Run("get with multiple keys", func(t *testing.T) {
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
key := strings.ToLower(t.Name()) key := strings.ToLower(t.Name())
@@ -462,8 +469,7 @@ func TestHandler(t *testing.T) {
_, err := rand.Read(contents[i]) _, err := rand.Read(contents[i])
require.NoError(t, err) require.NoError(t, err)
uploadCacheNormally(t, base, keys[i], version, contents[i]) uploadCacheNormally(t, base, keys[i], version, contents[i])
// ensure CreatedAt of caches are different, in upload order time.Sleep(time.Second) // ensure CreatedAt of caches are different
backdateCache(t, handler, keys[i], time.Duration(len(contents)-i)*time.Second)
} }
reqKeys := strings.Join([]string{ reqKeys := strings.Join([]string{
@@ -548,8 +554,7 @@ func TestHandler(t *testing.T) {
_, err := rand.Read(contents[i]) _, err := rand.Read(contents[i])
require.NoError(t, err) require.NoError(t, err)
uploadCacheNormally(t, base, keys[i], version, contents[i]) uploadCacheNormally(t, base, keys[i], version, contents[i])
// ensure CreatedAt of caches are different, in upload order time.Sleep(time.Second) // ensure CreatedAt of caches are different
backdateCache(t, handler, keys[i], time.Duration(len(contents)-i)*time.Second)
} }
reqKeys := strings.Join([]string{ reqKeys := strings.Join([]string{
@@ -602,8 +607,7 @@ func TestHandler(t *testing.T) {
_, err := rand.Read(contents[i]) _, err := rand.Read(contents[i])
require.NoError(t, err) require.NoError(t, err)
uploadCacheNormally(t, base, keys[i], version, contents[i]) uploadCacheNormally(t, base, keys[i], version, contents[i])
// ensure CreatedAt of caches are different, in upload order time.Sleep(time.Second) // ensure CreatedAt of caches are different
backdateCache(t, handler, keys[i], time.Duration(len(contents)-i)*time.Second)
} }
reqKeys := strings.Join([]string{ reqKeys := strings.Join([]string{
@@ -642,20 +646,6 @@ func TestHandler(t *testing.T) {
}) })
} }
// backdateCache rewrites a cache's CreatedAt. It has one-second resolution, so age-ordering
// tests set it directly instead of sleeping a second between uploads.
func backdateCache(t *testing.T, handler *Handler, key string, age time.Duration) {
db, err := handler.openDB()
require.NoError(t, err)
defer db.Close()
var caches []*Cache
require.NoError(t, db.Find(&caches, bolthold.Where("Key").Eq(key)))
require.Len(t, caches, 1)
caches[0].CreatedAt = time.Now().Add(-age).Unix()
require.NoError(t, db.Update(caches[0].ID, caches[0]))
}
func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { //nolint:unparam // pre-existing issue from nektos/act func uploadCacheNormally(t *testing.T, base, key, version string, content []byte) { //nolint:unparam // pre-existing issue from nektos/act
var id uint64 var id uint64
{ {
@@ -890,7 +880,7 @@ func TestHandler_UnregisterRevokes(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
unregister := handler.RegisterJob("tmp-token", JobCredential{Repo: testRepo}) unregister := handler.RegisterJob("tmp-token", testRepo)
base := handler.ExternalURL() + apiPath base := handler.ExternalURL() + apiPath
req, err := http.NewRequest(http.MethodGet, base+"/cache?keys=x&version=y", nil) req, err := http.NewRequest(http.MethodGet, base+"/cache?keys=x&version=y", nil)
@@ -920,8 +910,8 @@ func TestHandler_CrossRepoIsolation(t *testing.T) {
handler, err := StartHandler(dir, "", 0, "", nil) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob("token-a", JobCredential{Repo: "owner/repoA"}) handler.RegisterJob("token-a", "owner/repoA")
handler.RegisterJob("token-b", JobCredential{Repo: "owner/repoB"}) handler.RegisterJob("token-b", "owner/repoB")
base := handler.ExternalURL() + apiPath base := handler.ExternalURL() + apiPath
key := "shared-key" key := "shared-key"
@@ -986,7 +976,7 @@ func TestHandler_ArtifactSignature(t *testing.T) {
handler, err := StartHandler(dir, "", 0, "", nil) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob(testToken, JobCredential{Repo: testRepo}) handler.RegisterJob(testToken, testRepo)
base := handler.ExternalURL() + apiPath base := handler.ExternalURL() + apiPath
@@ -1041,14 +1031,14 @@ func TestHandler_SecretPersistsAcrossRestarts(t *testing.T) {
first, err := StartHandler(dir, "127.0.0.1", 0, "", nil) first, err := StartHandler(dir, "127.0.0.1", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
exp := time.Now().Add(artifactURLTTL).Unix() exp := time.Now().Add(artifactURLTTL).Unix()
sig := first.computeSignature("", 42, exp) sig := first.computeSignature(42, exp)
require.NoError(t, first.Close()) require.NoError(t, first.Close())
second, err := StartHandler(dir, "127.0.0.1", 0, "", nil) second, err := StartHandler(dir, "127.0.0.1", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer second.Close() defer second.Close()
assert.Equal(t, sig, second.computeSignature("", 42, exp)) assert.Equal(t, sig, second.computeSignature(42, exp))
} }
// TestHandler_ArtifactSignatureDownload is a happy-path round trip that // TestHandler_ArtifactSignatureDownload is a happy-path round trip that
@@ -1059,7 +1049,7 @@ func TestHandler_ArtifactSignatureDownload(t *testing.T) {
handler, err := StartHandler(dir, "", 0, "", nil) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob(testToken, JobCredential{Repo: testRepo}) handler.RegisterJob(testToken, testRepo)
base := handler.ExternalURL() + apiPath base := handler.ExternalURL() + apiPath
key := "download-key" key := "download-key"
@@ -1100,8 +1090,8 @@ func TestHandler_RegisterJob_RefCounted(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
first := handler.RegisterJob("shared", JobCredential{Repo: testRepo}) first := handler.RegisterJob("shared", testRepo)
second := handler.RegisterJob("shared", JobCredential{Repo: testRepo}) second := handler.RegisterJob("shared", testRepo)
base := handler.ExternalURL() + apiPath base := handler.ExternalURL() + apiPath
probe := func() int { probe := func() int {
@@ -1131,8 +1121,8 @@ func TestHandler_GC_PerRepoDedup(t *testing.T) {
handler, err := StartHandler(dir, "", 0, "", nil) handler, err := StartHandler(dir, "", 0, "", nil)
require.NoError(t, err) require.NoError(t, err)
defer handler.Close() defer handler.Close()
handler.RegisterJob("tok-a", JobCredential{Repo: "owner/repoA"}) handler.RegisterJob("tok-a", "owner/repoA")
handler.RegisterJob("tok-b", JobCredential{Repo: "owner/repoB"}) handler.RegisterJob("tok-b", "owner/repoB")
key := "shared-dedup-key" key := "shared-dedup-key"
version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20" version := "c19da02a2bd7e77277f1ac29ab45c09b7d46a4ee758284e26bb3045ad11d9d20"
@@ -5,15 +5,12 @@
package artifactcache package artifactcache
import ( import (
"crypto/sha256"
"encoding/hex"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
) )
type Storage struct { type Storage struct {
@@ -40,10 +37,7 @@ func (s *Storage) Exist(id uint64) (bool, error) {
} }
func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error { func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
return s.writeFile(s.tempName(id, offset), reader) name := s.tempName(id, offset)
}
func (s *Storage) writeFile(name string, reader io.Reader) error {
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
return err return err
} }
@@ -57,26 +51,6 @@ func (s *Storage) writeFile(name string, reader io.Reader) error {
return err return err
} }
func (s *Storage) WriteBlock(id uint64, blockID string, reader io.Reader) error {
return s.writeFile(s.blockName(id, blockID), reader)
}
// OrderBlocks renames the staged blocks into the order the block list gives. A block the list
// does not name keeps its staged name, which is how Commit leaves it out, as Azure drops it. One
// rename pass is safe because a staged name always carries blockFilePrefix and a target name
// never does, so no rename can collide with a block not yet moved.
func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
for i, blockID := range blockIDs {
if err := os.Rename(s.blockName(id, blockID), s.tempName(id, int64(i))); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("block %q of cache %d was never uploaded: %w", blockID, id, err)
}
return err
}
}
return nil
}
func (s *Storage) Commit(id uint64, size int64) (int64, error) { func (s *Storage) Commit(id uint64, size int64) (int64, error) {
defer func() { defer func() {
_ = os.RemoveAll(s.tempDir(id)) _ = os.RemoveAll(s.tempDir(id))
@@ -91,31 +65,6 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil { if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
return 0, err return 0, err
} }
written, err := assemble(name, tempNames)
if err != nil {
return 0, err
}
// If size is less than 0, it means the size is unknown.
// We can't check the size of the file, just skip the check.
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
if size >= 0 && written != size {
_ = os.Remove(name)
return 0, fmt.Errorf("broken file: %v != %v", written, size)
}
return written, nil
}
// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
// produces below the client's block threshold, is already the whole archive and is moved.
func assemble(name string, tempNames []string) (int64, error) {
if len(tempNames) == 1 {
info, err := os.Stat(tempNames[0])
if err != nil {
return 0, err
}
return info.Size(), os.Rename(tempNames[0], name)
}
file, err := os.Create(name) file, err := os.Create(name)
if err != nil { if err != nil {
return 0, err return 0, err
@@ -135,6 +84,16 @@ func assemble(name string, tempNames []string) (int64, error) {
} }
written += n written += n
} }
// If size is less than 0, it means the size is unknown.
// We can't check the size of the file, just skip the check.
// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
if size >= 0 && written != size {
_ = file.Close()
_ = os.Remove(name)
return 0, fmt.Errorf("broken file: %v != %v", written, size)
}
return written, nil return written, nil
} }
@@ -160,17 +119,6 @@ func (s *Storage) tempName(id uint64, offset int64) string {
return filepath.Join(s.tempDir(id), fmt.Sprintf("%016x", offset)) return filepath.Join(s.tempDir(id), fmt.Sprintf("%016x", offset))
} }
// blockFilePrefix marks a staged, not yet ordered block, so that tempNames can keep it out of
// Commit's name-ordered concatenation.
const blockFilePrefix = "block-"
func (s *Storage) blockName(id uint64, blockID string) string {
// The block id is client-chosen (base64), so it is hashed rather than trusted as a
// path element.
sum := sha256.Sum256([]byte(blockID))
return filepath.Join(s.tempDir(id), blockFilePrefix+hex.EncodeToString(sum[:]))
}
func (s *Storage) tempNames(id uint64) ([]string, error) { func (s *Storage) tempNames(id uint64) ([]string, error) {
dir := s.tempDir(id) dir := s.tempDir(id)
files, err := os.ReadDir(dir) files, err := os.ReadDir(dir)
@@ -179,7 +127,7 @@ func (s *Storage) tempNames(id uint64) ([]string, error) {
} }
var names []string var names []string
for _, v := range files { for _, v := range files {
if !v.IsDir() && !strings.HasPrefix(v.Name(), blockFilePrefix) { if !v.IsDir() {
names = append(names, filepath.Join(dir, v.Name())) names = append(names, filepath.Join(dir, v.Name()))
} }
} }
@@ -17,7 +17,7 @@ import (
"strings" "strings"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
) )
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// Copyright 2020 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import "slices"
// CartesianProduct takes map of lists and returns list of unique tuples
func CartesianProduct(mapOfLists map[string][]any) []map[string]any {
listNames := make([]string, 0)
lists := make([][]any, 0)
for k, v := range mapOfLists {
listNames = append(listNames, k)
lists = append(lists, v)
}
listCart := cartN(lists...)
rtn := make([]map[string]any, 0)
for _, list := range listCart {
vMap := make(map[string]any)
for i, v := range list {
vMap[listNames[i]] = v
}
rtn = append(rtn, vMap)
}
return rtn
}
func cartN(a ...[]any) [][]any {
c := 1
for _, a := range a {
c *= len(a)
}
if c == 0 || len(a) == 0 {
return nil
}
p := make([][]any, c)
b := make([]any, c*len(a))
n := make([]int, len(a))
s := 0
for i := range p {
e := s + len(a)
pi := b[s:e]
p[i] = pi
s = e
for j, n := range n {
pi[j] = a[j][n]
}
for j := range slices.Backward(n) {
n[j]++
if n[j] < len(a[j]) {
break
}
n[j] = 0
}
}
return p
}
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// Copyright 2020 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestCartesianProduct(t *testing.T) {
assert := assert.New(t)
input := map[string][]any{
"foo": {1, 2, 3, 4},
"bar": {"a", "b", "c"},
"baz": {false, true},
}
output := CartesianProduct(input)
assert.Len(output, 24)
for _, v := range output {
assert.Len(v, 3)
assert.Contains(v, "foo")
assert.Contains(v, "bar")
assert.Contains(v, "baz")
}
input = map[string][]any{
"foo": {1, 2, 3, 4},
"bar": {},
"baz": {false, true},
}
output = CartesianProduct(input)
assert.Empty(output)
input = map[string][]any{}
output = CartesianProduct(input)
assert.Empty(output)
}
+89
View File
@@ -0,0 +1,89 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
// Simple fast test that verifies max-parallel: 2 limits concurrency
func TestMaxParallel2Quick(t *testing.T) {
ctx := context.Background()
var currentRunning atomic.Int32
var maxSimultaneous atomic.Int32
executors := make([]Executor, 4)
for i := range 4 {
executors[i] = func(ctx context.Context) error {
current := currentRunning.Add(1)
// Update max if needed
for {
maxValue := maxSimultaneous.Load()
if current <= maxValue || maxSimultaneous.CompareAndSwap(maxValue, current) {
break
}
}
time.Sleep(10 * time.Millisecond)
currentRunning.Add(-1)
return nil
}
}
err := NewParallelExecutor(2, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.LessOrEqual(t, maxSimultaneous.Load(), int32(2),
"Should not exceed max-parallel: 2")
}
// Test that verifies max-parallel: 1 enforces sequential execution
func TestMaxParallel1Sequential(t *testing.T) {
ctx := context.Background()
var currentRunning atomic.Int32
var maxSimultaneous atomic.Int32
var executionOrder []int
var orderMutex sync.Mutex
executors := make([]Executor, 5)
for i := range 5 {
taskID := i
executors[i] = func(ctx context.Context) error {
current := currentRunning.Add(1)
// Track execution order
orderMutex.Lock()
executionOrder = append(executionOrder, taskID)
orderMutex.Unlock()
// Update max if needed
for {
maxValue := maxSimultaneous.Load()
if current <= maxValue || maxSimultaneous.CompareAndSwap(maxValue, current) {
break
}
}
time.Sleep(20 * time.Millisecond)
currentRunning.Add(-1)
return nil
}
}
err := NewParallelExecutor(1, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, int32(1), maxSimultaneous.Load(),
"max-parallel: 1 should only run 1 task at a time")
assert.Len(t, executionOrder, 5, "All 5 tasks should have executed")
}
@@ -0,0 +1,221 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package common
import (
"context"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
// TestMaxParallelJobExecution tests actual job execution with max-parallel
func TestMaxParallelJobExecution(t *testing.T) {
t.Run("MaxParallel=1 Sequential", func(t *testing.T) {
var currentRunning atomic.Int32
var maxConcurrent int32
var executionOrder []int
var mu sync.Mutex
executors := make([]Executor, 5)
for i := range 5 {
taskID := i
executors[i] = func(ctx context.Context) error {
current := currentRunning.Add(1)
// Track max concurrent
for {
maxValue := atomic.LoadInt32(&maxConcurrent)
if current <= maxValue || atomic.CompareAndSwapInt32(&maxConcurrent, maxValue, current) {
break
}
}
mu.Lock()
executionOrder = append(executionOrder, taskID)
mu.Unlock()
time.Sleep(10 * time.Millisecond)
currentRunning.Add(-1)
return nil
}
}
ctx := context.Background()
err := NewParallelExecutor(1, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, int32(1), maxConcurrent, "Should never exceed 1 concurrent execution")
assert.Len(t, executionOrder, 5, "All tasks should execute")
})
t.Run("MaxParallel=3 Limited", func(t *testing.T) {
var currentRunning atomic.Int32
var maxConcurrent int32
executors := make([]Executor, 10)
for i := range 10 {
executors[i] = func(ctx context.Context) error {
current := currentRunning.Add(1)
for {
maxValue := atomic.LoadInt32(&maxConcurrent)
if current <= maxValue || atomic.CompareAndSwapInt32(&maxConcurrent, maxValue, current) {
break
}
}
time.Sleep(20 * time.Millisecond)
currentRunning.Add(-1)
return nil
}
}
ctx := context.Background()
err := NewParallelExecutor(3, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.LessOrEqual(t, int(maxConcurrent), 3, "Should never exceed 3 concurrent executions")
assert.GreaterOrEqual(t, int(maxConcurrent), 1, "Should have at least 1 concurrent execution")
})
t.Run("MaxParallel=0 Uses1Worker", func(t *testing.T) {
var maxConcurrent int32
var currentRunning atomic.Int32
executors := make([]Executor, 5)
for i := range 5 {
executors[i] = func(ctx context.Context) error {
current := currentRunning.Add(1)
for {
maxValue := atomic.LoadInt32(&maxConcurrent)
if current <= maxValue || atomic.CompareAndSwapInt32(&maxConcurrent, maxValue, current) {
break
}
}
time.Sleep(10 * time.Millisecond)
currentRunning.Add(-1)
return nil
}
}
ctx := context.Background()
// When maxParallel is 0 or negative, it defaults to 1
err := NewParallelExecutor(0, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, int32(1), maxConcurrent, "Should use 1 worker when max-parallel is 0")
})
}
// TestMaxParallelWithErrors tests error handling with max-parallel
func TestMaxParallelWithErrors(t *testing.T) {
t.Run("OneTaskFailsOthersContinue", func(t *testing.T) {
var successCount int32
executors := make([]Executor, 5)
for i := range 5 {
taskID := i
executors[i] = func(ctx context.Context) error {
if taskID == 2 {
return assert.AnError
}
atomic.AddInt32(&successCount, 1)
return nil
}
}
ctx := context.Background()
err := NewParallelExecutor(2, executors...)(ctx)
// Should return the error from task 2
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
// Other tasks should still execute
assert.Equal(t, int32(4), successCount, "4 tasks should succeed")
})
t.Run("ContextCancellation", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
var startedCount int32
executors := make([]Executor, 10)
for i := range 10 {
executors[i] = func(ctx context.Context) error {
atomic.AddInt32(&startedCount, 1)
time.Sleep(100 * time.Millisecond)
return nil
}
}
// Cancel after a short delay
go func() {
time.Sleep(30 * time.Millisecond)
cancel()
}()
err := NewParallelExecutor(3, executors...)(ctx)
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.ErrorIs(t, err, context.Canceled) //nolint:testifylint // pre-existing issue from nektos/act
// Not all tasks should start due to cancellation (but timing may vary)
// Just verify cancellation occurred
t.Logf("Started %d tasks before cancellation", startedCount)
})
}
// TestMaxParallelResourceSharing tests resource sharing scenarios
func TestMaxParallelResourceSharing(t *testing.T) {
t.Run("SharedResourceWithMutex", func(t *testing.T) {
var sharedCounter int
var mu sync.Mutex
executors := make([]Executor, 100)
for i := range 100 {
executors[i] = func(ctx context.Context) error {
mu.Lock()
sharedCounter++
mu.Unlock()
return nil
}
}
ctx := context.Background()
err := NewParallelExecutor(10, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, 100, sharedCounter, "All tasks should increment counter")
})
t.Run("ChannelCommunication", func(t *testing.T) {
resultChan := make(chan int, 50)
executors := make([]Executor, 50)
for i := range 50 {
taskID := i
executors[i] = func(ctx context.Context) error {
resultChan <- taskID
return nil
}
}
ctx := context.Background()
err := NewParallelExecutor(5, executors...)(ctx)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
close(resultChan)
results := make(map[int]bool)
for result := range resultChan {
results[result] = true
}
assert.Len(t, results, 50, "All task IDs should be received")
})
}
@@ -9,9 +9,9 @@ import (
"errors" "errors"
"reflect" "reflect"
"strings" "strings"
"sync"
"sync/atomic" "sync/atomic"
"testing" "testing"
"time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -82,45 +82,44 @@ func TestNewConditionalExecutor(t *testing.T) {
assert.Equal(1, falseCount) assert.Equal(1, falseCount)
} }
// concurrencyProbe returns an executor recording the peak number of concurrent copies. Copies func TestNewParallelExecutor(t *testing.T) {
// block until wantActive are in flight so the peak is exact without sleeping, and later copies assert := assert.New(t)
// find the gate already open so the last one still finishes with no partner left.
func concurrencyProbe(wantActive int32) (exec Executor, count, maxActive *atomic.Int32) {
var counted, active, peak atomic.Int32
var once sync.Once
reached := make(chan struct{})
return func(ctx context.Context) error { ctx := context.Background()
counted.Add(1)
running := active.Add(1) var count, activeCount, maxCount atomic.Int32
emptyWorkflow := NewPipelineExecutor(func(ctx context.Context) error {
count.Add(1)
active := activeCount.Add(1)
for { for {
seen := peak.Load() m := maxCount.Load()
if running <= seen || peak.CompareAndSwap(seen, running) { if active <= m || maxCount.CompareAndSwap(m, active) {
break break
} }
} }
if running >= wantActive { time.Sleep(2 * time.Second)
once.Do(func() { close(reached) }) activeCount.Add(-1)
}
<-reached
active.Add(-1)
return nil return nil
}, &counted, &peak })
}
func TestNewParallelExecutor(t *testing.T) { err := NewParallelExecutor(2, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx)
ctx := context.Background()
exec, count, maxActive := concurrencyProbe(2) assert.Equal(int32(3), count.Load(), "should run all 3 executors")
require.NoError(t, NewParallelExecutor(2, exec, exec, exec)(ctx)) assert.Equal(int32(2), maxCount.Load(), "should run at most 2 executors in parallel")
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors") assert.NoError(err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, int32(2), maxActive.Load(), "should run at most 2 executors in parallel")
// parallelism below 1 falls back to a single worker // Reset to test running the executor with 0 parallelism
exec, count, maxActive = concurrencyProbe(1) count.Store(0)
require.NoError(t, NewParallelExecutor(0, exec, exec, exec)(ctx)) activeCount.Store(0)
assert.Equal(t, int32(3), count.Load(), "should run all 3 executors") maxCount.Store(0)
assert.Equal(t, int32(1), maxActive.Load(), "should run at most 1 executor in parallel")
errSingle := NewParallelExecutor(0, emptyWorkflow, emptyWorkflow, emptyWorkflow)(ctx)
assert.Equal(int32(3), count.Load(), "should run all 3 executors")
assert.Equal(int32(1), maxCount.Load(), "should run at most 1 executors in parallel")
assert.NoError(errSingle)
} }
func TestNewParallelExecutorEmpty(t *testing.T) { func TestNewParallelExecutorEmpty(t *testing.T) {
@@ -174,23 +173,6 @@ func TestNewParallelExecutorCanceled(t *testing.T) {
assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act assert.Error(errExpected, err) //nolint:testifylint // pre-existing issue from nektos/act
} }
func TestNewParallelExecutorRunsRemainingAfterFailure(t *testing.T) {
var successCount atomic.Int32
executors := make([]Executor, 5)
for i := range executors {
executors[i] = func(ctx context.Context) error {
if i == 2 {
return errors.New("fake error")
}
successCount.Add(1)
return nil
}
}
require.Error(t, NewParallelExecutor(2, executors...)(context.Background()))
assert.Equal(t, int32(4), successCount.Load(), "a failing executor must not stop the others")
}
func TestExecutorConditionalsAndFinally(t *testing.T) { func TestExecutorConditionalsAndFinally(t *testing.T) {
ctx := context.Background() ctx := context.Background()
var calls []string var calls []string
@@ -15,8 +15,7 @@ import (
"strings" "strings"
"sync" "sync"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/internal/pkg/lock"
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/config"
@@ -33,7 +32,7 @@ var (
githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`) githubHTTPRegex = regexp.MustCompile(`^https?://.*github.com.*/(.+)/(.+?)(?:.git)?$`)
githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`) githubSSHRegex = regexp.MustCompile(`github.com[:/](.+)/(.+?)(?:.git)?$`)
cloneLocks lock.Keyed[string] // key: clone target directory cloneLocks sync.Map // key: clone target directory; value: *sync.Mutex
ErrShortRef = errors.New("short SHA references are not supported") ErrShortRef = errors.New("short SHA references are not supported")
ErrNoRepo = errors.New("unable to find git repo") ErrNoRepo = errors.New("unable to find git repo")
@@ -44,7 +43,10 @@ var (
// Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too, // Callers reading files inside dir (e.g. tarring a checked-out action into a job container) must hold this lock too,
// otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read. // otherwise a concurrent NewGitCloneExecutor on the same dir can mutate the worktree mid-read.
func AcquireCloneLock(dir string) func() { func AcquireCloneLock(dir string) func() {
return cloneLocks.Lock(dir) v, _ := cloneLocks.LoadOrStore(dir, &sync.Mutex{})
mu := v.(*sync.Mutex)
mu.Lock()
return mu.Unlock
} }
type Error struct { type Error struct {
@@ -12,11 +12,12 @@ import (
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings" "strings"
"sync"
"syscall" "syscall"
"testing" "testing"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
logrustest "github.com/sirupsen/logrus/hooks/test" logrustest "github.com/sirupsen/logrus/hooks/test"
@@ -609,4 +610,12 @@ func TestAcquireCloneLock(t *testing.T) {
t.Fatal("acquire on a different directory must not block") t.Fatal("acquire on a different directory must not block")
} }
}) })
t.Run("same directory reuses the same mutex", func(t *testing.T) {
dir := t.TempDir()
v1, _ := cloneLocks.LoadOrStore(dir, &sync.Mutex{})
v2, _ := cloneLocks.LoadOrStore(dir, &sync.Mutex{})
require.Same(t, v1, v2)
})
} }
@@ -19,9 +19,7 @@ func GetOutboundIP() net.IP {
conn, err := net.Dial("udp", "8.8.8.8:80") conn, err := net.Dial("udp", "8.8.8.8:80")
if err == nil { if err == nil {
defer conn.Close() defer conn.Close()
if addr, ok := conn.LocalAddr().(*net.UDPAddr); ok { return conn.LocalAddr().(*net.UDPAddr).IP
return addr.IP
}
} }
// So the machine cannot access the internet. Pick an IP address from network interfaces. // So the machine cannot access the internet. Pick an IP address from network interfaces.
@@ -6,14 +6,12 @@ package container
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io" "io"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/docker/go-connections/nat" "github.com/docker/go-connections/nat"
"github.com/moby/moby/api/types/container"
) )
// ExitCodeError reports a non-zero process exit code from a container command. // ExitCodeError reports a non-zero process exit code from a container command.
@@ -59,32 +57,6 @@ type FileEntry struct {
Body string Body string
} }
// Container and healthcheck states, as plain strings so a caller of Info needs no docker
// SDK of its own.
const (
StateRunning = string(container.StateRunning)
HealthNone = string(container.NoHealthcheck)
HealthStarting = string(container.Starting)
HealthHealthy = string(container.Healthy)
HealthUnhealthy = string(container.Unhealthy)
)
// ErrContainerNotFound reports a container the daemon no longer knows. Its text is a
// fragment, missingContainerError composes it into the message every operation shares.
var ErrContainerNotFound = errors.New("does not exist")
// Info is a snapshot of a container, as of one inspect.
type Info struct {
ID string
State string // the docker container state: "created", "running", "exited", ...
ExitCode int
Health string // one of the Health* constants
// HealthOutput is the last healthcheck probe's output.
HealthOutput string
Ports map[string]string // container port ("5432") to the host port it is published on
}
// Container for managing docker run containers // Container for managing docker run containers
type Container interface { type Container interface {
Create(capAdd, capDrop []string) common.Executor Create(capAdd, capDrop []string) common.Executor
@@ -93,8 +65,6 @@ type Container interface {
CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error CopyTarStream(ctx context.Context, destPath string, tarStream io.Reader) error
CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor CopyDir(destPath, srcPath string, useGitIgnore bool) common.Executor
GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error)
Inspect(ctx context.Context) (*Info, error)
DumpLogs(ctx context.Context) error
Pull(forcePull bool) common.Executor Pull(forcePull bool) common.Executor
Start(attach bool) common.Executor Start(attach bool) common.Executor
Exec(command []string, env map[string]string, user, workdir string) common.Executor Exec(command []string, env map[string]string, user, workdir string) common.Executor
@@ -112,14 +82,12 @@ type NewDockerBuildExecutorInput struct {
BuildContext io.Reader BuildContext io.Reader
ImageTag string ImageTag string
Platform string Platform string
BuildArgs map[string]*string
} }
// NewDockerNetworkCreateExecutorInput the input for the NewDockerNetworkCreateExecutor function // NewDockerNetworkCreateExecutorInput the input for the NewDockerNetworkCreateExecutor function
type NewDockerNetworkCreateExecutorInput struct { type NewDockerNetworkCreateExecutorInput struct {
EnableIPv4 *bool EnableIPv4 *bool
EnableIPv6 *bool EnableIPv6 *bool
RunnerUUID string
} }
// NewDockerPullExecutorInput the input for the NewDockerPullExecutor function // NewDockerPullExecutorInput the input for the NewDockerPullExecutor function
@@ -9,7 +9,7 @@ package container
import ( import (
"context" "context"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/distribution/reference" "github.com/distribution/reference"
"github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config"
@@ -12,7 +12,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/moby/go-archive" "github.com/moby/go-archive"
"github.com/moby/go-archive/compression" "github.com/moby/go-archive/compression"
@@ -49,7 +49,6 @@ func NewDockerBuildExecutor(input NewDockerBuildExecutorInput) common.Executor {
Remove: true, Remove: true,
AuthConfigs: LoadDockerAuthConfigs(ctx), AuthConfigs: LoadDockerAuthConfigs(ctx),
Dockerfile: input.Dockerfile, Dockerfile: input.Dockerfile,
BuildArgs: input.BuildArgs,
} }
platform, err := parsePlatform(input.Platform) platform, err := parsePlatform(input.Platform)
if err != nil { if err != nil {
@@ -4,10 +4,8 @@
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd)) //go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts.go with: // This file is exact copy of https://github.com/docker/cli/blob/9a471180cb7d39c236d090399a9d362c3f5a8ebd/cli/command/container/opts.go
// * appended with license information // appended with license information.
// * regexp and loader.ParseVolume in place of the import-restricted internal/lazyregexp and internal/volumespec
// * invalidParameter from the package's errors.go, and convertPortSet/convertPortMap for the callers in docker_run.go
// //
// docker/cli is licensed under the Apache License, Version 2.0. // docker/cli is licensed under the Apache License, Version 2.0.
// See DOCKER_LICENSE for the full license text. // See DOCKER_LICENSE for the full license text.
@@ -32,7 +30,6 @@ import (
"strings" "strings"
"time" "time"
cerrdefs "github.com/containerd/errdefs"
"github.com/docker/cli/cli/compose/loader" "github.com/docker/cli/cli/compose/loader"
"github.com/docker/cli/opts" "github.com/docker/cli/opts"
"github.com/docker/go-connections/nat" "github.com/docker/go-connections/nat"
@@ -383,7 +380,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
var binds []string var binds []string
volumes := copts.volumes.GetMap() volumes := copts.volumes.GetMap()
// add any bind targets to the list of container volumes // add any bind targets to the list of container volumes
for bind := range volumes { for bind := range copts.volumes.GetMap() {
parsed, err := loader.ParseVolume(bind) parsed, err := loader.ParseVolume(bind)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -518,13 +515,13 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*con
// collect all the environment variables for the container // collect all the environment variables for the container
envVariables, err := opts.ReadKVEnvStrings(copts.envFile.GetSlice(), copts.env.GetSlice()) envVariables, err := opts.ReadKVEnvStrings(copts.envFile.GetSlice(), copts.env.GetSlice())
if err != nil { if err != nil {
return nil, fmt.Errorf("--env-file: %w", err) return nil, err
} }
// collect all the labels for the container // collect all the labels for the container
labels, err := opts.ReadKVStrings(copts.labelsFile.GetSlice(), copts.labels.GetSlice()) labels, err := opts.ReadKVStrings(copts.labelsFile.GetSlice(), copts.labels.GetSlice())
if err != nil { if err != nil {
return nil, fmt.Errorf("--label-file: %w", err) return nil, err
} }
pidMode := container.PidMode(copts.pidMode) pidMode := container.PidMode(copts.pidMode)
@@ -1167,17 +1164,16 @@ func toNetipAddrSlice(ips []string) []netip.Addr {
} }
// invalidParameter wraps an error to indicate it was caused by invalid input. // invalidParameter wraps an error to indicate it was caused by invalid input.
// This is a copy of docker/cli's cli/command/container/errors.go, which is not importable. // This is a local replacement for docker/docker/errdefs.InvalidParameter.
type invalidParameterErr struct{ error } type invalidParameterError struct{ error }
func (invalidParameterErr) InvalidParameter() {} func (e invalidParameterError) InvalidParameter() {}
func (e invalidParameterErr) Unwrap() error { return e.error }
func invalidParameter(err error) error { func invalidParameter(err error) error {
if err == nil || cerrdefs.IsInvalidArgument(err) { if err == nil {
return err return nil
} }
return invalidParameterErr{err} return invalidParameterError{err}
} }
func convertPortSet(ports nat.PortSet) (network.PortSet, error) { func convertPortSet(ports nat.PortSet) (network.PortSet, error) {
@@ -2,22 +2,20 @@
// Copyright 2022 The nektos/act Authors. All rights reserved. // Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT // SPDX-License-Identifier: MIT
// This file is exact copy of https://github.com/docker/cli/blob/dfc4efb1e2ab8c06d70d2a1366ad448d2f917e90/cli/command/container/opts_test.go with: // This file is exact copy of https://github.com/docker/cli/blob/9ac8584acfd501c3f4da0e845e3a40ed15c85041/cli/command/container/opts_test.go with:
// * appended with license information // * appended with license information
// * added tests for the locally changed parseDevice, validateDevice and invalidParameter // * commented out case 'invalid-mixed-network-types' in test TestParseNetworkConfig
// //
// docker/cli is licensed under the Apache License, Version 2.0. // docker/cli is licensed under the Apache License, Version 2.0.
// See DOCKER_LICENSE for the full license text. // See DOCKER_LICENSE for the full license text.
// //
//nolint:gocritic // verbatim copy from docker/cli tests //nolint:depguard,gocritic // verbatim copy from docker/cli tests
package container package container
import ( import (
"errors"
"fmt" "fmt"
"io" "io"
"net"
"net/netip" "net/netip"
"os" "os"
"runtime" "runtime"
@@ -25,23 +23,18 @@ import (
"testing" "testing"
"time" "time"
"github.com/docker/go-connections/nat"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts" "github.com/google/go-cmp/cmp/cmpopts"
"github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/container"
networktypes "github.com/moby/moby/api/types/network" networktypes "github.com/moby/moby/api/types/network"
"github.com/pkg/errors"
"github.com/spf13/pflag" "github.com/spf13/pflag"
"gotest.tools/v3/assert" "gotest.tools/v3/assert"
is "gotest.tools/v3/assert/cmp" is "gotest.tools/v3/assert/cmp"
"gotest.tools/v3/skip" "gotest.tools/v3/skip"
) )
func mustParseMAC(s string) networktypes.HardwareAddr {
mac, err := net.ParseMAC(s)
if err != nil {
panic(err)
}
return networktypes.HardwareAddr(mac)
}
func TestValidateAttach(t *testing.T) { func TestValidateAttach(t *testing.T) {
valid := []string{ valid := []string{
"stdin", "stdin",
@@ -71,12 +64,12 @@ func parseRun(args []string) (*container.Config, *container.HostConfig, *network
if err := flags.Parse(args); err != nil { if err := flags.Parse(args); err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
// TODO(dnephin): fix tests to accept ContainerConfig; see https://github.com/moby/moby/pull/31621 // TODO: fix tests to accept ContainerConfig
containerCfg, err := parse(flags, copts, runtime.GOOS) containerConfig, err := parse(flags, copts, runtime.GOOS)
if err != nil { if err != nil {
return nil, nil, nil, err return nil, nil, nil, err
} }
return containerCfg.Config, containerCfg.HostConfig, containerCfg.NetworkingConfig, err return containerConfig.Config, containerConfig.HostConfig, containerConfig.NetworkingConfig, err
} }
func setupRunFlags() (*pflag.FlagSet, *containerOptions) { func setupRunFlags() (*pflag.FlagSet, *containerOptions) {
@@ -89,81 +82,20 @@ func setupRunFlags() (*pflag.FlagSet, *containerOptions) {
func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) { func mustParse(t *testing.T, args string) (*container.Config, *container.HostConfig, *networktypes.NetworkingConfig) {
t.Helper() t.Helper()
config, hostConfig, nwConfig, err := parseRun(append(strings.Split(args, " "), "ubuntu", "bash")) config, hostConfig, networkingConfig, err := parseRun(append(strings.Split(args, " "), "ubuntu", "bash"))
assert.NilError(t, err) assert.NilError(t, err)
return config, hostConfig, nwConfig return config, hostConfig, networkingConfig
} }
func TestParseRunLinks(t *testing.T) { func TestParseRunLinks(t *testing.T) {
tests := []struct { if _, hostConfig, _ := mustParse(t, "--link a:b"); len(hostConfig.Links) == 0 || hostConfig.Links[0] != "a:b" {
name string t.Fatalf("Error parsing links. Expected []string{\"a:b\"}, received: %v", hostConfig.Links)
input string
expHostConfigLinks []string
expNetConfigLinks map[string][]string
}{
// Default bridge - legacy links ...
{
name: "default/onelink",
input: "--link a:b",
expHostConfigLinks: []string{"a:b"},
expNetConfigLinks: map[string][]string{"default": nil},
},
{
name: "default/twolinks",
input: "--link a:b --link c:d",
expHostConfigLinks: []string{"a:b", "c:d"},
expNetConfigLinks: map[string][]string{"default": nil},
},
{
name: "bridge/onelink",
input: "--network bridge --link a:b",
expHostConfigLinks: []string{"a:b"},
// expNetConfigLinks - no EndpointsConfig is created for a single named network with no options set.
// See the "For backward compatibility" comment in parseNetworkOpts().
},
{
name: "default/nolinks",
expNetConfigLinks: map[string][]string{"default": nil},
},
// User-defined bridge - links become DNS aliases ...
{
name: "userdefnet/onelink",
input: "--network userdefnet --link a:b",
expHostConfigLinks: []string{"a:b"},
expNetConfigLinks: map[string][]string{"userdefnet": {"a:b"}},
},
{
name: "userdefnet/twolinks",
input: "--network userdefnet --link a:b --link c:d",
expHostConfigLinks: []string{"a:b", "c:d"},
expNetConfigLinks: map[string][]string{"userdefnet": {"a:b", "c:d"}},
},
{
name: "userdefnet/nolinks",
input: "--network userdefnet",
},
{
// Link options are applied to the first network (and there's no "advanced syntax"
// link key, like "--network name=userdefnet,link=a:b").
name: "links apply to the first network",
input: "--network userdefnet --link a:b --network bar --link c:d",
expHostConfigLinks: []string{"a:b", "c:d"},
expNetConfigLinks: map[string][]string{"userdefnet": {"a:b", "c:d"}, "bar": nil},
},
} }
if _, hostConfig, _ := mustParse(t, "--link a:b --link c:d"); len(hostConfig.Links) < 2 || hostConfig.Links[0] != "a:b" || hostConfig.Links[1] != "c:d" {
for _, tc := range tests { t.Fatalf("Error parsing links. Expected []string{\"a:b\", \"c:d\"}, received: %v", hostConfig.Links)
t.Run(tc.name, func(t *testing.T) {
_, hostConfig, netConfig := mustParse(t, tc.input)
assert.Check(t, is.DeepEqual(hostConfig.Links, tc.expHostConfigLinks))
assert.Check(t, is.Len(netConfig.EndpointsConfig, len(tc.expNetConfigLinks)))
for netName, expLinks := range tc.expNetConfigLinks {
nc, ok := netConfig.EndpointsConfig[netName]
assert.Assert(t, ok)
assert.Check(t, is.DeepEqual(nc.Links, expLinks))
} }
}) if _, hostConfig, _ := mustParse(t, ""); len(hostConfig.Links) != 0 {
t.Fatalf("Error parsing links. No link expected, received: %v", hostConfig.Links)
} }
} }
@@ -362,7 +294,37 @@ func compareRandomizedStrings(a, b, c, d string) error {
if a == d && b == c { if a == d && b == c {
return nil return nil
} }
return errors.New("strings don't match") return errors.Errorf("strings don't match")
}
func mustNetworkPort(t *testing.T, value string) networktypes.Port {
t.Helper()
port, err := networktypes.ParsePort(value)
if err != nil {
t.Fatalf("failed to parse network port %q: %v", value, err)
}
return port
}
func mustAddr(t *testing.T, value string) netip.Addr {
t.Helper()
addr, err := netip.ParseAddr(value)
if err != nil {
t.Fatalf("failed to parse address %q: %v", value, err)
}
return addr
}
func mustAddrs(t *testing.T, values ...string) []netip.Addr {
t.Helper()
addrs := make([]netip.Addr, 0, len(values))
for _, value := range values {
addrs = append(addrs, mustAddr(t, value))
}
return addrs
} }
// Simple parse with MacAddress validation // Simple parse with MacAddress validation
@@ -372,11 +334,10 @@ func TestParseWithMacAddress(t *testing.T) {
if _, _, _, err := parseRun([]string{invalidMacAddress, "img", "cmd"}); err != nil && err.Error() != "invalidMacAddress is not a valid mac address" { if _, _, _, err := parseRun([]string{invalidMacAddress, "img", "cmd"}); err != nil && err.Error() != "invalidMacAddress is not a valid mac address" {
t.Fatalf("Expected an error with %v mac-address, got %v", invalidMacAddress, err) t.Fatalf("Expected an error with %v mac-address, got %v", invalidMacAddress, err)
} }
_, hostConfig, nwConfig := mustParse(t, validMacAddress) _, hostConfig, networkingConfig := mustParse(t, validMacAddress)
defaultNw := hostConfig.NetworkMode.NetworkName() endpoint := networkingConfig.EndpointsConfig[string(hostConfig.NetworkMode)]
if nwConfig.EndpointsConfig[defaultNw].MacAddress.String() != "92:d0:c6:0a:29:33" { assert.Check(t, endpoint != nil)
t.Fatalf("Expected the default endpoint to have the MacAddress '92:d0:c6:0a:29:33' set, got '%v'", nwConfig.EndpointsConfig[defaultNw].MacAddress) assert.Equal(t, "92:d0:c6:0a:29:33", endpoint.MacAddress.String())
}
} }
func TestRunFlagsParseWithMemory(t *testing.T) { func TestRunFlagsParseWithMemory(t *testing.T) {
@@ -447,144 +408,93 @@ func TestParseHostnameDomainname(t *testing.T) {
} }
func TestParseWithExpose(t *testing.T) { func TestParseWithExpose(t *testing.T) {
t.Run("invalid", func(t *testing.T) { invalids := []string{
tests := map[string]string{ ":",
":": `invalid range format for --expose: invalid start port ':': invalid syntax`, "8080:9090",
"8080:9090": `invalid range format for --expose: invalid start port '8080:9090': invalid syntax`, "/tcp",
"/tcp": `invalid range format for --expose: invalid start port '': value is empty`, "/udp",
"/udp": `invalid range format for --expose: invalid start port '': value is empty`, "NaN/tcp",
"NaN/tcp": `invalid range format for --expose: invalid start port 'NaN': invalid syntax`, "NaN-NaN/tcp",
"NaN-NaN/tcp": `invalid range format for --expose: invalid start port 'NaN': invalid syntax`, "8080-NaN/tcp",
"8080-NaN/tcp": `invalid range format for --expose: invalid end port 'NaN': invalid syntax`, "1234567890-8080/tcp",
"1234567890-8080/tcp": `invalid range format for --expose: invalid start port '1234567890': value out of range`,
} }
for expose, expectedError := range tests { valids := map[string][]nat.Port{
t.Run(expose, func(t *testing.T) { "8080/tcp": {"8080/tcp"},
_, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"}) "8080/udp": {"8080/udp"},
assert.Error(t, err, expectedError) "8080/ncp": {"8080/ncp"},
}) "8080-8080/udp": {"8080/udp"},
"8080-8082/tcp": {"8080/tcp", "8081/tcp", "8082/tcp"},
} }
}) for _, expose := range invalids {
t.Run("valid", func(t *testing.T) { if _, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"}); err == nil {
tests := map[string][]networktypes.Port{ t.Fatalf("Expected error with '--expose=%v', got none", expose)
"8080/tcp": {networktypes.MustParsePort("8080/tcp")},
"8080/udp": {networktypes.MustParsePort("8080/udp")},
"8080/ncp": {networktypes.MustParsePort("8080/ncp")},
"8080-8080/udp": {networktypes.MustParsePort("8080/udp")},
"8080-8082/tcp": {networktypes.MustParsePort("8080/tcp"), networktypes.MustParsePort("8081/tcp"), networktypes.MustParsePort("8082/tcp")},
} }
for expose, exposedPorts := range tests { }
t.Run(expose, func(t *testing.T) { for expose, exposedPorts := range valids {
config, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"}) config, _, _, err := parseRun([]string{fmt.Sprintf("--expose=%v", expose), "img", "cmd"})
assert.NilError(t, err) if err != nil {
t.Fatal(err)
}
if len(config.ExposedPorts) != len(exposedPorts) {
t.Fatalf("Expected %v exposed port, got %v", len(exposedPorts), len(config.ExposedPorts))
}
for _, port := range exposedPorts { for _, port := range exposedPorts {
_, ok := config.ExposedPorts[port] if _, ok := config.ExposedPorts[mustNetworkPort(t, string(port))]; !ok {
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port]) t.Fatalf("Expected %v, got %v", exposedPorts, config.ExposedPorts)
}
} }
})
} }
})
t.Run("merge with published", func(t *testing.T) {
// Merge with actual published port // Merge with actual published port
config, _, _, err := parseRun([]string{"--publish=80", "--expose=80-81/tcp", "img", "cmd"}) config, _, _, err := parseRun([]string{"--publish=80", "--expose=80-81/tcp", "img", "cmd"})
assert.NilError(t, err) if err != nil {
assert.Check(t, is.Len(config.ExposedPorts, 2)) t.Fatal(err)
ports := []networktypes.Port{networktypes.MustParsePort("80/tcp"), networktypes.MustParsePort("81/tcp")} }
if len(config.ExposedPorts) != 2 {
t.Fatalf("Expected 2 exposed ports, got %v", config.ExposedPorts)
}
ports := []nat.Port{"80/tcp", "81/tcp"}
for _, port := range ports { for _, port := range ports {
_, ok := config.ExposedPorts[port] if _, ok := config.ExposedPorts[mustNetworkPort(t, string(port))]; !ok {
assert.Check(t, ok, "missing port %q in exposed ports: %#+v", port, config.ExposedPorts[port]) t.Fatalf("Expected %v, got %v", ports, config.ExposedPorts)
}
} }
})
} }
func TestParseDevice(t *testing.T) { func TestParseDevice(t *testing.T) {
skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side skip.If(t, runtime.GOOS != "linux") // Windows and macOS validate server-side
testCases := []struct { valids := map[string]container.DeviceMapping{
devices []string "/dev/snd": {
deviceMapping *container.DeviceMapping
deviceRequests []container.DeviceRequest
}{
{
devices: []string{"/dev/snd"},
deviceMapping: &container.DeviceMapping{
PathOnHost: "/dev/snd", PathOnHost: "/dev/snd",
PathInContainer: "/dev/snd", PathInContainer: "/dev/snd",
CgroupPermissions: "rwm", CgroupPermissions: "rwm",
}, },
}, "/dev/snd:rw": {
{
devices: []string{"/dev/snd:rw"},
deviceMapping: &container.DeviceMapping{
PathOnHost: "/dev/snd", PathOnHost: "/dev/snd",
PathInContainer: "/dev/snd", PathInContainer: "/dev/snd",
CgroupPermissions: "rw", CgroupPermissions: "rw",
}, },
}, "/dev/snd:/something": {
{
devices: []string{"/dev/snd:/something"},
deviceMapping: &container.DeviceMapping{
PathOnHost: "/dev/snd", PathOnHost: "/dev/snd",
PathInContainer: "/something", PathInContainer: "/something",
CgroupPermissions: "rwm", CgroupPermissions: "rwm",
}, },
}, "/dev/snd:/something:rw": {
{
devices: []string{"/dev/snd:/something:rw"},
deviceMapping: &container.DeviceMapping{
PathOnHost: "/dev/snd", PathOnHost: "/dev/snd",
PathInContainer: "/something", PathInContainer: "/something",
CgroupPermissions: "rw", CgroupPermissions: "rw",
}, },
},
{
devices: []string{"vendor.com/class=name"},
deviceMapping: nil,
deviceRequests: []container.DeviceRequest{
{
Driver: "cdi",
DeviceIDs: []string{"vendor.com/class=name"},
},
},
},
{
devices: []string{"vendor.com/class=name", "/dev/snd:/something:rw"},
deviceMapping: &container.DeviceMapping{
PathOnHost: "/dev/snd",
PathInContainer: "/something",
CgroupPermissions: "rw",
},
deviceRequests: []container.DeviceRequest{
{
Driver: "cdi",
DeviceIDs: []string{"vendor.com/class=name"},
},
},
},
} }
for device, deviceMapping := range valids {
for _, tc := range testCases { _, hostconfig, _, err := parseRun([]string{fmt.Sprintf("--device=%v", device), "img", "cmd"})
t.Run(fmt.Sprintf("%s", tc.devices), func(t *testing.T) { if err != nil {
var args []string t.Fatal(err)
for _, d := range tc.devices {
args = append(args, fmt.Sprintf("--device=%v", d))
} }
args = append(args, "img", "cmd") if len(hostconfig.Devices) != 1 {
t.Fatalf("Expected 1 devices, got %v", hostconfig.Devices)
_, hostconfig, _, err := parseRun(args)
assert.NilError(t, err)
if tc.deviceMapping != nil {
if assert.Check(t, is.Len(hostconfig.Devices, 1)) {
assert.Check(t, is.DeepEqual(*tc.deviceMapping, hostconfig.Devices[0]))
} }
} else { if hostconfig.Devices[0] != deviceMapping {
assert.Check(t, is.Len(hostconfig.Devices, 0)) t.Fatalf("Expected %v, got %v", deviceMapping, hostconfig.Devices)
} }
assert.Check(t, is.DeepEqual(tc.deviceRequests, hostconfig.DeviceRequests))
})
} }
} }
@@ -666,20 +576,20 @@ func TestParseNetworkConfig(t *testing.T) {
name string name string
flags []string flags []string
expected map[string]*networktypes.EndpointSettings expected map[string]*networktypes.EndpointSettings
expectedHostCfg container.HostConfig expectedCfg container.HostConfig
expectedErr string expectedErr string
}{ }{
{ {
name: "single-network-legacy", name: "single-network-legacy",
flags: []string{"--network", "net1"}, flags: []string{"--network", "net1"},
expected: map[string]*networktypes.EndpointSettings{}, expected: map[string]*networktypes.EndpointSettings{},
expectedHostCfg: container.HostConfig{NetworkMode: "net1"}, expectedCfg: container.HostConfig{NetworkMode: "net1"},
}, },
{ {
name: "single-network-advanced", name: "single-network-advanced",
flags: []string{"--network", "name=net1"}, flags: []string{"--network", "name=net1"},
expected: map[string]*networktypes.EndpointSettings{}, expected: map[string]*networktypes.EndpointSettings{},
expectedHostCfg: container.HostConfig{NetworkMode: "net1"}, expectedCfg: container.HostConfig{NetworkMode: "net1"},
}, },
{ {
name: "single-network-legacy-with-options", name: "single-network-legacy-with-options",
@@ -697,15 +607,15 @@ func TestParseNetworkConfig(t *testing.T) {
expected: map[string]*networktypes.EndpointSettings{ expected: map[string]*networktypes.EndpointSettings{
"net1": { "net1": {
IPAMConfig: &networktypes.EndpointIPAMConfig{ IPAMConfig: &networktypes.EndpointIPAMConfig{
IPv4Address: netip.MustParseAddr("172.20.88.22"), IPv4Address: mustAddr(t, "172.20.88.22"),
IPv6Address: netip.MustParseAddr("2001:db8::8822"), IPv6Address: mustAddr(t, "2001:db8::8822"),
LinkLocalIPs: []netip.Addr{netip.MustParseAddr("169.254.2.2"), netip.MustParseAddr("fe80::169:254:2:2")}, LinkLocalIPs: mustAddrs(t, "169.254.2.2", "fe80::169:254:2:2"),
}, },
Links: []string{"foo:bar", "bar:baz"}, Links: []string{"foo:bar", "bar:baz"},
Aliases: []string{"web1", "web2"}, Aliases: []string{"web1", "web2"},
}, },
}, },
expectedHostCfg: container.HostConfig{NetworkMode: "net1"}, expectedCfg: container.HostConfig{NetworkMode: "net1"},
}, },
{ {
name: "multiple-network-advanced-mixed", name: "multiple-network-advanced-mixed",
@@ -721,15 +631,14 @@ func TestParseNetworkConfig(t *testing.T) {
"--network-alias", "web2", "--network-alias", "web2",
"--network", "net2", "--network", "net2",
"--network", "name=net3,alias=web3,driver-opt=field3=value3,ip=172.20.88.22,ip6=2001:db8::8822", "--network", "name=net3,alias=web3,driver-opt=field3=value3,ip=172.20.88.22,ip6=2001:db8::8822",
"--network", "name=net4,mac-address=02:32:1c:23:00:04,link-local-ip=169.254.169.254",
}, },
expected: map[string]*networktypes.EndpointSettings{ expected: map[string]*networktypes.EndpointSettings{
"net1": { "net1": {
DriverOpts: map[string]string{"field1": "value1"}, DriverOpts: map[string]string{"field1": "value1"},
IPAMConfig: &networktypes.EndpointIPAMConfig{ IPAMConfig: &networktypes.EndpointIPAMConfig{
IPv4Address: netip.MustParseAddr("172.20.88.22"), IPv4Address: mustAddr(t, "172.20.88.22"),
IPv6Address: netip.MustParseAddr("2001:db8::8822"), IPv6Address: mustAddr(t, "2001:db8::8822"),
LinkLocalIPs: []netip.Addr{netip.MustParseAddr("169.254.2.2"), netip.MustParseAddr("fe80::169:254:2:2")}, LinkLocalIPs: mustAddrs(t, "169.254.2.2", "fe80::169:254:2:2"),
}, },
Links: []string{"foo:bar", "bar:baz"}, Links: []string{"foo:bar", "bar:baz"},
Aliases: []string{"web1", "web2"}, Aliases: []string{"web1", "web2"},
@@ -738,23 +647,17 @@ func TestParseNetworkConfig(t *testing.T) {
"net3": { "net3": {
DriverOpts: map[string]string{"field3": "value3"}, DriverOpts: map[string]string{"field3": "value3"},
IPAMConfig: &networktypes.EndpointIPAMConfig{ IPAMConfig: &networktypes.EndpointIPAMConfig{
IPv4Address: netip.MustParseAddr("172.20.88.22"), IPv4Address: mustAddr(t, "172.20.88.22"),
IPv6Address: netip.MustParseAddr("2001:db8::8822"), IPv6Address: mustAddr(t, "2001:db8::8822"),
}, },
Aliases: []string{"web3"}, Aliases: []string{"web3"},
}, },
"net4": {
MacAddress: mustParseMAC("02:32:1c:23:00:04"),
IPAMConfig: &networktypes.EndpointIPAMConfig{
LinkLocalIPs: []netip.Addr{netip.MustParseAddr("169.254.169.254")},
}, },
}, expectedCfg: container.HostConfig{NetworkMode: "net1"},
},
expectedHostCfg: container.HostConfig{NetworkMode: "net1"},
}, },
{ {
name: "single-network-advanced-with-options", name: "single-network-advanced-with-options",
flags: []string{"--network", "name=net1,alias=web1,alias=web2,driver-opt=field1=value1,driver-opt=field2=value2,ip=172.20.88.22,ip6=2001:db8::8822,mac-address=02:32:1c:23:00:04"}, flags: []string{"--network", "name=net1,alias=web1,alias=web2,driver-opt=field1=value1,driver-opt=field2=value2,ip=172.20.88.22,ip6=2001:db8::8822"},
expected: map[string]*networktypes.EndpointSettings{ expected: map[string]*networktypes.EndpointSettings{
"net1": { "net1": {
DriverOpts: map[string]string{ DriverOpts: map[string]string{
@@ -762,31 +665,19 @@ func TestParseNetworkConfig(t *testing.T) {
"field2": "value2", "field2": "value2",
}, },
IPAMConfig: &networktypes.EndpointIPAMConfig{ IPAMConfig: &networktypes.EndpointIPAMConfig{
IPv4Address: netip.MustParseAddr("172.20.88.22"), IPv4Address: mustAddr(t, "172.20.88.22"),
IPv6Address: netip.MustParseAddr("2001:db8::8822"), IPv6Address: mustAddr(t, "2001:db8::8822"),
}, },
Aliases: []string{"web1", "web2"}, Aliases: []string{"web1", "web2"},
MacAddress: mustParseMAC("02:32:1c:23:00:04"),
}, },
}, },
expectedHostCfg: container.HostConfig{NetworkMode: "net1"}, expectedCfg: container.HostConfig{NetworkMode: "net1"},
}, },
{ {
name: "multiple-networks", name: "multiple-networks",
flags: []string{"--network", "net1", "--network", "name=net2"}, flags: []string{"--network", "net1", "--network", "name=net2"},
expected: map[string]*networktypes.EndpointSettings{"net1": {}, "net2": {}}, expected: map[string]*networktypes.EndpointSettings{"net1": {}, "net2": {}},
expectedHostCfg: container.HostConfig{NetworkMode: "net1"}, expectedCfg: container.HostConfig{NetworkMode: "net1"},
},
{
name: "advanced-options-with-standalone-mac-address-flag",
flags: []string{"--network=name=net1,alias=foobar", "--mac-address", "52:0f:f3:dc:50:10"},
expected: map[string]*networktypes.EndpointSettings{
"net1": {
Aliases: []string{"foobar"},
MacAddress: mustParseMAC("52:0f:f3:dc:50:10"),
},
},
expectedHostCfg: container.HostConfig{NetworkMode: "net1"},
}, },
{ {
name: "conflict-network", name: "conflict-network",
@@ -808,26 +699,13 @@ func TestParseNetworkConfig(t *testing.T) {
flags: []string{"--network", "name=net1,ip=172.20.88.22,ip6=2001:db8::8822", "--ip6", "2001:db8::8822"}, flags: []string{"--network", "name=net1,ip=172.20.88.22,ip6=2001:db8::8822", "--ip6", "2001:db8::8822"},
expectedErr: `conflicting options: cannot specify both --ip6 and per-network IPv6 address`, expectedErr: `conflicting options: cannot specify both --ip6 and per-network IPv6 address`,
}, },
{ // case is skipped as it fails w/o any change
name: "invalid-mixed-network-types", //
flags: []string{"--network", "name=host", "--network", "net1"}, //{
expectedErr: `conflicting options: cannot attach both user-defined and non-user-defined network-modes`, // name: "invalid-mixed-network-types",
}, // flags: []string{"--network", "name=host", "--network", "net1"},
{ // expectedErr: `conflicting options: cannot attach both user-defined and non-user-defined network-modes`,
name: "conflict-options-link-local-ip", //},
flags: []string{"--network", "name=net1,link-local-ip=169.254.169.254", "--link-local-ip", "169.254.10.8"},
expectedErr: `conflicting options: cannot specify both --link-local-ip and per-network link-local IP addresses`,
},
{
name: "conflict-options-mac-address",
flags: []string{"--network", "name=net1,mac-address=02:32:1c:23:00:04", "--mac-address", "02:32:1c:23:00:04"},
expectedErr: `conflicting options: cannot specify both --mac-address and per-network MAC address`,
},
{
name: "invalid-mac-address",
flags: []string{"--network", "name=net1,mac-address=foobar"},
expectedErr: "foobar is not a valid mac address",
},
} }
for _, tc := range tests { for _, tc := range tests {
@@ -840,8 +718,10 @@ func TestParseNetworkConfig(t *testing.T) {
} }
assert.NilError(t, err) assert.NilError(t, err)
assert.DeepEqual(t, hConfig.NetworkMode, tc.expectedHostCfg.NetworkMode) assert.DeepEqual(t, hConfig.NetworkMode, tc.expectedCfg.NetworkMode)
assert.DeepEqual(t, nwConfig.EndpointsConfig, tc.expected, cmpopts.EquateComparable(netip.Addr{})) if diff := cmp.Diff(tc.expected, nwConfig.EndpointsConfig, cmpopts.EquateComparable(netip.Addr{})); diff != "" {
t.Fatalf("unexpected endpoints (-want +got):\n%s", diff)
}
}) })
} }
} }
@@ -890,84 +770,42 @@ func TestRunFlagsParseShmSize(t *testing.T) {
} }
func TestParseRestartPolicy(t *testing.T) { func TestParseRestartPolicy(t *testing.T) {
tests := []struct { invalids := map[string]string{
input string "always:2:3": "invalid restart policy format: maximum retry count must be an integer",
expected container.RestartPolicy "on-failure:invalid": "invalid restart policy format: maximum retry count must be an integer",
expectedErr string }
}{ valids := map[string]container.RestartPolicy{
{ "": {},
input: "", "always": {
Name: "always",
MaximumRetryCount: 0,
}, },
{ "on-failure:1": {
input: "no", Name: "on-failure",
expected: container.RestartPolicy{
Name: container.RestartPolicyDisabled,
},
},
{
input: ":1",
expectedErr: "invalid restart policy format: no policy provided before colon",
},
{
input: "always",
expected: container.RestartPolicy{
Name: container.RestartPolicyAlways,
},
},
{
input: "always:2:3",
expectedErr: "invalid restart policy format: maximum retry count must be an integer",
},
{
input: "on-failure:1",
expected: container.RestartPolicy{
Name: container.RestartPolicyOnFailure,
MaximumRetryCount: 1, MaximumRetryCount: 1,
}, },
},
{
input: "on-failure:invalid",
expectedErr: "invalid restart policy format: maximum retry count must be an integer",
},
{
input: "unless-stopped",
expected: container.RestartPolicy{
Name: container.RestartPolicyUnlessStopped,
},
},
{
input: "unless-stopped:invalid",
expectedErr: "invalid restart policy format: maximum retry count must be an integer",
},
// Unknown / invalid combinations: validation is handled by the daemon>
{
input: "anything:123",
expected: container.RestartPolicy{Name: "anything", MaximumRetryCount: 123},
},
{
input: "negative:-123",
expected: container.RestartPolicy{Name: "negative", MaximumRetryCount: -123},
},
} }
for _, tc := range tests { for restart, expectedError := range invalids {
t.Run(tc.input, func(t *testing.T) { if _, _, _, err := parseRun([]string{"--restart=" + restart, "img", "cmd"}); err == nil || err.Error() != expectedError {
_, hostConfig, _, err := parseRun([]string{"--restart=" + tc.input, "img", "cmd"}) t.Fatalf("Expected an error with message '%v' for %v, got %v", expectedError, restart, err)
if tc.expectedErr != "" { }
assert.Check(t, is.Error(err, tc.expectedErr)) }
assert.Check(t, is.Nil(hostConfig)) for restart, expected := range valids {
} else { _, hostconfig, _, err := parseRun([]string{fmt.Sprintf("--restart=%v", restart), "img", "cmd"})
assert.NilError(t, err) if err != nil {
assert.Check(t, is.DeepEqual(hostConfig.RestartPolicy, tc.expected)) t.Fatal(err)
}
if hostconfig.RestartPolicy != expected {
t.Fatalf("Expected %v, got %v", expected, hostconfig.RestartPolicy)
} }
})
} }
} }
func TestParseRestartPolicyAutoRemove(t *testing.T) { func TestParseRestartPolicyAutoRemove(t *testing.T) {
_, _, _, err := parseRun([]string{"--rm", "--restart=always", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests _, _, _, err := parseRun([]string{"--rm", "--restart=always", "img", "cmd"}) //nolint:dogsled // verbatim copy from docker/cli tests
const expected = "conflicting options: cannot specify both --restart and --rm" if err == nil {
assert.Check(t, is.Error(err, expected)) t.Fatal("Expected error for conflicting --restart and --rm, but got none")
}
} }
func TestParseHealth(t *testing.T) { func TestParseHealth(t *testing.T) {
@@ -1003,8 +841,8 @@ func TestParseHealth(t *testing.T) {
checkError("--no-healthcheck conflicts with --health-* options", checkError("--no-healthcheck conflicts with --health-* options",
"--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd") "--no-healthcheck", "--health-cmd=/check.sh -q", "img", "cmd")
health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "--health-start-interval=1s", "img", "cmd") health = checkOk("--health-timeout=2s", "--health-retries=3", "--health-interval=4.5s", "--health-start-period=5s", "img", "cmd")
if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second || health.StartInterval != 1*time.Second { if health.Timeout != 2*time.Second || health.Retries != 3 || health.Interval != 4500*time.Millisecond || health.StartPeriod != 5*time.Second {
t.Fatalf("--health-*: got %#v", health) t.Fatalf("--health-*: got %#v", health)
} }
} }
@@ -1025,13 +863,13 @@ func TestParseLoggingOpts(t *testing.T) {
} }
func TestParseEnvfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests func TestParseEnvfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
expErr := "--env-file: open nonexistent: no such file or directory" e := "open nonexistent: no such file or directory"
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
expErr = "--env-file: open nonexistent: The system cannot find the file specified." e = "open nonexistent: The system cannot find the file specified."
} }
// env ko // env ko
if _, _, _, err := parseRun([]string{"--env-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr { if _, _, _, err := parseRun([]string{"--env-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != e {
t.Fatalf("Expected an error with message '%s', got %v", expErr, err) t.Fatalf("Expected an error with message '%s', got %v", e, err)
} }
// env ok // env ok
config, _, _, err := parseRun([]string{"--env-file=testdata/valid.env", "img", "cmd"}) config, _, _, err := parseRun([]string{"--env-file=testdata/valid.env", "img", "cmd"})
@@ -1067,7 +905,7 @@ func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
} }
// UTF16 with BOM // UTF16 with BOM
e := "invalid utf8 bytes at line" e := "invalid env file"
if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) { if _, _, _, err := parseRun([]string{"--env-file=testdata/utf16.env", "img", "cmd"}); err == nil || !strings.Contains(err.Error(), e) {
t.Fatalf("Expected an error with message '%s', got %v", e, err) t.Fatalf("Expected an error with message '%s', got %v", e, err)
} }
@@ -1078,13 +916,13 @@ func TestParseEnvfileVariablesWithBOMUnicode(t *testing.T) {
} }
func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy from docker/cli tests
expErr := "--label-file: open nonexistent: no such file or directory" e := "open nonexistent: no such file or directory"
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
expErr = "--label-file: open nonexistent: The system cannot find the file specified." e = "open nonexistent: The system cannot find the file specified."
} }
// label ko // label ko
if _, _, _, err := parseRun([]string{"--label-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != expErr { if _, _, _, err := parseRun([]string{"--label-file=nonexistent", "img", "cmd"}); err == nil || err.Error() != e {
t.Fatalf("Expected an error with message '%s', got %v", expErr, err) t.Fatalf("Expected an error with message '%s', got %v", e, err)
} }
// label ok // label ok
config, _, _, err := parseRun([]string{"--label-file=testdata/valid.label", "img", "cmd"}) config, _, _, err := parseRun([]string{"--label-file=testdata/valid.label", "img", "cmd"})
@@ -1105,8 +943,12 @@ func TestParseLabelfileVariables(t *testing.T) { //nolint:dupl // verbatim copy
func TestParseEntryPoint(t *testing.T) { func TestParseEntryPoint(t *testing.T) {
config, _, _, err := parseRun([]string{"--entrypoint=anything", "cmd", "img"}) config, _, _, err := parseRun([]string{"--entrypoint=anything", "cmd", "img"})
assert.NilError(t, err) if err != nil {
assert.Check(t, is.DeepEqual(config.Entrypoint, []string{"anything"})) t.Fatal(err)
}
if len(config.Entrypoint) != 1 && config.Entrypoint[0] != "anything" {
t.Fatalf("Expected entrypoint 'anything', got %v", config.Entrypoint)
}
} }
func TestValidateDevice(t *testing.T) { func TestValidateDevice(t *testing.T) {
@@ -1153,11 +995,13 @@ func TestValidateDevice(t *testing.T) {
for path, expectedError := range invalid { for path, expectedError := range invalid {
if _, err := validateDevice(path, runtime.GOOS); err == nil { if _, err := validateDevice(path, runtime.GOOS); err == nil {
t.Fatalf("ValidateDevice(`%q`) should have failed validation", path) t.Fatalf("ValidateDevice(`%q`) should have failed validation", path)
} else if err.Error() != expectedError { } else {
if err.Error() != expectedError {
t.Fatalf("ValidateDevice(`%q`) error should contain %q, got %q", path, expectedError, err.Error()) t.Fatalf("ValidateDevice(`%q`) error should contain %q, got %q", path, expectedError, err.Error())
} }
} }
} }
}
func TestValidateDeviceByServerOS(t *testing.T) { func TestValidateDeviceByServerOS(t *testing.T) {
tests := []struct { tests := []struct {
@@ -1229,12 +1073,10 @@ func TestDeviceCgroupRulesAndInvalidParameter(t *testing.T) {
if invalidParameter(nil) != nil { if invalidParameter(nil) != nil {
t.Fatal("invalidParameter(nil) should be nil") t.Fatal("invalidParameter(nil) should be nil")
} }
cause := errors.New("bad input") err = invalidParameter(errors.New("bad input"))
err = invalidParameter(cause) assert.Assert(t, err != nil)
var invalid interface{ InvalidParameter() } var invalid interface{ InvalidParameter() }
assert.Assert(t, errors.As(err, &invalid)) assert.Assert(t, errors.As(err, &invalid))
assert.Assert(t, errors.Is(err, cause))
assert.Equal(t, invalidParameter(err), err) // already invalid, so not wrapped twice
} }
func TestParseSystemPaths(t *testing.T) { func TestParseSystemPaths(t *testing.T) {
@@ -57,7 +57,7 @@ func logDockerResponse(logger logrus.FieldLogger, dockerResponse io.ReadCloser,
if msg.ErrorDetail.Message != "" { if msg.ErrorDetail.Message != "" {
writeLog(logger, isError, "%s", msg.ErrorDetail.Message) writeLog(logger, isError, "%s", msg.ErrorDetail.Message)
return errors.New(msg.ErrorDetail.Message) return errors.New(msg.Error)
} }
if msg.Status != "" { if msg.Status != "" {
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// Copyright 2023 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
//go:build !(WITHOUT_DOCKER || !(linux || darwin || windows || netbsd))
package container
import (
"context"
"gitea.com/gitea/runner/act/common"
"github.com/moby/moby/client"
)
func NewDockerNetworkCreateExecutor(name string, opts NewDockerNetworkCreateExecutorInput) common.Executor {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
// Only create the network if it doesn't exist
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
for _, n := range networks.Items {
if n.Name == name {
common.Logger(ctx).Debugf("Network %v exists", name)
return nil
}
}
_, err = cli.NetworkCreate(ctx, name, client.NetworkCreateOptions{
Driver: "bridge",
Scope: "local",
EnableIPv4: opts.EnableIPv4,
EnableIPv6: opts.EnableIPv6,
})
if err != nil {
return err
}
return nil
}
}
func NewDockerNetworkRemoveExecutor(name string) common.Executor {
return func(ctx context.Context) error {
cli, err := GetDockerClient(ctx)
if err != nil {
return err
}
defer cli.Close()
// Make sure that all network of the specified name are removed
// cli.NetworkRemove refuses to remove a network if there are duplicates
networks, err := cli.NetworkList(ctx, client.NetworkListOptions{})
if err != nil {
return err
}
// For Gitea, reduce log noise
// common.Logger(ctx).Debugf("%v", networks)
for _, n := range networks.Items {
if n.Name == name {
result, err := cli.NetworkInspect(ctx, n.ID, client.NetworkInspectOptions{})
if err != nil {
return err
}
if len(result.Network.Containers) == 0 {
if _, err = cli.NetworkRemove(ctx, n.ID, client.NetworkRemoveOptions{}); err != nil {
common.Logger(ctx).Debugf("%v", err)
}
} else {
common.Logger(ctx).Debugf("Refusing to remove network %v because it still has active endpoints", name)
}
}
}
return err
}
}
@@ -11,7 +11,7 @@ import (
"fmt" "fmt"
"strings" "strings"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/distribution/reference" "github.com/distribution/reference"
"github.com/moby/moby/api/pkg/authconfig" "github.com/moby/moby/api/pkg/authconfig"
@@ -30,19 +30,23 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
return nil return nil
} }
// skip the pull when the image is already here: either none was forced, or a digest pull := input.ForcePull
// pins the content so a forced pull could only fetch the same bytes again if !pull {
if !input.ForcePull || isPinnedImage(input.Image) {
imageExists, err := ImageExistsLocally(ctx, input.Image, input.Platform) imageExists, err := ImageExistsLocally(ctx, input.Image, input.Platform)
logger.Debugf("Image exists? %v", imageExists) logger.Debugf("Image exists? %v", imageExists)
if err != nil { if err != nil {
return fmt.Errorf("unable to determine if image already exists for image '%s' (%s): %w", input.Image, input.Platform, err) return fmt.Errorf("unable to determine if image already exists for image '%s' (%s): %w", input.Image, input.Platform, err)
} }
if imageExists {
return nil if !imageExists {
pull = true
} }
} }
if !pull {
return nil
}
imageRef := cleanImage(ctx, input.Image) imageRef := cleanImage(ctx, input.Image)
logger.Debugf("pulling image '%v' (%s)", imageRef, input.Platform) logger.Debugf("pulling image '%v' (%s)", imageRef, input.Platform)
@@ -57,34 +61,24 @@ func NewDockerPullExecutor(input NewDockerPullExecutorInput) common.Executor {
return err return err
} }
// the daemon reports a failure that happens after the first progress line in the reader, err := cli.ImagePull(ctx, imageRef, imagePullOptions)
// stream rather than on the call itself, so both have to be checked
pullOnce := func(opts client.ImagePullOptions) error {
reader, err := cli.ImagePull(ctx, imageRef, opts)
streamErr := logDockerResponse(logger, reader, err != nil)
if err != nil {
return err
}
return streamErr
}
err = pullOnce(imagePullOptions) _ = logDockerResponse(logger, reader, err != nil)
if err != nil && imagePullOptions.RegistryAuth != "" && strings.Contains(err.Error(), "unauthorized") { if err != nil {
if imagePullOptions.RegistryAuth != "" && strings.Contains(err.Error(), "unauthorized") {
logger.Errorf("pulling image '%v' (%s) failed with credentials %s retrying without them, please check for stale docker config files", imageRef, input.Platform, err.Error()) logger.Errorf("pulling image '%v' (%s) failed with credentials %s retrying without them, please check for stale docker config files", imageRef, input.Platform, err.Error())
imagePullOptions.RegistryAuth = "" imagePullOptions.RegistryAuth = ""
err = pullOnce(imagePullOptions) reader, err = cli.ImagePull(ctx, imageRef, imagePullOptions)
}
if err == nil { _ = logDockerResponse(logger, reader, err != nil)
return nil
}
// a registry that is down should not fail a job whose image is already here
if exists, existsErr := ImageExistsLocally(ctx, input.Image, input.Platform); existsErr == nil && exists {
logger.Warnf("could not update image '%s' (%s), continuing with the local copy: %v", imageRef, input.Platform, err)
return nil
} }
if err != nil {
return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err) return fmt.Errorf("failed to pull image '%s' (%s): %w", imageRef, input.Platform, err)
} }
} }
return nil
}
}
func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) { func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput) (client.ImagePullOptions, error) {
imagePullOptions := client.ImagePullOptions{} imagePullOptions := client.ImagePullOptions{}
@@ -128,15 +122,6 @@ func getImagePullOptions(ctx context.Context, input NewDockerPullExecutorInput)
return imagePullOptions, nil return imagePullOptions, nil
} }
func isPinnedImage(image string) bool {
ref, err := reference.ParseAnyReference(image)
if err != nil {
return false
}
_, pinned := ref.(reference.Canonical)
return pinned
}
func cleanImage(ctx context.Context, imageName string) string { func cleanImage(ctx context.Context, imageName string) string {
ref, err := reference.ParseAnyReference(imageName) ref, err := reference.ParseAnyReference(imageName)
if err != nil { if err != nil {
@@ -6,15 +6,11 @@ package container
import ( import (
"context" "context"
"io"
"strings"
"testing" "testing"
"github.com/docker/cli/cli/config" "github.com/docker/cli/cli/config"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/test"
assert "github.com/stretchr/testify/assert" assert "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
) )
func init() { func init() {
@@ -69,21 +65,3 @@ func TestGetImagePullOptions(t *testing.T) {
assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act assert.NoError(t, err, "Failed to create ImagePullOptions") //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config") assert.Equal(t, "eyJ1c2VybmFtZSI6InVzZXJuYW1lIiwicGFzc3dvcmQiOiJwYXNzd29yZFxuIiwic2VydmVyYWRkcmVzcyI6Imh0dHBzOi8vaW5kZXguZG9ja2VyLmlvL3YxLyJ9", options.RegistryAuth, "RegistryAuth should be taken from local docker config")
} }
// A digest-pinned image is immutable, so its local copy is always current.
func TestIsPinnedImage(t *testing.T) {
assert.True(t, isPinnedImage("alpine@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b"))
assert.False(t, isPinnedImage("alpine:latest"))
}
// The pull path reports a failure the daemon sent mid-stream, so it must carry the reason
// whichever of the two shapes the daemon used.
func TestLogDockerResponseError(t *testing.T) {
logger, _ := test.NewNullLogger()
streamErr := func(line string) error {
return logDockerResponse(logger, io.NopCloser(strings.NewReader(line)), false)
}
require.EqualError(t, streamErr(`{"error":"toomanyrequests: rate limit exceeded"}`), "toomanyrequests: rate limit exceeded")
require.EqualError(t, streamErr(`{"errorDetail":{"message":"unexpected EOF"}}`), "unexpected EOF")
require.NoError(t, streamErr(`{"status":"Downloading"}`))
}
@@ -22,10 +22,11 @@ import (
"strings" "strings"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/internal/act/filecollector" "gitea.com/gitea/runner/act/filecollector"
"dario.cat/mergo" "dario.cat/mergo"
"github.com/Masterminds/semver"
cerrdefs "github.com/containerd/errdefs" cerrdefs "github.com/containerd/errdefs"
"github.com/docker/cli/cli/compose/loader" "github.com/docker/cli/cli/compose/loader"
"github.com/docker/cli/cli/connhelper" "github.com/docker/cli/cli/connhelper"
@@ -34,15 +35,15 @@ import (
"github.com/go-git/go-git/v5/plumbing/format/gitignore" "github.com/go-git/go-git/v5/plumbing/format/gitignore"
"github.com/gobwas/glob" "github.com/gobwas/glob"
"github.com/joho/godotenv" "github.com/joho/godotenv"
"github.com/kballard/go-shellquote"
"github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/pkg/stdcopy"
"github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount" "github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network" "github.com/moby/moby/api/types/network"
"github.com/moby/moby/api/types/system" "github.com/moby/moby/api/types/system"
"github.com/moby/moby/client" "github.com/moby/moby/client"
"github.com/moby/moby/client/pkg/versions"
specs "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus" "github.com/spf13/pflag"
) )
// drainGracePeriod bounds how long we wait for an output-copy goroutine to // drainGracePeriod bounds how long we wait for an output-copy goroutine to
@@ -56,12 +57,6 @@ const drainGracePeriod = 2 * time.Second
func NewContainer(input *NewContainerInput) ExecutionsEnvironment { func NewContainer(input *NewContainerInput) ExecutionsEnvironment {
cr := new(containerReference) cr := new(containerReference)
cr.input = input cr.input = input
// Resolved up front because the image pull runs before the container is created.
cf := createFlagsFromOptions(input.Options)
if cf.platform != "" {
cr.input.Platform = cf.platform
}
cr.pullPolicy = cf.pull
return cr return cr
} }
@@ -91,11 +86,19 @@ func (cr *containerReference) connectToNetwork(name string, aliases []string) co
// supportsContainerImagePlatform returns true if the underlying Docker server // supportsContainerImagePlatform returns true if the underlying Docker server
// API version is 1.41 and beyond // API version is 1.41 and beyond
func supportsContainerImagePlatform(ctx context.Context, cli client.APIClient) bool { func supportsContainerImagePlatform(ctx context.Context, cli client.APIClient) bool {
logger := common.Logger(ctx)
ver, err := cli.ServerVersion(ctx, client.ServerVersionOptions{}) ver, err := cli.ServerVersion(ctx, client.ServerVersionOptions{})
if err != nil { if err != nil {
common.Logger(ctx).Panicf("Failed to get Docker API Version: %s", err) logger.Panicf("Failed to get Docker API Version: %s", err)
return false
} }
return versions.GreaterThanOrEqualTo(ver.APIVersion, "1.41") sv, err := semver.NewVersion(ver.APIVersion)
if err != nil {
logger.Panicf("Failed to unmarshal Docker Version: %s", err)
return false
}
constraint, _ := semver.NewConstraint(">= 1.41")
return constraint.Check(sv)
} }
func (cr *containerReference) Create(capAdd, capDrop []string) common.Executor { func (cr *containerReference) Create(capAdd, capDrop []string) common.Executor {
@@ -134,11 +137,6 @@ func (cr *containerReference) Start(attach bool) common.Executor {
} }
func (cr *containerReference) Pull(forcePull bool) common.Executor { func (cr *containerReference) Pull(forcePull bool) common.Executor {
if cr.pullPolicy == pullPolicyNever {
return common.NewInfoExecutor("docker pull skipped image=%s, --pull=never in the options", cr.input.Image)
}
forcePull = forcePull || cr.pullPolicy == pullPolicyAlways
return common. return common.
NewInfoExecutor("docker pull image=%s platform=%s username=%s forcePull=%t", cr.input.Image, cr.input.Platform, cr.input.Username, forcePull). NewInfoExecutor("docker pull image=%s platform=%s username=%s forcePull=%t", cr.input.Image, cr.input.Platform, cr.input.Username, forcePull).
Then( Then(
@@ -197,109 +195,6 @@ func (cr *containerReference) GetContainerArchive(ctx context.Context, srcPath s
return result.Content, nil return result.Content, nil
} }
// Inspect resolves the container by name when its id is not known yet. One the daemon no
// longer knows is reported as ErrContainerNotFound.
func (cr *containerReference) Inspect(ctx context.Context) (*Info, error) {
if common.Dryrun(ctx) {
return &Info{Health: HealthNone, Ports: map[string]string{}}, nil
}
if err := cr.connect()(ctx); err != nil {
return nil, err
}
if cr.id == "" { // a known id is trusted, find() would spend a call validating it
if err := cr.find()(ctx); err != nil {
return nil, err
}
}
if cr.id == "" {
return nil, cr.missingContainerError("inspect it")
}
result, err := cr.cli.ContainerInspect(ctx, cr.id, client.ContainerInspectOptions{})
if cerrdefs.IsNotFound(err) {
return nil, cr.missingContainerError("inspect it")
} else if err != nil {
return nil, err
}
return containerInfoFromInspect(result.Container), nil
}
// DumpLogs copies the container's log so far to its output writers.
func (cr *containerReference) DumpLogs(ctx context.Context) error {
if common.Dryrun(ctx) {
return nil
}
if err := cr.connect()(ctx); err != nil {
return err
}
if cr.id == "" {
return cr.missingContainerError("read its logs")
}
logs, err := cr.cli.ContainerLogs(ctx, cr.id, client.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
if err != nil {
return err
}
defer logs.Close()
return cr.copyOutput(logs)
}
// copyOutput writes a container stream to the writers the container was created with,
// demultiplexing it unless the container has a TTY, which sends a single raw stream.
func (cr *containerReference) copyOutput(stream io.Reader) error {
outWriter := cr.input.Stdout
if outWriter == nil {
outWriter = os.Stdout
}
errWriter := cr.input.Stderr
if errWriter == nil {
errWriter = os.Stderr
}
var err error
if !cr.input.AllocatePTY || os.Getenv("NORAW") != "" {
_, err = stdcopy.StdCopy(outWriter, errWriter, stream)
} else {
_, err = io.Copy(outWriter, stream)
}
// Flush any buffered, not-yet-newline-terminated trailing line so the final line of
// the output is not lost when it is not newline-terminated.
common.FlushWriter(outWriter)
common.FlushWriter(errWriter)
return err
}
func containerInfoFromInspect(inspect container.InspectResponse) *Info {
info := &Info{
ID: inspect.ID,
Health: HealthNone,
Ports: map[string]string{}, // an empty map, never null, in the expression context
}
if state := inspect.State; state != nil {
info.State = string(state.Status)
info.ExitCode = state.ExitCode
if health := state.Health; health != nil {
info.Health = string(health.Status)
if len(health.Log) > 0 {
info.HealthOutput = strings.TrimSpace(health.Log[len(health.Log)-1].Output)
}
}
}
if settings := inspect.NetworkSettings; settings != nil {
for port, bindings := range settings.Ports {
for _, binding := range bindings { // the last binding wins, a port maps to one host port
if binding.HostPort != "" {
info.Ports[port.Port()] = binding.HostPort
}
}
}
}
return info
}
func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor { func (cr *containerReference) UpdateFromEnv(srcPath string, env *map[string]string) common.Executor {
return parseEnvFile(cr, srcPath, env).IfNot(common.Dryrun) return parseEnvFile(cr, srcPath, env).IfNot(common.Dryrun)
} }
@@ -340,7 +235,6 @@ type containerReference struct {
cli client.APIClient cli client.APIClient
id string id string
input *NewContainerInput input *NewContainerInput
pullPolicy string
UID int UID int
GID int GID int
// attachDone is closed by the attach() streaming goroutine once it has // attachDone is closed by the attach() streaming goroutine once it has
@@ -445,10 +339,10 @@ func (cr *containerReference) Close() common.Executor {
} }
} }
// missingContainerError is the shared "container X does not exist" error used by ops that // missingContainerError is the shared "container X does not exist" error
// need a live cr.id, wrapping ErrContainerNotFound so a caller can tell it from a failing daemon. // used by ops that need a live cr.id.
func (cr *containerReference) missingContainerError(format string, args ...any) error { func (cr *containerReference) missingContainerError(format string, args ...any) error {
return fmt.Errorf("container %q %w; cannot "+format, append([]any{cr.input.Name, ErrContainerNotFound}, args...)...) return fmt.Errorf("container %q does not exist; cannot "+format, append([]any{cr.input.Name}, args...)...)
} }
func (cr *containerReference) find() common.Executor { func (cr *containerReference) find() common.Executor {
@@ -482,71 +376,32 @@ func (cr *containerReference) find() common.Executor {
} }
} }
// isContainerGone reports whether a failed remove still left the container gone (NotFound or Conflict).
func isContainerGone(err error) bool {
return cerrdefs.IsNotFound(err) || cerrdefs.IsConflict(err)
}
func (cr *containerReference) remove() common.Executor { func (cr *containerReference) remove() common.Executor {
return func(ctx context.Context) error { return func(ctx context.Context) error {
idOrName := cr.id if cr.id == "" {
if idOrName == "" && cr.input != nil {
idOrName = cr.input.Name
}
if idOrName == "" {
return nil return nil
} }
logger := common.Logger(ctx) logger := common.Logger(ctx)
// Kill first so removal never waits out a daemon's stop timeout: Docker kills outright _, err := cr.cli.ContainerRemove(ctx, cr.id, client.ContainerRemoveOptions{
// on a forced remove, Podman sends SIGTERM and waits. Only worth it for a container
// this started, and removal can still deal with one it could not kill.
if cr.id != "" {
_, err := cr.cli.ContainerKill(ctx, cr.id, client.ContainerKillOptions{Signal: "SIGKILL"})
if err != nil && !cerrdefs.IsConflict(err) && !cerrdefs.IsNotFound(err) {
logger.Debugf("Container %s could not be killed: %v", cr.id, err)
}
}
_, err := cr.cli.ContainerRemove(ctx, idOrName, client.ContainerRemoveOptions{
RemoveVolumes: true, RemoveVolumes: true,
Force: true, Force: true,
}) })
switch { if err != nil && !isContainerGone(err) {
case cerrdefs.IsConflict(err): logger.Error(fmt.Errorf("failed to remove container: %w", err))
// the daemon's own AutoRemove teardown is running, and it releases the volume
// references and the network endpoint only once it finishes
cr.waitForRemoval(ctx, idOrName)
case err != nil && !cerrdefs.IsNotFound(err):
logger.Error(fmt.Errorf("failed to remove container %s: %w", idOrName, err))
return nil // keep the id, the container is still there for a later Remove()
} }
logger.Debugf("Removed container: %v", idOrName) logger.Debugf("Removed container: %v", cr.id)
cr.id = "" cr.id = ""
return nil return nil
} }
} }
func (cr *containerReference) waitForRemoval(ctx context.Context, idOrName string) {
// per container, against the one minute the post-job executor allows for the whole
// cleanup, so a job with several services can spend most of that budget here
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
waitResult := cr.cli.ContainerWait(ctx, idOrName, client.ContainerWaitOptions{
Condition: container.WaitConditionRemoved,
})
select {
case <-waitResult.Result:
case <-waitResult.Error:
case <-ctx.Done():
// the client delivers the result over an unbuffered channel, so leave a receiver
// behind or its goroutine parks on the send for the lifetime of the process
go func() {
select {
case <-waitResult.Result:
case <-waitResult.Error:
}
}()
common.Logger(ctx).Warnf("Timed out waiting for the daemon to remove container %s, its volumes and network may be left behind", idOrName)
}
}
func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) { func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig, error) {
logger := common.Logger(ctx) logger := common.Logger(ctx)
input := cr.input input := cr.input
@@ -556,13 +411,17 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
} }
// parse configuration from CLI container.options // parse configuration from CLI container.options
flags, copts, cf, err := parseContainerOptions(input.Options) flags := pflag.NewFlagSet("container_flags", pflag.ContinueOnError)
copts := addFlags(flags)
optionsArgs, err := shellquote.Split(input.Options)
if err != nil { if err != nil {
return nil, nil, err return nil, nil, fmt.Errorf("Cannot split container options: '%s': '%w'", input.Options, err)
} }
if err := cf.validate(); err != nil { err = flags.Parse(optionsArgs)
return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err) if err != nil {
return nil, nil, fmt.Errorf("Cannot parse container options: '%s': '%w'", input.Options, err)
} }
// FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment. // FIXME: If everything is fine after gitea/act v0.260.0, remove the following comment.
@@ -596,16 +455,6 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err) return nil, nil, fmt.Errorf("Cannot process container options: '%s': '%w'", input.Options, err)
} }
// For Gitea
// When privileged mode is disabled, container.options is workflow-controlled
// untrusted input. Strip the HostConfig fields that would let a workflow break
// out of the container (host namespaces, capability expansion, security profile
// overrides, device and runtime access). Otherwise these survive into the final
// HostConfig even though --privileged is forced off.
if !hostConfig.Privileged {
sanitizeOptionsHostConfig(logger, containerConfig.HostConfig)
}
logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config) logger.Debugf("Custom container.Config from options ==> %+v", containerConfig.Config)
err = mergo.Merge(config, containerConfig.Config, mergo.WithOverride, mergo.WithAppendSlice) err = mergo.Merge(config, containerConfig.Config, mergo.WithOverride, mergo.WithAppendSlice)
@@ -616,7 +465,8 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig) logger.Debugf("Custom container.HostConfig from options ==> %+v", containerConfig.HostConfig)
overlayVolumes(hostConfig, containerConfig.HostConfig) hostConfig.Binds = append(hostConfig.Binds, containerConfig.HostConfig.Binds...)
hostConfig.Mounts = append(hostConfig.Mounts, containerConfig.HostConfig.Mounts...)
binds := hostConfig.Binds binds := hostConfig.Binds
mounts := hostConfig.Mounts mounts := hostConfig.Mounts
networkMode := hostConfig.NetworkMode networkMode := hostConfig.NetworkMode
@@ -626,9 +476,6 @@ func (cr *containerReference) mergeContainerConfigs(ctx context.Context, config
} }
hostConfig.Binds = binds hostConfig.Binds = binds
hostConfig.Mounts = mounts hostConfig.Mounts = mounts
if cf.name != "" {
logger.Warn("--name in the options will be ignored.")
}
if len(copts.netMode.Value()) > 0 { if len(copts.netMode.Value()) > 0 {
logger.Warn("--network and --net in the options will be ignored.") logger.Warn("--network and --net in the options will be ignored.")
} }
@@ -682,7 +529,7 @@ func (cr *containerReference) create(capAdd, capDrop []string) common.Executor {
} }
var platSpecs *specs.Platform var platSpecs *specs.Platform
if cr.input.Platform != "" && supportsContainerImagePlatform(ctx, cr.cli) { if supportsContainerImagePlatform(ctx, cr.cli) && cr.input.Platform != "" {
platSpecs, err = parsePlatform(cr.input.Platform) platSpecs, err = parsePlatform(cr.input.Platform)
if err != nil { if err != nil {
return err return err
@@ -839,7 +686,7 @@ func (cr *containerReference) exec(cmd []string, env map[string]string, user, wo
} }
defer resp.Close() defer resp.Close()
err = cr.waitForCommand(ctx, resp.HijackedResponse, idResp, user, workdir) err = cr.waitForCommand(ctx, isTerminal, resp.HijackedResponse, idResp, user, workdir)
if err != nil { if err != nil {
return err return err
} }
@@ -897,7 +744,7 @@ func (cr *containerReference) tryReadGID() common.Executor {
return cr.tryReadID("-g", func(id int) { cr.GID = id }) return cr.tryReadID("-g", func(id int) { cr.GID = id })
} }
func (cr *containerReference) waitForCommand(ctx context.Context, resp client.HijackedResponse, _ client.ExecCreateResult, _, _ string) error { func (cr *containerReference) waitForCommand(ctx context.Context, isTerminal bool, resp client.HijackedResponse, _ client.ExecCreateResult, _, _ string) error {
logger := common.Logger(ctx) logger := common.Logger(ctx)
// Buffered so the copy goroutine never blocks on send if the grace-period // Buffered so the copy goroutine never blocks on send if the grace-period
@@ -905,7 +752,28 @@ func (cr *containerReference) waitForCommand(ctx context.Context, resp client.Hi
cmdResponse := make(chan error, 1) cmdResponse := make(chan error, 1)
go func() { go func() {
cmdResponse <- cr.copyOutput(resp.Reader) var outWriter io.Writer
outWriter = cr.input.Stdout
if outWriter == nil {
outWriter = os.Stdout
}
errWriter := cr.input.Stderr
if errWriter == nil {
errWriter = os.Stderr
}
var err error
if !isTerminal || os.Getenv("NORAW") != "" {
_, err = stdcopy.StdCopy(outWriter, errWriter, resp.Reader)
} else {
_, err = io.Copy(outWriter, resp.Reader)
}
// Flush any buffered, not-yet-newline-terminated trailing line so the
// final line of a command's output is not lost (e.g. an error message
// printed without a trailing newline before the process exits).
common.FlushWriter(outWriter)
common.FlushWriter(errWriter)
cmdResponse <- err
}() }()
select { select {
@@ -942,12 +810,11 @@ func (cr *containerReference) CopyTarStream(ctx context.Context, destPath string
if cr.id == "" { if cr.id == "" {
return cr.missingContainerError("copy to %s", destPath) return cr.missingContainerError("copy to %s", destPath)
} }
// Mkdir, with a path relative to the DestinationPath ("/") below. Docker 29.5+ // Mkdir
// rejects absolute tar entry names with "path escapes from parent".
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
tw := tar.NewWriter(buf) tw := tar.NewWriter(buf)
_ = tw.WriteHeader(&tar.Header{ _ = tw.WriteHeader(&tar.Header{
Name: strings.TrimPrefix(destPath, "/"), Name: destPath,
Mode: 0o777, Mode: 0o777,
Typeflag: tar.TypeDir, Typeflag: tar.TypeDir,
}) })
@@ -1101,11 +968,33 @@ func (cr *containerReference) attach() common.Executor {
if err != nil { if err != nil {
return fmt.Errorf("failed to attach to container: %w", err) return fmt.Errorf("failed to attach to container: %w", err)
} }
isTerminal := cr.input.AllocatePTY
var outWriter io.Writer
outWriter = cr.input.Stdout
if outWriter == nil {
outWriter = os.Stdout
}
errWriter := cr.input.Stderr
if errWriter == nil {
errWriter = os.Stderr
}
done := make(chan struct{}) done := make(chan struct{})
cr.attachDone = done cr.attachDone = done
go func() { go func() {
defer close(done) defer close(done)
if copyErr := cr.copyOutput(out.Reader); copyErr != nil { var copyErr error
if !isTerminal || os.Getenv("NORAW") != "" {
_, copyErr = stdcopy.StdCopy(outWriter, errWriter, out.Reader)
} else {
_, copyErr = io.Copy(outWriter, out.Reader)
}
// Flush any buffered, not-yet-newline-terminated trailing line once
// the stream reaches EOF, so the final line of the container's
// output is not lost when it is not newline-terminated.
common.FlushWriter(outWriter)
common.FlushWriter(errWriter)
if copyErr != nil {
common.Logger(ctx).Error(copyErr) common.Logger(ctx).Error(copyErr)
} }
}() }()
@@ -1165,77 +1054,6 @@ func (cr *containerReference) wait() common.Executor {
} }
} }
// For Gitea
// sanitizeOptionsHostConfig clears the HostConfig fields parsed from a
// workflow-controlled container.options string that could be used to escape the
// container when privileged mode is disabled. It must only be called when the
// runner has privileged mode turned off; with privileged mode enabled the
// administrator has already opted into host access.
func sanitizeOptionsHostConfig(logger logrus.FieldLogger, hostConfig *container.HostConfig) {
warn := func(option string) {
logger.Warnf("container option %q is not allowed when privileged mode is disabled and will be ignored", option)
}
if hostConfig.PidMode != "" {
warn("--pid")
hostConfig.PidMode = ""
}
if hostConfig.IpcMode != "" {
warn("--ipc")
hostConfig.IpcMode = ""
}
if hostConfig.UTSMode != "" {
warn("--uts")
hostConfig.UTSMode = ""
}
if hostConfig.CgroupnsMode != "" {
warn("--cgroupns")
hostConfig.CgroupnsMode = ""
}
// UsernsMode is set from the runner-controlled input; never let options
// override it (e.g. --userns=host disables user namespace remapping).
if hostConfig.UsernsMode != "" {
warn("--userns")
hostConfig.UsernsMode = ""
}
if len(hostConfig.CapAdd) > 0 {
warn("--cap-add")
hostConfig.CapAdd = nil
}
if len(hostConfig.SecurityOpt) > 0 {
warn("--security-opt")
hostConfig.SecurityOpt = nil
}
if len(hostConfig.Devices) > 0 {
warn("--device")
hostConfig.Devices = nil
}
if len(hostConfig.DeviceCgroupRules) > 0 {
warn("--device-cgroup-rule")
hostConfig.DeviceCgroupRules = nil
}
if len(hostConfig.DeviceRequests) > 0 {
warn("--gpus")
hostConfig.DeviceRequests = nil
}
if len(hostConfig.VolumesFrom) > 0 {
warn("--volumes-from")
hostConfig.VolumesFrom = nil
}
if hostConfig.Runtime != "" {
warn("--runtime")
hostConfig.Runtime = ""
}
if hostConfig.CgroupParent != "" {
warn("--cgroup-parent")
hostConfig.CgroupParent = ""
}
if len(hostConfig.Sysctls) > 0 {
warn("--sysctl")
hostConfig.Sysctls = nil
}
}
// For Gitea // For Gitea
// sanitizeConfig remove the invalid configurations from `config` and `hostConfig` // sanitizeConfig remove the invalid configurations from `config` and `hostConfig`
func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) { func (cr *containerReference) sanitizeConfig(ctx context.Context, config *container.Config, hostConfig *container.HostConfig) (*container.Config, *container.HostConfig) {
@@ -1281,34 +1099,6 @@ func (cr *containerReference) sanitizeConfig(ctx context.Context, config *contai
return config, hostConfig return config, hostConfig
} }
// bindTarget returns the container path a bind mounts onto, empty if it cannot be parsed.
func bindTarget(bind string) string {
parsed, err := loader.ParseVolume(bind)
if err != nil {
return ""
}
return parsed.Target
}
// overlayVolumes appends src's volumes to dst, dropping the dst ones they mount over. Docker
// rejects two mounts on one target, so the volumes declared last have to win.
func overlayVolumes(dst, src *container.HostConfig) {
claimed := map[string]bool{}
for _, bind := range src.Binds {
if target := bindTarget(bind); target != "" {
claimed[target] = true
}
}
for _, mt := range src.Mounts {
claimed[mt.Target] = true
}
dst.Binds = append(slices.DeleteFunc(slices.Clone(dst.Binds),
func(bind string) bool { return claimed[bindTarget(bind)] }), src.Binds...)
dst.Mounts = append(slices.DeleteFunc(slices.Clone(dst.Mounts),
func(mt mount.Mount) bool { return claimed[mt.Target] }), src.Mounts...)
}
type validVolumeMatcher struct { type validVolumeMatcher struct {
allowAll bool allowAll bool
named []glob.Glob named []glob.Glob
@@ -5,7 +5,6 @@
package container package container
import ( import (
"archive/tar"
"bufio" "bufio"
"bytes" "bytes"
"context" "context"
@@ -19,13 +18,11 @@ import (
"testing" "testing"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
cerrdefs "github.com/containerd/errdefs" cerrdefs "github.com/containerd/errdefs"
"github.com/moby/moby/api/pkg/stdcopy" "github.com/moby/moby/api/pkg/stdcopy"
"github.com/moby/moby/api/types/container" "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/mount"
"github.com/moby/moby/api/types/network"
mobyclient "github.com/moby/moby/client" mobyclient "github.com/moby/moby/client"
"github.com/sirupsen/logrus/hooks/test" "github.com/sirupsen/logrus/hooks/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -124,26 +121,6 @@ func (m *mockDockerClient) ContainerRemove(ctx context.Context, id string, opts
return args.Get(0).(mobyclient.ContainerRemoveResult), args.Error(1) return args.Get(0).(mobyclient.ContainerRemoveResult), args.Error(1)
} }
func (m *mockDockerClient) ContainerKill(ctx context.Context, id string, opts mobyclient.ContainerKillOptions) (mobyclient.ContainerKillResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.ContainerKillResult), args.Error(1)
}
func (m *mockDockerClient) NetworkList(ctx context.Context, opts mobyclient.NetworkListOptions) (mobyclient.NetworkListResult, error) {
args := m.Called(ctx, opts)
return args.Get(0).(mobyclient.NetworkListResult), args.Error(1)
}
func (m *mockDockerClient) NetworkInspect(ctx context.Context, id string, opts mobyclient.NetworkInspectOptions) (mobyclient.NetworkInspectResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.NetworkInspectResult), args.Error(1)
}
func (m *mockDockerClient) NetworkRemove(ctx context.Context, id string, opts mobyclient.NetworkRemoveOptions) (mobyclient.NetworkRemoveResult, error) {
args := m.Called(ctx, id, opts)
return args.Get(0).(mobyclient.NetworkRemoveResult), args.Error(1)
}
type endlessReader struct { type endlessReader struct {
io.Reader io.Reader
} }
@@ -360,44 +337,6 @@ func TestDockerCopyTarStream(t *testing.T) {
client.AssertExpectations(t) client.AssertExpectations(t)
} }
// Docker 29.5+ rejects absolute names in the mkdir tarball with
// "path escapes from parent", since it is extracted relative to "/".
func TestDockerCopyTarStreamMkdirEntryIsRelative(t *testing.T) {
ctx := context.Background()
var mkdirNames []string
client := &mockDockerClient{}
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
if opts.DestinationPath != "/" || opts.Content == nil {
return false
}
tr := tar.NewReader(opts.Content)
for {
hdr, err := tr.Next()
if err != nil {
break
}
mkdirNames = append(mkdirNames, hdr.Name)
}
return true
})).Return(mobyclient.CopyToContainerResult{}, nil)
client.On("CopyToContainer", ctx, "123", mock.MatchedBy(func(opts mobyclient.CopyToContainerOptions) bool {
return opts.DestinationPath == "/var/run/act" && opts.Content != nil
})).Return(mobyclient.CopyToContainerResult{}, nil)
cr := &containerReference{
id: "123",
cli: client,
input: &NewContainerInput{
Image: "image",
},
}
require.NoError(t, cr.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{}))
assert.Equal(t, []string{"var/run/act"}, mkdirNames)
client.AssertExpectations(t)
}
func TestDockerCopyTarStreamErrorInCopyFiles(t *testing.T) { func TestDockerCopyTarStreamErrorInCopyFiles(t *testing.T) {
ctx := context.Background() ctx := context.Background()
@@ -451,39 +390,29 @@ func TestDockerCopyTarStreamErrorInMkdir(t *testing.T) {
// be logged as one. // be logged as one.
func TestRemoveIgnoresAutoRemoveRace(t *testing.T) { func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
removeOpts := mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true} removeOpts := mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}
killOpts := mobyclient.ContainerKillOptions{Signal: "SIGKILL"}
for _, tc := range []struct { for _, tc := range []struct {
name string name string
err error err error
wantWait bool wantLogs bool
wantFailure bool
}{ }{
{name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress"), wantWait: true}, {name: "removal in progress", err: cerrdefs.ErrConflict.WithMessage("removal of container abc is already in progress")},
{name: "already removed", err: cerrdefs.ErrNotFound.WithMessage("No such container: abc")}, {name: "already removed", err: cerrdefs.ErrNotFound.WithMessage("No such container: abc")},
{name: "removed cleanly", err: nil}, {name: "removed cleanly", err: nil},
{name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantFailure: true}, {name: "real failure", err: errors.New("driver failed to remove root filesystem"), wantLogs: true},
} { } {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
logger, hook := test.NewNullLogger() logger, hook := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger) ctx := common.WithLogger(context.Background(), logger)
client := &mockDockerClient{} client := &mockDockerClient{}
client.On("ContainerKill", ctx, "abc", killOpts).Return(mobyclient.ContainerKillResult{}, nil)
client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err) client.On("ContainerRemove", ctx, "abc", removeOpts).Return(mobyclient.ContainerRemoveResult{}, tc.err)
if tc.wantWait {
removed := make(chan container.WaitResponse, 1)
removed <- container.WaitResponse{}
client.On("ContainerWait", mock.Anything, "abc", mobyclient.ContainerWaitOptions{Condition: container.WaitConditionRemoved}).
Return(mobyclient.ContainerWaitResult{Result: removed})
}
cr := &containerReference{id: "abc", cli: client} cr := &containerReference{id: "abc", cli: client}
require.NoError(t, cr.remove()(ctx)) require.NoError(t, cr.remove()(ctx))
// a failure keeps the id, so a later Remove() can retry it assert.Empty(t, cr.id)
if tc.wantFailure {
assert.Equal(t, "abc", cr.id) if tc.wantLogs {
assert.Len(t, hook.AllEntries(), 1) assert.Len(t, hook.AllEntries(), 1)
} else { } else {
assert.Empty(t, cr.id)
assert.Empty(t, hook.AllEntries()) assert.Empty(t, hook.AllEntries())
} }
client.AssertExpectations(t) client.AssertExpectations(t)
@@ -491,20 +420,6 @@ func TestRemoveIgnoresAutoRemoveRace(t *testing.T) {
} }
} }
// A container whose id was never learned, because find() could not reach the daemon or
// create() lost its reply, must still be removed rather than leaking with its network. It
// was never started here, so it is not worth a kill of its own.
func TestRemoveWithoutIDUsesName(t *testing.T) {
ctx := context.Background()
client := &mockDockerClient{}
client.On("ContainerRemove", ctx, "job-1", mobyclient.ContainerRemoveOptions{RemoveVolumes: true, Force: true}).
Return(mobyclient.ContainerRemoveResult{}, nil)
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
require.NoError(t, cr.remove()(ctx))
client.AssertExpectations(t)
}
// find() must drop a stale cached id so later Copy/Exec don't hit the // find() must drop a stale cached id so later Copy/Exec don't hit the
// daemon with a torn-down container. // daemon with a torn-down container.
func TestFindRevalidatesStaleID(t *testing.T) { func TestFindRevalidatesStaleID(t *testing.T) {
@@ -572,7 +487,7 @@ func TestRejectsMissingContainer(t *testing.T) {
cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}} cr := &containerReference{cli: client, input: &NewContainerInput{Name: "job-1"}}
check := func(op string, err error) { check := func(op string, err error) {
t.Helper() t.Helper()
require.ErrorIs(t, err, ErrContainerNotFound, op) require.Error(t, err, op)
assert.Contains(t, err.Error(), `container "job-1" does not exist`, op) assert.Contains(t, err.Error(), `container "job-1" does not exist`, op)
} }
check("copyContent", cr.copyContent("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx)) check("copyContent", cr.copyContent("/var/run/act", &FileEntry{Name: "x", Mode: 0o644})(ctx))
@@ -581,15 +496,6 @@ func TestRejectsMissingContainer(t *testing.T) {
check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx)) check("exec", cr.exec([]string{"echo"}, nil, "", "")(ctx))
_, err := cr.GetContainerArchive(ctx, "/var/run/act/x") _, err := cr.GetContainerArchive(ctx, "/var/run/act/x")
check("GetContainerArchive", err) check("GetContainerArchive", err)
_, err = cr.Inspect(ctx)
check("Inspect", err)
// a known id the daemon has since dropped
client.On("ContainerInspect", ctx, "gone", mobyclient.ContainerInspectOptions{}).
Return(mobyclient.ContainerInspectResult{}, cerrdefs.ErrNotFound)
removed := &containerReference{id: "gone", cli: client, input: &NewContainerInput{Name: "job-1"}}
_, err = removed.Inspect(ctx)
check("Inspect after removal", err)
} }
// End-to-end: a stale cr.id is cleared, repopulated from name lookup, // End-to-end: a stale cr.id is cleared, repopulated from name lookup,
@@ -638,7 +544,7 @@ func TestDockerCopyToSymlinkPath(t *testing.T) {
// CopyTarStream first creates the destination directory by extracting a tar at "/", // CopyTarStream first creates the destination directory by extracting a tar at "/",
// which makes the daemon mkdir var, then var/run (the symlink), then act — the exact // which makes the daemon mkdir var, then var/run (the symlink), then act — the exact
// step that fails on the broken daemon. // step that fails on the broken daemon.
err := rc.CopyTarStream(ctx, "/var/run/act/actions/", &bytes.Buffer{}) err := rc.CopyTarStream(ctx, "/var/run/act", &bytes.Buffer{})
require.NoError(t, err) require.NoError(t, err)
} }
@@ -718,110 +624,6 @@ func TestCheckVolumes(t *testing.T) {
} }
} }
func TestSanitizeOptionsHostConfig(t *testing.T) {
logger, _ := test.NewNullLogger()
dangerous := func() *container.HostConfig {
return &container.HostConfig{
PidMode: "host",
IpcMode: "host",
UTSMode: "host",
CgroupnsMode: "host",
UsernsMode: "host",
CapAdd: []string{"ALL"},
SecurityOpt: []string{"seccomp=unconfined", "apparmor=unconfined"},
VolumesFrom: []string{"other"},
Runtime: "runc",
Resources: container.Resources{
CgroupParent: "/custom",
Devices: []container.DeviceMapping{{PathOnHost: "/dev/sda", PathInContainer: "/dev/sda", CgroupPermissions: "rwm"}},
DeviceCgroupRules: []string{"a *:* rwm"},
},
Sysctls: map[string]string{"net.ipv4.ip_forward": "1"},
}
}
hostConfig := dangerous()
sanitizeOptionsHostConfig(logger, hostConfig)
assert.Empty(t, string(hostConfig.PidMode))
assert.Empty(t, string(hostConfig.IpcMode))
assert.Empty(t, string(hostConfig.UTSMode))
assert.Empty(t, string(hostConfig.CgroupnsMode))
assert.Empty(t, string(hostConfig.UsernsMode))
assert.Empty(t, hostConfig.CapAdd)
assert.Empty(t, hostConfig.SecurityOpt)
assert.Empty(t, hostConfig.Devices)
assert.Empty(t, hostConfig.DeviceCgroupRules)
assert.Empty(t, hostConfig.VolumesFrom)
assert.Empty(t, hostConfig.Runtime)
assert.Empty(t, hostConfig.CgroupParent)
assert.Empty(t, hostConfig.Sysctls)
}
func TestMergeContainerConfigsStripsDangerousOptionsWhenUnprivileged(t *testing.T) {
// OS-independent options only: --device parsing requires a linux/windows
// server OS, which is not guaranteed for the test host.
const dangerousOptions = "--pid=host --ipc=host --uts=host --cgroupns=host " +
"--userns=host --cap-add=ALL --security-opt seccomp=unconfined " +
"--security-opt apparmor=unconfined --volumes-from other " +
"--runtime runc --cgroup-parent /custom --sysctl net.ipv4.ip_forward=1"
t.Run("unprivileged strips host-escape options", func(t *testing.T) {
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
Options: dangerousOptions,
NetworkMode: "bridge",
UsernsMode: "private",
},
}
_, hostConfig, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
Privileged: false,
UsernsMode: container.UsernsMode("private"),
NetworkMode: container.NetworkMode("bridge"),
})
require.NoError(t, err)
assert.False(t, hostConfig.Privileged)
assert.Empty(t, string(hostConfig.PidMode))
assert.Empty(t, string(hostConfig.IpcMode))
assert.Empty(t, string(hostConfig.UTSMode))
assert.Empty(t, string(hostConfig.CgroupnsMode))
// UsernsMode must keep the runner-controlled value, not the one from options.
assert.Equal(t, "private", string(hostConfig.UsernsMode))
assert.Empty(t, hostConfig.CapAdd)
assert.Empty(t, hostConfig.SecurityOpt)
assert.Empty(t, hostConfig.VolumesFrom)
assert.Empty(t, hostConfig.Runtime)
assert.Empty(t, hostConfig.CgroupParent)
assert.Empty(t, hostConfig.Sysctls)
})
t.Run("privileged preserves options", func(t *testing.T) {
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
Options: "--pid=host --cap-add=ALL --security-opt seccomp=unconfined",
NetworkMode: "bridge",
},
}
_, hostConfig, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
Privileged: true,
NetworkMode: container.NetworkMode("bridge"),
})
require.NoError(t, err)
assert.Equal(t, "host", string(hostConfig.PidMode))
assert.Equal(t, []string{"ALL"}, hostConfig.CapAdd)
assert.Equal(t, []string{"seccomp=unconfined"}, hostConfig.SecurityOpt)
})
}
func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) { func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
logger, _ := test.NewNullLogger() logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger) ctx := common.WithLogger(context.Background(), logger)
@@ -858,75 +660,3 @@ func TestCheckVolumesRejectsEscapingHostPaths(t *testing.T) {
}) })
assert.Empty(t, hostConf.Binds) assert.Empty(t, hostConf.Binds)
} }
func TestContainerInfoFromInspect(t *testing.T) {
t.Run("reports no healthcheck when the image declares none", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
ID: "abc123",
State: &container.State{Status: "running", Running: true},
})
assert.Equal(t, "abc123", info.ID)
assert.Equal(t, "running", info.State)
assert.Equal(t, HealthNone, info.Health)
assert.Empty(t, info.Ports)
})
t.Run("reports the health status and the last probe output", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
State: &container.State{
Status: "running",
Health: &container.Health{
Status: container.Unhealthy,
Log: []*container.HealthcheckResult{
{Output: "first\n"},
{Output: "connection refused\n"},
},
},
},
})
assert.Equal(t, HealthUnhealthy, info.Health)
assert.Equal(t, "connection refused", info.HealthOutput)
})
t.Run("reports the published ports", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{
State: &container.State{Status: "running"},
NetworkSettings: &container.NetworkSettings{
Ports: network.PortMap{
network.MustParsePort("5432/tcp"): []network.PortBinding{{HostPort: "49153"}},
network.MustParsePort("6379/tcp"): nil,
},
},
})
assert.Equal(t, map[string]string{"5432": "49153"}, info.Ports)
})
t.Run("tolerates a container without state", func(t *testing.T) {
info := containerInfoFromInspect(container.InspectResponse{ID: "abc123"})
assert.Equal(t, "abc123", info.ID)
assert.Equal(t, HealthNone, info.Health)
})
}
func TestMergeContainerConfigsVolumesReplaceRunnerMounts(t *testing.T) {
logger, _ := test.NewNullLogger()
ctx := common.WithLogger(context.Background(), logger)
cr := &containerReference{
input: &NewContainerInput{
NetworkMode: "bridge",
Options: "--volume /host/tools:/opt/hostedtoolcache",
},
}
_, hostConf, err := cr.mergeContainerConfigs(ctx, &container.Config{}, &container.HostConfig{
Binds: []string{"/var/run/docker.sock:/var/run/docker.sock"},
Mounts: []mount.Mount{{Type: mount.TypeVolume, Source: "act-toolcache", Target: "/opt/hostedtoolcache"}},
})
require.NoError(t, err)
assert.Equal(t, []string{"/var/run/docker.sock:/var/run/docker.sock", "/host/tools:/opt/hostedtoolcache"}, hostConf.Binds)
assert.Empty(t, hostConf.Mounts)
}
@@ -8,13 +8,12 @@ package container
import ( import (
"context" "context"
"errors"
"runtime" "runtime"
"time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/moby/moby/api/types/system" "github.com/moby/moby/api/types/system"
"github.com/pkg/errors"
) )
// ImageExistsLocally returns a boolean indicating if an image with the // ImageExistsLocally returns a boolean indicating if an image with the
@@ -73,7 +72,3 @@ func NewDockerNetworkRemoveExecutor(name string) common.Executor {
return nil return nil
} }
} }
func RemoveOrphanNetworks(ctx context.Context, runnerUUID string, createdBefore time.Time) error {
return nil
}
@@ -9,7 +9,7 @@ package container
import ( import (
"context" "context"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/moby/moby/client" "github.com/moby/moby/client"
) )
@@ -21,9 +21,9 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/internal/act/filecollector" "gitea.com/gitea/runner/act/filecollector"
"gitea.com/gitea/runner/internal/act/lookpath" "gitea.com/gitea/runner/act/lookpath"
"gitea.com/gitea/runner/internal/pkg/process" "gitea.com/gitea/runner/internal/pkg/process"
"github.com/go-git/go-billy/v5/helper/polyfill" "github.com/go-git/go-billy/v5/helper/polyfill"
@@ -154,14 +154,6 @@ func (e *HostEnvironment) CopyDir(destPath, srcPath string, useGitIgnore bool) c
} }
} }
func (e *HostEnvironment) DumpLogs(_ context.Context) error {
return nil
}
func (e *HostEnvironment) Inspect(_ context.Context) (*Info, error) {
return &Info{Health: HealthNone, Ports: map[string]string{}}, nil
}
func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) { func (e *HostEnvironment) GetContainerArchive(ctx context.Context, srcPath string) (io.ReadCloser, error) {
buf := &bytes.Buffer{} buf := &bytes.Buffer{}
tw := tar.NewWriter(buf) tw := tar.NewWriter(buf)
@@ -338,10 +330,6 @@ func (e *HostEnvironment) exec(ctx context.Context, command []string, cmdline st
} else { } else {
wd = e.Path wd = e.Path
} }
// Flush any buffered, not-yet-newline-terminated trailing line, as the docker backend
// does in waitForCommand, so the final line of a command's output is not lost.
defer common.FlushWriter(e.StdOut)
f, err := lookupPathHost(command[0], env, e.StdOut) f, err := lookupPathHost(command[0], env, e.StdOut)
if err != nil { if err != nil {
return err return err
@@ -17,7 +17,7 @@ import (
"testing" "testing"
"time" "time"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -66,15 +66,12 @@ func (*LinuxContainerEnvironmentExtensions) JoinPathVariable(paths ...string) st
return strings.Join(paths, ":") return strings.Join(paths, ":")
} }
// DefaultToolCache is where the runner mounts the tool cache inside job containers.
const DefaultToolCache = "/opt/hostedtoolcache"
func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any { func (*LinuxContainerEnvironmentExtensions) GetRunnerContext(ctx context.Context) map[string]any {
return map[string]any{ return map[string]any{
"os": "Linux", "os": "Linux",
"arch": RunnerArch(ctx), "arch": RunnerArch(ctx),
"temp": "/tmp", "temp": "/tmp",
"tool_cache": DefaultToolCache, "tool_cache": "/opt/hostedtoolcache",
} }
} }
@@ -12,7 +12,7 @@ import (
"io" "io"
"strings" "strings"
"gitea.com/gitea/runner/internal/act/common" "gitea.com/gitea/runner/act/common"
"golang.org/x/text/encoding/unicode" "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform" "golang.org/x/text/transform"
+309
View File
@@ -0,0 +1,309 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package exprparser
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"gitea.com/gitea/runner/act/model"
"github.com/go-git/go-git/v5/plumbing/format/gitignore"
"github.com/rhysd/actionlint"
)
func (impl *interperterImpl) contains(search, item reflect.Value) (bool, error) {
switch search.Kind() {
case reflect.String, reflect.Int, reflect.Float64, reflect.Bool, reflect.Invalid:
return strings.Contains(
strings.ToLower(impl.coerceToString(search).String()),
strings.ToLower(impl.coerceToString(item).String()),
), nil
case reflect.Slice:
for i := 0; i < search.Len(); i++ {
arrayItem := search.Index(i).Elem()
result, err := impl.compareValues(arrayItem, item, actionlint.CompareOpNodeKindEq)
if err != nil {
return false, err
}
if isEqual, ok := result.(bool); ok && isEqual {
return true, nil
}
}
}
return false, nil
}
func (impl *interperterImpl) startsWith(searchString, searchValue reflect.Value) (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
return strings.HasPrefix(
strings.ToLower(impl.coerceToString(searchString).String()),
strings.ToLower(impl.coerceToString(searchValue).String()),
), nil
}
func (impl *interperterImpl) endsWith(searchString, searchValue reflect.Value) (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
return strings.HasSuffix(
strings.ToLower(impl.coerceToString(searchString).String()),
strings.ToLower(impl.coerceToString(searchValue).String()),
), nil
}
const (
passThrough = iota
bracketOpen
bracketClose
)
func (impl *interperterImpl) format(str reflect.Value, replaceValue ...reflect.Value) (string, error) {
input := impl.coerceToString(str).String()
var output strings.Builder
replacementIndex := ""
state := passThrough
for _, character := range input {
switch state {
case passThrough: // normal buffer output
switch character {
case '{':
state = bracketOpen
case '}':
state = bracketClose
default:
output.WriteRune(character)
}
case bracketOpen: // found {
switch character {
case '{':
output.WriteString("{")
replacementIndex = ""
state = passThrough
case '}':
index, err := strconv.ParseInt(replacementIndex, 10, 32)
if err != nil {
return "", fmt.Errorf("The following format string is invalid: '%s'", input)
}
replacementIndex = ""
if len(replaceValue) <= int(index) {
return "", fmt.Errorf("The following format string references more arguments than were supplied: '%s'", input)
}
output.WriteString(impl.coerceToString(replaceValue[index]).String())
state = passThrough
default:
replacementIndex += string(character)
}
case bracketClose: // found }
switch character {
case '}':
output.WriteString("}")
replacementIndex = ""
state = passThrough
default:
panic("Invalid format parser state")
}
}
}
if state != passThrough {
switch state {
case bracketOpen:
return "", fmt.Errorf("Unclosed brackets. The following format string is invalid: '%s'", input)
case bracketClose:
return "", fmt.Errorf("Closing bracket without opening one. The following format string is invalid: '%s'", input)
}
}
return output.String(), nil
}
func (impl *interperterImpl) join(array, sep reflect.Value) (string, error) { //nolint:unparam // pre-existing issue from nektos/act
separator := impl.coerceToString(sep).String()
switch array.Kind() {
case reflect.Slice:
var items []string
for i := 0; i < array.Len(); i++ {
items = append(items, impl.coerceToString(array.Index(i).Elem()).String())
}
return strings.Join(items, separator), nil
default:
return strings.Join([]string{impl.coerceToString(array).String()}, separator), nil
}
}
func (impl *interperterImpl) toJSON(value reflect.Value) (string, error) {
if value.Kind() == reflect.Invalid {
return "null", nil
}
json, err := json.MarshalIndent(value.Interface(), "", " ")
if err != nil {
return "", fmt.Errorf("Cannot convert value to JSON. Cause: %v", err)
}
return string(json), nil
}
func (impl *interperterImpl) fromJSON(value reflect.Value) (any, error) {
if value.Kind() != reflect.String {
return nil, fmt.Errorf("Cannot parse non-string type %v as JSON", value.Kind())
}
var data any
err := json.Unmarshal([]byte(value.String()), &data)
if err != nil {
return nil, fmt.Errorf("Invalid JSON: %v", err)
}
return data, nil
}
func (impl *interperterImpl) hashFiles(paths ...reflect.Value) (string, error) {
var ps []gitignore.Pattern
const cwdPrefix = "." + string(filepath.Separator)
const excludeCwdPrefix = "!" + cwdPrefix
for _, path := range paths {
if path.Kind() == reflect.String {
cleanPath := path.String()
if strings.HasPrefix(cleanPath, cwdPrefix) {
cleanPath = cleanPath[len(cwdPrefix):]
} else if strings.HasPrefix(cleanPath, excludeCwdPrefix) {
cleanPath = "!" + cleanPath[len(excludeCwdPrefix):]
}
ps = append(ps, gitignore.ParsePattern(cleanPath, nil))
} else {
return "", errors.New("Non-string path passed to hashFiles")
}
}
matcher := gitignore.NewMatcher(ps)
var files []string
if err := filepath.Walk(impl.config.WorkingDir, func(path string, fi fs.FileInfo, err error) error {
if err != nil {
return err
}
sansPrefix := strings.TrimPrefix(path, impl.config.WorkingDir+string(filepath.Separator))
parts := strings.Split(sansPrefix, string(filepath.Separator))
if fi.IsDir() || !matcher.Match(parts, fi.IsDir()) {
return nil
}
files = append(files, path)
return nil
}); err != nil {
return "", fmt.Errorf("Unable to filepath.Walk: %v", err)
}
if len(files) == 0 {
return "", nil
}
hasher := sha256.New()
for _, file := range files {
f, err := os.Open(file)
if err != nil {
return "", fmt.Errorf("Unable to os.Open: %v", err)
}
if _, err := io.Copy(hasher, f); err != nil {
return "", fmt.Errorf("Unable to io.Copy: %v", err)
}
if err := f.Close(); err != nil {
return "", fmt.Errorf("Unable to Close file: %v", err)
}
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}
func (impl *interperterImpl) getNeedsTransitive(job *model.Job) []string {
needs := job.Needs()
for _, need := range needs {
parentNeeds := impl.getNeedsTransitive(impl.config.Run.Workflow.GetJob(need))
needs = append(needs, parentNeeds...)
}
return needs
}
func (impl *interperterImpl) always() (bool, error) {
return true, nil
}
func (impl *interperterImpl) jobSuccess() (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
jobs := impl.config.Run.Workflow.Jobs
jobNeeds := impl.getNeedsTransitive(impl.config.Run.Job())
for _, needs := range jobNeeds {
if jobs[needs].NeedsResult() != "success" {
return false, nil
}
}
return true, nil
}
// jobStatus returns the current job status, treating a nil Job context as an
// empty status so status-check functions never panic on a nil dereference.
func (impl *interperterImpl) jobStatus() string {
if impl.env.Job == nil {
return ""
}
return impl.env.Job.Status
}
func (impl *interperterImpl) stepSuccess() (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
return impl.jobStatus() == "success", nil
}
func (impl *interperterImpl) jobFailure() (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
jobs := impl.config.Run.Workflow.Jobs
jobNeeds := impl.getNeedsTransitive(impl.config.Run.Job())
for _, needs := range jobNeeds {
if jobs[needs].NeedsResult() == "failure" {
return true, nil
}
}
return false, nil
}
func (impl *interperterImpl) stepFailure() (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
return impl.jobStatus() == "failure", nil
}
func (impl *interperterImpl) cancelled() (bool, error) { //nolint:unparam // pre-existing issue from nektos/act
return impl.jobStatus() == "cancelled", nil
}
+280
View File
@@ -0,0 +1,280 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package exprparser
import (
"path/filepath"
"testing"
"gitea.com/gitea/runner/act/model"
"github.com/stretchr/testify/assert"
)
func TestFunctionContains(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"contains('search', 'item') }}", false, "contains-str-str"},
{`cOnTaInS('Hello', 'll') }}`, true, "contains-str-casing"},
{`contains('HELLO', 'll') }}`, true, "contains-str-casing"},
{`contains('3.141592', 3.14) }}`, true, "contains-str-number"},
{`contains(3.141592, '3.14') }}`, true, "contains-number-str"},
{`contains(3.141592, 3.14) }}`, true, "contains-number-number"},
{`contains(true, 'u') }}`, true, "contains-bool-str"},
{`contains(null, '') }}`, true, "contains-null-str"},
{`contains(fromJSON('["first","second"]'), 'first') }}`, true, "contains-item"},
{`contains(fromJSON('[null,"second"]'), '') }}`, true, "contains-item-null-empty-str"},
{`contains(fromJSON('["","second"]'), null) }}`, true, "contains-item-empty-str-null"},
{`contains(fromJSON('[true,"second"]'), 'true') }}`, false, "contains-item-bool-arr"},
{`contains(fromJSON('["true","second"]'), true) }}`, false, "contains-item-str-bool"},
{`contains(fromJSON('[3.14,"second"]'), '3.14') }}`, true, "contains-item-number-str"},
{`contains(fromJSON('[3.14,"second"]'), 3.14) }}`, true, "contains-item-number-number"},
{`contains(fromJSON('["","second"]'), fromJSON('[]')) }}`, false, "contains-item-str-arr"},
{`contains(fromJSON('["","second"]'), fromJSON('{}')) }}`, false, "contains-item-str-obj"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionStartsWith(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"startsWith('search', 'se') }}", true, "startswith-string"},
{"startsWith('search', 'sa') }}", false, "startswith-string"},
{"startsWith('123search', '123s') }}", true, "startswith-string"},
{"startsWith(123, 's') }}", false, "startswith-string"},
{"startsWith(123, '12') }}", true, "startswith-string"},
{"startsWith('123', 12) }}", true, "startswith-string"},
{"startsWith(null, '42') }}", false, "startswith-string"},
{"startsWith('null', null) }}", true, "startswith-string"},
{"startsWith('null', '') }}", true, "startswith-string"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionEndsWith(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"endsWith('search', 'ch') }}", true, "endsWith-string"},
{"endsWith('search', 'sa') }}", false, "endsWith-string"},
{"endsWith('search123s', '123s') }}", true, "endsWith-string"},
{"endsWith(123, 's') }}", false, "endsWith-string"},
{"endsWith(123, '23') }}", true, "endsWith-string"},
{"endsWith('123', 23) }}", true, "endsWith-string"},
{"endsWith(null, '42') }}", false, "endsWith-string"},
{"endsWith('null', null) }}", true, "endsWith-string"},
{"endsWith('null', '') }}", true, "endsWith-string"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionJoin(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"join(fromJSON('[\"a\", \"b\"]'), ',')", "a,b", "join-arr"},
{"join('string', ',')", "string", "join-str"},
{"join(1, ',')", "1", "join-number"},
{"join(null, ',')", "", "join-number"},
{"join(fromJSON('[\"a\", \"b\", null]'), null)", "ab", "join-number"},
{"join(fromJSON('[\"a\", \"b\"]'))", "a,b", "join-number"},
{"join(fromJSON('[\"a\", \"b\", null]'), 1)", "a1b1", "join-number"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionToJSON(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"toJSON(env) }}", "{\n \"key\": \"value\"\n}", "toJSON"},
{"toJSON(null)", "null", "toJSON-null"},
}
env := &EvaluationEnvironment{
Env: map[string]string{
"key": "value",
},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionFromJSON(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"fromJSON('{\"foo\":\"bar\"}') }}", map[string]any{
"foo": "bar",
}, "fromJSON"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionHashFiles(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"hashFiles('**/non-extant-files') }}", "", "hash-non-existing-file"},
{"hashFiles('**/non-extant-files', '**/more-non-extant-files') }}", "", "hash-multiple-non-existing-files"},
{"hashFiles('./for-hashing-1.txt') }}", "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", "hash-single-file"},
{"hashFiles('./for-hashing-*.txt') }}", "8e5935e7e13368cd9688fe8f48a0955293676a021562582c7e848dafe13fb046", "hash-multiple-files"},
{"hashFiles('./for-hashing-*.txt', '!./for-hashing-2.txt') }}", "66a045b452102c59d840ec097d59d9467e13a3f34f6494e539ffd32c1bb35f18", "hash-negative-pattern"},
{"hashFiles('./for-hashing-**') }}", "c418ba693753c84115ced0da77f876cddc662b9054f4b129b90f822597ee2f94", "hash-multiple-files-and-directories"},
{"hashFiles('./for-hashing-3/**') }}", "6f5696b546a7a9d6d42a449dc9a56bef244aaa826601ef27466168846139d2c2", "hash-nested-directories"},
{"hashFiles('./for-hashing-3/**/nested-data.txt') }}", "8ecadfb49f7f978d0a9f3a957e9c8da6cc9ab871f5203b5d9f9d1dc87d8af18c", "hash-nested-directories-2"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
workdir, err := filepath.Abs("testdata")
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
output, err := NewInterpeter(env, Config{WorkingDir: workdir}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestFunctionFormat(t *testing.T) {
table := []struct {
input string
expected any
error any
name string
}{
{"format('text')", "text", nil, "format-plain-string"},
{"format('Hello {0} {1} {2}!', 'Mona', 'the', 'Octocat')", "Hello Mona the Octocat!", nil, "format-with-placeholders"},
{"format('{{Hello {0} {1} {2}!}}', 'Mona', 'the', 'Octocat')", "{Hello Mona the Octocat!}", nil, "format-with-escaped-braces"},
{"format('{{0}}', 'test')", "{0}", nil, "format-with-escaped-braces"},
{"format('{{{0}}}', 'test')", "{test}", nil, "format-with-escaped-braces-and-value"},
{"format('}}')", "}", nil, "format-output-closing-brace"},
{`format('Hello "{0}" {1} {2} {3} {4}', null, true, -3.14, NaN, Infinity)`, `Hello "" true -3.14 NaN Infinity`, nil, "format-with-primitives"},
{`format('Hello "{0}" {1} {2}', fromJSON('[0, true, "abc"]'), fromJSON('[{"a":1}]'), fromJSON('{"a":{"b":1}}'))`, `Hello "Array" Array Object`, nil, "format-with-complex-types"},
{"format(true)", "true", nil, "format-with-primitive-args"},
{"format('echo Hello {0} ${{Test}}', github.undefined_property)", "echo Hello ${Test}", nil, "format-with-undefined-value"},
{"format('{0}}', '{1}', 'World')", nil, "Closing bracket without opening one. The following format string is invalid: '{0}}'", "format-invalid-format-string"},
{"format('{0', '{1}', 'World')", nil, "Unclosed brackets. The following format string is invalid: '{0'", "format-invalid-format-string"},
{"format('{2}', '{1}', 'World')", "", "The following format string references more arguments than were supplied: '{2}'", "format-invalid-replacement-reference"},
{"format('{2147483648}')", "", "The following format string is invalid: '{2147483648}'", "format-invalid-replacement-reference"},
{"format('{0} {1} {2} {3}', 1.0, 1.1, 1234567890.0, 12345678901234567890.0)", "1 1.1 1234567890 1.23456789012346E+19", nil, "format-floats"},
}
env := &EvaluationEnvironment{
Github: &model.GithubContext{},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
if tt.error != nil {
assert.Equal(t, tt.error, err.Error())
} else {
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
}
})
}
}
func TestStatusFunctionsNilJob(t *testing.T) {
// A nil Job context must not panic: the status-check functions should treat
// it as an empty status and return false rather than dereferencing nil.
env := &EvaluationEnvironment{}
table := []struct {
input string
context string
name string
}{
{"cancelled()", "job", "cancelled-nil-job"},
{"success()", "step", "step-success-nil-job"},
{"failure()", "step", "step-failure-nil-job"},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{Context: tt.context}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, false, output)
})
}
}
+644
View File
@@ -0,0 +1,644 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package exprparser
import (
"encoding"
"errors"
"fmt"
"math"
"reflect"
"strings"
"gitea.com/gitea/runner/act/model"
"github.com/rhysd/actionlint"
)
type EvaluationEnvironment struct {
Github *model.GithubContext
Env map[string]string
Job *model.JobContext
Jobs *map[string]*model.WorkflowCallResult
Steps map[string]*model.StepResult
Runner map[string]any
Secrets map[string]string
Vars map[string]string
Strategy map[string]any
Matrix map[string]any
Needs map[string]Needs
Inputs map[string]any
HashFiles func([]reflect.Value) (any, error)
}
type Needs struct {
Outputs map[string]string `json:"outputs"`
Result string `json:"result"`
}
type Config struct {
Run *model.Run
WorkingDir string
Context string
}
type DefaultStatusCheck int
const (
DefaultStatusCheckNone DefaultStatusCheck = iota
DefaultStatusCheckSuccess
DefaultStatusCheckAlways
DefaultStatusCheckCanceled
DefaultStatusCheckFailure
)
func (dsc DefaultStatusCheck) String() string {
switch dsc {
case DefaultStatusCheckSuccess:
return "success"
case DefaultStatusCheckAlways:
return "always"
case DefaultStatusCheckCanceled:
return "cancelled"
case DefaultStatusCheckFailure:
return "failure"
}
return ""
}
type Interpreter interface {
Evaluate(input string, defaultStatusCheck DefaultStatusCheck) (any, error)
}
type interperterImpl struct {
env *EvaluationEnvironment
config Config
}
func NewInterpeter(env *EvaluationEnvironment, config Config) Interpreter {
return &interperterImpl{
env: env,
config: config,
}
}
func (impl *interperterImpl) Evaluate(input string, defaultStatusCheck DefaultStatusCheck) (any, error) {
input = strings.TrimPrefix(input, "${{")
if defaultStatusCheck != DefaultStatusCheckNone && input == "" {
input = "success()"
}
parser := actionlint.NewExprParser()
exprNode, err := parser.Parse(actionlint.NewExprLexer(input + "}}"))
if err != nil {
return nil, fmt.Errorf("Failed to parse: %s", err.Message)
}
if defaultStatusCheck != DefaultStatusCheckNone {
hasStatusCheckFunction := false
actionlint.VisitExprNode(exprNode, func(node, _ actionlint.ExprNode, entering bool) {
if funcCallNode, ok := node.(*actionlint.FuncCallNode); entering && ok {
switch strings.ToLower(funcCallNode.Callee) {
case "success", "always", "cancelled", "failure":
hasStatusCheckFunction = true
}
}
})
if !hasStatusCheckFunction {
exprNode = &actionlint.LogicalOpNode{
Kind: actionlint.LogicalOpNodeKindAnd,
Left: &actionlint.FuncCallNode{
Callee: defaultStatusCheck.String(),
Args: []actionlint.ExprNode{},
},
Right: exprNode,
}
}
}
result, err2 := impl.evaluateNode(exprNode)
return result, err2
}
func (impl *interperterImpl) evaluateNode(exprNode actionlint.ExprNode) (any, error) {
switch node := exprNode.(type) {
case *actionlint.VariableNode:
return impl.evaluateVariable(node)
case *actionlint.BoolNode:
return node.Value, nil
case *actionlint.NullNode:
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
case *actionlint.IntNode:
return node.Value, nil
case *actionlint.FloatNode:
return node.Value, nil
case *actionlint.StringNode:
return node.Value, nil
case *actionlint.IndexAccessNode:
return impl.evaluateIndexAccess(node)
case *actionlint.ObjectDerefNode:
return impl.evaluateObjectDeref(node)
case *actionlint.ArrayDerefNode:
return impl.evaluateArrayDeref(node)
case *actionlint.NotOpNode:
return impl.evaluateNot(node)
case *actionlint.CompareOpNode:
return impl.evaluateCompare(node)
case *actionlint.LogicalOpNode:
return impl.evaluateLogicalCompare(node)
case *actionlint.FuncCallNode:
return impl.evaluateFuncCall(node)
default:
return nil, fmt.Errorf("Fatal error! Unknown node type: %s node: %+v", reflect.TypeOf(exprNode), exprNode)
}
}
func (impl *interperterImpl) evaluateVariable(variableNode *actionlint.VariableNode) (any, error) {
switch strings.ToLower(variableNode.Name) {
case "github":
return impl.env.Github, nil
case "gitea": // compatible with Gitea
return impl.env.Github, nil
case "env":
return impl.env.Env, nil
case "job":
return impl.env.Job, nil
case "jobs":
if impl.env.Jobs == nil {
return nil, errors.New("Unavailable context: jobs")
}
return impl.env.Jobs, nil
case "steps":
return impl.env.Steps, nil
case "runner":
return impl.env.Runner, nil
case "secrets":
return impl.env.Secrets, nil
case "vars":
return impl.env.Vars, nil
case "strategy":
return impl.env.Strategy, nil
case "matrix":
return impl.env.Matrix, nil
case "needs":
return impl.env.Needs, nil
case "inputs":
return impl.env.Inputs, nil
case "infinity":
return math.Inf(1), nil
case "nan":
return math.NaN(), nil
default:
return nil, fmt.Errorf("Unavailable context: %s", variableNode.Name)
}
}
func (impl *interperterImpl) evaluateIndexAccess(indexAccessNode *actionlint.IndexAccessNode) (any, error) {
left, err := impl.evaluateNode(indexAccessNode.Operand)
if err != nil {
return nil, err
}
leftValue := reflect.ValueOf(left)
right, err := impl.evaluateNode(indexAccessNode.Index)
if err != nil {
return nil, err
}
rightValue := reflect.ValueOf(right)
switch rightValue.Kind() {
case reflect.String:
return impl.getPropertyValue(leftValue, rightValue.String())
case reflect.Int:
switch leftValue.Kind() {
case reflect.Slice:
if rightValue.Int() < 0 || rightValue.Int() >= int64(leftValue.Len()) {
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
}
return leftValue.Index(int(rightValue.Int())).Interface(), nil
default:
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
}
default:
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
}
}
func (impl *interperterImpl) evaluateObjectDeref(objectDerefNode *actionlint.ObjectDerefNode) (any, error) {
left, err := impl.evaluateNode(objectDerefNode.Receiver)
if err != nil {
return nil, err
}
return impl.getPropertyValue(reflect.ValueOf(left), objectDerefNode.Property)
}
func (impl *interperterImpl) evaluateArrayDeref(arrayDerefNode *actionlint.ArrayDerefNode) (any, error) {
left, err := impl.evaluateNode(arrayDerefNode.Receiver)
if err != nil {
return nil, err
}
return impl.getSafeValue(reflect.ValueOf(left)), nil
}
func (impl *interperterImpl) getPropertyValue(left reflect.Value, property string) (value any, err error) {
switch left.Kind() {
case reflect.Pointer:
return impl.getPropertyValue(left.Elem(), property)
case reflect.Struct:
leftType := left.Type()
for field := range leftType.Fields() {
jsonName := field.Tag.Get("json")
if jsonName == property {
property = field.Name
break
}
}
fieldValue := left.FieldByNameFunc(func(name string) bool {
return strings.EqualFold(name, property)
})
if fieldValue.Kind() == reflect.Invalid {
return "", nil
}
i := fieldValue.Interface()
// The type stepStatus int is an integer, but should be treated as string
if m, ok := i.(encoding.TextMarshaler); ok {
text, err := m.MarshalText()
if err != nil {
return nil, err
}
return string(text), nil
}
return i, nil
case reflect.Map:
iter := left.MapRange()
for iter.Next() {
key := iter.Key()
switch key.Kind() {
case reflect.String:
if strings.EqualFold(key.String(), property) {
return impl.getMapValue(iter.Value())
}
default:
return nil, fmt.Errorf("'%s' in map key not implemented", key.Kind())
}
}
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
case reflect.Slice:
var values []any
for i := 0; i < left.Len(); i++ {
value, err := impl.getPropertyValue(left.Index(i).Elem(), property)
if err != nil {
return nil, err
}
values = append(values, value)
}
return values, nil
}
return nil, nil //nolint:nilnil // pre-existing issue from nektos/act
}
func (impl *interperterImpl) getMapValue(value reflect.Value) (any, error) {
if value.Kind() == reflect.Pointer {
return impl.getMapValue(value.Elem())
}
return value.Interface(), nil
}
func (impl *interperterImpl) evaluateNot(notNode *actionlint.NotOpNode) (any, error) {
operand, err := impl.evaluateNode(notNode.Operand)
if err != nil {
return nil, err
}
return !IsTruthy(operand), nil
}
func (impl *interperterImpl) evaluateCompare(compareNode *actionlint.CompareOpNode) (any, error) {
left, err := impl.evaluateNode(compareNode.Left)
if err != nil {
return nil, err
}
right, err := impl.evaluateNode(compareNode.Right)
if err != nil {
return nil, err
}
leftValue := reflect.ValueOf(left)
rightValue := reflect.ValueOf(right)
return impl.compareValues(leftValue, rightValue, compareNode.Kind)
}
func (impl *interperterImpl) compareValues(leftValue, rightValue reflect.Value, kind actionlint.CompareOpNodeKind) (any, error) {
if leftValue.Kind() != rightValue.Kind() {
if !impl.isNumber(leftValue) {
leftValue = impl.coerceToNumber(leftValue)
}
if !impl.isNumber(rightValue) {
rightValue = impl.coerceToNumber(rightValue)
}
}
switch leftValue.Kind() {
case reflect.Bool:
return impl.compareNumber(float64(impl.coerceToNumber(leftValue).Int()), float64(impl.coerceToNumber(rightValue).Int()), kind)
case reflect.String:
return impl.compareString(strings.ToLower(leftValue.String()), strings.ToLower(rightValue.String()), kind)
case reflect.Int:
if rightValue.Kind() == reflect.Float64 {
return impl.compareNumber(float64(leftValue.Int()), rightValue.Float(), kind)
}
return impl.compareNumber(float64(leftValue.Int()), float64(rightValue.Int()), kind)
case reflect.Float64:
if rightValue.Kind() == reflect.Int {
return impl.compareNumber(leftValue.Float(), float64(rightValue.Int()), kind)
}
return impl.compareNumber(leftValue.Float(), rightValue.Float(), kind)
case reflect.Invalid:
if rightValue.Kind() == reflect.Invalid {
return true, nil
}
// not possible situation - params are converted to the same type in code above
return nil, fmt.Errorf("Compare params of Invalid type: left: %+v, right: %+v", leftValue.Kind(), rightValue.Kind())
default:
return nil, fmt.Errorf("Compare not implemented for types: left: %+v, right: %+v", leftValue.Kind(), rightValue.Kind())
}
}
func (impl *interperterImpl) coerceToNumber(value reflect.Value) reflect.Value {
switch value.Kind() {
case reflect.Invalid:
return reflect.ValueOf(0)
case reflect.Bool:
switch value.Bool() {
case true:
return reflect.ValueOf(1)
case false:
return reflect.ValueOf(0)
}
case reflect.String:
if value.String() == "" {
return reflect.ValueOf(0)
}
// try to parse the string as a number
evaluated, err := impl.Evaluate(value.String(), DefaultStatusCheckNone)
if err != nil {
return reflect.ValueOf(math.NaN())
}
if value := reflect.ValueOf(evaluated); impl.isNumber(value) {
return value
}
}
return reflect.ValueOf(math.NaN())
}
func (impl *interperterImpl) coerceToString(value reflect.Value) reflect.Value {
switch value.Kind() {
case reflect.Invalid:
return reflect.ValueOf("")
case reflect.Bool:
switch value.Bool() {
case true:
return reflect.ValueOf("true")
case false:
return reflect.ValueOf("false")
}
case reflect.String:
return value
case reflect.Int:
return reflect.ValueOf(fmt.Sprint(value))
case reflect.Float64:
if math.IsInf(value.Float(), 1) {
return reflect.ValueOf("Infinity")
} else if math.IsInf(value.Float(), -1) {
return reflect.ValueOf("-Infinity")
}
return reflect.ValueOf(fmt.Sprintf("%.15G", value.Float()))
case reflect.Slice:
return reflect.ValueOf("Array")
case reflect.Map:
return reflect.ValueOf("Object")
}
return value
}
func (impl *interperterImpl) compareString(left, right string, kind actionlint.CompareOpNodeKind) (bool, error) {
switch kind {
case actionlint.CompareOpNodeKindLess:
return left < right, nil
case actionlint.CompareOpNodeKindLessEq:
return left <= right, nil
case actionlint.CompareOpNodeKindGreater:
return left > right, nil
case actionlint.CompareOpNodeKindGreaterEq:
return left >= right, nil
case actionlint.CompareOpNodeKindEq:
return left == right, nil
case actionlint.CompareOpNodeKindNotEq:
return left != right, nil
default:
return false, fmt.Errorf("TODO: not implemented to compare '%+v'", kind)
}
}
func (impl *interperterImpl) compareNumber(left, right float64, kind actionlint.CompareOpNodeKind) (bool, error) {
switch kind {
case actionlint.CompareOpNodeKindLess:
return left < right, nil
case actionlint.CompareOpNodeKindLessEq:
return left <= right, nil
case actionlint.CompareOpNodeKindGreater:
return left > right, nil
case actionlint.CompareOpNodeKindGreaterEq:
return left >= right, nil
case actionlint.CompareOpNodeKindEq:
return left == right, nil
case actionlint.CompareOpNodeKindNotEq:
return left != right, nil
default:
return false, fmt.Errorf("TODO: not implemented to compare '%+v'", kind)
}
}
func IsTruthy(input any) bool {
value := reflect.ValueOf(input)
switch value.Kind() {
case reflect.Bool:
return value.Bool()
case reflect.String:
return value.String() != ""
case reflect.Int:
return value.Int() != 0
case reflect.Float64:
if math.IsNaN(value.Float()) {
return false
}
return value.Float() != 0
case reflect.Map, reflect.Slice:
return true
default:
return false
}
}
func (impl *interperterImpl) isNumber(value reflect.Value) bool {
switch value.Kind() {
case reflect.Int, reflect.Float64:
return true
default:
return false
}
}
func (impl *interperterImpl) getSafeValue(value reflect.Value) any {
switch value.Kind() {
case reflect.Invalid:
return nil
case reflect.Float64:
if value.Float() == 0 {
return 0
}
}
return value.Interface()
}
func (impl *interperterImpl) evaluateLogicalCompare(compareNode *actionlint.LogicalOpNode) (any, error) {
left, err := impl.evaluateNode(compareNode.Left)
if err != nil {
return nil, err
}
leftValue := reflect.ValueOf(left)
if IsTruthy(left) == (compareNode.Kind == actionlint.LogicalOpNodeKindOr) {
return impl.getSafeValue(leftValue), nil
}
right, err := impl.evaluateNode(compareNode.Right)
if err != nil {
return nil, err
}
rightValue := reflect.ValueOf(right)
switch compareNode.Kind {
case actionlint.LogicalOpNodeKindAnd:
return impl.getSafeValue(rightValue), nil
case actionlint.LogicalOpNodeKindOr:
return impl.getSafeValue(rightValue), nil
}
return nil, fmt.Errorf("Unable to compare incompatibles types '%s' and '%s'", leftValue.Kind(), rightValue.Kind())
}
func (impl *interperterImpl) evaluateFuncCall(funcCallNode *actionlint.FuncCallNode) (any, error) {
args := make([]reflect.Value, 0)
for _, arg := range funcCallNode.Args {
value, err := impl.evaluateNode(arg)
if err != nil {
return nil, err
}
args = append(args, reflect.ValueOf(value))
}
switch strings.ToLower(funcCallNode.Callee) {
case "contains":
return impl.contains(args[0], args[1])
case "startswith":
return impl.startsWith(args[0], args[1])
case "endswith":
return impl.endsWith(args[0], args[1])
case "format":
return impl.format(args[0], args[1:]...)
case "join":
if len(args) == 1 {
return impl.join(args[0], reflect.ValueOf(","))
}
return impl.join(args[0], args[1])
case "tojson":
return impl.toJSON(args[0])
case "fromjson":
return impl.fromJSON(args[0])
case "hashfiles":
if impl.env.HashFiles != nil {
return impl.env.HashFiles(args)
}
return impl.hashFiles(args...)
case "always":
return impl.always()
case "success":
if impl.config.Context == "job" {
return impl.jobSuccess()
}
if impl.config.Context == "step" {
return impl.stepSuccess()
}
return nil, fmt.Errorf("Context '%s' must be one of 'job' or 'step'", impl.config.Context)
case "failure":
if impl.config.Context == "job" {
return impl.jobFailure()
}
if impl.config.Context == "step" {
return impl.stepFailure()
}
return nil, fmt.Errorf("Context '%s' must be one of 'job' or 'step'", impl.config.Context)
case "cancelled":
return impl.cancelled()
default:
return nil, fmt.Errorf("TODO: '%s' not implemented", funcCallNode.Callee)
}
}
+632
View File
@@ -0,0 +1,632 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package exprparser
import (
"math"
"testing"
"gitea.com/gitea/runner/act/model"
"github.com/stretchr/testify/assert"
)
func TestLiterals(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"true", true, "true"},
{"false", false, "false"},
{"null", nil, "null"},
{"123", 123, "integer"},
{"-9.7", -9.7, "float"},
{"0xff", 255, "hex"},
{"-2.99e-2", -2.99e-2, "exponential"},
{"'foo'", "foo", "string"},
{"'it''s foo'", "it's foo", "string"},
}
env := &EvaluationEnvironment{}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestOperators(t *testing.T) {
table := []struct {
input string
expected any
name string
error string
}{
{"(false || (false || true))", true, "logical-grouping", ""},
{"github.action", "push", "property-dereference", ""},
{"github['action']", "push", "property-index", ""},
{"github.action[0]", nil, "string-index", ""},
{"github.action['0']", nil, "string-index", ""},
{"fromJSON('[0,1]')[1]", 1.0, "array-index", ""},
{"fromJSON('[0,1]')[1.1]", nil, "array-index", ""},
// Disabled weird things are happening
// {"fromJSON('[0,1]')['1.1']", nil, "array-index", ""},
{"(github.event.commits.*.author.username)[0]", "someone", "array-index-0", ""},
{"fromJSON('[0,1]')[2]", nil, "array-index-out-of-bounds-0", ""},
{"fromJSON('[0,1]')[34553]", nil, "array-index-out-of-bounds-1", ""},
{"fromJSON('[0,1]')[-1]", nil, "array-index-out-of-bounds-2", ""},
{"fromJSON('[0,1]')[-34553]", nil, "array-index-out-of-bounds-3", ""},
{"!true", false, "not", ""},
{"1 < 2", true, "less-than", ""},
{`'b' <= 'a'`, false, "less-than-or-equal", ""},
{"1 > 2", false, "greater-than", ""},
{`'b' >= 'a'`, true, "greater-than-or-equal", ""},
{`'a' == 'a'`, true, "equal", ""},
{`'a' != 'a'`, false, "not-equal", ""},
{`true && false`, false, "and", ""},
{`true || false`, true, "or", ""},
{`fromJSON('{}') && true`, true, "and-boolean-object", ""},
{`fromJSON('{}') || false`, make(map[string]any), "or-boolean-object", ""},
{"github.event.commits[0].author.username != github.event.commits[1].author.username", true, "property-comparison1", ""},
{"github.event.commits[0].author.username1 != github.event.commits[1].author.username", true, "property-comparison2", ""},
{"github.event.commits[0].author.username != github.event.commits[1].author.username1", true, "property-comparison3", ""},
{"github.event.commits[0].author.username1 != github.event.commits[1].author.username2", true, "property-comparison4", ""},
{"secrets != env", nil, "property-comparison5", "Compare not implemented for types: left: map, right: map"},
}
env := &EvaluationEnvironment{
Github: &model.GithubContext{
Action: "push",
Event: map[string]any{
"commits": []any{
map[string]any{
"author": map[string]any{
"username": "someone",
},
},
map[string]any{
"author": map[string]any{
"username": "someone-else",
},
},
},
},
},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
if tt.error != "" {
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.error, err.Error())
} else {
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
}
assert.Equal(t, tt.expected, output)
})
}
}
func TestOperatorsCompare(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"!null", true, "not-null"},
{"!-10", false, "not-neg-num"},
{"!0", true, "not-zero"},
{"!3.14", false, "not-pos-float"},
{"!''", true, "not-empty-str"},
{"!'abc'", false, "not-str"},
{"!fromJSON('{}')", false, "not-obj"},
{"!fromJSON('[]')", false, "not-arr"},
{`null == 0 }}`, true, "null-coercion"},
{`true == 1 }}`, true, "boolean-coercion"},
{`'' == 0 }}`, true, "string-0-coercion"},
{`'3' == 3 }}`, true, "string-3-coercion"},
{`0 == null }}`, true, "null-coercion-alt"},
{`1 == true }}`, true, "boolean-coercion-alt"},
{`0 == '' }}`, true, "string-0-coercion-alt"},
{`3 == '3' }}`, true, "string-3-coercion-alt"},
{`'TEST' == 'test' }}`, true, "string-casing"},
{"true > false }}", true, "bool-greater-than"},
{"true >= false }}", true, "bool-greater-than-eq"},
{"true >= true }}", true, "bool-greater-than-1"},
{"true != false }}", true, "bool-not-equal"},
{`fromJSON('{}') < 2 }}`, false, "object-with-less"},
{`fromJSON('{}') < fromJSON('[]') }}`, false, "object/arr-with-lt"},
{`fromJSON('{}') > fromJSON('[]') }}`, false, "object/arr-with-gt"},
}
env := &EvaluationEnvironment{
Github: &model.GithubContext{
Action: "push",
},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
func TestOperatorsBooleanEvaluation(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
// true &&
{"true && true", true, "true-and"},
{"true && false", false, "true-and"},
{"true && null", nil, "true-and"},
{"true && -10", -10, "true-and"},
{"true && 0", 0, "true-and"},
{"true && 10", 10, "true-and"},
{"true && 3.14", 3.14, "true-and"},
{"true && 0.0", 0, "true-and"},
{"true && Infinity", math.Inf(1), "true-and"},
// {"true && -Infinity", math.Inf(-1), "true-and"},
{"true && NaN", math.NaN(), "true-and"},
{"true && ''", "", "true-and"},
{"true && 'abc'", "abc", "true-and"},
// false &&
{"false && true", false, "false-and"},
{"false && false", false, "false-and"},
{"false && null", false, "false-and"},
{"false && -10", false, "false-and"},
{"false && 0", false, "false-and"},
{"false && 10", false, "false-and"},
{"false && 3.14", false, "false-and"},
{"false && 0.0", false, "false-and"},
{"false && Infinity", false, "false-and"},
// {"false && -Infinity", false, "false-and"},
{"false && NaN", false, "false-and"},
{"false && ''", false, "false-and"},
{"false && 'abc'", false, "false-and"},
// true ||
{"true || true", true, "true-or"},
{"true || false", true, "true-or"},
{"true || null", true, "true-or"},
{"true || -10", true, "true-or"},
{"true || 0", true, "true-or"},
{"true || 10", true, "true-or"},
{"true || 3.14", true, "true-or"},
{"true || 0.0", true, "true-or"},
{"true || Infinity", true, "true-or"},
// {"true || -Infinity", true, "true-or"},
{"true || NaN", true, "true-or"},
{"true || ''", true, "true-or"},
{"true || 'abc'", true, "true-or"},
// false ||
{"false || true", true, "false-or"},
{"false || false", false, "false-or"},
{"false || null", nil, "false-or"},
{"false || -10", -10, "false-or"},
{"false || 0", 0, "false-or"},
{"false || 10", 10, "false-or"},
{"false || 3.14", 3.14, "false-or"},
{"false || 0.0", 0, "false-or"},
{"false || Infinity", math.Inf(1), "false-or"},
// {"false || -Infinity", math.Inf(-1), "false-or"},
{"false || NaN", math.NaN(), "false-or"},
{"false || ''", "", "false-or"},
{"false || 'abc'", "abc", "false-or"},
// null &&
{"null && true", nil, "null-and"},
{"null && false", nil, "null-and"},
{"null && null", nil, "null-and"},
{"null && -10", nil, "null-and"},
{"null && 0", nil, "null-and"},
{"null && 10", nil, "null-and"},
{"null && 3.14", nil, "null-and"},
{"null && 0.0", nil, "null-and"},
{"null && Infinity", nil, "null-and"},
// {"null && -Infinity", nil, "null-and"},
{"null && NaN", nil, "null-and"},
{"null && ''", nil, "null-and"},
{"null && 'abc'", nil, "null-and"},
// null ||
{"null || true", true, "null-or"},
{"null || false", false, "null-or"},
{"null || null", nil, "null-or"},
{"null || -10", -10, "null-or"},
{"null || 0", 0, "null-or"},
{"null || 10", 10, "null-or"},
{"null || 3.14", 3.14, "null-or"},
{"null || 0.0", 0, "null-or"},
{"null || Infinity", math.Inf(1), "null-or"},
// {"null || -Infinity", math.Inf(-1), "null-or"},
{"null || NaN", math.NaN(), "null-or"},
{"null || ''", "", "null-or"},
{"null || 'abc'", "abc", "null-or"},
// -10 &&
{"-10 && true", true, "neg-num-and"},
{"-10 && false", false, "neg-num-and"},
{"-10 && null", nil, "neg-num-and"},
{"-10 && -10", -10, "neg-num-and"},
{"-10 && 0", 0, "neg-num-and"},
{"-10 && 10", 10, "neg-num-and"},
{"-10 && 3.14", 3.14, "neg-num-and"},
{"-10 && 0.0", 0, "neg-num-and"},
{"-10 && Infinity", math.Inf(1), "neg-num-and"},
// {"-10 && -Infinity", math.Inf(-1), "neg-num-and"},
{"-10 && NaN", math.NaN(), "neg-num-and"},
{"-10 && ''", "", "neg-num-and"},
{"-10 && 'abc'", "abc", "neg-num-and"},
// -10 ||
{"-10 || true", -10, "neg-num-or"},
{"-10 || false", -10, "neg-num-or"},
{"-10 || null", -10, "neg-num-or"},
{"-10 || -10", -10, "neg-num-or"},
{"-10 || 0", -10, "neg-num-or"},
{"-10 || 10", -10, "neg-num-or"},
{"-10 || 3.14", -10, "neg-num-or"},
{"-10 || 0.0", -10, "neg-num-or"},
{"-10 || Infinity", -10, "neg-num-or"},
// {"-10 || -Infinity", -10, "neg-num-or"},
{"-10 || NaN", -10, "neg-num-or"},
{"-10 || ''", -10, "neg-num-or"},
{"-10 || 'abc'", -10, "neg-num-or"},
// 0 &&
{"0 && true", 0, "zero-and"},
{"0 && false", 0, "zero-and"},
{"0 && null", 0, "zero-and"},
{"0 && -10", 0, "zero-and"},
{"0 && 0", 0, "zero-and"},
{"0 && 10", 0, "zero-and"},
{"0 && 3.14", 0, "zero-and"},
{"0 && 0.0", 0, "zero-and"},
{"0 && Infinity", 0, "zero-and"},
// {"0 && -Infinity", 0, "zero-and"},
{"0 && NaN", 0, "zero-and"},
{"0 && ''", 0, "zero-and"},
{"0 && 'abc'", 0, "zero-and"},
// 0 ||
{"0 || true", true, "zero-or"},
{"0 || false", false, "zero-or"},
{"0 || null", nil, "zero-or"},
{"0 || -10", -10, "zero-or"},
{"0 || 0", 0, "zero-or"},
{"0 || 10", 10, "zero-or"},
{"0 || 3.14", 3.14, "zero-or"},
{"0 || 0.0", 0, "zero-or"},
{"0 || Infinity", math.Inf(1), "zero-or"},
// {"0 || -Infinity", math.Inf(-1), "zero-or"},
{"0 || NaN", math.NaN(), "zero-or"},
{"0 || ''", "", "zero-or"},
{"0 || 'abc'", "abc", "zero-or"},
// 10 &&
{"10 && true", true, "pos-num-and"},
{"10 && false", false, "pos-num-and"},
{"10 && null", nil, "pos-num-and"},
{"10 && -10", -10, "pos-num-and"},
{"10 && 0", 0, "pos-num-and"},
{"10 && 10", 10, "pos-num-and"},
{"10 && 3.14", 3.14, "pos-num-and"},
{"10 && 0.0", 0, "pos-num-and"},
{"10 && Infinity", math.Inf(1), "pos-num-and"},
// {"10 && -Infinity", math.Inf(-1), "pos-num-and"},
{"10 && NaN", math.NaN(), "pos-num-and"},
{"10 && ''", "", "pos-num-and"},
{"10 && 'abc'", "abc", "pos-num-and"},
// 10 ||
{"10 || true", 10, "pos-num-or"},
{"10 || false", 10, "pos-num-or"},
{"10 || null", 10, "pos-num-or"},
{"10 || -10", 10, "pos-num-or"},
{"10 || 0", 10, "pos-num-or"},
{"10 || 10", 10, "pos-num-or"},
{"10 || 3.14", 10, "pos-num-or"},
{"10 || 0.0", 10, "pos-num-or"},
{"10 || Infinity", 10, "pos-num-or"},
// {"10 || -Infinity", 10, "pos-num-or"},
{"10 || NaN", 10, "pos-num-or"},
{"10 || ''", 10, "pos-num-or"},
{"10 || 'abc'", 10, "pos-num-or"},
// 3.14 &&
{"3.14 && true", true, "pos-float-and"},
{"3.14 && false", false, "pos-float-and"},
{"3.14 && null", nil, "pos-float-and"},
{"3.14 && -10", -10, "pos-float-and"},
{"3.14 && 0", 0, "pos-float-and"},
{"3.14 && 10", 10, "pos-float-and"},
{"3.14 && 3.14", 3.14, "pos-float-and"},
{"3.14 && 0.0", 0, "pos-float-and"},
{"3.14 && Infinity", math.Inf(1), "pos-float-and"},
// {"3.14 && -Infinity", math.Inf(-1), "pos-float-and"},
{"3.14 && NaN", math.NaN(), "pos-float-and"},
{"3.14 && ''", "", "pos-float-and"},
{"3.14 && 'abc'", "abc", "pos-float-and"},
// 3.14 ||
{"3.14 || true", 3.14, "pos-float-or"},
{"3.14 || false", 3.14, "pos-float-or"},
{"3.14 || null", 3.14, "pos-float-or"},
{"3.14 || -10", 3.14, "pos-float-or"},
{"3.14 || 0", 3.14, "pos-float-or"},
{"3.14 || 10", 3.14, "pos-float-or"},
{"3.14 || 3.14", 3.14, "pos-float-or"},
{"3.14 || 0.0", 3.14, "pos-float-or"},
{"3.14 || Infinity", 3.14, "pos-float-or"},
// {"3.14 || -Infinity", 3.14, "pos-float-or"},
{"3.14 || NaN", 3.14, "pos-float-or"},
{"3.14 || ''", 3.14, "pos-float-or"},
{"3.14 || 'abc'", 3.14, "pos-float-or"},
// Infinity &&
{"Infinity && true", true, "pos-inf-and"},
{"Infinity && false", false, "pos-inf-and"},
{"Infinity && null", nil, "pos-inf-and"},
{"Infinity && -10", -10, "pos-inf-and"},
{"Infinity && 0", 0, "pos-inf-and"},
{"Infinity && 10", 10, "pos-inf-and"},
{"Infinity && 3.14", 3.14, "pos-inf-and"},
{"Infinity && 0.0", 0, "pos-inf-and"},
{"Infinity && Infinity", math.Inf(1), "pos-inf-and"},
// {"Infinity && -Infinity", math.Inf(-1), "pos-inf-and"},
{"Infinity && NaN", math.NaN(), "pos-inf-and"},
{"Infinity && ''", "", "pos-inf-and"},
{"Infinity && 'abc'", "abc", "pos-inf-and"},
// Infinity ||
{"Infinity || true", math.Inf(1), "pos-inf-or"},
{"Infinity || false", math.Inf(1), "pos-inf-or"},
{"Infinity || null", math.Inf(1), "pos-inf-or"},
{"Infinity || -10", math.Inf(1), "pos-inf-or"},
{"Infinity || 0", math.Inf(1), "pos-inf-or"},
{"Infinity || 10", math.Inf(1), "pos-inf-or"},
{"Infinity || 3.14", math.Inf(1), "pos-inf-or"},
{"Infinity || 0.0", math.Inf(1), "pos-inf-or"},
{"Infinity || Infinity", math.Inf(1), "pos-inf-or"},
// {"Infinity || -Infinity", math.Inf(1), "pos-inf-or"},
{"Infinity || NaN", math.Inf(1), "pos-inf-or"},
{"Infinity || ''", math.Inf(1), "pos-inf-or"},
{"Infinity || 'abc'", math.Inf(1), "pos-inf-or"},
// -Infinity &&
// {"-Infinity && true", true, "neg-inf-and"},
// {"-Infinity && false", false, "neg-inf-and"},
// {"-Infinity && null", nil, "neg-inf-and"},
// {"-Infinity && -10", -10, "neg-inf-and"},
// {"-Infinity && 0", 0, "neg-inf-and"},
// {"-Infinity && 10", 10, "neg-inf-and"},
// {"-Infinity && 3.14", 3.14, "neg-inf-and"},
// {"-Infinity && 0.0", 0, "neg-inf-and"},
// {"-Infinity && Infinity", math.Inf(1), "neg-inf-and"},
// {"-Infinity && -Infinity", math.Inf(-1), "neg-inf-and"},
// {"-Infinity && NaN", math.NaN(), "neg-inf-and"},
// {"-Infinity && ''", "", "neg-inf-and"},
// {"-Infinity && 'abc'", "abc", "neg-inf-and"},
// -Infinity ||
// {"-Infinity || true", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || false", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || null", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || -10", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || 0", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || 10", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || 3.14", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || 0.0", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || Infinity", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || -Infinity", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || NaN", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || ''", math.Inf(-1), "neg-inf-or"},
// {"-Infinity || 'abc'", math.Inf(-1), "neg-inf-or"},
// NaN &&
{"NaN && true", math.NaN(), "nan-and"},
{"NaN && false", math.NaN(), "nan-and"},
{"NaN && null", math.NaN(), "nan-and"},
{"NaN && -10", math.NaN(), "nan-and"},
{"NaN && 0", math.NaN(), "nan-and"},
{"NaN && 10", math.NaN(), "nan-and"},
{"NaN && 3.14", math.NaN(), "nan-and"},
{"NaN && 0.0", math.NaN(), "nan-and"},
{"NaN && Infinity", math.NaN(), "nan-and"},
// {"NaN && -Infinity", math.NaN(), "nan-and"},
{"NaN && NaN", math.NaN(), "nan-and"},
{"NaN && ''", math.NaN(), "nan-and"},
{"NaN && 'abc'", math.NaN(), "nan-and"},
// NaN ||
{"NaN || true", true, "nan-or"},
{"NaN || false", false, "nan-or"},
{"NaN || null", nil, "nan-or"},
{"NaN || -10", -10, "nan-or"},
{"NaN || 0", 0, "nan-or"},
{"NaN || 10", 10, "nan-or"},
{"NaN || 3.14", 3.14, "nan-or"},
{"NaN || 0.0", 0, "nan-or"},
{"NaN || Infinity", math.Inf(1), "nan-or"},
// {"NaN || -Infinity", math.Inf(-1), "nan-or"},
{"NaN || NaN", math.NaN(), "nan-or"},
{"NaN || ''", "", "nan-or"},
{"NaN || 'abc'", "abc", "nan-or"},
// "" &&
{"'' && true", "", "empty-str-and"},
{"'' && false", "", "empty-str-and"},
{"'' && null", "", "empty-str-and"},
{"'' && -10", "", "empty-str-and"},
{"'' && 0", "", "empty-str-and"},
{"'' && 10", "", "empty-str-and"},
{"'' && 3.14", "", "empty-str-and"},
{"'' && 0.0", "", "empty-str-and"},
{"'' && Infinity", "", "empty-str-and"},
// {"'' && -Infinity", "", "empty-str-and"},
{"'' && NaN", "", "empty-str-and"},
{"'' && ''", "", "empty-str-and"},
{"'' && 'abc'", "", "empty-str-and"},
// "" ||
{"'' || true", true, "empty-str-or"},
{"'' || false", false, "empty-str-or"},
{"'' || null", nil, "empty-str-or"},
{"'' || -10", -10, "empty-str-or"},
{"'' || 0", 0, "empty-str-or"},
{"'' || 10", 10, "empty-str-or"},
{"'' || 3.14", 3.14, "empty-str-or"},
{"'' || 0.0", 0, "empty-str-or"},
{"'' || Infinity", math.Inf(1), "empty-str-or"},
// {"'' || -Infinity", math.Inf(-1), "empty-str-or"},
{"'' || NaN", math.NaN(), "empty-str-or"},
{"'' || ''", "", "empty-str-or"},
{"'' || 'abc'", "abc", "empty-str-or"},
// "abc" &&
{"'abc' && true", true, "str-and"},
{"'abc' && false", false, "str-and"},
{"'abc' && null", nil, "str-and"},
{"'abc' && -10", -10, "str-and"},
{"'abc' && 0", 0, "str-and"},
{"'abc' && 10", 10, "str-and"},
{"'abc' && 3.14", 3.14, "str-and"},
{"'abc' && 0.0", 0, "str-and"},
{"'abc' && Infinity", math.Inf(1), "str-and"},
// {"'abc' && -Infinity", math.Inf(-1), "str-and"},
{"'abc' && NaN", math.NaN(), "str-and"},
{"'abc' && ''", "", "str-and"},
{"'abc' && 'abc'", "abc", "str-and"},
// "abc" ||
{"'abc' || true", "abc", "str-or"},
{"'abc' || false", "abc", "str-or"},
{"'abc' || null", "abc", "str-or"},
{"'abc' || -10", "abc", "str-or"},
{"'abc' || 0", "abc", "str-or"},
{"'abc' || 10", "abc", "str-or"},
{"'abc' || 3.14", "abc", "str-or"},
{"'abc' || 0.0", "abc", "str-or"},
{"'abc' || Infinity", "abc", "str-or"},
// {"'abc' || -Infinity", "abc", "str-or"},
{"'abc' || NaN", "abc", "str-or"},
{"'abc' || ''", "abc", "str-or"},
{"'abc' || 'abc'", "abc", "str-or"},
// extra tests
{"0.0 && true", 0, "float-evaluation-0-alt"},
{"-1.5 && true", true, "float-evaluation-neg-alt"},
}
env := &EvaluationEnvironment{
Github: &model.GithubContext{
Action: "push",
},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
if expected, ok := tt.expected.(float64); ok && math.IsNaN(expected) {
assert.True(t, math.IsNaN(output.(float64)))
} else {
assert.Equal(t, tt.expected, output)
}
})
}
}
func TestContexts(t *testing.T) {
table := []struct {
input string
expected any
name string
}{
{"github.action", "push", "github-context"},
{"github.event.commits[0].message", nil, "github-context-noexist-prop"},
{"fromjson('{\"commits\":[]}').commits[0].message", nil, "github-context-noexist-prop"},
{"github.event.pull_request.labels.*.name", nil, "github-context-noexist-prop"},
{"env.TEST", "value", "env-context"},
{"job.status", "success", "job-context"},
{"steps.step-id.outputs.name", "value", "steps-context"},
{"steps.step-id.conclusion", "success", "steps-context-conclusion"},
{"steps.step-id.conclusion && true", true, "steps-context-conclusion"},
{"steps.step-id2.conclusion", "skipped", "steps-context-conclusion"},
{"steps.step-id2.conclusion && true", true, "steps-context-conclusion"},
{"steps.step-id.outcome", "success", "steps-context-outcome"},
{"steps.step-id['outcome']", "success", "steps-context-outcome"},
{"steps.step-id.outcome == 'success'", true, "steps-context-outcome"},
{"steps.step-id['outcome'] == 'success'", true, "steps-context-outcome"},
{"steps.step-id.outcome && true", true, "steps-context-outcome"},
{"steps['step-id']['outcome'] && true", true, "steps-context-outcome"},
{"steps.step-id2.outcome", "failure", "steps-context-outcome"},
{"steps.step-id2.outcome && true", true, "steps-context-outcome"},
// Disabled, since the interpreter is still too broken
// {"contains(steps.*.outcome, 'success')", true, "steps-context-array-outcome"},
// {"contains(steps.*.outcome, 'failure')", true, "steps-context-array-outcome"},
// {"contains(steps.*.outputs.name, 'value')", true, "steps-context-array-outputs"},
{"runner.os", "Linux", "runner-context"},
{"secrets.name", "value", "secrets-context"},
{"vars.name", "value", "vars-context"},
{"strategy.fail-fast", true, "strategy-context"},
{"matrix.os", "Linux", "matrix-context"},
{"needs.job-id.outputs.output-name", "value", "needs-context"},
{"needs.job-id.result", "success", "needs-context"},
{"inputs.name", "value", "inputs-context"},
}
env := &EvaluationEnvironment{
Github: &model.GithubContext{
Action: "push",
},
Env: map[string]string{
"TEST": "value",
},
Job: &model.JobContext{
Status: "success",
},
Steps: map[string]*model.StepResult{
"step-id": {
Outputs: map[string]string{
"name": "value",
},
},
"step-id2": {
Outcome: model.StepStatusFailure,
Conclusion: model.StepStatusSkipped,
},
},
Runner: map[string]any{
"os": "Linux",
"temp": "/tmp",
"tool_cache": "/opt/hostedtoolcache",
},
Secrets: map[string]string{
"name": "value",
},
Vars: map[string]string{
"name": "value",
},
Strategy: map[string]any{
"fail-fast": true,
},
Matrix: map[string]any{
"os": "Linux",
},
Needs: map[string]Needs{
"job-id": {
Outputs: map[string]string{
"output-name": "value",
},
Result: "success",
},
},
Inputs: map[string]any{
"name": "value",
},
}
for _, tt := range table {
t.Run(tt.name, func(t *testing.T) {
output, err := NewInterpeter(env, Config{}).Evaluate(tt.input, DefaultStatusCheckNone)
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Equal(t, tt.expected, output)
})
}
}
+1
View File
@@ -0,0 +1 @@
Hello
+1
View File
@@ -0,0 +1 @@
World!
+1
View File
@@ -0,0 +1 @@
Knock knock!
@@ -0,0 +1 @@
Anybody home?
+136
View File
@@ -0,0 +1,136 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// Copyright 2020 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
import (
"fmt"
"io"
"strings"
"go.yaml.in/yaml/v4"
)
// ActionRunsUsing is the type of runner for the action
type ActionRunsUsing string
func (a *ActionRunsUsing) UnmarshalYAML(unmarshal func(any) error) error {
var using string
if err := unmarshal(&using); err != nil {
return err
}
// Force input to lowercase for case insensitive comparison
format := ActionRunsUsing(strings.ToLower(using))
switch format {
case ActionRunsUsingNode24, ActionRunsUsingNode20, ActionRunsUsingNode16, ActionRunsUsingNode12, ActionRunsUsingDocker, ActionRunsUsingComposite, ActionRunsUsingGo:
*a = format
default:
return fmt.Errorf("The runs.using key in action.yml must be one of: %v, got %s", []string{
ActionRunsUsingComposite,
ActionRunsUsingDocker,
ActionRunsUsingNode12,
ActionRunsUsingNode16,
ActionRunsUsingNode20,
ActionRunsUsingNode24,
ActionRunsUsingGo,
}, format)
}
return nil
}
const (
// ActionRunsUsingNode12 for running with node12
ActionRunsUsingNode12 = "node12"
// ActionRunsUsingNode16 for running with node16
ActionRunsUsingNode16 = "node16"
// ActionRunsUsingNode20 for running with node20
ActionRunsUsingNode20 = "node20"
// ActionRunsUsingNode24 for running with node24
ActionRunsUsingNode24 = "node24"
// ActionRunsUsingDocker for running with docker
ActionRunsUsingDocker = "docker"
// ActionRunsUsingComposite for running composite
ActionRunsUsingComposite = "composite"
// ActionRunsUsingGo for running with go
ActionRunsUsingGo = "go"
)
func (a ActionRunsUsing) IsNode() bool {
switch a {
case ActionRunsUsingNode12, ActionRunsUsingNode16, ActionRunsUsingNode20, ActionRunsUsingNode24:
return true
default:
return false
}
}
func (a ActionRunsUsing) IsDocker() bool {
return a == ActionRunsUsingDocker
}
func (a ActionRunsUsing) IsComposite() bool {
return a == ActionRunsUsingComposite
}
// ActionRuns are a field in Action
type ActionRuns struct {
Using ActionRunsUsing `yaml:"using"`
Env map[string]string `yaml:"env"`
Main string `yaml:"main"`
Pre string `yaml:"pre"`
PreIf string `yaml:"pre-if"`
Post string `yaml:"post"`
PostIf string `yaml:"post-if"`
Image string `yaml:"image"`
Entrypoint string `yaml:"entrypoint"`
Args []string `yaml:"args"`
Steps []Step `yaml:"steps"`
}
// Action describes a metadata file for GitHub actions. The metadata filename must be either action.yml or action.yaml. The data in the metadata file defines the inputs, outputs and main entrypoint for your action.
type Action struct {
Name string `yaml:"name"`
Author string `yaml:"author"`
Description string `yaml:"description"`
Inputs map[string]Input `yaml:"inputs"`
Outputs map[string]Output `yaml:"outputs"`
Runs ActionRuns `yaml:"runs"`
Branding struct {
Color string `yaml:"color"`
Icon string `yaml:"icon"`
} `yaml:"branding"`
}
// Input parameters allow you to specify data that the action expects to use during runtime. GitHub stores input parameters as environment variables. Input ids with uppercase letters are converted to lowercase during runtime. We recommended using lowercase input ids.
type Input struct {
Description string `yaml:"description"`
Required bool `yaml:"required"`
Default string `yaml:"default"`
}
// Output parameters allow you to declare data that an action sets. Actions that run later in a workflow can use the output data set in previously run actions. For example, if you had an action that performed the addition of two inputs (x + y = z), the action could output the sum (z) for other actions to use as an input.
type Output struct {
Description string `yaml:"description"`
Value string `yaml:"value"`
}
// ReadAction reads an action from a reader
func ReadAction(in io.Reader) (*Action, error) {
a := new(Action)
err := yaml.NewDecoder(in).Decode(a)
if err != nil {
return nil, err
}
// set defaults
if a.Runs.PreIf == "" {
a.Runs.PreIf = "always()"
}
if a.Runs.PostIf == "" {
a.Runs.PostIf = "always()"
}
return a, nil
}
+63
View File
@@ -0,0 +1,63 @@
// Copyright 2026 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
import (
"strings"
"testing"
)
func TestReadActionDefaultsAndCaseInsensitiveUsing(t *testing.T) {
action, err := ReadAction(strings.NewReader(`
name: example
runs:
using: NoDe24
main: dist/index.js
`))
if err != nil {
t.Fatal(err)
}
if action.Runs.Using != ActionRunsUsingNode24 {
t.Fatalf("using = %q, want %q", action.Runs.Using, ActionRunsUsingNode24)
}
if action.Runs.PreIf != "always()" {
t.Fatalf("pre-if = %q, want always()", action.Runs.PreIf)
}
if action.Runs.PostIf != "always()" {
t.Fatalf("post-if = %q, want always()", action.Runs.PostIf)
}
}
func TestReadActionPreservesExplicitConditions(t *testing.T) {
action, err := ReadAction(strings.NewReader(`
runs:
using: composite
pre-if: success()
post-if: failure()
steps:
- run: echo hello
`))
if err != nil {
t.Fatal(err)
}
if action.Runs.PreIf != "success()" || action.Runs.PostIf != "failure()" {
t.Fatalf("conditions = %q/%q, want explicit values", action.Runs.PreIf, action.Runs.PostIf)
}
if !action.Runs.Using.IsComposite() || action.Runs.Using.IsDocker() || action.Runs.Using.IsNode() {
t.Fatalf("unexpected using predicates for %q", action.Runs.Using)
}
}
func TestReadActionRejectsUnknownUsing(t *testing.T) {
_, err := ReadAction(strings.NewReader(`
runs:
using: node99
`))
if err == nil {
t.Fatal("expected unknown runs.using to fail")
}
if !strings.Contains(err.Error(), "node99") {
t.Fatalf("error = %q, want invalid value", err)
}
}
+222
View File
@@ -0,0 +1,222 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2021 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
import (
"context"
"fmt"
"strings"
"gitea.com/gitea/runner/act/common"
"gitea.com/gitea/runner/act/common/git"
)
type GithubContext struct {
Event map[string]any `json:"event"`
EventPath string `json:"event_path"`
Workflow string `json:"workflow"`
RunID string `json:"run_id"`
RunNumber string `json:"run_number"`
Actor string `json:"actor"`
Repository string `json:"repository"`
EventName string `json:"event_name"`
Sha string `json:"sha"`
Ref string `json:"ref"`
RefName string `json:"ref_name"`
RefType string `json:"ref_type"`
HeadRef string `json:"head_ref"`
BaseRef string `json:"base_ref"`
Token string `json:"token"`
Workspace string `json:"workspace"`
Action string `json:"action"`
ActionPath string `json:"action_path"`
ActionRef string `json:"action_ref"`
ActionRepository string `json:"action_repository"`
Job string `json:"job"`
JobName string `json:"job_name"`
RepositoryOwner string `json:"repository_owner"`
RetentionDays string `json:"retention_days"`
RunnerPerflog string `json:"runner_perflog"`
RunnerTrackingID string `json:"runner_tracking_id"`
ServerURL string `json:"server_url"`
APIURL string `json:"api_url"`
GraphQLURL string `json:"graphql_url"`
// For Gitea
RunAttempt string `json:"run_attempt"`
}
func asString(v any) string {
if v == nil {
return ""
} else if s, ok := v.(string); ok {
return s
}
return ""
}
func nestedMapLookup(m map[string]any, ks ...string) (rval any) {
var ok bool
if len(ks) == 0 { // degenerate input
return nil
}
if rval, ok = m[ks[0]]; !ok {
return nil
} else if len(ks) == 1 { // we've reached the final key
return rval
} else if m, ok = rval.(map[string]any); !ok {
return nil
} else { // 1+ more keys
return nestedMapLookup(m, ks[1:]...)
}
}
func withDefaultBranch(ctx context.Context, b string, event map[string]any) map[string]any {
repoI, ok := event["repository"]
if !ok {
repoI = make(map[string]any)
}
repo, ok := repoI.(map[string]any)
if !ok {
common.Logger(ctx).Warnf("unable to set default branch to %v", b)
return event
}
// if the branch is already there return with no changes
if _, ok = repo["default_branch"]; ok {
return event
}
repo["default_branch"] = b
event["repository"] = repo
return event
}
var (
findGitRef = git.FindGitRef
findGitRevision = git.FindGitRevision
)
func (ghc *GithubContext) SetRef(ctx context.Context, defaultBranch, repoPath string) {
logger := common.Logger(ctx)
// https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads
switch ghc.EventName {
case "pull_request_target":
ghc.Ref = "refs/heads/" + ghc.BaseRef
case "pull_request", "pull_request_review", "pull_request_review_comment":
ghc.Ref = fmt.Sprintf("refs/pull/%.0f/merge", ghc.Event["number"])
case "deployment", "deployment_status":
ghc.Ref = asString(nestedMapLookup(ghc.Event, "deployment", "ref"))
case "release":
ghc.Ref = "refs/tags/" + asString(nestedMapLookup(ghc.Event, "release", "tag_name"))
case "push", "create", "workflow_dispatch":
ghc.Ref = asString(ghc.Event["ref"])
default:
defaultBranch := asString(nestedMapLookup(ghc.Event, "repository", "default_branch"))
if defaultBranch != "" {
ghc.Ref = "refs/heads/" + defaultBranch
}
}
if ghc.Ref == "" {
ref, err := findGitRef(ctx, repoPath)
if err != nil {
logger.Warningf("unable to get git ref: %v", err)
} else {
logger.Debugf("using github ref: %s", ref)
ghc.Ref = ref
}
// set the branch in the event data
if defaultBranch != "" {
ghc.Event = withDefaultBranch(ctx, defaultBranch, ghc.Event)
} else {
ghc.Event = withDefaultBranch(ctx, "master", ghc.Event)
}
if ghc.Ref == "" {
ghc.Ref = "refs/heads/" + asString(nestedMapLookup(ghc.Event, "repository", "default_branch"))
}
}
}
func (ghc *GithubContext) SetSha(ctx context.Context, repoPath string) {
logger := common.Logger(ctx)
// https://docs.github.com/en/actions/learn-github-actions/events-that-trigger-workflows
// https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads
switch ghc.EventName {
case "pull_request_target":
ghc.Sha = asString(nestedMapLookup(ghc.Event, "pull_request", "base", "sha"))
case "deployment", "deployment_status":
ghc.Sha = asString(nestedMapLookup(ghc.Event, "deployment", "sha"))
case "push", "create", "workflow_dispatch":
if deleted, ok := ghc.Event["deleted"].(bool); ok && !deleted {
ghc.Sha = asString(ghc.Event["after"])
}
}
if ghc.Sha == "" {
_, sha, err := findGitRevision(ctx, repoPath)
if err != nil {
logger.Warningf("unable to get git revision: %v", err)
} else {
ghc.Sha = sha
}
}
}
func (ghc *GithubContext) SetRepositoryAndOwner(ctx context.Context, githubInstance, remoteName, repoPath string) {
if ghc.Repository == "" {
repo, err := git.FindGithubRepo(ctx, repoPath, githubInstance, remoteName)
if err != nil {
common.Logger(ctx).Warningf("unable to get git repo (githubInstance: %v; remoteName: %v, repoPath: %v): %v", githubInstance, remoteName, repoPath, err)
return
}
ghc.Repository = repo
}
ghc.RepositoryOwner = strings.Split(ghc.Repository, "/")[0]
}
func (ghc *GithubContext) SetRefTypeAndName() {
var refType, refName string
// https://docs.github.com/en/actions/learn-github-actions/environment-variables
if strings.HasPrefix(ghc.Ref, "refs/tags/") {
refType = "tag"
refName = ghc.Ref[len("refs/tags/"):]
} else if strings.HasPrefix(ghc.Ref, "refs/heads/") {
refType = "branch"
refName = ghc.Ref[len("refs/heads/"):]
} else if strings.HasPrefix(ghc.Ref, "refs/pull/") {
refType = ""
refName = ghc.Ref[len("refs/pull/"):]
}
if ghc.RefType == "" {
ghc.RefType = refType
}
if ghc.RefName == "" {
ghc.RefName = refName
}
}
func (ghc *GithubContext) SetBaseAndHeadRef() {
if ghc.EventName == "pull_request" || ghc.EventName == "pull_request_target" {
if ghc.BaseRef == "" {
ghc.BaseRef = asString(nestedMapLookup(ghc.Event, "pull_request", "base", "ref"))
}
if ghc.HeadRef == "" {
ghc.HeadRef = asString(nestedMapLookup(ghc.Event, "pull_request", "head", "ref"))
}
}
}
@@ -2,14 +2,13 @@
// Copyright 2022 The nektos/act Authors. All rights reserved. // Copyright 2022 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT // SPDX-License-Identifier: MIT
package ghcontext package model
import ( import (
"context" "context"
"errors" "errors"
"testing" "testing"
"gitea.dev/actionslib/pkg/model"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@@ -98,13 +97,13 @@ func TestSetRef(t *testing.T) {
for _, table := range tables { for _, table := range tables {
t.Run(table.eventName, func(t *testing.T) { t.Run(table.eventName, func(t *testing.T) {
ghc := &model.GithubContext{ ghc := &GithubContext{
EventName: table.eventName, EventName: table.eventName,
BaseRef: "master", BaseRef: "master",
Event: table.event, Event: table.event,
} }
SetRef(context.Background(), ghc, "main", "/some/dir") ghc.SetRef(context.Background(), "main", "/some/dir")
ghc.SetRefTypeAndName() ghc.SetRefTypeAndName()
assert.Equal(t, table.ref, ghc.Ref) assert.Equal(t, table.ref, ghc.Ref)
@@ -117,12 +116,12 @@ func TestSetRef(t *testing.T) {
return "", errors.New("no default branch") return "", errors.New("no default branch")
} }
ghc := &model.GithubContext{ ghc := &GithubContext{
EventName: "no-default-branch", EventName: "no-default-branch",
Event: map[string]any{}, Event: map[string]any{},
} }
SetRef(context.Background(), ghc, "", "/some/dir") ghc.SetRef(context.Background(), "", "/some/dir")
assert.Equal(t, "refs/heads/master", ghc.Ref) assert.Equal(t, "refs/heads/master", ghc.Ref)
}) })
@@ -203,13 +202,13 @@ func TestSetSha(t *testing.T) {
for _, table := range tables { for _, table := range tables {
t.Run(table.eventName, func(t *testing.T) { t.Run(table.eventName, func(t *testing.T) {
ghc := &model.GithubContext{ ghc := &GithubContext{
EventName: table.eventName, EventName: table.eventName,
BaseRef: "master", BaseRef: "master",
Event: table.event, Event: table.event,
} }
SetSha(context.Background(), ghc, "/some/dir") ghc.SetSha(context.Background(), "/some/dir")
assert.Equal(t, table.sha, ghc.Sha) assert.Equal(t, table.sha, ghc.Sha)
}) })
+16
View File
@@ -0,0 +1,16 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Copyright 2021 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
type JobContext struct {
Status string `json:"status"`
Container struct {
ID string `json:"id"`
Network string `json:"network"`
} `json:"container"`
Services map[string]struct {
ID string `json:"id"`
} `json:"services"`
}
+410
View File
@@ -0,0 +1,410 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2020 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
import (
"errors"
"fmt"
"io"
"io/fs"
"math"
"os"
"path/filepath"
"regexp"
"slices"
log "github.com/sirupsen/logrus"
)
// WorkflowPlanner contains methods for creating plans
type WorkflowPlanner interface {
PlanEvent(eventName string) (*Plan, error)
PlanJob(jobName string) (*Plan, error)
PlanAll() (*Plan, error)
GetEvents() []string
}
// Plan contains a list of stages to run in series
type Plan struct {
Stages []*Stage
}
// Stage contains a list of runs to execute in parallel
type Stage struct {
Runs []*Run
}
// Run represents a job from a workflow that needs to be run
type Run struct {
Workflow *Workflow
JobID string
}
func (r *Run) String() string {
jobName := r.Job().Name
if jobName == "" {
jobName = r.JobID
}
return jobName
}
// Job returns the job for this Run
func (r *Run) Job() *Job {
return r.Workflow.GetJob(r.JobID)
}
type WorkflowFiles struct {
workflowDirEntry os.DirEntry
dirPath string
}
// NewWorkflowPlanner will load a specific workflow, all workflows from a directory or all workflows from a directory and its subdirectories
func NewWorkflowPlanner(path string, noWorkflowRecurse bool) (WorkflowPlanner, error) {
path, err := filepath.Abs(path)
if err != nil {
return nil, err
}
fi, err := os.Stat(path)
if err != nil {
return nil, err
}
var workflows []WorkflowFiles
if fi.IsDir() {
log.Debugf("Loading workflows from '%s'", path)
if noWorkflowRecurse {
files, err := os.ReadDir(path)
if err != nil {
return nil, err
}
for _, v := range files {
workflows = append(workflows, WorkflowFiles{
dirPath: path,
workflowDirEntry: v,
})
}
} else {
log.Debug("Loading workflows recursively")
if err := filepath.Walk(path,
func(p string, f os.FileInfo, err error) error {
if err != nil {
return err
}
if !f.IsDir() {
log.Debugf("Found workflow '%s' in '%s'", f.Name(), p)
workflows = append(workflows, WorkflowFiles{
dirPath: filepath.Dir(p),
workflowDirEntry: fs.FileInfoToDirEntry(f),
})
}
return nil
}); err != nil {
return nil, err
}
}
} else {
log.Debugf("Loading workflow '%s'", path)
dirname := filepath.Dir(path)
workflows = append(workflows, WorkflowFiles{
dirPath: dirname,
workflowDirEntry: fs.FileInfoToDirEntry(fi),
})
}
wp := new(workflowPlanner)
for _, wf := range workflows {
ext := filepath.Ext(wf.workflowDirEntry.Name())
if ext == ".yml" || ext == ".yaml" {
f, err := os.Open(filepath.Join(wf.dirPath, wf.workflowDirEntry.Name()))
if err != nil {
return nil, err
}
log.Debugf("Reading workflow '%s'", f.Name())
workflow, err := ReadWorkflow(f)
if err != nil {
_ = f.Close()
if err == io.EOF {
return nil, fmt.Errorf("unable to read workflow '%s': file is empty: %w", wf.workflowDirEntry.Name(), err)
}
return nil, fmt.Errorf("workflow is not valid. '%s': %w", wf.workflowDirEntry.Name(), err)
}
_, err = f.Seek(0, 0)
if err != nil {
_ = f.Close()
return nil, fmt.Errorf("error occurring when resetting io pointer in '%s': %w", wf.workflowDirEntry.Name(), err)
}
workflow.File = wf.workflowDirEntry.Name()
if workflow.Name == "" {
workflow.Name = wf.workflowDirEntry.Name()
}
err = validateJobName(workflow)
if err != nil {
_ = f.Close()
return nil, err
}
wp.workflows = append(wp.workflows, workflow)
_ = f.Close()
}
}
return wp, nil
}
// CombineWorkflowPlanner combines workflows to a WorkflowPlanner
func CombineWorkflowPlanner(workflows ...*Workflow) WorkflowPlanner {
return &workflowPlanner{
workflows: workflows,
}
}
func NewSingleWorkflowPlanner(name string, f io.Reader) (WorkflowPlanner, error) {
wp := new(workflowPlanner)
log.Debugf("Reading workflow %s", name)
workflow, err := ReadWorkflow(f)
if err != nil {
if err == io.EOF {
return nil, fmt.Errorf("unable to read workflow '%s': file is empty: %w", name, err)
}
return nil, fmt.Errorf("workflow is not valid. '%s': %w", name, err)
}
workflow.File = name
if workflow.Name == "" {
workflow.Name = name
}
err = validateJobName(workflow)
if err != nil {
return nil, err
}
wp.workflows = append(wp.workflows, workflow)
return wp, nil
}
func validateJobName(workflow *Workflow) error {
jobNameRegex := regexp.MustCompile(`^([[:alpha:]_][[:alnum:]_\-]*)$`)
for k := range workflow.Jobs {
if ok := jobNameRegex.MatchString(k); !ok {
return fmt.Errorf("workflow is not valid. '%s': Job name '%s' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", workflow.Name, k)
}
}
return nil
}
type workflowPlanner struct {
workflows []*Workflow
}
// PlanEvent builds a new list of runs to execute in parallel for an event name
func (wp *workflowPlanner) PlanEvent(eventName string) (*Plan, error) {
plan := new(Plan)
if len(wp.workflows) == 0 {
log.Debug("no workflows found by planner")
return plan, nil
}
var lastErr error
for _, w := range wp.workflows {
events := w.On()
if len(events) == 0 {
log.Debugf("no events found for workflow: %s", w.File)
continue
}
for _, e := range events {
if e == eventName {
stages, err := createStages(w, w.GetJobIDs()...)
if err != nil {
log.Warn(err)
lastErr = err
} else {
plan.mergeStages(stages)
}
}
}
}
return plan, lastErr
}
// PlanJob builds a new run to execute in parallel for a job name
func (wp *workflowPlanner) PlanJob(jobName string) (*Plan, error) {
plan := new(Plan)
if len(wp.workflows) == 0 {
log.Debugf("no jobs found for workflow: %s", jobName)
}
var lastErr error
for _, w := range wp.workflows {
stages, err := createStages(w, jobName)
if err != nil {
log.Warn(err)
lastErr = err
} else {
plan.mergeStages(stages)
}
}
return plan, lastErr
}
// PlanAll builds a new run to execute in parallel all
func (wp *workflowPlanner) PlanAll() (*Plan, error) {
plan := new(Plan)
if len(wp.workflows) == 0 {
log.Debug("no workflows found by planner")
return plan, nil
}
var lastErr error
for _, w := range wp.workflows {
stages, err := createStages(w, w.GetJobIDs()...)
if err != nil {
log.Warn(err)
lastErr = err
} else {
plan.mergeStages(stages)
}
}
return plan, lastErr
}
// GetEvents gets all the events in the workflows file
func (wp *workflowPlanner) GetEvents() []string {
events := make([]string, 0)
for _, w := range wp.workflows {
found := false
for _, e := range events {
if slices.Contains(w.On(), e) {
found = true
}
if found {
break
}
}
if !found {
events = append(events, w.On()...)
}
}
// sort the list based on depth of dependencies
slices.Sort(events)
return events
}
// MaxRunNameLen determines the max name length of all jobs
func (p *Plan) MaxRunNameLen() int {
maxRunNameLen := 0
for _, stage := range p.Stages {
for _, run := range stage.Runs {
runNameLen := len(run.String())
if runNameLen > maxRunNameLen {
maxRunNameLen = runNameLen
}
}
}
return maxRunNameLen
}
// GetJobIDs will get all the job names in the stage
func (s *Stage) GetJobIDs() []string {
names := make([]string, 0)
for _, r := range s.Runs {
names = append(names, r.JobID)
}
return names
}
// Merge stages with existing stages in plan
func (p *Plan) mergeStages(stages []*Stage) {
newStages := make([]*Stage, int(math.Max(float64(len(p.Stages)), float64(len(stages)))))
for i := range newStages {
newStages[i] = new(Stage)
if i >= len(p.Stages) {
newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...)
} else if i >= len(stages) {
newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...)
} else {
newStages[i].Runs = append(newStages[i].Runs, p.Stages[i].Runs...)
newStages[i].Runs = append(newStages[i].Runs, stages[i].Runs...)
}
}
p.Stages = newStages
}
func createStages(w *Workflow, jobIDs ...string) ([]*Stage, error) {
// first, build a list of all the necessary jobs to run, and their dependencies
jobDependencies := make(map[string][]string)
for len(jobIDs) > 0 {
newJobIDs := make([]string, 0)
for _, jID := range jobIDs {
// make sure we haven't visited this job yet
if _, ok := jobDependencies[jID]; !ok {
if job := w.GetJob(jID); job != nil {
jobDependencies[jID] = job.Needs()
newJobIDs = append(newJobIDs, job.Needs()...)
}
}
}
jobIDs = newJobIDs
}
// next, build an execution graph
stages := make([]*Stage, 0)
for len(jobDependencies) > 0 {
stage := new(Stage)
for jID, jDeps := range jobDependencies {
// make sure all deps are in the graph already
if listInStages(jDeps, stages...) {
stage.Runs = append(stage.Runs, &Run{
Workflow: w,
JobID: jID,
})
delete(jobDependencies, jID)
}
}
if len(stage.Runs) == 0 {
return nil, fmt.Errorf("unable to build dependency graph for %s (%s)", w.Name, w.File)
}
stages = append(stages, stage)
}
if len(stages) == 0 {
return nil, errors.New("Could not find any stages to run. View the valid jobs with `act --list`. Use `act --help` to find how to filter by Job ID/Workflow/Event Name")
}
return stages, nil
}
// return true iff all strings in srcList exist in at least one of the stages
func listInStages(srcList []string, stages ...*Stage) bool {
for _, src := range srcList {
found := false
for _, stage := range stages {
for _, search := range stage.GetJobIDs() {
if src == search {
found = true
}
}
}
if !found {
return false
}
}
return true
}
+199
View File
@@ -0,0 +1,199 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// Copyright 2021 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
import (
"path/filepath"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type WorkflowPlanTest struct {
workflowPath string
errorMessage string
noWorkflowRecurse bool
}
func TestPlanner(t *testing.T) {
log.SetLevel(log.DebugLevel)
tables := []WorkflowPlanTest{
{"invalid-job-name/invalid-1.yml", "workflow is not valid. 'invalid-job-name-1': Job name 'invalid-JOB-Name-v1.2.3-docker_hub' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", false},
{"invalid-job-name/invalid-2.yml", "workflow is not valid. 'invalid-job-name-2': Job name '1234invalid-JOB-Name-v123-docker_hub' is invalid. Names must start with a letter or '_' and contain only alphanumeric characters, '-', or '_'", false},
{"invalid-job-name/valid-1.yml", "", false},
{"invalid-job-name/valid-2.yml", "", false},
{"empty-workflow", "unable to read workflow 'push.yml': file is empty: EOF", false},
{"nested", "unable to read workflow 'fail.yml': file is empty: EOF", false},
{"nested", "", true},
}
workdir, err := filepath.Abs("testdata")
assert.NoError(t, err, workdir) //nolint:testifylint // pre-existing issue from nektos/act
for _, table := range tables {
fullWorkflowPath := filepath.Join(workdir, table.workflowPath)
_, err = NewWorkflowPlanner(fullWorkflowPath, table.noWorkflowRecurse)
if table.errorMessage == "" {
assert.NoError(t, err, "WorkflowPlanner should exit without any error")
} else {
assert.EqualError(t, err, table.errorMessage)
}
}
}
func TestWorkflow(t *testing.T) {
log.SetLevel(log.DebugLevel)
workflow := Workflow{
Jobs: map[string]*Job{
"valid_job": {
Name: "valid_job",
},
},
}
// Check that an invalid job id returns error
result, err := createStages(&workflow, "invalid_job_id")
assert.Error(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.Nil(t, result)
// Check that an valid job id returns non-error
result, err = createStages(&workflow, "valid_job")
assert.NoError(t, err) //nolint:testifylint // pre-existing issue from nektos/act
assert.NotNil(t, result)
}
func TestNewSingleWorkflowPlannerAndPlanMethods(t *testing.T) {
planner, err := NewSingleWorkflowPlanner("ci.yml", strings.NewReader(`
name: CI
on: [push, pull_request]
jobs:
build:
name: Build project
runs-on: ubuntu-latest
steps:
- run: make build
test:
needs: build
runs-on: ubuntu-latest
steps:
- run: make test
`))
require.NoError(t, err)
assert.Equal(t, []string{"pull_request", "push"}, planner.GetEvents())
eventPlan, err := planner.PlanEvent("push")
require.NoError(t, err)
require.Len(t, eventPlan.Stages, 2)
assert.Equal(t, []string{"build"}, eventPlan.Stages[0].GetJobIDs())
assert.Equal(t, []string{"test"}, eventPlan.Stages[1].GetJobIDs())
assert.Equal(t, len("Build project"), eventPlan.MaxRunNameLen())
assert.Equal(t, "Build project", eventPlan.Stages[0].Runs[0].String())
assert.Equal(t, "build", eventPlan.Stages[0].Runs[0].JobID)
assert.NotNil(t, eventPlan.Stages[0].Runs[0].Job())
jobPlan, err := planner.PlanJob("test")
require.NoError(t, err)
require.Len(t, jobPlan.Stages, 2)
assert.Equal(t, []string{"build"}, jobPlan.Stages[0].GetJobIDs())
assert.Equal(t, []string{"test"}, jobPlan.Stages[1].GetJobIDs())
allPlan, err := planner.PlanAll()
require.NoError(t, err)
require.Len(t, allPlan.Stages, 2)
assert.Equal(t, []string{"build"}, allPlan.Stages[0].GetJobIDs())
assert.Equal(t, []string{"test"}, allPlan.Stages[1].GetJobIDs())
}
func TestCombineWorkflowPlannerMergesWorkflowStages(t *testing.T) {
first := mustReadWorkflow(t, `
name: First
on: push
jobs:
build:
runs-on: ubuntu-latest
steps:
- run: make build
`)
second := mustReadWorkflow(t, `
name: Second
on: push
jobs:
lint:
runs-on: ubuntu-latest
steps:
- run: make lint
test:
needs: lint
runs-on: ubuntu-latest
steps:
- run: make test
`)
planner := CombineWorkflowPlanner(first, second)
plan, err := planner.PlanEvent("push")
require.NoError(t, err)
require.Len(t, plan.Stages, 2)
assert.ElementsMatch(t, []string{"build", "lint"}, plan.Stages[0].GetJobIDs())
assert.Equal(t, []string{"test"}, plan.Stages[1].GetJobIDs())
empty, err := planner.PlanEvent("schedule")
require.NoError(t, err)
assert.Empty(t, empty.Stages)
}
func TestPlannerErrorsForMissingAndCyclicJobs(t *testing.T) {
workflow := mustReadWorkflow(t, `
name: Cyclic
on: push
jobs:
a:
needs: b
runs-on: ubuntu-latest
steps:
- run: echo a
b:
needs: a
runs-on: ubuntu-latest
steps:
- run: echo b
`)
planner := CombineWorkflowPlanner(workflow)
plan, err := planner.PlanJob("missing")
require.Error(t, err)
assert.Empty(t, plan.Stages)
assert.Contains(t, err.Error(), "Could not find any stages")
plan, err = planner.PlanEvent("push")
require.Error(t, err)
assert.Empty(t, plan.Stages)
assert.Contains(t, err.Error(), "unable to build dependency graph")
}
func TestNewSingleWorkflowPlannerErrors(t *testing.T) {
_, err := NewSingleWorkflowPlanner("empty.yml", strings.NewReader(""))
require.Error(t, err)
assert.Contains(t, err.Error(), "file is empty")
_, err = NewSingleWorkflowPlanner("invalid.yml", strings.NewReader("jobs: ["))
require.Error(t, err)
assert.Contains(t, err.Error(), "workflow is not valid")
}
func mustReadWorkflow(t *testing.T, content string) *Workflow {
t.Helper()
workflow, err := ReadWorkflow(strings.NewReader(content))
require.NoError(t, err)
if workflow.Name == "" {
workflow.Name = "workflow"
}
return workflow
}
+49
View File
@@ -0,0 +1,49 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// Copyright 2021 The nektos/act Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package model
import "fmt"
type stepStatus int
const (
StepStatusSuccess stepStatus = iota
StepStatusFailure
StepStatusSkipped
)
var stepStatusStrings = [...]string{
"success",
"failure",
"skipped",
}
func (s stepStatus) MarshalText() ([]byte, error) {
return []byte(s.String()), nil
}
func (s *stepStatus) UnmarshalText(b []byte) error {
str := string(b)
for i, name := range stepStatusStrings {
if name == str {
*s = stepStatus(i)
return nil
}
}
return fmt.Errorf("invalid step status %q", str)
}
func (s stepStatus) String() string {
if int(s) >= len(stepStatusStrings) {
return ""
}
return stepStatusStrings[s]
}
type StepResult struct {
Outputs map[string]string `json:"outputs"`
Conclusion stepStatus `json:"conclusion"`
Outcome stepStatus `json:"outcome"`
}

Some files were not shown because too many files have changed in this diff Show More