feat!: add cache service v2, add toolkit patches (#1110)

Serves `github.actions.results.api.v1.CacheService` next to the v1 cache API, sharing its store, plus the subset of the Azure blob protocol the toolkit uploads with. On by default via `cache.v2`, and works with `external_server`.

Clients reach it through two edits in the action's own bundle: the GHES check is opened, and the cache service URL is taken from `ACTIONS_CACHE_URL`.

The same GHES check is what makes the stock `actions/upload-artifact` and `download-artifact` abort on Gitea. Opening it makes them work without the `gitea-upload-artifact` fork, from `upload-artifact@v4.4.0` on.

Verified against 118 real bundles, every major version of 16 actions: 92 patched, the rest deliberately left alone, and every patched bundle checked with `node --check`. Also end to end against pinned `actions/cache@v6.1.0` with an unreachable results URL, so only the patch can make the cache work.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: bircni <me@silverwind.io>
Reviewed-on: https://gitea.com/gitea/runner/pulls/1110
Reviewed-by: silverwind <2021+silverwind@noreply.gitea.com>

act/artifactcache/storage.go

@@ -5,12 +5,15 @@
 package artifactcache
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 )
 
 type Storage struct {
@@ -37,7 +40,10 @@ func (s *Storage) Exist(id uint64) (bool, error) {
 }
 
 func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
-	name := s.tempName(id, offset)
+	return s.writeFile(s.tempName(id, offset), reader)
+}
+
+func (s *Storage) writeFile(name string, reader io.Reader) error {
 	if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
 		return err
 	}
@@ -51,6 +57,26 @@ func (s *Storage) Write(id uint64, offset int64, reader io.Reader) error {
 	return err
 }
 
+func (s *Storage) WriteBlock(id uint64, blockID string, reader io.Reader) error {
+	return s.writeFile(s.blockName(id, blockID), reader)
+}
+
+// OrderBlocks renames the staged blocks into the order the block list gives. A block the list
+// does not name keeps its staged name, which is how Commit leaves it out, as Azure drops it. One
+// rename pass is safe because a staged name always carries blockFilePrefix and a target name
+// never does, so no rename can collide with a block not yet moved.
+func (s *Storage) OrderBlocks(id uint64, blockIDs []string) error {
+	for i, blockID := range blockIDs {
+		if err := os.Rename(s.blockName(id, blockID), s.tempName(id, int64(i))); err != nil {
+			if os.IsNotExist(err) {
+				return fmt.Errorf("block %q of cache %d was never uploaded: %w", blockID, id, err)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *Storage) Commit(id uint64, size int64) (int64, error) {
 	defer func() {
 		_ = os.RemoveAll(s.tempDir(id))
@@ -65,6 +91,31 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
 	if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
 		return 0, err
 	}
+	written, err := assemble(name, tempNames)
+	if err != nil {
+		return 0, err
+	}
+	// If size is less than 0, it means the size is unknown.
+	// We can't check the size of the file, just skip the check.
+	// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
+	if size >= 0 && written != size {
+		_ = os.Remove(name)
+		return 0, fmt.Errorf("broken file: %v != %v", written, size)
+	}
+	return written, nil
+}
+
+// assemble concatenates the uploaded parts into name. A single part, which is what the v2 API
+// produces below the client's block threshold, is already the whole archive and is moved.
+func assemble(name string, tempNames []string) (int64, error) {
+	if len(tempNames) == 1 {
+		info, err := os.Stat(tempNames[0])
+		if err != nil {
+			return 0, err
+		}
+		return info.Size(), os.Rename(tempNames[0], name)
+	}
+
 	file, err := os.Create(name)
 	if err != nil {
 		return 0, err
@@ -84,16 +135,6 @@ func (s *Storage) Commit(id uint64, size int64) (int64, error) {
 		}
 		written += n
 	}
-
-	// If size is less than 0, it means the size is unknown.
-	// We can't check the size of the file, just skip the check.
-	// It happens when the request comes from old versions of actions, like `actions/cache@v2`.
-	if size >= 0 && written != size {
-		_ = file.Close()
-		_ = os.Remove(name)
-		return 0, fmt.Errorf("broken file: %v != %v", written, size)
-	}
-
 	return written, nil
 }
 
@@ -119,6 +160,17 @@ func (s *Storage) tempName(id uint64, offset int64) string {
 	return filepath.Join(s.tempDir(id), fmt.Sprintf("%016x", offset))
 }
 
+// blockFilePrefix marks a staged, not yet ordered block, so that tempNames can keep it out of
+// Commit's name-ordered concatenation.
+const blockFilePrefix = "block-"
+
+func (s *Storage) blockName(id uint64, blockID string) string {
+	// The block id is client-chosen (base64), so it is hashed rather than trusted as a
+	// path element.
+	sum := sha256.Sum256([]byte(blockID))
+	return filepath.Join(s.tempDir(id), blockFilePrefix+hex.EncodeToString(sum[:]))
+}
+
 func (s *Storage) tempNames(id uint64) ([]string, error) {
 	dir := s.tempDir(id)
 	files, err := os.ReadDir(dir)
@@ -127,7 +179,7 @@ func (s *Storage) tempNames(id uint64) ([]string, error) {
 	}
 	var names []string
 	for _, v := range files {
-		if !v.IsDir() {
+		if !v.IsDir() && !strings.HasPrefix(v.Name(), blockFilePrefix) {
 			names = append(names, filepath.Join(dir, v.Name()))
 		}
 	}