Support cache (#25)

See [Caching dependencies to speed up workflows](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows). Reviewed-on: https://gitea.com/gitea/act_runner/pulls/25 Reviewed-by: Lunny Xiao <xiaolunwen@gmail.com> Reviewed-by: Zettat123 <zettat123@noreply.gitea.io> Co-authored-by: Jason Song <i@wolfogre.com> Co-committed-by: Jason Song <i@wolfogre.com>
2023-02-28 23:39:30 +08:00
parent 552dbcdda9
commit 436b441cad
11 changed files with 1091 additions and 6 deletions
--- a/artifactcache/README.md
+++ b/artifactcache/README.md
@@ -0,0 +1,7 @@
+Inspired by:
+https://github.com/sp-ricard-valverde/github-act-cache-server
+
+TODO:
+- Authorization
+- [Restrictions for accessing a cache](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache)
+- [Force deleting cache entries](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries)
--- a/artifactcache/handler.go
+++ b/artifactcache/handler.go
--- a/artifactcache/model.go
+++ b/artifactcache/model.go
@@ -0,0 +1,30 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package artifactcache
+
+import (
+	"fmt"
+	"net/http"
+)
+
+type Cache struct {
+	ID        int64  `xorm:"id pk autoincr" json:"-"`
+	Key       string `xorm:"TEXT index unique(key_version)" json:"key"`
+	Version   string `xorm:"TEXT unique(key_version)" json:"version"`
+	Size      int64  `json:"cacheSize"`
+	Complete  bool   `xorm:"index(complete_used_at)" json:"-"`
+	UsedAt    int64  `xorm:"index(complete_used_at) updated" json:"-"`
+	CreatedAt int64  `xorm:"index created" json:"-"`
+}
+
+// Bind implements render.Binder
+func (c *Cache) Bind(_ *http.Request) error {
+	if c.Key == "" {
+		return fmt.Errorf("missing key")
+	}
+	if c.Version == "" {
+		return fmt.Errorf("missing version")
+	}
+	return nil
+}
--- a/artifactcache/storage.go
+++ b/artifactcache/storage.go
@@ -0,0 +1,129 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package artifactcache
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+)
+
+type Storage struct {
+	rootDir string
+}
+
+func NewStorage(rootDir string) (*Storage, error) {
+	if err := os.MkdirAll(rootDir, 0o755); err != nil {
+		return nil, err
+	}
+	return &Storage{
+		rootDir: rootDir,
+	}, nil
+}
+
+func (s *Storage) Exist(id int64) (bool, error) {
+	name := s.filename(id)
+	if _, err := os.Stat(name); os.IsNotExist(err) {
+		return false, nil
+	} else if err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *Storage) Write(id int64, offset int64, reader io.Reader) error {
+	name := s.tempName(id, offset)
+	if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
+		return err
+	}
+	file, err := os.Create(name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, err = io.Copy(file, reader)
+	return err
+}
+
+func (s *Storage) Commit(id int64, size int64) error {
+	defer func() {
+		_ = os.RemoveAll(s.tempDir(id))
+	}()
+
+	name := s.filename(id)
+	tempNames, err := s.tempNames(id)
+	if err != nil {
+		return err
+	}
+
+	if err := os.MkdirAll(filepath.Dir(name), 0o755); err != nil {
+		return err
+	}
+	file, err := os.Create(name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	var written int64
+	for _, v := range tempNames {
+		f, err := os.Open(v)
+		if err != nil {
+			return err
+		}
+		n, err := io.Copy(file, f)
+		_ = f.Close()
+		if err != nil {
+			return err
+		}
+		written += n
+	}
+
+	if written != size {
+		_ = file.Close()
+		_ = os.Remove(name)
+		return fmt.Errorf("broken file: %v != %v", written, size)
+	}
+	return nil
+}
+
+func (s *Storage) Serve(w http.ResponseWriter, r *http.Request, id int64) {
+	name := s.filename(id)
+	http.ServeFile(w, r, name)
+}
+
+func (s *Storage) Remove(id int64) {
+	_ = os.Remove(s.filename(id))
+	_ = os.RemoveAll(s.tempDir(id))
+}
+
+func (s *Storage) filename(id int64) string {
+	return filepath.Join(s.rootDir, fmt.Sprintf("%02x", id%0xff), fmt.Sprint(id))
+}
+
+func (s *Storage) tempDir(id int64) string {
+	return filepath.Join(s.rootDir, "tmp", fmt.Sprint(id))
+}
+
+func (s *Storage) tempName(id, offset int64) string {
+	return filepath.Join(s.tempDir(id), fmt.Sprintf("%016x", offset))
+}
+
+func (s *Storage) tempNames(id int64) ([]string, error) {
+	dir := s.tempDir(id)
+	files, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, v := range files {
+		if !v.IsDir() {
+			names = append(names, filepath.Join(dir, v.Name()))
+		}
+	}
+	return names, nil
+}
--- a/artifactcache/util.go
+++ b/artifactcache/util.go
@@ -0,0 +1,72 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package artifactcache
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/go-chi/render"
+	"xorm.io/xorm"
+)
+
+func responseJson(w http.ResponseWriter, r *http.Request, code int, v ...any) {
+	render.Status(r, code)
+	if len(v) == 0 || v[0] == nil {
+		render.JSON(w, r, struct{}{})
+	} else if err, ok := v[0].(error); ok {
+		logger.Errorf("%v %v: %v", r.Method, r.RequestURI, err)
+		render.JSON(w, r, map[string]any{
+			"error": err.Error(),
+		})
+	} else {
+		render.JSON(w, r, v[0])
+	}
+}
+
+func parseContentRange(s string) (int64, int64, error) {
+	// support the format like "bytes 11-22/*" only
+	s, _, _ = strings.Cut(strings.TrimPrefix(s, "bytes "), "/")
+	s1, s2, _ := strings.Cut(s, "-")
+
+	start, err := strconv.ParseInt(s1, 10, 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("parse %q: %w", s, err)
+	}
+	stop, err := strconv.ParseInt(s2, 10, 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("parse %q: %w", s, err)
+	}
+	return start, stop, nil
+}
+
+// engine is a wrapper of *xorm.Engine, with a lock.
+// To avoid racing of sqlite, we don't careperformance here.
+type engine struct {
+	e *xorm.Engine
+	m sync.Mutex
+}
+
+func (e *engine) Exec(f func(*xorm.Session) error) error {
+	e.m.Lock()
+	defer e.m.Unlock()
+
+	sess := e.e.NewSession()
+	defer sess.Close()
+
+	return f(sess)
+}
+
+func (e *engine) ExecBool(f func(*xorm.Session) (bool, error)) (bool, error) {
+	e.m.Lock()
+	defer e.m.Unlock()
+
+	sess := e.e.NewSession()
+	defer sess.Close()
+
+	return f(sess)
+}
--- a/build.go
+++ b/build.go
@@ -0,0 +1,11 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build vendor
+
+package main
+
+import (
+	// for vet
+	_ "code.gitea.io/gitea-vet"
+)
--- a/cmd/daemon.go
+++ b/cmd/daemon.go
@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"

+	"gitea.com/gitea/act_runner/artifactcache"
 	"gitea.com/gitea/act_runner/client"
 	"gitea.com/gitea/act_runner/config"
 	"gitea.com/gitea/act_runner/engine"
@@ -52,6 +53,12 @@ func runDaemon(ctx context.Context, envFile string) func(cmd *cobra.Command, arg
 			}
 		}

+		handler, err := artifactcache.NewHandler()
+		if err != nil {
+			return err
+		}
+		log.Infof("cache handler listens on: %v", handler.ExternalURL())
+
 		var g errgroup.Group

 		cli := client.New(
@@ -67,6 +74,7 @@ func runDaemon(ctx context.Context, envFile string) func(cmd *cobra.Command, arg
 			ForgeInstance: cfg.Client.Address,
 			Environ:       cfg.Runner.Environ,
 			Labels:        cfg.Runner.Labels,
+			CacheHandler:  handler,
 		}

 		poller := poller.New(
--- a/go.mod
+++ b/go.mod
@@ -4,9 +4,12 @@ go 1.18

 require (
 	code.gitea.io/actions-proto-go v0.2.0
+	code.gitea.io/gitea-vet v0.2.3-0.20230113022436-2b1561217fa5
 	github.com/avast/retry-go/v4 v4.3.1
 	github.com/bufbuild/connect-go v1.3.1
 	github.com/docker/docker v20.10.21+incompatible
+	github.com/go-chi/chi/v5 v5.0.8
+	github.com/go-chi/render v1.0.2
 	github.com/joho/godotenv v1.4.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/mattn/go-isatty v0.0.16
@@ -15,15 +18,18 @@ require (
 	github.com/spf13/cobra v1.6.1
 	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
 	google.golang.org/protobuf v1.28.1
+	modernc.org/sqlite v1.14.2
+	xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978
+	xorm.io/xorm v1.3.2
 )

 require (
-	code.gitea.io/gitea-vet v0.2.3-0.20230113022436-2b1561217fa5 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/Microsoft/hcsshim v0.9.3 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20220404123522-616f957b79ad // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
+	github.com/ajg/form v1.5.1 // indirect
 	github.com/containerd/cgroups v1.0.3 // indirect
 	github.com/containerd/containerd v1.6.6 // indirect
 	github.com/creack/pty v1.1.18 // indirect
@@ -38,12 +44,16 @@ require (
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/goccy/go-json v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/google/uuid v1.3.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
@@ -54,29 +64,44 @@ require (
 	github.com/moby/buildkit v0.10.6 // indirect
 	github.com/moby/sys/mount v0.3.1 // indirect
 	github.com/moby/sys/mountinfo v0.6.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect
 	github.com/opencontainers/runc v1.1.2 // indirect
 	github.com/opencontainers/selinux v1.10.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/rhysd/actionlint v1.6.22 // indirect
 	github.com/rivo/uniseg v0.3.4 // indirect
 	github.com/robfig/cron v1.2.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 // indirect
+	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c // indirect
 	golang.org/x/sys v0.0.0-20220818161305-2296e01440c6 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/tools v0.1.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	lukechampine.com/uint128 v1.1.1 // indirect
+	modernc.org/cc/v3 v3.35.18 // indirect
+	modernc.org/ccgo/v3 v3.12.82 // indirect
+	modernc.org/libc v1.11.87 // indirect
+	modernc.org/mathutil v1.4.1 // indirect
+	modernc.org/memory v1.0.5 // indirect
+	modernc.org/opt v0.1.1 // indirect
+	modernc.org/strutil v1.1.1 // indirect
+	modernc.org/token v1.0.0 // indirect
 )

 replace github.com/nektos/act => gitea.com/gitea/act v0.234.3-0.20230224062034-1252e551b867
--- a/go.sum
+++ b/go.sum
--- a/runtime/runtime.go
+++ b/runtime/runtime.go
@@ -8,6 +8,8 @@ import (
 	"strings"

 	runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
+
+	"gitea.com/gitea/act_runner/artifactcache"
 	"gitea.com/gitea/act_runner/client"
 )

@@ -18,11 +20,17 @@ type Runner struct {
 	Environ       map[string]string
 	Client        client.Client
 	Labels        []string
+	CacheHandler  *artifactcache.Handler
 }

 // Run runs the pipeline stage.
 func (s *Runner) Run(ctx context.Context, task *runnerv1.Task) error {
-	return NewTask(s.ForgeInstance, task.Id, s.Client, s.Environ, s.platformPicker).Run(ctx, task)
+	env := map[string]string{}
+	for k, v := range s.Environ {
+		env[k] = v
+	}
+	env["ACTIONS_CACHE_URL"] = s.CacheHandler.ExternalURL() + "/"
+	return NewTask(s.ForgeInstance, task.Id, s.Client, env, s.platformPicker).Run(ctx, task)
 }

 func (s *Runner) platformPicker(labels []string) string {
--- a/runtime/task.go
+++ b/runtime/task.go
@@ -14,13 +14,13 @@ import (
 	"time"

 	runnerv1 "code.gitea.io/actions-proto-go/runner/v1"
-	"gitea.com/gitea/act_runner/client"
-
 	"github.com/nektos/act/pkg/artifacts"
 	"github.com/nektos/act/pkg/common"
 	"github.com/nektos/act/pkg/model"
 	"github.com/nektos/act/pkg/runner"
 	log "github.com/sirupsen/logrus"
+
+	"gitea.com/gitea/act_runner/client"
 )

 var globalTaskMap sync.Map