lianguan/gitea
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Actions Artifacts v4 backend (#28965)

Browse Source 
Fixes #28853 

Needs both https://gitea.com/gitea/act_runner/pulls/473 and
https://gitea.com/gitea/act_runner/pulls/471 on the runner side and
patched `actions/upload-artifact@v4` / `actions/download-artifact@v4`,
like `christopherhx/gitea-upload-artifact@v4` and
`christopherhx/gitea-download-artifact@v4`, to not return errors due to
GHES not beeing supported yet.
This commit is contained in:
ChristopherHX
2024-03-02 10:12:17 +01:00
committed by GitHub
parent 8a0a83a1b5
commit a53d268aca
11 changed files with 2034 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
models/fixtures/action_run.yml
View File

@ -17,3 +17,22 @@
updated: 1683636626
need_approval: 0
approved_by: 0
-
id: 792
title: "update actions"
repo_id: 4
owner_id: 1
workflow_id: "artifact.yaml"
index: 188
trigger_user_id: 1
ref: "refs/heads/master"
commit_sha: "c2d72f548424103f01ee1dc02889c1e2bff816b0"
event: "push"
is_fork_pull_request: 0
status: 1
started: 1683636528
stopped: 1683636626
created: 1683636108
updated: 1683636626
need_approval: 0
approved_by: 0

14
models/fixtures/action_run_job.yml
View File

@ -12,3 +12,17 @@
status: 1
started: 1683636528
stopped: 1683636626
-
id: 193
run_id: 792
repo_id: 4
owner_id: 1
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
is_fork_pull_request: 0
name: job_2
attempt: 1
job_id: job_2
task_id: 48
status: 1
started: 1683636528
stopped: 1683636626

20
models/fixtures/action_task.yml
View File

@ -18,3 +18,23 @@
log_length: 707
log_size: 90179
log_expired: 0
-
id: 48
job_id: 193
attempt: 1
runner_id: 1
status: 6 # 6 is the status code for "running", running task can upload artifacts
started: 1683636528
stopped: 1683636626
repo_id: 4
owner_id: 1
commit_sha: c2d72f548424103f01ee1dc02889c1e2bff816b0
is_fork_pull_request: 0
token_hash: ffffcfffffffbffffffffffffffffefffffffafffffffffffffffffffffffffffffdffffffffffffffffffffffffffffffff
token_salt: ffffffffff
token_last_eight: ffffffff
log_filename: artifact-test2/2f/47.log
log_in_storage: 1
log_length: 707
log_size: 90179
log_expired: 0

1058
routers/api/actions/artifact.pb.go Normal file
View File

File diff suppressed because it is too large Load Diff

73
routers/api/actions/artifact.proto Normal file
View File

@ -0,0 +1,73 @@
syntax = "proto3";
import "google/protobuf/timestamp.proto";
import "google/protobuf/wrappers.proto";
package github.actions.results.api.v1;
message CreateArtifactRequest {
string workflow_run_backend_id = 1;
string workflow_job_run_backend_id = 2;
string name = 3;
google.protobuf.Timestamp expires_at = 4;
int32 version = 5;
}
message CreateArtifactResponse {
bool ok = 1;
string signed_upload_url = 2;
}
message FinalizeArtifactRequest {
string workflow_run_backend_id = 1;
string workflow_job_run_backend_id = 2;
string name = 3;
int64 size = 4;
google.protobuf.StringValue hash = 5;
}
message FinalizeArtifactResponse {
bool ok = 1;
int64 artifact_id = 2;
}
message ListArtifactsRequest {
string workflow_run_backend_id = 1;
string workflow_job_run_backend_id = 2;
google.protobuf.StringValue name_filter = 3;
google.protobuf.Int64Value id_filter = 4;
}
message ListArtifactsResponse {
repeated ListArtifactsResponse_MonolithArtifact artifacts = 1;
}
message ListArtifactsResponse_MonolithArtifact {
string workflow_run_backend_id = 1;
string workflow_job_run_backend_id = 2;
int64 database_id = 3;
string name = 4;
int64 size = 5;
google.protobuf.Timestamp created_at = 6;
}
message GetSignedArtifactURLRequest {
string workflow_run_backend_id = 1;
string workflow_job_run_backend_id = 2;
string name = 3;
}
message GetSignedArtifactURLResponse {
string signed_url = 1;
}
message DeleteArtifactRequest {
string workflow_run_backend_id = 1;
string workflow_job_run_backend_id = 2;
string name = 3;
}
message DeleteArtifactResponse {
bool ok = 1;
int64 artifact_id = 2;
}

107
routers/api/actions/artifacts_chunks.go
View File

@ -5,11 +5,16 @@ package actions
import (
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"hash"
"io"
"path/filepath"
"sort"
"strings"
"time"
"code.gitea.io/gitea/models/actions"
@ -18,6 +23,52 @@ import (
"code.gitea.io/gitea/modules/storage"
)
func saveUploadChunkBase(st storage.ObjectStorage, ctx *ArtifactContext,
artifact *actions.ActionArtifact,
contentSize, runID, start, end, length int64, checkMd5 bool,
) (int64, error) {
// build chunk store path
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end)
var r io.Reader = ctx.Req.Body
var hasher hash.Hash
if checkMd5 {
// use io.TeeReader to avoid reading all body to md5 sum.
// it writes data to hasher after reading end
// if hash is not matched, delete the read-end result
hasher = md5.New()
r = io.TeeReader(r, hasher)
}
// save chunk to storage
writtenSize, err := st.Save(storagePath, r, -1)
if err != nil {
return -1, fmt.Errorf("save chunk to storage error: %v", err)
}
var checkErr error
if checkMd5 {
// check md5
reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header)
chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
log.Info("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String)
// if md5 not match, delete the chunk
if reqMd5String != chunkMd5String {
checkErr = fmt.Errorf("md5 not match")
}
}
if writtenSize != contentSize {
checkErr = errors.Join(checkErr, fmt.Errorf("contentSize not match body size"))
}
if checkErr != nil {
if err := st.Delete(storagePath); err != nil {
log.Error("Error deleting chunk: %s, %v", storagePath, err)
}
return -1, checkErr
}
log.Info("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d",
storagePath, contentSize, artifact.ID, start, end)
// return chunk total size
return length, nil
}
func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
artifact *actions.ActionArtifact,
contentSize, runID int64,
@ -29,33 +80,15 @@ func saveUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
log.Warn("parse content range error: %v, content-range: %s", err, contentRange)
return -1, fmt.Errorf("parse content range error: %v", err)
}
// build chunk store path
storagePath := fmt.Sprintf("tmp%d/%d-%d-%d-%d.chunk", runID, runID, artifact.ID, start, end)
// use io.TeeReader to avoid reading all body to md5 sum.
// it writes data to hasher after reading end
// if hash is not matched, delete the read-end result
hasher := md5.New()
r := io.TeeReader(ctx.Req.Body, hasher)
// save chunk to storage
writtenSize, err := st.Save(storagePath, r, -1)
if err != nil {
return -1, fmt.Errorf("save chunk to storage error: %v", err)
}
// check md5
reqMd5String := ctx.Req.Header.Get(artifactXActionsResultsMD5Header)
chunkMd5String := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
log.Info("[artifact] check chunk md5, sum: %s, header: %s", chunkMd5String, reqMd5String)
// if md5 not match, delete the chunk
if reqMd5String != chunkMd5String || writtenSize != contentSize {
if err := st.Delete(storagePath); err != nil {
log.Error("Error deleting chunk: %s, %v", storagePath, err)
}
return -1, fmt.Errorf("md5 not match")
}
log.Info("[artifact] save chunk %s, size: %d, artifact id: %d, start: %d, end: %d",
storagePath, contentSize, artifact.ID, start, end)
// return chunk total size
return length, nil
return saveUploadChunkBase(st, ctx, artifact, contentSize, runID, start, end, length, true)
}
func appendUploadChunk(st storage.ObjectStorage, ctx *ArtifactContext,
artifact *actions.ActionArtifact,
start, contentSize, runID int64,
) (int64, error) {
end := start + contentSize - 1
return saveUploadChunkBase(st, ctx, artifact, contentSize, runID, start, end, contentSize, false)
}
type chunkFileItem struct {
@ -111,14 +144,14 @@ func mergeChunksForRun(ctx *ArtifactContext, st storage.ObjectStorage, runID int
log.Debug("artifact %d chunks not found", art.ID)
continue
}
if err := mergeChunksForArtifact(ctx, chunks, st, art); err != nil {
if err := mergeChunksForArtifact(ctx, chunks, st, art, ""); err != nil {
return err
}
}
return nil
}
func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact) error {
func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st storage.ObjectStorage, artifact *actions.ActionArtifact, checksum string) error {
sort.Slice(chunks, func(i, j int) bool {
return chunks[i].Start < chunks[j].Start
})
@ -157,6 +190,14 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
readers = append(readers, readCloser)
}
mergedReader := io.MultiReader(readers...)
shaPrefix := "sha256:"
var hash hash.Hash
if strings.HasPrefix(checksum, shaPrefix) {
hash = sha256.New()
}
if hash != nil {
mergedReader = io.TeeReader(mergedReader, hash)
}
// if chunk is gzip, use gz as extension
// download-artifact action will use content-encoding header to decide if it should decompress the file
@ -185,6 +226,14 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st
}
}()
if hash != nil {
rawChecksum := hash.Sum(nil)
actualChecksum := hex.EncodeToString(rawChecksum)
if !strings.HasSuffix(checksum, actualChecksum) {
return fmt.Errorf("update artifact error checksum is invalid")
}
}
// save storage path to artifact
log.Debug("[artifact] merge chunks to artifact: %d, %s, old:%s", artifact.ID, storagePath, artifact.StoragePath)
// if artifact is already uploaded, delete the old file

11
routers/api/actions/artifacts_utils.go
View File

@ -43,6 +43,17 @@ func validateRunID(ctx *ArtifactContext) (*actions.ActionTask, int64, bool) {
return task, runID, true
}
func validateRunIDV4(ctx *ArtifactContext, rawRunID string) (*actions.ActionTask, int64, bool) {
task := ctx.ActionTask
runID, err := strconv.ParseInt(rawRunID, 10, 64)
if err != nil || task.Job.RunID != runID {
log.Error("Error runID not match")
ctx.Error(http.StatusBadRequest, "run-id does not match")
return nil, 0, false
}
return task, runID, true
}
func validateArtifactHash(ctx *ArtifactContext, artifactName string) bool {
paramHash := ctx.Params("artifact_hash")
// use artifact name to create upload url

512
routers/api/actions/artifactsv4.go Normal file
View File

File diff suppressed because it is too large Load Diff

2
routers/init.go
View File

@ -198,6 +198,8 @@ func NormalRoutes() *web.Route {
// TODO: this prefix should be generated with a token string with runner ?
prefix = "/api/actions_pipeline"
r.Mount(prefix, actions_router.ArtifactsRoutes(prefix))
prefix = actions_router.ArtifactV4RouteBase
r.Mount(prefix, actions_router.ArtifactsV4Routes(prefix))
}
return r

23
routers/web/repo/actions/view.go
View File

@ -22,6 +22,7 @@ import (
"code.gitea.io/gitea/models/unit"
"code.gitea.io/gitea/modules/actions"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@ -602,6 +603,28 @@ func ArtifactsDownloadView(ctx *context_module.Context) {
ctx.Resp.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s.zip; filename*=UTF-8''%s.zip", url.PathEscape(artifactName), artifactName))
// Artifacts using the v4 backend are stored as a single combined zip file per artifact on the backend
// The v4 backend enshures ContentEncoding is set to "application/zip", which is not the case for the old backend
if len(artifacts) == 1 && artifacts[0].ArtifactName+".zip" == artifacts[0].ArtifactPath && artifacts[0].ContentEncoding == "application/zip" {
art := artifacts[0]
if setting.Actions.ArtifactStorage.MinioConfig.ServeDirect {
u, err := storage.ActionsArtifacts.URL(art.StoragePath, art.ArtifactPath)
if u != nil && err == nil {
ctx.Redirect(u.String())
return
}
}
f, err := storage.ActionsArtifacts.Open(art.StoragePath)
if err != nil {
ctx.Error(http.StatusInternalServerError, err.Error())
return
}
_, _ = io.Copy(ctx.Resp, f)
return
}
// Artifacts using the v1-v3 backend are stored as multiple individual files per artifact on the backend
// Those need to be zipped for download
writer := zip.NewWriter(ctx.Resp)
defer writer.Close()
for _, art := range artifacts {

224
tests/integration/api_actions_artifact_v4_test.go Normal file
View File

@ -0,0 +1,224 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package integration
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"io"
"net/http"
"strings"
"testing"
"time"
"code.gitea.io/gitea/routers/api/actions"
actions_service "code.gitea.io/gitea/services/actions"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/timestamppb"
"google.golang.org/protobuf/types/known/wrapperspb"
)
func toProtoJSON(m protoreflect.ProtoMessage) io.Reader {
resp, _ := protojson.Marshal(m)
buf := bytes.Buffer{}
buf.Write(resp)
return &buf
}
func TestActionsArtifactV4UploadSingleFile(t *testing.T) {
defer tests.PrepareTestEnv(t)()
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: 4,
Name: "artifact",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
// upload artifact chunk
body := strings.Repeat("A", 1024)
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
MakeRequest(t, req, http.StatusCreated)
t.Logf("Create artifact confirm")
sha := sha256.Sum256([]byte(body))
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: "artifact",
Size: 1024,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
}
func TestActionsArtifactV4UploadSingleFileWrongChecksum(t *testing.T) {
defer tests.PrepareTestEnv(t)()
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: 4,
Name: "artifact-invalid-checksum",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
// upload artifact chunk
body := strings.Repeat("B", 1024)
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
MakeRequest(t, req, http.StatusCreated)
t.Logf("Create artifact confirm")
sha := sha256.Sum256([]byte(strings.Repeat("A", 1024)))
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: "artifact-invalid-checksum",
Size: 1024,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
MakeRequest(t, req, http.StatusInternalServerError)
}
func TestActionsArtifactV4UploadSingleFileWithRetentionDays(t *testing.T) {
defer tests.PrepareTestEnv(t)()
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/CreateArtifact", toProtoJSON(&actions.CreateArtifactRequest{
Version: 4,
ExpiresAt: timestamppb.New(time.Now().Add(5 * 24 * time.Hour)),
Name: "artifactWithRetentionDays",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var uploadResp actions.CreateArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &uploadResp)
assert.True(t, uploadResp.Ok)
assert.Contains(t, uploadResp.SignedUploadUrl, "/twirp/github.actions.results.api.v1.ArtifactService/UploadArtifact")
// get upload url
idx := strings.Index(uploadResp.SignedUploadUrl, "/twirp/")
url := uploadResp.SignedUploadUrl[idx:] + "&comp=block"
// upload artifact chunk
body := strings.Repeat("A", 1024)
req = NewRequestWithBody(t, "PUT", url, strings.NewReader(body))
MakeRequest(t, req, http.StatusCreated)
t.Logf("Create artifact confirm")
sha := sha256.Sum256([]byte(body))
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/FinalizeArtifact", toProtoJSON(&actions.FinalizeArtifactRequest{
Name: "artifactWithRetentionDays",
Size: 1024,
Hash: wrapperspb.String("sha256:" + hex.EncodeToString(sha[:])),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.FinalizeArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.True(t, finalizeResp.Ok)
}
func TestActionsArtifactV4DownloadSingle(t *testing.T) {
defer tests.PrepareTestEnv(t)()
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// acquire artifact upload url
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/ListArtifacts", toProtoJSON(&actions.ListArtifactsRequest{
NameFilter: wrapperspb.String("artifact"),
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var listResp actions.ListArtifactsResponse
protojson.Unmarshal(resp.Body.Bytes(), &listResp)
assert.Len(t, listResp.Artifacts, 1)
// confirm artifact upload
req = NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/GetSignedArtifactURL", toProtoJSON(&actions.GetSignedArtifactURLRequest{
Name: "artifact",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).
AddTokenAuth(token)
resp = MakeRequest(t, req, http.StatusOK)
var finalizeResp actions.GetSignedArtifactURLResponse
protojson.Unmarshal(resp.Body.Bytes(), &finalizeResp)
assert.NotEmpty(t, finalizeResp.SignedUrl)
req = NewRequest(t, "GET", finalizeResp.SignedUrl)
resp = MakeRequest(t, req, http.StatusOK)
body := strings.Repeat("A", 1024)
assert.Equal(t, resp.Body.String(), body)
}
func TestActionsArtifactV4Delete(t *testing.T) {
defer tests.PrepareTestEnv(t)()
token, err := actions_service.CreateAuthorizationToken(48, 792, 193)
assert.NoError(t, err)
// delete artifact by name
req := NewRequestWithBody(t, "POST", "/twirp/github.actions.results.api.v1.ArtifactService/DeleteArtifact", toProtoJSON(&actions.DeleteArtifactRequest{
Name: "artifact",
WorkflowRunBackendId: "792",
WorkflowJobRunBackendId: "193",
})).AddTokenAuth(token)
resp := MakeRequest(t, req, http.StatusOK)
var deleteResp actions.DeleteArtifactResponse
protojson.Unmarshal(resp.Body.Bytes(), &deleteResp)
assert.True(t, deleteResp.Ok)
}