From 3646ea440bb81b70d7f138b4585b336c90e5d1fa Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 14:35:02 +0800 Subject: [PATCH 01/13] chore: update actions-proto-go --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 944f6d2c918c8..3363c384f9453 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module code.gitea.io/gitea go 1.19 require ( - code.gitea.io/actions-proto-go v0.2.0 + code.gitea.io/actions-proto-go v0.2.1 code.gitea.io/gitea-vet v0.2.2 code.gitea.io/sdk/gitea v0.15.1 codeberg.org/gusted/mcaptcha v0.0.0-20220723083913-4f3072e1d570 diff --git a/go.sum b/go.sum index df57c65918b05..783b3b5f93ecc 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= code.gitea.io/actions-proto-go v0.2.0 h1:nYh9nhhfk67YA4wVNLsCzd//RCvXnljwXClJ33+HPVk= code.gitea.io/actions-proto-go v0.2.0/go.mod h1:00ys5QDo1iHN1tHNvvddAcy2W/g+425hQya1cCSvq9A= +code.gitea.io/actions-proto-go v0.2.1 h1:ToMN/8thz2q10TuCq8dL2d8mI+/pWpJcHCvG+TELwa0= +code.gitea.io/actions-proto-go v0.2.1/go.mod h1:00ys5QDo1iHN1tHNvvddAcy2W/g+425hQya1cCSvq9A= code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= code.gitea.io/gitea-vet v0.2.2 h1:TEOV/Glf38iGmKzKP0EB++Z5OSL4zGg3RrAvlwaMuvk= code.gitea.io/gitea-vet v0.2.2/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= From 29ad93f95e4406bdabbadefc5cb478625077cd71 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 14:35:36 +0800 Subject: [PATCH 02/13] test: debug --- routers/api/actions/runner/runner.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go index a445864858931..b9b1c59a4e556 100644 --- a/routers/api/actions/runner/runner.go +++ b/routers/api/actions/runner/runner.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/actions-proto-go/runner/v1/runnerv1connect" "github.com/bufbuild/connect-go" gouuid "github.com/google/uuid" + "github.com/nektos/act/pkg/jobparser" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -109,12 +110,33 @@ func (s *Service) FetchTask( task = t } + if task != nil { + var workflowJob *jobparser.Job + if gots, err := jobparser.Parse(task.WorkflowPayload); err != nil { + panic(err) + } else if len(gots) != 1 { + panic(err) + } else { + _, workflowJob = gots[0].Job() + } + if workflowJob.Name == "job2" { + task.Needs = map[string]*runnerv1.TaskNeed{ + "job1": { + Outputs: debugOutputs, + Result: runnerv1.Result_RESULT_SUCCESS, + }, + } + } + } + res := connect.NewResponse(&runnerv1.FetchTaskResponse{ Task: task, }) return res, nil } +var debugOutputs = map[string]string{} + // UpdateTask updates the task status. func (s *Service) UpdateTask( ctx context.Context, @@ -145,6 +167,11 @@ func (s *Service) UpdateTask( return nil, status.Errorf(codes.Internal, "update task: %v", err) } + for k, v := range req.Msg.Outputs { + log.Info("task %d output %s: %s", task.ID, k, v) + debugOutputs[k] = v + } + if err := task.LoadJob(ctx); err != nil { return nil, status.Errorf(codes.Internal, "load job: %v", err) } From 1294750f59a002e9fb8197ccf2b1c0189d2f25fb Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 14:35:43 +0800 Subject: [PATCH 03/13] Revert "test: debug" This reverts commit 29ad93f95e4406bdabbadefc5cb478625077cd71. --- routers/api/actions/runner/runner.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go index b9b1c59a4e556..a445864858931 100644 --- a/routers/api/actions/runner/runner.go +++ b/routers/api/actions/runner/runner.go @@ -19,7 +19,6 @@ import ( "code.gitea.io/actions-proto-go/runner/v1/runnerv1connect" "github.com/bufbuild/connect-go" gouuid "github.com/google/uuid" - "github.com/nektos/act/pkg/jobparser" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -110,33 +109,12 @@ func (s *Service) FetchTask( task = t } - if task != nil { - var workflowJob *jobparser.Job - if gots, err := jobparser.Parse(task.WorkflowPayload); err != nil { - panic(err) - } else if len(gots) != 1 { - panic(err) - } else { - _, workflowJob = gots[0].Job() - } - if workflowJob.Name == "job2" { - task.Needs = map[string]*runnerv1.TaskNeed{ - "job1": { - Outputs: debugOutputs, - Result: runnerv1.Result_RESULT_SUCCESS, - }, - } - } - } - res := connect.NewResponse(&runnerv1.FetchTaskResponse{ Task: task, }) return res, nil } -var debugOutputs = map[string]string{} - // UpdateTask updates the task status. func (s *Service) UpdateTask( ctx context.Context, @@ -167,11 +145,6 @@ func (s *Service) UpdateTask( return nil, status.Errorf(codes.Internal, "update task: %v", err) } - for k, v := range req.Msg.Outputs { - log.Info("task %d output %s: %s", task.ID, k, v) - debugOutputs[k] = v - } - if err := task.LoadJob(ctx); err != nil { return nil, status.Errorf(codes.Internal, "load job: %v", err) } From 8b543c444eb9c67ab517530ee68b491490176224 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 15:06:18 +0800 Subject: [PATCH 04/13] feat: ActionTaskOutput --- models/actions/task_output.go | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 models/actions/task_output.go diff --git a/models/actions/task_output.go b/models/actions/task_output.go new file mode 100644 index 0000000000000..6812b1487df63 --- /dev/null +++ b/models/actions/task_output.go @@ -0,0 +1,56 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "context" + "crypto/sha1" + "fmt" + + "code.gitea.io/gitea/models/db" +) + +// ActionTaskOutput represents an output of ActionTask. +// So the outputs are bound to a task, that means when a completed job has been rerun, +// the outputs of the job will be reset because the task is new. +// It's by design, to avoid the outputs of the old task to be mixed with the new task. +type ActionTaskOutput struct { + ID int64 + TaskID int64 `xorm:"INDEX UNIQUE(task_id_key)"` + Key string `xorm:"VARCHAR(255)"` + KeyHash string `xorm:"CHAR(40) UNIQUE(task_id_key)"` + Value string `xorm:"TEXT"` +} + +// FindTaskOutputByTaskID returns the outputs of the task. +func FindTaskOutputByTaskID(ctx context.Context, taskID int64) ([]*ActionTaskOutput, error) { + var outputs []*ActionTaskOutput + return outputs, db.GetEngine(ctx).Where("task_id=?", taskID).Find(&outputs) +} + +// FindTaskOutputKeyByTaskID returns the keys of the outputs of the task. +func FindTaskOutputKeyByTaskID(ctx context.Context, taskID int64) ([]string, error) { + var keys []string + return keys, db.GetEngine(ctx).Table(ActionTaskOutput{}).Where("task_id=?", taskID).Cols("key").Find(&keys) +} + +// InsertTaskOutputIfNotExist inserts a new task output if it does not exist. +func InsertTaskOutputIfNotExist(ctx context.Context, taskID int64, key, value string) error { + keyHash := fmt.Sprintf("%x", sha1.Sum([]byte(key))) + return db.WithTx(ctx, func(ctx context.Context) error { + sess := db.GetEngine(ctx) + if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, KeyHash: keyHash}); err != nil { + return err + } else if exist { + return nil + } + _, err := sess.Insert(&ActionTaskOutput{ + TaskID: taskID, + Key: key, + KeyHash: keyHash, + Value: value, + }) + return err + }) +} From 13fbbea256c427472b63c6fb95042e579f29202c Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 15:43:50 +0800 Subject: [PATCH 05/13] fix: send needs outputs --- routers/api/actions/runner/runner.go | 2 +- routers/api/actions/runner/utils.go | 54 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go index a445864858931..ebce3802cf6f4 100644 --- a/routers/api/actions/runner/runner.go +++ b/routers/api/actions/runner/runner.go @@ -97,7 +97,7 @@ func (s *Service) Register( // FetchTask assigns a task to the runner func (s *Service) FetchTask( ctx context.Context, - req *connect.Request[runnerv1.FetchTaskRequest], + _ *connect.Request[runnerv1.FetchTaskRequest], ) (*connect.Response[runnerv1.FetchTaskResponse], error) { runner := GetRunner(ctx) diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go index cbfcd79a9d999..b0b842c38523e 100644 --- a/routers/api/actions/runner/utils.go +++ b/routers/api/actions/runner/utils.go @@ -37,6 +37,17 @@ func pickTask(ctx context.Context, runner *actions_model.ActionRunner) (*runnerv Context: generateTaskContext(t), Secrets: getSecretsOfTask(ctx, t), } + + if needs, err := findTaskNeeds(ctx, t); err != nil { + log.Error("Cannot find needs for task %v: %v", t.ID, err) + // Go on with empty needs. + // If return error, the task will be wild, which means the runner will never get it when it has been assigned to the runner. + // In contrast, missing needs is less serious. + // And the task will fail and the runner will report the error in the logs. + } else { + task.Needs = needs + } + return task, true, nil } @@ -124,3 +135,46 @@ func generateTaskContext(t *actions_model.ActionTask) *structpb.Struct { return taskContext } + +func findTaskNeeds(ctx context.Context, task *actions_model.ActionTask) (map[string]*runnerv1.TaskNeed, error) { + if err := task.LoadAttributes(ctx); err != nil { + return nil, fmt.Errorf("LoadAttributes: %w", err) + } + if len(task.Job.Needs) == 0 { + return nil, nil + } + needs := map[string]struct{}{} + for _, v := range task.Job.Needs { + needs[v] = struct{}{} + } + + jobs, _, err := actions_model.FindRunJobs(ctx, actions_model.FindRunJobOptions{RunID: task.Job.RunID}) + if err != nil { + return nil, fmt.Errorf("FindRunJobs: %w", err) + } + + ret := make(map[string]*runnerv1.TaskNeed, len(needs)) + for _, job := range jobs { + if _, ok := needs[job.JobID]; !ok { + continue + } + if job.TaskID == 0 || !job.Status.IsDone() { + // it shouldn't happen, or the job has been rerun + continue + } + outputs := make(map[string]string) + if out, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID); err != nil { + return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) + } else { + for _, v := range out { + outputs[v.Key] = v.Value + } + } + ret[job.JobID] = &runnerv1.TaskNeed{ + Outputs: outputs, + Result: runnerv1.Result(job.Status), + } + } + + return ret, nil +} From 673d58a571075fe8a1e6cbcd7c41333b20813b62 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 15:55:12 +0800 Subject: [PATCH 06/13] fix: send outputs --- routers/api/actions/runner/runner.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go index ebce3802cf6f4..ebb9130b42417 100644 --- a/routers/api/actions/runner/runner.go +++ b/routers/api/actions/runner/runner.go @@ -145,6 +145,22 @@ func (s *Service) UpdateTask( return nil, status.Errorf(codes.Internal, "update task: %v", err) } + for k, v := range req.Msg.Outputs { + if len(k) > 255 { + log.Warn("Ignore the output of task %d because the key is too long: %q", task.ID, k) + continue + } + if err := actions_model.InsertTaskOutputIfNotExist(ctx, task.ID, k, v); err != nil { + log.Warn("Failed to insert the output %q of task %d: %v", k, task.ID, err) + // It's ok not to return errors, the runner will resend the outputs. + } + } + sentOutputs, err := actions_model.FindTaskOutputKeyByTaskID(ctx, task.ID) + if err != nil { + log.Warn("Failed to find the sent outputs of task %d: %v", task.ID, err) + // It's not to return errors, it can be handled when the runner resends sent outputs. + } + if err := task.LoadJob(ctx); err != nil { return nil, status.Errorf(codes.Internal, "load job: %v", err) } @@ -162,6 +178,7 @@ func (s *Service) UpdateTask( Id: req.Msg.State.Id, Result: task.Status.AsResult(), }, + SentOutputs: sentOutputs, }), nil } From 3c975ff93a5b9d4010ad30000e9bdc4f70a6b509 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 16:42:02 +0800 Subject: [PATCH 07/13] chore: NewMigration --- models/migrations/migrations.go | 2 ++ models/migrations/v1_20/v254.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 models/migrations/v1_20/v254.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 42806a808f791..9de5931d71468 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -485,6 +485,8 @@ var migrations = []Migration{ NewMigration("Fix incorrect admin team unit access mode", v1_20.FixIncorrectAdminTeamUnitAccessMode), // v253 -> v254 NewMigration("Fix ExternalTracker and ExternalWiki accessMode in owner and admin team", v1_20.FixExternalTrackerAndExternalWikiAccessModeInOwnerAndAdminTeam), + // v254 -> v255 + NewMigration("Add ActionTaskOutput table", v1_20.AddActionTaskOutputTable), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v254.go new file mode 100644 index 0000000000000..614911690ea4d --- /dev/null +++ b/models/migrations/v1_20/v254.go @@ -0,0 +1,19 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package v1_20 //nolint + +import ( + "xorm.io/xorm" +) + +func AddActionTaskOutputTable(x *xorm.Engine) error { + type ActionTaskOutput struct { + ID int64 + TaskID int64 `xorm:"INDEX UNIQUE(task_id_key)"` + Key string `xorm:"VARCHAR(255)"` + KeyHash string `xorm:"CHAR(40) UNIQUE(task_id_key)"` + Value string `xorm:"TEXT"` + } + return x.Sync(new(ActionTaskOutput)) +} From ce5eb4716d6935286dfc5b62ab36528b1e75916d Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 16:58:53 +0800 Subject: [PATCH 08/13] fix: lint --- routers/api/actions/runner/utils.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go index b0b842c38523e..c37030e22f968 100644 --- a/routers/api/actions/runner/utils.go +++ b/routers/api/actions/runner/utils.go @@ -163,12 +163,12 @@ func findTaskNeeds(ctx context.Context, task *actions_model.ActionTask) (map[str continue } outputs := make(map[string]string) - if out, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID); err != nil { + got, err := actions_model.FindTaskOutputByTaskID(ctx, job.TaskID) + if err != nil { return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) - } else { - for _, v := range out { - outputs[v.Key] = v.Value - } + } + for _, v := range got { + outputs[v.Key] = v.Value } ret[job.JobID] = &runnerv1.TaskNeed{ Outputs: outputs, From 737446db73375dfe4b9bd01e1dff90a3584bc6d9 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 17:18:37 +0800 Subject: [PATCH 09/13] chore: tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 783b3b5f93ecc..7a47f4ee74db9 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -code.gitea.io/actions-proto-go v0.2.0 h1:nYh9nhhfk67YA4wVNLsCzd//RCvXnljwXClJ33+HPVk= -code.gitea.io/actions-proto-go v0.2.0/go.mod h1:00ys5QDo1iHN1tHNvvddAcy2W/g+425hQya1cCSvq9A= code.gitea.io/actions-proto-go v0.2.1 h1:ToMN/8thz2q10TuCq8dL2d8mI+/pWpJcHCvG+TELwa0= code.gitea.io/actions-proto-go v0.2.1/go.mod h1:00ys5QDo1iHN1tHNvvddAcy2W/g+425hQya1cCSvq9A= code.gitea.io/gitea-vet v0.2.1/go.mod h1:zcNbT/aJEmivCAhfmkHOlT645KNOf9W2KnkLgFjGGfE= From 9eb634e03953432c4709407f583cffdea8be7371 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 17:30:51 +0800 Subject: [PATCH 10/13] fix: update field name --- models/actions/task_output.go | 22 +++++++++++----------- models/migrations/v1_20/v254.go | 10 +++++----- routers/api/actions/runner/utils.go | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/models/actions/task_output.go b/models/actions/task_output.go index 6812b1487df63..6e79327bbd54a 100644 --- a/models/actions/task_output.go +++ b/models/actions/task_output.go @@ -16,11 +16,11 @@ import ( // the outputs of the job will be reset because the task is new. // It's by design, to avoid the outputs of the old task to be mixed with the new task. type ActionTaskOutput struct { - ID int64 - TaskID int64 `xorm:"INDEX UNIQUE(task_id_key)"` - Key string `xorm:"VARCHAR(255)"` - KeyHash string `xorm:"CHAR(40) UNIQUE(task_id_key)"` - Value string `xorm:"TEXT"` + ID int64 + TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` + OutputKey string `xorm:"VARCHAR(255)"` + OutputKeyHash string `xorm:"CHAR(40) UNIQUE(task_id_output_key)"` + OutputValue string `xorm:"TEXT"` } // FindTaskOutputByTaskID returns the outputs of the task. @@ -32,7 +32,7 @@ func FindTaskOutputByTaskID(ctx context.Context, taskID int64) ([]*ActionTaskOut // FindTaskOutputKeyByTaskID returns the keys of the outputs of the task. func FindTaskOutputKeyByTaskID(ctx context.Context, taskID int64) ([]string, error) { var keys []string - return keys, db.GetEngine(ctx).Table(ActionTaskOutput{}).Where("task_id=?", taskID).Cols("key").Find(&keys) + return keys, db.GetEngine(ctx).Table(ActionTaskOutput{}).Where("task_id=?", taskID).Cols("output_key").Find(&keys) } // InsertTaskOutputIfNotExist inserts a new task output if it does not exist. @@ -40,16 +40,16 @@ func InsertTaskOutputIfNotExist(ctx context.Context, taskID int64, key, value st keyHash := fmt.Sprintf("%x", sha1.Sum([]byte(key))) return db.WithTx(ctx, func(ctx context.Context) error { sess := db.GetEngine(ctx) - if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, KeyHash: keyHash}); err != nil { + if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, OutputKeyHash: keyHash}); err != nil { return err } else if exist { return nil } _, err := sess.Insert(&ActionTaskOutput{ - TaskID: taskID, - Key: key, - KeyHash: keyHash, - Value: value, + TaskID: taskID, + OutputKey: key, + OutputKeyHash: keyHash, + OutputValue: value, }) return err }) diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v254.go index 614911690ea4d..a9a603aa438ab 100644 --- a/models/migrations/v1_20/v254.go +++ b/models/migrations/v1_20/v254.go @@ -9,11 +9,11 @@ import ( func AddActionTaskOutputTable(x *xorm.Engine) error { type ActionTaskOutput struct { - ID int64 - TaskID int64 `xorm:"INDEX UNIQUE(task_id_key)"` - Key string `xorm:"VARCHAR(255)"` - KeyHash string `xorm:"CHAR(40) UNIQUE(task_id_key)"` - Value string `xorm:"TEXT"` + ID int64 + TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` + OutputKey string `xorm:"VARCHAR(255)"` + OutputKeyHash string `xorm:"CHAR(40) UNIQUE(task_id_output_key)"` + OutputValue string `xorm:"TEXT"` } return x.Sync(new(ActionTaskOutput)) } diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go index c37030e22f968..f9e9263edadb7 100644 --- a/routers/api/actions/runner/utils.go +++ b/routers/api/actions/runner/utils.go @@ -168,7 +168,7 @@ func findTaskNeeds(ctx context.Context, task *actions_model.ActionTask) (map[str return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) } for _, v := range got { - outputs[v.Key] = v.Value + outputs[v.OutputKey] = v.OutputKeyHash } ret[job.JobID] = &runnerv1.TaskNeed{ Outputs: outputs, From 70f8437ddaf45283113e0e1ad583ba8c1cf713b9 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 18:00:11 +0800 Subject: [PATCH 11/13] fix: use sha256 --- models/actions/task_output.go | 7 ++++--- models/migrations/v1_20/v254.go | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/models/actions/task_output.go b/models/actions/task_output.go index 6e79327bbd54a..3d700785b35c5 100644 --- a/models/actions/task_output.go +++ b/models/actions/task_output.go @@ -5,7 +5,7 @@ package actions import ( "context" - "crypto/sha1" + "crypto/sha256" "fmt" "code.gitea.io/gitea/models/db" @@ -19,7 +19,7 @@ type ActionTaskOutput struct { ID int64 TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` OutputKey string `xorm:"VARCHAR(255)"` - OutputKeyHash string `xorm:"CHAR(40) UNIQUE(task_id_output_key)"` + OutputKeyHash string `xorm:"CHAR(32) UNIQUE(task_id_output_key)"` OutputValue string `xorm:"TEXT"` } @@ -37,7 +37,8 @@ func FindTaskOutputKeyByTaskID(ctx context.Context, taskID int64) ([]string, err // InsertTaskOutputIfNotExist inserts a new task output if it does not exist. func InsertTaskOutputIfNotExist(ctx context.Context, taskID int64, key, value string) error { - keyHash := fmt.Sprintf("%x", sha1.Sum([]byte(key))) + keyHash := fmt.Sprintf("%x", sha256.Sum256([]byte(key))) + keyHash = keyHash[:32] // 32 chars (16 bytes) is enough to avoid collision inner a task return db.WithTx(ctx, func(ctx context.Context) error { sess := db.GetEngine(ctx) if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, OutputKeyHash: keyHash}); err != nil { diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v254.go index a9a603aa438ab..0bd5f78c3eabc 100644 --- a/models/migrations/v1_20/v254.go +++ b/models/migrations/v1_20/v254.go @@ -12,7 +12,7 @@ func AddActionTaskOutputTable(x *xorm.Engine) error { ID int64 TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` OutputKey string `xorm:"VARCHAR(255)"` - OutputKeyHash string `xorm:"CHAR(40) UNIQUE(task_id_output_key)"` + OutputKeyHash string `xorm:"CHAR(32) UNIQUE(task_id_output_key)"` OutputValue string `xorm:"TEXT"` } return x.Sync(new(ActionTaskOutput)) From 5519d9d87be1d24c48b026003255a60f38139033 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 18:30:12 +0800 Subject: [PATCH 12/13] fix: drop key hash --- models/actions/task_output.go | 22 ++++++++-------------- models/migrations/v1_20/v254.go | 9 ++++----- routers/api/actions/runner/utils.go | 2 +- 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/models/actions/task_output.go b/models/actions/task_output.go index 3d700785b35c5..9957c5b185d20 100644 --- a/models/actions/task_output.go +++ b/models/actions/task_output.go @@ -5,8 +5,6 @@ package actions import ( "context" - "crypto/sha256" - "fmt" "code.gitea.io/gitea/models/db" ) @@ -16,11 +14,10 @@ import ( // the outputs of the job will be reset because the task is new. // It's by design, to avoid the outputs of the old task to be mixed with the new task. type ActionTaskOutput struct { - ID int64 - TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` - OutputKey string `xorm:"VARCHAR(255)"` - OutputKeyHash string `xorm:"CHAR(32) UNIQUE(task_id_output_key)"` - OutputValue string `xorm:"TEXT"` + ID int64 + TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` + OutputKey string `xorm:"VARCHAR(255) UNIQUE(task_id_output_key)"` + OutputValue string `xorm:"TEXT"` } // FindTaskOutputByTaskID returns the outputs of the task. @@ -37,20 +34,17 @@ func FindTaskOutputKeyByTaskID(ctx context.Context, taskID int64) ([]string, err // InsertTaskOutputIfNotExist inserts a new task output if it does not exist. func InsertTaskOutputIfNotExist(ctx context.Context, taskID int64, key, value string) error { - keyHash := fmt.Sprintf("%x", sha256.Sum256([]byte(key))) - keyHash = keyHash[:32] // 32 chars (16 bytes) is enough to avoid collision inner a task return db.WithTx(ctx, func(ctx context.Context) error { sess := db.GetEngine(ctx) - if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, OutputKeyHash: keyHash}); err != nil { + if exist, err := sess.Exist(&ActionTaskOutput{TaskID: taskID, OutputKey: key}); err != nil { return err } else if exist { return nil } _, err := sess.Insert(&ActionTaskOutput{ - TaskID: taskID, - OutputKey: key, - OutputKeyHash: keyHash, - OutputValue: value, + TaskID: taskID, + OutputKey: key, + OutputValue: value, }) return err }) diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v254.go index 0bd5f78c3eabc..d44042a8b616a 100644 --- a/models/migrations/v1_20/v254.go +++ b/models/migrations/v1_20/v254.go @@ -9,11 +9,10 @@ import ( func AddActionTaskOutputTable(x *xorm.Engine) error { type ActionTaskOutput struct { - ID int64 - TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` - OutputKey string `xorm:"VARCHAR(255)"` - OutputKeyHash string `xorm:"CHAR(32) UNIQUE(task_id_output_key)"` - OutputValue string `xorm:"TEXT"` + ID int64 + TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` + OutputKey string `xorm:"VARCHAR(255) UNIQUE(task_id_output_key)"` + OutputValue string `xorm:"TEXT"` } return x.Sync(new(ActionTaskOutput)) } diff --git a/routers/api/actions/runner/utils.go b/routers/api/actions/runner/utils.go index f9e9263edadb7..705867a9b1067 100644 --- a/routers/api/actions/runner/utils.go +++ b/routers/api/actions/runner/utils.go @@ -168,7 +168,7 @@ func findTaskNeeds(ctx context.Context, task *actions_model.ActionTask) (map[str return nil, fmt.Errorf("FindTaskOutputByTaskID: %w", err) } for _, v := range got { - outputs[v.OutputKey] = v.OutputKeyHash + outputs[v.OutputKey] = v.OutputValue } ret[job.JobID] = &runnerv1.TaskNeed{ Outputs: outputs, From d6db381c2ee6b05ef62f0eb326676cb4f053ca05 Mon Sep 17 00:00:00 2001 From: Jason Song Date: Thu, 20 Apr 2023 18:58:15 +0800 Subject: [PATCH 13/13] fix: limitation for value --- models/actions/task_output.go | 2 +- models/migrations/v1_20/v254.go | 2 +- routers/api/actions/runner/runner.go | 9 +++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/models/actions/task_output.go b/models/actions/task_output.go index 9957c5b185d20..5291a783d6697 100644 --- a/models/actions/task_output.go +++ b/models/actions/task_output.go @@ -17,7 +17,7 @@ type ActionTaskOutput struct { ID int64 TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` OutputKey string `xorm:"VARCHAR(255) UNIQUE(task_id_output_key)"` - OutputValue string `xorm:"TEXT"` + OutputValue string `xorm:"MEDIUMTEXT"` } // FindTaskOutputByTaskID returns the outputs of the task. diff --git a/models/migrations/v1_20/v254.go b/models/migrations/v1_20/v254.go index d44042a8b616a..1e26979a5b2a2 100644 --- a/models/migrations/v1_20/v254.go +++ b/models/migrations/v1_20/v254.go @@ -12,7 +12,7 @@ func AddActionTaskOutputTable(x *xorm.Engine) error { ID int64 TaskID int64 `xorm:"INDEX UNIQUE(task_id_output_key)"` OutputKey string `xorm:"VARCHAR(255) UNIQUE(task_id_output_key)"` - OutputValue string `xorm:"TEXT"` + OutputValue string `xorm:"MEDIUMTEXT"` } return x.Sync(new(ActionTaskOutput)) } diff --git a/routers/api/actions/runner/runner.go b/routers/api/actions/runner/runner.go index ebb9130b42417..73c6b746a0fef 100644 --- a/routers/api/actions/runner/runner.go +++ b/routers/api/actions/runner/runner.go @@ -150,6 +150,15 @@ func (s *Service) UpdateTask( log.Warn("Ignore the output of task %d because the key is too long: %q", task.ID, k) continue } + // The value can be a maximum of 1 MB + if l := len(v); l > 1024*1024 { + log.Warn("Ignore the output %q of task %d because the value is too long: %v", k, task.ID, l) + continue + } + // There's another limitation on GitHub that the total of all outputs in a workflow run can be a maximum of 50 MB. + // We don't check the total size here because it's not easy to do, and it doesn't really worth it. + // See https://docs.github.com/en/actions/using-jobs/defining-outputs-for-jobs + if err := actions_model.InsertTaskOutputIfNotExist(ctx, task.ID, k, v); err != nil { log.Warn("Failed to insert the output %q of task %d: %v", k, task.ID, err) // It's ok not to return errors, the runner will resend the outputs.