chore: Image scanning refactoring (#4802)

* added sql

* updated event type for job in workflow request

* added comments, handling for global cm/cs

* fixed json string global cmcs

* updated global cm/cs spec

* wip

* updated sql script number

* updated sql script number

Author: kartik-579

pkg/pipeline/GlobalCMCSService.go

@@ -36,7 +36,7 @@ func NewGlobalCMCSServiceImpl(logger *zap.SugaredLogger,
 type GlobalCMCSDataUpdateDto struct {
 	Id                 int               `json:"id"`
 	Data               map[string]string `json:"data"  validate:"required"`
-	SecretIngestionFor string            `json:"SecretIngestionFor"` // value can be one of [ci, cd, ci/cd]
+	SecretIngestionFor string            `json:"secretIngestionFor"` // value can be one of [ci, cd, ci/cd]
 	UserId             int32             `json:"-"`
 }
 

pkg/pipeline/WorkflowService.go

@@ -204,7 +204,7 @@ func (impl *WorkflowServiceImpl) appendGlobalCMCS(workflowRequest *types.Workflo
 	var workflowSecrets []bean.ConfigSecretMap
 	if !workflowRequest.IsExtRun {
 		// inject global variables only if IsExtRun is false
-		globalCmCsConfigs, err := impl.globalCMCSService.FindAllActiveByPipelineType(workflowRequest.GetEventTypeForWorkflowRequest())
+		globalCmCsConfigs, err := impl.globalCMCSService.FindAllActiveByPipelineType(workflowRequest.GetPipelineTypeForGlobalCMCS())
 		if err != nil {
 			impl.Logger.Errorw("error in getting all global cm/cs config", "err", err)
 			return nil, nil, err

pkg/pipeline/bean/WorkflowTemplate.go

@@ -37,6 +37,7 @@ const (
 	CI_WORKFLOW_NAME        = "ci"
 	CI_WORKFLOW_WITH_STAGES = "ci-stages-with-env"
 	CiStage                 = "CI"
+	JobStage                = "JOB"
 	CdStage                 = "CD"
 	CD_WORKFLOW_NAME        = "cd"
 	CD_WORKFLOW_WITH_STAGES = "cd-stages-with-env"

pkg/pipeline/types/Workflow.go

@@ -211,8 +211,10 @@ func (workflowRequest *WorkflowRequest) GetWorkflowJson(config *CiCdConfig) ([]b
 
 func (workflowRequest *WorkflowRequest) GetEventTypeForWorkflowRequest() string {
 	switch workflowRequest.Type {
-	case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE:
+	case bean.CI_WORKFLOW_PIPELINE_TYPE:
 		return bean.CiStage
+	case bean.JOB_WORKFLOW_PIPELINE_TYPE:
+		return bean.JobStage
 	case bean.CD_WORKFLOW_PIPELINE_TYPE:
 		return bean.CdStage
 	default:
@@ -222,7 +224,7 @@ func (workflowRequest *WorkflowRequest) GetEventTypeForWorkflowRequest() string
 
 func (workflowRequest *WorkflowRequest) GetWorkflowTypeForWorkflowRequest() string {
 	switch workflowRequest.Type {
-	case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE:
+	case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE: //TODO: separate job as did in eventType, will need changes in wf template for this
 		return bean.CI_WORKFLOW_NAME
 	case bean.CD_WORKFLOW_PIPELINE_TYPE:
 		return bean.CD_WORKFLOW_NAME
@@ -231,6 +233,17 @@ func (workflowRequest *WorkflowRequest) GetWorkflowTypeForWorkflowRequest() stri
 	}
 }
 
+func (workflowRequest *WorkflowRequest) GetPipelineTypeForGlobalCMCS() string {
+	switch workflowRequest.Type {
+	case bean.CI_WORKFLOW_PIPELINE_TYPE, bean.JOB_WORKFLOW_PIPELINE_TYPE:
+		return bean.CiStage //although for job, event type is changed to job from ci but for backward compatibility still sending ci for global cm/cs
+	case bean.CD_WORKFLOW_PIPELINE_TYPE:
+		return bean.CdStage
+	default:
+		return ""
+	}
+}
+
 func (workflowRequest *WorkflowRequest) getContainerEnvVariables(config *CiCdConfig, workflowJson []byte) (containerEnvVariables []v1.EnvVar) {
 	containerEnvVariables = []v1.EnvVar{{Name: bean.IMAGE_SCANNER_ENDPOINT, Value: config.ImageScannerEndpoint}, {Name: "NATS_SERVER_HOST", Value: config.NatsServerHost}}
 	eventEnv := v1.EnvVar{Name: "CI_CD_EVENT", Value: string(workflowJson)}

scripts/sql/232_trivy_alter.down.sql

@@ -0,0 +1,15 @@
+UPDATE scan_tool_metadata
+SET result_descriptor_template = '[{{$size1:= len .Results}}{{range $i1, $v1 := .Results}}{{ if  $v1.Vulnerabilities}}{{$size2:= len $v1.Vulnerabilities}}{{range $i2, $v2 := $v1.Vulnerabilities}}{{if and (eq $i1 (add $size1 -1)) (eq $i2 (add $size2 -1)) }}
+{
+"package": "{{$v2.PkgName}}",
+"packageVersion": "{{$v2.InstalledVersion}}",
+"fixedInVersion": "{{$v2.FixedVersion}}",
+"severity": "{{$v2.Severity}}",
+"name": "{{$v2.VulnerabilityID}}"
+}{{else}}{
+"package": "{{$v2.PkgName}}",
+"packageVersion": "{{$v2.InstalledVersion}}",
+"fixedInVersion": "{{$v2.FixedVersion}}",
+"severity": "{{$v2.Severity}}",
+"name": "{{$v2.VulnerabilityID}}"
+},{{end}}{{end}}{{end}}{{end}}]' where name = 'TRIVY' and version ='V1';

scripts/sql/232_trivy_alter.up.sql

@@ -0,0 +1,12 @@
+UPDATE scan_tool_metadata
+SET result_descriptor_template = '[
+    {
+        "pathToVulnerabilitiesArray": "Results.#.Vulnerabilities",
+        "name": "VulnerabilityID",
+        "package": "PkgName",
+        "packageVersion": "InstalledVersion",
+        "fixedInVersion": "FixedVersion",
+        "severity": "Severity"
+    }
+]' where name = 'TRIVY' and version ='V1';
+

specs/global_cm_cs.yaml

@@ -60,6 +60,13 @@ components:
           type: object
           additionalProperties:
             type: string
+        secretIngestionFor:
+          type: string
+          description: field for defining at where this config is to be ingested. If not set, "CI/CD" will be used as default.
+          enum:
+            - "CI"
+            - "CD"
+            - "CI/CD"
     Error:
       required:
         - code