Skip to content

Commit 1f2608b

Browse files
abhinavGupta16abhinavGupta16
authored
Minor bugfix for signArtifacts.groovy (#1437)
* importing the key outside of if condition Signed-off-by: Abhinav Gupta <[email protected]> * added test cases for pgp and sign-artifacts job Signed-off-by: Abhinav Gupta <[email protected]> * moved variables for job in the job test case Signed-off-by: Abhinav Gupta <[email protected]> * add printurl library Signed-off-by: Abhinav Gupta <[email protected]> * added test case for printArtifactUrls Signed-off-by: Abhinav Gupta <[email protected]> * single line download and import key Signed-off-by: Abhinav Gupta <[email protected]> * remove extra line Signed-off-by: Abhinav Gupta <[email protected]> * added regression test file in tests folder Signed-off-by: Abhinav Gupta <[email protected]>
1 parent 84d8da0 commit 1f2608b

11 files changed

+133
-104
lines changed

‎jenkins/sign-artifacts/Jenkinsfile renamed to ‎jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,15 @@ pipeline {
4747
distributionPlatform: DISTRIBUTION_PLATFORM
4848
)
4949

50+
filenamesForUrls = []
51+
52+
println("Note: only supported file types will be signed")
53+
54+
for(filename in downloadedFiles){
55+
filenamesForUrls.add(filename)
56+
filenamesForUrls.add(filename + SIGNATURE_TYPE)
57+
}
58+
5059
finalUploadPath = ([
5160
"${JOB_NAME}",
5261
"${S3_FILE_UPLOAD_PATH}",
@@ -62,7 +71,10 @@ pipeline {
6271
path: finalUploadPath
6372
)
6473

65-
printUrls(downloadedFiles)
74+
printArtifactDownloadUrlsForStaging(
75+
artifactFileNames: filenamesForUrls,
76+
uploadPath: finalUploadPath
77+
)
6678

6779
}
6880
}
@@ -77,19 +89,6 @@ pipeline {
7789
}
7890
}
7991

80-
void printUrls(downloadedFiles){
81-
82-
println("Note: only supported file types will be signed")
83-
84-
for(filename in downloadedFiles){
85-
86-
println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" )
87-
println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" )
88-
89-
}
90-
91-
}
92-
9392
List downloadArtifactsFromUrls() {
9493

9594
listOfURls = URLs.split(",")

‎tests/jenkins/TestPrintArtifactDownloadUrlsForStaging.groovy

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package jenkins.tests
10+
11+
import org.junit.*
12+
13+
14+
class TestPrintArtifactDownloadUrlsForStaging extends BuildPipelineTest {
15+
16+
@Before
17+
void setUp() {
18+
super.setUp()
19+
binding.setVariable('filenamesForUrls', ['dummy_file.tar.gz', 'dummy_file.tar.gz.sig'])
20+
binding.setVariable('UPLOAD_PATH', 'dummy/upload/path')
21+
}
22+
23+
@Test
24+
void testPrintArtifactDownloadUrlsForStaging() {
25+
super.testPipeline("tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile")
26+
}
27+
28+
}

‎tests/jenkins/TestSignArtifacts.groovy

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,28 @@ class TestSignArtifacts extends BuildPipelineTest {
1818
void setUp() {
1919
super.setUp()
2020

21+
binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
22+
binding.setVariable('WORKSPACE', 'workspace')
23+
binding.setVariable('DISTRIBUTION_PLATFORM', 'linux')
24+
binding.setVariable('SIGNATURE_TYPE', '.sig')
25+
26+
binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role')
27+
binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id')
28+
binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket')
29+
binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket')
30+
31+
helper.registerAllowedMethod("git", [Map])
32+
helper.registerAllowedMethod("withCredentials", [Map])
33+
34+
}
35+
36+
@Test
37+
void testSignArtifacts() {
38+
super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile")
39+
}
40+
41+
@Test
42+
void testSignArtifactsJob() {
2143
binding.setVariable('URLs', 'https://www.dummy.com/dummy_1_artifact.tar.gz,' +
2244
' https://www.dummy.com/dummy_2_artifact.tar.gz')
2345
binding.setVariable('S3_FILE_UPLOAD_PATH', '/dummy/upload/path/')
@@ -27,26 +49,14 @@ class TestSignArtifacts extends BuildPipelineTest {
2749
binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'Dummy_Upload_Role')
2850
binding.setVariable('AWS_ACCOUNT_PUBLIC', 'dummy_account')
2951
binding.setVariable('ARTIFACT_BUCKET_NAME', 'dummy_bucket_name')
30-
binding.setVariable('DISTRIBUTION_PLATFORM', 'linux')
31-
binding.setVariable('SIGNATURE_TYPE', '.sig')
32-
binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
33-
binding.setVariable('WORKSPACE', 'workspace')
3452

35-
binding.setVariable('SIGNER_CLIENT_ROLE', 'dummy_signer_client_role')
36-
binding.setVariable('SIGNER_CLIENT_EXTERNAL_ID', 'signer_client_external_id')
37-
binding.setVariable('SIGNER_CLIENT_UNSIGNED_BUCKET', 'signer_client_unsigned_bucket')
38-
binding.setVariable('SIGNER_CLIENT_SIGNED_BUCKET', 'signer_client_signed_bucket')
39-
40-
helper.registerAllowedMethod("git", [Map])
53+
helper.registerAllowedMethod("cleanWs", [Map])
4154
helper.registerAllowedMethod("s3Upload", [Map])
4255
helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
4356
closure.delegate = delegate
4457
return helper.callClosure(closure)
4558
})
46-
}
4759

48-
@Test
49-
void testSignArtifacts() {
50-
super.testPipeline("tests/jenkins/jobs/SignArtifacts_Jenkinsfile")
60+
super.testPipeline("jenkins/sign-artifacts/sign-standalone-artifacts.jenkinsfile", "tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile")
5161
}
5262
}

‎tests/jenkins/jenkinsjob-regression-files/sign-standalone-artifacts.jenkinsfile.txt

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
sign-standalone-artifacts.run()
2+
sign-standalone-artifacts.legacySCM(groovy.lang.Closure)
3+
sign-standalone-artifacts.library({identifier=jenkins@20211123, retriever=null})
4+
sign-standalone-artifacts.pipeline(groovy.lang.Closure)
5+
sign-standalone-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
6+
sign-standalone-artifacts.stage(sign, groovy.lang.Closure)
7+
sign-standalone-artifacts.script(groovy.lang.Closure)
8+
sign-standalone-artifacts.sh(mkdir workspace/artifacts)
9+
sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz)
10+
sign-standalone-artifacts.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz)
11+
sign-standalone-artifacts.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux})
12+
signArtifacts.fileExists(workspace/sign.sh)
13+
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
14+
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
15+
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
16+
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
17+
signArtifacts.sh(
18+
#!/bin/bash
19+
set +x
20+
export ROLE=dummy_signer_client_role
21+
export EXTERNAL_ID=signer_client_external_id
22+
export UNSIGNED_BUCKET=signer_client_unsigned_bucket
23+
export SIGNED_BUCKET=signer_client_signed_bucket
24+
25+
workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null
26+
)
27+
sign-standalone-artifacts.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
28+
uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
29+
uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
30+
sign-standalone-artifacts.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_1_artifact.tar.gz, dummy_1_artifact.tar.gz.sig, dummy_2_artifact.tar.gz, dummy_2_artifact.tar.gz.sig], uploadPath=sign_artifacts_job/dummy/upload/path/20/dist/signed})
31+
sign-standalone-artifacts.script(groovy.lang.Closure)
32+
sign-standalone-artifacts.postCleanup()
33+
postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})

‎tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
pipeline {
2+
agent none
3+
stages {
4+
stage('print_urls') {
5+
steps {
6+
script {
7+
printArtifactDownloadUrlsForStaging(
8+
artifactFileNames: filenamesForUrls,
9+
uploadPath: UPLOAD_PATH
10+
)
11+
}
12+
}
13+
}
14+
}
15+
}

‎tests/jenkins/jobs/PrintArtifactDownloadUrlsForStaging_Jenkinsfile.txt

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.run()
2+
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.pipeline(groovy.lang.Closure)
3+
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.echo(Executing on agent [label:none])
4+
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.stage(print_urls, groovy.lang.Closure)
5+
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.script(groovy.lang.Closure)
6+
PrintArtifactDownloadUrlsForStaging_Jenkinsfile.printArtifactDownloadUrlsForStaging({artifactFileNames=[dummy_file.tar.gz, dummy_file.tar.gz.sig], uploadPath=dummy/upload/path})

‎tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile.txt

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,7 @@
2828
createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/artifacts/vars-build/1.3.0/33/x64/linux/builds/opensearch/core-plugins})
2929
signArtifacts.fileExists(workspace/sign.sh)
3030
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
31-
signArtifacts.fileExists(workspace/opensearch.pgp)
32-
signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
33-
signArtifacts.sh(gpg --import workspace/opensearch.pgp)
31+
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
3432
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
3533
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
3634
signArtifacts.sh(
@@ -53,9 +51,7 @@
5351
createSignatureFiles.signArtifacts({signatureType=.sig, artifactPath=workspace/workspace/file/found.zip})
5452
signArtifacts.fileExists(workspace/sign.sh)
5553
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
56-
signArtifacts.fileExists(workspace/opensearch.pgp)
57-
signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
58-
signArtifacts.sh(gpg --import workspace/opensearch.pgp)
54+
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
5955
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
6056
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
6157
signArtifacts.sh(

‎tests/jenkins/jobs/SignArtifacts_Jenkinsfile

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -4,70 +4,13 @@ pipeline {
44
stage('sign') {
55
steps {
66
script {
7-
if (URLs == '' || S3_FILE_UPLOAD_PATH == '') {
8-
currentBuild.result = 'ABORTED'
9-
error('URLs or S3_FILE_UPLOAD_PATH params are not set')
10-
}
11-
S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('/$', "")
12-
S3_FILE_UPLOAD_PATH = S3_FILE_UPLOAD_PATH.replaceAll('^/+', "")
13-
downloadedFiles = downloadArtifactsFromUrls()
147
signArtifacts(
158
artifactPath: "$WORKSPACE/artifacts/",
169
signatureType: SIGNATURE_TYPE,
1710
distributionPlatform: DISTRIBUTION_PLATFORM
1811
)
19-
20-
finalUploadPath = ([
21-
"${JOB_NAME}",
22-
"${S3_FILE_UPLOAD_PATH}",
23-
"${BUILD_NUMBER}",
24-
"dist",
25-
"signed"
26-
].join('/'))
27-
28-
// uploading artifacts with signatures on s3
29-
uploadToS3(
30-
sourcePath: "$WORKSPACE/artifacts",
31-
bucket: "${ARTIFACT_BUCKET_NAME}",
32-
path: finalUploadPath
33-
)
34-
35-
printUrls(downloadedFiles)
36-
3712
}
3813
}
3914
}
4015
}
4116
}
42-
43-
void printUrls(downloadedFiles){
44-
45-
println("Note: only supported file types will be signed")
46-
47-
for(filename in downloadedFiles){
48-
49-
println("Artifacts can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}" )
50-
println("Signatures can be accessed using the url - https://ci.opensearch.org/ci/dbc/${finalUploadPath}/${filename}${SIGNATURE_TYPE}" )
51-
52-
}
53-
54-
}
55-
56-
List downloadArtifactsFromUrls() {
57-
58-
listOfURls = URLs.split(",")
59-
60-
def downloadedFiles = []
61-
62-
sh "mkdir ${WORKSPACE}/artifacts"
63-
64-
for (url in listOfURls) {
65-
trimmedUrl = url.trim()
66-
filename = trimmedUrl.substring(trimmedUrl.lastIndexOf('/') + 1)
67-
downloadedFiles.add(filename)
68-
sh "curl -SL ${trimmedUrl} -o ${WORKSPACE}/artifacts/${filename}"
69-
}
70-
71-
return downloadedFiles
72-
73-
}

‎tests/jenkins/jobs/SignArtifacts_Jenkinsfile.txt

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,10 @@
33
SignArtifacts_Jenkinsfile.echo(Executing on agent [label:none])
44
SignArtifacts_Jenkinsfile.stage(sign, groovy.lang.Closure)
55
SignArtifacts_Jenkinsfile.script(groovy.lang.Closure)
6-
SignArtifacts_Jenkinsfile.sh(mkdir workspace/artifacts)
7-
SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_1_artifact.tar.gz -o workspace/artifacts/dummy_1_artifact.tar.gz)
8-
SignArtifacts_Jenkinsfile.sh(curl -SL https://www.dummy.com/dummy_2_artifact.tar.gz -o workspace/artifacts/dummy_2_artifact.tar.gz)
96
SignArtifacts_Jenkinsfile.signArtifacts({artifactPath=workspace/artifacts/, signatureType=.sig, distributionPlatform=linux})
107
signArtifacts.fileExists(workspace/sign.sh)
118
signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
12-
signArtifacts.fileExists(workspace/opensearch.pgp)
13-
signArtifacts.sh(curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o workspace/opensearch.pgp)
14-
signArtifacts.sh(gpg --import workspace/opensearch.pgp)
9+
signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
1510
signArtifacts.usernamePassword({credentialsId=github_bot_token_name, usernameVariable=GITHUB_USER, passwordVariable=GITHUB_TOKEN})
1611
signArtifacts.withCredentials([[GITHUB_USER, GITHUB_TOKEN]], groovy.lang.Closure)
1712
signArtifacts.sh(
@@ -24,6 +19,3 @@
2419

2520
workspace/sign.sh workspace/artifacts/ --sigtype=.sig --component=null --type=null
2621
)
27-
SignArtifacts_Jenkinsfile.uploadToS3({sourcePath=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})
28-
uploadToS3.withAWS({role=Dummy_Upload_Role, roleAccount=dummy_account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
29-
uploadToS3.s3Upload({file=workspace/artifacts, bucket=dummy_bucket_name, path=sign_artifacts_job/dummy/upload/path/20/dist/signed})

‎vars/printArtifactDownloadUrlsForStaging.groovy

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
void call(Map args = [:]){
2+
3+
for(filename in args.artifactFileNames){
4+
url = "https://ci.opensearch.org/ci/dbc/${args.uploadPath}/${filename}"
5+
println("File ${filename} can be accessed using the url - ${url}" )
6+
}
7+
8+
}

‎vars/signArtifacts.groovy

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,7 @@ void call(Map args = [:]) {
3030
}
3131

3232
void importPGPKey(){
33-
if( !fileExists("$WORKSPACE/opensearch.pgp")) {
34-
sh("curl -SL https://artifacts.opensearch.org/publickeys/opensearch.pgp -o $WORKSPACE/opensearch.pgp")
35-
sh("gpg --import $WORKSPACE/opensearch.pgp")
36-
}
33+
34+
sh "curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -"
35+
3736
}

0 commit comments

Comments
 (0)