Skip to content

Commit 4194dfd

Browse files
zylzulugaiksaya
authored and
peterzhuamazon
committed
Promote min and bundle artifacts to release-candidate after every bundle build (opensearch-project#1378)
* Promote to release candidate after every bundle build Signed-off-by: Yilin Zhang <[email protected]> Signed-off-by: Sayali Gaikawad <[email protected]> Co-authored-by: Sayali Gaikawad <[email protected]>
1 parent 82eb8e9 commit 4194dfd

11 files changed

+95
-14
lines changed

‎jenkins/opensearch/distribution-build.jenkinsfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ pipeline {
1919
}
2020
}
2121
steps {
22-
script {
22+
script {
2323
dockerAgent = detectDockerAgent()
2424
currentBuild.description = "$INPUT_MANIFEST"
2525
}

‎src/jenkins/BuildManifest.groovy

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -25,30 +25,72 @@ class BuildManifest implements Serializable {
2525
String getFilename() {
2626
return this.name.toLowerCase().replaceAll(' ', '-')
2727
}
28+
29+
String getPackageName() {
30+
return [
31+
this.getFilename(),
32+
this.version,
33+
this.platform,
34+
this.architecture,
35+
].join('-') + '.tar.gz'
36+
}
37+
}
38+
39+
class Components extends HashMap<String, Component> {
40+
41+
Components(ArrayList data) {
42+
data.each { item ->
43+
Component component = new Component(item)
44+
this[component.name] = component
45+
}
46+
}
47+
}
48+
49+
class Component implements Serializable {
50+
String name
51+
String version
52+
String ref
53+
String commit_id
54+
String repository
55+
Map<String, ArrayList> artifacts
56+
57+
Component(Map data) {
58+
this.name = data.name
59+
this.version = data.version
60+
this.ref = data.ref
61+
this.commit_id = data.commit_id
62+
this.repository = data.repository
63+
this.artifacts = new HashMap<>(data.artifacts)
64+
}
65+
2866
}
2967

3068
Build build
69+
Components components
3170

3271
BuildManifest(Map data) {
3372
this.build = new BuildManifest.Build(data.build)
73+
this.components = new BuildManifest.Components(data.components)
3474
}
3575

3676
public String getArtifactRoot(String jobName, String buildNumber) {
3777
return [
38-
jobName,
39-
this.build.version,
40-
buildNumber,
41-
this.build.platform,
42-
this.build.architecture
78+
jobName,
79+
this.build.version,
80+
buildNumber,
81+
this.build.platform,
82+
this.build.architecture
4383
].join("/")
4484
}
4585

4686
public String getArtifactRootUrl(String publicArtifactUrl = 'https://ci.opensearch.org/ci/dbc', String jobName, String buildNumber) {
4787
return [
48-
publicArtifactUrl,
49-
this.getArtifactRoot(jobName, buildNumber)
88+
publicArtifactUrl,
89+
this.getArtifactRoot(jobName, buildNumber)
5090
].join('/')
5191
}
52-
}
53-
5492

93+
public String getMinArtifact() {
94+
components.get(build.name.replace(' ','-'))?.artifacts?.get("dist")?.first()
95+
}
96+
}

‎tests/jenkins/TestArchiveAssembleUpload.groovy

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ class TestArchiveAssembleUpload extends BuildPipelineTest {
2424
binding.setVariable('STAGE_NAME', 'stage')
2525
binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
2626
binding.setVariable('BUILD_NUMBER', '33')
27+
binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
28+
binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
29+
binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
2730
binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
2831

2932
helper.registerAllowedMethod("s3Upload", [Map])

‎tests/jenkins/TestAssembleUpload.groovy

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ class TestAssembleUpload extends BuildPipelineTest {
2222
binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
2323
binding.setVariable('STAGE_NAME', 'stage')
2424
binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
25+
binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
26+
binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
27+
binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
2528
binding.setVariable('BUILD_NUMBER', '33')
2629
binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
2730

‎tests/jenkins/TestBuildAssembleUpload.groovy

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ class TestBuildAssembleUpload extends BuildPipelineTest {
2424
binding.setVariable('ARTIFACT_BUCKET_NAME', 'artifact-bucket')
2525
binding.setVariable('AWS_ACCOUNT_PUBLIC', 'account')
2626
binding.setVariable('STAGE_NAME', 'stage')
27+
binding.setVariable('ARTIFACT_PROMOTION_ROLE_NAME', 'role')
28+
binding.setVariable('AWS_ACCOUNT_ARTIFACT', 'dummy')
29+
binding.setVariable('ARTIFACT_PRODUCTION_BUCKET_NAME', 'bucket')
2730
binding.setVariable('ARTIFACT_UPLOAD_ROLE_NAME', 'upload_role')
2831

2932
helper.registerAllowedMethod("withCredentials", [List, Closure], { list, closure ->

‎tests/jenkins/jobs/ArchiveAssembleUpload_Jenkinsfile.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
2828
uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml})
2929
BuildManifest.asBoolean()
30+
BuildManifest.getMinArtifact()
3031
BuildManifest.getArtifactRoot(vars-build, 33)
3132
uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
3233
uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds})
@@ -35,6 +36,10 @@
3536
uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
3637
uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
3738
uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
39+
uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
40+
uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
41+
uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
42+
uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
3843
BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
3944
Messages.asBoolean()
4045
Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

‎tests/jenkins/jobs/AssembleUpload_Jenkinsfile.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
1919
uploadArtifacts.readYaml({file=tests/data/opensearch-build-1.1.0.yml})
2020
BuildManifest.asBoolean()
21+
BuildManifest.getMinArtifact()
2122
BuildManifest.getArtifactRoot(vars-build, 33)
2223
uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
2324
uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds})
@@ -26,6 +27,10 @@
2627
uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
2728
uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
2829
uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
30+
uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
31+
uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
32+
uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
33+
uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
2934
BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
3035
Messages.asBoolean()
3136
Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

‎tests/jenkins/jobs/BuildAssembleUpload_Jenkinsfile.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
uploadArtifacts.library({identifier=jenkins@20211123, retriever=null})
2626
uploadArtifacts.readYaml({file=builds/opensearch/manifest.yml})
2727
BuildManifest.asBoolean()
28+
BuildManifest.getMinArtifact()
2829
BuildManifest.getArtifactRoot(vars-build, 33)
2930
uploadArtifacts.echo(Uploading to s3://artifact-bucket/vars-build/1.1.0/33/linux/x64)
3031
uploadArtifacts.uploadToS3({sourcePath=builds, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/builds})
@@ -33,6 +34,10 @@
3334
uploadArtifacts.uploadToS3({sourcePath=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
3435
uploadToS3.withAWS({role=upload_role, roleAccount=account, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
3536
uploadToS3.s3Upload({file=dist, bucket=artifact-bucket, path=vars-build/1.1.0/33/linux/x64/dist})
37+
uploadArtifacts.echo(Uploading to s3://bucket/vars-build/1.1.0/33/linux/x64)
38+
uploadArtifacts.withAWS({role=role, roleAccount=dummy, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
39+
uploadArtifacts.s3Upload({file=builds/opensearch/dist/opensearch-min-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/core/opensearch/1.1.0/})
40+
uploadArtifacts.s3Upload({file=dist/opensearch/opensearch-1.1.0-linux-x64.tar.gz, bucket=bucket, path=release-candidates/bundle/opensearch/1.1.0/})
3641
BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/dbc, vars-build, 33)
3742
Messages.asBoolean()
3843
Messages.add(stage, https://ci.opensearch.org/dbc/vars-build/1.1.0/33/linux/x64/builds/opensearch/manifest.yml

‎tests/jenkins/jobs/BuildManifest_Jenkinsfile

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* this file be licensed under the Apache-2.0 license or a
66
* compatible open source license.
77
*/
8-
8+
99
def lib = library("jenkins")
1010

1111
pipeline {
@@ -22,8 +22,10 @@ pipeline {
2222
echo buildManifest.build.getFilename()
2323
echo buildManifest.getArtifactRoot('bundle-build', '1')
2424
echo buildManifest.getArtifactRootUrl('https://ci.opensearch.org/ci/dbc', 'bundle-build', '1')
25+
echo buildManifest.build.getPackageName()
26+
echo buildManifest.components.getMinArtifact()
2527
}
2628
}
2729
}
2830
}
29-
}
31+
}

‎tests/jenkins/jobs/BuildManifest_Jenkinsfile.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,6 @@
1515
BuildManifest_Jenkinsfile.echo(bundle-build/1.1.0/1/linux/x64)
1616
BuildManifest.getArtifactRootUrl(https://ci.opensearch.org/ci/dbc, bundle-build, 1)
1717
BuildManifest_Jenkinsfile.echo(https://ci.opensearch.org/ci/dbc/bundle-build/1.1.0/1/linux/x64)
18+
BuildManifest_Jenkinsfile.echo(opensearch-1.1.0-linux-x64.tar.gz)
19+
BuildManifest.getMinArtifact()
20+
BuildManifest_Jenkinsfile.echo(dist/opensearch-min-1.1.0-linux-x64.tar.gz)

‎vars/uploadArtifacts.groovy

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@ void call(Map args = [:]) {
22
def lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
33

44
def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.manifest))
5+
def minArtifactPath = buildManifest.getMinArtifact()
6+
def productFilename = buildManifest.build.getFilename()
7+
def packageName = buildManifest.build.getPackageName()
58

69
def artifactPath = buildManifest.getArtifactRoot("${JOB_NAME}", "${BUILD_NUMBER}")
710
echo "Uploading to s3://${ARTIFACT_BUCKET_NAME}/${artifactPath}"
@@ -18,10 +21,17 @@ void call(Map args = [:]) {
1821
path: "${artifactPath}/dist"
1922
)
2023

24+
echo "Uploading to s3://${ARTIFACT_PRODUCTION_BUCKET_NAME}/${artifactPath}"
25+
26+
withAWS(role: "${ARTIFACT_PROMOTION_ROLE_NAME}", roleAccount: "${AWS_ACCOUNT_ARTIFACT}", duration: 900, roleSessionName: 'jenkins-session') {
27+
s3Upload(file: "builds/${productFilename}/${minArtifactPath}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/core/${productFilename}/${buildManifest.build.version}/")
28+
s3Upload(file: "dist/${productFilename}/${packageName}", bucket: "${ARTIFACT_PRODUCTION_BUCKET_NAME}", path: "release-candidates/bundle/${productFilename}/${buildManifest.build.version}/")
29+
}
30+
2131
def baseUrl = buildManifest.getArtifactRootUrl("${PUBLIC_ARTIFACT_URL}", "${JOB_NAME}", "${BUILD_NUMBER}")
2232
lib.jenkins.Messages.new(this).add("${STAGE_NAME}", [
23-
"${baseUrl}/builds/${buildManifest.build.getFilename()}/manifest.yml",
24-
"${baseUrl}/dist/${buildManifest.build.getFilename()}/manifest.yml"
33+
"${baseUrl}/builds/${productFilename}/manifest.yml",
34+
"${baseUrl}/dist/${productFilename}/manifest.yml"
2535
].join('\n')
2636
)
2737
}

0 commit comments

Comments
 (0)