Skip to content

Commit 4064185

Browse files
peterzhuamazongaiksaya
peterzhuamazon
authored and
gaiksaya
committed
Update incremental build to support qualifier on Jenkins (opensearch-project#585)
Signed-off-by: Peter Zhu <[email protected]>
1 parent 0fbcc01 commit 4064185

7 files changed

+249
-7
lines changed

‎build.gradle

+1-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ jacocoTestReport {
128128
}
129129
}
130130

131-
String version = '8.2.0'
131+
String version = '8.2.1'
132132

133133
task updateVersion {
134134
doLast {

‎tests/data/opensearch-input-3.0.0-alpha1.yml

+213
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
---
2+
schema-version: '1.1'
3+
build:
4+
name: OpenSearch
5+
version: 3.0.0
6+
qualifier: alpha1
7+
ci:
8+
image:
9+
name: opensearchstaging/ci-runner:ci-runner-al2-opensearch-build-v1
10+
args: -e JAVA_HOME=/opt/java/openjdk-23
11+
components:
12+
- name: OpenSearch
13+
repository: https://github.com/opensearch-project/OpenSearch.git
14+
ref: main
15+
- name: common-utils
16+
repository: https://github.com/opensearch-project/common-utils.git
17+
ref: main
18+
platforms:
19+
- linux
20+
- windows
21+
- name: opensearch-learning-to-rank-base
22+
repository: https://github.com/opensearch-project/opensearch-learning-to-rank-base.git
23+
ref: main
24+
platforms:
25+
- linux
26+
- windows
27+
- name: opensearch-remote-metadata-sdk
28+
repository: https://github.com/opensearch-project/opensearch-remote-metadata-sdk.git
29+
ref: main
30+
platforms:
31+
- linux
32+
- windows
33+
- name: job-scheduler
34+
repository: https://github.com/opensearch-project/job-scheduler.git
35+
ref: main
36+
platforms:
37+
- linux
38+
- windows
39+
- name: security
40+
repository: https://github.com/opensearch-project/security.git
41+
ref: main
42+
platforms:
43+
- linux
44+
- windows
45+
- name: k-NN
46+
repository: https://github.com/opensearch-project/k-NN.git
47+
ref: main
48+
platforms:
49+
- linux
50+
- windows
51+
- name: geospatial
52+
repository: https://github.com/opensearch-project/geospatial.git
53+
ref: main
54+
platforms:
55+
- linux
56+
- windows
57+
depends_on:
58+
- job-scheduler
59+
- name: cross-cluster-replication
60+
repository: https://github.com/opensearch-project/cross-cluster-replication.git
61+
ref: main
62+
platforms:
63+
- linux
64+
- windows
65+
depends_on:
66+
- common-utils
67+
- name: ml-commons
68+
repository: https://github.com/opensearch-project/ml-commons.git
69+
ref: main
70+
platforms:
71+
- linux
72+
- windows
73+
depends_on:
74+
- common-utils
75+
- name: neural-search
76+
repository: https://github.com/opensearch-project/neural-search.git
77+
ref: main
78+
platforms:
79+
- linux
80+
- windows
81+
depends_on:
82+
- ml-commons
83+
- k-NN
84+
- name: notifications-core
85+
repository: https://github.com/opensearch-project/notifications.git
86+
ref: main
87+
working_directory: notifications
88+
platforms:
89+
- linux
90+
- windows
91+
depends_on:
92+
- common-utils
93+
- name: notifications
94+
repository: https://github.com/opensearch-project/notifications.git
95+
ref: main
96+
working_directory: notifications
97+
platforms:
98+
- linux
99+
- windows
100+
depends_on:
101+
- common-utils
102+
- name: opensearch-observability
103+
repository: https://github.com/opensearch-project/observability.git
104+
ref: main
105+
platforms:
106+
- linux
107+
- windows
108+
depends_on:
109+
- common-utils
110+
- name: opensearch-reports
111+
repository: https://github.com/opensearch-project/reporting.git
112+
ref: main
113+
platforms:
114+
- linux
115+
- windows
116+
depends_on:
117+
- common-utils
118+
- job-scheduler
119+
- name: sql
120+
repository: https://github.com/opensearch-project/sql.git
121+
ref: main
122+
platforms:
123+
- linux
124+
- windows
125+
depends_on:
126+
- ml-commons
127+
- name: asynchronous-search
128+
repository: https://github.com/opensearch-project/asynchronous-search.git
129+
ref: main
130+
platforms:
131+
- linux
132+
- windows
133+
depends_on:
134+
- common-utils
135+
- name: anomaly-detection
136+
repository: https://github.com/opensearch-project/anomaly-detection.git
137+
ref: main
138+
platforms:
139+
- linux
140+
- windows
141+
depends_on:
142+
- common-utils
143+
- job-scheduler
144+
- name: alerting
145+
repository: https://github.com/opensearch-project/alerting.git
146+
ref: main
147+
platforms:
148+
- linux
149+
- windows
150+
depends_on:
151+
- common-utils
152+
- name: security-analytics
153+
repository: https://github.com/opensearch-project/security-analytics.git
154+
ref: main
155+
platforms:
156+
- linux
157+
- windows
158+
depends_on:
159+
- common-utils
160+
- alerting
161+
- job-scheduler
162+
- name: index-management
163+
repository: https://github.com/opensearch-project/index-management.git
164+
ref: main
165+
platforms:
166+
- linux
167+
- windows
168+
depends_on:
169+
- common-utils
170+
- job-scheduler
171+
- name: performance-analyzer
172+
repository: https://github.com/opensearch-project/performance-analyzer.git
173+
ref: main
174+
platforms:
175+
- linux
176+
- name: custom-codecs
177+
repository: https://github.com/opensearch-project/custom-codecs.git
178+
ref: main
179+
platforms:
180+
- linux
181+
- windows
182+
- name: flow-framework
183+
repository: https://github.com/opensearch-project/flow-framework.git
184+
ref: main
185+
platforms:
186+
- linux
187+
- windows
188+
depends_on:
189+
- common-utils
190+
- opensearch-remote-metadata-sdk
191+
- name: skills
192+
repository: https://github.com/opensearch-project/skills.git
193+
ref: main
194+
platforms:
195+
- linux
196+
- windows
197+
depends_on:
198+
- job-scheduler
199+
- anomaly-detection
200+
- sql
201+
- ml-commons
202+
- name: query-insights
203+
repository: https://github.com/opensearch-project/query-insights.git
204+
ref: main
205+
platforms:
206+
- linux
207+
- windows
208+
- name: opensearch-system-templates
209+
repository: https://github.com/opensearch-project/opensearch-system-templates.git
210+
ref: main
211+
platforms:
212+
- linux
213+
- windows

‎tests/jenkins/TestRetrievePreviousBuild.groovy

+3
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ class TestRetrievePreviousBuild extends BuildPipelineTest {
3939
assertThat(shCommands, hasItems('rm -rf zip && mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip/* /tmp/workspace/zip'))
4040
assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r zip/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')))
4141

42+
assertThat(shCommands, hasItems('rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/3.0.0-alpha1/123123/linux/x64/tar/* /tmp/workspace/tar'))
43+
assertThat(shCommands, not(hasItems('mkdir -p ~/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ ~/.m2/repository/org/')))
44+
4245
def s3DownloadCommands = getCommands('s3Download', 'bucket').findAll {
4346
shCommand -> shCommand.contains('bucket')
4447
}

‎tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile

+7
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,13 @@ pipeline {
2727
platform: "windows",
2828
distributionBuildNumber: "1234"
2929
)
30+
retrievePreviousBuild(
31+
inputManifest: "tests/data/opensearch-input-3.0.0-alpha1.yml",
32+
distribution: "tar",
33+
architecture: "x64",
34+
platform: "linux",
35+
distributionBuildNumber: "123123"
36+
)
3037
}
3138
}
3239
}

‎tests/jenkins/jobs/RetrievePreviousBuild_Jenkinsfile.txt

+15
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,18 @@
3131
downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
3232
downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/2.12.0/1234/windows/x64/zip/, force=true})
3333
retrievePreviousBuild.sh(rm -rf zip && mkdir -p zip && mv -v /tmp/workspace/download/dummy_job/2.12.0/1234/windows/x64/zip/* /tmp/workspace/zip)
34+
RetrievePreviousBuild_Jenkinsfile.retrievePreviousBuild({inputManifest=tests/data/opensearch-input-3.0.0-alpha1.yml, distribution=tar, architecture=x64, platform=linux, distributionBuildNumber=123123})
35+
retrievePreviousBuild.legacySCM(groovy.lang.Closure)
36+
retrievePreviousBuild.library({identifier=jenkins@main, retriever=null})
37+
retrievePreviousBuild.readYaml({file=tests/data/opensearch-input-3.0.0-alpha1.yml})
38+
InputManifest.asBoolean()
39+
retrievePreviousBuild.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
40+
retrievePreviousBuild.withCredentials([ARTIFACT_BUCKET_NAME], groovy.lang.Closure)
41+
retrievePreviousBuild.downloadFromS3({assumedRoleName=opensearch-bundle, roleAccountNumberCred=jenkins-aws-account-public, downloadPath=dummy_job/3.0.0-alpha1/123123/linux/x64/tar/, bucketName=ARTIFACT_BUCKET_NAME, localPath=/tmp/workspace/download, force=true})
42+
downloadFromS3.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_NUMBER})
43+
downloadFromS3.withCredentials([AWS_ACCOUNT_NUMBER], groovy.lang.Closure)
44+
downloadFromS3.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_NUMBER, duration=900, roleSessionName=jenkins-session, region=us-east-1}, groovy.lang.Closure)
45+
downloadFromS3.s3Download({file=/tmp/workspace/download, bucket=ARTIFACT_BUCKET_NAME, path=dummy_job/3.0.0-alpha1/123123/linux/x64/tar/, force=true})
46+
retrievePreviousBuild.sh(rm -rf tar && mkdir -p tar && mv -v /tmp/workspace/download/dummy_job/3.0.0-alpha1/123123/linux/x64/tar/* /tmp/workspace/tar)
47+
retrievePreviousBuild.echo(Setting up Maven Local for OpenSearch build.)
48+
retrievePreviousBuild.sh(mkdir -p $HOME/.m2/repository/org/ && cp -r tar/builds/opensearch/maven/org/opensearch/ $HOME/.m2/repository/org/)

‎vars/buildManifest.groovy

+7-5
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ void call(Map args = [:]) {
2626

2727
def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}"
2828
def version = inputManifestObj.build.version
29+
def qualifier = inputManifestObj.build.qualifier ? '-' + inputManifestObj.build.qualifier : ''
30+
def revision = version + qualifier
2931

3032
def DISTRIBUTION_PLATFORM = args.platform
3133
def DISTRIBUTION_ARCHITECTURE = args.architecture
@@ -35,27 +37,27 @@ void call(Map args = [:]) {
3537

3638
if (incremental_enabled && previousBuildId.equalsIgnoreCase("latest")) {
3739
def latestIndexStatus = sh (
38-
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1",
40+
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\" > /dev/null 2>&1",
3941
returnStatus: true
4042
)
4143
def latestIndexStatusOld = sh (
42-
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\" > /dev/null 2>&1",
44+
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\" > /dev/null 2>&1",
4345
returnStatus: true
4446
)
4547
if (latestIndexStatus == 0) {
4648
echo("Use new URL path for the latest index.")
4749
DISTRIBUTION_BUILD_NUMBER = sh(
48-
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"",
50+
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}/index.json | jq -r \".latest\"",
4951
returnStdout: true
5052
).trim()
5153
} else if (latestIndexStatusOld == 0) {
5254
echo("Use old URL path for the latest index.")
5355
DISTRIBUTION_BUILD_NUMBER = sh(
54-
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${version}/index.json | jq -r \".latest\"",
56+
script: "curl -sL ${PUBLIC_ARTIFACT_URL}/${DISTRIBUTION_JOB_NAME}/${revision}/index.json | jq -r \".latest\"",
5557
returnStdout: true
5658
).trim()
5759
} else {
58-
echo("No latest build for ${version} is available. Building all components from the manifest.")
60+
echo("No latest build for ${revision} is available. Building all components from the manifest.")
5961
incremental_enabled = false
6062
}
6163
} else {

‎vars/retrievePreviousBuild.groovy

+3-1
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,16 @@ void call(Map args = [:]) {
2121

2222
def DISTRIBUTION_JOB_NAME = args.jobName ?: "${JOB_NAME}"
2323
def version = inputManifestObj.build.version
24+
def qualifier = inputManifestObj.build.qualifier ? '-' + inputManifestObj.build.qualifier : ''
25+
def revision = version + qualifier
2426

2527
def DISTRIBUTION_PLATFORM = args.platform
2628
def DISTRIBUTION_ARCHITECTURE = args.architecture
2729
def distribution = args.distribution
2830
def prefixPath = "${WORKSPACE}/download"
2931
def DISTRIBUTION_BUILD_NUMBER = args.distributionBuildNumber
3032

31-
def artifactPath = "${DISTRIBUTION_JOB_NAME}/${version}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}"
33+
def artifactPath = "${DISTRIBUTION_JOB_NAME}/${revision}/${DISTRIBUTION_BUILD_NUMBER}/${DISTRIBUTION_PLATFORM}/${DISTRIBUTION_ARCHITECTURE}/${distribution}"
3234

3335
withCredentials([string(credentialsId: 'jenkins-artifact-bucket-name', variable: 'ARTIFACT_BUCKET_NAME')]) {
3436
downloadFromS3(

0 commit comments

Comments
 (0)