[1.3] Remove zips and download from remote at build time (#337) (#422)

joshuali925 · ps48 · web-flow · commit 64bb6b9f8c6a · 2022-08-16T13:53:18.000-07:00
* Remove zips and download from remote at build time (#337) Signed-off-by: Joshua Li <joshuali925@gmail.com> * Remove odfe bwc test (#408) Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> Signed-off-by: Joshua Li <joshuali925@gmail.com> Signed-off-by: Shenoy Pratik <sgguruda@amazon.com> Co-authored-by: Shenoy Pratik <sgguruda@amazon.com>
diff --git a/.github/workflows/reports-scheduler-test-and-build-workflow.yml b/.github/workflows/reports-scheduler-test-and-build-workflow.yml
@@ -25,13 +25,6 @@ jobs:
       - name: Checkout Reports Scheduler
         uses: actions/checkout@v2
 
-      - name: RunBackwards Compatibility Tests
-        run: |
-          cd reports-scheduler
-          echo "Running backwards compatibility tests ..."
-          ./gradlew bwcTestSuite
-
-
       - name: Build with Gradle
         run: |
           cd reports-scheduler
@@ -54,4 +47,3 @@ jobs:
         with:
           name: reports-scheduler
           path: reports-scheduler-builds
-
diff --git a/reports-scheduler/build.gradle b/reports-scheduler/build.gradle
@@ -240,6 +240,8 @@ Zip bundle = (Zip) project.getTasks().getByName("bundlePlugin");
 integTest.dependsOn(bundle)
 integTest.getClusters().forEach{c -> c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))}
 
+String jobSchedulerURL = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_version.replace("-SNAPSHOT", "") + "/latest/linux/x64/builds/opensearch/plugins/opensearch-job-scheduler-" + opensearch_build.replace("-SNAPSHOT", "") + ".zip"
+
 testClusters.integTest {
     testDistribution = "INTEG_TEST"
     // need to install job-scheduler first, need to assemble job-scheduler first
@@ -249,6 +251,14 @@ testClusters.integTest {
             return new RegularFile() {
                 @Override
                 File getAsFile() {
+                    File dir = new File("src/test/resources/job-scheduler")
+                    if (!dir.exists()) {
+                        dir.mkdirs()
+                    }
+                    File file = new File(dir, "opensearch-job-scheduler-" + opensearch_build + ".zip")
+                    if (!file.exists()) {
+                        new URL(jobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                    }
                     return fileTree("src/test/resources/job-scheduler").getSingleFile()
                 }
             }
@@ -274,19 +284,29 @@ testClusters.integTest {
 String bwcVersion = "1.13.0.0"
 String baseName = "reportsSchedulerBwcCluster"
 String bwcFilePath = "src/test/resources/bwc"
+String bwcJobSchedulerURL = "https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-job-scheduler/opendistro-job-scheduler-" + bwcVersion + ".zip"
+String bwcReportsSchedulerURL = "https://d3g5vo6xdbdb9a.cloudfront.net/downloads/elasticsearch-plugins/opendistro-reports-scheduler/opendistro-reports-scheduler-" + bwcVersion + ".zip"
 
 2.times {i ->
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["7.10.2","1.3.5-SNAPSHOT"]
+            versions = ["7.10.2", opensearch_version]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>(){
                 @Override
                 RegularFile call() throws Exception {
                     return new RegularFile() {
                         @Override
                         File getAsFile() {
+                            File dir = new File(bwcFilePath + "/job-scheduler/" + bwcVersion)
+                            if (!dir.exists()) {
+                                dir.mkdirs()
+                            }
+                            File file = new File(dir, "opendistro-job-scheduler-" + bwcVersion + ".zip")
+                            if (!file.exists()) {
+                                new URL(bwcJobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                            }
                             return fileTree(bwcFilePath + "/job-scheduler/" + bwcVersion).getSingleFile()
                         }
                     }
@@ -298,6 +318,14 @@ String bwcFilePath = "src/test/resources/bwc"
                     return new RegularFile() {
                         @Override
                         File getAsFile() {
+                            File dir = new File(bwcFilePath + "/reports-scheduler/" + bwcVersion)
+                            if (!dir.exists()) {
+                                dir.mkdirs()
+                            }
+                            File file = new File(dir, "opendistro-reports-scheduler-" + bwcVersion + ".zip")
+                            if (!file.exists()) {
+                                new URL(bwcReportsSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                            }
                             return fileTree(bwcFilePath + "/reports-scheduler/" + bwcVersion).getSingleFile()
                         }
                     }
@@ -322,6 +350,14 @@ task prepareBwcTests {
                         return new RegularFile() {
                             @Override
                             File getAsFile() {
+                                File dir = new File(bwcFilePath + "/job-scheduler/" + project.version)
+                                if (!dir.exists()) {
+                                    dir.mkdirs()
+                                }
+                                File file = new File(dir, "opendistro-reports-scheduler-" + project.version + ".zip")
+                                if (!file.exists()) {
+                                    new URL(jobSchedulerURL).withInputStream{ ins -> file.withOutputStream{ it << ins }}
+                                }
                                 return fileTree(bwcFilePath + "/job-scheduler/" + project.version).getSingleFile()
                             }
                         }
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/TestHelpers.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/TestHelpers.kt
@@ -93,4 +93,4 @@ internal fun createReportDefinitionDetails(): ReportDefinitionDetails {
         listOf(),
         reportDefinition = createReportDefinitionObject()
     )
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/CreateReportDefinitionRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/CreateReportDefinitionRequestTests.kt
@@ -98,4 +98,4 @@ internal class CreateReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { CreateReportDefinitionRequest(it) }
         Assertions.assertEquals(reportDefinitionObject, recreatedObject.reportDefinition)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/CreateReportDefinitionResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/CreateReportDefinitionResponseTests.kt
@@ -68,4 +68,4 @@ internal class CreateReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { CreateReportDefinitionResponse.parse(it) }
         Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/DeleteReportDefinitionRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/DeleteReportDefinitionRequestTests.kt
@@ -56,4 +56,4 @@ internal class DeleteReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { DeleteReportDefinitionRequest.parse(it) }
         assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/DeleteReportDefinitionResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/DeleteReportDefinitionResponseTests.kt
@@ -64,4 +64,4 @@ internal class DeleteReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { DeleteReportDefinitionResponse.parse(it) }
         assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportDefinitionsRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportDefinitionsRequestTests.kt
@@ -69,4 +69,4 @@ internal class GetAllReportDefinitionsRequestTests {
         assertEquals(0, recreatedObject.fromIndex)
         assertEquals(defaultMaxItems, recreatedObject.maxItems)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportDefinitionsResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportDefinitionsResponseTests.kt
@@ -132,4 +132,4 @@ internal class GetAllReportDefinitionsResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetAllReportDefinitionsResponse(it) }
         Assertions.assertEquals(reportDefinitionDetailsSearchResults.objectList, recreatedObject.reportDefinitionList.objectList)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportInstancesRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportInstancesRequestTests.kt
@@ -68,4 +68,4 @@ internal class GetAllReportInstancesRequestTests {
         assertEquals(0, recreatedObject.fromIndex)
         assertEquals(defaultMaxItems, recreatedObject.maxItems)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportInstancesResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetAllReportInstancesResponseTests.kt
@@ -103,4 +103,4 @@ internal class GetAllReportInstancesResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetAllReportInstancesResponse(it) }
         Assertions.assertEquals(reportInstanceSearchResults.objectList, recreatedObject.reportInstanceList.objectList)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportDefinitionRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportDefinitionRequestTests.kt
@@ -67,4 +67,4 @@ internal class GetReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportDefinitionRequest.parse(it) }
         assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportDefinitionResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportDefinitionResponseTests.kt
@@ -111,4 +111,4 @@ internal class GetReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportDefinitionResponse(it) }
         assertEquals(reportDefinitionDetails, recreatedObject.reportDefinitionDetails)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportInstanceRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportInstanceRequestTests.kt
@@ -67,4 +67,4 @@ internal class GetReportInstanceRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportInstanceRequest.parse(it) }
         Assertions.assertEquals(reportInstanceId, recreatedObject.reportInstanceId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportInstanceResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/GetReportInstanceResponseTests.kt
@@ -84,4 +84,4 @@ internal class GetReportInstanceResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { GetReportInstanceResponse(it) }
         Assertions.assertEquals(reportInstance, recreatedObject.reportInstance)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/InContextReportCreateRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/InContextReportCreateRequestTests.kt
@@ -80,4 +80,4 @@ internal class InContextReportCreateRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { InContextReportCreateRequest(it) }
         verify(request, recreatedObject)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/InContextReportCreateResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/InContextReportCreateResponseTests.kt
@@ -84,4 +84,4 @@ internal class InContextReportCreateResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { InContextReportCreateResponse(it) }
         Assertions.assertEquals(reportInstance, recreatedObject.reportInstance)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/OnDemandReportCreateRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/OnDemandReportCreateRequestTests.kt
@@ -56,4 +56,4 @@ internal class OnDemandReportCreateRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { OnDemandReportCreateRequest.parse(it) }
         Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/OnDemandReportCreateResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/OnDemandReportCreateResponseTests.kt
@@ -83,4 +83,4 @@ internal class OnDemandReportCreateResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { OnDemandReportCreateResponse(it) }
         Assertions.assertEquals(reportInstance, recreatedObject.reportInstance)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportDefinitionRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportDefinitionRequestTests.kt
@@ -100,4 +100,4 @@ internal class UpdateReportDefinitionRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportDefinitionRequest(it) }
         Assertions.assertEquals(reportDefinitionRequest.reportDefinition, recreatedObject.reportDefinition)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportDefinitionResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportDefinitionResponseTests.kt
@@ -66,4 +66,4 @@ internal class UpdateReportDefinitionResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportDefinitionResponse.parse(it) }
         Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportInstanceStatusRequestTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportInstanceStatusRequestTests.kt
@@ -68,4 +68,4 @@ internal class UpdateReportInstanceStatusRequestTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportInstanceStatusRequest.parse(it) }
         verify(request, recreatedObject)
     }
-}
+}
diff --git a/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportInstanceStatusResponseTests.kt b/reports-scheduler/src/test/kotlin/org/opensearch/reportsscheduler/model/UpdateReportInstanceStatusResponseTests.kt
@@ -67,4 +67,4 @@ internal class UpdateReportInstanceStatusResponseTests {
         val recreatedObject = createObjectFromJsonString(jsonString) { UpdateReportInstanceStatusResponse.parse(it) }
         Assertions.assertEquals(reportInstanceId, recreatedObject.reportInstanceId)
     }
-}
+}
diff --git a/reports-scheduler/src/test/resources/bwc/job-scheduler/.gitignore b/reports-scheduler/src/test/resources/bwc/job-scheduler/.gitignore
@@ -0,0 +1,4 @@
+# ignore artifacts in this directory
+*
+*/
+!.gitignore
diff --git a/reports-scheduler/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip b/reports-scheduler/src/test/resources/bwc/job-scheduler/1.13.0.0/opendistro-job-scheduler-1.13.0.0.zip
diff --git a/reports-scheduler/src/test/resources/bwc/reports-scheduler/.gitignore b/reports-scheduler/src/test/resources/bwc/reports-scheduler/.gitignore
@@ -0,0 +1,4 @@
+# ignore artifacts in this directory
+*
+*/
+!.gitignore
diff --git a/reports-scheduler/src/test/resources/bwc/reports-scheduler/1.13.0.0/opendistro-reports-scheduler-1.13.0.0.zip b/reports-scheduler/src/test/resources/bwc/reports-scheduler/1.13.0.0/opendistro-reports-scheduler-1.13.0.0.zip
diff --git a/reports-scheduler/src/test/resources/job-scheduler/.gitignore b/reports-scheduler/src/test/resources/job-scheduler/.gitignore
@@ -0,0 +1,4 @@
+# ignore artifacts in this directory
+*
+*/
+!.gitignore

Original file line number	Diff line number	Diff line change
`@@ -93,4 +93,4 @@ internal fun createReportDefinitionDetails(): ReportDefinitionDetails {`
`93`	`93`	`listOf(),`
`94`	`94`	`reportDefinition = createReportDefinitionObject()`
`95`	`95`	`)`
`96`		`-}`
	`96`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -98,4 +98,4 @@ internal class CreateReportDefinitionRequestTests {`
`98`	`98`	`val recreatedObject = createObjectFromJsonString(jsonString) { CreateReportDefinitionRequest(it) }`
`99`	`99`	`Assertions.assertEquals(reportDefinitionObject, recreatedObject.reportDefinition)`
`100`	`100`	`}`
`101`		`-}`
	`101`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -68,4 +68,4 @@ internal class CreateReportDefinitionResponseTests {`
`68`	`68`	`val recreatedObject = createObjectFromJsonString(jsonString) { CreateReportDefinitionResponse.parse(it) }`
`69`	`69`	`Assertions.assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)`
`70`	`70`	`}`
`71`		`-}`
	`71`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -56,4 +56,4 @@ internal class DeleteReportDefinitionRequestTests {`
`56`	`56`	`val recreatedObject = createObjectFromJsonString(jsonString) { DeleteReportDefinitionRequest.parse(it) }`
`57`	`57`	`assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)`
`58`	`58`	`}`
`59`		`-}`
	`59`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -64,4 +64,4 @@ internal class DeleteReportDefinitionResponseTests {`
`64`	`64`	`val recreatedObject = createObjectFromJsonString(jsonString) { DeleteReportDefinitionResponse.parse(it) }`
`65`	`65`	`assertEquals(reportDefinitionId, recreatedObject.reportDefinitionId)`
`66`	`66`	`}`
`67`		`-}`
	`67`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -69,4 +69,4 @@ internal class GetAllReportDefinitionsRequestTests {`
`69`	`69`	`assertEquals(0, recreatedObject.fromIndex)`
`70`	`70`	`assertEquals(defaultMaxItems, recreatedObject.maxItems)`
`71`	`71`	`}`
`72`		`-}`
	`72`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -132,4 +132,4 @@ internal class GetAllReportDefinitionsResponseTests {`
`132`	`132`	`val recreatedObject = createObjectFromJsonString(jsonString) { GetAllReportDefinitionsResponse(it) }`
`133`	`133`	`Assertions.assertEquals(reportDefinitionDetailsSearchResults.objectList, recreatedObject.reportDefinitionList.objectList)`
`134`	`134`	`}`
`135`		`-}`
	`135`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -68,4 +68,4 @@ internal class GetAllReportInstancesRequestTests {`
`68`	`68`	`assertEquals(0, recreatedObject.fromIndex)`
`69`	`69`	`assertEquals(defaultMaxItems, recreatedObject.maxItems)`
`70`	`70`	`}`
`71`		`-}`
	`71`	`+}`