Merge remote-tracking branch 'upstream/main' into small-fix-ci

Author: zelinh

manifests/1.3.0/opensearch-1.3.0.yml

@@ -26,12 +26,24 @@ components:
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+  - name: alerting
+    repository: https://github.com/opensearch-project/alerting.git
+    ref: main
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version: alerting
   - name: asynchronous-search
     repository: https://github.com/opensearch-project/asynchronous-search.git
     ref: main
     checks:
       - gradle:properties:version
       - gradle:dependencies:opensearch.version
+  - name: index-management
+    repository: https://github.com/opensearch-project/index-management.git
+    ref: main
+    checks:
+      - gradle:properties:version
+      - gradle:dependencies:opensearch.version
   - name: k-NN
     repository: https://github.com/opensearch-project/k-NN.git
     ref: main

src/assemble_workflow/bundle.py

@@ -14,6 +14,7 @@
 
 from assemble_workflow.bundle_recorder import BundleRecorder
 from assemble_workflow.dist import Dist
+from assemble_workflow.dists import Dists
 from manifests.build_manifest import BuildComponent, BuildComponents, BuildManifest
 from paths.script_finder import ScriptFinder
 from system.temporary_directory import TemporaryDirectory
@@ -132,7 +133,8 @@ def __get_min_dist(self, build_components: BuildComponents) -> Dist:
         min_dist_path = self._copy_component(min_bundle, "dist")
         logging.info(f"Copied min bundle to {min_dist_path}.")
         min_path = f"{self.build.filename}-{self.build.version}".replace("-SNAPSHOT", "")
-        min_dist = Dist.from_path(min_bundle.name, min_dist_path, min_path)
+        logging.info(f"Start creating distribution {self.build.distribution} for {min_bundle.name}.")
+        min_dist = Dists.create_dist(min_bundle.name, min_dist_path, min_path, self.build.distribution)
         logging.info(f"Extracting dist into {self.tmp_dir.name}.")
         min_dist.extract(self.tmp_dir.name)
         logging.info(f"Extracted dist into {self.tmp_dir.name}.")

src/assemble_workflow/bundle_recorder.py

@@ -13,11 +13,17 @@
 
 
 class BundleRecorder:
+    EXTENSIONS = {
+        "tar": ".tar.gz",
+        "zip": ".zip",
+    }
+
     def __init__(self, build: BuildManifest.Build, output_dir: str, artifacts_dir: str, bundle_location: BundleLocation) -> None:
         self.output_dir = output_dir
         self.build_id = build.id
         self.bundle_location = bundle_location
         self.version = build.version
+        self.distribution = build.distribution
         self.package_name = self.__get_package_name(build)
         self.artifacts_dir = artifacts_dir
         self.architecture = build.architecture
@@ -27,6 +33,7 @@ def __init__(self, build: BuildManifest.Build, output_dir: str, artifacts_dir: s
             build.version,
             build.platform,
             build.architecture,
+            build.distribution,
             self.__get_package_location(),
         )
 
@@ -37,7 +44,8 @@ def __get_package_name(self, build: BuildManifest.Build) -> str:
             build.platform,
             build.architecture,
         ]
-        return "-".join(parts) + (".zip" if build.platform == "windows" else ".tar.gz")
+        extension = self.EXTENSIONS[self.distribution] if self.distribution else self.EXTENSIONS['tar']
+        return "-".join(parts) + extension
 
     # Assembled output are expected to be served from a separate "dist" folder
     # Example: https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/build-id/linux/x64/dist/
@@ -66,14 +74,15 @@ def write_manifest(self, folder: str) -> None:
         self.get_manifest().to_file(manifest_path)
 
     class BundleManifestBuilder:
-        def __init__(self, build_id: str, name: str, version: str, platform: str, architecture: str, location: str) -> None:
+        def __init__(self, build_id: str, name: str, version: str, platform: str, architecture: str, distribution: str, location: str) -> None:
             self.data: Dict[str, Any] = {}
             self.data["build"] = {}
             self.data["build"]["id"] = build_id
             self.data["build"]["name"] = name
             self.data["build"]["version"] = str(version)
             self.data["build"]["platform"] = platform
             self.data["build"]["architecture"] = architecture
+            self.data["build"]["distribution"] = distribution if distribution else "tar"
             self.data["build"]["location"] = location
             self.data["schema-version"] = "1.1"
             # We need to store components as a hash so that we can append artifacts by component name

src/assemble_workflow/dist.py

@@ -29,7 +29,7 @@ def __extract__(self, dest: str) -> None:
     def __build__(self, name: str, dest: str) -> None:
         pass
 
-    def __find_min_archive_path(self, dest: str) -> str:
+    def find_min_archive_path(self, dest: str) -> str:
         '''
         Return the single folder at the top level of the tar.
         '''
@@ -41,7 +41,7 @@ def __find_min_archive_path(self, dest: str) -> str:
 
         raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), os.path.join(dest, "*"))
 
-    def __rename_archive_path(self, path: str) -> str:
+    def rename_archive_path(self, path: str) -> str:
         '''
         Rename the single folder at the top level of the tar that contains the min distribution to match current version.
         For example, when OpenSearch 1.1.1 is built using the 1.1.0 artifact, we rename opensearch-1.1.0 to opensearch-1.1.1.
@@ -58,8 +58,8 @@ def __rename_archive_path(self, path: str) -> str:
 
     def extract(self, dest: str) -> str:
         self.__extract__(dest)
-        self.archive_path = self.__rename_archive_path(
-            self.__find_min_archive_path(dest)
+        self.archive_path = self.rename_archive_path(
+            self.find_min_archive_path(dest)
         )
         return self.archive_path
 
@@ -69,16 +69,6 @@ def build(self, name: str, dest: str) -> None:
         shutil.copyfile(name, path)
         logging.info(f"Published {path}.")
 
-    @classmethod
-    def from_path(cls, name: str, path: str, min_path: str) -> 'Dist':
-        ext = os.path.splitext(path)[1]
-        if ext == ".gz":
-            return DistTar(name, path, min_path)
-        elif ext == ".zip":
-            return DistZip(name, path, min_path)
-        else:
-            raise ValueError(f'Invalid min "dist" extension in input artifacts: {ext} ({path}).')
-
 
 class DistZip(Dist):
     def __extract__(self, dest: str) -> None:

src/assemble_workflow/dists.py

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+
+from assemble_workflow.dist import Dist, DistTar, DistZip
+
+
+class Dists:
+    DISTRIBUTIONS_MAP = {
+        "tar": DistTar,
+        "zip": DistZip,
+    }
+
+    @classmethod
+    def create_dist(cls, name: str, path: str, min_path: str, distribution: str) -> Dist:
+        if distribution is None:
+            logging.info("Distribution not specified, default to tar")
+            distribution = 'tar'
+
+        return cls.DISTRIBUTIONS_MAP[distribution](name, path, min_path)

src/manifests/build_manifest.py

@@ -22,6 +22,8 @@
   version: string
   platform: linux, darwin or windows
   architecture: x64 or arm64
+  distribution: tar, zip, and rpm
+  id: build id
 components:
   - name: string
     repository: URL of git repository

src/manifests/bundle_manifest.py

@@ -23,6 +23,8 @@ class BundleManifest(ComponentManifest['BundleManifest', 'BundleComponents']):
           version: string
           platform: linux, darwin or windows
           architecture: x64 or arm64
+          distribution: tar, zip, and rpm
+          id: build id
           location: /relative/path/to/tarball
         components:
           - name: string
@@ -39,6 +41,7 @@ class BundleManifest(ComponentManifest['BundleManifest', 'BundleComponents']):
             "schema": {
                 "platform": {"required": True, "type": "string"},  # added in 1.1
                 "architecture": {"required": True, "type": "string"},
+                "distribution": {"type": "string"},
                 "id": {"required": True, "type": "string"},
                 "location": {"required": True, "type": "string"},
                 "name": {"required": True, "type": "string"},
@@ -80,6 +83,7 @@ def __init__(self, data: Dict[str, str]):
             self.version = data["version"]
             self.platform = data["platform"]
             self.architecture = data["architecture"]
+            self.distribution: str = data.get('distribution', None)
             self.location = data["location"]
             self.id = data["id"]
 
@@ -89,10 +93,15 @@ def __to_dict__(self) -> dict:
                 "version": self.version,
                 "platform": self.platform,
                 "architecture": self.architecture,
+                "distribution": self.distribution,
                 "location": self.location,
                 "id": self.id,
             }
 
+        @property
+        def filename(self) -> str:
+            return self.name.lower().replace(" ", "-")
+
 
 class BundleComponents(Components['BundleComponent']):
     @classmethod

tests/tests_assemble_workflow/data/artifacts/dist/opensearch-1.3.0/opensearch.txt

@@ -0,0 +1 @@
+This page intentionally left blank.