[RPM M1] (Re-open) Update build workflow to build based on distribution (OpenSearch)

scripts/components/OpenSearch/build.sh

@@ -16,11 +16,12 @@ function usage() {
     echo -e "-s SNAPSHOT\t[Optional] Build a snapshot, default is 'false'."
     echo -e "-p PLATFORM\t[Optional] Platform, default is 'uname -s'."
     echo -e "-a ARCHITECTURE\t[Optional] Build architecture, default is 'uname -m'."
+    echo -e "-d DISTRIBUTION\t[Optional] Distribution, default is 'tar'."
     echo -e "-o OUTPUT\t[Optional] Output path, default is 'artifacts'."
     echo -e "-h help"
 }
 
-while getopts ":h:v:s:o:p:a:" arg; do
+while getopts ":h:v:s:o:p:a:d:" arg; do
     case $arg in
         h)
             usage
@@ -41,6 +42,9 @@ while getopts ":h:v:s:o:p:a:" arg; do
         a)
             ARCHITECTURE=$OPTARG
             ;;
+        d)
+            DISTRIBUTION=$OPTARG
+            ;;
         :)
             echo "Error: -${OPTARG} requires an argument"
             usage
@@ -77,48 +81,66 @@ cp -r ./build/local-test-repo/org/opensearch "${OUTPUT}"/maven/org
 
 [ -z "$PLATFORM" ] && PLATFORM=$(uname -s | awk '{print tolower($0)}')
 [ -z "$ARCHITECTURE" ] && ARCHITECTURE=`uname -m`
+[ -z "$DISTRIBUTION" ] && DISTRIBUTION="tar"
 
-case $PLATFORM in
-    linux*)
+case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in
+    linux-tar-x64|darwin-tar-x64)
         PACKAGE="tar"
         EXT="tar.gz"
+        TYPE="archives"
+        TARGET="$PLATFORM-$PACKAGE"
+        QUALIFIER="$PLATFORM-x64"
         ;;
-    darwin*)
+    linux-tar-arm64|darwin-tar-arm64)
         PACKAGE="tar"
         EXT="tar.gz"
+        TYPE="archives"
+        TARGET="$PLATFORM-arm64-$PACKAGE"
+        QUALIFIER="$PLATFORM-arm64"
         ;;
-    windows*)
-        PACKAGE="zip"
-        EXT="zip"
+    linux-rpm-x64)
+        PACKAGE="rpm"
+        EXT="rpm"
+        TYPE="packages"
+        TARGET="rpm"
+        QUALIFIER="x86_64"
         ;;
-    *)
-        echo "Unsupported platform: $PLATFORM"
-        exit 1
+    linux-rpm-arm64)
+        PACKAGE="rpm"
+        EXT="rpm"
+        TYPE="packages"
+        TARGET="arm64-rpm"
+        QUALIFIER="aarch64"
         ;;
-esac
-
-case $ARCHITECTURE in
-    x64)
+    windows-zip-x64)
+        PACKAGE="zip"
+        EXT="zip"
+        TYPE="archives"
         TARGET="$PLATFORM-$PACKAGE"
         QUALIFIER="$PLATFORM-x64"
         ;;
-    arm64)
+    windows-zip-arm64)
+        PACKAGE="zip"
+        EXT="zip"
+        TYPE="archives"
         TARGET="$PLATFORM-arm64-$PACKAGE"
         QUALIFIER="$PLATFORM-arm64"
         ;;
     *)
-        echo "Unsupported architecture: $ARCHITECTURE"
+        echo "Unsupported platform-distribution-architecture combo: $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
         exit 1
         ;;
 esac
 
-./gradlew :distribution:archives:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT
+echo "Building OpenSearch for $PLATFORM-$DISTRIBUTION-$ARCHITECTURE"
+
+./gradlew :distribution:$TYPE:$TARGET:assemble -Dbuild.snapshot=$SNAPSHOT
 
 # Copy artifact to dist folder in bundle build output
 [[ "$SNAPSHOT" == "true" ]] && IDENTIFIER="-SNAPSHOT"
-ARTIFACT_BUILD_NAME=`ls distribution/archives/$TARGET/build/distributions/ | grep "opensearch-min.*$QUALIFIER.$EXT"`
+ARTIFACT_BUILD_NAME=`ls distribution/$TYPE/$TARGET/build/distributions/ | grep "opensearch-min.*$QUALIFIER.$EXT"`
 mkdir -p "${OUTPUT}/dist"
-cp distribution/archives/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME
+cp distribution/$TYPE/$TARGET/build/distributions/$ARTIFACT_BUILD_NAME "${OUTPUT}"/dist/$ARTIFACT_BUILD_NAME
 
 echo "Building core plugins..."
 mkdir -p "${OUTPUT}/core-plugins"

src/build_workflow/build_args.py

@@ -16,13 +16,15 @@ class BuildArgs:
         "x64",
         "arm64",
     ]
+    SUPPORTED_DISTRIBUTIONS = ["tar", "zip", "rpm"]
 
     manifest: IO
     snapshot: bool
     component: str
     keep: bool
     platform: str
     architecture: str
+    distribution: str
 
     def __init__(self) -> None:
         parser = argparse.ArgumentParser(description="Build an OpenSearch Bundle")
@@ -60,6 +62,15 @@ def __init__(self) -> None:
             const=logging.DEBUG,
             dest="logging_level",
         )
+        parser.add_argument(
+            "-d",
+            "--distribution",
+            type=str,
+            choices=self.SUPPORTED_DISTRIBUTIONS,
+            help="Distribution to build.",
+            default=None,
+            dest="distribution"
+        )
 
         args = parser.parse_args()
         self.logging_level = args.logging_level
@@ -70,6 +81,7 @@ def __init__(self) -> None:
         self.keep = args.keep
         self.platform = args.platform
         self.architecture = args.architecture
+        self.distribution = args.distribution
         self.script_path = sys.argv[0].replace("/src/run_build.py", "/build.sh")
 
     def component_command(self, name: str) -> str:

src/build_workflow/build_recorder.py

@@ -60,6 +60,7 @@ def __init__(self, target: BuildTarget):
             self.data["build"]["version"] = target.opensearch_version
             self.data["build"]["platform"] = target.platform
             self.data["build"]["architecture"] = target.architecture
+            self.data["build"]["distribution"] = target.distribution if target.distribution else "tar"
             self.data["schema-version"] = "1.2"
             self.components_hash: Dict[str, Dict[str, Any]] = {}
 
@@ -87,3 +88,4 @@ def to_manifest(self) -> 'BuildManifest':
             if len(components):
                 self.data["components"] = list(components)
             return BuildManifest(self.data)
+

src/build_workflow/build_target.py

@@ -17,6 +17,7 @@ class BuildTarget:
     version: str
     platform: str
     architecture: str
+    distribution: str
     snapshot: bool
     output_dir: str
 
@@ -26,6 +27,7 @@ def __init__(
         patches: List[str] = [],
         platform: str = None,
         architecture: str = None,
+        distribution: str = None,
         name: str = None,
         snapshot: bool = True,
         build_id: str = None,
@@ -37,6 +39,7 @@ def __init__(
         self.patches = patches
         self.snapshot = snapshot
         self.architecture = architecture or current_architecture()
+        self.distribution = distribution
         self.platform = platform or current_platform()
         self.output_dir = output_dir
 

src/build_workflow/builder_from_source.py

@@ -31,20 +31,18 @@ def build(self, build_recorder: BuildRecorder) -> None:
         build_script = ScriptFinder.find_build_script(self.target.name, self.component.name, self.git_repo.working_directory)
 
         build_command = " ".join(
-            [
-                "bash",
-                build_script,
-                "-v",
-                self.target.version,
-                "-p",
-                self.target.platform,
-                "-a",
-                self.target.architecture,
-                "-s",
-                str(self.target.snapshot).lower(),
-                "-o",
-                self.output_path,
-            ]
+            filter(None,
+                [
+                    "bash",
+                    build_script,
+                    f"-v {self.target.version}",
+                    f"-p {self.target.platform}",
+                    f"-a {self.target.architecture}",
+                    f"-d {self.target.distribution}" if self.target.distribution else None,
+                    f"-s {str(self.target.snapshot).lower()}",
+                    f"-o {self.output_path}",
+                ]
+            )
         )
 
         self.git_repo.execute(build_command)

src/manifests/build_manifest.py

@@ -55,6 +55,7 @@ class BuildManifest(ComponentManifest['BuildManifest', 'BuildComponents']):
             "schema": {
                 "platform": {"required": True, "type": "string"},  # added in 1.2
                 "architecture": {"required": True, "type": "string"},
+                "distribution": {"required": False, "type": "string"},
                 "id": {"required": True, "type": "string"},
                 "name": {"required": True, "type": "string"},
                 "version": {"required": True, "type": "string"},
@@ -104,6 +105,7 @@ def __init__(self, data: Any):
             self.version: str = data["version"]
             self.platform: str = data["platform"]
             self.architecture: str = data["architecture"]
+            self.distribution: str = data["distribution"]
             self.id: str = data["id"]
 
         def __to_dict__(self) -> dict:
@@ -112,6 +114,7 @@ def __to_dict__(self) -> dict:
                 "version": self.version,
                 "platform": self.platform,
                 "architecture": self.architecture,
+                "distribution": self.distribution,
                 "id": self.id
             }
 

src/run_build.py

@@ -49,6 +49,7 @@ def main():
             patches=manifest.build.patches,
             snapshot=args.snapshot if args.snapshot is not None else manifest.build.snapshot,
             output_dir=output_dir,
+            distribution=args.distribution,
             platform=args.platform or manifest.build.platform,
             architecture=args.architecture or manifest.build.architecture,
         )