Support subfolders for distribution types for build and assemble locally and in Jenkins

.gitignore

@@ -21,6 +21,10 @@ out.txt
 /dist/
 /test-results/
 
+/rpm/
+/tar/
+/zip/
+
 /.vscode/
 
 .project

src/assemble_workflow/bundle_file_location.py

@@ -10,8 +10,8 @@
 
 
 class BundleFileLocation(BundleLocation):
-    def __init__(self, path: str, filename: str) -> None:
-        super().__init__(path, filename)
+    def __init__(self, path: str, filename: str, distribution: str) -> None:
+        super().__init__(path, filename, distribution)
 
     def join(self, *args: str) -> str:
         return os.path.join(self.path, *args)

src/assemble_workflow/bundle_location.py

@@ -8,16 +8,17 @@
 
 
 class BundleLocation(abc.ABC):
-    def __init__(self, path: str, filename: str) -> None:
+    def __init__(self, path: str, filename: str, distribution: str) -> None:
         self.path = path
         self.filename = filename
+        self.distribution = distribution
 
     @abc.abstractmethod
     def join(self, *args: str) -> str:
         pass
 
     def get_build_location(self, target_name: str) -> str:
-        return self.join("builds", self.filename, target_name)
+        return self.join(self.distribution, "builds", self.filename, target_name)
 
     def get_bundle_location(self, target_name: str) -> str:
-        return self.join("dist", self.filename, target_name)
+        return self.join(self.distribution, "dist", self.filename, target_name)

src/assemble_workflow/bundle_locations.py

@@ -13,5 +13,5 @@
 class BundleLocations:
 
     @classmethod
-    def from_path(cls, url_path: str, file_path: str, filename: str) -> BundleLocation:
-        return BundleUrlLocation(url_path, filename) if url_path else BundleFileLocation(file_path, filename)
+    def from_path(cls, url_path: str, file_path: str, filename: str, distribution: str) -> BundleLocation:
+        return BundleUrlLocation(url_path, filename, distribution) if url_path else BundleFileLocation(file_path, filename, distribution)

src/assemble_workflow/bundle_url_location.py

@@ -10,8 +10,8 @@
 
 
 class BundleUrlLocation(BundleLocation):
-    def __init__(self, path: str, filename: str) -> None:
-        super().__init__(path, filename)
+    def __init__(self, path: str, filename: str, distribution: str) -> None:
+        super().__init__(path, filename, distribution)
 
     def join(self, *args: str) -> str:
         sub_path = "/".join(args)

src/build_workflow/build_args.py

@@ -68,7 +68,7 @@ def __init__(self) -> None:
             type=str,
             choices=self.SUPPORTED_DISTRIBUTIONS,
             help="Distribution to build.",
-            default=None,
+            default="tar",
             dest="distribution"
         )
 

src/build_workflow/builder_from_source.py

@@ -45,7 +45,7 @@ def build(self, build_recorder: BuildRecorder) -> None:
                     f"-q {self.target.qualifier}" if self.target.qualifier else None,
                     f"-p {self.target.platform}",
                     f"-a {self.target.architecture}",
-                    f"-d {self.target.distribution}" if self.target.distribution and (self.component.name in DISTRIBUTION_SUPPORTED_COMPONENTS) else None,
+                    f"-d {self.target.distribution}" if self.component.name in DISTRIBUTION_SUPPORTED_COMPONENTS else None,
                     f"-s {str(self.target.snapshot).lower()}",
                     f"-o {self.output_path}",
                 ]

src/jenkins/BuildManifest.groovy

@@ -15,13 +15,15 @@ class BuildManifest implements Serializable {
         String version
         String platform
         String architecture
+        String distribution
 
         Build(Map data) {
             this.id = data.id
             this.name = data.name
             this.version = data.version
             this.platform = data.platform
             this.architecture = data.architecture
+            this.distribution = data.distribution
         }
 
         String getFilename() {
@@ -31,7 +33,17 @@ class BuildManifest implements Serializable {
         String getFilenameWithExtension(String platform = null, String architecture = null) {
             String resolvedPlatform = platform ?: this.platform
             String resolvedArchitecture = architecture ?: this.architecture
-            return "${this.getFilename()}-${this.version}-${resolvedPlatform}-${resolvedArchitecture}.${resolvedPlatform == 'windows' ? 'zip' : 'tar.gz'}"
+            return "${this.getFilename()}-${this.version}-${resolvedPlatform}-${resolvedArchitecture}.${getExtension()}"
+        }
+
+        String getExtension() {
+            if (this.distribution == 'zip') {
+                return 'zip'
+            } else if (this.distribution == 'rpm') {
+                return 'rpm'
+            } else {
+                return 'tar.gz'
+            }
         }
 
         String getPackageName() {
@@ -40,7 +52,7 @@ class BuildManifest implements Serializable {
                     this.version,
                     this.platform,
                     this.architecture,
-            ].join('-') + '.tar.gz'
+            ].join('-') + '.' + getExtension()
         }
     }
 
@@ -87,7 +99,8 @@ class BuildManifest implements Serializable {
                 this.build.version,
                 buildNumber,
                 this.build.platform,
-                this.build.architecture
+                this.build.architecture,
+                this.build.distribution
         ].join("/")
     }
 

src/paths/assemble_output_dir.py

@@ -8,5 +8,5 @@
 
 
 class AssembleOutputDir(OutputDir):
-    def __init__(cls, filename: str, cwd: str = None, makedirs: bool = True) -> None:
-        super().__init__("dist", filename, cwd, makedirs)
+    def __init__(cls, filename: str, distribution: str, cwd: str = None, makedirs: bool = True) -> None:
+        super().__init__("dist", filename, distribution, cwd, makedirs)

src/paths/build_output_dir.py

@@ -9,5 +9,5 @@
 
 
 class BuildOutputDir(OutputDir):
-    def __init__(self, filename: str, cwd: Optional[str] = None, makedirs: bool = True) -> None:
-        super().__init__("builds", filename, cwd, makedirs)
+    def __init__(self, filename: str, distribution: str, cwd: Optional[str] = None, makedirs: bool = True) -> None:
+        super().__init__("builds", filename, distribution, cwd, makedirs)

src/paths/output_dir.py

@@ -10,9 +10,10 @@
 
 
 class OutputDir(ABC):
-    def __init__(cls, parent_dir: str, filename: str, cwd: Optional[str] = None, makedirs: bool = True) -> None:
+    def __init__(cls, parent_dir: str, filename: str, distribution: str, cwd: Optional[str] = None, makedirs: bool = True) -> None:
         cls.dir = os.path.join(
             cwd or os.getcwd(),
+            distribution,
             parent_dir,
             filename
         )

src/run_assemble.py

@@ -28,15 +28,15 @@ def main() -> int:
     build = build_manifest.build
     artifacts_dir = os.path.dirname(os.path.realpath(args.manifest.name))
 
-    output_dir = AssembleOutputDir(build.filename).dir
+    output_dir = AssembleOutputDir(build.filename, build.distribution).dir
 
     logging.info(f"Bundling {build.name} ({build.architecture}) on {build.platform} into {output_dir} ...")
 
     bundle_recorder = BundleRecorder(
         build,
         output_dir,
         artifacts_dir,
-        BundleLocations.from_path(args.base_url, os.getcwd(), build.filename)
+        BundleLocations.from_path(args.base_url, os.getcwd(), build.filename, build.distribution)
     )
 
     with Bundles.create(build_manifest, artifacts_dir, bundle_recorder, args.keep) as bundle:

src/run_build.py

@@ -38,7 +38,7 @@ def main():
             manifest.to_file(args.ref_manifest)
         exit(0)
 
-    output_dir = BuildOutputDir(manifest.build.filename).dir
+    output_dir = BuildOutputDir(manifest.build.filename, args.distribution).dir
 
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
         logging.info(f"Building in {work_dir.name}")

tests/data/opensearch-build-1.1.0.yml

@@ -5,6 +5,7 @@ build:
   id: c3ff7a232d25403fa8cc14c97799c323
   name: OpenSearch
   version: 1.1.0
+  distribution: tar
 components:
   - artifacts:
       dist: