Package and deploy in one command (#1532)

* feat: beginnings of package and deploy together

* fix: plumb through configuration

- need to add additional parameters over from package.

* fix: conditional check to print text on package during deploy

* fix: exceptions for `package`

* lint: local variables rule

- `samcli/commands/deploy/command.py` has high number of local
  variables, because of the nature of command, it has high number
  of arguments.

Author: sriram-mv

.pylintrc

@@ -59,7 +59,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=R0201,W0613,W0640,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,C0301
+disable=R0201,W0613,W0640,I0021,I0020,W1618,W1619,R0902,R0903,W0231,W0611,R0913,W0703,C0330,R0204,I0011,R0904,R0914,C0301
 
 
 [REPORTS]

samcli/cli/types.py

@@ -22,8 +22,9 @@ class CfnParameterOverridesType(click.ParamType):
     # https://regex101.com/r/xqfSjW/2
     # https://regex101.com/r/xqfSjW/5
 
-    # If Both ParameterKey pattern and KeyPairName=MyKey, should not be fixed. if they are it can
-    # result in unpredicatable behavior.
+    # If Both ParameterKey pattern and KeyPairName=MyKey should not be present
+    # while adding parameter overrides, if they are, it
+    # can result in unpredicatable behavior.
     KEY_REGEX = '([A-Za-z0-9\\"]+)'
     VALUE_REGEX = '(\\"(?:\\\\.|[^\\"\\\\]+)*\\"|(?:\\\\.|[^ \\"\\\\]+)+))'
 

samcli/commands/_utils/options.py

@@ -122,6 +122,7 @@ def parameter_override_click_option():
         "--parameter-overrides",
         cls=OptionNargs,
         type=CfnParameterOverridesType(),
+        default={},
         help="Optional. A string that contains CloudFormation parameter overrides encoded as key=value "
         "pairs. Use the same format as the AWS CLI, e.g. 'ParameterKey=KeyPairName,"
         "ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro'",
@@ -148,8 +149,7 @@ def capabilities_click_option():
     return click.option(
         "--capabilities",
         cls=OptionNargs,
-        type=FuncParamType(lambda value: value.split(" ")),
-        required=True,
+        type=FuncParamType(lambda value: value.split(" ") if not isinstance(value, tuple) else value),
         help="A list of  capabilities  that  you  must  specify"
         "before  AWS  Cloudformation  can create certain stacks. Some stack tem-"
         "plates might include resources that can affect permissions in your  AWS"
@@ -187,7 +187,7 @@ def notification_arns_click_option():
     return click.option(
         "--notification-arns",
         cls=OptionNargs,
-        type=FuncParamType(lambda value: value.split(" ")),
+        type=FuncParamType(lambda value: value.split(" ") if not isinstance(value, tuple) else value),
         required=False,
         help="Amazon  Simple  Notification  Service  topic"
         "Amazon  Resource  Names  (ARNs) that AWS CloudFormation associates with"

samcli/commands/deploy/command.py

@@ -2,16 +2,19 @@
 CLI command for "deploy" command
 """
 
+import tempfile
+
 import click
 
-from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.commands._utils.options import (
     parameter_override_option,
     capabilities_override_option,
     tags_override_option,
     notification_arns_override_option,
     template_click_option,
+    metadata_override_option,
 )
+from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.cli.main import pass_context, common_options, aws_creds_options
 from samcli.lib.telemetry.metrics import track_command
 
@@ -28,14 +31,14 @@
 """
 
 
-@configuration_option(provider=TomlProvider(section="parameters"))
 @click.command(
     "deploy",
     short_help=SHORT_HELP,
     context_settings={"ignore_unknown_options": False, "allow_interspersed_args": True, "allow_extra_args": True},
     help=HELP_TEXT,
 )
-@template_click_option(include_build=False)
+@configuration_option(provider=TomlProvider(section="parameters"))
+@template_click_option(include_build=True)
 @click.option(
     "--stack-name",
     required=True,
@@ -96,6 +99,14 @@
     "changes to be made to the stack. The default behavior is  to  return  a"
     "non-zero exit code.",
 )
+@click.option(
+    "--use-json",
+    required=False,
+    is_flag=True,
+    help="Indicates whether to use JSON as the format for "
+    "the output AWS CloudFormation template. YAML is used by default.",
+)
+@metadata_override_option
 @notification_arns_override_option
 @tags_override_option
 @parameter_override_option
@@ -118,7 +129,9 @@ def cli(
     role_arn,
     notification_arns,
     fail_on_empty_changeset,
+    use_json,
     tags,
+    metadata,
 ):
 
     # All logic must be implemented in the ``do_cli`` method. This helps with easy unit testing
@@ -135,7 +148,9 @@ def cli(
         role_arn,
         notification_arns,
         fail_on_empty_changeset,
+        use_json,
         tags,
+        metadata,
         ctx.region,
         ctx.profile,
     )  # pragma: no cover
@@ -154,27 +169,47 @@ def do_cli(
     role_arn,
     notification_arns,
     fail_on_empty_changeset,
+    use_json,
     tags,
+    metadata,
     region,
     profile,
 ):
+    from samcli.commands.package.package_context import PackageContext
     from samcli.commands.deploy.deploy_context import DeployContext
 
-    with DeployContext(
-        template_file=template_file,
-        stack_name=stack_name,
-        s3_bucket=s3_bucket,
-        force_upload=force_upload,
-        s3_prefix=s3_prefix,
-        kms_key_id=kms_key_id,
-        parameter_overrides=parameter_overrides,
-        capabilities=capabilities,
-        no_execute_changeset=no_execute_changeset,
-        role_arn=role_arn,
-        notification_arns=notification_arns,
-        fail_on_empty_changeset=fail_on_empty_changeset,
-        tags=tags,
-        region=region,
-        profile=profile,
-    ) as deploy_context:
-        deploy_context.run()
+    with tempfile.NamedTemporaryFile() as output_template_file:
+
+        with PackageContext(
+            template_file=template_file,
+            s3_bucket=s3_bucket,
+            s3_prefix=s3_prefix,
+            output_template_file=output_template_file.name,
+            kms_key_id=kms_key_id,
+            use_json=use_json,
+            force_upload=force_upload,
+            metadata=metadata,
+            on_deploy=True,
+            region=region,
+            profile=profile,
+        ) as package_context:
+            package_context.run()
+
+        with DeployContext(
+            template_file=output_template_file.name,
+            stack_name=stack_name,
+            s3_bucket=s3_bucket,
+            force_upload=force_upload,
+            s3_prefix=s3_prefix,
+            kms_key_id=kms_key_id,
+            parameter_overrides=parameter_overrides,
+            capabilities=capabilities,
+            no_execute_changeset=no_execute_changeset,
+            role_arn=role_arn,
+            notification_arns=notification_arns,
+            fail_on_empty_changeset=fail_on_empty_changeset,
+            tags=tags,
+            region=region,
+            profile=profile,
+        ) as deploy_context:
+            deploy_context.run()

samcli/commands/package/command.py

@@ -1,18 +1,12 @@
 """
 CLI command for "package" command
 """
-from functools import partial
-
 import click
 
-from samcli.cli.cli_config_file import TomlProvider, configuration_option
+
+from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.cli.main import pass_context, common_options, aws_creds_options
-from samcli.commands._utils.options import (
-    metadata_override_option,
-    _TEMPLATE_OPTION_DEFAULT_VALUE,
-    get_or_default_template_file_name,
-    template_click_option,
-)
+from samcli.commands._utils.options import metadata_override_option, template_click_option
 from samcli.commands._utils.resources import resources_generator
 from samcli.lib.telemetry.metrics import track_command
 

samcli/commands/package/exceptions.py

@@ -72,3 +72,12 @@ def __init__(self, template_file, ex):
         super(PackageFailedError, self).__init__(
             message=message_fmt.format(template_file=self.template_file, ex=self.ex)
         )
+
+
+class NoSuchBucketError(UserException):
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+        message_fmt = "\n S3 Bucket does not exist."
+
+        super(NoSuchBucketError, self).__init__(message=message_fmt.format(**self.kwargs))

samcli/commands/package/package_context.py

@@ -34,7 +34,7 @@
 class PackageContext:
 
     MSG_PACKAGED_TEMPLATE_WRITTEN = (
-        "Successfully packaged artifacts and wrote output template "
+        "\nSuccessfully packaged artifacts and wrote output template "
         "to file {output_file_name}."
         "\n"
         "Execute the following command to deploy the packaged template"
@@ -56,6 +56,7 @@ def __init__(
         metadata,
         region,
         profile,
+        on_deploy=False,
     ):
         self.template_file = template_file
         self.s3_bucket = s3_bucket
@@ -67,6 +68,7 @@ def __init__(
         self.metadata = metadata
         self.region = region
         self.profile = profile
+        self.on_deploy = on_deploy
         self.s3_uploader = None
 
     def __enter__(self):
@@ -91,7 +93,7 @@ def run(self):
 
             self.write_output(self.output_template_file, exported_str)
 
-            if self.output_template_file:
+            if self.output_template_file and not self.on_deploy:
                 msg = self.MSG_PACKAGED_TEMPLATE_WRITTEN.format(
                     output_file_name=self.output_template_file,
                     output_file_path=os.path.abspath(self.output_template_file),

samcli/lib/package/artifact_exporter.py

@@ -171,7 +171,6 @@ def zip_folder(folder_path):
     :param folder_path:
     :return: Name of the zipfile
     """
-
     filename = os.path.join(tempfile.gettempdir(), "data-" + uuid.uuid4().hex)
 
     zipfile_name = make_zip(filename, folder_path)
@@ -539,7 +538,6 @@ def __init__(
         """
         Reads the template and makes it ready for export
         """
-
         if not (is_local_folder(parent_dir) and os.path.isabs(parent_dir)):
             raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}".format(parent_dir))
 

samcli/lib/package/s3_uploader.py

@@ -27,19 +27,11 @@
 
 from boto3.s3 import transfer
 
+from samcli.commands.package.exceptions import NoSuchBucketError
 
 LOG = logging.getLogger(__name__)
 
 
-class NoSuchBucketError(Exception):
-    def __init__(self, **kwargs):
-        msg = self.fmt.format(**kwargs)
-        Exception.__init__(self, msg)
-        self.kwargs = kwargs
-
-    fmt = "S3 Bucket does not exist. " "Execute the command to create a new bucket" "\n" "aws s3 mb s3://{bucket_name}"
-
-
 class S3Uploader:
     """
     Class to upload objects to S3 bucket that use versioning. If bucket
@@ -125,7 +117,6 @@ def upload_with_dedup(self, file_name, extension=None):
         # uploads of same object. Uploader will check if the file exists in S3
         # and re-upload only if necessary. So the template points to same file
         # in multiple places, this will upload only once
-
         filemd5 = self.file_checksum(file_name)
         remote_path = filemd5
         if extension:

tests/unit/commands/deploy/test_command.py

@@ -1,5 +1,5 @@
 from unittest import TestCase
-from unittest.mock import patch, Mock
+from unittest.mock import patch, Mock, ANY
 
 from samcli.commands.deploy.command import do_cli
 
@@ -23,13 +23,17 @@ def setUp(self):
         self.metadata = {"abc": "def"}
         self.region = None
         self.profile = None
+        self.use_json = True
+        self.metadata = {}
 
+    @patch("samcli.commands.package.command.click")
+    @patch("samcli.commands.package.package_context.PackageContext")
     @patch("samcli.commands.deploy.command.click")
     @patch("samcli.commands.deploy.deploy_context.DeployContext")
-    def test_all_args(self, deploy_command_context, click_mock):
+    def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_context, mock_package_click):
 
         context_mock = Mock()
-        deploy_command_context.return_value.__enter__.return_value = context_mock
+        mock_deploy_context.return_value.__enter__.return_value = context_mock
 
         do_cli(
             template_file=self.template_file,
@@ -47,10 +51,12 @@ def test_all_args(self, deploy_command_context, click_mock):
             tags=self.tags,
             region=self.region,
             profile=self.profile,
+            use_json=self.use_json,
+            metadata=self.metadata,
         )
 
-        deploy_command_context.assert_called_with(
-            template_file=self.template_file,
+        mock_deploy_context.assert_called_with(
+            template_file=ANY,
             stack_name=self.stack_name,
             s3_bucket=self.s3_bucket,
             force_upload=self.force_upload,

tests/unit/lib/package/test_s3_uploader.py

@@ -7,7 +7,8 @@
 from pathlib import Path
 from botocore.exceptions import ClientError
 
-from samcli.lib.package.s3_uploader import S3Uploader, NoSuchBucketError
+from samcli.commands.package.exceptions import NoSuchBucketError
+from samcli.lib.package.s3_uploader import S3Uploader
 
 
 class TestS3Uploader(TestCase):