Skip to content

Commit c7fffc4

Browse files
committed
feat: configuration file for sam cli commands (#1502)
* feat: configuration file for sam cli commands - add passthroughs to the CLI command line interface via a configuration file - local defaults are set at: `.aws-sam/samconfig.toml` - This commit contains code copied and modified from https://github.com/phha/click_config_file/blob/master/click_config_file.py under MIT license * fix: add identifier name argument instead of configuration files. * rework: change command key construction logic - descope environment variables - move samconfig.toml back to project root * docstring: TomlProvider class * fix: allow spaces on certain deploy options * fix: REGEX for parameter overrides * fix: infer config path name from ctx * fix: set abs path to `ctx.config_path` * fix: set dirname for config_path not template file name.
1 parent 250941c commit c7fffc4

File tree

20 files changed

+437
-91
lines changed

20 files changed

+437
-91
lines changed

requirements/base.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,4 +13,5 @@ requests==2.22.0
1313
serverlessrepo==0.1.9
1414
aws_lambda_builders==0.6.0
1515
# https://github.com/mhammond/pywin32/issues/1439
16-
pywin32 < 226; sys_platform == 'win32'
16+
pywin32 < 226; sys_platform == 'win32'
17+
toml==0.10.0

requirements/isolated.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ requests==2.22.0
3232
s3transfer==0.2.1
3333
serverlessrepo==0.1.9
3434
six==1.11.0
35+
toml==0.10.0
3536
tzlocal==2.0.0
3637
urllib3==1.25.3
3738
websocket-client==0.56.0

samcli/cli/cli_config_file.py

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
"""
2+
CLI configuration decorator to use TOML configuration files for click commands.
3+
"""
4+
5+
## This section contains code copied and modified from [click_config_file][https://github.com/phha/click_config_file/blob/master/click_config_file.py]
6+
## SPDX-License-Identifier: MIT
7+
8+
import functools
9+
import os
10+
import logging
11+
12+
import click
13+
import toml
14+
15+
__all__ = ("TomlProvider", "configuration_option", "get_ctx_defaults")
16+
17+
LOG = logging.getLogger("samcli")
18+
DEFAULT_CONFIG_FILE_NAME = "samconfig.toml"
19+
DEFAULT_IDENTIFER = "default"
20+
21+
22+
class TomlProvider:
23+
"""
24+
A parser for toml configuration files
25+
:param cmd: sam command name as defined by click
26+
:param section: section defined in the configuration file nested within `cmd`
27+
"""
28+
29+
def __init__(self, section=None):
30+
self.section = section
31+
32+
def __call__(self, file_path, config_env, cmd_name):
33+
"""
34+
Get resolved config based on the `file_path` for the configuration file,
35+
`config_env` targeted inside the config file and corresponding `cmd_name`
36+
as denoted by `click`.
37+
38+
:param file_path: The path to the configuration file
39+
:param config_env: The name of the sectional config_env within configuration file.
40+
:param cmd_name: sam command name as defined by click
41+
:returns dictionary containing the configuration parameters under specified config_env
42+
"""
43+
resolved_config = {}
44+
try:
45+
config = toml.load(file_path)
46+
except Exception as ex:
47+
LOG.error("Error reading configuration file :%s %s", file_path, str(ex))
48+
return resolved_config
49+
if self.section:
50+
try:
51+
resolved_config = self._get_config_env(config, config_env)[cmd_name][self.section]
52+
except KeyError:
53+
LOG.debug(
54+
"Error reading configuration file at %s with config_env %s, command %s, section %s",
55+
file_path,
56+
config_env,
57+
cmd_name,
58+
self.section,
59+
)
60+
return resolved_config
61+
62+
def _get_config_env(self, config, config_env):
63+
"""
64+
65+
:param config: loaded TOML configuration file into dictionary representation
66+
:param config_env: top level section defined within TOML configuration file
67+
:return:
68+
"""
69+
return config.get(config_env, config.get(DEFAULT_IDENTIFER, {}))
70+
71+
72+
def configuration_callback(cmd_name, option_name, config_env_name, saved_callback, provider, ctx, param, value):
73+
"""
74+
Callback for reading the config file.
75+
76+
Also takes care of calling user specified custom callback afterwards.
77+
78+
:param cmd_name: `sam` command name derived from click.
79+
:param option_name: The name of the option. This is used for error messages.
80+
:param config_env_name: `top` level section within configuration file
81+
:param saved_callback: User-specified callback to be called later.
82+
:param provider: A callable that parses the configuration file and returns a dictionary
83+
of the configuration parameters. Will be called as
84+
`provider(file_path, config_env, cmd_name)`.
85+
:param ctx: Click context
86+
:param param: Click parameter
87+
:param value: Specified value for config_env
88+
:returns specified callback or the specified value for config_env.
89+
"""
90+
91+
# ctx, param and value are default arguments for click specified callbacks.
92+
ctx.default_map = ctx.default_map or {}
93+
cmd_name = cmd_name or ctx.info_name
94+
param.default = DEFAULT_IDENTIFER
95+
config_env_name = value or config_env_name
96+
config = get_ctx_defaults(cmd_name, provider, ctx, config_env_name=config_env_name)
97+
ctx.default_map.update(config)
98+
99+
return saved_callback(ctx, param, value) if saved_callback else value
100+
101+
102+
def get_ctx_defaults(cmd_name, provider, ctx, config_env_name=DEFAULT_IDENTIFER):
103+
"""
104+
Get the set of the parameters that are needed to be set into the click command.
105+
This function also figures out the command name by looking up current click context's parent
106+
and constructing the parsed command name that is used in default configuration file.
107+
If a given cmd_name is start-api, the parsed name is "local_start_api".
108+
provider is called with `config_file`, `config_env_name` and `parsed_cmd_name`.
109+
110+
:param cmd_name: `sam` command name
111+
:param provider: provider to be called for reading configuration file
112+
:param ctx: Click context
113+
:param config_env_name: config-env within configuration file
114+
:return: dictionary of defaults for parameters
115+
"""
116+
117+
cwd = getattr(ctx, "config_path", None)
118+
config_file = os.path.join(cwd if cwd else os.getcwd(), DEFAULT_CONFIG_FILE_NAME)
119+
config = {}
120+
if os.path.isfile(config_file):
121+
LOG.debug("Config file location: %s", os.path.abspath(config_file))
122+
123+
# Find parent of current context
124+
_parent = ctx.parent
125+
_cmd_names = []
126+
# Need to find the total set of commands that current command is part of.
127+
if cmd_name != ctx.info_name:
128+
_cmd_names = [cmd_name]
129+
_cmd_names.append(ctx.info_name)
130+
# Go through all parents till a parent of a context exists.
131+
while _parent.parent:
132+
info_name = _parent.info_name
133+
_cmd_names.append(info_name)
134+
_parent = _parent.parent
135+
136+
# construct a parsed name that is of the format: a_b_c_d
137+
parsed_cmd_name = "_".join(reversed([cmd.replace("-", "_").replace(" ", "_") for cmd in _cmd_names]))
138+
139+
config = provider(config_file, config_env_name, parsed_cmd_name)
140+
141+
return config
142+
143+
144+
def configuration_option(*param_decls, **attrs):
145+
"""
146+
Adds configuration file support to a click application.
147+
148+
This will create an option of type `STRING` expecting the config_env in the
149+
configuration file, by default this config_env is `default`. When specified,
150+
the requisite portion of the configuration file is considered as the
151+
source of truth.
152+
153+
The default name of the option is `--config-env`.
154+
155+
This decorator accepts the same arguments as `click.option`.
156+
In addition, the following keyword arguments are available:
157+
:param cmd_name: The command name. Default: `ctx.info_name`
158+
:param config_env_name: The config_env name. This is used to determine which part of the configuration
159+
needs to be read.
160+
:param provider: A callable that parses the configuration file and returns a dictionary
161+
of the configuration parameters. Will be called as
162+
`provider(file_path, config_env, cmd_name)
163+
"""
164+
param_decls = param_decls or ("--config-env",)
165+
option_name = param_decls[0]
166+
167+
def decorator(f):
168+
169+
attrs.setdefault("is_eager", True)
170+
attrs.setdefault("help", "Read config-env from Configuration File.")
171+
attrs.setdefault("expose_value", False)
172+
# --config-env is hidden and can potentially be opened up in the future.
173+
attrs.setdefault("hidden", True)
174+
# explicitly ignore values passed to --config-env, can be opened up in the future.
175+
config_env_name = DEFAULT_IDENTIFER
176+
provider = attrs.pop("provider")
177+
attrs["type"] = click.STRING
178+
saved_callback = attrs.pop("callback", None)
179+
partial_callback = functools.partial(
180+
configuration_callback, None, option_name, config_env_name, saved_callback, provider
181+
)
182+
attrs["callback"] = partial_callback
183+
return click.option(*param_decls, **attrs)(f)
184+
185+
return decorator
186+
187+
188+
# End section copied from [[click_config_file][https://github.com/phha/click_config_file/blob/master/click_config_file.py]

samcli/cli/types.py

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,17 @@ class CfnParameterOverridesType(click.ParamType):
1818
__EXAMPLE_1 = "ParameterKey=KeyPairName,ParameterValue=MyKey ParameterKey=InstanceType,ParameterValue=t1.micro"
1919
__EXAMPLE_2 = "KeyPairName=MyKey InstanceType=t1.micro"
2020

21-
# Regex that parses CloudFormation parameter key-value pairs: https://regex101.com/r/xqfSjW/2
22-
_pattern_1 = r"(?:ParameterKey=([A-Za-z0-9\"]+),ParameterValue=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
23-
_pattern_2 = r"(?:([A-Za-z0-9\"]+)=(\"(?:\\.|[^\"\\]+)*\"|(?:\\.|[^ \"\\]+)+))"
21+
# Regex that parses CloudFormation parameter key-value pairs:
22+
# https://regex101.com/r/xqfSjW/2
23+
# https://regex101.com/r/xqfSjW/5
24+
25+
# If Both ParameterKey pattern and KeyPairName=MyKey, should not be fixed. if they are it can
26+
# result in unpredicatable behavior.
27+
KEY_REGEX = '([A-Za-z0-9\\"]+)'
28+
VALUE_REGEX = '(\\"(?:\\\\.|[^\\"\\\\]+)*\\"|(?:\\\\.|[^ \\"\\\\]+)+))'
29+
30+
_pattern_1 = r"(?:ParameterKey={key},ParameterValue={value}".format(key=KEY_REGEX, value=VALUE_REGEX)
31+
_pattern_2 = r"(?:(?: ){key}={value}".format(key=KEY_REGEX, value=VALUE_REGEX)
2432

2533
ordered_pattern_match = [_pattern_1, _pattern_2]
2634

@@ -34,7 +42,11 @@ def convert(self, value, param, ctx):
3442
if value == ("",):
3543
return result
3644

45+
value = (value,) if isinstance(value, str) else value
3746
for val in value:
47+
val.strip()
48+
# Add empty string to start of the string to help match `_pattern2`
49+
val = " " + val
3850

3951
try:
4052
# NOTE(TheSriram): find the first regex that matched.
@@ -159,6 +171,9 @@ def convert(self, value, param, ctx):
159171
if value == ("",):
160172
return result
161173

174+
# if value comes in a via configuration file, we should still convert it.
175+
# value = (value, ) if not isinstance(value, tuple) else value
176+
162177
for val in value:
163178

164179
groups = re.findall(self._pattern, val)

samcli/commands/_utils/options.py

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from functools import partial
88

99
import click
10+
from click.types import FuncParamType
1011
from samcli.cli.types import CfnParameterOverridesType, CfnMetadataType, CfnTags
1112
from samcli.commands._utils.custom_options.option_nargs import OptionNargs
1213

@@ -43,8 +44,10 @@ def get_or_default_template_file_name(ctx, param, provided_value, include_build)
4344
if os.path.exists(option):
4445
provided_value = option
4546
break
46-
4747
result = os.path.abspath(provided_value)
48+
49+
if ctx:
50+
setattr(ctx, "config_path", os.path.dirname(result))
4851
LOG.debug("Using SAM Template at %s", result)
4952
return result
5053

@@ -74,13 +77,15 @@ def template_click_option(include_build=True):
7477
Click Option for template option
7578
"""
7679
return click.option(
80+
"--template-file",
7781
"--template",
7882
"-t",
7983
default=_TEMPLATE_OPTION_DEFAULT_VALUE,
8084
type=click.Path(),
8185
envvar="SAM_TEMPLATE_FILE",
8286
callback=partial(get_or_default_template_file_name, include_build=include_build),
8387
show_default=True,
88+
is_eager=True,
8489
help="AWS SAM template file",
8590
)
8691

@@ -143,7 +148,7 @@ def capabilities_click_option():
143148
return click.option(
144149
"--capabilities",
145150
cls=OptionNargs,
146-
type=click.STRING,
151+
type=FuncParamType(lambda value: value.split(" ")),
147152
required=True,
148153
help="A list of capabilities that you must specify"
149154
"before AWS Cloudformation can create certain stacks. Some stack tem-"
@@ -182,7 +187,7 @@ def notification_arns_click_option():
182187
return click.option(
183188
"--notification-arns",
184189
cls=OptionNargs,
185-
type=click.STRING,
190+
type=FuncParamType(lambda value: value.split(" ")),
186191
required=False,
187192
help="Amazon Simple Notification Service topic"
188193
"Amazon Resource Names (ARNs) that AWS CloudFormation associates with"

samcli/commands/build/command.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
parameter_override_option
1111
from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options
1212
from samcli.lib.telemetry.metrics import track_command
13+
from samcli.cli.cli_config_file import configuration_option, TomlProvider
1314

1415

1516
LOG = logging.getLogger(__name__)
@@ -53,6 +54,7 @@
5354
"""
5455

5556

57+
@configuration_option(provider=TomlProvider(section="parameters"))
5658
@click.command("build", help=HELP_TEXT, short_help="Build your Lambda function code")
5759
@click.option('--build-dir', '-b',
5860
default=DEFAULT_BUILD_DIR,
@@ -82,7 +84,7 @@
8284
@track_command
8385
def cli(ctx,
8486
function_identifier,
85-
template,
87+
template_file,
8688
base_dir,
8789
build_dir,
8890
use_container,
@@ -95,7 +97,7 @@ def cli(ctx,
9597

9698
mode = _get_mode_value_from_envvar("SAM_BUILD_MODE", choices=["debug"])
9799

98-
do_cli(function_identifier, template, base_dir, build_dir, True, use_container, manifest, docker_network,
100+
do_cli(function_identifier, template_file, base_dir, build_dir, True, use_container, manifest, docker_network,
99101
skip_pull_image, parameter_overrides, mode) # pragma: no cover
100102

101103

samcli/commands/deploy/command.py

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,13 @@
44

55
import click
66

7-
7+
from samcli.cli.cli_config_file import configuration_option, TomlProvider
88
from samcli.commands._utils.options import (
99
parameter_override_option,
1010
capabilities_override_option,
1111
tags_override_option,
1212
notification_arns_override_option,
13+
template_click_option,
1314
)
1415
from samcli.cli.main import pass_context, common_options, aws_creds_options
1516
from samcli.lib.telemetry.metrics import track_command
@@ -27,20 +28,14 @@
2728
"""
2829

2930

31+
@configuration_option(provider=TomlProvider(section="parameters"))
3032
@click.command(
3133
"deploy",
3234
short_help=SHORT_HELP,
3335
context_settings={"ignore_unknown_options": False, "allow_interspersed_args": True, "allow_extra_args": True},
3436
help=HELP_TEXT,
3537
)
36-
@click.option(
37-
"--template-file",
38-
"--template",
39-
"-t",
40-
required=True,
41-
type=click.Path(),
42-
help="The path where your AWS SAM template is located",
43-
)
38+
@template_click_option(include_build=False)
4439
@click.option(
4540
"--stack-name",
4641
required=True,

samcli/commands/init/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
import click
1010

11+
from samcli.cli.cli_config_file import configuration_option, TomlProvider
1112
from samcli.commands.exceptions import UserException
1213
from samcli.cli.main import pass_context, common_options, global_cfg
1314
from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS
@@ -55,6 +56,7 @@
5556
"""
5657

5758

59+
@configuration_option(provider=TomlProvider(section="parameters"))
5860
@click.command(
5961
"init",
6062
help=HELP_TEXT,

0 commit comments

Comments
 (0)