chore: Move schema source/generation out of samtranslator package

Author: aahung

Makefile

@@ -26,7 +26,7 @@ black:
 black-check:
 	# Checking latest schema was generated (run `make schema` if this fails)
 	mkdir -p .tmp
-	python samtranslator/schema/schema.py --sam-schema .tmp/sam.schema.json --cfn-schema samtranslator/schema/cloudformation.schema.json --unified-schema .tmp/schema.json
+	python -m schema_source.schema --sam-schema .tmp/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema .tmp/schema.json
 	diff -u samtranslator/schema/sam.schema.json .tmp/sam.schema.json
 	diff -u samtranslator/schema/schema.json .tmp/schema.json
 	black --check setup.py samtranslator/* tests/* integration/* bin/*.py
@@ -51,21 +51,21 @@ update-schema-data:
 	# Update and parse SAM docs
 	rm -rf .tmp/aws-sam-developer-guide
 	git clone --depth 1 https://github.com/awsdocs/aws-sam-developer-guide.git .tmp/aws-sam-developer-guide
-	bin/parse_docs.py .tmp/aws-sam-developer-guide/doc_source > samtranslator/schema/docs.json
+	bin/parse_docs.py .tmp/aws-sam-developer-guide/doc_source > schema_source/docs.json
 
 	# Update and parse CloudFormation docs
 	rm -rf .tmp/aws-cloudformation-user-guide
 	git clone --depth 1 [email protected]:awsdocs/aws-cloudformation-user-guide.git .tmp/aws-cloudformation-user-guide
-	bin/parse_docs.py --cfn .tmp/aws-cloudformation-user-guide/doc_source > samtranslator/schema/cloudformation-docs.json
+	bin/parse_docs.py --cfn .tmp/aws-cloudformation-user-guide/doc_source > schema_source/cloudformation-docs.json
 
 	# Update CloudFormation schema
 	curl -o .tmp/cloudformation.schema.json https://raw.githubusercontent.com/awslabs/goformation/master/schema/cloudformation.schema.json
 
 	# Add CloudFormation docs to CloudFormation schema
-	python bin/add_docs_cfn_schema.py --schema .tmp/cloudformation.schema.json --docs samtranslator/schema/cloudformation-docs.json > samtranslator/schema/cloudformation.schema.json
+	python bin/add_docs_cfn_schema.py --schema .tmp/cloudformation.schema.json --docs schema_source/cloudformation-docs.json > schema_source/cloudformation.schema.json
 
 schema:
-	python samtranslator/schema/schema.py --sam-schema samtranslator/schema/sam.schema.json --cfn-schema samtranslator/schema/cloudformation.schema.json --unified-schema samtranslator/schema/schema.json
+	python -m schema_source.schema --sam-schema samtranslator/schema/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema samtranslator/schema/schema.json
 
 # Command to run everytime you make changes to verify everything works
 dev: test

requirements/base.txt

@@ -1,4 +1,3 @@
 boto3>=1.19.5,==1.*
 jsonschema<5,>=3.2  # TODO: evaluate risk of removing jsonschema 3.x support
-pydantic~=1.8
 typing_extensions~=4.4.0 # 3.7 doesn't have Literal

requirements/dev.txt

@@ -35,3 +35,6 @@ mypy==0.971
 boto3-stubs[appconfig,serverlessrepo]>=1.19.5,==1.*
 types-PyYAML~=5.4
 types-jsonschema~=3.2
+
+# schema generation, requiring features in >=1.10
+pydantic~=1.10

samtranslator/model/api/api_generator.py

@@ -24,8 +24,8 @@
     InvalidTemplateException,
 )
 from samtranslator.model.s3_utils.uri_parser import parse_s3_uri
+from samtranslator.model.types import PassThrough
 from samtranslator.region_configuration import RegionConfiguration
-from samtranslator.schema.common import PassThrough
 from samtranslator.swagger.swagger import SwaggerEditor
 from samtranslator.model.intrinsics import is_intrinsic, fnSub
 from samtranslator.model.lambda_ import LambdaPermission

samtranslator/model/apigateway.py

@@ -5,8 +5,7 @@
 from samtranslator.model import PropertyType, Resource
 from samtranslator.model.exceptions import InvalidResourceException
 from samtranslator.model.intrinsics import fnSub, ref
-from samtranslator.model.types import IS_DICT, IS_STR, is_type, list_of, one_of
-from samtranslator.schema.common import PassThrough
+from samtranslator.model.types import IS_DICT, IS_STR, is_type, list_of, one_of, PassThrough
 from samtranslator.translator import logical_id_generator
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.utils.py27hash_fix import Py27Dict, Py27UniStr

samtranslator/model/apigatewayv2.py

@@ -1,10 +1,9 @@
 from typing import Any, Dict, List, Optional, Union
 
 from samtranslator.model import PropertyType, Resource
-from samtranslator.model.types import IS_DICT, is_type, one_of, IS_STR, list_of
+from samtranslator.model.types import IS_DICT, is_type, one_of, IS_STR, list_of, PassThrough
 from samtranslator.model.intrinsics import ref, fnSub
 from samtranslator.model.exceptions import ExpectedType, InvalidResourceException
-from samtranslator.schema.common import PassThrough
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.utils.types import Intrinsicable
 from samtranslator.validator.value_validator import sam_expect

samtranslator/model/eventsources/pull.py

@@ -4,8 +4,7 @@
 from samtranslator.metrics.method_decorator import cw_timer
 from samtranslator.model import ResourceMacro, PropertyType, PassThroughProperty
 from samtranslator.model.eventsources import FUNCTION_EVETSOURCE_METRIC_PREFIX
-from samtranslator.model.types import IS_DICT, is_type, IS_STR
-from samtranslator.schema.common import PassThrough
+from samtranslator.model.types import IS_DICT, is_type, IS_STR, PassThrough
 from samtranslator.model.intrinsics import is_intrinsic
 
 from samtranslator.model.lambda_ import LambdaEventSourceMapping

samtranslator/model/eventsources/push.py

@@ -20,7 +20,7 @@
 from samtranslator.model.eventbridge_utils import EventBridgeRuleUtils
 from samtranslator.model.iot import IotTopicRule
 from samtranslator.model.cognito import CognitoUserPool
-from samtranslator.schema.common import PassThrough
+from samtranslator.model.types import PassThrough
 from samtranslator.translator import logical_id_generator
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.model.exceptions import InvalidEventException, InvalidResourceException, InvalidDocumentException

samtranslator/model/sam_resources.py

@@ -62,7 +62,7 @@
     LambdaUrl,
     LambdaPermission,
 )
-from samtranslator.model.types import dict_of, IS_STR, is_type, IS_DICT, list_of, one_of, any_type
+from samtranslator.model.types import dict_of, IS_STR, is_type, IS_DICT, list_of, one_of, any_type, PassThrough
 from samtranslator.translator import logical_id_generator
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.model.intrinsics import (
@@ -80,7 +80,6 @@
 from samtranslator.model.role_utils import construct_role_for_resource
 from samtranslator.model.xray_utils import get_xray_managed_policy_name
 from samtranslator.utils.types import Intrinsicable
-from samtranslator.schema.common import PassThrough
 from samtranslator.validator.value_validator import sam_expect
 
 

samtranslator/model/stepfunctions/events.py

@@ -6,9 +6,8 @@
 from samtranslator.model import Property, PropertyType, ResourceMacro, Resource
 from samtranslator.model.events import EventsRule
 from samtranslator.model.iam import IAMRole, IAMRolePolicies
-from samtranslator.model.types import IS_DICT, IS_STR, is_type
+from samtranslator.model.types import IS_DICT, IS_STR, is_type, PassThrough
 from samtranslator.model.intrinsics import fnSub
-from samtranslator.schema.common import PassThrough
 from samtranslator.translator import logical_id_generator
 from samtranslator.model.exceptions import InvalidEventException
 from samtranslator.model.eventbridge_utils import EventBridgeRuleUtils

samtranslator/model/types.py

@@ -147,3 +147,7 @@ def is_str() -> Validator:
     https://github.com/aws/aws-sam-cli/commit/d18f57c5f39273a04fb582f90e6c5817a4651912
     """
     return IS_STR
+
+
+# Value passed directly to CloudFormation; not used by SAM
+PassThrough = Any  # TODO: Make it behave like typescript's unknown