Fix training data discovery (#5)

* Fix package data inclusion

* Fix getting outputs of a erroneously deployed stack

* Remove ResponseMetadata from configuration

* Remove ResponseMetadata from API response

* Update data-capture destination

* Fix contents look up

* Fix typo

* Remove stack outputs cashing

* Add CopyZips function

* Add template to package distribution

* Remove template URL

* Convert traffic_shadowing to package

* Update source bucket

* Rename function parameters

* Update IAM policy

* Update distribution bucket

* Check response status_code before searching through objects

* Check for number of training files first

* Update stub for lambda tests

Author: tidylobster

README.md

@@ -20,31 +20,31 @@ The following rights are required for deploying traffic-shadowing stack.
             "Sid": "VisualEditor0",
             "Effect": "Allow",
             "Action": [
-                "iam:GetRole",
-                "iam:CreateRole",
-                "iam:DeleteRole",
-                "iam:GetRolePolicy",
-                "iam:PutRolePolicy",
-                "iam:DeleteRolePolicy",
-                "iam:PassRole",
                 "lambda:CreateFunction",
-                "lambda:DeleteFunction",
+                "iam:GetRole",
                 "lambda:InvokeFunction",
                 "lambda:ListVersionsByFunction",
+                "iam:CreateRole",
+                "iam:DeleteRole",
                 "lambda:GetFunctionConfiguration",
-                "lambda:PutFunctionConcurrency",
-                "lambda:AddPermission",
-                "lambda:RemovePermission"
-                "lambda:PublishVersion",
+                "iam:PutRolePolicy",
                 "cloudformation:DescribeStacks",
+                "lambda:PutFunctionConcurrency",
+                "iam:PassRole",
                 "cloudformation:DescribeStackEvents",
+                "lambda:AddPermission",
                 "cloudformation:CreateStack",
+                "iam:DeleteRolePolicy",
                 "cloudformation:DeleteStack",
+                "lambda:DeleteFunction",
+                "lambda:PublishVersion",
+                "lambda:RemovePermission",
+                "iam:GetRolePolicy"
             ],
             "Resource": [
-                "arn:aws:iam:::role/*",
-                "arn:aws:cloudformation:::stack/*/*",
-                "arn:aws:lambda:::function:*"
+                "arn:aws:cloudformation:*:*:stack/*/*",
+                "arn:aws:lambda:*:*:function:*",
+                "arn:aws:iam::*:role/*"
             ]
         },
         {
@@ -53,9 +53,10 @@ The following rights are required for deploying traffic-shadowing stack.
             "Action": [
                 "s3:PutBucketNotification",
                 "s3:CreateBucket",
-                "s3:DeleteBucket"
-                "s3:GetBucketLocation",
                 "s3:GetBucketNotification",
+                "s3:GetBucketLocation",
+                "s3:DeleteBucket",
+                "s3:GetObject"
             ],
             "Resource": "*"
         }

aws/traffic_shadowing/src/utils.py

@@ -110,10 +110,15 @@ def get_largest_csv(self, bucket: str, prefix: str, model_name: str) -> str:
         """Parse largest csv file from bucket/prefix."""
         path = '/'.join([prefix, model_name])
         response = self._s3_client.list_objects_v2(Bucket=bucket, Prefix=path)
-        candidates = list(filter(
-            lambda x: os.path.splitext(x['Key'])[1].lower() == '.csv',
-            response.get('Contents')
-        ))
+        if response['ResponseMetadata']['HTTPStatusCode'] != 200:
+            logger.error(response)
+            raise ValueError(f"Could not list objects from s3://{bucket}/{path}")
+        candidates = []
+        if response['KeyCount'] > 0:
+            candidates = list(filter(
+                lambda x: os.path.splitext(x['Key'])[1].lower() == '.csv',
+                response['Contents']
+            ))
         if not candidates:
             raise errors.DataNotFound(f'Didn\'t find .csv training data under "{path}" path')
         candidates.sort(key=lambda x: x['Size'], reverse=True)

aws/traffic_shadowing/tests/stubs/http/aws.py

@@ -24,6 +24,21 @@ def expected_params(self) -> dict:
     @property
     def service_response(self) -> dict:
         return {
+            'ResponseMetadata': {
+                'RequestId': 'xxxxxxxxxxxxxxxx', 
+                'HostId': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 
+                'HTTPStatusCode': 200, 
+                'HTTPHeaders': {
+                    'x-amz-id-2': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 
+                    'x-amz-request-id': 'xxxxxxxxxxxxxxxx', 
+                    'date': 'Wed, 20 May 2020 11:38:02 GMT', 
+                    'x-amz-bucket-region': 'eu-west-3', 
+                    'content-type': 'application/xml', 
+                    'transfer-encoding': 'chunked', 
+                    'server': 'AmazonS3'
+                },
+                'RetryAttempts': 1,
+            },
             'IsTruncated': False,
             'Contents': [
                 {
@@ -41,7 +56,6 @@ def service_response(self) -> dict:
             'KeyCount': 1
         }
 
-
 class GetObjectStub(StubBase):
     method = 'get_object'
 

hydro_integrations/aws/sagemaker/traffic_shadowing/template.yaml

@@ -57,10 +57,10 @@ Resources:
       ServiceToken: !GetAtt CopyZipsFunction.Arn
       DestRegion: !Ref AWS::Region
       DestBucket: !Ref LambdaZipsBucket
-      SourceBucket: hydrosphere-integrations-eu-west-3
+      SourceBucket: hydrosphere-integrations
       Prefix: ""
       Objects:
-      - lambda/traffic_shadowing/1a46812f177b62fc77b02818f04c0ff4
+      - lambda/traffic_shadowing/d2ca18fd62456c06ceba108712f65373
   CopyZipsFunction:
     Type: AWS::Lambda::Function
     Properties:
@@ -128,7 +128,7 @@ Resources:
         production data for analysis.
       Code:
         S3Bucket: !Ref LambdaZipsBucket
-        S3Key: lambda/traffic_shadowing/1a46812f177b62fc77b02818f04c0ff4
+        S3Key: lambda/traffic_shadowing/d2ca18fd62456c06ceba108712f65373
       Timeout: 240
       MemorySize: 256
       Runtime: python3.7