Merge pull request #790 from jettero/onto-4.0

s3-fix merge into 4.0 branch

Author: jettero

hubblestack/extmods/fileserver/s3fs.py

@@ -105,7 +105,7 @@
 
 log = logging.getLogger(__name__)
 
-S3_CACHE_EXPIRE = 30  # cache for 30 seconds
+S3_CACHE_EXPIRE = 1800  # cache for 30 minutes
 S3_SYNC_ON_UPDATE = True  # sync cache on update rather than jit
 
 
@@ -336,6 +336,7 @@ def _get_s3_key():
         'service_url': None,
         'keyid': None,
         'key': None,
+        'cache_expire': S3_CACHE_EXPIRE,
     }
 
     ret = dict()
@@ -351,14 +352,17 @@ def _get_s3_key():
 
     return ret
 
+
 def _init():
     """
     Connect to S3 and download the metadata for each file in all buckets
     specified and cache the data to disk.
     """
     cache_file = _get_buckets_cache_filename()
-    exp = time.time() - S3_CACHE_EXPIRE
+    cache_expire_time = float(_get_s3_key().get('cache_expire'))
+    exp = time.time() - cache_expire_time
 
+    log.debug('S3 cache expire time is %ds', cache_expire_time)
     # check mtime of the buckets files cache
     metadata = None
     try:
@@ -443,6 +447,9 @@ def __get_s3_meta(bucket, key=s3_key_kwargs['key'], keyid=s3_key_kwargs['keyid']
                                         path_style=s3_key_kwargs['path_style'],
                                         https_enable=s3_key_kwargs['https_enable'],
                                         params={'marker': marker})
+            if not tmp:
+                return None
+
             headers = []
             for header in tmp:
                 if 'Key' in header:

hubblestack/extmods/utils/s3.py

@@ -17,6 +17,7 @@
     HAS_REQUESTS = False  # pylint: disable=W0612
 
 # Import Salt libs
+import os
 import salt.utils.aws
 import salt.utils.files
 import salt.utils.hashutils
@@ -203,6 +204,11 @@ def query(key, keyid, method='GET', params=None, headers=None,
             err_code = 'http-{0}'.format(result.status_code)
             err_msg = err_text
 
+    if os.environ.get('MOCK_SLOW_DOWN'):
+        result.status_code = 503
+        err_code = 'SlowDown'
+        err_msg = 'MOCK_SLOW_DOWN environment variable set. All S3 queries will fail for testing purposes."
+
     log.debug('S3 Response Status Code: %s', result.status_code)
 
     if method == 'PUT':
@@ -219,7 +225,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             log.debug('Uploaded from %s to %s', local_file, path)
         else:
             log.debug('Created bucket %s', bucket)
-        return
+        return None
 
     if method == 'DELETE':
         if not six.text_type(result.status_code).startswith('2'):
@@ -235,7 +241,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             log.debug('Deleted %s from bucket %s', path, bucket)
         else:
             log.debug('Deleted bucket %s', bucket)
-        return
+        return None
 
     # This can be used to save a binary object to disk
     if local_file and method == 'GET':
@@ -250,6 +256,10 @@ def query(key, keyid, method='GET', params=None, headers=None,
         return 'Saved to local file: {0}'.format(local_file)
 
     if result.status_code < 200 or result.status_code >= 300:
+        if err_code in ['SlowDown', 'ServiceUnavailable', 'RequestTimeTooSkewed',
+                        'RequestTimeout', 'OperationAborted', 'InternalError']:
+            log.error('Failed s3 operation: %s, %s', err_code, err_msg)
+            return None
         raise CommandExecutionError(
             'Failed s3 operation. {0}: {1}'.format(err_code, err_msg))
 
@@ -268,7 +278,7 @@ def query(key, keyid, method='GET', params=None, headers=None,
             return ret, requesturl
     else:
         if result.status_code != requests.codes.ok:
-            return
+            return None
         ret = {'headers': []}
         if full_headers:
             ret['headers'] = dict(result.headers)