Merge pull request #847 from jettero/s3-access-denied

S3 access denied

Author: jettero

hubblestack/extmods/utils/s3.py

@@ -22,14 +22,31 @@
 import salt.utils.files
 import salt.utils.hashutils
 import salt.utils.xmlutil as xml
+import time
 from salt._compat import ElementTree as ET
 from salt.exceptions import CommandExecutionError
 from salt.ext.six.moves.urllib.parse import quote as _quote   # pylint: disable=import-error,no-name-in-module
 from salt.ext import six
 
 log = logging.getLogger(__name__)
 
-
+def thirty_second_memoize(f):
+    memo = dict()
+    def inner(*a, **kw):
+        k = '-'.join([ str(x) for x in a ] + [ str(kw[x]) for x in sorted(kw) ])
+        now = time.time()
+        if k in memo:
+            v,t = memo[k]
+            if now - t < 30:
+                log_k = kw.get('path', k)
+                log.info('returning memoized result for %s', log_k)
+                return v
+        v = f(*a, **kw)
+        memo[k] = (v,now)
+        return v
+    return inner
+
+@thirty_second_memoize
 def query(key, keyid, method='GET', params=None, headers=None,
           requesturl=None, return_url=False, bucket=None, service_url=None,
           path='', return_bin=False, action=None, local_file=None,
@@ -243,11 +260,18 @@ def query(key, keyid, method='GET', params=None, headers=None,
             log.debug('Deleted bucket %s', bucket)
         return None
 
+    sortof_ok = ['SlowDown', 'ServiceUnavailable', 'RequestTimeTooSkewed',
+        'RequestTimeout', 'OperationAborted', 'InternalError',
+        'AccessDenied']
+
     # This can be used to save a binary object to disk
     if local_file and method == 'GET':
         if result.status_code < 200 or result.status_code >= 300:
+            if err_code in sortof_ok:
+                log.error('Failed to get file=%s. %s: %s', path, err_code, err_msg)
+                return None
             raise CommandExecutionError(
-                'Failed to get file. {0}: {1}'.format(err_code, err_msg))
+                'Failed to get file=%s. {0}: {1}'.format(path, err_code, err_msg))
 
         log.debug('Saving to local file: %s', local_file)
         with salt.utils.files.fopen(local_file, 'wb') as out:
@@ -256,9 +280,8 @@ def query(key, keyid, method='GET', params=None, headers=None,
         return 'Saved to local file: {0}'.format(local_file)
 
     if result.status_code < 200 or result.status_code >= 300:
-        if err_code in ['SlowDown', 'ServiceUnavailable', 'RequestTimeTooSkewed',
-                        'RequestTimeout', 'OperationAborted', 'InternalError']:
-            log.error('Failed s3 operation: %s, %s', err_code, err_msg)
+        if err_code in sortof_ok:
+            log.error('Failed s3 operation. %s: %s', err_code, err_msg)
             return None
         raise CommandExecutionError(
             'Failed s3 operation. {0}: {1}'.format(err_code, err_msg))

requirements.txt

@@ -25,6 +25,7 @@ markupsafe
 mock
 msgpack
 ntplib
+objgraph
 patch==1.*
 pprint
 pycryptodome