remove unnecessary conversion to list

Author: Tenebriso

hubblestack/extmods/fileserver/s3fs.py

@@ -683,7 +683,7 @@ def _get_file():
             path_style=s3_key_kwargs['path_style'],
             https_enable=s3_key_kwargs['https_enable'])
         if ret:
-            for header_name, header_value in list(ret['headers'].items()):
+            for header_name, header_value in ret['headers'].items():
                 header_name = header_name.strip()
                 header_value = header_value.strip()
                 if six.text_type(header_name).lower() == 'last-modified':

hubblestack/extmods/grains/cloud_details.py

@@ -33,20 +33,21 @@ def _get_aws_details():
     aws['cloud_account_id'] = None
     aws['cloud_type'] = 'aws'
 
-    proxies = { 'http': None }
+    proxies = {'http': None}
 
     try:
         res = requests.get('http://169.254.169.254/latest/dynamic/instance-identity/document',
-                            timeout=3, proxies=proxies).json()
+                           timeout=3, proxies=proxies).json()
         aws['cloud_account_id'] = res.get('accountId', 'unknown')
 
         # AWS account id is always an integer number
         # So if it's an aws machine it must be a valid integer number
         # Else it will throw an Exception
         int(aws['cloud_account_id'])
 
-        aws['cloud_instance_id'] = requests.get('http://169.254.169.254/latest/meta-data/instance-id',
-                                                timeout=3, proxies=proxies).text
+        aws['cloud_instance_id'] = requests.get(
+            'http://169.254.169.254/latest/meta-data/instance-id',
+            timeout=3, proxies=proxies).text
     except (requests.exceptions.RequestException, ValueError):
         # Not on an AWS box
         aws = None
@@ -57,16 +58,19 @@ def _get_aws_details():
             aws_extra['cloud_availability_zone'] = res.get('availabilityZone')
             aws_extra['cloud_ami_id'] = res.get('imageId')
             aws_extra['cloud_region'] = res.get('region')
-            r = requests.get('http://169.254.169.254/latest/meta-data/public-hostname', timeout=3, proxies=proxies)
+            r = requests.get('http://169.254.169.254/latest/meta-data/public-hostname', timeout=3,
+                             proxies=proxies)
             if r.status_code == requests.codes.ok:
                 aws_extra['cloud_public_hostname'] = r.text
-            r = requests.get('http://169.254.169.254/latest/meta-data/public-ipv4', timeout=3, proxies=proxies)
+            r = requests.get('http://169.254.169.254/latest/meta-data/public-ipv4', timeout=3,
+                             proxies=proxies)
             if r.status_code == requests.codes.ok:
                 aws_extra['cloud_public_ipv4'] = r.text
-            r = requests.get('http://169.254.169.254/latest/meta-data/local-hostname', timeout=3, proxies=proxies)
+            r = requests.get('http://169.254.169.254/latest/meta-data/local-hostname', timeout=3,
+                             proxies=proxies)
             if r.status_code == requests.codes.ok:
                 aws_extra['cloud_private_hostname'] = r.text
-            for key in list(aws_extra.keys()):
+            for key in aws_extra.keys():
                 if not aws_extra[key]:
                     aws_extra.pop(key)
 
@@ -87,7 +91,7 @@ def _get_azure_details():
     azure['cloud_account_id'] = None
     azure['cloud_type'] = 'azure'
     azureHeader = {'Metadata': 'true'}
-    proxies = { 'http': None }
+    proxies = {'http': None}
 
     try:
         # Reminder: rev the api version for access to more details
@@ -111,19 +115,22 @@ def _get_azure_details():
             azure_extra['cloud_tags'] = id['tags']
             azure_extra['cloud_image_version'] = id['version']
             azure_extra['cloud_size'] = id['vmSize']
-            interface_list = requests.get('http://169.254.169.254/metadata/instance/network/interface?api-version=2017-08-01',
-                                          headers=azureHeader, timeout=3, proxies=proxies).json()
+            interface_list = requests.get(
+                'http://169.254.169.254/metadata/instance/network/interface?api-version=2017-08-01',
+                headers=azureHeader, timeout=3, proxies=proxies).json()
             for counter, value in enumerate(interface_list):
                 grain_name_private_ipv4 = "cloud_interface_{0}_private_ipv4".format(counter)
-                azure_extra[grain_name_private_ipv4] = value['ipv4']['ipAddress'][0]['privateIpAddress']
+                azure_extra[grain_name_private_ipv4] = value['ipv4']['ipAddress'][0][
+                    'privateIpAddress']
 
                 grain_name_public_ipv4 = "cloud_interface_{0}_public_ipv4".format(counter)
-                azure_extra[grain_name_public_ipv4] = value['ipv4']['ipAddress'][0]['publicIpAddress']
+                azure_extra[grain_name_public_ipv4] = value['ipv4']['ipAddress'][0][
+                    'publicIpAddress']
 
                 grain_name_mac = "cloud_interface_{0}_mac_address".format(counter)
                 azure_extra[grain_name_mac] = value['macAddress']
 
-            for key in list(azure_extra.keys()):
+            for key in azure_extra.keys():
                 if not azure_extra[key]:
                     azure_extra.pop(key)
 
@@ -134,6 +141,7 @@ def _get_azure_details():
     ret['cloud_details_extra'] = azure_extra
     return ret
 
+
 def _get_gcp_details():
     # Gather google compute platform information if present
     ret = {}
@@ -143,34 +151,45 @@ def _get_gcp_details():
     gcp['cloud_account_id'] = None
     gcp['cloud_type'] = 'gcp'
     gcp_header = {'Metadata-Flavor': 'Google'}
-    proxies = { 'http': None }
+    proxies = {'http': None}
 
     try:
-        gcp['cloud_instance_id'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/id',
-                                                headers=gcp_header, timeout=3, proxies=proxies).text
-        gcp['cloud_account_id'] = requests.get('http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id',
-                                               headers=gcp_header, timeout=3, proxies=proxies).text
+        gcp['cloud_instance_id'] = requests.get(
+            'http://metadata.google.internal/computeMetadata/v1/instance/id',
+            headers=gcp_header, timeout=3, proxies=proxies).text
+        gcp['cloud_account_id'] = requests.get(
+            'http://metadata.google.internal/computeMetadata/v1/project/numeric-project-id',
+            headers=gcp_header, timeout=3, proxies=proxies).text
     except (requests.exceptions.RequestException, ValueError):
         # Not on gcp box
         gcp = None
     if gcp:
         try:
-            gcp_extra['cloud_project_id'] = requests.get('http://metadata.google.internal/computeMetadata/v1/project/project-id',
-                                                         headers=gcp_header, timeout=3, proxies=proxies).text
-            gcp_extra['cloud_name'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/name',
-                                                   headers=gcp_header, timeout=3, proxies=proxies).text
-            gcp_extra['cloud_hostname'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/hostname',
-                                                       headers=gcp_header, timeout=3, proxies=proxies).text
-            gcp_extra['cloud_zone'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/zone',
-                                                   headers=gcp_header, timeout=3, proxies=proxies).text
-            gcp_extra['cloud_image'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/image',
-                                                    headers=gcp_header, timeout=3, proxies=proxies).text
-            gcp_extra['cloud_machine_type'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/machine-type',
-                                                           headers=gcp_header, timeout=3, proxies=proxies).text
-            gcp_extra['cloud_tags'] = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/tags?recursive=true',
-                                                   headers=gcp_header, timeout=3, proxies=proxies).json()
-            interface_list = requests.get('http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/?recursive=true',
-                                          headers=gcp_header, timeout=3, proxies=proxies).json()
+            gcp_extra['cloud_project_id'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/project/project-id',
+                headers=gcp_header, timeout=3, proxies=proxies).text
+            gcp_extra['cloud_name'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/name',
+                headers=gcp_header, timeout=3, proxies=proxies).text
+            gcp_extra['cloud_hostname'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/hostname',
+                headers=gcp_header, timeout=3, proxies=proxies).text
+            gcp_extra['cloud_zone'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/zone',
+                headers=gcp_header, timeout=3, proxies=proxies).text
+            gcp_extra['cloud_image'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/image',
+                headers=gcp_header, timeout=3, proxies=proxies).text
+            gcp_extra['cloud_machine_type'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/machine-type',
+                headers=gcp_header, timeout=3, proxies=proxies).text
+            gcp_extra['cloud_tags'] = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/tags?recursive=true',
+                headers=gcp_header, timeout=3, proxies=proxies).json()
+            interface_list = requests.get(
+                'http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces'
+                '/?recursive=true',
+                headers=gcp_header, timeout=3, proxies=proxies).json()
             for counter, value in enumerate(interface_list):
                 grain_name_network = "cloud_interface_{0}_network".format(counter)
                 gcp_extra[grain_name_network] = value['network']
@@ -190,11 +209,14 @@ def _get_gcp_details():
                 grain_name_targetips = "cloud_interface_{0}_target_ips".format(counter)
                 gcp_extra[grain_name_targetips] = ','.join(value['targetInstanceIps'])
 
-                grain_name_accessconfig_external_ips = "cloud_interface_{0}_accessconfigs_external_ips".format(counter)
-                external_ips_list = [ item['externalIp'] for item in value['accessConfigs'] if 'externalIp' in item ]
+                grain_name_accessconfig_external_ips = "cloud_interface_{" \
+                                                       "0}_accessconfigs_external_ips".format(
+                                                            counter)
+                external_ips_list = [item['externalIp'] for item in value['accessConfigs'] if
+                                     'externalIp' in item]
                 gcp_extra[grain_name_accessconfig_external_ips] = ','.join(external_ips_list)
 
-            for key in list(gcp_extra.keys()):
+            for key in gcp_extra.keys():
                 if not gcp_extra[key]:
                     gcp_extra.pop(key)
 

hubblestack/extmods/modules/conf_publisher.py

@@ -9,15 +9,16 @@
 
 log = logging.getLogger(__name__)
 
+
 def get_splunk_options(**kwargs):
     if not kwargs:
         kwargs['sourcetype'] = 'hubble_osquery'
     if '_nick' not in kwargs or not isinstance(kwargs['_nick'], dict):
         kwargs['_nick'] = {'sourcetype_nebula': 'sourcetype'}
     return gso(**kwargs)
 
-def publish(report_directly_to_splunk=True, remove_dots=True, *args):
 
+def publish(report_directly_to_splunk=True, remove_dots=True, *args):
     """
     Publishes config to splunk at an interval defined in schedule
 
@@ -44,7 +45,7 @@ def publish(report_directly_to_splunk=True, remove_dots=True, *args):
             opts_to_log.pop('grains')
     else:
         for arg in args:
-            if arg in  __opts__:
+            if arg in __opts__:
                 opts_to_log[arg] = __opts__[arg]
 
     filtered_conf = hubblestack.log.filter_logs(opts_to_log, remove_dots=remove_dots)
@@ -63,7 +64,7 @@ def _filter_config(opts_to_log, remove_dots=True):
     patterns_to_filter = ["password", "token", "passphrase", "privkey", "keyid", "s3.key"]
     filtered_conf = _remove_sensitive_info(opts_to_log, patterns_to_filter)
     if remove_dots:
-        for key in list(filtered_conf.keys()):
+        for key in filtered_conf.keys():
             if '.' in key:
                 filtered_conf[key.replace('.', '_')] = filtered_conf.pop(key)
     return filtered_conf

hubblestack/extmods/modules/hubble.py

@@ -333,7 +333,7 @@ def _run_audit(configs, tags, debug, labels, **kwargs):
     for failure_index in reversed(sorted(set(failures_to_remove))):
         results['Failure'].pop(failure_index)
 
-    for key in list(results.keys()):
+    for key in results.keys():
         if not results[key]:
             results.pop(key)
 
@@ -581,7 +581,7 @@ def _clean_up_results(results, show_success):
     if show_success was not passed, adding an error message if
     results is empty
     """
-    for key in list(results.keys()):
+    for key in results.keys():
         if not results[key]:
             results.pop(key)
 

hubblestack/extmods/modules/pulsar.py

@@ -558,16 +558,18 @@ def _wrapped(val):
     the_list = []
     for e in excludes:
         if isinstance(e,dict):
-            if list(e.values())[0].get('regex'):
-                r = list(e.keys())[0]
+            first_val = list(e.values())[0]
+            first_key = list(e.keys())[0]
+            if first_val.get('regex'):
+                r = first_key
                 try:
                     c = re.compile(r)
                     the_list.append(re_wrapper(c))
                 except Exception as e:
-                    log.warn('Failed to compile regex "%s": %s', r,e)
+                    log.warning('Failed to compile regex "%s": %s', r, e)
                 continue
             else:
-                e = list(e.keys())[0]
+                e = first_key
         if '*' in e:
             the_list.append(fn_wrapper(e))
         else:
@@ -576,7 +578,7 @@ def _wrapped(val):
     # finally, wrap the whole decision set in a decision wrapper
     def _final(val):
         for i in the_list:
-            if i( val ):
+            if i(val):
                 return True
         return False
     return _final
@@ -972,7 +974,7 @@ def _dict_update(dest, upd, recursive_update=True, merge_lists=False):
         return dest
     else:
         try:
-            for k in list(upd.keys()):
+            for k in upd.keys():
                 dest[k] = upd[k]
         except AttributeError:
             # this mapping is not a dict

hubblestack/extmods/returners/graylog_nebula_return.py

@@ -53,7 +53,7 @@ def returner(ret):
 
     for opts in opts_list:
         for query in ret['return']:
-            for query_name, value in list(query.items()):
+            for query_name, value in query.items():
                 for query_data in value['data']:
                     args = {'query': query_name,
                             'job_id': ret['jid'],

hubblestack/files/hubblestack_nova/misc.py

@@ -449,7 +449,7 @@ def check_duplicate_uids(reason=''):
     """
     uids = _execute_shell_command("cat /etc/passwd | cut -f3 -d\":\"", python_shell=True).strip()
     uids = uids.split('\n') if uids != "" else []
-    duplicate_uids = [k for k, v in list(Counter(uids).items()) if v > 1]
+    duplicate_uids = [k for k, v in Counter(uids).items() if v > 1]
     if duplicate_uids is None or duplicate_uids == []:
         return True
     return str(duplicate_uids)
@@ -461,7 +461,7 @@ def check_duplicate_gids(reason=''):
     """
     gids = _execute_shell_command("cat /etc/group | cut -f3 -d\":\"", python_shell=True).strip()
     gids = gids.split('\n') if gids != "" else []
-    duplicate_gids = [k for k, v in list(Counter(gids).items()) if v > 1]
+    duplicate_gids = [k for k, v in Counter(gids).items() if v > 1]
     if duplicate_gids is None or duplicate_gids == []:
         return True
     return str(duplicate_gids)
@@ -473,7 +473,7 @@ def check_duplicate_unames(reason=''):
     """
     unames = _execute_shell_command("cat /etc/passwd | cut -f1 -d\":\"", python_shell=True).strip()
     unames = unames.split('\n') if unames != "" else []
-    duplicate_unames = [k for k, v in list(Counter(unames).items()) if v > 1]
+    duplicate_unames = [k for k, v in Counter(unames).items() if v > 1]
     if duplicate_unames is None or duplicate_unames == []:
         return True
     return str(duplicate_unames)
@@ -485,7 +485,7 @@ def check_duplicate_gnames(reason=''):
     """
     gnames = _execute_shell_command("cat /etc/group | cut -f1 -d\":\"", python_shell=True).strip()
     gnames = gnames.split('\n') if gnames != "" else []
-    duplicate_gnames = [k for k, v in list(Counter(gnames).items()) if v > 1]
+    duplicate_gnames = [k for k, v in Counter(gnames).items() if v > 1]
     if duplicate_gnames is None or duplicate_gnames == []:
         return True
     return str(duplicate_gnames)

hubblestack/files/hubblestack_nova/oval_scanner.py

@@ -90,7 +90,7 @@ def audit(data_list, tags, labels, debug=False, **kwargs):
 
 def parse_impact_report(report, local_pkgs, hubble_format, impacted_pkgs=[]):
     """Parse into Hubble friendly format"""
-    for key, value in list(report.items()):
+    for key, value in report.items():
         pkg_desc = 'Vulnerable Package(s): '
         for pkg in value['installed']:
             pkg_desc += '{0}-{1}, '.format(pkg['name'], pkg['version'])
@@ -125,7 +125,7 @@ def get_impact_report(vulns, local_pkgs, distro_name):
 def build_impact(vulns, local_pkgs, distro_name, result={}):
     """Build impacts based on pkg comparisons"""
     logging.debug('build_impact')
-    for data in list(vulns.values()):
+    for data in vulns.values():
         for pkg in data['pkg']:
             name = pkg['name']
             ver = pkg['version']
@@ -152,7 +152,7 @@ def build_impact(vulns, local_pkgs, distro_name, result={}):
 def build_impact_report(impact, report={}):
     """Build a report based on impacts"""
     logging.debug('build_impact_report')
-    for adv, detail in list(impact.items()):
+    for adv, detail in impact.items():
         if adv not in report:
             report[adv] = {
                 'updated_pkg': [],
@@ -187,7 +187,7 @@ def create_vulns(oval_and_maps, vulns={}):
     logging.debug('create_vulns')
     id_maps = oval_and_maps[0]
     oval = oval_and_maps[1]
-    for definition, data in list(id_maps.items()):
+    for definition, data in id_maps.items():
         if definition in oval['definitions']:
             vulns[definition] = oval['definitions'][definition]
             vulns[definition]['pkg'] = []
@@ -217,7 +217,7 @@ def create_vulns(oval_and_maps, vulns={}):
 def map_oval_ids(oval, id_maps={}):
     """For every test, grab only tests with both state and obj references"""
     logging.debug('map_oval_ids')
-    for definition, data in list(oval['definitions'].items()):
+    for definition, data in oval['definitions'].items():
         id_maps[definition] = {'objects': []}
         objects = id_maps[definition]['objects']
         tests = data['tests']