Azdev Fixes to work with Azure-CLI CI (#36)

* Pin pytest version.

* Fix linter to use long or short formats. Remove --ci flag.

* Split out version checking from updating setup.py.

* Make azdev test run all tests by default. Improve FileNotFound handling.

* Fix #14.

* CI fixes.

Author: tjprescott

azdev/commands.py

@@ -27,7 +27,10 @@ def operation_group(name):
 
     with CommandGroup(self, 'verify', operation_group('pypi')) as g:
         g.command('history', 'check_history')
-        g.command('version', 'verify_versions')
+
+    with CommandGroup(self, 'cli', operation_group('pypi')) as g:
+        g.command('check-versions', 'verify_versions')
+        g.command('update-setup', 'update_setup_py')
 
     with CommandGroup(self, 'verify', operation_group('help')) as g:
         g.command('document-map', 'check_document_map')

azdev/help.py

@@ -29,6 +29,33 @@
 """
 
 
+helps['cli'] = """
+    short-summary: Commands for working with CLI modules.
+"""
+
+helps['cli check-versions'] = """
+    short-summary: Verify package versions against those hosted on PyPI and/or in the azure-cli setup.py file.
+    long-summary: >
+        This is used to ensure the correct module versions are bumped prior to release.
+    examples:
+        - name: Verify all versions and audit them against azure-cli's setup.py ONLY.
+          text: azdev cli check-versions
+
+        - name: Verify all versions and update azure-cli's setup.py with each module's verison.
+          text: azdev cli check-versions --update --pin
+"""
+
+helps['cli update-setup'] = """
+    short-summary: Update the azure-cli setup.py file.
+    examples:
+        - name: Update azure-cli's setup.py with unpinned module verions.
+          text: azdev cli update-setup
+
+        - name: Update azure-cli's setup.py with pinned module versions.
+          text: azdev cli update-setup --pin
+"""
+
+
 helps['configure'] = """
     short-summary: Configure azdev for use without installing anything.
 """
@@ -80,20 +107,6 @@
 """
 
 
-helps['verify version'] = """
-    short-summary: Verify package versions against those hosted on PyPI and/or in the azure-cli setup.py file.
-    long-summary: >
-        This is used to ensure the correct module versions are bumped prior to release and that all
-        module versions are present in the azure-cli setup.py with their versions pinned.
-    examples:
-        - name: Verify all versions and audit them against azure-cli's setup.py ONLY.
-          text: azdev verify version
-
-        - name: Verify all versions and update azure-cli's setup.py with each module's verison.
-          text: azdev verify version --update
-"""
-
-
 helps['style'] = """
     short-summary: Check code style (pylint and PEP8).
 """

azdev/operations/linter/__init__.py

@@ -26,7 +26,7 @@
 
 
 # pylint:disable=too-many-locals
-def run_linter(modules=None, rule_types=None, rules=None, ci_mode=False):
+def run_linter(modules=None, rule_types=None, rules=None):
 
     require_azure_cli()
 
@@ -39,12 +39,18 @@ def run_linter(modules=None, rule_types=None, rules=None, ci_mode=False):
     # needed to remove helps from azdev
     azdev_helps = helps.copy()
     exclusions = {}
-    path_table = get_path_table(include_only=modules)
+    selected_modules = get_path_table(include_only=modules)
 
-    selected_mod_names = list(path_table['mod'].keys()) + list(path_table['core'].keys()) + \
-        list(path_table['ext'].keys())
-    selected_mod_paths = list(path_table['mod'].values()) + list(path_table['core'].values()) + \
-        list(path_table['ext'].values())
+    if not selected_modules:
+        raise CLIError('No modules selected.')
+
+    selected_mod_names = list(selected_modules['mod'].keys()) + list(selected_modules['core'].keys()) + \
+        list(selected_modules['ext'].keys())
+    selected_mod_paths = list(selected_modules['mod'].values()) + list(selected_modules['core'].values()) + \
+        list(selected_modules['ext'].values())
+
+    if selected_mod_names:
+        display('Modules: {}\n'.format(', '.join(selected_mod_names)))
 
     # collect all rule exclusions
     for path in selected_mod_paths:
@@ -99,6 +105,5 @@ def run_linter(modules=None, rule_types=None, rules=None, ci_mode=False):
         run_params=not rule_types or 'params' in rule_types,
         run_commands=not rule_types or 'commands' in rule_types,
         run_command_groups=not rule_types or 'command_groups'in rule_types,
-        run_help_files_entries=not rule_types or 'help_entries' in rule_types,
-        ci=ci_mode)
+        run_help_files_entries=not rule_types or 'help_entries' in rule_types)
     sys.exit(exit_code)

azdev/operations/linter/linter.py

@@ -188,8 +188,8 @@ def exclusions(self):
     def exit_code(self):
         return self._exit_code
 
-    def run(self, run_params=None, run_commands=None, run_command_groups=None, run_help_files_entries=None, ci=False):
-        self._ci = ci
+    def run(self, run_params=None, run_commands=None, run_command_groups=None, run_help_files_entries=None):
+        self._ci = os.environ.get('CI', False)
         paths = import_module('{}.rules'.format(PACAKGE_NAME)).__path__
 
         if paths:

azdev/operations/linter/util.py

@@ -9,6 +9,8 @@
 
 from knack.log import get_logger
 
+from azdev.utilities import COMMAND_MODULE_PREFIX
+
 
 logger = get_logger(__name__)
 
@@ -38,6 +40,7 @@ def exclude_commands(command_loader, help_file_entries, module_exclusions):
 
 def _filter_mods(command_loader, help_file_entries, modules=None, exclude=False):
     modules = modules or []
+    modules = [x.replace(COMMAND_MODULE_PREFIX, '') for x in modules]
 
     # command tables and help entries must be copied to allow for seperate linter scope
     command_table = command_loader.command_table.copy()
@@ -64,6 +67,7 @@ def _filter_mods(command_loader, help_file_entries, modules=None, exclude=False)
     # Remove unneeded command groups
     retained_command_groups = {' '.join(x.split(' ')[:-1]) for x in command_loader.command_table}
     excluded_command_groups = set(command_loader.command_group_table.keys()) - retained_command_groups
+
     for group_name in excluded_command_groups:
         command_loader.command_group_table.pop(group_name, None)
         help_file_entries.pop(group_name, None)

azdev/operations/performance.py

@@ -15,26 +15,23 @@
 logger = get_logger(__name__)
 
 TOTAL = 'ALL'
+TOTAL_THRESHOLD = 300
 DEFAULT_THRESHOLD = 10
-
-# TODO: Treat everything as bubble instead of specific modules
-# explicit thresholds that deviate from the default
 THRESHOLDS = {
-    'network': 30,
-    'vm': 30,
-    'batch': 30,
-    'storage': 50,
-    TOTAL: 300
+    # threshold value: num of exceptions allowed
+    50: 1,
+    30: 4
 }
 
 
+# pylint: disable=too-many-statements
 def check_load_time(runs=3):
 
     require_azure_cli()
 
     heading('Module Load Performance')
 
-    regex = r"[^']*'([^']*)'[\D]*([\d\.]*)"
+    regex = r"[^']*'(?P<mod>[^']*)'[\D]*(?P<val>[\d\.]*)"
 
     results = {TOTAL: []}
     # Time the module loading X times
@@ -52,8 +49,8 @@ def check_load_time(runs=3):
         for line in lines:
             if line.startswith('DEBUG: Loaded module'):
                 matches = re.match(regex, line)
-                mod = matches.group(1)
-                val = float(matches.group(2)) * 1000
+                mod = matches.group('mod')
+                val = float(matches.group('val')) * 1000
                 total_time = total_time + val
                 if mod in results:
                     results[mod].append(val)
@@ -64,24 +61,33 @@ def check_load_time(runs=3):
     passed_mods = {}
     failed_mods = {}
 
+    def _claim_higher_threshold(val):
+        avail_thresholds = {k: v for k, v in THRESHOLDS.items() if v}
+        new_threshold = None
+        for threshold in sorted(avail_thresholds):
+            if val < threshold:
+                THRESHOLDS[threshold] = THRESHOLDS[threshold] - 1
+                new_threshold = threshold
+            break
+        return new_threshold
+
     mods = sorted(results.keys())
-    bubble_found = False
     for mod in mods:
         val = results[mod]
         mean_val = mean(val)
         stdev_val = pstdev(val)
-        threshold = THRESHOLDS.get(mod) or DEFAULT_THRESHOLD
+        threshold = TOTAL_THRESHOLD if mod == TOTAL else DEFAULT_THRESHOLD
         statistics = {
             'average': mean_val,
             'stdev': stdev_val,
             'threshold': threshold,
             'values': val
         }
         if mean_val > threshold:
-            if not bubble_found and mean_val < 30:
-                # This temporary measure allows one floating performance
-                # failure up to 30 ms. See issue #6224 and #6218.
-                bubble_found = True
+            # claim a threshold exception if available
+            new_threshold = _claim_higher_threshold(mean_val)
+            if new_threshold:
+                statistics['threshold'] = new_threshold
                 passed_mods[mod] = statistics
             else:
                 failed_mods[mod] = statistics