ESC (end stage cleanup) step 0: fix 'salt.' imports and move things around

Author: jettero

contrib/dep-orbital-strike.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+THIS_DIR="$(git rev-parse --show-toplevel)"
+
+UNINSTALL=(
+    salt-ssh
+    jinja2
+)
+
+UNDEPEND=(
+    "${UNINSTALL[@]}"
+    six
+)
+
+
+( set -x
+  pip uninstall -y hubblestack "${UNINSTALL[@]}"
+)
+
+IFS='|'
+KILL="${UNDEPEND[*]}"
+unset IFS
+
+( set -x
+  find "$THIS_DIR" -type f -name \*req\*.txt \
+      | grep -v /abandoned/ \
+      | xargs -r sed -i -re "/$KILL/d"
+  sed -i -re "/'$KILL.+',/d" setup.py
+)

contrib/unused-import-scan.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+find hubblestack tests -type f -name \*.py \
+    | xargs -r pylint --disable=all --enable=unused-import

doc/requirements.txt

@@ -1,4 +1,3 @@
 sphinx
 sphinx-autobuild
 sphinx_rtd_theme
-salt-ssh

hubblestack/daemon.py

@@ -21,13 +21,13 @@
 import uuid
 from datetime import datetime
 
-import salt.fileserver
-import salt.fileserver.gitfs
+import hubblestack.fileserver
+import hubblestack.fileserver.gitfs
 import hubblestack.modules.cmdmod
-import salt.utils
+import hubblestack.utils
 import hubblestack.utils.platform
-import salt.utils.jid
-import salt.utils.gitfs
+import hubblestack.utils.jid
+import hubblestack.utils.gitfs
 import hubblestack.utils.path
 from croniter import croniter
 
@@ -81,15 +81,15 @@ def _clear_gitfs_locks():
     # Clear old locks
     if 'gitfs' in __opts__['fileserver_backend'] or 'git' in __opts__['fileserver_backend']:
         git_objects = [
-            salt.utils.gitfs.GitFS(
+            hubblestack.utils.gitfs.GitFS(
                 __opts__,
                 __opts__['gitfs_remotes'],
-                per_remote_overrides=salt.fileserver.gitfs.PER_REMOTE_OVERRIDES,
-                per_remote_only=salt.fileserver.gitfs.PER_REMOTE_ONLY)]
+                per_remote_overrides=hubblestack.fileserver.gitfs.PER_REMOTE_OVERRIDES,
+                per_remote_only=hubblestack.fileserver.gitfs.PER_REMOTE_ONLY)]
         ret = {}
         for obj in git_objects:
             lock_type = 'update'
-            cleared, errors = salt.fileserver.clear_lock(obj.clear_lock, 'gitfs', remote=None,
+            cleared, errors = hubblestack.fileserver.clear_lock(obj.clear_lock, 'gitfs', remote=None,
                                                          lock_type=lock_type)
             if cleared:
                 ret.setdefault('cleared', []).extend(cleared)
@@ -143,7 +143,7 @@ def main():
     count = 0
     while True:
         try:
-            file_client = salt.fileclient.get_file_client(__opts__)
+            file_client = hubblestack.fileclient.get_file_client(__opts__)
             file_client.channel.fs.update()
             last_fc_update = time.time()
             break
@@ -362,7 +362,7 @@ def _execute_function(jobdata, func, returners, args, kwargs):
             continue
         log.debug('Returning job data to %s', returner)
         returner_ret = {'id': __grains__['id'],
-                        'jid': salt.utils.jid.gen_jid(__opts__),
+                        'jid': hubblestack.utils.jid.gen_jid(__opts__),
                         'fun': func,
                         'fun_args': args + ([kwargs] if kwargs else []),
                         'return': ret}
@@ -429,7 +429,7 @@ def run_function():
         else:
             log.info('Returning job data to %s', returner)
             returner_ret = {'id': __grains__['id'],
-                            'jid': salt.utils.jid.gen_jid(__opts__),
+                            'jid': hubblestack.utils.jid.gen_jid(__opts__),
                             'fun': __opts__['function'],
                             'fun_args': args + ([kwargs] if kwargs else []),
                             'return': ret}
@@ -483,7 +483,7 @@ def load_config(args=None):
         # them a signal 15 (otherwise refuse to run)
         if not __opts__.get('ignore_running', False):
             check_pidfile(kill_other=True, scan_proc=scan_proc)
-        salt.utils.daemonize()
+        hubblestack.utils.daemonize()
         create_pidfile()
     elif not __opts__['function'] and not __opts__['version'] and not __opts__['buildinfo']:
         # check the pidfile and possibly refuse to run (assuming this isn't a single function call)

hubblestack/extmods/utils/__init__.py

@@ -23,8 +23,8 @@
     CommandExecutionError, MinionError
 )
 import hubblestack.loader
-import salt.payload
-import hubblestack.extmods.fileserver
+import hubblestack.payload
+import hubblestack.fileserver
 import hubblestack.utils.data
 import hubblestack.utils.files
 import hubblestack.utils.gzip_util
@@ -82,7 +82,7 @@ class Client(object):
     def __init__(self, opts):
         self.opts = opts
         self.utils = hubblestack.loader.utils(self.opts)
-        self.serial = salt.payload.Serial(self.opts)
+        self.serial = hubblestack.payload.Serial(self.opts)
 
     # Add __setstate__ and __getstate__ so that the object may be
     # deep copied. It normally can't be deep copied because its
@@ -1113,7 +1113,7 @@ class FSClient(RemoteClient):
     def __init__(self, opts):  # pylint: disable=W0231
         Client.__init__(self, opts)  # pylint: disable=W0233
         self._closing = False
-        self.channel = hubblestack.extmods.fileserver.FSChan(opts)
+        self.channel = hubblestack.fileserver.FSChan(opts)
         self.auth = DumbAuth()
 
 

hubblestack/fileclient.py

@@ -6,7 +6,7 @@
 import errno
 
 import hubblestack.loader
-import salt.payload
+import hubblestack.payload
 import hubblestack.utils.path
 import hubblestack.utils.files
 import hubblestack.utils.data
@@ -130,7 +130,7 @@ def check_env_cache(opts, env_cache):
     try:
         with hubblestack.utils.files.fopen(env_cache, 'rb') as fp_:
             log.trace('Returning env cache data from %s', env_cache)
-            serial = salt.payload.Serial(opts)
+            serial = hubblestack.payload.Serial(opts)
             return hubblestack.utils.data.decode(serial.load(fp_))
     except (IOError, OSError):
         pass
@@ -227,7 +227,7 @@ def check_file_list_cache(opts, form, list_cache, w_lock):
     """
     refresh_cache = False
     save_cache = True
-    serial = salt.payload.Serial(opts)
+    serial = hubblestack.payload.Serial(opts)
     wait_lock(w_lock, list_cache, 5 * 60)
     if not os.path.isfile(list_cache) and _lock_cache(w_lock):
         refresh_cache = True
@@ -289,7 +289,7 @@ def write_file_list_cache(opts, data, list_cache, w_lock):
     returns the match (if found, along with booleans used by the fileserver
     backend to determine if the cache needs to be refreshed/written).
     '''
-    serial = salt.payload.Serial(opts)
+    serial = hubblestack.payload.Serial(opts)
     with hubblestack.utils.files.fopen(list_cache, 'w+b') as fp_:
         fp_.write(serial.dumps(data))
         _unlock_cache(w_lock)
@@ -888,4 +888,4 @@ def send(self, load, tries=None, timeout=None, raw=False):  # pylint: disable=un
         return getattr(self.fs, cmd)(load)
 
     def close(self):
-        pass
+        pass

hubblestack/extmods/fileserver/__init__.py hubblestack/fileserver/__init__.py

@@ -55,7 +55,7 @@
 import os.path
 import shutil
 
-import hubblestack.extmods.fileserver
+import hubblestack.fileserver
 import hubblestack.utils.files
 import hubblestack.utils.gzip_util
 import hubblestack.utils.hashutils
@@ -103,7 +103,7 @@ def find_file(path, saltenv='base', **kwargs):
         if container.get('saltenv', 'base') != saltenv:
             continue
         full = os.path.join(_get_container_path(container), path)
-        if os.path.isfile(full) and not hubblestack.extmods.fileserver.is_file_ignored(
+        if os.path.isfile(full) and not hubblestack.fileserver.is_file_ignored(
                                                                 __opts__, path):
             fnd['path'] = full
             fnd['rel'] = path
@@ -234,7 +234,7 @@ def update():
                 fname = os.path.join(root, f)
                 relpath = os.path.relpath(fname, path)
                 if relpath not in blob_set:
-                    hubblestack.extmods.fileserver.wait_lock(fname + '.lk', fname)
+                    hubblestack.fileserver.wait_lock(fname + '.lk', fname)
                     try:
                         os.unlink(fname)
                     except Exception:
@@ -260,7 +260,7 @@ def update():
                     os.makedirs(os.path.dirname(fname))
                 # Lock writes
                 lk_fn = fname + '.lk'
-                hubblestack.extmods.fileserver.wait_lock(lk_fn, fname)
+                hubblestack.fileserver.wait_lock(lk_fn, fname)
                 with hubblestack.utils.files.fopen(lk_fn, 'w+') as fp_:
                     fp_.write('')
 
@@ -298,7 +298,7 @@ def update():
         # Write out file list
         container_list = path + '.list'
         lk_fn = container_list + '.lk'
-        hubblestack.extmods.fileserver.wait_lock(lk_fn, container_list)
+        hubblestack.fileserver.wait_lock(lk_fn, container_list)
         with hubblestack.utils.files.fopen(lk_fn, 'w+') as fp_:
             fp_.write('')
         with hubblestack.utils.files.fopen(container_list, 'w') as fp_:
@@ -354,7 +354,7 @@ def file_list(load):
                 continue
             container_list = _get_container_path(container) + '.list'
             lk = container_list + '.lk'
-            hubblestack.extmods.fileserver.wait_lock(lk, container_list, 5)
+            hubblestack.fileserver.wait_lock(lk, container_list, 5)
             if not os.path.exists(container_list):
                 continue
             with hubblestack.utils.files.fopen(container_list, 'r') as fp_:

hubblestack/extmods/fileserver/azurefs.py hubblestack/fileserver/azurefs.py

@@ -66,7 +66,7 @@
                'insecure_auth')
 
 from hubblestack.utils.signing import find_wrapf
-from hubblestack.extmods.utils.gitfs import GitFS
+from hubblestack.utils.gitfs import GitFS
 from hubblestack.utils.exceptions import FileserverConfigError
 
 log = logging.getLogger(__name__)

hubblestack/extmods/fileserver/gitfs.py hubblestack/fileserver/gitfs.py

@@ -22,7 +22,7 @@
 import errno
 import logging
 
-import hubblestack.extmods.fileserver
+import hubblestack.fileserver
 import hubblestack.utils.files
 import hubblestack.utils.gzip_util
 import hubblestack.utils.hashutils
@@ -86,14 +86,14 @@ def _add_file_stat(fnd):
             # An invalid index option was passed
             return fnd
         full = os.path.join(root, path)
-        if os.path.isfile(full) and not hubblestack.extmods.fileserver.is_file_ignored(__opts__, full):
+        if os.path.isfile(full) and not hubblestack.fileserver.is_file_ignored(__opts__, full):
             fnd['path'] = full
             fnd['rel'] = path
             return _add_file_stat(fnd)
         return fnd
     for root in __opts__['file_roots'][saltenv]:
         full = os.path.join(root, path)
-        if os.path.isfile(full) and not hubblestack.extmods.fileserver.is_file_ignored(__opts__, full):
+        if os.path.isfile(full) and not hubblestack.fileserver.is_file_ignored(__opts__, full):
             fnd['path'] = full
             fnd['rel'] = path
             return _add_file_stat(fnd)
@@ -139,7 +139,7 @@ def update():
     When we are asked to update (regular interval) lets reap the cache
     """
     try:
-        hubblestack.extmods.fileserver.reap_fileserver_cache_dir(
+        hubblestack.fileserver.reap_fileserver_cache_dir(
             os.path.join(__opts__['cachedir'], 'roots', 'hash'),
             find_file
         )
@@ -154,7 +154,7 @@ def update():
             'backend': 'roots'}
 
     # generate the new map
-    new_mtime_map = hubblestack.extmods.fileserver.generate_mtime_map(__opts__, __opts__['file_roots'])
+    new_mtime_map = hubblestack.fileserver.generate_mtime_map(__opts__, __opts__['file_roots'])
 
     old_mtime_map = {}
     # if you have an old map, load that
@@ -175,7 +175,7 @@ def update():
                     )
 
     # compare the maps, set changed to the return value
-    data['changed'] = hubblestack.extmods.fileserver.diff_mtime_map(old_mtime_map, new_mtime_map)
+    data['changed'] = hubblestack.fileserver.diff_mtime_map(old_mtime_map, new_mtime_map)
 
     # compute files that were removed and added
     old_files = set(old_mtime_map.keys())
@@ -293,7 +293,7 @@ def _file_lists(load, form):
     list_cache = os.path.join(list_cachedir, '{0}.p'.format(load['saltenv']))
     w_lock = os.path.join(list_cachedir, '.{0}.w'.format(load['saltenv']))
     cache_match, refresh_cache, save_cache = \
-        hubblestack.extmods.fileserver.check_file_list_cache(
+        hubblestack.fileserver.check_file_list_cache(
             __opts__, form, list_cache, w_lock
         )
     if cache_match is not None:
@@ -328,7 +328,7 @@ def _translate_sep(path):
                     continue
                 rel_path = _translate_sep(os.path.relpath(abs_path, fs_root))
                 log.trace('roots: %s relative path is %s', abs_path, rel_path)
-                if hubblestack.extmods.fileserver.is_file_ignored(__opts__, rel_path):
+                if hubblestack.fileserver.is_file_ignored(__opts__, rel_path):
                     continue
                 tgt.add(rel_path)
                 try:
@@ -390,7 +390,7 @@ def _translate_sep(path):
 
         if save_cache:
             try:
-                hubblestack.extmods.fileserver.write_file_list_cache(
+                hubblestack.fileserver.write_file_list_cache(
                     __opts__, ret, list_cache, w_lock
                 )
             except NameError:

hubblestack/extmods/fileserver/roots.py hubblestack/fileserver/roots.py

@@ -87,7 +87,7 @@
 import logging
 
 # Import salt libs
-import hubblestack.extmods.fileserver as fs
+import hubblestack.fileserver as fs
 import hubblestack.utils.files
 import hubblestack.utils.gzip_util
 import hubblestack.utils.hashutils

hubblestack/extmods/fileserver/s3fs.py hubblestack/fileserver/s3fs.py

@@ -50,13 +50,6 @@
 
 log = logging.getLogger(__name__)
 
-# XXX: we only need this while we're still loading salt modules
-def __salt_basepath():
-    import salt.syspaths
-    return os.path.abspath(salt.syspaths.INSTALL_DIR)
-
-# XXX: we only need this while we're still loading salt modules
-SALT_BASE_PATH = __salt_basepath()
 HUBBLE_BASE_PATH = os.path.abspath(hubblestack.syspaths.INSTALL_DIR)
 LOADED_BASE_NAME = 'hubble.loaded'
 
@@ -107,9 +100,6 @@ def _module_dirs(
     hubblestack_type = 'hubblestack_' + (int_type or ext_type)
     files_base_types = os.path.join(base_path or HUBBLE_BASE_PATH, 'files', hubblestack_type)
 
-    # XXX should be removed eventually:
-    salt_base_types = os.path.join(SALT_BASE_PATH, int_type or ext_type)
-
     ext_type_types = []
     if ext_dirs:
         if tag is not None and ext_type_dirs is None:
@@ -139,8 +129,8 @@ def _module_dirs(
             cli_module_dirs.insert(0, maybe_dir)
 
     if explain:
-        return (cli_module_dirs, ext_type_types, [files_base_types, salt_base_types, ext_types, sys_types])
-    return cli_module_dirs + ext_type_types + [files_base_types, salt_base_types, ext_types, sys_types]
+        return (cli_module_dirs, ext_type_types, [files_base_types, ext_types, sys_types])
+    return cli_module_dirs + ext_type_types + [files_base_types, ext_types, sys_types]
 
 
 def modules(
@@ -301,7 +291,7 @@ def grains(opts, force_refresh=False, proxy=None):
     if 'conf_file' in opts:
         pre_opts = {}
         pre_opts.update(hubblestack.config.load_config(
-            opts['conf_file'], 'SALT_MINION_CONFIG',
+            opts['conf_file'], 'HUBBLE_CONFIG',
             hubblestack.config.DEFAULT_OPTS['conf_file']
         ))
         default_include = pre_opts.get(
@@ -444,10 +434,8 @@ def _generate_module(name):
 def _mod_type(module_path):
     if module_path.startswith(HUBBLE_BASE_PATH):
         if 'extmods' in module_path:
-            return 'e_int' # XXX: we should remove this when we nolonger have internal extmods
+            return 'e_int'
         return 'int'
-    if module_path.startswith(SALT_BASE_PATH):
-        return 'salt' # XXX: we should remove this after we nolonger depend on salt
     return 'ext'