tweak requirements and provide methods to generate it

Author: jettero

.travis.yml

@@ -20,12 +20,12 @@ python:
   - "3.8"
 
 install:
-  - pip install -r test-requirements.txt
+  - pip --upgarde pip
+  - ./mk-requires.sh
+  - pip install -r requirements.txt
 
 script:
   - wget -q "$LIBGIT2_SRC_URL" -O $LIBGIT2_TAR_NAME
   - tar -xzf $LIBGIT2_TAR_NAME
   - (cd $LIBGIT2_DIR_NAME; cmake . -DMAKE_INSTALL_PREFIX=${LIBGIT2} && make -j 5 install)
-  - pip install --upgrade pip
-  - pip install --upgrade -r <( bash pkg/generate-requirements.sh )
   - pytest tests/unittests

mk-requires.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+D="$(dirname "$0")"
+F="$D/requirements.txt"
+
+echo "# This file was generated by $0" > "$F"
+echo "# $(date -u)" >> "$F"
+echo >> "$F"
+
+bash "$D/pkg/generate-requirements.sh" "$@" | tee -a "$F"
+
+CMD=( pip install --upgrade -r requirements.txt )
+
+read -ep "issue ${CMD[*]}? [Y/n]" YN
+[[ "$YN" =~ [Nn] ]] || "${CMD[@]}"

pkg/generate-requirements.sh

@@ -8,5 +8,7 @@ cat "$setup_py_dir/hubblestack.egg-info/requires.txt" \
     "$setup_py_dir/optional-requirements.txt" \
     "$setup_py_dir/package-requirements.txt" \
     "$setup_py_dir/test-requirements.txt" \
+    \
     | tr A-Z a-z | grep -v ^# | grep . | sort -u \
-    | perl -ne 'chomp; $P{$1} = $_ if m/^([^<>=]+)/; END { print "$_\n" for sort values %P }'
+    | perl -ne 'chomp; $P{$1} = $_ if m/^([^<>=]+)/ and length($_) > length($P{$1});
+      END { print "$_\n" for sort values %P }'

requirements.txt

@@ -1,21 +1,52 @@
-ntplib
-distro
-pyinstaller==3.3.1
-Crypto
-pyopenssl>=16.2.0
+# This file was generated by ./mk-requires.sh
+# Wed Mar 18 14:02:09 UTC 2020
+
+ansi2html
 argparse
-requests>=2.13.0
-pprint
-daemon
+azure-storage-blob==2.1.0
+azure-storage-common==2.1.0
+azure==4.0.0
+backports-abc
+bandit
 boto3
 botocore
-pygit2<0.27.0
-salt-ssh
-gitpython
-pyinotify
+certifi
 cffi
-azure
-azure-storage-common
-azure-storage-blob
+chardet
 croniter
-vulners
+cryptography
+daemon
+futures
+gitdb2
+gitpython
+gprof2dot
+idna
+jinja2
+markupsafe
+mock
+msgpack
+ntplib
+patch==1.*
+pprint
+pycryptodome
+pygit2<0.27.0
+pyinotify
+pyinstaller-hooks
+pyinstaller==3.6
+pylint
+pyopenssl>=16.2.0
+pytest
+pytest-cov
+pytest-html
+pytest-pythonpath
+python-dateutil
+pyyaml
+requests>=2.13.0
+salt-ssh==2019.2.3
+saltpylint
+singledispatch
+six
+smmap2
+tornado
+urllib3
+vulners

setup.py

@@ -13,7 +13,8 @@
 # Default to CentOS7
 data_files = [('/usr/lib/systemd/system', ['pkg/source/hubble.service']),
               ('/etc/hubble', ['conf/hubble']), ]
-dependencies = [
+
+build_dependencies = [
     'pycryptodome',
     'cryptography',
     'pyopenssl>=16.2.0',
@@ -28,7 +29,7 @@
     'vulners',
     'ntplib',
     'patch==1.*',
-    ]
+]
 
 if distro == 'redhat' or distro == 'centos':
     if version.startswith('6'):
@@ -42,7 +43,7 @@
                   ('/etc/hubble', ['conf/hubble']), ]
 
 if platform_name == 'Windows':
-    dependencies.remove('pyinotify')
+    build_dependencies.remove('pyinotify')
 
 with open('hubblestack/__init__.py', 'r') as fd:
     version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
@@ -64,10 +65,7 @@
             'hubble = hubblestack.daemon:run',
         ],
     },
-    tests_require=[
-        'mock',
-    ],
-    install_requires=dependencies,
+    install_requires=build_dependencies,
     data_files=data_files,
     options={
 #        'build_scripts': {

test-requirements.txt

@@ -1,5 +1,4 @@
 argparse
-crypto
 daemon
 pprint
 backports-abc
@@ -17,8 +16,6 @@ markupsafe
 mock
 msgpack
 ntplib
-cryptography
-pycryptodome
 pyinotify
 pytest
 pytest-cov