flake8 + travis

Author: lightforever

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+exclude = .git,__pycache__,docs,build,dist,mlcomp/server/front

.style.yapf

@@ -0,0 +1,6 @@
+[style]
+based_on_style=pep8
+allow_split_before_dict_value=False
+join_multiple_lines=False
+split_before_named_assigns=True
+dedent_closing_brackets=True

.travis.yml

@@ -0,0 +1,35 @@
+group: travis_latest
+language: python
+cache: pip
+
+branches:
+  only:
+  - master
+
+python:
+  - '3.6'
+
+before_install: &before_install
+  sudo ls
+
+install: &requirements
+  pip install -r requirements.txt
+
+jobs:
+  include:
+    - stage: Tests
+      install:
+        - *requirements
+        - pip install flake8 flake8-quotes yapf
+        - pip install -U pytest
+      script:
+        # stop the build if there are any unexpected flake8 issues
+        - flake8 . --count
+          --max-complexity=16
+          --inline-quotes "single"
+          --multiline-quotes "single"
+          --docstring-quotes "single"
+          --show-source --statistics
+        # test to make sure the code is yapf compliant
+        #- ./yapf.sh --all
+        - pytest

README.md

@@ -1,4 +1,4 @@
-# mlcomp
+# MLComp
 Machine learning pipelines. Especially, for competitions, like Kaggle
 
 ![MLComp logo](mlcomp/server/front/src/assets/img/mlcomp_logo.jpg)

examples/cifar/__init__.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 from catalyst.dl import SupervisedRunner as Runner
 from experiment import Experiment
-from model import SimpleNet
+from model import SimpleNet

examples/mnist/__init__.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 from catalyst.dl import SupervisedRunner as Runner
 from experiment import Experiment
 from model import Net

examples/mnist/dataset.py

@@ -5,12 +5,13 @@
 
 
 class MnistDataset(Dataset):
-    def __init__(self,
-                 file: str,
-                 fold_csv: str = None,
-                 fold_number: int = 0,
-                 is_test: bool = False
-                 ):
+    def __init__(
+        self,
+        file: str,
+        fold_csv: str = None,
+        fold_number: int = 0,
+        is_test: bool = False
+    ):
         df = pd.read_csv(file)
         if fold_csv is not None:
             fold = pd.read_csv(fold_csv)
@@ -29,7 +30,7 @@ def __init__(self,
 
     def __len__(self):
         return len(self.x)
-    
+
     def __getitem__(self, index):
         res = {'features': self.x[index] / 255}
         if self.y is not None:

mlcomp/__init__.py

@@ -1 +1 @@
-from .__version__ import __version__  # noqa: F401
+from .__version__ import __version__  # noqa: F401

mlcomp/__main__.py

@@ -8,9 +8,13 @@
 
 from mlcomp.db.enums import DagType, ComponentType, TaskStatus
 from mlcomp.db.models import Computer
+from mlcomp.db.providers import \
+    ComputerProvider, \
+    TaskProvider, \
+    StepProvider, \
+    ProjectProvider
 from mlcomp.utils.io import yaml_load
 from mlcomp.utils.logging import create_logger
-from mlcomp.db.providers import *
 from multiprocessing import cpu_count
 
 from mlcomp.utils.settings import ROOT_FOLDER, DATA_FOLDER, MODEL_FOLDER
@@ -26,25 +30,26 @@ def _dag(config: str, debug: bool = False):
 
     type_name = config_parsed['info'].get('type', "standard")
     if type_name == DagType.Standard.name.lower():
-        return dag_standard(config_parsed,
-                            debug=debug,
-                            config_text=config_text)
+        return dag_standard(
+            config_parsed, debug=debug, config_text=config_text
+        )
 
-    return dag_pipe(config_parsed,
-                    config_text=config_text)
+    return dag_pipe(config_parsed, config_text=config_text)
 
 
 def _create_computer():
     tot_m, used_m, free_m = memory()
     tot_d, used_d, free_d = disk(ROOT_FOLDER)
-    computer = Computer(name=socket.gethostname(),
-                        gpu=len(GPUtil.getGPUs()),
-                        cpu=cpu_count(), memory=tot_m,
-                        ip=os.getenv('IP'),
-                        port=int(os.getenv('PORT')),
-                        user=os.getenv('USER'),
-                        disk=tot_d
-                        )
+    computer = Computer(
+        name=socket.gethostname(),
+        gpu=len(GPUtil.getGPUs()),
+        cpu=cpu_count(),
+        memory=tot_m,
+        ip=os.getenv('IP'),
+        port=int(os.getenv('PORT')),
+        user=os.getenv('USER'),
+        disk=tot_d
+    )
     ComputerProvider().create_or_update(computer, 'name')
 
 
@@ -72,16 +77,15 @@ def execute(config: str, debug: bool):
     provider = TaskProvider()
     step_provider = StepProvider()
 
-    for t in provider.by_status(TaskStatus.InProgress,
-                                worker_index=worker_index):
+    for t in provider.by_status(
+        TaskStatus.InProgress, worker_index=worker_index
+    ):
         step = step_provider.last_for_task(t.id)
         logger.error(
             f'Task Id = {t.id} was in InProgress state '
             f'when another tasks arrived to the same worker',
-            ComponentType.Worker,
-            t.computer_assigned,
-            t.id,
-            step)
+            ComponentType.Worker, t.computer_assigned, t.id, step
+        )
         provider.change_status(t, TaskStatus.Failed)
 
     # Create dag
@@ -93,12 +97,16 @@ def execute(config: str, debug: bool):
 
 @main.command()
 @click.option('--computer', help='sync computer with all the others')
-@click.option('--only_from',
-              is_flag=True,
-              help='only copy files from the computer to all the others')
-@click.option('--only_to',
-              is_flag=True,
-              help='only copy files from all the others to the computer')
+@click.option(
+    '--only_from',
+    is_flag=True,
+    help='only copy files from the computer to all the others'
+)
+@click.option(
+    '--only_to',
+    is_flag=True,
+    help='only copy files from all the others to the computer'
+)
 def sync(computer: str, only_from: bool, only_to: bool):
     computer = computer or socket.gethostname()
     provider = ComputerProvider()

mlcomp/contrib/catalyst/__init__.py

@@ -1 +1,3 @@
-from .register import register
+from .register import register
+
+__all__ = ['register']

mlcomp/contrib/catalyst/optim/__init__.py

@@ -1 +1,3 @@
-from .cosineanneal import OneCycleCosineAnnealLR
+from .cosineanneal import OneCycleCosineAnnealLR
+
+__all__ = ['OneCycleCosineAnnealLR']

mlcomp/contrib/catalyst/register.py

@@ -13,4 +13,4 @@ def register():
 
     registry.Callback(InferBestCallback)
 
-    registry.Scheduler(OneCycleCosineAnnealLR)
+    registry.Scheduler(OneCycleCosineAnnealLR)

mlcomp/contrib/criterion/__init__.py

@@ -1 +1,3 @@
-from .ring import RingLoss
+from .ring import RingLoss
+
+__all__ = ['RingLoss']

mlcomp/contrib/model/__init__.py

@@ -1 +1,3 @@
-from .pretrained import Pretrained
+from .pretrained import Pretrained
+
+__all__ = ['Pretrained']

mlcomp/contrib/scripts/split.py

@@ -5,12 +5,14 @@
 from sklearn.model_selection import GroupKFold
 
 
-def file_group_kfold(n_splits: int,
-                     output: str,
-                     get_group=None,
-                     sort=False,
-                     must_equal = (),
-                     **files):
+def file_group_kfold(
+    n_splits: int,
+    output: str,
+    get_group=None,
+    sort=False,
+    must_equal=(),
+    **files
+):
     assert len(files) > 0, 'at lease 1 type of files is required'
     fold = GroupKFold(n_splits)
     keys = sorted(list(files))
@@ -38,16 +40,18 @@ def get_name(file):
             names_equal = get_name(v[i]) == get_name(file_first[i])
             assert names_equal, \
                 f'file name in {k} does not equal to {keys[0]}, ' \
-                    f'file name = {basename(v[i])}'
+                f'file name = {basename(v[i])}'
 
     df = pd.DataFrame(files)[keys]
     df['fold'] = 0
 
-    groups = [i if not get_group else get_group(file) for i, file in
-              enumerate(file_first)]
+    groups = [
+        i if not get_group else get_group(file)
+        for i, file in enumerate(file_first)
+    ]
 
-    for i, (train_index, test_index) in enumerate(
-            fold.split(groups, groups=groups)):
+    for i, (train_index,
+            test_index) in enumerate(fold.split(groups, groups=groups)):
         df.loc[test_index, 'fold'] = i
 
     df = df.sample(frac=1)

mlcomp/contrib/split/__init__.py

@@ -1 +1,3 @@
 from .frame import split_frame
+
+__all__ = ['split_frame']

mlcomp/db/__init__.py

@@ -1 +1,2 @@
+# flake8: noqa
 from .signals import *

mlcomp/db/conf.py

@@ -8,7 +8,7 @@
     'host': os.getenv('POSTGRES_HOST', 'localhost'),
     'port': int(os.getenv('POSTGRES_PORT', 5432)),
 }
-if DB_DRIVER=='sqlite':
+if DB_DRIVER == 'sqlite':
     f = os.path.dirname(__file__)
     f = os.path.join(f, '../migration')
     f = os.path.abspath(f)
@@ -18,6 +18,4 @@
         f"{DATABASE['password']}@{DATABASE['host']}:" \
         f"{DATABASE['port']}/{DATABASE['dbname']}"
 
-__all__ = [
-    'SA_CONNECTION_STRING'
-]
+__all__ = ['SA_CONNECTION_STRING']

mlcomp/db/core/__init__.py

@@ -1,2 +1,4 @@
-from .db import *
-from .options import *
+from .db import Session
+from .options import PaginatorOptions
+
+__all__ = ['Session', 'PaginatorOptions']

mlcomp/db/core/db.py

@@ -2,11 +2,9 @@
 import sqlalchemy.orm.session as session
 from sqlalchemy.orm import scoped_session, sessionmaker
 
-from mlcomp.db.conf import *
+from mlcomp.db.conf import SA_CONNECTION_STRING
 from mlcomp.utils.misc import adapt_db_types
 
-__all__ = ['Session']
-
 
 class Session(session.Session):
     __session = dict()
@@ -23,19 +21,20 @@ def create_session(connection_string: str = None, key='default'):
             return Session.__session[key][0]
 
         session_factory = scoped_session(sessionmaker(class_=Session, key=key))
-        engine = sa.create_engine(connection_string or SA_CONNECTION_STRING,
-                                  echo=False)
+        engine = sa.create_engine(
+            connection_string or SA_CONNECTION_STRING, echo=False
+        )
         session_factory.configure(bind=engine)
-        session = session_factory()
+        s = session_factory()
 
-        Session.__session[key] = [session, engine]
-        return session
+        Session.__session[key] = [s, engine]
+        return s
 
     @classmethod
     def cleanup(cls):
-        for k, (session, engine) in cls.__session.items():
+        for k, (s, engine) in cls.__session.items():
             try:
-                session.close()
+                s.close()
             except Exception:
                 pass
             try:
@@ -94,3 +93,6 @@ def update(self):
         except Exception as e:
             self.rollback()
             raise e
+
+
+__all__ = ['Session']

mlcomp/db/core/options.py

@@ -1,16 +1,18 @@
 class PaginatorOptions:
-    def __init__(self,
-                 page_number: int,
-                 page_size: int,
-                 sort_column: str = None,
-                 sort_descending: bool = None):
+    def __init__(
+        self,
+        page_number: int,
+        page_size: int,
+        sort_column: str = None,
+        sort_descending: bool = None
+    ):
         self.sort_column = sort_column
         self.sort_descending = sort_descending
         self.page_number = page_number
         self.page_size = page_size
 
         assert (page_number is not None and page_size) \
-               or (not page_number is None and not page_size), \
+            or (page_number is not None and not page_size), \
             'Specify both page_number and page_size'
 
         if not sort_column: