last changes

Author: lightforever

examples/cifar_simple/catalyst.yml

@@ -35,7 +35,7 @@ stages:
       accuracy_args: [1]
     scheduler:
       callback: SchedulerCallback
-      reduce_metric: *reduce_metric
+      reduced_metric: *reduce_metric
     saver:
       callback: CheckpointCallback
 

mlcomp/__version__.py

@@ -1 +1 @@
-__version__ = '20.3'
+__version__ = '20.3.1a'

mlcomp/db/models/__init__.py

@@ -10,10 +10,12 @@
 from .docker import Docker
 from .model import Model
 from .auxilary import Auxiliary
+from .memory import Memory
+from .space import Space
 
 __all__ = [
     'Project', 'Task', 'TaskDependence', 'File', 'DagStorage', 'DagLibrary',
     'Computer', 'ComputerUsage', 'Log', 'Step', 'Dag', 'ReportSeries',
     'ReportImg', 'ReportTasks', 'Report', 'ReportLayout', 'Docker', 'Model',
-    'Auxiliary', 'TaskSynced'
+    'Auxiliary', 'TaskSynced', 'Memory', 'Space'
 ]

mlcomp/db/models/memory.py

@@ -0,0 +1,18 @@
+import sqlalchemy as sa
+
+from mlcomp.db.models.base import Base
+
+
+class Memory(Base):
+    __tablename__ = 'memory'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    model = sa.Column(sa.String, nullable=False)
+    variant = sa.Column(sa.String)
+    num_classes = sa.Column(sa.Integer)
+    img_size = sa.Column(sa.Integer)
+    batch_size = sa.Column(sa.Integer, nullable=False)
+    memory = sa.Column(sa.Float, nullable=False)
+
+
+__all__ = ['Memory']

mlcomp/db/models/space.py

@@ -0,0 +1,25 @@
+import sqlalchemy as sa
+from sqlalchemy import ForeignKey
+
+from mlcomp.db.models.base import Base
+
+
+class Space(Base):
+    __tablename__ = 'space'
+
+    name = sa.Column(sa.String, nullable=False, primary_key=True)
+    created = sa.Column(sa.DateTime, nullable=False)
+    changed = sa.Column(sa.DateTime, nullable=False)
+    content = sa.Column(sa.String, nullable=False)
+
+
+class SpaceRelation(Base):
+    __tablename__ = 'space_relation'
+
+    parent = sa.Column(sa.String, ForeignKey('space.name'),
+                       primary_key=True)
+    child = sa.Column(sa.String, ForeignKey('space.name'),
+                      primary_key=True)
+
+
+__all__ = ['Space', 'SpaceRelation']

mlcomp/db/providers/__init__.py

@@ -16,12 +16,14 @@
 from .model import ModelProvider
 from .auxiliary import AuxiliaryProvider
 from .task_synced import TaskSyncedProvider
+from .memory import MemoryProvider
+from .space import SpaceProvider
 
 __all__ = [
     'ProjectProvider', 'TaskProvider', 'FileProvider', 'DagStorageProvider',
     'DagLibraryProvider', 'LogProvider', 'StepProvider', 'ComputerProvider',
     'DagProvider', 'ReportImgProvider', 'ReportProvider',
     'ReportLayoutProvider', 'ReportSeriesProvider', 'ReportTasksProvider',
     'DockerProvider', 'ModelProvider', 'AuxiliaryProvider',
-    'TaskSyncedProvider'
+    'TaskSyncedProvider', 'MemoryProvider', 'SpaceProvider'
 ]

mlcomp/db/providers/base.py

@@ -51,11 +51,11 @@ def add(self, obj: Base, commit=True):
         return obj
 
     def bulk_save_objects(
-        self,
-        objs,
-        return_defaults=False,
-        update_changed_only=True,
-        preserve_order=True,
+            self,
+            objs,
+            return_defaults=False,
+            update_changed_only=True,
+            preserve_order=True,
     ):
         for obj in objs:
             adapt_db_types(obj)
@@ -68,8 +68,9 @@ def bulk_save_objects(
         )
         self._session.commit()
 
-    def by_id(self, id: int, joined_load=None):
-        res = self.query(self.model).filter(getattr(self.model, 'id') == id)
+    def by_id(self, id: int, joined_load=None, key_column: str = 'id'):
+        res = self.query(self.model).filter(
+            getattr(self.model, key_column) == id)
         if joined_load is not None:
             for n in joined_load:
                 res = res.options(joinedload(n, innerjoin=True))

mlcomp/db/providers/memory.py

@@ -0,0 +1,39 @@
+from mlcomp.db.core import PaginatorOptions
+from mlcomp.db.models import Memory
+from mlcomp.db.providers.base import BaseDataProvider
+
+
+class MemoryProvider(BaseDataProvider):
+    model = Memory
+
+    def get(self, filter: dict, options: PaginatorOptions = None):
+        query = self.query(Memory)
+        if filter.get('model'):
+            query = query.filter(Memory.model.contains(filter['model']))
+        if filter.get('variant'):
+            query = query.filter(Memory.variant.contains(filter['variant']))
+
+        total = query.count()
+        paginator = self.paginator(query, options) if options else query
+        data = []
+        for p in paginator.all():
+            item = self.to_dict(p)
+            data.append(item)
+
+        return {
+            'total': total,
+            'data': data
+        }
+
+    def find(self, data: dict):
+        query = self.query(Memory)
+        for k, v in data.items():
+            if k in ['batch_size']:
+                continue
+
+            if hasattr(Memory, k):
+                query = query.filter(getattr(Memory, k) == v)
+        return query.all()
+
+
+__all__ = ['MemoryProvider']

mlcomp/db/providers/space.py

@@ -0,0 +1,61 @@
+from sqlalchemy import literal_column
+
+from mlcomp.db.core import PaginatorOptions
+from mlcomp.db.models import Space
+from mlcomp.db.models.space import SpaceRelation
+from mlcomp.db.providers.base import BaseDataProvider
+
+
+class SpaceProvider(BaseDataProvider):
+    model = Space
+
+    def get(self, filter: dict, options: PaginatorOptions = None):
+        query = self.query(Space, literal_column('0').label('relation'))
+        if 'parent' in filter:
+            query = query.filter(Space.name != filter['parent'])
+
+        if filter.get('name'):
+            query = query.filter(Space.name.contains(filter['name']))
+        if filter.get('parent'):
+            relation = literal_column('1').label('relation')
+            query2 = self.query(Space, relation). \
+                join(SpaceRelation, SpaceRelation.child == Space.name).filter(
+                SpaceRelation.parent == filter['parent']
+            )
+            query2_names = [space.name for space, _ in query2.all()]
+            query = query.filter(Space.name.notin_(query2_names))
+            query = query.union_all(query2)
+            query = query.order_by(relation.desc())
+
+        total = query.count()
+        paginator = self.paginator(query, options) if options else query
+        data = []
+        for space, relation in paginator.all():
+            item = self.to_dict(space)
+            item['relation'] = relation
+            data.append(item)
+
+        return {
+            'total': total,
+            'data': data
+        }
+
+    def add_relation(self, parent: str, child: str):
+        self.add(SpaceRelation(parent=parent, child=child))
+
+    def remove_relation(self, parent: str, child: str):
+        self.query(SpaceRelation).filter(
+            SpaceRelation.parent == parent).filter(
+            SpaceRelation.child == child).delete(synchronize_session=False)
+
+        self.session.commit()
+
+    def related(self, parent: str):
+        res = self.query(Space).join(SpaceRelation,
+                                     SpaceRelation.child == Space.name) \
+            .filter(SpaceRelation.parent == parent) \
+            .all()
+        return res
+
+
+__all__ = ['SpaceProvider']

mlcomp/db/providers/task.py

@@ -315,5 +315,13 @@ def find_dependents(self, task_id: int):
             all()
         return res
 
+    def get_dependencies(self, dag: int):
+        res = self.query(TaskDependence).join(
+            Task, Task.id == TaskDependence.task_id).\
+            filter(Task.dag == dag).\
+            all()
+
+        return res
+
 
 __all__ = ['TaskProvider']