|
| 1 | +""" |
| 2 | + Analysis |
| 3 | +""" |
| 4 | +import os |
| 5 | +import re |
| 6 | +import importlib |
| 7 | +import inspect |
| 8 | + |
| 9 | +from app import app, db |
| 10 | +from app.models.sample import Sample, AnalysisStatus |
| 11 | +from app.controllers.jobpool import JobPool |
| 12 | + |
| 13 | + |
| 14 | +class AnalysisFactory(object): |
| 15 | + """ |
| 16 | + Dynamically loads tasks from directory |
| 17 | + """ |
| 18 | + tasks_classes_container = None |
| 19 | + |
| 20 | + def __init__(self): |
| 21 | + self.tasks_classes_container = [] |
| 22 | + self.load_tasks() |
| 23 | + |
| 24 | + def load_tasks(self): |
| 25 | + """ |
| 26 | + Dynamically loads the tasks in the tasks/ folder. The tasks must |
| 27 | + be loaded here in order to avoid too much memory usage. |
| 28 | + """ |
| 29 | + app.logger.info("Loading tasks") |
| 30 | + srcre = re.compile('.py$', re.IGNORECASE) |
| 31 | + tasks_files = filter(srcre.search, |
| 32 | + os.listdir(app.config['TASKS_PATH'])) |
| 33 | + form_module = lambda fp: os.path.splitext(fp)[0] |
| 34 | + tasks_modules = map(form_module, tasks_files) |
| 35 | + for task_filename in tasks_modules: |
| 36 | + if not task_filename.startswith('__'): |
| 37 | + try: |
| 38 | + package_name = app.config['TASKS_PATH'].replace("/", ".") |
| 39 | + task_module = importlib.import_module( |
| 40 | + "." + task_filename, package=package_name) |
| 41 | + for task_name, task_class in inspect.getmembers( |
| 42 | + task_module): |
| 43 | + if task_name == task_filename and inspect.isclass( |
| 44 | + task_class): |
| 45 | + self.tasks_classes_container.append( |
| 46 | + (task_class, task_filename)) |
| 47 | + app.logger.info("Loaded task %s" % (task_filename)) |
| 48 | + except Exception as e: |
| 49 | + app.logger.error( |
| 50 | + "Could not load %s : %s" % |
| 51 | + (task_filename, e)) |
| 52 | + continue |
| 53 | + return True |
| 54 | + |
| 55 | + def create_analysis(self, sample): |
| 56 | + """ |
| 57 | + Creates a simple analysis from a sample. |
| 58 | + """ |
| 59 | + analysis = Analysis(sample) |
| 60 | + if analysis is None: |
| 61 | + app.logger.error("The factory couldn't generate an analysis...") |
| 62 | + return None |
| 63 | + self.assign_tasks(analysis, sample) |
| 64 | + return analysis |
| 65 | + |
| 66 | + def assign_tasks(self, analysis, sample): |
| 67 | + """ |
| 68 | + Creates tasks, and, if they will run on the sample, add them to the |
| 69 | + analysis. |
| 70 | + """ |
| 71 | + for p_class, p_name in self.tasks_classes_container: |
| 72 | + try: |
| 73 | + p_instance = p_class(sample) |
| 74 | + if p_instance.will_run(): |
| 75 | + analysis.add_task(p_instance, p_name) |
| 76 | + except Exception as e: |
| 77 | + app.logger.error("Could not load task %s : %s" % (p_name, e)) |
| 78 | + app.logger.exception(e) |
| 79 | + pass |
| 80 | + return True |
| 81 | + |
| 82 | + |
| 83 | +class AnalysisController(object): |
| 84 | + """ |
| 85 | + Manages the creation, dispatch and management of analysis tasks |
| 86 | + """ |
| 87 | + jobpool = None |
| 88 | + factory = None |
| 89 | + |
| 90 | + def __init__(self, max_instances=4): |
| 91 | + self.jobpool = JobPool(max_instances) |
| 92 | + self.factory = AnalysisFactory() |
| 93 | + |
| 94 | + def create_analysis(self, sid, force=False): |
| 95 | + """ |
| 96 | + Creates an analysis for SID sample. If force, will create the analysis |
| 97 | + even if the analysis status is FINISHED or RUNNING. |
| 98 | + """ |
| 99 | + sample = Sample.query.get(sid) |
| 100 | + if sample is None: |
| 101 | + return None |
| 102 | + if sample.analysis_status == AnalysisStatus.RUNNING and not force: |
| 103 | + return None |
| 104 | + if sample.analysis_status == AnalysisStatus.FINISHED and not force: |
| 105 | + return None |
| 106 | + return self.factory.create_analysis(sample) |
| 107 | + |
| 108 | + def dispatch_analysis(self, analysis): |
| 109 | + """ |
| 110 | + Send the analysis to the job queue. |
| 111 | + """ |
| 112 | + if analysis.tasks is None or len(analysis.tasks) == 0: |
| 113 | + return False |
| 114 | + self.jobpool.add_analysis(analysis) |
| 115 | + return True |
| 116 | + |
| 117 | + def schedule_sample_analysis(self, sid, force=False): |
| 118 | + """ |
| 119 | + Create analysis, and dispatch it to execution pool. |
| 120 | + """ |
| 121 | + analysis = self.create_analysis(sid, force) |
| 122 | + if analysis is None: |
| 123 | + app.logger.error("No analysis generated for sample %d" % (sid)) |
| 124 | + return False |
| 125 | + app.logger.info("Launching full analysis of sample %d" % (sid)) |
| 126 | + self.dispatch_analysis(analysis) |
| 127 | + return True |
| 128 | + |
| 129 | + def reschedule_all_analysis(self, force=False): |
| 130 | + """ |
| 131 | + Schedule all analyses in database. If "force" has been set to True, |
| 132 | + even FINISHED analyses are re-scheduled. RUNNING are also scheduled |
| 133 | + in order to recover from crashes. |
| 134 | + """ |
| 135 | + for sample in Sample.query.all(): |
| 136 | + if force or sample.analysis_status == AnalysisStatus.TOSTART: |
| 137 | + self.schedule_sample_analysis(sample.id, force) |
| 138 | + elif sample.analysis_status == AnalysisStatus.RUNNING: |
| 139 | + self.schedule_sample_analysis(sample.id, force) |
| 140 | + |
| 141 | + |
| 142 | +class Analysis(object): |
| 143 | + """ |
| 144 | + Analysis object, contains tasks, and manages samples status. |
| 145 | + """ |
| 146 | + sid = None |
| 147 | + tasks = None |
| 148 | + |
| 149 | + def __init__(self, sample=None): |
| 150 | + """ |
| 151 | + Only the sample ID is copyed, not the sample itself: on different |
| 152 | + processes/threads, several SQLAlchemy synchronization issues may |
| 153 | + appear. |
| 154 | + """ |
| 155 | + self.sid = sample.id |
| 156 | + self.tasks = [] |
| 157 | + return |
| 158 | + |
| 159 | + def set_started(self): |
| 160 | + """ |
| 161 | + Sets the analysis status to RUNNING (scheduled). Sets on dispatch. |
| 162 | + """ |
| 163 | + if self.sid: |
| 164 | + s = Sample.query.get(self.sid) |
| 165 | + if s: |
| 166 | + s.analysis_status = AnalysisStatus.RUNNING |
| 167 | + db.session.commit() |
| 168 | + return True |
| 169 | + |
| 170 | + def set_finished(self): |
| 171 | + """ |
| 172 | + Sets the analysis status to FINISHED. Sets by the jobpool after tasks |
| 173 | + execution. |
| 174 | + """ |
| 175 | + if self.sid: |
| 176 | + sample = Sample.query.get(self.sid) |
| 177 | + if sample: |
| 178 | + sample.analysis_status = AnalysisStatus.FINISHED |
| 179 | + db.session.commit() |
| 180 | + return True |
| 181 | + |
| 182 | + def add_task(self, task, tname): |
| 183 | + """ |
| 184 | + Adds a new task to the analysis. The task object is given, and the |
| 185 | + list is provided along with its execution level, in order to be |
| 186 | + priorized when the jobpool will execute them. |
| 187 | + """ |
| 188 | + try: |
| 189 | + execution_level = task.execution_level |
| 190 | + except Exception as e: |
| 191 | + app.logger.warning( |
| 192 | + "Could not read execution_level for task %s, default to 0" % |
| 193 | + (tname)) |
| 194 | + execution_level = 0 |
| 195 | + if execution_level < 0: |
| 196 | + execution_level = 0 |
| 197 | + if execution_level > 32: |
| 198 | + execution_level = 32 |
| 199 | + self.tasks.append((execution_level, task)) |
| 200 | + app.logger.info("Task loaded: %s" % (tname)) |
| 201 | + return True |
0 commit comments