Introducing the taXaminer-dashboard extension

Author: lukekoch

flask-backend/src/Routes/annotations.py

@@ -13,6 +13,7 @@
     fetchFeatures,
     import_annotation,
     updateAnnotationLabel,
+    grepFeature
 )
 
 # setup blueprint name
@@ -173,6 +174,31 @@ def annotations_bp_fetchFeatures():
         return REQUESTMETHODERROR
 
 
+# FETCH ALL ASSEMBLIES
+@annotations_bp.route("/grepFeatures", methods=["POST"])
+def annotations_bp_grepFeatures():
+    if request.method == "POST":
+        req = request.get_json(force=True)
+        userID = req.get("userID", None)
+        token = req.get("token", None)
+
+        # token still active
+        valid_token, error = validateActiveToken(userID, token, ACCESS_LVL_1)
+        if not valid_token:
+            response = jsonify({"payload": {}, "notification": error})
+            response.headers.add("Access-Control-Allow-Origin", "*")
+            return response
+        
+        # grep input
+        search = req.get("search", None)
+        annotation_id = req.get("annotationID", None)
+
+        coords = grepFeature(search, annotation_id)
+        return jsonify({"coords": coords})
+    else:
+        return REQUESTMETHODERROR
+
+
 # FETCH ALL UNIQUE FEATURE TYPES
 @annotations_bp.route("/fetchFeatureSeqIDs", methods=["POST"])
 def annotations_bp_fetchFeatureSeqIDs():

flask-backend/src/Routes/taxaminer_data.py

@@ -1,7 +1,9 @@
 # general imports
+import ast
 from email.mime import base
 import json
 from pathlib import Path
+import sys
 from urllib import response
 from flask import Blueprint, jsonify, request, abort, Response
 from . import file_io
@@ -117,7 +119,8 @@ def main_data():
 
     path = get_basepath(assembly_id=assembly_id, analysis_id=analysis_id)
 
-
+    if not path:
+        return abort(500)
     json_data = file_io.indexed_data(f"{path}gene_table_taxon_assignment.csv")
 
     # return as json
@@ -145,9 +148,36 @@ def diamond_data():
         return response
 
     try: 
-        json_data = fetchTaxaminerDiamond(assembly_id, analysis_id, qseq_id)
-        return jsonify(json_data)
-    except Exception:
+        DIAMOND_FIELDS = fields = ['qseqid', 'sseqid', 'pident', 'length', 'mismatch', 'gapopen', 'qstart', 'qend', 'sstart', 'send', 'evalue', 'bitscore', 'taxids', 'taxname', 'assemblyID', 'analysisID']
+        index_data = fetchTaxaminerDiamond(assembly_id, analysis_id, qseq_id)
+        if index_data == []:
+            return jsonify([])
+        total_count = int(index_data['stop']) - int(index_data['start'])
+
+        # get on disk
+        path = get_basepath(assembly_id=assembly_id, analysis_id=analysis_id)
+        print((index_data, path), file=sys.stderr)
+
+        final_lines = []
+        with open(path + "taxonomic_hits.txt") as f:
+            for i in range(int(index_data['start'])):
+                next(f)
+            counter = 0
+            for line in f:
+                # ensure we stop at the end of the block
+                if counter == total_count:
+                    break
+
+                temp_dict = {}
+                fields = line.split("\t")
+                for i, field in enumerate(fields):
+                    temp_dict[DIAMOND_FIELDS[i]] = field
+                final_lines.append(temp_dict)
+                counter += 1
+
+        return jsonify(final_lines)
+    except Exception as e:
+        print(e, file=sys.stderr)
         return abort(404)
 
 
@@ -237,16 +267,18 @@ def get_config():
         fields = fetchTaxaminerSettings(userID, analysisID)
         # no previous settings
         if not fields:
-            setTaxaminerSettings(userID, analysisID, "[]")
-            return jsonify("[]")
+            setTaxaminerSettings(userID, analysisID, "[]", "[]")
+            return jsonify({"custom_fields": [], "selection": []})
         else:
-            fields_json = json.loads(fields[0])
-            return jsonify(fields_json)
+            data_json = json.loads(fields[0])
+            print(fields, file=sys.stderr)
+            return jsonify({"custom_fields": ast.literal_eval(fields[0]), "selection": ast.literal_eval(fields[1])})
     # store settings in database
     elif request.method == "PUT":
         # TODO: add support for additional settings
         new_fields = request.json['fields']
-        setTaxaminerSettings(userID, analysisID, json.dumps(new_fields))
+        new_seletion = request.json['selection']
+        setTaxaminerSettings(userID, analysisID, json.dumps(new_fields), json.dumps(new_seletion))
         return jsonify(new_fields)
 
 
@@ -272,7 +304,7 @@ def pca_contributions():
     basepath = get_basepath(assembly_id=assembly_id, analysis_id=analysis_id)
 
     if basepath:
-        with open(f"{basepath}pca_loadings.csv", 'r') as file:
+        with open(f"{basepath}contribution_of_variables.csv", 'r') as file:
             lines = file.readlines()
 
         final_lines = []

flask-backend/src/modules/analyses.py

@@ -13,7 +13,7 @@
 
 import json
 
-DIAMOND_FIELDS = fields = ['qseqid', 'sseqid', 'pident', 'length', 'mismatch', 'gapopen', 'qstart', 'qend', 'sstart', 'send', 'evalue', 'bitscore', 'taxids', 'taxname', 'assemblyID', 'analysisID']
+DIAMOND_FIELDS = fields = ['assemblyID', 'analysisID', 'qseqid', 'start', 'stop']
 
 ## ============================ IMPORT AND DELETE ============================ ##
 # full import of analyses
@@ -547,38 +547,39 @@ def __importTaxaminer(assemblyID, analysisID, base_path):
         cursor.execute("INSERT INTO analysesTaxaminer (analysisID) VALUES (%s)", (analysisID,))
         connection.commit()
 
-        """
-        # parse diamond
+        # Load taxonomic hits
         diamond_path = base_path + "taxonomic_hits.txt"
+        print(diamond_path)
         if not os.path.isfile(diamond_path):
             return 0, createNotification(message=f"taXaminerImportDBError: Diamond data is missing!")
-        
-        FIELDS = ['qseqid', 'sseqid', 'pident', 'length', 'mismatch', 'gapopen', 'qstart', 'qend', 'sstart', 'send', 'evalue', 'bitscore', 'taxids', 'taxname']
-        TYPES = {'qseqid': str, 'sseqid': str, 'pident': float, 'length': float, 'mismatch': float, 'gapopen': float, 'qstart': float,
-         'qend': float, 'sstart': float, 'send': float, 'evalue': float, 'bitscore': float, 'taxids': str, 'taxname': str}
-        rows = []
+
+        # build data rows
+        # => save assemblyID, analysisID, qseqID together with the row number to index file
+        sql_rows = []
         with open(diamond_path) as file:
-            my_reader = csv.DictReader(file, delimiter='\t', fieldnames=FIELDS)
-            for row in my_reader:
-                # manually set types
-                for field in FIELDS:
-                    if TYPES.get(field) != str:
-                        if TYPES.get(field) == int:
-                            row[field] = int(row[field])
-                        elif TYPES.get(field) == float:
-                            row[field] = float(row[field])
-                # cleared for db insert
-                rows.append((assemblyID, analysisID, row['qseqid'], json.dumps(row)))
-
-        print("Database Inserts look like this:" + str(rows[0]))
-
-        # .executemany() exceeds the 'max_allowed_packet'
-        # if you encounter this error use 'SET SESSION max_allowed_packet=500*1024*1024' or 'SET GLOBAL max_allowed_packet=500*1024*1024'
-        # TLDR: MOOOOOOOOOOREEEEEEE RAM
+            start_index = 0
+            curr_id = ""
+            outer_index = 0
+            for i, line in enumerate(file.readlines()):
+                # primer
+                if i == 0:
+                    curr_id = line.split("\t")[0]
+                
+                # determine new id
+                next_id = line.split("\t")[0]
+                if next_id != curr_id:
+                    # start -> stop
+                    sql_rows.append((assemblyID, analysisID, curr_id, start_index, i-1))
+                    curr_id = next_id
+                    start_index = i
+                outer_index = i
+            
+            # final row
+            sql_rows.append((assemblyID, analysisID, curr_id, start_index, outer_index))
+        
         connection, cursor, error = connect()
-        cursor.executemany("INSERT INTO taxaminerDiamond (assemblyID, analysisID, qseqID, data) VALUES (%s, %s, %s, %s)", rows)
+        cursor.executemany("INSERT INTO taxaminerDiamond (assemblyID, analysisID, qseqID, start, stop) VALUES (%s, %s, %s, %s, %s)", sql_rows)
         connection.commit()
-        """
 
         return 1, []
     except Exception as err:
@@ -707,10 +708,10 @@ def deleteAnalysesByAnalysesID(analyses_id):
     try:
         connection, cursor, error = connect()
         cursor.execute(
-            "SELECT assemblies.id, assemblies.name, analyses.path FROM assemblies, analyses WHERE analyses.id=%s AND analyses.assemblyID=assemblies.id",
+            "SELECT assemblies.id, assemblies.name, analyses.path, analyses.type FROM assemblies, analyses WHERE analyses.id=%s AND analyses.assemblyID=assemblies.id",
             (analyses_id,),
         )
-        assembly_id, assembly_name, analyses_path = cursor.fetchone()
+        assembly_id, assembly_name, analyses_path, analysis_type = cursor.fetchone()
 
         cursor.execute(
             "SELECT taxa.* FROM assemblies, taxa WHERE assemblies.id=%s AND assemblies.taxonID=taxa.id",
@@ -725,7 +726,7 @@ def deleteAnalysesByAnalysesID(analyses_id):
             status, error = __deleteAnalysesEntryByAnalysesID(analyses_id)
 
         if status and taxon and assembly_name and analyses_path:
-            status, error = __deleteAnalysesFile(taxon, assembly_name, analyses_path)
+            status, error = __deleteAnalysesFile(taxon, assembly_name, analyses_path, type=analysis_type)
         else:
             return 0, error
 
@@ -740,7 +741,7 @@ def deleteAnalysesByAnalysesID(analyses_id):
 
 
 # deletes files for annotation
-def __deleteAnalysesFile(taxon, assembly_name, analyses_path):
+def __deleteAnalysesFile(taxon, assembly_name, analyses_path, type=""):
     """
     Deletes data for specific annotation.
     """
@@ -749,6 +750,11 @@ def __deleteAnalysesFile(taxon, assembly_name, analyses_path):
         path = f"{BASE_PATH_TO_STORAGE}taxa/{scientificName}"
 
         run(args=["rm", "-r", analyses_path])
+        if type == "taxaminer":
+            print("Analysis is taXaminer, deleting parent directory as well")
+            # go one folder up
+            taxaminer_folder = "/".join(analyses_path.split("/")[0:-1])
+            run(args=["rm", "-r", taxaminer_folder])
 
         return 1, createNotification("Success", "Successfully deleted analyses", "success")
     except Exception as err:
@@ -759,6 +765,7 @@ def __deleteAnalysesEntryByAnalysesID(id):
     try:
         connection, cursor, error = connect()
         cursor.execute("DELETE FROM analyses WHERE id=%s", (id,))
+        cursor.execute("DELETE FROM taxaminerDiamond WHERE analysisID=%s", (id,))
         connection.commit()
         return 1, []
     except Exception as err:
@@ -1210,18 +1217,20 @@ def fetchRepeatmaskerAnalysesByAssemblyID(assemblyID):
 def fetchTaxaminerDiamond(assemblyID, analysisID, qseqid):
     try:
         connection, cursor, error = connect()
-        cursor.execute("SELECT * FROM taxaminerDiamond WHERE assemblyID=%s AND analysisID=%s AND qseqID=%s",
+        cursor.execute("SELECT * FROM taxaminerDiamond, analysesTaxaminer WHERE taxaminerDiamond.analysisID=analysesTaxaminer.analysisID AND taxaminerDiamond.assemblyID=%s AND analysesTaxaminer.id=%s AND qseqID=%s",
         (assemblyID, analysisID, qseqid)
         )
-        rows = cursor.fetchall()
-        final_rows = []
-        for row in rows:
-            temp_dict = dict()
-            for i in range(len(row)):
-                temp_dict[DIAMOND_FIELDS[i]] = row[i]
-            final_rows.append(temp_dict)
-
-        return final_rows
+        row = cursor.fetchone()
+
+        # catch no entries
+        if not row:
+            return []
+        
+        temp_dict = dict()
+        for i in range(0, 5):
+            temp_dict[DIAMOND_FIELDS[i]] = row[i]
+
+        return temp_dict
     except Exception as err:
         return 0, createNotification(message=str(err))
 

flask-backend/src/modules/annotations.py

@@ -6,6 +6,8 @@
 from subprocess import run
 from glob import glob
 from operator import contains, is_, is_not, lt, le, eq, ne, ge, gt
+import subprocess
+import sys
 
 from .notifications import createNotification
 from .db_connection import connect, DB_NAME
@@ -871,6 +873,27 @@ def fetchFeatureSeqIDs(assemblyID=0, taxonIDs=[]):
         return [], createNotification(message=f"FeatureTypesFetchingError: {str(err)}")
 
 
+def grepFeature(search, annotation_id):
+    try:
+        connection, cursor, error = connect()
+        cursor.execute("SELECT path FROM genomicAnnotations WHERE id=35;")
+        annotation_path = cursor.fetchone()[0]
+
+        zcat_annot = subprocess.Popen(["zcat", annotation_path], stdout=subprocess.PIPE)
+        grep_feature = subprocess.run(["grep", search], stdin=zcat_annot.stdout, capture_output=True, text=True)
+        my_result = grep_feature.stdout.split("\n")[0]
+        if my_result != "":
+            cols = my_result.split("\t")
+            pos_string = f"{cols[0]}:{cols[3]}..{cols[4]}"
+            return pos_string
+        else:
+            return ""
+
+    except Exception as err:
+        return 0, createNotification(message=str(err))
+
+
+
 # fetches all unique feature types from all features
 def fetchFeatureTypes(assemblyID=0, taxonIDs=[], seqIDs=[]):
     """

flask-backend/src/modules/assemblies.py

@@ -40,6 +40,7 @@ def import_assembly(taxon, dataset, userID, taskID=""):
             return 0, createNotification(message="Missing user ID!")
 
         assembly_id, error = __get_new_assembly_ID()
+        print(assembly_id, flush=True)
         if not assembly_id:
             return 0, error
 
@@ -51,7 +52,7 @@ def import_assembly(taxon, dataset, userID, taskID=""):
         main_file_path, assembly_name, error = __store_assembly(dataset, taxon, assembly_id)
         if not main_file_path or not exists(main_file_path):
             deleteAssemblyByAssemblyID(assembly_id)
-            print(error)
+            print(error, flush=True)
             return 0, error
 
         fasta_content, error = parseFasta(main_file_path, taskID)
@@ -129,7 +130,7 @@ def __get_new_assembly_ID():
         else:
             next_id = auto_increment_counter
 
-        cursor.execute("ALTER TABLE assemblies AUTO_INCREMENT = %s", (next_id + 1,))
+        # cursor.execute("ALTER TABLE assemblies AUTO_INCREMENT = %s", (next_id + 1,))
         connection.commit()
     except Exception as err:
         return 0, createNotification(message=f"AssemblyCreationError: {str(err)}")

flask-backend/src/modules/combined_imports.py

@@ -6,6 +6,7 @@
 from re import compile
 from sys import argv
 from datetime import datetime
+from .db_connection import connect
 
 from modules.environment import BASE_PATH_TO_IMPORT
 from modules.assemblies import (
@@ -295,8 +296,9 @@ def importDataset(
                 return summary, createNotification(message="Exact one assembly needs to be supplied!")
 
             assembly = assembly[0]
-
             assembly_id, notification = import_assembly(taxon, assembly, userID, taskID)
+
+
             if not assembly_id:
                 return summary, notification
             summary["assemblyID"] = assembly_id

flask-backend/src/modules/users.py

@@ -326,7 +326,7 @@ def removeBookmark(userID, assemblyID):
 def fetchTaxaminerSettings(userID, analysisID):
     try:
         connection, cursor, error = connect()
-        cursor.execute("SELECT custom_fields FROM settingsTaxaminer WHERE analysisID=%s AND userID=%s",
+        cursor.execute("SELECT custom_fields, selection FROM settingsTaxaminer WHERE analysisID=%s AND userID=%s",
         (analysisID, userID)
         )
         settings = cursor.fetchone()
@@ -336,11 +336,11 @@ def fetchTaxaminerSettings(userID, analysisID):
 
 
 """ set taXaminer settings"""
-def setTaxaminerSettings(userID, analysisID, json_string, setting="fields"):
+def setTaxaminerSettings(userID, analysisID, custom_fields_json, selection_json, setting="fields"):
     try:
         connection, cursor, error = connect()
-        cursor.execute("INSERT INTO settingsTaxaminer (userID, analysisID, custom_fields) VALUES (%s,%s,%s) ON DUPLICATE KEY UPDATE custom_fields=%s;",
-        (userID, analysisID, json_string, json_string)
+        cursor.execute("INSERT INTO settingsTaxaminer (userID, analysisID, custom_fields, selection) VALUES (%s,%s,%s,%s) ON DUPLICATE KEY UPDATE custom_fields=%s, selection=%s;",
+        (userID, analysisID, custom_fields_json, selection_json, custom_fields_json, selection_json)
         )
         connection.commit()
     except Exception as err: