Merging staging branch into prod branch

Author: atarashansky

.happy/terraform/envs/rdev/main.tf

@@ -14,8 +14,8 @@ module stack {
   batch_container_memory_limit = 28000
   wmg_batch_container_memory_limit = 248000
   wmg_desired_vcpus                = 128
-  cg_desired_vcpus                 = 48
-  cg_batch_container_memory_limit  = 92000
+  cg_desired_vcpus                 = 128
+  cg_batch_container_memory_limit  = 248000
   backend_instance_count       = 1
   frontend_instance_count      = 1
   backend_memory               = 8192

.happy/terraform/modules/ecs-stack/variables.tf

@@ -103,13 +103,13 @@ variable wmg_desired_vcpus {
 variable cg_batch_container_memory_limit {
   type        = number
   description = "Memory hard limit for the cellguide pipeline batch container"
-  default     = 92000
+  default     = 248000
 }
 
 variable cg_desired_vcpus {
   type        = number
   description = "Number of desired vCPUs for cellguide pipeline"
-  default     = 48
+  default     = 128
 }
 
 variable frontend_instance_count {

backend/cellguide/pipeline/computational_marker_genes/computational_markers.py

@@ -1,6 +1,7 @@
 import concurrent.futures
 import itertools
 import logging
+import os
 import warnings
 from typing import Tuple
 
@@ -42,6 +43,12 @@
 
 MARKER_SCORE_THRESHOLD = 0.5
 
+# This file contains blacklisted marker genes that match the following criteria:
+# Ensembl 104 gene IDs that are annotated with the GO terms: ['GO:0005840', 'GO:0005739']
+# These GO terms correspond to the cellular components "ribosome" and "mitochondrion".
+file_dir = os.path.dirname(os.path.realpath(__file__))
+MARKER_GENE_BLACKLIST_FILENAME = os.path.join(file_dir, "marker_gene_blacklist.txt")
+
 
 class MarkerGenesCalculator:
     def __init__(self, *, snapshot: WmgSnapshot, all_cell_type_ids_in_corpus: list[str], groupby_terms: list[str]):
@@ -477,6 +484,12 @@ def get_computational_marker_genes(
 
         # reset the index to convert MultiIndex back into columns
         markers_df = markers_df.reset_index()
+        with open(MARKER_GENE_BLACKLIST_FILENAME, "r") as f:
+            marker_gene_blacklist = f.read().split(",")
+        blacklisted_genes = markers_df["gene_ontology_term_id"].isin(marker_gene_blacklist)
+
+        logger.info(f"Removing {blacklisted_genes.sum()} blacklisted marker genes")
+        markers_df = markers_df[~blacklisted_genes]
 
         # get the top `num_marker_genes` genes per metadata group
         top_per_group = (

backend/cellguide/pipeline/computational_marker_genes/marker_gene_blacklist.txt

@@ -30,7 +30,7 @@ requests==2.31.0
 s3transfer==0.6.1
 scipy==1.11.1
 six==1.16.0
-tiledb==0.22.3
+tiledb==0.24.0
 typing_extensions==4.7.1
 tzdata==2023.3
 urllib3==1.26.16