feat(wmg): census-based wmg builder - snapshot v3 (#6020)

Author: atarashansky

Dockerfile.wmg_pipeline

@@ -19,6 +19,8 @@ ADD backend/wmg/__init__.py backend/wmg/__init__.py
 ADD backend/wmg/config.py backend/wmg/config.py
 ADD backend/wmg/data backend/wmg/data
 ADD backend/wmg/pipeline backend/wmg/pipeline
+ADD backend/wmg/api backend/wmg/api
+ADD backend/cellguide/pipeline backend/cellguide/pipeline
 ADD backend/layers backend/layers
 ADD backend/common backend/common
 
@@ -29,4 +31,4 @@ LABEL commit=${HAPPY_COMMIT}
 ENV COMMIT_SHA=${HAPPY_COMMIT}
 ENV COMMIT_BRANCH=${HAPPY_BRANCH}
 
-CMD ["python3", "-m", "backend.wmg.pipeline.cube_pipeline"]
+CMD ["python3", "-m", "backend.wmg.pipeline"]

backend/cellguide/pipeline/computational_marker_genes/__init__.py

@@ -1,7 +1,9 @@
 import logging
 
-from backend.cellguide.pipeline.computational_marker_genes.computational_markers import MarkerGenesCalculator
-from backend.cellguide.pipeline.computational_marker_genes.constants import MARKER_SCORE_THRESHOLD
+from backend.cellguide.pipeline.computational_marker_genes.computational_markers import (
+    MARKER_SCORE_THRESHOLD,
+    MarkerGenesCalculator,
+)
 from backend.cellguide.pipeline.constants import COMPUTATIONAL_MARKER_GENES_FOLDERNAME, MARKER_GENE_PRESENCE_FILENAME
 from backend.cellguide.pipeline.ontology_tree import get_ontology_tree_builder
 from backend.cellguide.pipeline.ontology_tree.tree_builder import OntologyTreeBuilder

backend/cellguide/pipeline/computational_marker_genes/computational_markers.py

@@ -10,9 +10,6 @@
 from dask.diagnostics import ProgressBar
 from tqdm import tqdm
 
-from backend.cellguide.pipeline.computational_marker_genes.constants import (
-    MARKER_SCORE_THRESHOLD,
-)
 from backend.cellguide.pipeline.computational_marker_genes.types import ComputationalMarkerGenes
 from backend.cellguide.pipeline.computational_marker_genes.utils import (
     bootstrap_rows_percentiles,
@@ -43,6 +40,8 @@
 or any arbitrary combinations of metadata dimensions.
 """
 
+MARKER_SCORE_THRESHOLD = 0.5
+
 
 class MarkerGenesCalculator:
     def __init__(self, *, snapshot: WmgSnapshot, all_cell_type_ids_in_corpus: list[str], groupby_terms: list[str]):

backend/cellguide/pipeline/computational_marker_genes/constants.py

@@ -1,16 +1,16 @@
 from backend.cellguide.pipeline.canonical_marker_genes.utils import format_citation_dp
 from backend.cellguide.pipeline.source_collections.types import SourceCollectionsData
 from backend.common.utils.rollup import descendants
-from backend.wmg.data.utils import get_collections_from_curation_api, get_datasets_from_curation_api
+from backend.wmg.data.utils import get_collections_from_discover_api, get_datasets_from_discover_api
 
 
 def generate_source_collections_data(all_cell_type_ids_in_corpus: list[str]) -> dict[str, list[SourceCollectionsData]]:
     """
     For each cell type id in the corpus, we want to generate a SourceCollectionsData object, which contains
     metadata about the source data for each cell type
     """
-    all_datasets = get_datasets_from_curation_api()
-    all_collections = get_collections_from_curation_api()
+    all_datasets = get_datasets_from_discover_api()
+    all_collections = get_collections_from_discover_api()
 
     collections_dict = {collection["collection_id"]: collection for collection in all_collections}
     datasets_dict = {dataset["dataset_id"]: dataset for dataset in all_datasets}

backend/cellguide/pipeline/source_collections/source_collections_generator.py

@@ -23,9 +23,5 @@ class NonExistentDatasetException(CorporaException):
     pass
 
 
-class CubeValidationException(Exception):
-    pass
-
-
 class MarkerGeneCalculationException(Exception):
     pass

backend/common/utils/exceptions.py

@@ -85,7 +85,7 @@ def gen_wmg_pipeline_failure_message(failure_info: str) -> dict:
     }
 
 
-def gen_wmg_pipeline_success_message(snapshot_path: str, dataset_count: int, cell_count: int, gene_count: int) -> dict:
+def gen_wmg_pipeline_success_message(snapshot_path: str, dataset_count: int, cell_count: int) -> dict:
     return {
         "blocks": [
             {
@@ -101,8 +101,7 @@ def gen_wmg_pipeline_success_message(snapshot_path: str, dataset_count: int, cel
                 "text": {
                     "type": "mrkdwn",
                     "text": f"\n* WMG snapshot stored in {snapshot_path}"
-                    f"\n* The cube contains {cell_count} cells from {dataset_count} "
-                    f"\n  datasets, with expression scores across {gene_count} genes.",
+                    f"\n* The cube contains {cell_count} cells from {dataset_count} datasets.",
                 },
             },
         ]

backend/common/utils/result_notification.py

@@ -11,7 +11,6 @@
 from backend.wmg.data import query
 from backend.wmg.data.snapshot import (
     EXPRESSION_SUMMARY_CUBE_NAME,
-    EXPRESSION_SUMMARY_FMG_CUBE_NAME,
     WmgSnapshot,
     _open_cube,
 )
@@ -27,20 +26,9 @@ def load_snapshot(snapshot_id) -> WmgSnapshot:
     cube = _open_cube(
         f's3://cellxgene-wmg-{os.environ["DEPLOYMENT_STAGE"]}/{snapshot_id}/{EXPRESSION_SUMMARY_CUBE_NAME}/'
     )
-    cube_fmg = _open_cube(
-        f's3://cellxgene-wmg-{os.environ["DEPLOYMENT_STAGE"]}/{snapshot_id}/{EXPRESSION_SUMMARY_FMG_CUBE_NAME}/'
-    )
     return WmgSnapshot(
         snapshot_identifier=snapshot_id,
         expression_summary_cube=cube,
-        cell_counts_cube=None,
-        cell_type_orderings=pd.DataFrame(),
-        primary_filter_dimensions=pd.DataFrame(),
-        expression_summary_fmg_cube=cube_fmg,
-        dataset_to_gene_ids={},
-        marker_genes_cube=None,
-        filter_relationships=None,
-        dataset_metadata=None,
     )
 
 

backend/scripts/wmg_query_examples.py

@@ -1,12 +1,3 @@
-# Minimum number of expressed genes for a cell to be included in the corpus.
-# See the following document for further details:
-# https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/scExpression/scExpression-documentation.md#removal-of-low-coverage-cells
-GENE_EXPRESSION_COUNT_MIN_THRESHOLD = 500
-
-# Minimum value for raw expression counts that will be used to filter out computed RankIt values. Details:
-# https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/scExpression/scExpression-documentation.md#removal-of-noisy-ultra-low-expression-values
-RANKIT_RAW_EXPR_COUNT_FILTERING_MIN_THRESHOLD = 2
-
 # wmg only includes data generated by assays that normalize for gene length
 INCLUDED_ASSAYS = {
     "EFO:0010550": "sci-RNA-seq",

backend/wmg/data/constants.py

@@ -51,12 +51,6 @@ class WmgFiltersQueryCriteria(BaseModel):
     publication_citations: List[str] = Field(default=[], unique_items=True, min_items=0)
 
 
-class FmgQueryCriteria(BaseModel):
-    organism_ontology_term_id: str  # required!
-    tissue_ontology_term_ids: List[str] = Field(default=[], unique_items=True, min_items=0)
-    cell_type_ontology_term_ids: List[str] = Field(default=[], unique_items=True, min_items=0, max_items=1)
-
-
 class MarkerGeneQueryCriteria(BaseModel):
     organism_ontology_term_id: str  # required!
     tissue_ontology_term_id: str  # required!
@@ -102,19 +96,13 @@ def expression_summary_default(self, criteria: WmgQueryCriteria) -> DataFrame:
             criteria=criteria,
         )
 
-    def expression_summary_fmg(self, criteria: FmgQueryCriteria) -> DataFrame:
-        return self._query(
-            cube=self._snapshot.expression_summary_fmg_cube,
-            criteria=criteria,
-        )
-
     def marker_genes(self, criteria: MarkerGeneQueryCriteria) -> DataFrame:
         return self._query(
             cube=self._snapshot.marker_genes_cube,
             criteria=criteria,
         )
 
-    def cell_counts(self, criteria: Union[WmgQueryCriteria, FmgQueryCriteria], compare_dimension=None) -> DataFrame:
+    def cell_counts(self, criteria: WmgQueryCriteria, compare_dimension=None) -> DataFrame:
         cell_counts = self._query(
             cube=self._snapshot.cell_counts_cube,
             criteria=criteria.copy(exclude={"gene_ontology_term_ids"}),
@@ -128,7 +116,7 @@ def cell_counts(self, criteria: Union[WmgQueryCriteria, FmgQueryCriteria], compa
     def _query(
         self,
         cube: Array,
-        criteria: Union[WmgQueryCriteria, WmgQueryCriteriaV2, FmgQueryCriteria, MarkerGeneQueryCriteria],
+        criteria: Union[WmgQueryCriteria, WmgQueryCriteriaV2, MarkerGeneQueryCriteria],
         compare_dimension=None,
     ) -> DataFrame:
         indexed_dims = self._cube_query_params.get_indexed_dims_to_lookup_query_criteria(
@@ -165,12 +153,6 @@ def _query(
         attrs = self._cube_query_params.get_attrs_for_cube_query(cube)
         if compare_dimension is not None:
             attrs.append(compare_dimension)
-        if (
-            isinstance(criteria, FmgQueryCriteria)
-            and compare_dimension != "dataset_id"
-            and "dataset_id" in [i.name for i in cube.schema]
-        ):
-            attrs.append("dataset_id")
 
         attrs += numeric_attrs