Change to imported stratum-haul mapping for INPFC

Author: brandynlucca

docs/example_notebooks/example_echopop_workflow.ipynb

@@ -271,8 +271,9 @@
     "  * `specimen_df`: aged length and weight measurements\n",
     "* `spatial`\n",
     "  * `strata_df`: the `KS` stratum definitions and fraction of hake for each haul\n",
-    "  * `geo_strata_df`: latitudinal limits of the `KS` strata\n",
-    "  * `inpfc_strata_df`: the `INPFC` stratum definitions and their respective latitudinal limits\n",
+    "  * `inpfc_strata_df`: the `INPFC` stratum definitions and fraction of hake for each haul\n",
+    "  * `geo_strata_df`: latitudinal limits of the `KS` strata and their respective latitudinal limits\n",
+    "  * `inpfc_geo_strata_df`: the `INPFC` stratum definitions and their respective latitudinal limits\n",
     "* `statistics`\n",
     "  * `kriging`\n",
     "    * `mesh_df`: kriging mesh\n",

docs/implementation/preprocessing_data.md

@@ -111,16 +111,17 @@ A color-coded schematic that provides a visual overview of how data are loaded a
   - `specimen` ➡️ `Survey.input["biology"]["specimen_df"]`
 - `Configuration` ➡️ `Survey.input["biology"]`
   - `bio_hake_len_bin`/`bio_hake_age_bin` ➡️ `Survey.input["biology"]["distributions]`
-- `Kriging` ➡️ `Survey.input["statistics"]`
+- `Kriging` ➡️ `Survey.input["statistics"]` 
   - `mesh` ➡️ `Survey.input["statistics"]["kriging"]["mesh_df"]`
   - `isobath_200m` ➡️ `Survey.input["statistics"]["kriging"]["isobath_200m_df"]`
   - `vario_krig_para` ➡️ `Survey.input["statistics"]["kriging"]["vario_krig_para"]`
 - `NASC` ➡️ `Survey.input["acoustics"]`
   - `{GROUP}` (all) ➡️ `Survey.input["acoustics"]["nasc_df"]`
 - `Stratification` ➡️ `Survey.input["spatial"]`
   - `strata` ➡️: `Survey.input["spatial"]["strata_df"]`
+  - `strata` ➡️: `Survey.input["spatial"]["inpfc_strata_df"]`
   - `geo_strata` ➡️ `Survey.input["spatial"]["geo_strata_df"]`
-  - `geo_strata` ➡️ `Survey.input["spatial"]["inpfc_strata_df"]`
+  - `geo_strata` ➡️ `Survey.input["spatial"]["inpfc_geo_strata_df"]`
 
 Echoview exports can be **<span style="color:#6666FF">alternatively processed and loaded</span>** into `Echopop` by incorporating the `nasc_exports` parameters within `initialization_config.yml`. These files can also processed outside of the same `Echopop` workflow whereby the processed exports can then be saved and used to parameterize the `NASC` dataset definiations within the `survey_year_{YEAR}_config.yml` configuration file.
 

echopop/core.py

@@ -120,8 +120,9 @@
         },
         "spatial": {
             "strata_df": pd.DataFrame(),
-            "geo_strata_df": pd.DataFrame(),
             "inpfc_strata_df": pd.DataFrame(),
+            "geo_strata_df": pd.DataFrame(),
+            "inpfc_geo_strata_df": pd.DataFrame(),
         },
         "statistics": {
             "kriging": {
@@ -182,7 +183,7 @@
     },
     "stratification": {
         "name": "spatial",
-        "data": ["strata", "geo_strata", "inpfc_strata"],
+        "data": ["strata", "inpfc_strata", "geo_strata", "inpfc_geo_strata"],
         "data_label": [
             "strata:KS strata",
             "geo_strata:Georeferenced KS strata",

echopop/survey.py

@@ -260,7 +260,9 @@ def transect_analysis(
                     "unique_strata": (
                         np.unique(self.input["spatial"]["strata_df"]["stratum_num"])
                         if stratum == "ks"
-                        else np.unique(self.input["spatial"]["inpfc_strata_df"]["stratum_inpfc"])
+                        else np.unique(
+                            self.input["spatial"]["inpfc_geo_strata_df"]["stratum_inpfc"]
+                        )
                     ),
                     "exclude_age1": exclude_age1,
                 }

echopop/utils/load.py

@@ -31,9 +31,6 @@ def load_configuration(init_config_path: Path, survey_year_config_path: Path):
     the Survey class object. This initializes the `config` attribute that
     becomes available for future reference and functions.
     """
-    # Validate configuration files
-    # Retrieve the module directory to begin mapping the configuration file location
-    # current_directory = os.path.dirname(os.path.abspath(__file__))
 
     # Build the full configuration file paths and verify they exist
     config_files = [init_config_path, survey_year_config_path]
@@ -297,21 +294,29 @@ def read_validated_data(
 
         # A single dataframe per entry is expected, so no other fancy operations are needed
         if sheet_name.lower() == "inpfc":
-            df_list = [input_dict[sub_attribute]["inpfc_strata_df"], df]
-            input_dict[sub_attribute]["inpfc_strata_df"] = pd.concat(df_list)
+            # ---- Create the full key name
+            keyname = "inpfc_" + config_map[-1] + "_df"
+            # ---- Create DataFrame list
+            df_list = [input_dict[sub_attribute][keyname], df]
+            # ---- Concatenate/update
+            input_dict[sub_attribute][keyname] = pd.concat(df_list, ignore_index=True)
         else:
             if config_map[0] == "kriging" and config_map[1] == "vario_krig_para":
                 df_list = [input_dict[sub_attribute]["kriging"][config_map[1] + "_df"], df]
                 input_dict[sub_attribute]["kriging"][config_map[1] + "_df"] = pd.concat(
-                    df_list
+                    df_list,
+                    ignore_index=True
                 ).tail(1)
             elif config_map[0] == "kriging":
                 df_list = [input_dict[sub_attribute]["kriging"][config_map[1] + "_df"], df]
-                input_dict[sub_attribute]["kriging"][config_map[1] + "_df"] = pd.concat(df_list)
+                input_dict[sub_attribute]["kriging"][config_map[1] + "_df"] = pd.concat(
+                    df_list,
+                    ignore_index=True
+                )
             else:
                 df_list = [input_dict[sub_attribute][config_map[1] + "_df"], df]
-                input_dict[sub_attribute][config_map[1] + "_df"] = pd.concat(df_list)
-    # TODO: This can be refactored out
+                input_dict[sub_attribute][config_map[1] + "_df"] = pd.concat(df_list, 
+                                                                             ignore_index=True)
     elif sub_attribute == "acoustics":
 
         # Toggle through including and excluding age-1
@@ -539,30 +544,36 @@ def preprocess_spatial(input_dict: dict) -> None:
     """
 
     # Update column names
-    # ---- `geo_strata`
-    input_dict["spatial"]["geo_strata_df"].columns = input_dict["spatial"][
-        "geo_strata_df"
-    ].columns.str.replace(" ", "_")
-    # ---- `inpfc_strata`
-    input_dict["spatial"]["inpfc_strata_df"].columns = input_dict["spatial"][
-        "inpfc_strata_df"
-    ].columns.str.replace(" ", "_")
-    # ---- `inpfc_strata`: rename stratum column name to avoid conflicts
+    # ---- INPFC entries
+    # -------- `inpfc_strata`: rename stratum column name to avoid conflicts
     input_dict["spatial"]["inpfc_strata_df"].rename(
         columns={"stratum_num": "stratum_inpfc"}, inplace=True
     )
+    # -------- `inpfc_geo_strata`: rename stratum column name to avoid conflicts
+    input_dict["spatial"]["inpfc_geo_strata_df"].rename(
+        columns={"stratum_num": "stratum_inpfc"}, inplace=True
+    )
 
-    # Bin data
-    # ---- Create latitude intervals to bin the strata
-    latitude_bins = np.concatenate(
-        [[-90], 
-         input_dict["spatial"]["inpfc_strata_df"]["northlimit_latitude"].unique(), 
-         [90]]
+
+    # Bin the geo-strata latitudes
+    # ---- INPFC
+    # -------- Latitude bins
+    latitude_bins_inpfc = np.concatenate(
+        [[-90], input_dict["spatial"]["inpfc_geo_strata_df"]["northlimit_latitude"].unique(), [90]]
     )
-    # ---- Add categorical intervals
-    input_dict["spatial"]["inpfc_strata_df"]["latitude_interval"] = pd.cut(
-        input_dict["spatial"]["inpfc_strata_df"]["northlimit_latitude"] * 0.99, 
-        latitude_bins,
+    # -------- Add categorical intervals
+    input_dict["spatial"]["inpfc_geo_strata_df"]["latitude_interval"] = pd.cut(
+        input_dict["spatial"]["inpfc_geo_strata_df"]["northlimit_latitude"], 
+        latitude_bins_inpfc,
+    )
+    # ---- KS
+    latitude_bins_ks = np.concatenate(
+        [[-90], input_dict["spatial"]["geo_strata_df"]["northlimit_latitude"].unique(), [90]]
+    )
+    # -------- Add categorical intervals
+    input_dict["spatial"]["geo_strata_df"]["latitude_interval"] = pd.cut(
+        input_dict["spatial"]["geo_strata_df"]["northlimit_latitude"], 
+        latitude_bins_ks,
     )
 
 
@@ -580,7 +591,7 @@ def preprocess_acoustic_spatial(input_dict: dict) -> None:
     # ---- Create latitude intervals to bin the strata
     latitude_bins = np.concatenate(
         [[-90], 
-         input_dict["spatial"]["inpfc_strata_df"]["northlimit_latitude"].unique(), 
+         input_dict["spatial"]["inpfc_geo_strata_df"]["northlimit_latitude"].unique(), 
          [90]]
     )
     # ---- Bin NASC transects into appropriate INPFC strata
@@ -628,41 +639,13 @@ def preprocess_biology_spatial(input_dict: dict) -> None:
         Dictionary corresponding to the `input` attribute belonging to `Survey`-class
     """
 
-    # Merge haul numbers and spatial information across biological variables
-    # ---- Create interval key for haul numbers to assign INPFC stratum
-    haul_bins = np.sort(
-        np.unique(
-            np.concatenate(
-                [
-                    input_dict["spatial"]["inpfc_strata_df"]["haul_start"] - int(1),
-                    input_dict["spatial"]["inpfc_strata_df"]["haul_end"],
-                ]
-            )
-        )
-    )
-    # ---- Quantize the INPFC dataframe hauls based on strata
-    input_dict["spatial"]["inpfc_strata_df"]["haul_bin"] = pd.cut(
-        (
-            input_dict["spatial"]["inpfc_strata_df"]["haul_start"]
-            + input_dict["spatial"]["inpfc_strata_df"]["haul_end"]
-        )
-        / 2,
-        haul_bins,
-    )
-    # ---- Rename `stratum_num` column
-    input_dict["spatial"]["inpfc_strata_df"].rename(
-        columns={"stratum_num": "stratum_inpfc"}, inplace=True
-    )
-    # ---- Set the index to `haul_bins`
-    inpfc_df = (
-        input_dict["spatial"]["inpfc_strata_df"].copy()
-        .drop_duplicates("haul_bin")
-        .set_index(["haul_bin"])
-    )
-    
-    # Get the KS-strata
+    # Get the KS-strata (indexed by haul)
     strata_df = input_dict["spatial"]["strata_df"].copy().set_index(["haul_num"])
-    
+
+    # Get the INPFC strata (indexed by haul)
+    inpfc_strata_df = input_dict["spatial"]["inpfc_strata_df"].copy().set_index(["haul_num"])
+
+
     # Loop through the KS-strata to map the correct strata values
     for keys, values in input_dict["biology"].items():
         if isinstance(values, pd.DataFrame) and "haul_num" in values.columns:
@@ -674,26 +657,16 @@ def preprocess_biology_spatial(input_dict: dict) -> None:
             input_dict["biology"][keys]["stratum_num"] = (
                 input_dict["biology"][keys]["stratum_num"].fillna(0.0).astype(int)
             )
-            # ---- Reset the index
-            input_dict["biology"][keys].reset_index(inplace=True)
-            # ---- Bin for `stratum_inpfc`
-            input_dict["biology"][keys]["haul_bin"] = pd.cut(
-                input_dict["biology"][keys]["haul_num"], haul_bins
-            )
+            # ---- Map the correct `stratum_inpfc` value
+            input_dict["biology"][keys]["stratum_inpfc"] = inpfc_strata_df["stratum_inpfc"]
             # ---- NaN mask
-            nan_mask = input_dict["biology"][keys]['haul_bin'].isna()
-            # # ---- Valid haul bins
+            nan_mask = input_dict["biology"][keys]["stratum_inpfc"].isna()
+            # ---- Valid haul bins
             valid_haul_bins = input_dict["biology"][keys].copy().loc[~nan_mask]
-            # ---- Set index to `haul_bins`
-            valid_haul_bins.set_index(["haul_bin"], inplace=True)
-            # ---- Merge
-            valid_haul_bins["stratum_inpfc"] = inpfc_df["stratum_inpfc"]
-            # ---- Reset indices
-            valid_haul_bins.reset_index(inplace=True)
-            # ---- Drop `haul_bin`
-            valid_haul_bins.drop(columns=["haul_bin"], inplace=True)
+            # ---- Change to integer
+            valid_haul_bins["stratum_inpfc"] = valid_haul_bins["stratum_inpfc"].astype(int)
             # ---- Set
-            input_dict["biology"][keys] = valid_haul_bins
+            input_dict["biology"][keys] = valid_haul_bins.reset_index()
 
 
 def preprocess_acoustic_biology_spatial(input_dict: dict, configuration_dict: dict) -> None: