Skip to content

Commit 287687f

Browse files
brandynluccabrandynlucca
committed
Adjustments to transect region file that is called for the 2011 survey year
1 parent 6fbee49 commit 287687f

File tree

5 files changed

+281
-168
lines changed

5 files changed

+281
-168
lines changed

‎config_files/survey_year_2011_config.yml

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,7 @@ NASC:
6868
filename: Exports/US_CAN_NASC_2011_table_all_ages.xlsx
6969
sheetname: Sheet1
7070
export_regions:
71-
all_ages:
72-
filename: Stratification/US&CAN_2011_transect_region_haul_age1+ auto final_4-20-2019.xlsx
73-
sheetname: Sheet1
74-
no_age1:
75-
filename: Stratification/US&CAN_2011_transect_region_haul_age2+ auto final_4-20-2019.xlsx
76-
sheetname: Sheet1
71+
filename: Stratification/US&CAN_T_reg_haul_final.csv
7772
kriging:
7873
mesh:
7974
filename: Kriging_files/Kriging_grid_files/krig_grid2_5nm_cut_centroids_2013.xlsx

‎echopop/compatibility_parameters_test.json

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{
22
"2023": {
3+
"transect_pattern": "T(\\d+)",
34
"bootstrap_ci_method": "percentile",
45
"exclude_age1": [
56
true,
@@ -19,6 +20,7 @@
1920
"transect_replicates": 10
2021
},
2122
"2021": {
23+
"transect_pattern": "T(\\d+)",
2224
"bootstrap_ci_method": "percentile",
2325
"exclude_age1": [
2426
true,
@@ -38,6 +40,7 @@
3840
"transect_replicates": 10
3941
},
4042
"2019": {
43+
"transect_pattern": "T(\\d+)",
4144
"bootstrap_ci_method": "percentile",
4245
"exclude_age1": [
4346
true,
@@ -57,6 +60,7 @@
5760
"transect_replicates": 10
5861
},
5962
"2017": {
63+
"transect_pattern": "T(\\d+)",
6064
"bootstrap_ci_method": "percentile",
6165
"exclude_age1": [
6266
true,
@@ -76,6 +80,7 @@
7680
"transect_replicates": 10
7781
},
7882
"2015": {
83+
"transect_pattern": "T(\\d+)",
7984
"bootstrap_ci_method": "percentile",
8085
"exclude_age1": [
8186
true,
@@ -95,6 +100,7 @@
95100
"transect_replicates": 10
96101
},
97102
"2013": {
103+
"transect_pattern": "T(\\d+)",
98104
"bootstrap_ci_method": "percentile",
99105
"exclude_age1": [
100106
true,
@@ -114,6 +120,7 @@
114120
"transect_replicates": 10
115121
},
116122
"2012": {
123+
"transect_pattern": "T(\\d+)",
117124
"bootstrap_ci_method": "percentile",
118125
"exclude_age1": [
119126
true,
@@ -133,6 +140,7 @@
133140
"transect_replicates": 10
134141
},
135142
"2011": {
143+
"transect_pattern": "T(\\d+(?:\\.\\d+)?)(?=-)",
136144
"bootstrap_ci_method": "percentile",
137145
"exclude_age1": [
138146
true,
@@ -148,7 +156,7 @@
148156
"inpfc",
149157
"ks"
150158
],
151-
"write_transect_region_file": true,
159+
"write_transect_region_file": false,
152160
"transect_replicates": 10
153161
}
154162
}

‎echopop/test_survey.py

Lines changed: 137 additions & 136 deletions
Original file line numberDiff line numberDiff line change
@@ -1,142 +1,143 @@
1-
# # ####################################################################################################
2-
# # # TEST BACKWARDS COMPATIBILITY WITH OTHER SURVEY YEARS
3-
# # # LINKED ISSUE: https://github.com/OSOceanAcoustics/echopop/issues/307
4-
# # ####################################################################################################
5-
# # # SUCCESSFUL TESTS
6-
# # # 2015, 2017, 2019, 2021, 2023
7-
# # ####################################################################################################
8-
# # # FAILURE NOTES
9-
# # # 2012: Transect-region mapping
10-
# # # Initialize year-specific survey parameters
11-
# # ####################################################################################################
12-
# # # Import libraries
13-
# from echopop.survey import Survey
14-
# from echopop.extensions import generate_reports
15-
# import pandas as pd
16-
# from pathlib import Path
17-
# import glob
18-
# import json
19-
# import os
1+
####################################################################################################
2+
# TEST BACKWARDS COMPATIBILITY WITH OTHER SURVEY YEARS
3+
# LINKED ISSUE: https://github.com/OSOceanAcoustics/echopop/issues/307
4+
####################################################################################################
5+
# SUCCESSFUL TESTS
6+
# 2015, 2017, 2019, 2021, 2023
7+
####################################################################################################
8+
# FAILURE NOTES
9+
# 2012: Transect-region mapping
10+
# Initialize year-specific survey parameters
11+
####################################################################################################
12+
# Import libraries
13+
from echopop.survey import Survey
14+
from echopop.extensions import generate_reports
15+
import pandas as pd
16+
from pathlib import Path
17+
import glob
18+
import json
19+
import os
2020

21-
# # ####################################################################################################
22-
# # # CURRENT SURVEY YEAR BEING TESTED: 2019
23-
# # ####################################################################################################
24-
# # # Define current survey year
25-
# SURVEY_YEAR = 2019
21+
####################################################################################################
22+
# CURRENT SURVEY YEAR BEING TESTED: 2019
23+
####################################################################################################
24+
# Define current survey year
25+
SURVEY_YEAR = 2011
2626

27-
# # Initialization configuration
28-
# init_config_path = f"C:/Users/Brandyn/Documents/GitHub/echopop/config_files/initialization_\
29-
# config_{SURVEY_YEAR}.yml"
27+
# Initialization configuration
28+
init_config_path = f"C:/Users/Brandyn/Documents/GitHub/echopop/config_files/initialization_\
29+
config_{SURVEY_YEAR}.yml"
3030

31-
# # Filepath/dataset configuration
32-
# survey_year_config_path = f"C:/Users/Brandyn/Documents/GitHub/echopop/config_files\
33-
# /survey_year_{SURVEY_YEAR}_config.yml"
31+
# Filepath/dataset configuration
32+
survey_year_config_path = f"C:/Users/Brandyn/Documents/GitHub/echopop/config_files\
33+
/survey_year_{SURVEY_YEAR}_config.yml"
3434

35-
# # Load json settings
36-
# # ---- File open
37-
# with open(Path(os.getcwd() + "\\echopop\\compatibility_parameters_test.json").as_posix()) as f:
38-
# json_dict = json.load(f)
39-
# # ---- Load
40-
# parameters = json_dict[f"{SURVEY_YEAR}"]
35+
# Load json settings
36+
# ---- File open
37+
with open(Path(os.getcwd() + "\\echopop\\compatibility_parameters_test.json").as_posix()) as f:
38+
json_dict = json.load(f)
39+
# ---- Load
40+
parameters = json_dict[f"{SURVEY_YEAR}"]
4141

42-
# ####################################################################################################
43-
# # Run
44-
# # ---- Create object
45-
# survey = Survey(init_config_path, survey_year_config_path)
46-
# # ---- Load acoustic data (Echoview export ingestion)
47-
# survey.load_acoustic_data(ingest_exports="echoview",
48-
# read_transect_region_file=parameters["read_transect_region_file"],
49-
# write_transect_region_file=parameters["write_transect_region_file"])
50-
# # ---- Load acoustic data (Already-defined file)
51-
# if parameters["default_acoustics"]:
52-
# survey = Survey(init_config_path, survey_year_config_path)
53-
# survey.load_acoustic_data()
54-
# else:
55-
# Survey(init_config_path, survey_year_config_path).load_acoustic_data()
56-
# # ---- Load survey data
57-
# survey.load_survey_data()
58-
# # ---- Initial transect analysis test
59-
# survey.transect_analysis()
60-
# # ---- Counter
61-
# counter = 1
62-
# # ---- Iterate across multiple strata types
63-
# for stratum in parameters["strata_types"]:
64-
# # ---- Iterate across different age-1 exclusion definitions
65-
# for excl in parameters["exclude_age1"]:
66-
# # ---- Transect analysis
67-
# survey.transect_analysis(exclude_age1=excl, stratum=stratum, verbose=False)
68-
# print(
69-
# f"{SURVEY_YEAR} transect analysis [{counter}/8] success!"
70-
# )
71-
# # ---- Stratified analysis (transect analysis)
72-
# survey.stratified_analysis(bootstrap_ci_method=parameters["bootstrap_ci_method"],
73-
# transect_replicates=parameters["transect_replicates"],
74-
# verbose=False)
75-
# print(
76-
# f"{SURVEY_YEAR} stratified analysis (transect) [{counter}/8] success!"
77-
# )
78-
# # ---- Fit variogram
79-
# survey.fit_variogram(verbose=False)
80-
# print(
81-
# f"{SURVEY_YEAR} variogram fitting [{counter}/8] success!"
82-
# )
83-
# # ---- Iterate across different extrapolation schema
84-
# for extrap in parameters["extrapolate"]:
85-
# # ---- Kriging analysis (no variogram fitting)
86-
# survey.kriging_analysis(extrapolate=extrap, verbose=False)
87-
# print(
88-
# f"{SURVEY_YEAR} kriging analysis (default variogram) [{counter}/8] success!"
89-
# )
90-
# # ---- Apply best-fit variogram
91-
# survey.kriging_analysis(best_fit_variogram=True, extrapolate=extrap, verbose=False)
92-
# print(
93-
# f"{SURVEY_YEAR} kriging analysis (best-fit variogram) [{counter}/8] success!"
94-
# )
95-
# # ---- Stratified analysis (kriging analysis)
96-
# survey.stratified_analysis(dataset="kriging",
97-
# bootstrap_ci_method=parameters["bootstrap_ci_method"],
98-
# transect_replicates=parameters["transect_replicates"],
99-
# verbose=False)
100-
# print(
101-
# f"{SURVEY_YEAR} stratified analysis (kriging) [{counter}/8] success!"
102-
# )
103-
# # ---- Test reports
104-
# survey.generate_reports(reports=["aged_length_haul_counts",
105-
# "kriging_input",
106-
# "kriged_length_age_abundance",
107-
# "kriged_length_age_biomass",
108-
# "kriged_mesh_results",
109-
# "total_length_haul_counts",
110-
# "transect_length_age_abundance",
111-
# "transect_length_age_biomass",
112-
# "transect_population_results"])
113-
# # ---- Print out success
114-
# print(
115-
# f"""
116-
# Year: {SURVEY_YEAR} success [{counter}/8 configurations]!
117-
# Stratum: {stratum}
118-
# Age-1 fish excluded: {excl}
119-
# Extrapolated: {extrap}
120-
# """
121-
# )
122-
# # ---- Advance counter
123-
# counter += 1
42+
####################################################################################################
43+
# Run
44+
# ---- Create object
45+
survey = Survey(init_config_path, survey_year_config_path)
46+
# ---- Load acoustic data (Echoview export ingestion)
47+
survey.load_acoustic_data(ingest_exports="echoview",
48+
transect_pattern=parameters["transect_pattern"].replace(r"\\", "\\"),
49+
read_transect_region_file=parameters["read_transect_region_file"],
50+
write_transect_region_file=parameters["write_transect_region_file"])
51+
# ---- Load acoustic data (Already-defined file)
52+
if parameters["default_acoustics"]:
53+
survey = Survey(init_config_path, survey_year_config_path)
54+
survey.load_acoustic_data()
55+
else:
56+
Survey(init_config_path, survey_year_config_path).load_acoustic_data()
57+
# ---- Load survey data
58+
survey.load_survey_data()
59+
# ---- Initial transect analysis test
60+
survey.transect_analysis()
61+
# ---- Counter
62+
counter = 1
63+
# ---- Iterate across multiple strata types
64+
for stratum in parameters["strata_types"]:
65+
# ---- Iterate across different age-1 exclusion definitions
66+
for excl in parameters["exclude_age1"]:
67+
# ---- Transect analysis
68+
survey.transect_analysis(exclude_age1=excl, stratum=stratum, verbose=False)
69+
print(
70+
f"{SURVEY_YEAR} transect analysis [{counter}/8] success!"
71+
)
72+
# ---- Stratified analysis (transect analysis)
73+
survey.stratified_analysis(bootstrap_ci_method=parameters["bootstrap_ci_method"],
74+
transect_replicates=parameters["transect_replicates"],
75+
verbose=False)
76+
print(
77+
f"{SURVEY_YEAR} stratified analysis (transect) [{counter}/8] success!"
78+
)
79+
# ---- Fit variogram
80+
survey.fit_variogram(verbose=False)
81+
print(
82+
f"{SURVEY_YEAR} variogram fitting [{counter}/8] success!"
83+
)
84+
# ---- Iterate across different extrapolation schema
85+
for extrap in parameters["extrapolate"]:
86+
# ---- Kriging analysis (no variogram fitting)
87+
survey.kriging_analysis(extrapolate=extrap, verbose=False)
88+
print(
89+
f"{SURVEY_YEAR} kriging analysis (default variogram) [{counter}/8] success!"
90+
)
91+
# ---- Apply best-fit variogram
92+
survey.kriging_analysis(best_fit_variogram=True, extrapolate=extrap, verbose=False)
93+
print(
94+
f"{SURVEY_YEAR} kriging analysis (best-fit variogram) [{counter}/8] success!"
95+
)
96+
# ---- Stratified analysis (kriging analysis)
97+
survey.stratified_analysis(dataset="kriging",
98+
bootstrap_ci_method=parameters["bootstrap_ci_method"],
99+
transect_replicates=parameters["transect_replicates"],
100+
verbose=False)
101+
print(
102+
f"{SURVEY_YEAR} stratified analysis (kriging) [{counter}/8] success!"
103+
)
104+
# ---- Test reports
105+
survey.generate_reports(reports=["aged_length_haul_counts",
106+
"kriging_input",
107+
"kriged_length_age_abundance",
108+
"kriged_length_age_biomass",
109+
"kriged_mesh_results",
110+
"total_length_haul_counts",
111+
"transect_length_age_abundance",
112+
"transect_length_age_biomass",
113+
"transect_population_results"])
114+
# ---- Print out success
115+
print(
116+
f"""
117+
Year: {SURVEY_YEAR} success [{counter}/8 configurations]!
118+
Stratum: {stratum}
119+
Age-1 fish excluded: {excl}
120+
Extrapolated: {extrap}
121+
"""
122+
)
123+
# ---- Advance counter
124+
counter += 1
124125

125-
# ####################################################################################################
126-
# # Get report outputs
127-
# # ---- Define directory
128-
# output_dir = f"C:/Users/Brandyn/Documents/GitHub/EchoPro_data/echopo_backtest_years/Outputs/\
129-
# Historical Outputs (KS Stratification with aged data)/without extrapolation/{SURVEY_YEAR}"
130-
# # ---- Get files
131-
# files = glob.glob(output_dir + "/EchoPro_un-kriged_output*")
132-
# # ---- Filter files
133-
# filtered_files = [f for f in files if f.endswith("_1.xlsx")]
134-
# # ---- Read
135-
# echopro_data = pd.read_excel(filtered_files[0])
136-
# ####################################################################################################
137-
# # Compare
138-
# echopop_data = survey.analysis["transect"]["acoustics"]["adult_transect_df"].copy()
139-
# # ---- Calculate: NASC
140-
# echopro_data["NASC"].sum() - echopop_data["nasc"].sum()
141-
# # ---- Calculate: biomass
142-
# (echopro_data["wgt_total"].sum() - echopop_data["biomass"].sum()) * 1e-6
126+
####################################################################################################
127+
# Get report outputs
128+
# ---- Define directory
129+
output_dir = f"C:/Users/Brandyn/Documents/GitHub/EchoPro_data/echopo_backtest_years/Outputs/\
130+
Historical Outputs (KS Stratification with aged data)/without extrapolation/{SURVEY_YEAR}"
131+
# ---- Get files
132+
files = glob.glob(output_dir + "/EchoPro_un-kriged_output*")
133+
# ---- Filter files
134+
filtered_files = [f for f in files if f.endswith("_1.xlsx")]
135+
# ---- Read
136+
echopro_data = pd.read_excel(filtered_files[0])
137+
####################################################################################################
138+
# Compare
139+
echopop_data = survey.analysis["transect"]["acoustics"]["adult_transect_df"].copy()
140+
# ---- Calculate: NASC
141+
echopro_data["NASC"].sum() - echopop_data["nasc"].sum()
142+
# ---- Calculate: biomass
143+
(echopro_data["wgt_total"].sum() - echopop_data["biomass"].sum()) * 1e-6

0 commit comments

Comments
 (0)