Add codespell exception

.pre-commit-config.yaml

@@ -39,4 +39,4 @@ repos:
       - id: codespell
         # Checks spelling in `docs/source` and `echopype` dirs ONLY
         # Ignores `.ipynb` files and `_build` folders
-        args: ["-L", "soop", "--skip=*.ipynb,docs/source/_build,echopype/test_data", "-w", "docs/source", "echopype"]
+        args: ["-L", "soop,anc,indx", "--skip=*.ipynb,docs/source/_build,echopype/test_data", "-w", "docs/source", "echopype"]

echopype/convert/parse_ad2cp.py

@@ -328,7 +328,8 @@ def timestamp(self) -> np.datetime64:
         microsec100 = self.data["microsec100"]
         try:
             return np.datetime64(
-                f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{seconds:02}.{microsec100:04}"
+                f"{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{seconds:02}.{microsec100:04}",  # noqa
+                "ns",
             )  # type: ignore
         except ValueError:
             return np.datetime64("NaT")  # type: ignore

echopype/convert/set_groups_ad2cp.py

@@ -1,5 +1,5 @@
 from enum import Enum, auto, unique
-from importlib import resources
+from importlib.resources import files
 from typing import Dict, List, Optional, Set, Tuple, Union
 
 import numpy as np
@@ -35,7 +35,7 @@ def __init__(self, *args, **kwargs):
         # resulting in index error in setting ds["pulse_compressed"]
         self.pulse_compressed = self.parser_obj.get_pulse_compressed()
         self._make_time_coords()
-        with resources.open_text(convert, "ad2cp_fields.yaml") as f:
+        with files(convert).joinpath("ad2cp_fields.yaml").open("r") as f:
             self.field_attrs: Dict[str, Dict[str, Dict[str, str]]] = yaml.safe_load(f)  # type: ignore # noqa
 
     def _make_time_coords(self):

echopype/convert/set_groups_ek80.py

@@ -8,6 +8,7 @@
 from ..utils.coding import set_time_encodings
 from ..utils.log import _init_logger
 from .set_groups_base import SetGroupsBase
+from .utils.ek_duplicates import check_unique_ping_time_duplicates
 
 logger = _init_logger(__name__)
 
@@ -1145,6 +1146,17 @@ def set_beam(self) -> List[xr.Dataset]:
 
             ds_data = self._attach_vars_to_ds_data(ds_data, ch, rs_size=ds_data.range_sample.size)
 
+            # Access the 'ping_time' coordinate as a NumPy array
+            ping_times = ds_data["ping_time"].values
+
+            # Check if ping time duplicates exist
+            if len(ping_times) > len(np.unique(ping_times)):
+                # Check for unique ping time duplicates and if they are not unique, raise warning.
+                check_unique_ping_time_duplicates(ds_data, logger)
+
+                # Drop duplicates
+                ds_data = ds_data.drop_duplicates(dim="ping_time")
+
             if ch in self.sorted_channel["complex"]:
                 ds_complex.append(ds_data)
             else:

echopype/convert/utils/ek_duplicates.py

@@ -0,0 +1,44 @@
+import logging
+
+import xarray as xr
+
+
+def check_unique_ping_time_duplicates(ds_data: xr.Dataset, logger: logging.Logger) -> None:
+    """
+    Raises a warning if the data stored in duplicate pings is not unique.
+
+    Parameters
+    ----------
+    ds_data : xr.Dataset
+        Single freq beam dataset being processed in the `SetGroupsEK80.set_beams` class function.
+    logger : logging.Logger
+        Warning logger initialized in `SetGroupsEK80` file.
+    """
+    # Group the dataset by the "ping_time" coordinate
+    groups = ds_data.groupby("ping_time")
+
+    # Loop through each ping_time group
+    for ping_time_val, group in groups:
+        # Extract all data variable names to check
+        data_vars = list(group.data_vars)
+
+        # Use the first duplicate ping time index as a reference
+        ref_duplicate_ping_time_index = 0
+
+        # Iterate over each data variable in the group
+        for var in data_vars:
+            # Extract data array corresponding to the iterated variable
+            data_array = group[var]
+
+            # Use the slice corresponding to the reference index as the reference slice
+            ref_slice = data_array.isel({"ping_time": ref_duplicate_ping_time_index})
+
+            # Iterate over the remaining entries
+            for i in range(1, data_array.sizes["ping_time"]):
+                if not ref_slice.equals(data_array.isel({"ping_time": i})):
+                    logger.warning(
+                        f"Duplicate slices in variable '{var}' corresponding to 'ping_time' "
+                        f"{ping_time_val} differ in data. All duplicate 'ping_time' entries "
+                        "will be removed, which will result in data loss."
+                    )
+                    break

echopype/echodata/convention/conv.py

@@ -1,4 +1,4 @@
-from importlib import resources
+from importlib.resources import files
 from typing import Optional
 
 import yaml
@@ -21,7 +21,7 @@ def yaml_dict(self):
         if self._yaml_dict:  # Data has already been read, return it directly
             return self._yaml_dict
 
-        with resources.open_text(package=convention, resource=f"{self.version}.yml") as fid:
+        with files(convention).joinpath(f"{self.version}.yml").open("r") as fid:
             convention_yaml = yaml.load(fid, Loader=yaml.SafeLoader)
 
         self._yaml_dict = convention_yaml

echopype/tests/consolidate/test_add_depth.py

@@ -196,7 +196,7 @@ def test_ek_depth_utils_group_variable_NaNs_logger_warnings(caplog):
     ds_Sv = ep.calibrate.compute_Sv(ed, **{"waveform_mode":"CW", "encode_mode":"power"})
 
     # Set first index of group variables to NaN
-    ed["Platform"]["water_level"] = np.nan # Is a scalar
+    ed["Platform"]["water_level"].values = np.nan # Is a scalar
     ed["Platform"]["vertical_offset"].values[0] = np.nan
     ed["Platform"]["transducer_offset_z"].values[0] = np.nan
     ed["Platform"]["pitch"].values[0] = np.nan
@@ -337,7 +337,7 @@ def test_add_depth_errors():
     with pytest.raises(NotImplementedError, match=(
         "`use_platform/beam_...` not implemented yet for `AZFP`."
     )):
-        ed["Sonar"].attrs["sonar_model"] = "AZFP"
+        ed["Sonar"] = ed["Sonar"].assign_attrs(sonar_model="AZFP")
         ep.consolidate.add_depth(ds_Sv, ed, use_platform_angles=True)
 
 
@@ -375,9 +375,9 @@ def test_add_depth_EK_with_platform_vertical_offsets(file, sonar_model, compute_
     ds_Sv = ds_Sv.isel(range_sample=slice(0,5))
 
     # Replace any Platform Vertical Offset NaN values with 0
-    ed["Platform"]["water_level"] = ed["Platform"]["water_level"].fillna(0)
-    ed["Platform"]["vertical_offset"] = ed["Platform"]["vertical_offset"].fillna(0)
-    ed["Platform"]["transducer_offset_z"] = ed["Platform"]["transducer_offset_z"].fillna(0)
+    ed["Platform"]["water_level"].values = ed["Platform"]["water_level"].fillna(0).values
+    ed["Platform"]["vertical_offset"].values = ed["Platform"]["vertical_offset"].fillna(0).values
+    ed["Platform"]["transducer_offset_z"].values = ed["Platform"]["transducer_offset_z"].fillna(0).values
 
     # Compute `depth` using platform vertical offset values
     ds_Sv_with_depth = ep.consolidate.add_depth(ds_Sv, ed, use_platform_vertical_offsets=True)
@@ -431,8 +431,8 @@ def test_add_depth_EK_with_platform_angles(file, sonar_model, compute_Sv_kwargs)
     ds_Sv = ep.calibrate.compute_Sv(ed, **compute_Sv_kwargs)
 
     # Replace any Beam Angle NaN values with 0
-    ed["Platform"]["pitch"] = ed["Platform"]["pitch"].fillna(0)
-    ed["Platform"]["roll"] = ed["Platform"]["roll"].fillna(0)
+    ed["Platform"]["pitch"].values = ed["Platform"]["pitch"].fillna(0).values
+    ed["Platform"]["roll"].values = ed["Platform"]["roll"].fillna(0).values
 
     # Compute `depth` using platform angle values
     ds_Sv_with_depth = ep.consolidate.add_depth(ds_Sv, ed, use_platform_angles=True)
@@ -485,9 +485,9 @@ def test_add_depth_EK_with_beam_angles(file, sonar_model, compute_Sv_kwargs):
     ds_Sv = ep.calibrate.compute_Sv(ed, **compute_Sv_kwargs)
 
     # Replace Beam Angle NaN values
-    ed["Sonar/Beam_group1"]["beam_direction_x"] = ed["Sonar/Beam_group1"]["beam_direction_x"].fillna(0)
-    ed["Sonar/Beam_group1"]["beam_direction_y"] = ed["Sonar/Beam_group1"]["beam_direction_y"].fillna(0)
-    ed["Sonar/Beam_group1"]["beam_direction_z"] = ed["Sonar/Beam_group1"]["beam_direction_z"].fillna(1)
+    ed["Sonar/Beam_group1"]["beam_direction_x"].values = ed["Sonar/Beam_group1"]["beam_direction_x"].fillna(0).values
+    ed["Sonar/Beam_group1"]["beam_direction_y"].values = ed["Sonar/Beam_group1"]["beam_direction_y"].fillna(0).values
+    ed["Sonar/Beam_group1"]["beam_direction_z"].values = ed["Sonar/Beam_group1"]["beam_direction_z"].fillna(1).values
 
     # Compute `depth` using beam angle values
     ds_Sv_with_depth = ep.consolidate.add_depth(ds_Sv, ed, use_beam_angles=True)

echopype/tests/convert/test_convert_ek80.py

@@ -4,11 +4,14 @@
 import numpy as np
 import pandas as pd
 from scipy.io import loadmat
+import xarray as xr
 
 from echopype import open_raw, open_converted
 from echopype.testing import TEST_DATA_FOLDER
 from echopype.convert.parse_ek80 import ParseEK80
 from echopype.convert.set_groups_ek80 import WIDE_BAND_TRANS, PULSE_COMPRESS, FILTER_IMAG, FILTER_REAL, DECIMATION
+from echopype.utils import log
+from echopype.convert.utils.ek_duplicates import check_unique_ping_time_duplicates
 
 
 @pytest.fixture
@@ -512,6 +515,62 @@ def test_parse_missing_sound_velocity_profile():
     shutil.rmtree(save_path)
 
 
+@pytest.mark.unit
+def test_duplicate_ping_times(caplog):
+    """
+    Tests that RAW file with duplicate ping times can be parsed and that the correct warning has been raised.
+    """
+    # Turn on logger verbosity
+    log.verbose(override=False)
+
+    # Open RAW
+    ed = open_raw("echopype/test_data/ek80_duplicate_ping_times/Hake-D20210913-T130612.raw", sonar_model="EK80")
+
+    # Check that there are no ping time duplicates in Beam group
+    assert ed["Sonar/Beam_group1"].equals(
+        ed["Sonar/Beam_group1"].drop_duplicates(dim="ping_time")
+    )
+
+    # Check that no warning is logged since the data for all duplicate pings is unique
+    not_expected_warning = ("All duplicate ping_time entries' will be removed, resulting in potential data loss.")
+    assert not any(not_expected_warning in record.message for record in caplog.records)
+
+    # Turn off logger verbosity
+    log.verbose(override=True)
+
+
+@pytest.mark.unit
+def test_check_unique_ping_time_duplicates(caplog):
+    """
+    Checks that `check_unique_ping_time_duplicates` raises a warning when the data for duplicate ping times is not unique.
+    """
+    # Initialize logger
+    logger = log._init_logger(__name__)
+
+    # Turn on logger verbosity
+    log.verbose(override=False)
+
+    # Open duplicate ping time beam dataset
+    ds_data = xr.open_zarr("echopype/test_data/ek80_duplicate_ping_times/duplicate_beam_ds.zarr")
+
+    # Modify a single entry to ensure that there exists duplicate ping times that do not share the same backscatter data
+    ds_data["backscatter_r"][0,0,0] = 0
+
+    # Check for ping time duplicates
+    check_unique_ping_time_duplicates(ds_data, logger)
+
+    # Turn off logger verbosity
+    log.verbose(override=True)
+
+    # Check if the expected warning is logged
+    expected_warning = (
+        "Duplicate slices in variable 'backscatter_r' corresponding to 'ping_time' "
+        f"{str(ds_data['ping_time'].values[0])} differ in data. All duplicate "
+        "'ping_time' entries will be removed, which will result in data loss."
+    )
+    assert any(expected_warning in record.message for record in caplog.records)
+
+
 @pytest.mark.unit
 def test_parse_ek80_with_invalid_env_datagrams():
     """

requirements.txt

@@ -9,7 +9,7 @@ pytz
 scipy
 xarray>=2024.11.0
 pandas
-zarr
+zarr>=2,<3
 fsspec
 s3fs
 requests