Move preprocess functions and tests new clean and commongrid subpackages

.ci_helpers/run-test.py

@@ -27,13 +27,14 @@
     "root": {},  # This is to test the root folder.
     "convert": {},
     "calibrate": {},
+    "consolidate": {},
     "echodata": {},
+    "filter": {},
+    "mask": {},
+    "metrics": {},
     "preprocess": {},
     "utils": {},
     "visualize": {},
-    "metrics": {},
-    "mask": {},
-    "consolidate": {},
 }
 
 if __name__ == "__main__":

echopype/__init__.py

@@ -2,7 +2,7 @@
 
 from _echopype_version import version as __version__  # noqa
 
-from . import calibrate, consolidate, mask, preprocess, utils
+from . import calibrate, consolidate, filter, mask, preprocess, utils
 from .convert.api import open_raw
 from .echodata.api import open_converted
 from .echodata.combine import combine_echodata
@@ -20,6 +20,7 @@
     "combine_echodata",
     "calibrate",
     "consolidate",
+    "filter",
     "mask",
     "preprocess",
     "utils",

echopype/filter/__init__.py

@@ -0,0 +1,5 @@
+from .api import remove_noise
+
+__all__ = [
+    "remove_noise",
+]

echopype/filter/api.py

@@ -0,0 +1,72 @@
+"""
+Functions for enhancing the spatial and temporal coherence of data.
+"""
+
+from ..utils.prov import echopype_prov_attrs
+from .noise_est import NoiseEst
+
+
+def estimate_noise(ds_Sv, ping_num, range_sample_num, noise_max=None):
+    """
+    Remove noise by using estimates of background noise
+    from mean calibrated power of a collection of pings.
+
+    See ``remove_noise`` for reference.
+
+    Parameters
+    ----------
+    ds_Sv : xr.Dataset
+        dataset containing ``Sv`` and ``echo_range`` [m]
+    ping_num : int
+        number of pings to obtain noise estimates
+    range_sample_num : int
+        number of samples along the ``range_sample`` dimension to obtain noise estimates
+    noise_max : float
+        the upper limit for background noise expected under the operating conditions
+
+    Returns
+    -------
+    A DataArray containing noise estimated from the input ``ds_Sv``
+    """
+    noise_obj = NoiseEst(ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num)
+    noise_obj.estimate_noise(noise_max=noise_max)
+    return noise_obj.Sv_noise
+
+
+def remove_noise(ds_Sv, ping_num, range_sample_num, noise_max=None, SNR_threshold=3):
+    """
+    Remove noise by using estimates of background noise
+    from mean calibrated power of a collection of pings.
+
+    Reference: De Robertis & Higginbottom. 2007.
+    A post-processing technique to estimate the signal-to-noise ratio
+    and remove echosounder background noise.
+    ICES Journal of Marine Sciences 64(6): 1282–1291.
+
+    Parameters
+    ----------
+    ds_Sv : xr.Dataset
+        dataset containing ``Sv`` and ``echo_range`` [m]
+    ping_num : int
+        number of pings to obtain noise estimates
+    range_sample_num : int
+        number of samples along the ``range_sample`` dimension to obtain noise estimates
+    noise_max : float
+        the upper limit for background noise expected under the operating conditions
+    SNR_threshold : float
+        acceptable signal-to-noise ratio, default to 3 dB
+
+    Returns
+    -------
+    The input dataset with additional variables, including
+    the corrected Sv (``Sv_corrected``) and the noise estimates (``Sv_noise``)
+    """
+    noise_obj = NoiseEst(ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num)
+    noise_obj.remove_noise(noise_max=noise_max, SNR_threshold=SNR_threshold)
+    ds_Sv = noise_obj.ds_Sv
+
+    prov_dict = echopype_prov_attrs(process_type="processing")
+    prov_dict["processing_function"] = "preprocess.remove_noise"
+    ds_Sv = ds_Sv.assign_attrs(prov_dict)
+
+    return ds_Sv

echopype/preprocess/__init__.py

@@ -1,7 +1,6 @@
-from .api import compute_MVBS, compute_MVBS_index_binning, remove_noise
+from .api import compute_MVBS, compute_MVBS_index_binning
 
 __all__ = [
     "compute_MVBS",
     "compute_MVBS_index_binning",
-    "remove_noise",
 ]

echopype/preprocess/api.py

@@ -8,7 +8,6 @@
 
 from ..utils.prov import echopype_prov_attrs
 from .mvbs import get_MVBS_along_channels
-from .noise_est import NoiseEst
 
 
 def _set_MVBS_attrs(ds):
@@ -214,71 +213,5 @@ def compute_MVBS_index_binning(ds_Sv, range_sample_num=100, ping_num=100):
     return ds_MVBS
 
 
-def estimate_noise(ds_Sv, ping_num, range_sample_num, noise_max=None):
-    """
-    Remove noise by using estimates of background noise
-    from mean calibrated power of a collection of pings.
-
-    See ``remove_noise`` for reference.
-
-    Parameters
-    ----------
-    ds_Sv : xr.Dataset
-        dataset containing ``Sv`` and ``echo_range`` [m]
-    ping_num : int
-        number of pings to obtain noise estimates
-    range_sample_num : int
-        number of samples along the ``range_sample`` dimension to obtain noise estimates
-    noise_max : float
-        the upper limit for background noise expected under the operating conditions
-
-    Returns
-    -------
-    A DataArray containing noise estimated from the input ``ds_Sv``
-    """
-    noise_obj = NoiseEst(ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num)
-    noise_obj.estimate_noise(noise_max=noise_max)
-    return noise_obj.Sv_noise
-
-
-def remove_noise(ds_Sv, ping_num, range_sample_num, noise_max=None, SNR_threshold=3):
-    """
-    Remove noise by using estimates of background noise
-    from mean calibrated power of a collection of pings.
-
-    Reference: De Robertis & Higginbottom. 2007.
-    A post-processing technique to estimate the signal-to-noise ratio
-    and remove echosounder background noise.
-    ICES Journal of Marine Sciences 64(6): 1282–1291.
-
-    Parameters
-    ----------
-    ds_Sv : xr.Dataset
-        dataset containing ``Sv`` and ``echo_range`` [m]
-    ping_num : int
-        number of pings to obtain noise estimates
-    range_sample_num : int
-        number of samples along the ``range_sample`` dimension to obtain noise estimates
-    noise_max : float
-        the upper limit for background noise expected under the operating conditions
-    SNR_threshold : float
-        acceptable signal-to-noise ratio, default to 3 dB
-
-    Returns
-    -------
-    The input dataset with additional variables, including
-    the corrected Sv (``Sv_corrected``) and the noise estimates (``Sv_noise``)
-    """
-    noise_obj = NoiseEst(ds_Sv=ds_Sv.copy(), ping_num=ping_num, range_sample_num=range_sample_num)
-    noise_obj.remove_noise(noise_max=noise_max, SNR_threshold=SNR_threshold)
-    ds_Sv = noise_obj.ds_Sv
-
-    prov_dict = echopype_prov_attrs(process_type="processing")
-    prov_dict["processing_function"] = "preprocess.remove_noise"
-    ds_Sv = ds_Sv.assign_attrs(prov_dict)
-
-    return ds_Sv
-
-
 def regrid():
     return 1

echopype/tests/filter/test_noise.py

@@ -0,0 +1,98 @@
+import numpy as np
+import xarray as xr
+import echopype as ep
+import pytest
+from numpy.random import default_rng
+
+
+def test_remove_noise():
+    """Test remove_noise on toy data"""
+
+    # Parameters for fake data
+    nchan, npings, nrange_samples = 1, 10, 100
+    chan = np.arange(nchan).astype(str)
+    ping_index = np.arange(npings)
+    range_sample = np.arange(nrange_samples)
+    data = np.ones(nrange_samples)
+
+    # Insert noise points
+    np.put(data, 30, -30)
+    np.put(data, 60, -30)
+    # Add more pings
+    data = np.array([data] * npings)
+    # Make DataArray
+    Sv = xr.DataArray(
+        [data],
+        coords=[
+            ('channel', chan),
+            ('ping_time', ping_index),
+            ('range_sample', range_sample),
+        ],
+    )
+    Sv.name = "Sv"
+    ds_Sv = Sv.to_dataset()
+
+    ds_Sv = ds_Sv.assign(
+        echo_range=xr.DataArray(
+            np.array([[np.linspace(0, 10, nrange_samples)] * npings]),
+            coords=Sv.coords,
+        )
+    )
+    ds_Sv = ds_Sv.assign(sound_absorption=0.001)
+    # Run noise removal
+    ds_Sv = ep.filter.remove_noise(
+        ds_Sv, ping_num=2, range_sample_num=5, SNR_threshold=0
+    )
+
+    # Test if noise points are nan
+    assert np.isnan(
+        ds_Sv.Sv_corrected.isel(channel=0, ping_time=0, range_sample=30)
+    )
+    assert np.isnan(
+        ds_Sv.Sv_corrected.isel(channel=0, ping_time=0, range_sample=60)
+    )
+
+    # Test remove noise on a normal distribution
+    np.random.seed(1)
+    data = np.random.normal(
+        loc=-100, scale=2, size=(nchan, npings, nrange_samples)
+    )
+    # Make Dataset to pass into remove_noise
+    Sv = xr.DataArray(
+        data,
+        coords=[
+            ('channel', chan),
+            ('ping_time', ping_index),
+            ('range_sample', range_sample),
+        ],
+    )
+    Sv.name = "Sv"
+    ds_Sv = Sv.to_dataset()
+    # Attach required echo_range and sound_absorption values
+    ds_Sv = ds_Sv.assign(
+        echo_range=xr.DataArray(
+            np.array([[np.linspace(0, 3, nrange_samples)] * npings]),
+            coords=Sv.coords,
+        )
+    )
+    ds_Sv = ds_Sv.assign(sound_absorption=0.001)
+    # Run noise removal
+    ds_Sv = ep.filter.remove_noise(
+        ds_Sv, ping_num=2, range_sample_num=5, SNR_threshold=0
+    )
+    null = ds_Sv.Sv_corrected.isnull()
+    # Test to see if the right number of points are removed before the range gets too large
+    assert (
+        np.count_nonzero(null.isel(channel=0, range_sample=slice(None, 50)))
+        == 6
+    )
+
+
+def test_remove_noise_no_sound_absorption():
+    """
+    Tests remove_noise on toy data that does
+    not have sound absorption as a variable.
+    """
+
+    pytest.xfail(f"Tests for remove_noise have not been implemented" +
+                 " when no sound absorption is provided!")