Support Python 3.12 and Migrate to Xarray DataTree (#1419)

* initial commit to support python 3.12 and migrate to xarray DataTree

* remove additional py3.11 file

* update imports

* add todo note to search through pr history to learn how to correctly work with rendertree

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix data tree render import

* pin xarray to later 2024 november version

* bump minimum python testing version and supported version to 3.10

* set time3 in nmea to time_nmea

* resolve merge conflicts

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* use channel all for ek80 sonar group

* add support for nmea_time

* import xarray import open_datatree test change

* fix datatree import pt 2

* support 3.12 in setup cfg

* update dim check

* resolve merge conflict

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove test notebook

* fix import

* comment out TestEchoData and save it for issue 1420

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* set max python to <= 3.12

* set to  less than ython 3.13

* set python 3.12 in setup cfg

* add ek80 channel test with two beam groups

* remove unnecessary comment

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: ctuguinay

.ci_helpers/py3.9.yaml renamed to .ci_helpers/py3.12.yaml

@@ -2,7 +2,7 @@ name: echopype
 channels:
   - conda-forge
 dependencies:
-  - python=3.9
+  - python=3.12
   - pip
   - pip:
     - -r ../requirements.txt

.github/workflows/build.yaml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         runs-on: [ubuntu-latest]
         experimental: [false]
     services:

.github/workflows/pr.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         runs-on: [ubuntu-latest]
         experimental: [false]
     services:

.github/workflows/windows-utils.yaml

@@ -21,7 +21,7 @@ jobs:
         fail-fast: false
         matrix:
           include:
-          - python-version: 3.9
+          - python-version: 3.12
             experimental: false
       defaults:
         run:

.github/workflows/windows.yaml

@@ -20,7 +20,7 @@ jobs:
         fail-fast: false
         matrix:
           include:
-          - python-version: 3.9
+          - python-version: 3.12
             experimental: false
       defaults:
         run:

docs/source/contributing.md

@@ -50,7 +50,7 @@ To create an environment for developing Echopype, we recommend the following ste
     ```shell
     # Create a conda environment using the supplied requirements files
     # Note the last one docs/requirements.txt is only required for building docs
-    conda create -c conda-forge -n echopype --yes python=3.9 --file requirements.txt --file requirements-dev.txt --file docs/requirements.txt
+    conda create -c conda-forge -n echopype --yes python=3.12 --file requirements.txt --file requirements-dev.txt --file docs/requirements.txt
 
     # Switch to the newly built environment
     conda activate echopype

docs/source/installation.md

@@ -2,7 +2,7 @@
 
 ## Installation
 
-Echopype is available and tested for Python 3.9-3.11. The latest release can be installed through conda (or mamba, see below) via the [conda-forge channel](https://anaconda.org/conda-forge/echopype):
+Echopype is available and tested for Python 3.10-3.12. The latest release can be installed through conda (or mamba, see below) via the [conda-forge channel](https://anaconda.org/conda-forge/echopype):
 ```shell
 # Install via conda-forge
 $ conda install -c conda-forge echopype
@@ -14,10 +14,6 @@ It is available via [PyPI](https://pypi.org/project/echopype):
 $ pip install echopype
 ```
 
-:::{note}
-We are working on adding support for Python 3.12 soon!
-:::
-
 :::{attention}
 It's common to encounter the situation that installing packages using Conda is slow or fails,
 because Conda is unable to resolve dependencies.

echopype/convert/api.py

@@ -2,7 +2,7 @@
 from typing import TYPE_CHECKING, Dict, Literal, Optional, Tuple, Union
 
 import fsspec
-from datatree import DataTree
+from xarray import DataTree
 
 # fmt: off
 # black and isort have conflicting ideas about how this should be formatted

echopype/convert/set_groups_base.py

@@ -149,14 +149,14 @@ def set_nmea(self) -> xr.Dataset:
         ds = xr.Dataset(
             {
                 "NMEA_datagram": (
-                    ["time1"],
+                    ["nmea_time"],
                     raw_nmea,
                     {"long_name": "NMEA datagram"},
                 )
             },
             coords={
-                "time1": (
-                    ["time1"],
+                "nmea_time": (
+                    ["nmea_time"],
                     time,
                     {
                         "axis": "T",

echopype/convert/set_groups_ek80.py

@@ -227,7 +227,7 @@ def set_sonar(self, beam_group_type: list = ["power", None]) -> xr.Dataset:
         # Create dataset
         sonar_vars = {
             "frequency_nominal": (
-                ["channel"],
+                ["channel_all"],
                 var["transducer_frequency"],
                 {
                     "units": "Hz",
@@ -237,21 +237,21 @@ def set_sonar(self, beam_group_type: list = ["power", None]) -> xr.Dataset:
                 },
             ),
             "transceiver_serial_number": (
-                ["channel"],
+                ["channel_all"],
                 var["serial_number"],
                 {
                     "long_name": "Transceiver serial number",
                 },
             ),
             "transducer_name": (
-                ["channel"],
+                ["channel_all"],
                 var["transducer_name"],
                 {
                     "long_name": "Transducer name",
                 },
             ),
             "transducer_serial_number": (
-                ["channel"],
+                ["channel_all"],
                 var["transducer_serial_number"],
                 {
                     "long_name": "Transducer serial number",
@@ -261,8 +261,8 @@ def set_sonar(self, beam_group_type: list = ["power", None]) -> xr.Dataset:
         ds = xr.Dataset(
             {**sonar_vars, **beam_groups_vars},
             coords={
-                "channel": (
-                    ["channel"],
+                "channel_all": (
+                    ["channel_all"],
                     self.sorted_channel["all"],
                     self._varattrs["beam_coord_default"]["channel"],
                 ),

echopype/echodata/combine.py

@@ -9,7 +9,7 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
-from datatree import DataTree
+from xarray import DataTree
 
 from ..utils.io import validate_output_path
 from ..utils.log import _init_logger
@@ -18,7 +18,7 @@
 
 logger = _init_logger(__name__)
 
-POSSIBLE_TIME_DIMS = {"time1", "time2", "time3", "time4", "ping_time"}
+POSSIBLE_TIME_DIMS = {"time1", "time2", "time3", "time4", "nmea_time", "ping_time"}
 APPEND_DIMS = {"filenames"}.union(POSSIBLE_TIME_DIMS)
 DATE_CREATED_ATTR = "date_created"
 CONVERSION_TIME_ATTR = "conversion_time"

echopype/echodata/echodata.py

@@ -8,7 +8,7 @@
 import fsspec
 import numpy as np
 import xarray as xr
-from datatree import DataTree, open_datatree
+from xarray import DataTree, open_datatree
 from zarr.errors import GroupNotFoundError, PathNotFoundError
 
 if TYPE_CHECKING:
@@ -242,7 +242,7 @@ def group_paths(self) -> Set[str]:
     def __get_dataset(node: DataTree) -> Optional[xr.Dataset]:
         if node.has_data or node.has_attrs:
             # validate and clean dtypes
-            return sanitize_dtypes(node.ds)
+            return sanitize_dtypes(node.to_dataset())
         return None
 
     def __get_node(self, key: Optional[str]) -> DataTree:
@@ -265,7 +265,7 @@ def __setitem__(self, __key: Optional[str], __newvalue: Any) -> Optional[xr.Data
         if self._tree:
             try:
                 node = self.__get_node(__key)
-                node.ds = __newvalue
+                node.dataset = __newvalue
                 return self.__get_dataset(node)
             except KeyError:
                 raise GroupNotFoundError(__key)
@@ -283,9 +283,9 @@ def __setattr__(self, __name: str, __value: Any) -> None:
                 group_path = group["ep_group"]
                 if self._tree:
                     if __name == "top":
-                        self._tree.ds = __value
+                        self._tree.dataset = __value
                     else:
-                        self._tree[group_path].ds = __value
+                        self._tree[group_path].dataset = __value
         super().__setattr__(__name, attr_value)
 
     @add_processing_level("L1A")

echopype/echodata/widgets/utils.py

@@ -1,8 +1,8 @@
 import uuid
 from hashlib import md5
 
-from datatree import DataTree
-from datatree.render import RenderTree
+from xarray import DataTree
+from xarray.core.datatree_render import RenderDataTree
 
 from ..convention.utils import _get_sonar_groups
 
@@ -63,7 +63,7 @@ def _single_node_repr(node: DataTree) -> str:
 
 
 def tree_repr(tree: DataTree) -> str:
-    renderer = RenderTree(tree)
+    renderer = RenderDataTree(tree)
     lines = []
     for pre, _, node in renderer:
         if node.has_data or node.has_attrs:

echopype/tests/convert/test_convert_ek60.py

@@ -238,7 +238,7 @@ def test_convert_ek60_different_num_channel_mode_values(file_path, ek60_path):
             np.float32
         )
 
-@pytest.mark.test
+
 @pytest.mark.integration
 def test_converting_ek60_raw_with_missing_channel_power():
     """

echopype/tests/convert/test_convert_ek80.py

@@ -446,7 +446,6 @@ def test_convert_ek80_mru1(ek80_path):
     np.all(echodata["Platform"]["vertical_offset"].data == np.array(parser.mru1["heave"]))
     np.all(echodata["Platform"]["heading"].data == np.array(parser.mru1["heading"]))
 
-
 @pytest.mark.unit
 def test_skip_ec150(ek80_path):
     """Make sure we skip EC150 datagrams correctly."""
@@ -457,7 +456,7 @@ def test_skip_ec150(ek80_path):
     assert "backscatter_i" in echodata["Sonar/Beam_group1"].data_vars
     assert (
         echodata["Sonar/Beam_group1"].dims
-        == {'channel': 1, 'ping_time': 2, 'range_sample': 115352, 'beam': 4}
+        == {'channel_all': 1, 'beam_group': 1, 'channel': 1, 'ping_time': 2, 'range_sample': 115352, 'beam': 4}
     )
 
 
@@ -531,3 +530,25 @@ def test_parse_ek80_with_invalid_env_datagrams():
         env_var = ed["Environment"][var]
         assert env_var.notnull().all() and env_var.dtype == np.float64
 
+
+@pytest.mark.unit
+def test_ek80_sonar_all_channel():
+    """
+    Checks that when an EK80 raw file has 2 beam groups, the converted Echodata object contains
+    all channels in the 'channel_all' dimension of the Sonar group.
+    """
+    # Convert EK80 Raw File
+    ed = open_raw(
+        raw_file="echopype/test_data/ek80_new/echopype-test-D20211004-T235714.raw",
+        sonar_model="EK80"
+    )
+
+    # Grab channels from Sonar group
+    channel_set = set(ed["Sonar"]["channel_all"].data)
+
+    # Grab channels from both beam groups
+    target_channel_set = set(ed["Sonar/Beam_group1"]["channel"].data)
+    target_channel_set.update(set(ed["Sonar/Beam_group2"]["channel"].data))
+
+    # Check that channel sets are equal
+    assert channel_set == target_channel_set

echopype/tests/convert/test_convert_source_target_locs.py

@@ -12,7 +12,7 @@
 import fsspec
 import xarray as xr
 import pytest
-from datatree import open_datatree
+from xarray import open_datatree
 from tempfile import TemporaryDirectory
 from echopype import open_raw
 from echopype.utils.coding import DEFAULT_ENCODINGS