Merge pull request #303 from ecmwf/feature/avoid_duplicate_merging

fix duplicate merging

Author: mathleur

polytope_feature/datacube/backends/datacube.py

@@ -48,11 +48,8 @@ def validate(self, axes):
     def _create_axes(self, name, values, transformation_type_key, transformation_options):
         # first check what the final axes are for this axis name given transformations
         transformation_options = transformation_type_key
-        final_axis_names = DatacubeAxisTransformation.get_final_axes(
-            name, transformation_type_key.name, transformation_options
-        )
-        transformation = DatacubeAxisTransformation.create_transform(
-            name, transformation_type_key.name, transformation_options
+        (final_axis_names, transformation) = DatacubeAxisTransformation.get_final_axes(
+            name, transformation_type_key.name, transformation_options, self
         )
 
         # do not compress merged axes

polytope_feature/datacube/transformations/datacube_cyclic/datacube_cyclic.py

@@ -8,7 +8,7 @@
 class DatacubeAxisCyclic(DatacubeAxisTransformation):
     # The transformation here will be to point the old axes to the new cyclic axes
 
-    def __init__(self, name, cyclic_options):
+    def __init__(self, name, cyclic_options, datacube=None):
         self.name = name
         self.transformation_options = cyclic_options
         self.range = cyclic_options.range

polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py

@@ -7,7 +7,7 @@
 class DatacubeMapper(DatacubeAxisTransformation):
     # Needs to implements DatacubeAxisTransformation methods
 
-    def __init__(self, name, mapper_options):
+    def __init__(self, name, mapper_options, datacube=None):
         self.transformation_options = mapper_options
         self.grid_type = mapper_options.type
         self.grid_resolution = mapper_options.resolution

polytope_feature/datacube/transformations/datacube_merger/datacube_merger.py

@@ -7,12 +7,13 @@
 
 
 class DatacubeAxisMerger(DatacubeAxisTransformation):
-    def __init__(self, name, merge_options):
+    def __init__(self, name, merge_options, datacube=None):
         self.transformation_options = merge_options
         self.name = name
         self._first_axis = name
         self._second_axis = merge_options.other_axis
         self._linkers = merge_options.linkers
+        self._merged_values = self.merged_values(datacube)
 
     def blocked_axes(self):
         return [self._second_axis]
@@ -34,6 +35,7 @@ def merged_values(self, datacube):
         )
         merged_values = pd.to_datetime(combined_strings).to_numpy().astype("datetime64[s]")
         merged_values = np.array(merged_values)
+        merged_values.sort()
         logging.info(
             f"Merged values {first_ax_vals} on axis {self.name} and \
                      values {second_ax_vals} on axis {second_ax_name} to values {merged_values}"
@@ -74,7 +76,7 @@ def change_val_type(self, axis_name, values):
 
     def find_modified_indexes(self, indexes, path, datacube, axis):
         if axis.name == self._first_axis:
-            return self.merged_values(datacube)
+            return self._merged_values
 
     def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis):
         new_key_value_path = {}

polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py

@@ -3,7 +3,7 @@
 
 
 class DatacubeAxisReverse(DatacubeAxisTransformation):
-    def __init__(self, name, mapper_options):
+    def __init__(self, name, mapper_options, datacube=None):
         self.name = name
         self.transformation_options = mapper_options
 

polytope_feature/datacube/transformations/datacube_transformations.py

@@ -8,25 +8,25 @@ def __init__(self):
         self.parent = None
 
     @staticmethod
-    def create_transform(name, transformation_type_key, transformation_options):
+    def create_transform(name, transformation_type_key, transformation_options, datacube):
         transformation_type = _type_to_datacube_transformation_lookup[transformation_type_key]
         transformation_file_name = _type_to_transformation_file_lookup[transformation_type_key]
         file_name = ".datacube_" + transformation_file_name
         module = import_module("polytope_feature.datacube.transformations" + file_name + file_name)
         constructor = getattr(module, transformation_type)
         transformation_type_option = transformation_options
-        new_transformation = deepcopy(constructor(name, transformation_type_option))
+        new_transformation = deepcopy(constructor(name, transformation_type_option, datacube))
 
         new_transformation.name = name
         return new_transformation
 
     @staticmethod
-    def get_final_axes(name, transformation_type_key, transformation_options):
+    def get_final_axes(name, transformation_type_key, transformation_options, datacube):
         new_transformation = DatacubeAxisTransformation.create_transform(
-            name, transformation_type_key, transformation_options
+            name, transformation_type_key, transformation_options, datacube
         )
         transformation_axis_names = new_transformation.transformation_axes_final()
-        return transformation_axis_names
+        return (transformation_axis_names, new_transformation)
 
     def name(self):
         pass

polytope_feature/datacube/transformations/datacube_type_change/datacube_type_change.py

@@ -7,7 +7,7 @@
 class DatacubeAxisTypeChange(DatacubeAxisTransformation):
     # The transformation here will be to point the old axes to the new cyclic axes
 
-    def __init__(self, name, type_options):
+    def __init__(self, name, type_options, datacube=None):
         self.name = name
         self.transformation_options = type_options
         self.new_type = type_options.type