Reorder inputs before multiplications

This should be a stable order. Ideally this would be configurable to the
user or tool auther, but for now this is based on the order within which
data paramaters appear in tool.inputs, which depends on the order in the
XML/YML.

If this isn't stable the order of the input combinations may change, and
elements will jump in the collection structure. For instance the
reverse:forward output for a pair:pair in inputA multiplied by pair:pair
in input B may end up in forward:reverse if re-run.

Author: mvdbeek

lib/galaxy/tools/parameters/meta.py

@@ -3,13 +3,15 @@
 import copy
 import itertools
 import logging
+from collections import OrderedDict
 
 from galaxy import (
     exceptions,
     model,
     util
 )
 from galaxy.util import permutations
+from . import visit_input_values
 
 log = logging.getLogger(__name__)
 
@@ -70,6 +72,28 @@ def expand_workflow_inputs(inputs):
     return params, params_keys
 
 
+def process_key(incoming_key, d):
+    key_parts = incoming_key.split('|')
+    if len(key_parts) == 1:
+        # Regular parameter
+        d[incoming_key] = object()
+    elif key_parts[0].rsplit('_', maxsplit=1)[-1].isdigit():
+        # Repeat
+        input_name_index = key_parts[0].rsplit('_', maxsplit=1)
+        input_name, index = input_name_index
+        if input_name not in d:
+            d[input_name] = []
+        subdict = {}
+        d[input_name].append(subdict)
+        process_key("|".join(key_parts[1:]), d=subdict)
+    else:
+        # Section / Conditional
+        input_name = key_parts[0]
+        subdict = {}
+        d[input_name] = subdict
+        process_key("|".join(key_parts[1:]), d=subdict)
+
+
 def expand_meta_parameters(trans, tool, incoming):
     """
     Take in a dictionary of raw incoming parameters and expand to a list
@@ -84,6 +108,25 @@ def expand_meta_parameters(trans, tool, incoming):
     for key in to_remove:
         incoming.pop(key)
 
+    # If we're going to multiply input dataset combinations
+    # order matters, so the following reorders incoming
+    # according to tool.inputs (which is ordered).
+    incoming_copy = incoming.copy()
+    nested_dict = {}
+    for incoming_key in incoming_copy:
+        if not incoming_key.startswith('__'):
+            process_key(incoming_key, d=nested_dict)
+
+    reordered_incoming = OrderedDict()
+
+    def visitor(input, value, prefix, prefixed_name, prefixed_label, error, **kwargs):
+        if prefixed_name in incoming_copy:
+            reordered_incoming[prefixed_name] = incoming_copy[prefixed_name]
+            del incoming_copy[prefixed_name]
+
+    visit_input_values(inputs=tool.inputs, input_values=nested_dict, callback=visitor)
+    reordered_incoming.update(incoming_copy)
+
     def classifier(input_key):
         value = incoming[input_key]
         if isinstance(value, dict) and 'values' in value:
@@ -111,7 +154,7 @@ def classifier(input_key):
 
     # Stick an unexpanded version of multirun keys so they can be replaced,
     # by expand_mult_inputs.
-    incoming_template = incoming.copy()
+    incoming_template = reordered_incoming
 
     expanded_incomings = permutations.expand_multi_inputs(incoming_template, classifier)
     if collections_to_match.has_collections():

lib/galaxy/util/permutations.py

@@ -6,6 +6,8 @@
 Maybe this doesn't make sense and maybe much of this stuff could be replaced
 with itertools product and permutations. These are open questions.
 """
+from collections import OrderedDict
+
 from galaxy.exceptions import MessageException
 from galaxy.util.bunch import Bunch
 
@@ -40,9 +42,9 @@ def expand_multi_inputs(inputs, classifier, key_filter=None):
 def __split_inputs(inputs, classifier, key_filter):
     key_filter = key_filter or (lambda x: True)
 
-    single_inputs = {}
-    matched_multi_inputs = {}
-    multiplied_multi_inputs = {}
+    single_inputs = OrderedDict()
+    matched_multi_inputs = OrderedDict()
+    multiplied_multi_inputs = OrderedDict()
 
     for input_key in filter(key_filter, inputs):
         input_type, expanded_val = classifier(input_key)