Introduce --dimParams compiler option, and use it for dynamic backend tests on NNPA to utilze NNPA.

Signed-off-by: Yasushi Negishi <[email protected]>

Author: negiyas

docs/Testing.md

@@ -54,7 +54,6 @@ The onnx node tests usually have known dimension size for input tensors. So, to
 IMPORTER_FORCE_DYNAMIC='-1:-1' all dimensions of all the inputs will be changed
 IMPORTER_FORCE_DYNAMIC='0:-1' all dimensions of the first input will be changed
 IMPORTER_FORCE_DYNAMIC='0:-1|1:0,1' all dimensions of the first input and the 1st and 2nd dimensions of the second input will be changed
-IMPORTER_FORCE_DYNAMIC='0:0=a,1=b,2=c|1:0=a,1=b,2=c' the first three dimensions of the first of and the second inputs are changed. And assume that the first dimensions of the first and second arguments are the same, and same for the the second and third dimensions.
 ```
 
 The Backus-Naur Form (BNF) for `IMPORTER_FORCE_DYNAMIC` is as follows.
@@ -64,11 +63,10 @@ The Backus-Naur Form (BNF) for `IMPORTER_FORCE_DYNAMIC` is as follows.
             <inputString ::= <inputIndex> `:` <dimString>
              <dimString> ::= <dimIndex> | <dimIndex> `,` <dimString>
             <inputIndex> ::= <index>
-              <dimIndex> ::= <index> | <index> '=' <symbol>
+              <dimIndex> ::= <index>
                  <index> ::= -1 | <number>
                 <number> ::= <digit> | <digit><number>
                  <digit> ::= 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9
-                <symbol> ::= 'a', 'b', 'c', ..., 'z'
 ```
 Value `-1` semantically represents all inputs or all dimensions, and it has the highest priority. E.g. `'0: -1, 0'` means all dimensions of the first input will be changed. Input and dimension indices start from 0.
 
@@ -84,12 +82,10 @@ with `IMPORTER_FORCE_DYNAMIC='0:-1'`, the result is:
 ```
 func @main_graph(%arg0: tensor<?x?x?xf32>, %arg1: tensor<3x4x5xf32>) -> tensor<3x4x5xf32>
 ```
-
-with `IMPORTER_FORCE_DYNAMIC='0:0=a,2=b|1:1=a'`, the result is:
+with `IMPORTER_FORCE_DYNAMIC='0:0,2|1:1'`, the result is:
 ```
-func @main_graph(%arg0: tensor<?x4x?xf32>{onnx.name = "x“, onnx.dim_params = "0:a,2:b"}, %arg1: tensor<3x?x5xf32>{onnx.name = "y“, onnx.dim_params = "1:a"}) -> tensor<3x4x5xf32>
+func @main_graph(%arg0: tensor<?x4x?xf32>, %arg1: tensor<3x?x5xf32>) -> tensor<3x4x5xf32>
 ```
-The onnx.dim_params attributes are used to specify the assumptions about dimensions of the first and second arguments.
 This is a way to use existing node test for dynamic tensors. Since not all test case can pass with dynamic tensor, there is a list in test/backend/test.py, test_not_for_dynamic, to specify which test can not pass with `IMPORTER_FORCE_DYNAMIC` is defined.
 
 ### Tests with constant inputs

src/Builder/FrontendDialectTransformer.cpp

@@ -452,6 +452,27 @@ class FrontendGenImpl {
     return attributes;
   }
 
+  // Generate a string vector from the dimParams option string
+  void getInputDimParamsVecFromOption(std::string optionStr, size_t numOfArgs,
+      SmallVector<std::string> &paramStrVec) {
+    std::stringstream paramStrStream;
+    paramStrStream << optionStr;
+    std::string dimParamStr;
+    while (std::getline(paramStrStream, dimParamStr, '|')) {
+      size_t pos = dimParamStr.find(':');
+      assert((pos > 0) && "invalid dimParams option string");
+      int idx = stoi(dimParamStr.substr(0, pos));
+      dimParamStr = dimParamStr.substr(pos + 1);
+      std::replace(dimParamStr.begin(), dimParamStr.end(), '=', ':');
+      if (idx < 0) // set all arguments
+        for (size_t i = 0; i < numOfArgs; i++)
+          paramStrVec[i] = dimParamStr;
+      else
+        paramStrVec[idx] = dimParamStr;
+    }
+    return;
+  }
+
   /*!
    * An alternative graph importing procedure for importing ONNX subgraphs.
    * ONNX subgraphs, unlike the main computation graph, are imported as regions
@@ -492,6 +513,10 @@ class FrontendGenImpl {
     // See https://github.com/onnx/onnx/blob/main/docs/IR.md for more
     // information about dim_param.
     llvm::SmallVector<std::string, 4> inputDimParams, outputDimParams;
+    size_t numOfArgs = graph.input().size();
+    llvm::SmallVector<std::string> inputDimParamsFromOption(numOfArgs);
+    getInputDimParamsVecFromOption(
+        options_.dimParams, numOfArgs, inputDimParamsFromOption);
 
     // Import the input tensor types that are not constant and not initialized.
     int inputIndex = 0;
@@ -502,7 +527,9 @@ class FrontendGenImpl {
         std::string dimParams = "";
         Type argTy = ImportType(input.type(), &dimParams);
         argTy = modelInputShaper_.reshape(inputIndex, argTy);
-        if (!dimParams.empty())
+        if (!inputDimParamsFromOption[inputIndex].empty())
+          inputDimParams.emplace_back(inputDimParamsFromOption[inputIndex]);
+        else if (!dimParams.empty())
           inputDimParams.emplace_back(dimParams);
 
         argTypes.emplace_back(argTy);
@@ -1388,56 +1415,6 @@ class FrontendGenImpl {
     ret_vals.push_back(val);
   }
 
-  void getParamStr(std::string envStr, std::string &paramStr) {
-    std::replace(envStr.begin(), envStr.end(), '=', ':');
-    paramStr = envStr;
-    return;
-  }
-
-  // Generate string for input_dim_params attr from the IMPORTER_FORCE_DYNAMIC
-  // string.
-  void getInputDimParamStrVec(std::string envInputString, func::FuncOp funcOp,
-      size_t numOfArgs, SmallVector<std::string> &paramStrVec) {
-    std::stringstream paramStrStream;
-    paramStrStream << envInputString;
-    std::string envStr;
-    while (std::getline(paramStrStream, envStr, '|')) {
-      size_t pos = envStr.find(':');
-      assert((pos > 0) && "invalid IMPORTER_FORCE_DYNAMIC environment");
-      int idx = stoi(envStr.substr(0, pos));
-      envStr = envStr.substr(pos + 1);
-      if (idx < 0) { // set all arguments
-        for (size_t i = 0; i < numOfArgs; i++) {
-          getParamStr(envStr, paramStrVec[i]);
-        }
-      } else {
-        getParamStr(envStr, paramStrVec[idx]);
-      }
-    }
-    return;
-  }
-
-  // Get named attributesfrom IMPORTER_FORCE_DYNAMIC environment.
-  SmallVector<NamedAttribute> getNamedAttrFromImporterForceDynamic(
-      func::FuncOp funcOp, size_t numOfArgs) {
-    SmallVector<NamedAttribute> argNamedAttrs;
-    const char *envInputString = std::getenv("IMPORTER_FORCE_DYNAMIC");
-    if (envInputString == NULL)
-      return argNamedAttrs;
-    SmallVector<std::string> inputDimParamStrVec(numOfArgs);
-    getInputDimParamStrVec(
-        envInputString, funcOp, numOfArgs, inputDimParamStrVec);
-    for (size_t i = 0; i < numOfArgs; ++i) {
-      if (inputDimParamStrVec[i].length() != 0) {
-        auto name = builder_.getStringAttr(inputDimParamStrVec[i]);
-        NamedAttribute namedAttr =
-            builder_.getNamedAttr("onnx.dim_params", name);
-        argNamedAttrs.emplace_back(namedAttr);
-      }
-    }
-    return argNamedAttrs;
-  }
-
   // Move function attributes for argument/result names and dim_params into
   // argument/result attributes.
   void moveFuncAttrsToArgAttrs(func::FuncOp funcOp,
@@ -1448,10 +1425,6 @@ class FrontendGenImpl {
     Operation *op = funcOp.getOperation();
     size_t numOfArgs =
         (isArg) ? funcOp.getNumArguments() : funcOp.getNumResults();
-    SmallVector<NamedAttribute> argAttrsFromImporterForceDynamic;
-    if (isArg)
-      argAttrsFromImporterForceDynamic =
-          getNamedAttrFromImporterForceDynamic(funcOp, numOfArgs);
 
     // Only move attributes that exists.
     SmallVector<ArrayAttr, 2> funcAttrsToMove;
@@ -1476,8 +1449,6 @@ class FrontendGenImpl {
             argAttrs.emplace_back(namedAttr);
           }
         }
-        if (isArg && (i < argAttrsFromImporterForceDynamic.size()))
-          argAttrs.emplace_back(argAttrsFromImporterForceDynamic[i]);
       }
       if (!argAttrs.empty()) {
         if (isArg)

src/Builder/FrontendDialectTransformer.hpp

@@ -55,6 +55,16 @@ struct ImportOptions {
   //   - (arg0: tensor<3x4x5xf32>, arg1: tensor<10x5xf32>)
   //
   std::string shapeInformation = "";
+  // Custom onnx.dim_params attributes for the graph inputs for specifying
+  // relationship among their ranks.
+  // Its format is 'input_id:dim=sym,dim=sym,dim=sym|input_id:dim=sym,dim=sym..'
+  // E.g. An ONNX model has two dynamic inputs
+  //   - (arg0: tensor<?x5xf32>, arg1: tensor<?x5xf32>)
+  // If we want to specify that the first ranks of the first and second
+  // arguments are the same, we can use:
+  //   - dimParams = '0:0=batch|1:0=batch'
+  //
+  std::string dimParams = "";
   // Directory to look for external data if any tensor has external
   // data location. If empty then external data is disabled.
   std::string externalDataDir = "";

src/Builder/ModelInputShaper.cpp

@@ -36,8 +36,7 @@ ModelInputShaper::ModelInputShaper() : force_dim_dynamic_enabled_(false) {
       std::string dimIndex;
       std::vector<int> dims;
       while (std::getline(dimIndices, dimIndex, ',')) {
-        pos = dimIndex.find('='); // Ignore symbol part after '='
-        dims.emplace_back(stoi(dimIndex.substr(0, pos)));
+        dims.emplace_back(stoi(dimIndex));
       }
       // Default to the all dimensions if dims are not specified.
       if (dims.empty())

src/Compiler/CompilerOptions.cpp

@@ -51,6 +51,7 @@ bool preserveMLIR;                                     // onnx-mlir only
 bool useOnnxModelTypes;                                // onnx-mlir only
 int repeatOnnxTransform;                               // onnx-mlir only
 std::string shapeInformation;                          // onnx-mlir only
+std::string dimParams;                                 // onnx-mlir only
 ModelSize modelSize;                                   // onnx-mlir only
 bool storeConstantsToFile;                             // onnx-mlir only
 float constantsToFileTotalThreshold;                   // onnx-mlir only
@@ -281,6 +282,19 @@ static llvm::cl::opt<std::string, true> shapeInformationOpt("shapeInformation",
     llvm::cl::value_desc("value"), llvm::cl::location(shapeInformation),
     llvm::cl::cat(OnnxMlirOptions));
 
+static llvm::cl::opt<std::string, true> dimParamsOpt("dimParams",
+    llvm::cl::desc(
+        "Custom onnx.dim_params attributes for the inputs of the ONNX model for"
+        "specifying relationship among ranks of the inputs.\n"
+        "\"value\" is in the format of "
+        "\"INPUT_ID1:D1=S1,D2=S2,...,Dn=Sn|INPUT_ID2:D1=T1,D2=T2,...Dn=Tn\""
+        "where \"INPUT_ID1, INPUT_ID2, ...\" are input indices (starting from "
+        "0 or being -1 for all input indices), and\n"
+        "\"S1, S2, ...\" and \"T2, T2, ...\" are symbols to specify that same "
+        "symbols have the same value"),
+    llvm::cl::value_desc("value"), llvm::cl::location(dimParams),
+    llvm::cl::cat(OnnxMlirOptions));
+
 // Default value is defined by the OnnxMlirEnvOptionName constant string
 // variable, but the default setting mechanism here cannot be used here as we
 // need to evaluate this value prior to the compiler options being set. Proper

src/Compiler/CompilerOptions.hpp

@@ -94,6 +94,7 @@ extern bool preserveMLIR;                                     // onnx-mlir only
 extern bool useOnnxModelTypes;                                // onnx-mlir only
 extern int repeatOnnxTransform;                               // onnx-mlir only
 extern std::string shapeInformation;                          // onnx-mlir only
+extern std::string dimParams;                                 // onnx-mlir only
 extern ModelSize modelSize;                                   // onnx-mlir only
 extern bool storeConstantsToFile;                             // onnx-mlir only
 extern float constantsToFileTotalThreshold;                   // onnx-mlir only

src/Compiler/CompilerUtils.cpp

@@ -623,6 +623,7 @@ int processInputFile(StringRef inputFilename, mlir::MLIRContext &context,
     options.useOnnxModelTypes = useOnnxModelTypes;
     options.invokeOnnxVersionConverter = invokeOnnxVersionConverter;
     options.shapeInformation = shapeInformation;
+    options.dimParams = dimParams;
     options.allowSorting = allowSorting;
     options.externalDataDir = dirName(inputFilename);
     options.functionsToDecompose.insert(options.functionsToDecompose.end(),

test/accelerators/NNPA/backend/CMakeLists.txt

@@ -93,14 +93,15 @@ endif()
 # Followings are test cases of test_to_enable_dict in test/backend/inference_backend.py.
 # Only test cases for operations supported by zDNN are listed. Non-supported cases are
 # commented out. This list is set in the environment variable `TEST_CASE_BY_USER`. So,
-# instruction name and relationship among input arguments are added after test
-# case name in each test case if necessary. For example, 
+# instruction name and dimParams are added after test case name if necessary.
+# For example, the following line
 #    test_add_cpu,zdnn_add,"0:0=a,1=b,2=c|1:0=a,1=b,2=c"
-# means that (1) the test name is "test_add_cpu", (2) instruction name to be
-# checked if it is used is "zdnn_add", (3) relationship among input arguments
-# are "0:0=a,1=b,2=c|1:0=a,1=b,2=c", which means that the first ranks of the
-# first, second and third ranks of the first and second input arguments are the
-# same respectively.
+# means that (1) the test name is "test_add_cpu", (2) instruction name is
+# "zdnn_add" to check the function is called, (3) when dimParams is
+# "NO_DYNAMIC_SHAPE_TEST", backend test is skipped, otherwise the string is
+# passed as --dimParams option. "0:0=a,1=b,2=c|1:0=a,1=b,2=c" means that the
+# first ranks of the first, second and third ranks of the first and second
+# input arguments are the same respectively.
 set(NNPA_TEST_LIST
 
     # ==ARCH== NNPA
@@ -133,8 +134,8 @@ set(NNPA_TEST_LIST
     # ==OP== BatchNormalization
     # ==MIN== 6
     # ==LIM== Input and output tensor must be 4D(N x C x H x W).
-    test_batchnorm_epsilon_cpu,zdnn_mul,"0:0=a,1=b,2=c,3=d"
-    test_batchnorm_example_cpu,zdnn_mul,"0:0=a,1=b,2=c,3=d"
+    test_batchnorm_epsilon_cpu,zdnn_mul,NO_DYNAMIC_SHAPE_TEST
+    test_batchnorm_example_cpu,zdnn_mul,NO_DYNAMIC_SHAPE_TEST
 
     # ==OP== Conv
     # ==MIN== 1

test/backend/common.py

@@ -34,9 +34,7 @@ def determine_dynamic_parameters(test_name):
     selected_list = {args.input: {args.dim}}
     test_name_cpu = test_name + "_cpu"
     if test_name_cpu in variables.test_for_dynamic:
-        if test_name_cpu in test_to_enable_dynshape_dict:
-            selected_list = variables.test_to_enable_dynshape_dict[test_name_cpu]
-        elif len(variables.test_to_enable_dict[test_name_cpu]) > 1:
+        if len(variables.test_to_enable_dict[test_name_cpu]) > 1:
             selected_list = variables.test_to_enable_dict[test_name_cpu].get(
                 DYNAMIC_SHAPE
             )
@@ -52,25 +50,28 @@ def execute_commands(cmds, dynamic_inputs_dims):
     env_string = ""
     if dynamic_inputs_dims is not None:
         first_input = True
-        if isinstance(dynamic_inputs_dims, dict):
-            for input_index, dim_indices in dynamic_inputs_dims.items():
-                if first_input:
-                    env_string += str(input_index)
-                    first_input = False
+        for input_index, dim_indices in dynamic_inputs_dims.items():
+            if first_input:
+                env_string += str(input_index)
+                first_input = False
+            else:
+                env_string += "|" + str(input_index)
+            first_dim = True
+            for dim_index in dim_indices:
+                if first_dim:
+                    env_string += ":" + str(dim_index)
+                    first_dim = False
                 else:
-                    env_string += "|" + str(input_index)
-                first_dim = True
-                for dim_index in dim_indices:
-                    if first_dim:
-                        env_string += ":" + str(dim_index)
-                        first_dim = False
-                    else:
-                        env_string += "," + str(dim_index)
-        else:
-            env_string = dynamic_inputs_dims
+                    env_string += "," + str(dim_index)
         my_env["IMPORTER_FORCE_DYNAMIC"] = env_string
     subprocess.run(cmds, env=my_env, check=True)
 
+def get_compile_option(test_name):
+    if args.dynamic and test_name in variables.test_to_enable_dimparams_dict:
+        return ["--dimParams=" + variables.test_to_enable_dimparams_dict[test_name]]
+    else:
+        return []
+
 
 def check_instruction(test_name, exec_name):
     if args.instruction_check and test_name in variables.test_to_enable_symbol_dict:
@@ -137,6 +138,7 @@ def compile_model(model, emit):
             "--constants-to-file-total-threshold="
             + str(args.constants_to_file_total_threshold)
         )
+    command_list += get_compile_option(name + "_cpu")
 
     command_list.append(target[emit])
     command_list.append(model_name)

test/backend/inference_backend.py

@@ -3538,10 +3538,10 @@ def get_test_models():
     if TEST_CASE_BY_USER is not None and TEST_CASE_BY_USER != "":
         variables.test_by_user = TEST_CASE_BY_USER.split()
         variables.node_test_by_user = [
-            x for x in variables.test_by_user if x in node_test_to_enable
+            x for x in variables.test_by_user if x.split(",")[0] in node_test_to_enable
         ]
         variables.model_test_by_user = [
-            x for x in variables.test_by_user if x in model_test_to_enable
+            x for x in variables.test_by_user if x.split(",")[0] in model_test_to_enable
         ]
 
         node_test_to_enable = variables.node_test_by_user

test/backend/test.py

@@ -82,14 +82,14 @@
         quit()
 
     # Ensure that test names specified in test_to_enable actually exist.
-    for test_name_symbol_dynshape in (
+    for test_name_symbol_dimparam in (
         test_to_enable if not args.type else test_by_type[args.type]
     ):
-        test_name_symbol_dynshape_list = test_name_symbol_dynshape.split(",")
-        test_name = test_name_symbol_dynshape_list[0]
-        if args.instruction_check and len(test_name_symbol_dynshape_list) >= 2:
+        test_name_symbol_dimparam_list = test_name_symbol_dimparam.split(",")
+        test_name = test_name_symbol_dimparam_list[0]
+        if args.instruction_check and len(test_name_symbol_dimparam_list) >= 2:
             variables.test_to_enable_symbol_dict[test_name] = (
-                test_name_symbol_dynshape_list[1]
+                test_name_symbol_dimparam_list[1]
             )
         assert (
             test_name in all_test_names
@@ -99,9 +99,9 @@
             test_name
         )
         backend_test.include(r"^{}$".format(test_name))
-        if len(test_name_symbol_dynshape_list) >= 3:
-            variables.test_to_enable_dynshape_dict[test_name] = ",".join(
-                test_name_symbol_dynshape_list[2:]
+        if len(test_name_symbol_dimparam_list) >= 3:
+            variables.test_to_enable_dimparams_dict[test_name] = ",".join(
+                test_name_symbol_dimparam_list[2:]
             )
 
 # import all test cases at global scope to make them visible to python.unittest

test/backend/variables.py

@@ -326,8 +326,8 @@ def get_runtime_vars():
 
 # test_to_enable_dict
 try:
-    _ = test_to_enable_dict, test_to_enable_symbol_dict, test_to_enable_dynshape_dict
+    _ = test_to_enable_dict, test_to_enable_symbol_dict, test_to_enable_dimparams_dict
 except NameError:
     test_to_enable_dict = {}
     test_to_enable_symbol_dict = {}
-    test_to_enable_dynshape_dict = {}
+    test_to_enable_dimparams_dict = {}