Resolves #113 Refactor Parameterization

devtools/debug_parameterization.py

@@ -0,0 +1,40 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# The following is intended for developers of fabric-cicd to debug parameter.yml file locally against the github repo
+
+import sys
+from pathlib import Path
+
+from azure.identity import ClientSecretCredential
+
+from fabric_cicd import change_log_level
+from fabric_cicd._parameter._utils import validate_parameter_file
+from fabric_cicd.constants import PARAMETER_FILE_NAME
+
+root_directory = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(root_directory / "src"))
+
+# Uncomment to enable debug
+# change_log_level()
+
+# In this example, the parameter.yml file sits within the root/sample/workspace directory
+repository_directory = str(root_directory / "sample" / "workspace")
+
+# Explicitly define valid item types
+item_type_in_scope = ["DataPipeline", "Notebook", "Environment", "SemanticModel", "Report"]
+
+# Set target environment
+environment = "PPE"
+
+validate_parameter_file(
+    repository_directory=repository_directory,
+    item_type_in_scope=item_type_in_scope,
+    # Comment to exclude target environment in validation
+    environment=environment,
+    # Uncomment to pass in an alternative parameter file name
+    # Assign to the constant in constants.py or pass in a string directly
+    # parameter_file_name=PARAMETER_FILE_NAME,
+    # Uncomment to use SPN auth
+    # token_credential=token_credential,
+)

sample/workspace/parameter.yml

@@ -1,15 +1,53 @@
 find_replace:
-    # SQL Connection Guid
-    "db52be81-c2b2-4261-84fa-840c67f4bbd0":
+    # Lakehouse Connection Guid
+    - find_value: "db52be81-c2b2-4261-84fa-840c67f4bbd0"
+      replace_value:
         PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733"
         PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0"
+      # Optional fields:
+      item_type: "Notebook"
+      item_name: ["Hello World", "Hello World Subfolder"]
+      file_path:
+       - "/Hello World.Notebook/notebook-content.py"
+       - "\\subfolder\\Hello World Subfolder.Notebook/notebook-content.py"
 
 spark_pool:
     # CapacityPool_Large
-    "72c68dbc-0775-4d59-909d-a47896f4573b":
-        type: "Capacity"
-        name: "CapacityPool_Large"
+    - instance_pool_id: "72c68dbc-0775-4d59-909d-a47896f4573b"
+      replace_value:
+        PPE: 
+           type: "Capacity"
+           name: "CapacityPool_Large_PPE"
+        PROD: 
+           type: "Capacity"
+           name: "CapacityPool_Large_PROD"
+      # Optional field:
+      item_name: "World"
+    # CapacityPool_Medium
+    - instance_pool_id: "e7b8f1c4-4a6e-4b8b-9b2e-8f1e5d6a9c3d"
+      replace_value:
+        PPE:
+           type: "Workspace"
+           name: "WorkspacePool_Medium"
+        PROD:
+           type: "Workspace"
+           name: "WorkspacePool_Medium"
+      # Optional field:
+      item_name:
+
+# Legacy structure
+#find_replace:
+    # SQL Connection Guid
+    #"db52be81-c2b2-4261-84fa-840c67f4bbd0":
+        #PPE: "81bbb339-8d0b-46e8-bfa6-289a159c0733"
+        #PROD: "5d6a1b16-447f-464a-b959-45d0fed35ca0"
+
+#spark_pool:
+    # CapacityPool_Large
+    #"72c68dbc-0775-4d59-909d-a47896f4573b":
+        #type: "Capacity"
+        #name: "CapacityPool_Large"
     # CapacityPool_Medium
-    "e7b8f1c4-4a6e-4b8b-9b2e-8f1e5d6a9c3d":
-        type: "Workspace"
-        name: "WorkspacePool_Medium"
+    #"e7b8f1c4-4a6e-4b8b-9b2e-8f1e5d6a9c3d":
+        #type: "Workspace"
+        #name: "WorkspacePool_Medium"

src/fabric_cicd/_common/_exceptions.py

@@ -44,3 +44,7 @@ class ItemDependencyError(BaseCustomError):
 
 class FileTypeError(BaseCustomError):
     pass
+
+
+class ParameterFileError(BaseCustomError):
+    pass

src/fabric_cicd/_items/_environment.py

@@ -12,6 +12,7 @@
 
 from fabric_cicd import FabricWorkspace
 from fabric_cicd._common._fabric_endpoint import handle_retry
+from fabric_cicd._parameter._utils import check_parameter_structure
 
 logger = logging.getLogger(__name__)
 
@@ -60,7 +61,7 @@ def _publish_environment_metadata(fabric_workspace_obj: FabricWorkspace, item_na
     _check_environment_publish_state(fabric_workspace_obj, item_guid, initial_check=True)
 
     # Update compute settings
-    _update_compute_settings(fabric_workspace_obj, item_path, item_guid)
+    _update_compute_settings(fabric_workspace_obj, item_path, item_guid, item_name)
 
     repo_library_files = _get_repo_libraries(item_path)
 
@@ -128,14 +129,17 @@ def _check_environment_publish_state(
             iteration += 1
 
 
-def _update_compute_settings(fabric_workspace_obj: FabricWorkspace, item_path: Path, item_guid: str) -> None:
+def _update_compute_settings(
+    fabric_workspace_obj: FabricWorkspace, item_path: Path, item_guid: str, item_name: str
+) -> None:
     """
     Update spark compute settings.
 
     Args:
         fabric_workspace_obj: The FabricWorkspace object.
         item_path: The path to the environment item.
         item_guid: The GUID of the environment item.
+        item_name: Name of the environment item.
     """
     # Read compute settings from YAML file
     with Path.open(Path(item_path, "Setting", "Sparkcompute.yml"), "r+", encoding="utf-8") as f:
@@ -145,8 +149,22 @@ def _update_compute_settings(fabric_workspace_obj: FabricWorkspace, item_path: P
         if "instance_pool_id" in yaml_body:
             pool_id = yaml_body["instance_pool_id"]
             if "spark_pool" in fabric_workspace_obj.environment_parameter:
+                structure_type = check_parameter_structure(fabric_workspace_obj.environment_parameter, "spark_pool")
                 parameter_dict = fabric_workspace_obj.environment_parameter["spark_pool"]
-                if pool_id in parameter_dict:
+                # Handle new parameter file format
+                if structure_type == "new":
+                    for key in parameter_dict:
+                        instance_pool_id = key["instance_pool_id"]
+                        replace_value = key["replace_value"]
+                        input_name = key.get("item_name")
+                        if instance_pool_id == pool_id and (input_name == item_name or not input_name):
+                            # replace any found references with specified environment value
+                            yaml_body["instancePool"] = replace_value[fabric_workspace_obj.environment]
+                            del yaml_body["instance_pool_id"]
+
+                # Handle original parameter file format
+                # TODO: Deprecate old structure handling by April 24, 2025
+                if structure_type == "old" and pool_id in parameter_dict:
                     # replace any found references with specified environment value
                     yaml_body["instancePool"] = parameter_dict[pool_id]
                     del yaml_body["instance_pool_id"]

src/fabric_cicd/_parameter/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+