Add Parallelism getter property to Accelerator class (#3703)

* Add rank property to Accelerator class

Signed-off-by: WoosungMyung <[email protected]>

* Raise errors when parallelism configuration is not enabled

Signed-off-by: WoosungMyung <[email protected]>

* Fix: PR feedback

Signed-off-by: WoosungMyung <[email protected]>

* Fix: style

---------

Signed-off-by: WoosungMyung <[email protected]>
Co-authored-by: S1ro1 <[email protected]>

Author: WoosungMyung

src/accelerate/accelerator.py

@@ -784,6 +784,56 @@ def _setup_parallelism_config(
 
         return parallelism_config
 
+    @property
+    def tensor_parallel_rank(self) -> int:
+        """
+        Returns the local rank for tensor parallelism. If tensor parallelism is configured but not enabled, returns 0
+        since all ranks are assumed to be the same.
+        """
+        if self.parallelism_config:
+            if self.parallelism_config.tp_enabled:
+                return self.torch_device_mesh.get_local_rank("tp")
+            return 0
+        raise RuntimeError("Tensor parallelism is not configured. Set `parallelism_config` first.")
+
+    @property
+    def pipeline_parallel_rank(self) -> int:
+        """
+        Pipeline parallelism is not supported yet.
+        """
+        raise NotImplementedError("Pipeline parallelism is currently not supported in Accelerate.")
+
+    @property
+    def context_parallel_rank(self) -> int:
+        """
+        Context parallelism is not supported yet.
+        """
+        raise NotImplementedError("Context parallelism is currently not supported in Accelerate.")
+
+    @property
+    def data_parallel_rank(self) -> int:
+        """
+        Returns the local rank for replicate-based data parallelism. If replicate-based data parallelism is configured
+        but not enabled, returns 0 since all ranks are assumed to be the same.
+        """
+        if self.parallelism_config:
+            if self.parallelism_config.dp_replicate_enabled:
+                return self.torch_device_mesh.get_local_rank("dp_replicate")
+            return 0
+        raise RuntimeError("Data parallelism is not configured. Set `parallelism_config` first.")
+
+    @property
+    def data_parallel_shard_rank(self) -> int:
+        """
+        Returns the local rank for shard-based data parallelism. If shard-based data parallelism is configured but not
+        enabled, returns 0 since all ranks are assumed to be the same.
+        """
+        if self.parallelism_config:
+            if self.parallelism_config.dp_shard_enabled:
+                return self.torch_device_mesh.get_local_rank("dp_shard")
+            return 0
+        raise RuntimeError("Shard-based data parallelism is not configured. Set `parallelism_config` first.")
+
     def _build_torch_device_mesh(self, parallelism_config):
         if PartialState._shared_state != {} and getattr(PartialState(), "device_mesh", None) is not None:
             device_mesh = PartialState().device_mesh