Fix data_parallel_shard_degree description (#659)

https://github.com/pytorch/torchtitan/blob/7310abea8782bbe459b662bc6d8411fe8d55f62c/torchtitan/parallelisms/parallel_dims.py#L35-L36 does not allow `data_parallel_replicate_degree` to be -1
pytorch · Oct 29, 2024 · dbb0520 · dbb0520
1 parent 7310abe
commit dbb0520
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/torchtitan/config_manager.py b/torchtitan/config_manager.py
@@ -249,9 +249,7 @@ def __init__(self):
             parallelism method used is FSDP (Fully Sharded Data Parallelism).
 
             -1 means leftover ranks will be used (After DP_REPLICATE/SP/PP). Note that
-            only one of `data_parallel_replicate_degree` and `data_parallel_shard_degree`
-            can be negative.
-            1 means disabled.""",
+            only `data_parallel_shard_degree` can be negative. 1 means disabled.""",
         )
         self.parser.add_argument(
             "--training.enable_cpu_offload",