add exception handler in _get_master_addr_port since the port might b…

…e null

dlrover/python/elastic_agent/torch/training.py

@@ -485,7 +485,7 @@ def _rendezvous(self, worker_group: WorkerGroup) -> None:
  spec.master_port,
  )
 
- master_addr, master_port = self._get_master_addr_port(store)
+ master_addr, master_port = self._safe_get_master_addr_port(store)
 
  # compatible with torch 2.4
  if not version_less_than_240():
@@ -543,6 +543,17 @@ def _get_master_addr_port(self, store: Store) -> Tuple[str, int]:
  master_port = int(store.get("MASTER_PORT").decode(encoding="UTF-8"))
  return (master_addr, master_port)
 
+ def _safe_get_master_addr_port(self, store: Store) -> Tuple[str, int]:
+ for i in range(1,5):
+ try:
+ addr, port = self._get_master_addr_port(store)
+ return (addr, port)
+ except Exception as e:
+ logger.warning(f"_get_master_addr_port failed with exception {e}, will try again")
+ time.sleep(10)
+
+ raise ValueError("invalid value in _get_master_addr_port")
+
  def _get_socket_with_port(self) -> socket.socket:
  """Return a free port on localhost.