diff --git a/.gitignore b/.gitignore index db2eae9b9..53506501c 100644 --- a/.gitignore +++ b/.gitignore @@ -115,5 +115,13 @@ ENV/ trains/ .nnunetv2-venv/ +.venv_s3/ sanbox.ipynb !documentation/assets/scribble_example.png +# s3 files +config.yaml +train_s3.sh +build-and-push.yc.sh +Dockerfile +.dockerignore +s3_logs_[0-3][0-9].[0-1][0-9].[0-9][0-9][0-9][0-9]-[0-2][0-9]:[0-5][0-9]:[0-5][0-9]/ \ No newline at end of file diff --git a/nnunetv2/run/run_training.py b/nnunetv2/run/run_training.py index fff2ecd2e..4150b9674 100644 --- a/nnunetv2/run/run_training.py +++ b/nnunetv2/run/run_training.py @@ -15,6 +15,8 @@ from nnunetv2.utilities.find_class_by_name import recursive_find_python_class from torch.backends import cudnn +multiprocessing.set_start_method('spawn') # FOR CLUSTER TRAIN + def find_free_network_port() -> int: """Finds a free port on localhost. diff --git a/nnunetv2/training/dataloading/utils.py b/nnunetv2/training/dataloading/utils.py index 603f1ae67..9e2401263 100644 --- a/nnunetv2/training/dataloading/utils.py +++ b/nnunetv2/training/dataloading/utils.py @@ -9,6 +9,7 @@ from batchgenerators.utilities.file_and_folder_operations import isfile, subfiles from nnunetv2.configuration import default_num_processes +multiprocessing.set_start_method('spawn') # FOR CLUSTER TRAIN def _convert_to_npy(npz_file: str, unpack_segmentation: bool = True, overwrite_existing: bool = False, verify_npy: bool = False, fail_ctr: int = 0) -> None: diff --git a/scripts/train.sh b/scripts/train.sh old mode 100644 new mode 100755