From 76f373767c8708a2774798e69a929b56f582d0fc Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 20 Feb 2023 11:04:56 +0000 Subject: [PATCH 01/58] =?UTF-8?q?=E2=9E=95=20Add=20minimum=20requirements?= =?UTF-8?q?=20for=20triggering=20AzureML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ++++ environment-dev.yml | 1 + monai/apps/auto3dseg/auto_runner.py | 5 ++++- monai/apps/auto3dseg/utils.py | 18 ++++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3da001d0ce..2945c7160b 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,7 @@ tests/testing_data/eval_data_stats.yaml # profiling results *.prof + +# AzureML Configuration + +azureml_config.json diff --git a/environment-dev.yml b/environment-dev.yml index 400822aaf3..734a374543 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -10,5 +10,6 @@ dependencies: - torchvision - pytorch-cuda=11.6 - pip + - python>=3.8 - pip: - -r requirements-dev.txt diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 290a79e324..a6543418d0 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -30,7 +30,9 @@ AlgoEnsembleBuilder, ) from monai.apps.auto3dseg.hpo_gen import NNIGen -from monai.apps.auto3dseg.utils import export_bundle_algo_history, import_bundle_algo_history +from monai.apps.auto3dseg.utils import ( + export_bundle_algo_history, import_bundle_algo_history, submit_to_azureml_if_needed, +) from monai.apps.utils import get_logger from monai.auto3dseg.utils import algo_to_pickle from monai.bundle import ConfigParser @@ -223,6 +225,7 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): + run_info = submit_to_azureml_if_needed() logger.info(f"AutoRunner using work directory {work_dir}") os.makedirs(work_dir, exist_ok=True) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 67cde64a2c..0eba6cc8b0 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -15,7 +15,10 @@ from monai.apps.auto3dseg.bundle_gen import BundleAlgo from monai.auto3dseg import algo_from_pickle, algo_to_pickle +from monai.utils import optional_import +# health_azure, has_health_azure = optional_import("health_azure") +import health_azure def import_bundle_algo_history( output_folder: str = ".", template_path: str | None = None, only_trained: bool = True @@ -66,3 +69,18 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: for task in history: for _, algo in task.items(): algo_to_pickle(algo, template_path=algo.template_path) + + +def submit_to_azureml_if_needed() -> health_azure.AzureRunInfo: + run_info = health_azure.submit_to_azure_if_needed( + compute_cluster_name="lite-testing-ds2", + workspace_config_file="azureml_config.json", + input_datasets=["dataset_goes_here"], + default_datastore="innereyedatasets", + docker_base_image="mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", + snapshot_root_directory=os.getcwd(), + conda_environment_file=os.path.join(os.getcwd(), "environment-dev.yml"), + entry_script=os.getcwd() + "/" + __file__, + ) + print(f"Run info generated: {str(run_info)}") + return run_info From c762d9e0b5269a0d6f482a46e7de41f02cd1bdbf Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 20 Feb 2023 11:14:20 +0000 Subject: [PATCH 02/58] Set AzureML to SDK v1 only Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 0eba6cc8b0..eaac5f913a 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -81,6 +81,7 @@ def submit_to_azureml_if_needed() -> health_azure.AzureRunInfo: snapshot_root_directory=os.getcwd(), conda_environment_file=os.path.join(os.getcwd(), "environment-dev.yml"), entry_script=os.getcwd() + "/" + __file__, + strictly_aml_v1=True, ) print(f"Run info generated: {str(run_info)}") return run_info From 6288cdcc2f0588af406ad5e639ab74c1d8408390 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 21 Feb 2023 09:13:42 +0000 Subject: [PATCH 03/58] =?UTF-8?q?=F0=9F=9A=A7=20Set=20up=20AzureML=20envir?= =?UTF-8?q?onment=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environment-azureml.yml | 68 +++++++++++++++++++++++++++++ monai/apps/auto3dseg/auto_runner.py | 5 ++- monai/apps/auto3dseg/utils.py | 22 ++++++---- 3 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 environment-azureml.yml diff --git a/environment-azureml.yml b/environment-azureml.yml new file mode 100644 index 0000000000..537a28f22f --- /dev/null +++ b/environment-azureml.yml @@ -0,0 +1,68 @@ +name: monai +channels: + - pytorch + - defaults + - nvidia + - conda-forge +dependencies: + - numpy>=1.20 + - pytorch>=1.8 + - torchvision + - pytorch-cuda=11.6 + - pip + - python>=3.8 + - pip: + - pytorch-ignite==0.4.10 + - gdown>=4.4.0 + - scipy + - itk>=5.2 + - nibabel + - pillow!=8.3.0 # https://github.com/python-pillow/Pillow/issues/5571 + - tensorboard>=2.6 # https://github.com/Project-MONAI/MONAI/issues/5776 + - scikit-image>=0.19.0 + - tqdm>=4.47.0 + - lmdb + - flake8>=3.8.1 + - flake8-bugbear + - flake8-comprehensions + - flake8-executable + - pylint!=2.13 # https://github.com/PyCQA/pylint/issues/5969 + - mccabe + - pep8-naming + - pycodestyle + - pyflakes + - black + - isort + - pytype>=2020.6.1; platform_system != "Windows" + - types-pkg_resources + - mypy>=0.790 + - ninja + - torchvision + - psutil + - cucim==22.8.1; platform_system == "Linux" + - openslide-python==1.1.2 + - imagecodecs; platform_system == "Linux" or platform_system == "Darwin" + - tifffile; platform_system == "Linux" or platform_system == "Darwin" + - pandas + - requests + - einops + - transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 + - mlflow>=1.28.0 + - matplotlib!=3.5.0 + - tensorboardX + - types-PyYAML + - pyyaml + - fire + - jsonschema + - pynrrd + - pre-commit + - pydicom + - h5py + - nni + - optuna + - git+https://github.com/Project-MONAI/MetricsReloaded@monai-support#egg=MetricsReloaded + - setuptools>=50.3.0,<66.0.0,!=60.6.0 + - coverage>=5.5 + - parameterized + - torch>=1.8 + - numpy>=1.20 diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index a6543418d0..df6ac26b5a 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -31,7 +31,7 @@ ) from monai.apps.auto3dseg.hpo_gen import NNIGen from monai.apps.auto3dseg.utils import ( - export_bundle_algo_history, import_bundle_algo_history, submit_to_azureml_if_needed, + export_bundle_algo_history, import_bundle_algo_history, submit_auto3dseg_module_to_azureml_if_needed, ) from monai.apps.utils import get_logger from monai.auto3dseg.utils import algo_to_pickle @@ -225,7 +225,8 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): - run_info = submit_to_azureml_if_needed() + run_info = submit_auto3dseg_module_to_azureml_if_needed(module_name="AutoRunner", module_function="run") + logger.info(f"AutoRunner using work directory {work_dir}") os.makedirs(work_dir, exist_ok=True) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index eaac5f913a..5a821be05b 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -12,13 +12,15 @@ from __future__ import annotations import os +from typing import Optional + +# health_azure, has_health_azure = optional_import("health_azure") +import health_azure from monai.apps.auto3dseg.bundle_gen import BundleAlgo from monai.auto3dseg import algo_from_pickle, algo_to_pickle from monai.utils import optional_import -# health_azure, has_health_azure = optional_import("health_azure") -import health_azure def import_bundle_algo_history( output_folder: str = ".", template_path: str | None = None, only_trained: bool = True @@ -71,17 +73,21 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -def submit_to_azureml_if_needed() -> health_azure.AzureRunInfo: +def submit_auto3dseg_module_to_azureml_if_needed( + module_name: str, module_function: str, args: Optional[dict] = None +) -> health_azure.AzureRunInfo: + run_info = health_azure.submit_to_azure_if_needed( compute_cluster_name="lite-testing-ds2", workspace_config_file="azureml_config.json", - input_datasets=["dataset_goes_here"], - default_datastore="innereyedatasets", + input_datasets=["hello_world"], + default_datastore="himldatasets", docker_base_image="mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", snapshot_root_directory=os.getcwd(), - conda_environment_file=os.path.join(os.getcwd(), "environment-dev.yml"), - entry_script=os.getcwd() + "/" + __file__, - strictly_aml_v1=True, + conda_environment_file=os.path.join("environment-azureml.yml"), + entry_script=os.path.join(os.getcwd(), __file__), + # entry_script=f"python -m monai.apps.auto3dseg {module_name} {module_function}", + strictly_aml_v1=False, ) print(f"Run info generated: {str(run_info)}") return run_info From 9d0b98f830dd4ba50a6d8c960c7f63233299fc24 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 21 Feb 2023 10:24:12 +0000 Subject: [PATCH 04/58] =?UTF-8?q?=E2=9C=A8=20Enable=20job=20submission=20t?= =?UTF-8?q?o=20run=20correct=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 5a821be05b..128cd2c223 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -85,8 +85,7 @@ def submit_auto3dseg_module_to_azureml_if_needed( docker_base_image="mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", snapshot_root_directory=os.getcwd(), conda_environment_file=os.path.join("environment-azureml.yml"), - entry_script=os.path.join(os.getcwd(), __file__), - # entry_script=f"python -m monai.apps.auto3dseg {module_name} {module_function}", + entry_script="-m monai.apps.auto3dseg", strictly_aml_v1=False, ) print(f"Run info generated: {str(run_info)}") From dc8101b1b71e3dcfe978ea1e45ef943a535e8b3c Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 21 Feb 2023 10:26:52 +0000 Subject: [PATCH 05/58] =?UTF-8?q?=F0=9F=93=8C=20Add=20hi-ml=20to=20AzureML?= =?UTF-8?q?=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environment-azureml.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-azureml.yml b/environment-azureml.yml index 537a28f22f..d1e01ab06b 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -12,6 +12,7 @@ dependencies: - pip - python>=3.8 - pip: + - hi-ml-azure>=0.2.18 - pytorch-ignite==0.4.10 - gdown>=4.4.0 - scipy @@ -60,7 +61,6 @@ dependencies: - h5py - nni - optuna - - git+https://github.com/Project-MONAI/MetricsReloaded@monai-support#egg=MetricsReloaded - setuptools>=50.3.0,<66.0.0,!=60.6.0 - coverage>=5.5 - parameterized From 2b88637509c1a25f4dc6b9e7d6dc3848e05c7d79 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 21 Feb 2023 16:38:45 +0000 Subject: [PATCH 06/58] =?UTF-8?q?=F0=9F=93=8C=20AzureML=20environment=20wo?= =?UTF-8?q?rking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environment-azureml.yml | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/environment-azureml.yml b/environment-azureml.yml index d1e01ab06b..34c0b3a4b8 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -1,4 +1,4 @@ -name: monai +name: monai-azureml channels: - pytorch - defaults @@ -23,46 +23,21 @@ dependencies: - scikit-image>=0.19.0 - tqdm>=4.47.0 - lmdb - - flake8>=3.8.1 - - flake8-bugbear - - flake8-comprehensions - - flake8-executable - - pylint!=2.13 # https://github.com/PyCQA/pylint/issues/5969 - - mccabe - - pep8-naming - - pycodestyle - - pyflakes - - black - - isort - - pytype>=2020.6.1; platform_system != "Windows" - - types-pkg_resources - - mypy>=0.790 - ninja - torchvision - psutil - cucim==22.8.1; platform_system == "Linux" - openslide-python==1.1.2 - - imagecodecs; platform_system == "Linux" or platform_system == "Darwin" - - tifffile; platform_system == "Linux" or platform_system == "Darwin" - pandas - requests - einops - transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 - mlflow>=1.28.0 - - matplotlib!=3.5.0 - - tensorboardX - - types-PyYAML - - pyyaml - fire - - jsonschema - - pynrrd - - pre-commit - pydicom - h5py - nni - optuna - - setuptools>=50.3.0,<66.0.0,!=60.6.0 - - coverage>=5.5 - parameterized - torch>=1.8 - numpy>=1.20 From 136ec66b6508391f77a4616dab9d613443ea72ba Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 22 Feb 2023 14:16:36 +0000 Subject: [PATCH 07/58] =?UTF-8?q?=F0=9F=9A=A7=20Enable=20configuring=20tra?= =?UTF-8?q?ining=20using=20task.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .amlignore | 2 ++ azureml_configs/lung_datalist.json | 43 +++++++++++++++++++++++ azureml_configs/task.yaml | 7 ++++ environment-azureml.yml | 1 + monai/apps/auto3dseg/auto_runner.py | 17 +++++---- monai/apps/auto3dseg/utils.py | 53 ++++++++++++++++++++--------- work_dir/input.yaml | 7 ++++ 7 files changed, 107 insertions(+), 23 deletions(-) create mode 100644 .amlignore create mode 100644 azureml_configs/lung_datalist.json create mode 100644 azureml_configs/task.yaml create mode 100644 work_dir/input.yaml diff --git a/.amlignore b/.amlignore new file mode 100644 index 0000000000..58feb18d7a --- /dev/null +++ b/.amlignore @@ -0,0 +1,2 @@ +tests/ +docs/ diff --git a/azureml_configs/lung_datalist.json b/azureml_configs/lung_datalist.json new file mode 100644 index 0000000000..15927a9a21 --- /dev/null +++ b/azureml_configs/lung_datalist.json @@ -0,0 +1,43 @@ +{ + "testing": [ + + { + "image": "0/ct.nii.gz", + "label": "0/label.nii.gz" + }, + { + "image": "1/ct.nii.gz", + "label": "1/label.nii.gz" + }, + { + "image": "2/ct.nii.gz", + "label": "2/label.nii.gz" + }, + { + "image": "3/ct.nii.gz", + "label": "3/label.nii.gz" + } + ], + "training": [ + { + "fold": 0, + "image": "10/ct.nii.gz", + "label": "10/label.nii.gz" + }, + { + "fold": 0, + "image": "11/ct.nii.gz", + "label": "11/label.nii.gz" + }, + { + "fold": 0, + "image": "12/ct.nii.gz", + "label": "12/label.nii.gz" + }, + { + "fold": 0, + "image": "13/ct.nii.gz", + "label": "13/label.nii.gz" + } + ] +} diff --git a/azureml_configs/task.yaml b/azureml_configs/task.yaml new file mode 100644 index 0000000000..d01b8709fb --- /dev/null +++ b/azureml_configs/task.yaml @@ -0,0 +1,7 @@ +modality: CT +datalist: "./azureml_configs/lung_datalist.json" +dataroot: "/workspace/data/task" +azureml_compute_cluster_name: "dedicated-nc24s-v2" +azureml_input_datasets: +- "lung_tcia_single_label_12_22_v3" +azureml_default_datastore: "himldatasets" diff --git a/environment-azureml.yml b/environment-azureml.yml index 34c0b3a4b8..c24311eaef 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -13,6 +13,7 @@ dependencies: - python>=3.8 - pip: - hi-ml-azure>=0.2.18 + - monai>=1.1.0 - pytorch-ignite==0.4.10 - gdown>=4.4.0 - scipy diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index df6ac26b5a..ceb0495d6c 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -20,14 +20,12 @@ import numpy as np import torch +from health_azure.utils import is_running_in_azure_ml from monai.apps.auto3dseg.bundle_gen import BundleGen from monai.apps.auto3dseg.data_analyzer import DataAnalyzer from monai.apps.auto3dseg.ensemble_builder import ( - AlgoEnsemble, - AlgoEnsembleBestByFold, - AlgoEnsembleBestN, - AlgoEnsembleBuilder, + AlgoEnsemble, AlgoEnsembleBestByFold, AlgoEnsembleBestN, AlgoEnsembleBuilder, ) from monai.apps.auto3dseg.hpo_gen import NNIGen from monai.apps.auto3dseg.utils import ( @@ -225,7 +223,9 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): - run_info = submit_auto3dseg_module_to_azureml_if_needed(module_name="AutoRunner", module_function="run") + + if is_running_in_azure_ml(): + work_dir = os.path.join("outputs", work_dir) logger.info(f"AutoRunner using work directory {work_dir}") os.makedirs(work_dir, exist_ok=True) @@ -252,11 +252,16 @@ def __init__( else: raise ValueError(f"{input} is not a valid file or dict") + run_info = submit_auto3dseg_module_to_azureml_if_needed(cfg=self.data_src_cfg) + missing_keys = {"dataroot", "datalist", "modality"}.difference(self.data_src_cfg.keys()) if len(missing_keys) > 0: raise ValueError(f"Config keys are missing {missing_keys}") - self.dataroot = self.data_src_cfg["dataroot"] + if run_info.input_datasets: + self.dataroot = run_info.input_datasets[0] + else: + self.dataroot = self.data_src_cfg["dataroot"] self.datalist_filename = self.data_src_cfg["datalist"] self.datastats_filename = os.path.join(self.work_dir, "datastats.yaml") diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 128cd2c223..9136a4629c 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -12,7 +12,7 @@ from __future__ import annotations import os -from typing import Optional +from typing import Any # health_azure, has_health_azure = optional_import("health_azure") import health_azure @@ -73,20 +73,39 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -def submit_auto3dseg_module_to_azureml_if_needed( - module_name: str, module_function: str, args: Optional[dict] = None -) -> health_azure.AzureRunInfo: - - run_info = health_azure.submit_to_azure_if_needed( - compute_cluster_name="lite-testing-ds2", - workspace_config_file="azureml_config.json", - input_datasets=["hello_world"], - default_datastore="himldatasets", - docker_base_image="mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", - snapshot_root_directory=os.getcwd(), - conda_environment_file=os.path.join("environment-azureml.yml"), - entry_script="-m monai.apps.auto3dseg", - strictly_aml_v1=False, - ) - print(f"Run info generated: {str(run_info)}") +def is_running_in_azureml() -> bool: + return health_azure.utils.is_running_in_azure_ml() + + +def extract_azureml_args_from_cfg(cfg: dict[str, Any]) -> dict[str, Any]: + azureml_key_prefix = "azureml_" + + azureml_args = {} + + for key, value in cfg.items(): + if key.startswith(azureml_key_prefix): + azureml_args[key.removeprefix(azureml_key_prefix)] = value + + return azureml_args + + +def submit_auto3dseg_module_to_azureml_if_needed(cfg: dict[str, Any]) -> health_azure.AzureRunInfo: + user_defined_azureml_args = extract_azureml_args_from_cfg(cfg) + azureml_args = { + "workspace_config_file": "azureml_configs/azureml_config.json", + "docker_base_image": "mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", + "snapshot_root_directory": os.getcwd(), + "conda_environment_file": "environment-azureml.yml", + "entry_script": "-m monai.apps.auto3dseg", + "strictly_aml_v1": False, + } + azureml_args.update(user_defined_azureml_args) + + needed_keys = {"compute_cluster_name", "default_datastore"} + missing_keys = needed_keys.difference(azureml_args.keys()) + if len(missing_keys) > 0: + raise ValueError(f"Missing keys in azureml_args: {missing_keys}") + + run_info = health_azure.submit_to_azure_if_needed(**azureml_args) + return run_info diff --git a/work_dir/input.yaml b/work_dir/input.yaml new file mode 100644 index 0000000000..d01b8709fb --- /dev/null +++ b/work_dir/input.yaml @@ -0,0 +1,7 @@ +modality: CT +datalist: "./azureml_configs/lung_datalist.json" +dataroot: "/workspace/data/task" +azureml_compute_cluster_name: "dedicated-nc24s-v2" +azureml_input_datasets: +- "lung_tcia_single_label_12_22_v3" +azureml_default_datastore: "himldatasets" From 49dbb0483228c0d25f5dcf6285dd098fe75080b9 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 23 Feb 2023 11:03:39 +0000 Subject: [PATCH 08/58] =?UTF-8?q?=F0=9F=9A=A7=20Pass=20in=20azureml=20args?= =?UTF-8?q?=20as=20dict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .amlignore | 2 +- azureml_configs/task.yaml | 9 +++++---- monai/apps/auto3dseg/auto_runner.py | 11 +++++++---- monai/apps/auto3dseg/utils.py | 23 ++++------------------- work_dir/input.yaml | 9 +++++---- 5 files changed, 22 insertions(+), 32 deletions(-) diff --git a/.amlignore b/.amlignore index 58feb18d7a..59a45cc78b 100644 --- a/.amlignore +++ b/.amlignore @@ -1,2 +1,2 @@ tests/ -docs/ +docs/ \ No newline at end of file diff --git a/azureml_configs/task.yaml b/azureml_configs/task.yaml index d01b8709fb..b8c67a33ab 100644 --- a/azureml_configs/task.yaml +++ b/azureml_configs/task.yaml @@ -1,7 +1,8 @@ modality: CT datalist: "./azureml_configs/lung_datalist.json" dataroot: "/workspace/data/task" -azureml_compute_cluster_name: "dedicated-nc24s-v2" -azureml_input_datasets: -- "lung_tcia_single_label_12_22_v3" -azureml_default_datastore: "himldatasets" +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "lung_tcia_single_label_12_22_v3" + default_datastore: "himldatasets" diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index ceb0495d6c..970c8d3cfa 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -14,12 +14,14 @@ import os import shutil import subprocess +import sys from copy import deepcopy from time import sleep from typing import Any, cast import numpy as np import torch +from health_azure.himl import AZUREML_FLAG from health_azure.utils import is_running_in_azure_ml from monai.apps.auto3dseg.bundle_gen import BundleGen @@ -29,7 +31,8 @@ ) from monai.apps.auto3dseg.hpo_gen import NNIGen from monai.apps.auto3dseg.utils import ( - export_bundle_algo_history, import_bundle_algo_history, submit_auto3dseg_module_to_azureml_if_needed, + AZUREML_CONFIG_KEY, export_bundle_algo_history, + import_bundle_algo_history, submit_auto3dseg_module_to_azureml_if_needed, ) from monai.apps.utils import get_logger from monai.auto3dseg.utils import algo_to_pickle @@ -252,16 +255,16 @@ def __init__( else: raise ValueError(f"{input} is not a valid file or dict") - run_info = submit_auto3dseg_module_to_azureml_if_needed(cfg=self.data_src_cfg) - missing_keys = {"dataroot", "datalist", "modality"}.difference(self.data_src_cfg.keys()) if len(missing_keys) > 0: raise ValueError(f"Config keys are missing {missing_keys}") - if run_info.input_datasets: + if AZUREML_FLAG in sys.argv or is_running_in_azure_ml(): + run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) self.dataroot = run_info.input_datasets[0] else: self.dataroot = self.data_src_cfg["dataroot"] + self.datalist_filename = self.data_src_cfg["datalist"] self.datastats_filename = os.path.join(self.work_dir, "datastats.yaml") diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 9136a4629c..dce5866b32 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -21,6 +21,8 @@ from monai.auto3dseg import algo_from_pickle, algo_to_pickle from monai.utils import optional_import +AZUREML_CONFIG_KEY = "azureml_config" + def import_bundle_algo_history( output_folder: str = ".", template_path: str | None = None, only_trained: bool = True @@ -73,24 +75,7 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -def is_running_in_azureml() -> bool: - return health_azure.utils.is_running_in_azure_ml() - - -def extract_azureml_args_from_cfg(cfg: dict[str, Any]) -> dict[str, Any]: - azureml_key_prefix = "azureml_" - - azureml_args = {} - - for key, value in cfg.items(): - if key.startswith(azureml_key_prefix): - azureml_args[key.removeprefix(azureml_key_prefix)] = value - - return azureml_args - - -def submit_auto3dseg_module_to_azureml_if_needed(cfg: dict[str, Any]) -> health_azure.AzureRunInfo: - user_defined_azureml_args = extract_azureml_args_from_cfg(cfg) +def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> health_azure.AzureRunInfo: azureml_args = { "workspace_config_file": "azureml_configs/azureml_config.json", "docker_base_image": "mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", @@ -99,7 +84,7 @@ def submit_auto3dseg_module_to_azureml_if_needed(cfg: dict[str, Any]) -> health_ "entry_script": "-m monai.apps.auto3dseg", "strictly_aml_v1": False, } - azureml_args.update(user_defined_azureml_args) + azureml_args.update(azure_cfg) needed_keys = {"compute_cluster_name", "default_datastore"} missing_keys = needed_keys.difference(azureml_args.keys()) diff --git a/work_dir/input.yaml b/work_dir/input.yaml index d01b8709fb..b8c67a33ab 100644 --- a/work_dir/input.yaml +++ b/work_dir/input.yaml @@ -1,7 +1,8 @@ modality: CT datalist: "./azureml_configs/lung_datalist.json" dataroot: "/workspace/data/task" -azureml_compute_cluster_name: "dedicated-nc24s-v2" -azureml_input_datasets: -- "lung_tcia_single_label_12_22_v3" -azureml_default_datastore: "himldatasets" +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "lung_tcia_single_label_12_22_v3" + default_datastore: "himldatasets" From c4b8d4ecce1e503c409668871799a9b3d83b7c92 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 23 Feb 2023 16:59:41 +0000 Subject: [PATCH 09/58] =?UTF-8?q?=F0=9F=9A=A7=20Fix=20AzureML=20training?= =?UTF-8?q?=20arg=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- azureml_configs/lung_datalist.json | 20 ++++++++++++++++++++ azureml_configs/task.yaml | 4 ++++ monai/apps/auto3dseg/auto_runner.py | 14 +++++++++++--- work_dir/cache.yaml | 1 + work_dir/input.yaml | 4 ++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 work_dir/cache.yaml diff --git a/azureml_configs/lung_datalist.json b/azureml_configs/lung_datalist.json index 15927a9a21..e81923c2a3 100644 --- a/azureml_configs/lung_datalist.json +++ b/azureml_configs/lung_datalist.json @@ -38,6 +38,26 @@ "fold": 0, "image": "13/ct.nii.gz", "label": "13/label.nii.gz" + }, + { + "fold": 1, + "image": "14/ct.nii.gz", + "label": "14/label.nii.gz" + }, + { + "fold": 1, + "image": "15/ct.nii.gz", + "label": "15/label.nii.gz" + }, + { + "fold": 1, + "image": "16/ct.nii.gz", + "label": "16/label.nii.gz" + }, + { + "fold": 1, + "image": "17/ct.nii.gz", + "label": "17/label.nii.gz" } ] } diff --git a/azureml_configs/task.yaml b/azureml_configs/task.yaml index b8c67a33ab..62f22f7a3b 100644 --- a/azureml_configs/task.yaml +++ b/azureml_configs/task.yaml @@ -6,3 +6,7 @@ azureml_config: input_datasets: - "lung_tcia_single_label_12_22_v3" default_datastore: "himldatasets" + # compute_cluster_name: "lite-test-ds2" + # input_datasets: + # - "test_dataset" + # default_datastore: "test_datastore" diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 970c8d3cfa..5c12f18df0 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -219,6 +219,8 @@ def __init__( analyze: bool | None = None, algo_gen: bool | None = None, train: bool | None = None, + training_params: dict[str, Any] | None = None, + num_fold: int | None = 5, hpo: bool = False, hpo_backend: str = "nni", ensemble: bool = True, @@ -259,6 +261,7 @@ def __init__( if len(missing_keys) > 0: raise ValueError(f"Config keys are missing {missing_keys}") + logger.info(f"Argv: {sys.argv}") if AZUREML_FLAG in sys.argv or is_running_in_azure_ml(): run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) self.dataroot = run_info.input_datasets[0] @@ -280,9 +283,9 @@ def __init__( self.ensemble = ensemble # last step, no need to check # intermediate variables - self.num_fold = 5 + self.num_fold = num_fold self.ensemble_method_name = "AlgoEnsembleBestByFold" - self.set_training_params() + self.set_training_params(training_params) self.set_prediction_params() self.set_analyze_params() @@ -385,7 +388,7 @@ def set_gpu_customization( if gpu_customization_specs is not None: self.gpu_customization_specs = gpu_customization_specs - def set_num_fold(self, num_fold: int = 5) -> None: + def set_num_fold(self, num_fold: int) -> None: """ Set the number of cross validation folds for all algos. @@ -416,6 +419,7 @@ def set_training_params(self, params: dict[str, Any] | None = None) -> None: """ self.train_params = deepcopy(params) if params is not None else {} + logger.info(f"Set AutoRunner training params to {self.train_params}") def set_prediction_params(self, params: dict[str, Any] | None = None) -> None: """ @@ -565,6 +569,7 @@ def _train_algo_in_sequence(self, history: list[dict[str, Any]]) -> None: """ for task in history: for _, algo in task.items(): + logger.info(f"Type of train_params when calling algo.train: {type(self.train_params)}") algo.train(self.train_params) acc = algo.get_score() algo_to_pickle(algo, template_path=algo.template_path, best_metrics=acc) @@ -639,6 +644,7 @@ def run(self): """ Run the AutoRunner pipeline """ + logger.info(f"Type of train_params at start of run () is {type(self.train_params)}") # step 1: data analysis if self.analyze and self.analyze_params is not None: logger.info("Running data analysis...") @@ -683,6 +689,8 @@ def run(self): # step 3: algo training if self.train: + logger.info(f"Type of train_params at start of step 3 is {type(self.train_params)}") + history = import_bundle_algo_history(self.work_dir, only_trained=False) if len(history) == 0: diff --git a/work_dir/cache.yaml b/work_dir/cache.yaml new file mode 100644 index 0000000000..be326f097b --- /dev/null +++ b/work_dir/cache.yaml @@ -0,0 +1 @@ +{analyze: false, datastats: null, algo_gen: false, train: false} diff --git a/work_dir/input.yaml b/work_dir/input.yaml index b8c67a33ab..62f22f7a3b 100644 --- a/work_dir/input.yaml +++ b/work_dir/input.yaml @@ -6,3 +6,7 @@ azureml_config: input_datasets: - "lung_tcia_single_label_12_22_v3" default_datastore: "himldatasets" + # compute_cluster_name: "lite-test-ds2" + # input_datasets: + # - "test_dataset" + # default_datastore: "test_datastore" From ce16775fd14603dc37e9a7fadcba28360dcb155e Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 24 Feb 2023 14:44:59 +0000 Subject: [PATCH 10/58] =?UTF-8?q?=F0=9F=90=9B=20=F0=9F=9A=A7=20Fix=20train?= =?UTF-8?q?ing=20to=20work=20with=20AzureML=20dataroot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/auto_runner.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 5c12f18df0..4ea5abd180 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -10,7 +10,6 @@ # limitations under the License. from __future__ import annotations - import os import shutil import subprocess @@ -46,6 +45,9 @@ nni, has_nni = optional_import("nni") +RUNNING_IN_AZURE_ML = is_running_in_azure_ml() + + class AutoRunner: """ An interface for handling Auto3Dseg with minimal inputs and understanding of the internal states in Auto3Dseg. @@ -229,7 +231,7 @@ def __init__( **kwargs: Any, ): - if is_running_in_azure_ml(): + if RUNNING_IN_AZURE_ML: work_dir = os.path.join("outputs", work_dir) logger.info(f"AutoRunner using work directory {work_dir}") @@ -246,9 +248,7 @@ def __init__( if isinstance(input, dict): self.data_src_cfg = input - ConfigParser.export_config_file( - config=input, filepath=self.data_src_cfg_name, fmt="yaml", default_flow_style=None, sort_keys=False - ) + self.write_data_src_cfg() elif isinstance(input, str) and os.path.isfile(input): self.data_src_cfg = ConfigParser.load_config_file(input) logger.info(f"Loading input config {input}") @@ -262,9 +262,11 @@ def __init__( raise ValueError(f"Config keys are missing {missing_keys}") logger.info(f"Argv: {sys.argv}") - if AZUREML_FLAG in sys.argv or is_running_in_azure_ml(): + if AZUREML_FLAG in sys.argv or RUNNING_IN_AZURE_ML: run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) self.dataroot = run_info.input_datasets[0] + self.data_src_cfg["dataroot"] = str(self.dataroot) + self.write_data_src_cfg() else: self.dataroot = self.data_src_cfg["dataroot"] @@ -305,6 +307,16 @@ def __init__( self.search_space: dict[str, dict[str, Any]] = {} self.hpo_tasks = 0 + def write_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: + output_data_src_cfg = data_src_cfg if data_src_cfg is not None else self.data_src_cfg + ConfigParser.export_config_file( + config=output_data_src_cfg, + filepath=self.data_src_cfg_name, + fmt="yaml", + default_flow_style=None, + sort_keys=False + ) + def read_cache(self): """ Check if the intermediate result is cached after each step in the current working directory @@ -645,6 +657,7 @@ def run(self): Run the AutoRunner pipeline """ logger.info(f"Type of train_params at start of run () is {type(self.train_params)}") + # step 1: data analysis if self.analyze and self.analyze_params is not None: logger.info("Running data analysis...") @@ -664,6 +677,7 @@ def run(self): "Possibly the required data analysis step was not completed." ) + bundle_generator = BundleGen( algos=self.algos, algo_path=self.work_dir, From f5ea9d60442c91322647800c92b538ef6cf53209 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 27 Feb 2023 09:54:47 +0000 Subject: [PATCH 11/58] =?UTF-8?q?=F0=9F=9A=A7=20Attempt=20to=20run=20on=20?= =?UTF-8?q?Prostate=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- azureml_configs/prostate_datalist.json | 204 ++++++++++++++++++ azureml_configs/{task.yaml => task_lung.yaml} | 0 azureml_configs/task_prostate.yaml | 20 ++ work_dir/cache.yaml | 1 - work_dir/input.yaml | 12 -- 5 files changed, 224 insertions(+), 13 deletions(-) create mode 100644 azureml_configs/prostate_datalist.json rename azureml_configs/{task.yaml => task_lung.yaml} (100%) create mode 100644 azureml_configs/task_prostate.yaml delete mode 100644 work_dir/cache.yaml delete mode 100644 work_dir/input.yaml diff --git a/azureml_configs/prostate_datalist.json b/azureml_configs/prostate_datalist.json new file mode 100644 index 0000000000..fba52eebb6 --- /dev/null +++ b/azureml_configs/prostate_datalist.json @@ -0,0 +1,204 @@ +{ + "testing": [ + { + "image": "imagesTs/prostate_08.nii.gz" + }, + { + "image": "imagesTs/prostate_22.nii.gz" + }, + { + "image": "imagesTs/prostate_30.nii.gz" + }, + { + "image": "imagesTs/prostate_45.nii.gz" + }, + { + "image": "imagesTs/prostate_26.nii.gz" + }, + { + "image": "imagesTs/prostate_36.nii.gz" + }, + { + "image": "imagesTs/prostate_12.nii.gz" + }, + { + "image": "imagesTs/prostate_33.nii.gz" + }, + { + "image": "imagesTs/prostate_09.nii.gz" + }, + { + "image": "imagesTs/prostate_05.nii.gz" + }, + { + "image": "imagesTs/prostate_23.nii.gz" + }, + { + "image": "imagesTs/prostate_19.nii.gz" + }, + { + "image": "imagesTs/prostate_15.nii.gz" + }, + { + "image": "imagesTs/prostate_11.nii.gz" + }, + { + "image": "imagesTs/prostate_03.nii.gz" + }, + { + "image": "imagesTs/prostate_27.nii.gz" + } + ], + "training": [ + { + "fold": 0, + "image": "imagesTr/prostate_16.nii.gz", + "label": "labelsTr/prostate_16.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_04.nii.gz", + "label": "labelsTr/prostate_04.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_20.nii.gz", + "label": "labelsTr/prostate_20.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_43.nii.gz", + "label": "labelsTr/prostate_43.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_06.nii.gz", + "label": "labelsTr/prostate_06.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_14.nii.gz", + "label": "labelsTr/prostate_14.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_41.nii.gz", + "label": "labelsTr/prostate_41.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_34.nii.gz", + "label": "labelsTr/prostate_34.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_38.nii.gz", + "label": "labelsTr/prostate_38.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_10.nii.gz", + "label": "labelsTr/prostate_10.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_02.nii.gz", + "label": "labelsTr/prostate_02.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_24.nii.gz", + "label": "labelsTr/prostate_24.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_47.nii.gz", + "label": "labelsTr/prostate_47.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_28.nii.gz", + "label": "labelsTr/prostate_28.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_00.nii.gz", + "label": "labelsTr/prostate_00.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_42.nii.gz", + "label": "labelsTr/prostate_42.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_21.nii.gz", + "label": "labelsTr/prostate_21.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_17.nii.gz", + "label": "labelsTr/prostate_17.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_40.nii.gz", + "label": "labelsTr/prostate_40.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_31.nii.gz", + "label": "labelsTr/prostate_31.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_07.nii.gz", + "label": "labelsTr/prostate_07.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_35.nii.gz", + "label": "labelsTr/prostate_35.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_44.nii.gz", + "label": "labelsTr/prostate_44.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_39.nii.gz", + "label": "labelsTr/prostate_39.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_01.nii.gz", + "label": "labelsTr/prostate_01.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_13.nii.gz", + "label": "labelsTr/prostate_13.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_46.nii.gz", + "label": "labelsTr/prostate_46.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_25.nii.gz", + "label": "labelsTr/prostate_25.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_29.nii.gz", + "label": "labelsTr/prostate_29.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_37.nii.gz", + "label": "labelsTr/prostate_37.nii.gz" + } + ] +} diff --git a/azureml_configs/task.yaml b/azureml_configs/task_lung.yaml similarity index 100% rename from azureml_configs/task.yaml rename to azureml_configs/task_lung.yaml diff --git a/azureml_configs/task_prostate.yaml b/azureml_configs/task_prostate.yaml new file mode 100644 index 0000000000..a9c437b4a9 --- /dev/null +++ b/azureml_configs/task_prostate.yaml @@ -0,0 +1,20 @@ +name: Task05_Prostate +task: segmentation + +modality: MRI +datalist: ./azureml_configs/prostate_datalist.json # list of files +dataroot: /data/Task05_Prostate # data location + +multigpu: True + +class_names: [val_acc_pz, val_acc_tz] + +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "medical_decathlon_prostate_task_05" + default_datastore: "himldatasets" + # compute_cluster_name: "lite-test-ds2" + # input_datasets: + # - "test_dataset" + # default_datastore: "test_datastore" diff --git a/work_dir/cache.yaml b/work_dir/cache.yaml deleted file mode 100644 index be326f097b..0000000000 --- a/work_dir/cache.yaml +++ /dev/null @@ -1 +0,0 @@ -{analyze: false, datastats: null, algo_gen: false, train: false} diff --git a/work_dir/input.yaml b/work_dir/input.yaml deleted file mode 100644 index 62f22f7a3b..0000000000 --- a/work_dir/input.yaml +++ /dev/null @@ -1,12 +0,0 @@ -modality: CT -datalist: "./azureml_configs/lung_datalist.json" -dataroot: "/workspace/data/task" -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "lung_tcia_single_label_12_22_v3" - default_datastore: "himldatasets" - # compute_cluster_name: "lite-test-ds2" - # input_datasets: - # - "test_dataset" - # default_datastore: "test_datastore" From 218cb8b62c0d80acad975d5eced74e3bf9945e42 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Feb 2023 10:33:24 +0000 Subject: [PATCH 12/58] =?UTF-8?q?=F0=9F=94=87=20Remove=20old=20logs=20+=20?= =?UTF-8?q?configs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- azureml_configs/lung_datalist.json | 63 -------- azureml_configs/prostate_datalist.json | 204 ------------------------- azureml_configs/task_lung.yaml | 12 -- azureml_configs/task_prostate.yaml | 20 --- monai/apps/auto3dseg/auto_runner.py | 3 - 5 files changed, 302 deletions(-) delete mode 100644 azureml_configs/lung_datalist.json delete mode 100644 azureml_configs/prostate_datalist.json delete mode 100644 azureml_configs/task_lung.yaml delete mode 100644 azureml_configs/task_prostate.yaml diff --git a/azureml_configs/lung_datalist.json b/azureml_configs/lung_datalist.json deleted file mode 100644 index e81923c2a3..0000000000 --- a/azureml_configs/lung_datalist.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "testing": [ - - { - "image": "0/ct.nii.gz", - "label": "0/label.nii.gz" - }, - { - "image": "1/ct.nii.gz", - "label": "1/label.nii.gz" - }, - { - "image": "2/ct.nii.gz", - "label": "2/label.nii.gz" - }, - { - "image": "3/ct.nii.gz", - "label": "3/label.nii.gz" - } - ], - "training": [ - { - "fold": 0, - "image": "10/ct.nii.gz", - "label": "10/label.nii.gz" - }, - { - "fold": 0, - "image": "11/ct.nii.gz", - "label": "11/label.nii.gz" - }, - { - "fold": 0, - "image": "12/ct.nii.gz", - "label": "12/label.nii.gz" - }, - { - "fold": 0, - "image": "13/ct.nii.gz", - "label": "13/label.nii.gz" - }, - { - "fold": 1, - "image": "14/ct.nii.gz", - "label": "14/label.nii.gz" - }, - { - "fold": 1, - "image": "15/ct.nii.gz", - "label": "15/label.nii.gz" - }, - { - "fold": 1, - "image": "16/ct.nii.gz", - "label": "16/label.nii.gz" - }, - { - "fold": 1, - "image": "17/ct.nii.gz", - "label": "17/label.nii.gz" - } - ] -} diff --git a/azureml_configs/prostate_datalist.json b/azureml_configs/prostate_datalist.json deleted file mode 100644 index fba52eebb6..0000000000 --- a/azureml_configs/prostate_datalist.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "testing": [ - { - "image": "imagesTs/prostate_08.nii.gz" - }, - { - "image": "imagesTs/prostate_22.nii.gz" - }, - { - "image": "imagesTs/prostate_30.nii.gz" - }, - { - "image": "imagesTs/prostate_45.nii.gz" - }, - { - "image": "imagesTs/prostate_26.nii.gz" - }, - { - "image": "imagesTs/prostate_36.nii.gz" - }, - { - "image": "imagesTs/prostate_12.nii.gz" - }, - { - "image": "imagesTs/prostate_33.nii.gz" - }, - { - "image": "imagesTs/prostate_09.nii.gz" - }, - { - "image": "imagesTs/prostate_05.nii.gz" - }, - { - "image": "imagesTs/prostate_23.nii.gz" - }, - { - "image": "imagesTs/prostate_19.nii.gz" - }, - { - "image": "imagesTs/prostate_15.nii.gz" - }, - { - "image": "imagesTs/prostate_11.nii.gz" - }, - { - "image": "imagesTs/prostate_03.nii.gz" - }, - { - "image": "imagesTs/prostate_27.nii.gz" - } - ], - "training": [ - { - "fold": 0, - "image": "imagesTr/prostate_16.nii.gz", - "label": "labelsTr/prostate_16.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_04.nii.gz", - "label": "labelsTr/prostate_04.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_20.nii.gz", - "label": "labelsTr/prostate_20.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_43.nii.gz", - "label": "labelsTr/prostate_43.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_06.nii.gz", - "label": "labelsTr/prostate_06.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_14.nii.gz", - "label": "labelsTr/prostate_14.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_41.nii.gz", - "label": "labelsTr/prostate_41.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_34.nii.gz", - "label": "labelsTr/prostate_34.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_38.nii.gz", - "label": "labelsTr/prostate_38.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_10.nii.gz", - "label": "labelsTr/prostate_10.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_02.nii.gz", - "label": "labelsTr/prostate_02.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_24.nii.gz", - "label": "labelsTr/prostate_24.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_47.nii.gz", - "label": "labelsTr/prostate_47.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_28.nii.gz", - "label": "labelsTr/prostate_28.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_00.nii.gz", - "label": "labelsTr/prostate_00.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_42.nii.gz", - "label": "labelsTr/prostate_42.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_21.nii.gz", - "label": "labelsTr/prostate_21.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_17.nii.gz", - "label": "labelsTr/prostate_17.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_40.nii.gz", - "label": "labelsTr/prostate_40.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_31.nii.gz", - "label": "labelsTr/prostate_31.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_07.nii.gz", - "label": "labelsTr/prostate_07.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_35.nii.gz", - "label": "labelsTr/prostate_35.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_44.nii.gz", - "label": "labelsTr/prostate_44.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_39.nii.gz", - "label": "labelsTr/prostate_39.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_01.nii.gz", - "label": "labelsTr/prostate_01.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_13.nii.gz", - "label": "labelsTr/prostate_13.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_46.nii.gz", - "label": "labelsTr/prostate_46.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_25.nii.gz", - "label": "labelsTr/prostate_25.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_29.nii.gz", - "label": "labelsTr/prostate_29.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_37.nii.gz", - "label": "labelsTr/prostate_37.nii.gz" - } - ] -} diff --git a/azureml_configs/task_lung.yaml b/azureml_configs/task_lung.yaml deleted file mode 100644 index 62f22f7a3b..0000000000 --- a/azureml_configs/task_lung.yaml +++ /dev/null @@ -1,12 +0,0 @@ -modality: CT -datalist: "./azureml_configs/lung_datalist.json" -dataroot: "/workspace/data/task" -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "lung_tcia_single_label_12_22_v3" - default_datastore: "himldatasets" - # compute_cluster_name: "lite-test-ds2" - # input_datasets: - # - "test_dataset" - # default_datastore: "test_datastore" diff --git a/azureml_configs/task_prostate.yaml b/azureml_configs/task_prostate.yaml deleted file mode 100644 index a9c437b4a9..0000000000 --- a/azureml_configs/task_prostate.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Task05_Prostate -task: segmentation - -modality: MRI -datalist: ./azureml_configs/prostate_datalist.json # list of files -dataroot: /data/Task05_Prostate # data location - -multigpu: True - -class_names: [val_acc_pz, val_acc_tz] - -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "medical_decathlon_prostate_task_05" - default_datastore: "himldatasets" - # compute_cluster_name: "lite-test-ds2" - # input_datasets: - # - "test_dataset" - # default_datastore: "test_datastore" diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 4ea5abd180..d21e001d6d 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -581,7 +581,6 @@ def _train_algo_in_sequence(self, history: list[dict[str, Any]]) -> None: """ for task in history: for _, algo in task.items(): - logger.info(f"Type of train_params when calling algo.train: {type(self.train_params)}") algo.train(self.train_params) acc = algo.get_score() algo_to_pickle(algo, template_path=algo.template_path, best_metrics=acc) @@ -656,7 +655,6 @@ def run(self): """ Run the AutoRunner pipeline """ - logger.info(f"Type of train_params at start of run () is {type(self.train_params)}") # step 1: data analysis if self.analyze and self.analyze_params is not None: @@ -703,7 +701,6 @@ def run(self): # step 3: algo training if self.train: - logger.info(f"Type of train_params at start of step 3 is {type(self.train_params)}") history = import_bundle_algo_history(self.work_dir, only_trained=False) From f54793004cd7b78d08a82f1fcd3f227661d2577a Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Feb 2023 10:34:05 +0000 Subject: [PATCH 13/58] =?UTF-8?q?=F0=9F=99=88=20Update=20gitignore=20for?= =?UTF-8?q?=20azureml=5Fconfigs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2945c7160b..f1d88b50e1 100644 --- a/.gitignore +++ b/.gitignore @@ -148,4 +148,4 @@ tests/testing_data/eval_data_stats.yaml # AzureML Configuration -azureml_config.json +azureml_configs/ From 9a1e7c7daf506e528b6cf7b032121b783dd226a0 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Feb 2023 10:55:43 +0000 Subject: [PATCH 14/58] =?UTF-8?q?=F0=9F=90=9B=20Fix=20creating=20local=20w?= =?UTF-8?q?ork=5Fdir=20when=20running=20in=20Azure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/auto_runner.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index d21e001d6d..f8028856c2 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -232,10 +232,9 @@ def __init__( ): if RUNNING_IN_AZURE_ML: - work_dir = os.path.join("outputs", work_dir) + work_dir = os.path("outputs") logger.info(f"AutoRunner using work directory {work_dir}") - os.makedirs(work_dir, exist_ok=True) self.work_dir = os.path.abspath(work_dir) self.data_src_cfg_name = os.path.join(self.work_dir, "input.yaml") @@ -248,28 +247,26 @@ def __init__( if isinstance(input, dict): self.data_src_cfg = input - self.write_data_src_cfg() elif isinstance(input, str) and os.path.isfile(input): - self.data_src_cfg = ConfigParser.load_config_file(input) logger.info(f"Loading input config {input}") - if input != self.data_src_cfg_name: - shutil.copy(input, self.data_src_cfg_name) + self.data_src_cfg = ConfigParser.load_config_file(input) else: - raise ValueError(f"{input} is not a valid file or dict") + raise ValueError(f"Input: {input} is not a valid file or dict") missing_keys = {"dataroot", "datalist", "modality"}.difference(self.data_src_cfg.keys()) if len(missing_keys) > 0: raise ValueError(f"Config keys are missing {missing_keys}") - logger.info(f"Argv: {sys.argv}") if AZUREML_FLAG in sys.argv or RUNNING_IN_AZURE_ML: run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) self.dataroot = run_info.input_datasets[0] self.data_src_cfg["dataroot"] = str(self.dataroot) - self.write_data_src_cfg() else: self.dataroot = self.data_src_cfg["dataroot"] + os.makedirs(work_dir, exist_ok=True) + self.write_data_src_cfg() + self.datalist_filename = self.data_src_cfg["datalist"] self.datastats_filename = os.path.join(self.work_dir, "datastats.yaml") From 4c024bccc73148fa4b62e56bb3931c0daad772d4 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Feb 2023 13:31:11 +0000 Subject: [PATCH 15/58] =?UTF-8?q?=F0=9F=99=88=20=F0=9F=90=9B=20Update=20gi?= =?UTF-8?q?tignore=20and=20fix=20work=20dir=20creation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ---- monai/apps/auto3dseg/auto_runner.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index f1d88b50e1..3da001d0ce 100644 --- a/.gitignore +++ b/.gitignore @@ -145,7 +145,3 @@ tests/testing_data/eval_data_stats.yaml # profiling results *.prof - -# AzureML Configuration - -azureml_configs/ diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index f8028856c2..3150c8e3f9 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -232,7 +232,7 @@ def __init__( ): if RUNNING_IN_AZURE_ML: - work_dir = os.path("outputs") + work_dir = "outputs" logger.info(f"AutoRunner using work directory {work_dir}") From e6e34563909524037f3d1671796c248e8861bbe4 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 1 Mar 2023 16:54:08 +0000 Subject: [PATCH 16/58] =?UTF-8?q?=F0=9F=9A=A8=20Pass=20linting=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/auto_runner.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 3150c8e3f9..9a4c93747a 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -10,6 +10,7 @@ # limitations under the License. from __future__ import annotations + import os import shutil import subprocess @@ -26,12 +27,17 @@ from monai.apps.auto3dseg.bundle_gen import BundleGen from monai.apps.auto3dseg.data_analyzer import DataAnalyzer from monai.apps.auto3dseg.ensemble_builder import ( - AlgoEnsemble, AlgoEnsembleBestByFold, AlgoEnsembleBestN, AlgoEnsembleBuilder, + AlgoEnsemble, + AlgoEnsembleBestByFold, + AlgoEnsembleBestN, + AlgoEnsembleBuilder, ) from monai.apps.auto3dseg.hpo_gen import NNIGen from monai.apps.auto3dseg.utils import ( - AZUREML_CONFIG_KEY, export_bundle_algo_history, - import_bundle_algo_history, submit_auto3dseg_module_to_azureml_if_needed, + AZUREML_CONFIG_KEY, + export_bundle_algo_history, + import_bundle_algo_history, + submit_auto3dseg_module_to_azureml_if_needed, ) from monai.apps.utils import get_logger from monai.auto3dseg.utils import algo_to_pickle @@ -230,7 +236,6 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): - if RUNNING_IN_AZURE_ML: work_dir = "outputs" @@ -311,7 +316,7 @@ def write_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None filepath=self.data_src_cfg_name, fmt="yaml", default_flow_style=None, - sort_keys=False + sort_keys=False, ) def read_cache(self): @@ -672,7 +677,6 @@ def run(self): "Possibly the required data analysis step was not completed." ) - bundle_generator = BundleGen( algos=self.algos, algo_path=self.work_dir, @@ -698,7 +702,6 @@ def run(self): # step 3: algo training if self.train: - history = import_bundle_algo_history(self.work_dir, only_trained=False) if len(history) == 0: From d35d6e769e519789a34114298d48aa3fe0ce971d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 17:14:41 +0000 Subject: [PATCH 17/58] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .amlignore | 2 +- monai/apps/auto3dseg/auto_runner.py | 1 - monai/apps/auto3dseg/utils.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.amlignore b/.amlignore index 59a45cc78b..58feb18d7a 100644 --- a/.amlignore +++ b/.amlignore @@ -1,2 +1,2 @@ tests/ -docs/ \ No newline at end of file +docs/ diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 9a4c93747a..1457897b3e 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -12,7 +12,6 @@ from __future__ import annotations import os -import shutil import subprocess import sys from copy import deepcopy diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index dce5866b32..9e438d464e 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -19,7 +19,6 @@ from monai.apps.auto3dseg.bundle_gen import BundleAlgo from monai.auto3dseg import algo_from_pickle, algo_to_pickle -from monai.utils import optional_import AZUREML_CONFIG_KEY = "azureml_config" From 2ed7ab0ac3d29a48a7af26c6087bb6b1ac2dc77d Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 1 Mar 2023 17:24:44 +0000 Subject: [PATCH 18/58] =?UTF-8?q?=F0=9F=90=9B=20Update=20optional=20import?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/auto_runner.py | 11 +++-------- monai/apps/auto3dseg/utils.py | 5 ++--- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 9a4c93747a..6a4ad0343b 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -12,7 +12,6 @@ from __future__ import annotations import os -import shutil import subprocess import sys from copy import deepcopy @@ -21,8 +20,6 @@ import numpy as np import torch -from health_azure.himl import AZUREML_FLAG -from health_azure.utils import is_running_in_azure_ml from monai.apps.auto3dseg.bundle_gen import BundleGen from monai.apps.auto3dseg.data_analyzer import DataAnalyzer @@ -49,9 +46,7 @@ logger = get_logger(module_name=__name__) nni, has_nni = optional_import("nni") - - -RUNNING_IN_AZURE_ML = is_running_in_azure_ml() +health_azure, has_health_azure = optional_import("health_azure") class AutoRunner: @@ -236,7 +231,7 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): - if RUNNING_IN_AZURE_ML: + if health_azure.utils.is_running_in_azure_ml(): work_dir = "outputs" logger.info(f"AutoRunner using work directory {work_dir}") @@ -262,7 +257,7 @@ def __init__( if len(missing_keys) > 0: raise ValueError(f"Config keys are missing {missing_keys}") - if AZUREML_FLAG in sys.argv or RUNNING_IN_AZURE_ML: + if health_azure.himl.AZUREML_FLAG in sys.argv or health_azure.utils.is_running_in_azure_ml(): run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) self.dataroot = run_info.input_datasets[0] self.data_src_cfg["dataroot"] = str(self.dataroot) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index dce5866b32..d0eb7e4bdf 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -14,13 +14,12 @@ import os from typing import Any -# health_azure, has_health_azure = optional_import("health_azure") -import health_azure - from monai.apps.auto3dseg.bundle_gen import BundleAlgo from monai.auto3dseg import algo_from_pickle, algo_to_pickle from monai.utils import optional_import +health_azure, has_health_azure = optional_import("health_azure") + AZUREML_CONFIG_KEY = "azureml_config" From a3342e4edb00e657db566f113144fe3fb1c3cd51 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 1 Mar 2023 17:33:05 +0000 Subject: [PATCH 19/58] DCO Remediation Commit for Peter Hessey I, Peter Hessey , hereby add my Signed-off-by to this commit: 76f373767c8708a2774798e69a929b56f582d0fc I, Peter Hessey , hereby add my Signed-off-by to this commit: 6288cdcc2f0588af406ad5e639ab74c1d8408390 I, Peter Hessey , hereby add my Signed-off-by to this commit: 9d0b98f830dd4ba50a6d8c960c7f63233299fc24 I, Peter Hessey , hereby add my Signed-off-by to this commit: dc8101b1b71e3dcfe978ea1e45ef943a535e8b3c I, Peter Hessey , hereby add my Signed-off-by to this commit: 2b88637509c1a25f4dc6b9e7d6dc3848e05c7d79 I, Peter Hessey , hereby add my Signed-off-by to this commit: 136ec66b6508391f77a4616dab9d613443ea72ba I, Peter Hessey , hereby add my Signed-off-by to this commit: 49dbb0483228c0d25f5dcf6285dd098fe75080b9 I, Peter Hessey , hereby add my Signed-off-by to this commit: c4b8d4ecce1e503c409668871799a9b3d83b7c92 I, Peter Hessey , hereby add my Signed-off-by to this commit: ce16775fd14603dc37e9a7fadcba28360dcb155e I, Peter Hessey , hereby add my Signed-off-by to this commit: f5ea9d60442c91322647800c92b538ef6cf53209 I, Peter Hessey , hereby add my Signed-off-by to this commit: 218cb8b62c0d80acad975d5eced74e3bf9945e42 I, Peter Hessey , hereby add my Signed-off-by to this commit: f54793004cd7b78d08a82f1fcd3f227661d2577a I, Peter Hessey , hereby add my Signed-off-by to this commit: 9a1e7c7daf506e528b6cf7b032121b783dd226a0 I, Peter Hessey , hereby add my Signed-off-by to this commit: 4c024bccc73148fa4b62e56bb3931c0daad772d4 I, Peter Hessey , hereby add my Signed-off-by to this commit: e6e34563909524037f3d1671796c248e8861bbe4 I, Peter Hessey , hereby add my Signed-off-by to this commit: 2ed7ab0ac3d29a48a7af26c6087bb6b1ac2dc77d Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index ee31a5e3bd..5d85fdb30a 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -16,6 +16,7 @@ from monai.apps.auto3dseg.bundle_gen import BundleAlgo from monai.auto3dseg import algo_from_pickle, algo_to_pickle +from monai.utils.module import optional_import health_azure, has_health_azure = optional_import("health_azure") @@ -74,6 +75,12 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> health_azure.AzureRunInfo: + """ + Submit Auto3dSeg modules to run as AzureML jobs if the user has requested it. + + Args: + azure_cfg: Dictionary containing arguments to be used for AzureML job submission. + """ azureml_args = { "workspace_config_file": "azureml_configs/azureml_config.json", "docker_base_image": "mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", From 91d73202ac1ae32b8571d837e8fea0c2605bfbe1 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 1 Mar 2023 17:41:07 +0000 Subject: [PATCH 20/58] Fix mypy Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/auto_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 6a4ad0343b..eaf3ac7d8f 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -223,7 +223,7 @@ def __init__( algo_gen: bool | None = None, train: bool | None = None, training_params: dict[str, Any] | None = None, - num_fold: int | None = 5, + num_fold: int = 5, hpo: bool = False, hpo_backend: str = "nni", ensemble: bool = True, From 244cf81881a4c65ad4a950851cd7e477e8effb4b Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 6 Mar 2023 09:29:39 +0000 Subject: [PATCH 21/58] =?UTF-8?q?=F0=9F=93=8C=20Pin=20packages=20to=20remo?= =?UTF-8?q?ve=20Azure=20warnings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environment-azureml.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment-azureml.yml b/environment-azureml.yml index c24311eaef..2956c25177 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -12,6 +12,8 @@ dependencies: - pip - python>=3.8 - pip: + - argcomplete<2.0.0 + - importlib-metadata<5,>=0.23 - hi-ml-azure>=0.2.18 - monai>=1.1.0 - pytorch-ignite==0.4.10 From d7d2af8e98c02e13bd1fabb8f5e2c777532610fe Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 7 Mar 2023 10:28:43 +0000 Subject: [PATCH 22/58] =?UTF-8?q?=E2=9C=85=20=F0=9F=9A=A7=20Add=20azureml?= =?UTF-8?q?=20auto3dseg=20test=20for=20heart=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_auto3dseg_azureml.py | 53 +++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/test_auto3dseg_azureml.py diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py new file mode 100644 index 0000000000..c02ba33f0f --- /dev/null +++ b/tests/test_auto3dseg_azureml.py @@ -0,0 +1,53 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import unittest + +from monai.apps.auto3dseg import AutoRunner +from monai.utils import optional_import +from tests.utils import skip_if_quick + +health_azure, has_health_azure = optional_import("health_azure") + + +@skip_if_quick +class TestAuto3DSegAzureML(unittest.TestCase): + def setUp(self) -> None: + return super().setUp() + + def test_submit_autorunner_job_to_azureml(self) -> None: + test_input = './azureml_configs/auto3dseg_test_task.yaml' + test_params = { + "num_epochs_per_validation": 1, + "num_images_per_batch": 1, + "num_epochs": 2, + "num_warmup_epochs": 1 + } + test_num_fold = 2 + with self.assertRaises(SystemExit) as cm: + with unittest.mock.patch('sys.argv', [ + '__main__.py', + 'AutoRunner', + 'run', + f'--input={test_input}', + f'--training_params={test_params}', + f'--num_fold={test_num_fold}', + '--azureml' + ]): + AutoRunner(input=test_input, training_params=test_params, num_fold=test_num_fold) + + self.assertEqual(cm.exception.code, 0) + + +if __name__ == "__main__": + unittest.main() From dcfee01474e93e4496de47f661a4820023e240f4 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 7 Mar 2023 10:43:25 +0000 Subject: [PATCH 23/58] =?UTF-8?q?=F0=9F=9A=A7=20Start=20building=20AzureML?= =?UTF-8?q?=20test=20using=20synthetic=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- azureml_configs/auto3dseg_test_task.yaml | 15 ++ azureml_configs/azureml_config.json | 5 + azureml_configs/heart_datalist.json | 136 +++++++++++++++ azureml_configs/lung_datalist.json | 63 +++++++ azureml_configs/prostate_datalist.json | 204 +++++++++++++++++++++++ azureml_configs/task_heart.yaml | 14 ++ azureml_configs/task_lung.yaml | 8 + azureml_configs/task_prostate.yaml | 16 ++ tests/test_auto3dseg_azureml.py | 27 ++- tests/utils.py | 4 + 10 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 azureml_configs/auto3dseg_test_task.yaml create mode 100755 azureml_configs/azureml_config.json create mode 100644 azureml_configs/heart_datalist.json create mode 100644 azureml_configs/lung_datalist.json create mode 100644 azureml_configs/prostate_datalist.json create mode 100644 azureml_configs/task_heart.yaml create mode 100644 azureml_configs/task_lung.yaml create mode 100644 azureml_configs/task_prostate.yaml diff --git a/azureml_configs/auto3dseg_test_task.yaml b/azureml_configs/auto3dseg_test_task.yaml new file mode 100644 index 0000000000..c78f3d4f60 --- /dev/null +++ b/azureml_configs/auto3dseg_test_task.yaml @@ -0,0 +1,15 @@ +name: Task02_Heart +task: segmentation + +modality: MRI +datalist: ./azureml_configs/heart_datalist.json # list of files +dataroot: /data/Task02_heart # data location + +multigpu: True + +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "medical_decathlon_heart_task_02" + default_datastore: "himldatasets" + wait_for_completion: True diff --git a/azureml_configs/azureml_config.json b/azureml_configs/azureml_config.json new file mode 100755 index 0000000000..0cfa077f00 --- /dev/null +++ b/azureml_configs/azureml_config.json @@ -0,0 +1,5 @@ +{ + "subscription_id": "a85ceddd-892e-4637-ae4b-67d15ddf5f2b", + "resource_group": "health-ml", + "workspace_name": "hi-ml" +} \ No newline at end of file diff --git a/azureml_configs/heart_datalist.json b/azureml_configs/heart_datalist.json new file mode 100644 index 0000000000..e9b2d69f19 --- /dev/null +++ b/azureml_configs/heart_datalist.json @@ -0,0 +1,136 @@ +{ + "testing": [ + { + "image": "./imagesTs/la_015.nii.gz" + }, + { + "image": "./imagesTs/la_025.nii.gz" + }, + { + "image": "./imagesTs/la_013.nii.gz" + }, + { + "image": "./imagesTs/la_001.nii.gz" + }, + { + "image": "./imagesTs/la_027.nii.gz" + }, + { + "image": "./imagesTs/la_006.nii.gz" + }, + { + "image": "./imagesTs/la_008.nii.gz" + }, + { + "image": "./imagesTs/la_012.nii.gz" + }, + { + "image": "./imagesTs/la_028.nii.gz" + }, + { + "image": "./imagesTs/la_002.nii.gz" + } + ], + "training": [ + { + "fold": 0, + "image": "./imagesTr/la_007.nii.gz", + "label": "./labelsTr/la_007.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_019.nii.gz", + "label": "./labelsTr/la_019.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_023.nii.gz", + "label": "./labelsTr/la_023.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_005.nii.gz", + "label": "./labelsTr/la_005.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_009.nii.gz", + "label": "./labelsTr/la_009.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_017.nii.gz", + "label": "./labelsTr/la_017.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_021.nii.gz", + "label": "./labelsTr/la_021.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_029.nii.gz", + "label": "./labelsTr/la_029.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_003.nii.gz", + "label": "./labelsTr/la_003.nii.gz" + }, + { + "fold": 0, + "image": "./imagesTr/la_011.nii.gz", + "label": "./labelsTr/la_011.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_030.nii.gz", + "label": "./labelsTr/la_030.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_022.nii.gz", + "label": "./labelsTr/la_022.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_014.nii.gz", + "label": "./labelsTr/la_014.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_018.nii.gz", + "label": "./labelsTr/la_018.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_020.nii.gz", + "label": "./labelsTr/la_020.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_004.nii.gz", + "label": "./labelsTr/la_004.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_016.nii.gz", + "label": "./labelsTr/la_016.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_024.nii.gz", + "label": "./labelsTr/la_024.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_010.nii.gz", + "label": "./labelsTr/la_010.nii.gz" + }, + { + "fold": 1, + "image": "./imagesTr/la_026.nii.gz", + "label": "./labelsTr/la_026.nii.gz" + } + ] +} diff --git a/azureml_configs/lung_datalist.json b/azureml_configs/lung_datalist.json new file mode 100644 index 0000000000..e81923c2a3 --- /dev/null +++ b/azureml_configs/lung_datalist.json @@ -0,0 +1,63 @@ +{ + "testing": [ + + { + "image": "0/ct.nii.gz", + "label": "0/label.nii.gz" + }, + { + "image": "1/ct.nii.gz", + "label": "1/label.nii.gz" + }, + { + "image": "2/ct.nii.gz", + "label": "2/label.nii.gz" + }, + { + "image": "3/ct.nii.gz", + "label": "3/label.nii.gz" + } + ], + "training": [ + { + "fold": 0, + "image": "10/ct.nii.gz", + "label": "10/label.nii.gz" + }, + { + "fold": 0, + "image": "11/ct.nii.gz", + "label": "11/label.nii.gz" + }, + { + "fold": 0, + "image": "12/ct.nii.gz", + "label": "12/label.nii.gz" + }, + { + "fold": 0, + "image": "13/ct.nii.gz", + "label": "13/label.nii.gz" + }, + { + "fold": 1, + "image": "14/ct.nii.gz", + "label": "14/label.nii.gz" + }, + { + "fold": 1, + "image": "15/ct.nii.gz", + "label": "15/label.nii.gz" + }, + { + "fold": 1, + "image": "16/ct.nii.gz", + "label": "16/label.nii.gz" + }, + { + "fold": 1, + "image": "17/ct.nii.gz", + "label": "17/label.nii.gz" + } + ] +} diff --git a/azureml_configs/prostate_datalist.json b/azureml_configs/prostate_datalist.json new file mode 100644 index 0000000000..fba52eebb6 --- /dev/null +++ b/azureml_configs/prostate_datalist.json @@ -0,0 +1,204 @@ +{ + "testing": [ + { + "image": "imagesTs/prostate_08.nii.gz" + }, + { + "image": "imagesTs/prostate_22.nii.gz" + }, + { + "image": "imagesTs/prostate_30.nii.gz" + }, + { + "image": "imagesTs/prostate_45.nii.gz" + }, + { + "image": "imagesTs/prostate_26.nii.gz" + }, + { + "image": "imagesTs/prostate_36.nii.gz" + }, + { + "image": "imagesTs/prostate_12.nii.gz" + }, + { + "image": "imagesTs/prostate_33.nii.gz" + }, + { + "image": "imagesTs/prostate_09.nii.gz" + }, + { + "image": "imagesTs/prostate_05.nii.gz" + }, + { + "image": "imagesTs/prostate_23.nii.gz" + }, + { + "image": "imagesTs/prostate_19.nii.gz" + }, + { + "image": "imagesTs/prostate_15.nii.gz" + }, + { + "image": "imagesTs/prostate_11.nii.gz" + }, + { + "image": "imagesTs/prostate_03.nii.gz" + }, + { + "image": "imagesTs/prostate_27.nii.gz" + } + ], + "training": [ + { + "fold": 0, + "image": "imagesTr/prostate_16.nii.gz", + "label": "labelsTr/prostate_16.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_04.nii.gz", + "label": "labelsTr/prostate_04.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_20.nii.gz", + "label": "labelsTr/prostate_20.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_43.nii.gz", + "label": "labelsTr/prostate_43.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_06.nii.gz", + "label": "labelsTr/prostate_06.nii.gz" + }, + { + "fold": 0, + "image": "imagesTr/prostate_14.nii.gz", + "label": "labelsTr/prostate_14.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_41.nii.gz", + "label": "labelsTr/prostate_41.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_34.nii.gz", + "label": "labelsTr/prostate_34.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_38.nii.gz", + "label": "labelsTr/prostate_38.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_10.nii.gz", + "label": "labelsTr/prostate_10.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_02.nii.gz", + "label": "labelsTr/prostate_02.nii.gz" + }, + { + "fold": 1, + "image": "imagesTr/prostate_24.nii.gz", + "label": "labelsTr/prostate_24.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_47.nii.gz", + "label": "labelsTr/prostate_47.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_28.nii.gz", + "label": "labelsTr/prostate_28.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_00.nii.gz", + "label": "labelsTr/prostate_00.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_42.nii.gz", + "label": "labelsTr/prostate_42.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_21.nii.gz", + "label": "labelsTr/prostate_21.nii.gz" + }, + { + "fold": 2, + "image": "imagesTr/prostate_17.nii.gz", + "label": "labelsTr/prostate_17.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_40.nii.gz", + "label": "labelsTr/prostate_40.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_31.nii.gz", + "label": "labelsTr/prostate_31.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_07.nii.gz", + "label": "labelsTr/prostate_07.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_35.nii.gz", + "label": "labelsTr/prostate_35.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_44.nii.gz", + "label": "labelsTr/prostate_44.nii.gz" + }, + { + "fold": 3, + "image": "imagesTr/prostate_39.nii.gz", + "label": "labelsTr/prostate_39.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_01.nii.gz", + "label": "labelsTr/prostate_01.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_13.nii.gz", + "label": "labelsTr/prostate_13.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_46.nii.gz", + "label": "labelsTr/prostate_46.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_25.nii.gz", + "label": "labelsTr/prostate_25.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_29.nii.gz", + "label": "labelsTr/prostate_29.nii.gz" + }, + { + "fold": 4, + "image": "imagesTr/prostate_37.nii.gz", + "label": "labelsTr/prostate_37.nii.gz" + } + ] +} diff --git a/azureml_configs/task_heart.yaml b/azureml_configs/task_heart.yaml new file mode 100644 index 0000000000..e0aabad532 --- /dev/null +++ b/azureml_configs/task_heart.yaml @@ -0,0 +1,14 @@ +name: Task02_Heart +task: segmentation + +modality: MRI +datalist: ./azureml_configs/heart_datalist.json # list of files +dataroot: /data/Task02_heart # data location + +multigpu: True + +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "medical_decathlon_heart_task_02" + default_datastore: "himldatasets" diff --git a/azureml_configs/task_lung.yaml b/azureml_configs/task_lung.yaml new file mode 100644 index 0000000000..b8c67a33ab --- /dev/null +++ b/azureml_configs/task_lung.yaml @@ -0,0 +1,8 @@ +modality: CT +datalist: "./azureml_configs/lung_datalist.json" +dataroot: "/workspace/data/task" +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "lung_tcia_single_label_12_22_v3" + default_datastore: "himldatasets" diff --git a/azureml_configs/task_prostate.yaml b/azureml_configs/task_prostate.yaml new file mode 100644 index 0000000000..6026689736 --- /dev/null +++ b/azureml_configs/task_prostate.yaml @@ -0,0 +1,16 @@ +name: Task05_Prostate +task: segmentation + +modality: MRI +datalist: ./azureml_configs/prostate_datalist.json # list of files +dataroot: /data/Task05_Prostate # data location + +multigpu: True + +class_names: [val_acc_pz, val_acc_tz] + +azureml_config: + compute_cluster_name: "dedicated-nc24s-v2" + input_datasets: + - "medical_decathlon_prostate_task_05" + default_datastore: "himldatasets" diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index c02ba33f0f..2746b4f620 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -11,6 +11,8 @@ from __future__ import annotations +import os +import tempfile import unittest from monai.apps.auto3dseg import AutoRunner @@ -19,11 +21,31 @@ health_azure, has_health_azure = optional_import("health_azure") +fake_datalist: dict[str, list[dict]] = { + "testing": [{"image": "val_001.fake.nii.gz"}, {"image": "val_002.fake.nii.gz"}], + "training": [ + {"fold": 0, "image": "tr_image_001.fake.nii.gz", "label": "tr_label_001.fake.nii.gz"}, + {"fold": 0, "image": "tr_image_002.fake.nii.gz", "label": "tr_label_002.fake.nii.gz"}, + {"fold": 0, "image": "tr_image_003.fake.nii.gz", "label": "tr_label_003.fake.nii.gz"}, + {"fold": 0, "image": "tr_image_004.fake.nii.gz", "label": "tr_label_004.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_005.fake.nii.gz", "label": "tr_label_005.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_006.fake.nii.gz", "label": "tr_label_006.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_007.fake.nii.gz", "label": "tr_label_007.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_008.fake.nii.gz", "label": "tr_label_008.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_009.fake.nii.gz", "label": "tr_label_009.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_010.fake.nii.gz", "label": "tr_label_010.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_011.fake.nii.gz", "label": "tr_label_011.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_012.fake.nii.gz", "label": "tr_label_012.fake.nii.gz"}, + ], +} + @skip_if_quick +@unittest.skipIf(not has_health_azure, "health_azure package is required for this test.") class TestAuto3DSegAzureML(unittest.TestCase): def setUp(self) -> None: - return super().setUp() + self.test_dir = tempfile.TemporaryDirectory(dir=".") + os.makedirs(os.path.join(self.test_dir.name, "data")) def test_submit_autorunner_job_to_azureml(self) -> None: test_input = './azureml_configs/auto3dseg_test_task.yaml' @@ -48,6 +70,9 @@ def test_submit_autorunner_job_to_azureml(self) -> None: self.assertEqual(cm.exception.code, 0) + def tearDown(self) -> None: + self.test_dir.cleanup() + if __name__ == "__main__": unittest.main() diff --git a/tests/utils.py b/tests/utils.py index ef46678c06..48c87ae3ed 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -76,6 +76,10 @@ def get_testing_algo_template_path(): return os.environ.get("MONAI_TESTING_ALGO_TEMPLATE", None) +def generate_segmentation_test_data(dataroot: str, datalist: dict[str, Any]) -> None: + pass + + def clone(data: NdarrayTensor) -> NdarrayTensor: """ Clone data independent of type. From 4145a78f326b412154d256346f85458cbd533908 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 10:43:53 +0000 Subject: [PATCH 24/58] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- azureml_configs/azureml_config.json | 2 +- azureml_configs/lung_datalist.json | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/azureml_configs/azureml_config.json b/azureml_configs/azureml_config.json index 0cfa077f00..d967c4be81 100755 --- a/azureml_configs/azureml_config.json +++ b/azureml_configs/azureml_config.json @@ -2,4 +2,4 @@ "subscription_id": "a85ceddd-892e-4637-ae4b-67d15ddf5f2b", "resource_group": "health-ml", "workspace_name": "hi-ml" -} \ No newline at end of file +} diff --git a/azureml_configs/lung_datalist.json b/azureml_configs/lung_datalist.json index e81923c2a3..f2ca8752ca 100644 --- a/azureml_configs/lung_datalist.json +++ b/azureml_configs/lung_datalist.json @@ -1,6 +1,5 @@ { "testing": [ - { "image": "0/ct.nii.gz", "label": "0/label.nii.gz" From e3eacf4360ef2d5ecc5b281462fc853e1f79550b Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 7 Mar 2023 13:42:12 +0000 Subject: [PATCH 25/58] =?UTF-8?q?=F0=9F=8E=A8=20Restructure=20auto3dseg=20?= =?UTF-8?q?synth=20data=20functions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Hessey --- tests/test_auto3dseg_azureml.py | 2 +- tests/test_auto3dseg_ensemble.py | 16 ++----- tests/test_auto3dseg_hpo.py | 35 ++-------------- tests/test_integration_gpu_customization.py | 34 ++------------- tests/utils.py | 46 +++++++++++++++++++-- 5 files changed, 56 insertions(+), 77 deletions(-) diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index 2746b4f620..d2e664f821 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -17,7 +17,7 @@ from monai.apps.auto3dseg import AutoRunner from monai.utils import optional_import -from tests.utils import skip_if_quick +from tests.utils import generate_fake_segmentation_data, skip_if_quick health_azure, has_health_azure = optional_import("health_azure") diff --git a/tests/test_auto3dseg_ensemble.py b/tests/test_auto3dseg_ensemble.py index 979ebf744b..78d254b396 100644 --- a/tests/test_auto3dseg_ensemble.py +++ b/tests/test_auto3dseg_ensemble.py @@ -26,6 +26,8 @@ from monai.utils.enums import AlgoEnsembleKeys from tests.utils import ( SkipIfBeforePyTorchVersion, + export_fake_data_config_file, + generate_fake_segmentation_data, get_testing_algo_template_path, skip_if_downloading_fails, skip_if_no_cuda, @@ -94,20 +96,10 @@ def test_ensemble(self) -> None: os.makedirs(work_dir) # Generate a fake dataset - for d in fake_datalist["testing"] + fake_datalist["training"]: - im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) - nib_image = nib.Nifti1Image(im, affine=np.eye(4)) - image_fpath = os.path.join(dataroot, d["image"]) - nib.save(nib_image, image_fpath) - - if "label" in d: - nib_image = nib.Nifti1Image(seg, affine=np.eye(4)) - label_fpath = os.path.join(dataroot, d["label"]) - nib.save(nib_image, label_fpath) + generate_fake_segmentation_data(dataroot, fake_datalist) # write to a json file - fake_json_datalist = os.path.join(dataroot, "fake_input.json") - ConfigParser.export_config_file(fake_datalist, fake_json_datalist) + fake_json_datalist = export_fake_data_config_file(dataroot) da = DataAnalyzer(fake_json_datalist, dataroot, output_path=da_output_yaml) da.get_all_case_stats() diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py index 6501bb9363..509d08de22 100644 --- a/tests/test_auto3dseg_hpo.py +++ b/tests/test_auto3dseg_hpo.py @@ -26,6 +26,8 @@ from monai.utils import optional_import from tests.utils import ( SkipIfBeforePyTorchVersion, + export_fake_data_config_file, + generate_fake_segmentation_data, get_testing_algo_template_path, skip_if_downloading_fails, skip_if_no_cuda, @@ -55,25 +57,6 @@ def skip_if_no_optuna(obj): return unittest.skipUnless(has_optuna, "Skipping optuna tests")(obj) -fake_datalist: dict[str, list[dict]] = { - "testing": [{"image": "val_001.fake.nii.gz"}, {"image": "val_002.fake.nii.gz"}], - "training": [ - {"fold": 0, "image": "tr_image_001.fake.nii.gz", "label": "tr_label_001.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_002.fake.nii.gz", "label": "tr_label_002.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_003.fake.nii.gz", "label": "tr_label_003.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_004.fake.nii.gz", "label": "tr_label_004.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_005.fake.nii.gz", "label": "tr_label_005.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_006.fake.nii.gz", "label": "tr_label_006.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_007.fake.nii.gz", "label": "tr_label_007.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_008.fake.nii.gz", "label": "tr_label_008.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_009.fake.nii.gz", "label": "tr_label_009.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_010.fake.nii.gz", "label": "tr_label_010.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_011.fake.nii.gz", "label": "tr_label_011.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_012.fake.nii.gz", "label": "tr_label_012.fake.nii.gz"}, - ], -} - - @SkipIfBeforePyTorchVersion((1, 9, 1)) @unittest.skipIf(not has_tb, "no tensorboard summary writer") class TestHPO(unittest.TestCase): @@ -94,20 +77,10 @@ def setUp(self) -> None: os.makedirs(work_dir) # Generate a fake dataset - for d in fake_datalist["testing"] + fake_datalist["training"]: - im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) - nib_image = nib.Nifti1Image(im, affine=np.eye(4)) - image_fpath = os.path.join(dataroot, d["image"]) - nib.save(nib_image, image_fpath) - - if "label" in d: - nib_image = nib.Nifti1Image(seg, affine=np.eye(4)) - label_fpath = os.path.join(dataroot, d["label"]) - nib.save(nib_image, label_fpath) + generate_fake_segmentation_data(dataroot) # write to a json file - fake_json_datalist = os.path.join(dataroot, "fake_input.json") - ConfigParser.export_config_file(fake_datalist, fake_json_datalist) + fake_json_datalist = export_fake_data_config_file(dataroot) da = DataAnalyzer(fake_json_datalist, dataroot, output_path=da_output_yaml) da.get_all_case_stats() diff --git a/tests/test_integration_gpu_customization.py b/tests/test_integration_gpu_customization.py index 4c8772ea50..f7fbdc5ff2 100644 --- a/tests/test_integration_gpu_customization.py +++ b/tests/test_integration_gpu_customization.py @@ -26,6 +26,8 @@ from monai.utils.enums import AlgoEnsembleKeys from tests.utils import ( SkipIfBeforePyTorchVersion, + export_fake_data_config_file, + generate_fake_segmentation_data, get_testing_algo_template_path, skip_if_downloading_fails, skip_if_no_cuda, @@ -34,24 +36,6 @@ _, has_tb = optional_import("torch.utils.tensorboard", name="SummaryWriter") -fake_datalist: dict[str, list[dict]] = { - "testing": [{"image": "val_001.fake.nii.gz"}, {"image": "val_002.fake.nii.gz"}], - "training": [ - {"fold": 0, "image": "tr_image_001.fake.nii.gz", "label": "tr_label_001.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_002.fake.nii.gz", "label": "tr_label_002.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_003.fake.nii.gz", "label": "tr_label_003.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_004.fake.nii.gz", "label": "tr_label_004.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_005.fake.nii.gz", "label": "tr_label_005.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_006.fake.nii.gz", "label": "tr_label_006.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_007.fake.nii.gz", "label": "tr_label_007.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_008.fake.nii.gz", "label": "tr_label_008.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_009.fake.nii.gz", "label": "tr_label_009.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_010.fake.nii.gz", "label": "tr_label_010.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_011.fake.nii.gz", "label": "tr_label_011.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_012.fake.nii.gz", "label": "tr_label_012.fake.nii.gz"}, - ], -} - train_param = ( { "num_images_per_batch": 2, @@ -92,20 +76,10 @@ def test_ensemble_gpu_customization(self) -> None: os.makedirs(work_dir) # Generate a fake dataset - for d in fake_datalist["testing"] + fake_datalist["training"]: - im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) - nib_image = nib.Nifti1Image(im, affine=np.eye(4)) - image_fpath = os.path.join(dataroot, d["image"]) - nib.save(nib_image, image_fpath) - - if "label" in d: - nib_image = nib.Nifti1Image(seg, affine=np.eye(4)) - label_fpath = os.path.join(dataroot, d["label"]) - nib.save(nib_image, label_fpath) + generate_fake_segmentation_data(dataroot) # write to a json file - fake_json_datalist = os.path.join(dataroot, "fake_input.json") - ConfigParser.export_config_file(fake_datalist, fake_json_datalist) + fake_json_datalist = export_fake_data_config_file(dataroot) da = DataAnalyzer(fake_json_datalist, dataroot, output_path=da_output_yaml) da.get_all_case_stats() diff --git a/tests/utils.py b/tests/utils.py index 48c87ae3ed..73c1c3681b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -30,7 +30,7 @@ from contextlib import contextmanager from functools import partial, reduce from subprocess import PIPE, Popen -from typing import Callable +from typing import Any, Callable from urllib.error import ContentTooShortError, HTTPError import numpy as np @@ -38,6 +38,7 @@ import torch.distributed as dist from monai.apps.utils import download_url +from monai.bundle.config_parser import ConfigParser from monai.config import NdarrayTensor from monai.config.deviceconfig import USE_COMPILED from monai.config.type_definitions import NdarrayOrTensor @@ -55,6 +56,24 @@ _tf32_enabled = None _test_data_config: dict = {} +DEFAULT_FAKE_SEGMENTATION_DATALIST: dict[str, list[dict]] = { + "testing": [{"image": "val_001.fake.nii.gz"}, {"image": "val_002.fake.nii.gz"}], + "training": [ + {"fold": 0, "image": "tr_image_001.fake.nii.gz", "label": "tr_label_001.fake.nii.gz"}, + {"fold": 0, "image": "tr_image_002.fake.nii.gz", "label": "tr_label_002.fake.nii.gz"}, + {"fold": 0, "image": "tr_image_003.fake.nii.gz", "label": "tr_label_003.fake.nii.gz"}, + {"fold": 0, "image": "tr_image_004.fake.nii.gz", "label": "tr_label_004.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_005.fake.nii.gz", "label": "tr_label_005.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_006.fake.nii.gz", "label": "tr_label_006.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_007.fake.nii.gz", "label": "tr_label_007.fake.nii.gz"}, + {"fold": 1, "image": "tr_image_008.fake.nii.gz", "label": "tr_label_008.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_009.fake.nii.gz", "label": "tr_label_009.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_010.fake.nii.gz", "label": "tr_label_010.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_011.fake.nii.gz", "label": "tr_label_011.fake.nii.gz"}, + {"fold": 2, "image": "tr_image_012.fake.nii.gz", "label": "tr_label_012.fake.nii.gz"}, + ], +} + def testing_data_config(*keys): """get _test_data_config[keys0][keys1]...[keysN]""" @@ -76,8 +95,29 @@ def get_testing_algo_template_path(): return os.environ.get("MONAI_TESTING_ALGO_TEMPLATE", None) -def generate_segmentation_test_data(dataroot: str, datalist: dict[str, Any]) -> None: - pass +def generate_fake_segmentation_data( + dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST +) -> None: + for d in fake_datalist["testing"] + fake_datalist["training"]: + im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) + nib_image = nib.Nifti1Image(im, affine=np.eye(4)) + image_fpath = os.path.join(dataroot, d["image"]) + nib.save(nib_image, image_fpath) + + if "label" in d: + nib_image = nib.Nifti1Image(seg, affine=np.eye(4)) + label_fpath = os.path.join(dataroot, d["label"]) + nib.save(nib_image, label_fpath) + + +def export_fake_data_config_file( + dataroot: str, + fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST, + fake_json_datalist_filename: str = "fake_input.json" +) -> str: + fake_json_datalist = os.path.join(dataroot, fake_json_datalist_filename) + ConfigParser.export_config_file(fake_datalist, fake_json_datalist) + return fake_json_datalist def clone(data: NdarrayTensor) -> NdarrayTensor: From 9c0cc4dc6eff173058b9caa6cda27ea6bd46267e Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 7 Mar 2023 15:51:09 +0000 Subject: [PATCH 26/58] =?UTF-8?q?=E2=9C=85=20Add=20synthetic=20data=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/auto_runner.py | 7 ++-- tests/test_auto3dseg_azureml.py | 56 +++++++++++++++++------------ 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index eaf3ac7d8f..e05cb29daa 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -259,8 +259,11 @@ def __init__( if health_azure.himl.AZUREML_FLAG in sys.argv or health_azure.utils.is_running_in_azure_ml(): run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) - self.dataroot = run_info.input_datasets[0] - self.data_src_cfg["dataroot"] = str(self.dataroot) + if run_info.input_datasets: + self.dataroot = run_info.input_datasets[0] + self.data_src_cfg["dataroot"] = str(self.dataroot) + else: + self.dataroot = self.data_src_cfg["dataroot"] else: self.dataroot = self.data_src_cfg["dataroot"] diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index d2e664f821..e67b968b0f 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -15,40 +15,48 @@ import tempfile import unittest + from monai.apps.auto3dseg import AutoRunner +from monai.bundle.config_parser import ConfigParser from monai.utils import optional_import -from tests.utils import generate_fake_segmentation_data, skip_if_quick +from tests.utils import generate_fake_segmentation_data, export_fake_data_config_file, skip_if_quick health_azure, has_health_azure = optional_import("health_azure") -fake_datalist: dict[str, list[dict]] = { - "testing": [{"image": "val_001.fake.nii.gz"}, {"image": "val_002.fake.nii.gz"}], - "training": [ - {"fold": 0, "image": "tr_image_001.fake.nii.gz", "label": "tr_label_001.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_002.fake.nii.gz", "label": "tr_label_002.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_003.fake.nii.gz", "label": "tr_label_003.fake.nii.gz"}, - {"fold": 0, "image": "tr_image_004.fake.nii.gz", "label": "tr_label_004.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_005.fake.nii.gz", "label": "tr_label_005.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_006.fake.nii.gz", "label": "tr_label_006.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_007.fake.nii.gz", "label": "tr_label_007.fake.nii.gz"}, - {"fold": 1, "image": "tr_image_008.fake.nii.gz", "label": "tr_label_008.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_009.fake.nii.gz", "label": "tr_label_009.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_010.fake.nii.gz", "label": "tr_label_010.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_011.fake.nii.gz", "label": "tr_label_011.fake.nii.gz"}, - {"fold": 2, "image": "tr_image_012.fake.nii.gz", "label": "tr_label_012.fake.nii.gz"}, - ], -} - @skip_if_quick @unittest.skipIf(not has_health_azure, "health_azure package is required for this test.") class TestAuto3DSegAzureML(unittest.TestCase): def setUp(self) -> None: self.test_dir = tempfile.TemporaryDirectory(dir=".") - os.makedirs(os.path.join(self.test_dir.name, "data")) + self.dataroot = os.path.join(self.test_dir.name, "data") + os.makedirs(self.dataroot) def test_submit_autorunner_job_to_azureml(self) -> None: - test_input = './azureml_configs/auto3dseg_test_task.yaml' + + # generate fake data and datalist + generate_fake_segmentation_data(self.dataroot) + fake_json_datalist = export_fake_data_config_file(self.dataroot) + + # set up test task yaml + self.test_task_yaml = os.path.join(self.test_dir.name, "fake_task.yaml") + azureml_test_task_src = { + "name": "test_task", + "task": "segmentation", + "modality": "MRI", + "datalist": fake_json_datalist, + "dataroot": self.dataroot, + "multigpu": True, + + "azureml_config": { + "compute_cluster_name": "dedicated-nc24s-v2", + "default_datastore": "himldatasets", + "wait_for_completion": True + } + } + ConfigParser.export_config_file(azureml_test_task_src, self.test_task_yaml) + + # set up test training params test_params = { "num_epochs_per_validation": 1, "num_images_per_batch": 1, @@ -56,17 +64,19 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "num_warmup_epochs": 1 } test_num_fold = 2 + + # run AutoRunner in AzureML with self.assertRaises(SystemExit) as cm: with unittest.mock.patch('sys.argv', [ '__main__.py', 'AutoRunner', 'run', - f'--input={test_input}', + f'--input={self.test_task_yaml}', f'--training_params={test_params}', f'--num_fold={test_num_fold}', '--azureml' ]): - AutoRunner(input=test_input, training_params=test_params, num_fold=test_num_fold) + AutoRunner(input=self.test_task_yaml, training_params=test_params, num_fold=test_num_fold) self.assertEqual(cm.exception.code, 0) From 0eb2cd5ac9a7ad9493f21825e71b92e90e94083d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 15:51:50 +0000 Subject: [PATCH 27/58] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_auto3dseg_ensemble.py | 3 --- tests/test_auto3dseg_hpo.py | 3 --- tests/test_integration_gpu_customization.py | 3 --- 3 files changed, 9 deletions(-) diff --git a/tests/test_auto3dseg_ensemble.py b/tests/test_auto3dseg_ensemble.py index 78d254b396..f37b50e312 100644 --- a/tests/test_auto3dseg_ensemble.py +++ b/tests/test_auto3dseg_ensemble.py @@ -15,13 +15,10 @@ import tempfile import unittest -import nibabel as nib -import numpy as np import torch from monai.apps.auto3dseg import AlgoEnsembleBestByFold, AlgoEnsembleBestN, AlgoEnsembleBuilder, BundleGen, DataAnalyzer from monai.bundle.config_parser import ConfigParser -from monai.data import create_test_image_3d from monai.utils import optional_import, set_determinism from monai.utils.enums import AlgoEnsembleKeys from tests.utils import ( diff --git a/tests/test_auto3dseg_hpo.py b/tests/test_auto3dseg_hpo.py index 509d08de22..13acfeea1c 100644 --- a/tests/test_auto3dseg_hpo.py +++ b/tests/test_auto3dseg_hpo.py @@ -16,13 +16,10 @@ import unittest from functools import partial -import nibabel as nib -import numpy as np import torch from monai.apps.auto3dseg import BundleGen, DataAnalyzer, NNIGen, OptunaGen, import_bundle_algo_history from monai.bundle.config_parser import ConfigParser -from monai.data import create_test_image_3d from monai.utils import optional_import from tests.utils import ( SkipIfBeforePyTorchVersion, diff --git a/tests/test_integration_gpu_customization.py b/tests/test_integration_gpu_customization.py index f7fbdc5ff2..4a756fc202 100644 --- a/tests/test_integration_gpu_customization.py +++ b/tests/test_integration_gpu_customization.py @@ -15,13 +15,10 @@ import tempfile import unittest -import nibabel as nib -import numpy as np import torch from monai.apps.auto3dseg import AlgoEnsembleBestByFold, AlgoEnsembleBestN, AlgoEnsembleBuilder, BundleGen, DataAnalyzer from monai.bundle.config_parser import ConfigParser -from monai.data import create_test_image_3d from monai.utils import optional_import from monai.utils.enums import AlgoEnsembleKeys from tests.utils import ( From 301dc7dc100eeaaeb064eb44ffab383619920b5a Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 10 Mar 2023 09:00:58 +0000 Subject: [PATCH 28/58] =?UTF-8?q?=F0=9F=93=8C=20Update=20azureml=20env=20a?= =?UTF-8?q?nd=20remove=20configs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .amlignore | 2 +- azureml_configs/auto3dseg_test_task.yaml | 15 -- azureml_configs/azureml_config.json | 5 - azureml_configs/heart_datalist.json | 136 --------------- azureml_configs/lung_datalist.json | 62 ------- azureml_configs/prostate_datalist.json | 204 ----------------------- azureml_configs/task_heart.yaml | 14 -- azureml_configs/task_lung.yaml | 8 - azureml_configs/task_prostate.yaml | 16 -- environment-azureml.yml | 2 +- 10 files changed, 2 insertions(+), 462 deletions(-) delete mode 100644 azureml_configs/auto3dseg_test_task.yaml delete mode 100755 azureml_configs/azureml_config.json delete mode 100644 azureml_configs/heart_datalist.json delete mode 100644 azureml_configs/lung_datalist.json delete mode 100644 azureml_configs/prostate_datalist.json delete mode 100644 azureml_configs/task_heart.yaml delete mode 100644 azureml_configs/task_lung.yaml delete mode 100644 azureml_configs/task_prostate.yaml diff --git a/.amlignore b/.amlignore index 58feb18d7a..59a45cc78b 100644 --- a/.amlignore +++ b/.amlignore @@ -1,2 +1,2 @@ tests/ -docs/ +docs/ \ No newline at end of file diff --git a/azureml_configs/auto3dseg_test_task.yaml b/azureml_configs/auto3dseg_test_task.yaml deleted file mode 100644 index c78f3d4f60..0000000000 --- a/azureml_configs/auto3dseg_test_task.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: Task02_Heart -task: segmentation - -modality: MRI -datalist: ./azureml_configs/heart_datalist.json # list of files -dataroot: /data/Task02_heart # data location - -multigpu: True - -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "medical_decathlon_heart_task_02" - default_datastore: "himldatasets" - wait_for_completion: True diff --git a/azureml_configs/azureml_config.json b/azureml_configs/azureml_config.json deleted file mode 100755 index d967c4be81..0000000000 --- a/azureml_configs/azureml_config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "subscription_id": "a85ceddd-892e-4637-ae4b-67d15ddf5f2b", - "resource_group": "health-ml", - "workspace_name": "hi-ml" -} diff --git a/azureml_configs/heart_datalist.json b/azureml_configs/heart_datalist.json deleted file mode 100644 index e9b2d69f19..0000000000 --- a/azureml_configs/heart_datalist.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "testing": [ - { - "image": "./imagesTs/la_015.nii.gz" - }, - { - "image": "./imagesTs/la_025.nii.gz" - }, - { - "image": "./imagesTs/la_013.nii.gz" - }, - { - "image": "./imagesTs/la_001.nii.gz" - }, - { - "image": "./imagesTs/la_027.nii.gz" - }, - { - "image": "./imagesTs/la_006.nii.gz" - }, - { - "image": "./imagesTs/la_008.nii.gz" - }, - { - "image": "./imagesTs/la_012.nii.gz" - }, - { - "image": "./imagesTs/la_028.nii.gz" - }, - { - "image": "./imagesTs/la_002.nii.gz" - } - ], - "training": [ - { - "fold": 0, - "image": "./imagesTr/la_007.nii.gz", - "label": "./labelsTr/la_007.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_019.nii.gz", - "label": "./labelsTr/la_019.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_023.nii.gz", - "label": "./labelsTr/la_023.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_005.nii.gz", - "label": "./labelsTr/la_005.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_009.nii.gz", - "label": "./labelsTr/la_009.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_017.nii.gz", - "label": "./labelsTr/la_017.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_021.nii.gz", - "label": "./labelsTr/la_021.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_029.nii.gz", - "label": "./labelsTr/la_029.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_003.nii.gz", - "label": "./labelsTr/la_003.nii.gz" - }, - { - "fold": 0, - "image": "./imagesTr/la_011.nii.gz", - "label": "./labelsTr/la_011.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_030.nii.gz", - "label": "./labelsTr/la_030.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_022.nii.gz", - "label": "./labelsTr/la_022.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_014.nii.gz", - "label": "./labelsTr/la_014.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_018.nii.gz", - "label": "./labelsTr/la_018.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_020.nii.gz", - "label": "./labelsTr/la_020.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_004.nii.gz", - "label": "./labelsTr/la_004.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_016.nii.gz", - "label": "./labelsTr/la_016.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_024.nii.gz", - "label": "./labelsTr/la_024.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_010.nii.gz", - "label": "./labelsTr/la_010.nii.gz" - }, - { - "fold": 1, - "image": "./imagesTr/la_026.nii.gz", - "label": "./labelsTr/la_026.nii.gz" - } - ] -} diff --git a/azureml_configs/lung_datalist.json b/azureml_configs/lung_datalist.json deleted file mode 100644 index f2ca8752ca..0000000000 --- a/azureml_configs/lung_datalist.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "testing": [ - { - "image": "0/ct.nii.gz", - "label": "0/label.nii.gz" - }, - { - "image": "1/ct.nii.gz", - "label": "1/label.nii.gz" - }, - { - "image": "2/ct.nii.gz", - "label": "2/label.nii.gz" - }, - { - "image": "3/ct.nii.gz", - "label": "3/label.nii.gz" - } - ], - "training": [ - { - "fold": 0, - "image": "10/ct.nii.gz", - "label": "10/label.nii.gz" - }, - { - "fold": 0, - "image": "11/ct.nii.gz", - "label": "11/label.nii.gz" - }, - { - "fold": 0, - "image": "12/ct.nii.gz", - "label": "12/label.nii.gz" - }, - { - "fold": 0, - "image": "13/ct.nii.gz", - "label": "13/label.nii.gz" - }, - { - "fold": 1, - "image": "14/ct.nii.gz", - "label": "14/label.nii.gz" - }, - { - "fold": 1, - "image": "15/ct.nii.gz", - "label": "15/label.nii.gz" - }, - { - "fold": 1, - "image": "16/ct.nii.gz", - "label": "16/label.nii.gz" - }, - { - "fold": 1, - "image": "17/ct.nii.gz", - "label": "17/label.nii.gz" - } - ] -} diff --git a/azureml_configs/prostate_datalist.json b/azureml_configs/prostate_datalist.json deleted file mode 100644 index fba52eebb6..0000000000 --- a/azureml_configs/prostate_datalist.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "testing": [ - { - "image": "imagesTs/prostate_08.nii.gz" - }, - { - "image": "imagesTs/prostate_22.nii.gz" - }, - { - "image": "imagesTs/prostate_30.nii.gz" - }, - { - "image": "imagesTs/prostate_45.nii.gz" - }, - { - "image": "imagesTs/prostate_26.nii.gz" - }, - { - "image": "imagesTs/prostate_36.nii.gz" - }, - { - "image": "imagesTs/prostate_12.nii.gz" - }, - { - "image": "imagesTs/prostate_33.nii.gz" - }, - { - "image": "imagesTs/prostate_09.nii.gz" - }, - { - "image": "imagesTs/prostate_05.nii.gz" - }, - { - "image": "imagesTs/prostate_23.nii.gz" - }, - { - "image": "imagesTs/prostate_19.nii.gz" - }, - { - "image": "imagesTs/prostate_15.nii.gz" - }, - { - "image": "imagesTs/prostate_11.nii.gz" - }, - { - "image": "imagesTs/prostate_03.nii.gz" - }, - { - "image": "imagesTs/prostate_27.nii.gz" - } - ], - "training": [ - { - "fold": 0, - "image": "imagesTr/prostate_16.nii.gz", - "label": "labelsTr/prostate_16.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_04.nii.gz", - "label": "labelsTr/prostate_04.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_20.nii.gz", - "label": "labelsTr/prostate_20.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_43.nii.gz", - "label": "labelsTr/prostate_43.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_06.nii.gz", - "label": "labelsTr/prostate_06.nii.gz" - }, - { - "fold": 0, - "image": "imagesTr/prostate_14.nii.gz", - "label": "labelsTr/prostate_14.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_41.nii.gz", - "label": "labelsTr/prostate_41.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_34.nii.gz", - "label": "labelsTr/prostate_34.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_38.nii.gz", - "label": "labelsTr/prostate_38.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_10.nii.gz", - "label": "labelsTr/prostate_10.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_02.nii.gz", - "label": "labelsTr/prostate_02.nii.gz" - }, - { - "fold": 1, - "image": "imagesTr/prostate_24.nii.gz", - "label": "labelsTr/prostate_24.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_47.nii.gz", - "label": "labelsTr/prostate_47.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_28.nii.gz", - "label": "labelsTr/prostate_28.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_00.nii.gz", - "label": "labelsTr/prostate_00.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_42.nii.gz", - "label": "labelsTr/prostate_42.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_21.nii.gz", - "label": "labelsTr/prostate_21.nii.gz" - }, - { - "fold": 2, - "image": "imagesTr/prostate_17.nii.gz", - "label": "labelsTr/prostate_17.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_40.nii.gz", - "label": "labelsTr/prostate_40.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_31.nii.gz", - "label": "labelsTr/prostate_31.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_07.nii.gz", - "label": "labelsTr/prostate_07.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_35.nii.gz", - "label": "labelsTr/prostate_35.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_44.nii.gz", - "label": "labelsTr/prostate_44.nii.gz" - }, - { - "fold": 3, - "image": "imagesTr/prostate_39.nii.gz", - "label": "labelsTr/prostate_39.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_01.nii.gz", - "label": "labelsTr/prostate_01.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_13.nii.gz", - "label": "labelsTr/prostate_13.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_46.nii.gz", - "label": "labelsTr/prostate_46.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_25.nii.gz", - "label": "labelsTr/prostate_25.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_29.nii.gz", - "label": "labelsTr/prostate_29.nii.gz" - }, - { - "fold": 4, - "image": "imagesTr/prostate_37.nii.gz", - "label": "labelsTr/prostate_37.nii.gz" - } - ] -} diff --git a/azureml_configs/task_heart.yaml b/azureml_configs/task_heart.yaml deleted file mode 100644 index e0aabad532..0000000000 --- a/azureml_configs/task_heart.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Task02_Heart -task: segmentation - -modality: MRI -datalist: ./azureml_configs/heart_datalist.json # list of files -dataroot: /data/Task02_heart # data location - -multigpu: True - -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "medical_decathlon_heart_task_02" - default_datastore: "himldatasets" diff --git a/azureml_configs/task_lung.yaml b/azureml_configs/task_lung.yaml deleted file mode 100644 index b8c67a33ab..0000000000 --- a/azureml_configs/task_lung.yaml +++ /dev/null @@ -1,8 +0,0 @@ -modality: CT -datalist: "./azureml_configs/lung_datalist.json" -dataroot: "/workspace/data/task" -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "lung_tcia_single_label_12_22_v3" - default_datastore: "himldatasets" diff --git a/azureml_configs/task_prostate.yaml b/azureml_configs/task_prostate.yaml deleted file mode 100644 index 6026689736..0000000000 --- a/azureml_configs/task_prostate.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: Task05_Prostate -task: segmentation - -modality: MRI -datalist: ./azureml_configs/prostate_datalist.json # list of files -dataroot: /data/Task05_Prostate # data location - -multigpu: True - -class_names: [val_acc_pz, val_acc_tz] - -azureml_config: - compute_cluster_name: "dedicated-nc24s-v2" - input_datasets: - - "medical_decathlon_prostate_task_05" - default_datastore: "himldatasets" diff --git a/environment-azureml.yml b/environment-azureml.yml index 2956c25177..683c922549 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -15,7 +15,7 @@ dependencies: - argcomplete<2.0.0 - importlib-metadata<5,>=0.23 - hi-ml-azure>=0.2.18 - - monai>=1.1.0 + - git+https://github.com/peterhessey/MONAI.git@6054-integrate-azureml-for-autorunner-training - pytorch-ignite==0.4.10 - gdown>=4.4.0 - scipy From 53db45b600c993e5090bc3e979bfd4ed5d627660 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 10 Mar 2023 09:02:12 +0000 Subject: [PATCH 29/58] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .amlignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.amlignore b/.amlignore index 59a45cc78b..58feb18d7a 100644 --- a/.amlignore +++ b/.amlignore @@ -1,2 +1,2 @@ tests/ -docs/ \ No newline at end of file +docs/ From 1d9c2f25e27d2717f26449a88599f7a80fb586a2 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 10 Mar 2023 09:19:15 +0000 Subject: [PATCH 30/58] Update setup.py Signed-off-by: Peter Hessey --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b90d9d0976..a7fb1cd68a 100644 --- a/setup.py +++ b/setup.py @@ -146,6 +146,9 @@ def get_cmds(): cmdclass=get_cmds(), packages=find_packages(exclude=("docs", "examples", "tests")), zip_safe=False, - package_data={"monai": ["py.typed", *jit_extension_source]}, + package_data={ + "monai": ["py.typed", *jit_extension_source], + "azureml_env": ["environment-azureml.yml"], + }, ext_modules=get_extensions(), ) From 8a83655e911006197eb6f96f5fc0bdcc83d444a0 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 14 Mar 2023 13:07:19 +0000 Subject: [PATCH 31/58] =?UTF-8?q?=F0=9F=92=84=20Fix=20styling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Hessey --- setup.py | 5 +---- tests/test_auto3dseg_azureml.py | 32 ++++++++++++++++---------------- tests/utils.py | 4 ++-- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index a7fb1cd68a..b47fb81d1e 100644 --- a/setup.py +++ b/setup.py @@ -146,9 +146,6 @@ def get_cmds(): cmdclass=get_cmds(), packages=find_packages(exclude=("docs", "examples", "tests")), zip_safe=False, - package_data={ - "monai": ["py.typed", *jit_extension_source], - "azureml_env": ["environment-azureml.yml"], - }, + package_data={"monai": ["py.typed", *jit_extension_source], "azureml_env": ["environment-azureml.yml"]}, ext_modules=get_extensions(), ) diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index e67b968b0f..6e8c4e2bfe 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -15,11 +15,10 @@ import tempfile import unittest - from monai.apps.auto3dseg import AutoRunner from monai.bundle.config_parser import ConfigParser from monai.utils import optional_import -from tests.utils import generate_fake_segmentation_data, export_fake_data_config_file, skip_if_quick +from tests.utils import export_fake_data_config_file, generate_fake_segmentation_data, skip_if_quick health_azure, has_health_azure = optional_import("health_azure") @@ -33,7 +32,6 @@ def setUp(self) -> None: os.makedirs(self.dataroot) def test_submit_autorunner_job_to_azureml(self) -> None: - # generate fake data and datalist generate_fake_segmentation_data(self.dataroot) fake_json_datalist = export_fake_data_config_file(self.dataroot) @@ -47,12 +45,11 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "datalist": fake_json_datalist, "dataroot": self.dataroot, "multigpu": True, - "azureml_config": { "compute_cluster_name": "dedicated-nc24s-v2", "default_datastore": "himldatasets", - "wait_for_completion": True - } + "wait_for_completion": True, + }, } ConfigParser.export_config_file(azureml_test_task_src, self.test_task_yaml) @@ -61,21 +58,24 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "num_epochs_per_validation": 1, "num_images_per_batch": 1, "num_epochs": 2, - "num_warmup_epochs": 1 + "num_warmup_epochs": 1, } test_num_fold = 2 # run AutoRunner in AzureML with self.assertRaises(SystemExit) as cm: - with unittest.mock.patch('sys.argv', [ - '__main__.py', - 'AutoRunner', - 'run', - f'--input={self.test_task_yaml}', - f'--training_params={test_params}', - f'--num_fold={test_num_fold}', - '--azureml' - ]): + with unittest.mock.patch( + "sys.argv", + [ + "__main__.py", + "AutoRunner", + "run", + f"--input={self.test_task_yaml}", + f"--training_params={test_params}", + f"--num_fold={test_num_fold}", + "--azureml", + ], + ): AutoRunner(input=self.test_task_yaml, training_params=test_params, num_fold=test_num_fold) self.assertEqual(cm.exception.code, 0) diff --git a/tests/utils.py b/tests/utils.py index 73c1c3681b..3004f0a195 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -96,7 +96,7 @@ def get_testing_algo_template_path(): def generate_fake_segmentation_data( - dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST + dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST ) -> None: for d in fake_datalist["testing"] + fake_datalist["training"]: im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) @@ -113,7 +113,7 @@ def generate_fake_segmentation_data( def export_fake_data_config_file( dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST, - fake_json_datalist_filename: str = "fake_input.json" + fake_json_datalist_filename: str = "fake_input.json", ) -> str: fake_json_datalist = os.path.join(dataroot, fake_json_datalist_filename) ConfigParser.export_config_file(fake_datalist, fake_json_datalist) From b9e5bc7a99335911ab76841ad681a1e57b050d83 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 14 Mar 2023 13:07:19 +0000 Subject: [PATCH 32/58] =?UTF-8?q?=F0=9F=92=84=20Fix=20styling?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Hessey --- setup.py | 5 +---- tests/test_auto3dseg_azureml.py | 32 ++++++++++++++++---------------- tests/utils.py | 4 ++-- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index a7fb1cd68a..b47fb81d1e 100644 --- a/setup.py +++ b/setup.py @@ -146,9 +146,6 @@ def get_cmds(): cmdclass=get_cmds(), packages=find_packages(exclude=("docs", "examples", "tests")), zip_safe=False, - package_data={ - "monai": ["py.typed", *jit_extension_source], - "azureml_env": ["environment-azureml.yml"], - }, + package_data={"monai": ["py.typed", *jit_extension_source], "azureml_env": ["environment-azureml.yml"]}, ext_modules=get_extensions(), ) diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index e67b968b0f..6e8c4e2bfe 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -15,11 +15,10 @@ import tempfile import unittest - from monai.apps.auto3dseg import AutoRunner from monai.bundle.config_parser import ConfigParser from monai.utils import optional_import -from tests.utils import generate_fake_segmentation_data, export_fake_data_config_file, skip_if_quick +from tests.utils import export_fake_data_config_file, generate_fake_segmentation_data, skip_if_quick health_azure, has_health_azure = optional_import("health_azure") @@ -33,7 +32,6 @@ def setUp(self) -> None: os.makedirs(self.dataroot) def test_submit_autorunner_job_to_azureml(self) -> None: - # generate fake data and datalist generate_fake_segmentation_data(self.dataroot) fake_json_datalist = export_fake_data_config_file(self.dataroot) @@ -47,12 +45,11 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "datalist": fake_json_datalist, "dataroot": self.dataroot, "multigpu": True, - "azureml_config": { "compute_cluster_name": "dedicated-nc24s-v2", "default_datastore": "himldatasets", - "wait_for_completion": True - } + "wait_for_completion": True, + }, } ConfigParser.export_config_file(azureml_test_task_src, self.test_task_yaml) @@ -61,21 +58,24 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "num_epochs_per_validation": 1, "num_images_per_batch": 1, "num_epochs": 2, - "num_warmup_epochs": 1 + "num_warmup_epochs": 1, } test_num_fold = 2 # run AutoRunner in AzureML with self.assertRaises(SystemExit) as cm: - with unittest.mock.patch('sys.argv', [ - '__main__.py', - 'AutoRunner', - 'run', - f'--input={self.test_task_yaml}', - f'--training_params={test_params}', - f'--num_fold={test_num_fold}', - '--azureml' - ]): + with unittest.mock.patch( + "sys.argv", + [ + "__main__.py", + "AutoRunner", + "run", + f"--input={self.test_task_yaml}", + f"--training_params={test_params}", + f"--num_fold={test_num_fold}", + "--azureml", + ], + ): AutoRunner(input=self.test_task_yaml, training_params=test_params, num_fold=test_num_fold) self.assertEqual(cm.exception.code, 0) diff --git a/tests/utils.py b/tests/utils.py index 73c1c3681b..3004f0a195 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -96,7 +96,7 @@ def get_testing_algo_template_path(): def generate_fake_segmentation_data( - dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST + dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST ) -> None: for d in fake_datalist["testing"] + fake_datalist["training"]: im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) @@ -113,7 +113,7 @@ def generate_fake_segmentation_data( def export_fake_data_config_file( dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST, - fake_json_datalist_filename: str = "fake_input.json" + fake_json_datalist_filename: str = "fake_input.json", ) -> str: fake_json_datalist = os.path.join(dataroot, fake_json_datalist_filename) ConfigParser.export_config_file(fake_datalist, fake_json_datalist) From 18237ae71c9b49029ce3e541eb26a28135a5c24f Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 14 Mar 2023 13:45:02 +0000 Subject: [PATCH 33/58] DCO Remediation Commit for Peter Hessey I, Peter Hessey , hereby add my Signed-off-by to this commit: 244cf81881a4c65ad4a950851cd7e477e8effb4b I, Peter Hessey , hereby add my Signed-off-by to this commit: d7d2af8e98c02e13bd1fabb8f5e2c777532610fe I, Peter Hessey , hereby add my Signed-off-by to this commit: dcfee01474e93e4496de47f661a4820023e240f4 I, Peter Hessey , hereby add my Signed-off-by to this commit: 301dc7dc100eeaaeb064eb44ffab383619920b5a Signed-off-by: Peter Hessey --- tests/test_auto3dseg_azureml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index 6e8c4e2bfe..164d172b43 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -40,8 +40,8 @@ def test_submit_autorunner_job_to_azureml(self) -> None: self.test_task_yaml = os.path.join(self.test_dir.name, "fake_task.yaml") azureml_test_task_src = { "name": "test_task", - "task": "segmentation", "modality": "MRI", + "task": "segmentation", "datalist": fake_json_datalist, "dataroot": self.dataroot, "multigpu": True, From b63142cd962b1683190810fc9f43ddaae28456a9 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 14 Mar 2023 13:53:18 +0000 Subject: [PATCH 34/58] Fix typing Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 5d85fdb30a..91626845e2 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -74,7 +74,7 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> health_azure.AzureRunInfo: +def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]): """ Submit Auto3dSeg modules to run as AzureML jobs if the user has requested it. From 3a098fa142d8c1d12bd125b8ff23f31ced1983b9 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 14 Mar 2023 13:53:18 +0000 Subject: [PATCH 35/58] Fix typing Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 5d85fdb30a..6ee7fed52d 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -74,7 +74,8 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> health_azure.AzureRunInfo: +# can't give actual return type from health_azure because it's an optional import +def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> Any: """ Submit Auto3dSeg modules to run as AzureML jobs if the user has requested it. From 5e9fa1ca551f4ce0c8213e9dde89ced1b9c3d625 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 14 Mar 2023 14:05:47 +0000 Subject: [PATCH 36/58] Remove git merge content Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 453d1fc38e..6ee7fed52d 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -74,12 +74,8 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -<<<<<<< HEAD # can't give actual return type from health_azure because it's an optional import def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> Any: -======= -def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]): ->>>>>>> b63142cd962b1683190810fc9f43ddaae28456a9 """ Submit Auto3dSeg modules to run as AzureML jobs if the user has requested it. From c96b66ab13119cd0af5a88b2939f3614aad5113f Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 15 Mar 2023 11:10:44 +0000 Subject: [PATCH 37/58] Add hi-ml-azure to dev reqs Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 3 +-- requirements-dev.txt | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 6ee7fed52d..5d85fdb30a 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -74,8 +74,7 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -# can't give actual return type from health_azure because it's an optional import -def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> Any: +def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> health_azure.AzureRunInfo: """ Submit Auto3dSeg modules to run as AzureML jobs if the user has requested it. diff --git a/requirements-dev.txt b/requirements-dev.txt index 8b4433b39d..feb483fe4a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -50,3 +50,4 @@ h5py nni optuna git+https://github.com/Project-MONAI/MetricsReloaded@monai-support#egg=MetricsReloaded +hi-ml-azure>=0.2.18 From ac9ec8c7ffcfa3f14215a1f289231a05f158c3b6 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 16 Mar 2023 15:50:55 +0000 Subject: [PATCH 38/58] Test creating separate AML class Signed-off-by: Peter Hessey --- .amlignore | 1 + monai/apps/auto3dseg/__main__.py | 2 + monai/apps/auto3dseg/auto_runner.py | 64 +++++++--------- monai/apps/auto3dseg/azureml_auto_runner.py | 85 +++++++++++++++++++++ 4 files changed, 116 insertions(+), 36 deletions(-) create mode 100644 monai/apps/auto3dseg/azureml_auto_runner.py diff --git a/.amlignore b/.amlignore index 58feb18d7a..0e950bc2de 100644 --- a/.amlignore +++ b/.amlignore @@ -1,2 +1,3 @@ tests/ docs/ +monai/ diff --git a/monai/apps/auto3dseg/__main__.py b/monai/apps/auto3dseg/__main__.py index d169467ba9..6ea7c77134 100644 --- a/monai/apps/auto3dseg/__main__.py +++ b/monai/apps/auto3dseg/__main__.py @@ -12,6 +12,7 @@ from __future__ import annotations from monai.apps.auto3dseg.auto_runner import AutoRunner +from monai.apps.auto3dseg.azureml_auto_runner import AzureMLAutoRunner from monai.apps.auto3dseg.bundle_gen import BundleAlgo, BundleGen from monai.apps.auto3dseg.data_analyzer import DataAnalyzer from monai.apps.auto3dseg.ensemble_builder import AlgoEnsembleBuilder @@ -28,6 +29,7 @@ "BundleAlgo": BundleAlgo, "AlgoEnsembleBuilder": AlgoEnsembleBuilder, "AutoRunner": AutoRunner, + "AzureMLAutoRunner": AzureMLAutoRunner, "NNIGen": NNIGen, "OptunaGen": OptunaGen, } diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index e05cb29daa..ccf8d3bd4f 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -13,7 +13,6 @@ import os import subprocess -import sys from copy import deepcopy from time import sleep from typing import Any, cast @@ -31,10 +30,8 @@ ) from monai.apps.auto3dseg.hpo_gen import NNIGen from monai.apps.auto3dseg.utils import ( - AZUREML_CONFIG_KEY, export_bundle_algo_history, import_bundle_algo_history, - submit_auto3dseg_module_to_azureml_if_needed, ) from monai.apps.utils import get_logger from monai.auto3dseg.utils import algo_to_pickle @@ -46,7 +43,6 @@ logger = get_logger(module_name=__name__) nni, has_nni = optional_import("nni") -health_azure, has_health_azure = optional_import("health_azure") class AutoRunner: @@ -231,44 +227,17 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): - if health_azure.utils.is_running_in_azure_ml(): - work_dir = "outputs" - - logger.info(f"AutoRunner using work directory {work_dir}") - self.work_dir = os.path.abspath(work_dir) - self.data_src_cfg_name = os.path.join(self.work_dir, "input.yaml") - self.algos = algos - self.templates_path_or_url = templates_path_or_url - - if input is None and os.path.isfile(self.data_src_cfg_name): - input = self.data_src_cfg_name - logger.info(f"Input config is not provided, using the default {input}") - - if isinstance(input, dict): - self.data_src_cfg = input - elif isinstance(input, str) and os.path.isfile(input): - logger.info(f"Loading input config {input}") - self.data_src_cfg = ConfigParser.load_config_file(input) - else: - raise ValueError(f"Input: {input} is not a valid file or dict") + logger.info(f"AutoRunner using work directory {self.work_dir}") + self._set_up_data_src_cfg(input, algos, templates_path_or_url) missing_keys = {"dataroot", "datalist", "modality"}.difference(self.data_src_cfg.keys()) if len(missing_keys) > 0: raise ValueError(f"Config keys are missing {missing_keys}") - if health_azure.himl.AZUREML_FLAG in sys.argv or health_azure.utils.is_running_in_azure_ml(): - run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) - if run_info.input_datasets: - self.dataroot = run_info.input_datasets[0] - self.data_src_cfg["dataroot"] = str(self.dataroot) - else: - self.dataroot = self.data_src_cfg["dataroot"] - else: - self.dataroot = self.data_src_cfg["dataroot"] + self.dataroot = self.data_src_cfg["dataroot"] - os.makedirs(work_dir, exist_ok=True) - self.write_data_src_cfg() + self._create_work_dir_and_data_src_cfg() self.datalist_filename = self.data_src_cfg["datalist"] self.datastats_filename = os.path.join(self.work_dir, "datastats.yaml") @@ -307,7 +276,30 @@ def __init__( self.search_space: dict[str, dict[str, Any]] = {} self.hpo_tasks = 0 - def write_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: + def _set_up_data_src_cfg( + self, + input: dict[str, Any] | str | None = None, + algos: dict | list | str | None = None, + templates_path_or_url: str | None = None, + ): + self.data_src_cfg_name = os.path.join(self.work_dir, "input.yaml") + self.algos = algos + self.templates_path_or_url = templates_path_or_url + + if input is None and os.path.isfile(self.data_src_cfg_name): + input = self.data_src_cfg_name + logger.info(f"Input config is not provided, using the default {input}") + + if isinstance(input, dict): + self.data_src_cfg = input + elif isinstance(input, str) and os.path.isfile(input): + logger.info(f"Loading input config {input}") + self.data_src_cfg = ConfigParser.load_config_file(input) + else: + raise ValueError(f"Input: {input} is not a valid file or dict") + + def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: + os.makedirs(self.work_dir, exist_ok=True) output_data_src_cfg = data_src_cfg if data_src_cfg is not None else self.data_src_cfg ConfigParser.export_config_file( config=output_data_src_cfg, diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py new file mode 100644 index 0000000000..6281380a10 --- /dev/null +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -0,0 +1,85 @@ +# Copyright (c) MONAI Consortium +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Any + +from monai.apps.auto3dseg import AutoRunner +from monai.apps.auto3dseg.utils import AZUREML_CONFIG_KEY, submit_auto3dseg_module_to_azureml_if_needed +from monai.apps.utils import get_logger +from monai.utils.module import optional_import + +logger = get_logger(module_name=__name__) + +nni, has_nni = optional_import("nni") +health_azure, has_health_azure = optional_import("health_azure") + + +class AzureMLAutoRunner(AutoRunner): + def __init__( + self, + input: dict[str, Any] | str | None = None, + algos: dict | list | str | None = None, + analyze: bool | None = None, + algo_gen: bool | None = None, + train: bool | None = None, + training_params: dict[str, Any] | None = None, + num_fold: int = 5, + hpo: bool = False, + hpo_backend: str = "nni", + ensemble: bool = True, + not_use_cache: bool = False, + templates_path_or_url: str | None = None, + **kwargs: Any, + ): + + work_dir = "outputs" + super().__init__( + work_dir=work_dir, + input=input, + algos=algos, + analyze=analyze, + algo_gen=algo_gen, + train=train, + training_params=training_params, + num_fold=num_fold, + hpo=hpo, + hpo_backend=hpo_backend, + ensemble=ensemble, + not_use_cache=not_use_cache, + templates_path_or_url=templates_path_or_url, + **kwargs, + ) + + run_info = submit_auto3dseg_module_to_azureml_if_needed(self.data_src_cfg[AZUREML_CONFIG_KEY]) + if run_info.input_datasets: + self.dataroot = run_info.input_datasets[0] + self.data_src_cfg["dataroot"] = str(self.dataroot) + self._create_work_dir_and_data_src_cfg() + else: + self.dataroot = self.data_src_cfg["dataroot"] + + def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: + if health_azure.utils.is_running_in_azure_ml(): + super()._create_work_dir_and_data_src_cfg(data_src_cfg) + else: + pass + + def set_image_save_transform(self, kwargs): + if health_azure.utils.is_running_in_azure_ml(): + return super().set_image_save_transform(kwargs) + + def export_cache(self, **kwargs): + if health_azure.utils.is_running_in_azure_ml(): + return super().export_cache(**kwargs) + else: + pass From 354fde159ace4dc53f579d84695bfaa7e3bbdf72 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 23 Mar 2023 12:01:45 +0000 Subject: [PATCH 39/58] =?UTF-8?q?=E2=9C=85=20Pass=20Auto3dSeg=20AzureML=20?= =?UTF-8?q?test=20with=20subclass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/__init__.py | 1 + monai/apps/auto3dseg/utils.py | 2 +- tests/test_auto3dseg_azureml.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/monai/apps/auto3dseg/__init__.py b/monai/apps/auto3dseg/__init__.py index a90c626da9..70a85b3e2d 100644 --- a/monai/apps/auto3dseg/__init__.py +++ b/monai/apps/auto3dseg/__init__.py @@ -12,6 +12,7 @@ from __future__ import annotations from .auto_runner import AutoRunner +from .azureml_auto_runner import AzureMLAutoRunner from .bundle_gen import BundleAlgo, BundleGen from .data_analyzer import DataAnalyzer from .ensemble_builder import AlgoEnsemble, AlgoEnsembleBestByFold, AlgoEnsembleBestN, AlgoEnsembleBuilder diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 5d85fdb30a..554805a855 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -82,7 +82,7 @@ def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> h azure_cfg: Dictionary containing arguments to be used for AzureML job submission. """ azureml_args = { - "workspace_config_file": "azureml_configs/azureml_config.json", + "workspace_config_file": "config.json", "docker_base_image": "mcr.microsoft.com/azureml/openmpi3.1.2-cuda10.2-cudnn8-ubuntu18.04", "snapshot_root_directory": os.getcwd(), "conda_environment_file": "environment-azureml.yml", diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index 164d172b43..6b480230f0 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -15,7 +15,7 @@ import tempfile import unittest -from monai.apps.auto3dseg import AutoRunner +from monai.apps.auto3dseg import AzureMLAutoRunner from monai.bundle.config_parser import ConfigParser from monai.utils import optional_import from tests.utils import export_fake_data_config_file, generate_fake_segmentation_data, skip_if_quick @@ -76,7 +76,7 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "--azureml", ], ): - AutoRunner(input=self.test_task_yaml, training_params=test_params, num_fold=test_num_fold) + AzureMLAutoRunner(input=self.test_task_yaml, training_params=test_params, num_fold=test_num_fold) self.assertEqual(cm.exception.code, 0) From 3a60f6d10adce2d5ae170f39831213fafb12332d Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 23 Mar 2023 12:11:35 +0000 Subject: [PATCH 40/58] =?UTF-8?q?=F0=9F=93=9D=20Add=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- monai/apps/auto3dseg/auto_runner.py | 13 +++++++++++++ monai/apps/auto3dseg/azureml_auto_runner.py | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index ccf8d3bd4f..7adb55b5db 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -282,6 +282,13 @@ def _set_up_data_src_cfg( algos: dict | list | str | None = None, templates_path_or_url: str | None = None, ): + """Sets up the AutoRunner data source config using the provided input. + + Args: + input: The input to parse for AutoRunner configuration, defaults to None + algos: The algorithms to use during AutoRunner training, defaults to None + templates_path_or_url: The URL or filepath to the algorithm templates, defaults to None + """ self.data_src_cfg_name = os.path.join(self.work_dir, "input.yaml") self.algos = algos self.templates_path_or_url = templates_path_or_url @@ -299,6 +306,12 @@ def _set_up_data_src_cfg( raise ValueError(f"Input: {input} is not a valid file or dict") def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: + """ + Creates the work dir to be used by AutoRunner and exports the data source config to the specified filename + + Args + param data_src_cfg: dictionary containing the configuration for the AutoRunner, defaults to None + """ os.makedirs(self.work_dir, exist_ok=True) output_data_src_cfg = data_src_cfg if data_src_cfg is not None else self.data_src_cfg ConfigParser.export_config_file( diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 6281380a10..3e2ca177f7 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -25,6 +25,13 @@ class AzureMLAutoRunner(AutoRunner): + """ + Subclass of AutoRunner that runs the training in AzureML instead of on local resources. Inputs are idnetical to + those of AutoRunner, but the `input` argument must be a dictionary or input.yaml file with a key `azureml_config` + that contains the configuration for the AzureML run. + + """ + def __init__( self, input: dict[str, Any] | str | None = None, From a800423b290164946b7dfe920771cb27cd375086 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 23 Mar 2023 12:16:06 +0000 Subject: [PATCH 41/58] =?UTF-8?q?=F0=9F=93=9D=20Add=20more=20docstrings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/utils.py b/tests/utils.py index 3004f0a195..d6d1536c0c 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -98,6 +98,13 @@ def get_testing_algo_template_path(): def generate_fake_segmentation_data( dataroot: str, fake_datalist: dict[str, Any] = DEFAULT_FAKE_SEGMENTATION_DATALIST ) -> None: + """ + Given a fake datalist, generate fake data and save it to the specified dataroot. + + Args: + dataroot: The folder where fake data will be saved. + fake_datalist: The datalist used to generate the fake data. + """ for d in fake_datalist["testing"] + fake_datalist["training"]: im, seg = create_test_image_3d(24, 24, 24, rad_max=10, num_seg_classes=1) nib_image = nib.Nifti1Image(im, affine=np.eye(4)) From 662c6b41dd71cb4ec78d5ea550c53589c7c21c2a Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Mar 2023 09:32:43 +0100 Subject: [PATCH 42/58] Add typing and docstrings Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/auto_runner.py | 2 +- monai/apps/auto3dseg/azureml_auto_runner.py | 31 +++++++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 7adb55b5db..62deab29f4 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -518,7 +518,7 @@ def set_nni_search_space(self, search_space): self.search_space = search_space self.hpo_tasks = value_combinations - def set_image_save_transform(self, kwargs): + def set_image_save_transform(self, kwargs) -> SaveImage: """ Set the ensemble output transform. diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 3e2ca177f7..92bb4071ae 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -11,12 +11,13 @@ from __future__ import annotations -from typing import Any +from typing import Any, Optional from monai.apps.auto3dseg import AutoRunner from monai.apps.auto3dseg.utils import AZUREML_CONFIG_KEY, submit_auto3dseg_module_to_azureml_if_needed from monai.apps.utils import get_logger from monai.utils.module import optional_import +from monai.transforms import SaveImage logger = get_logger(module_name=__name__) @@ -27,8 +28,8 @@ class AzureMLAutoRunner(AutoRunner): """ Subclass of AutoRunner that runs the training in AzureML instead of on local resources. Inputs are idnetical to - those of AutoRunner, but the `input` argument must be a dictionary or input.yaml file with a key `azureml_config` - that contains the configuration for the AzureML run. + those of AutoRunner, but the `input` argument must be a dictionary or input.yaml file containing the key + `azureml_config` that contains the configuration for the AzureML run. """ @@ -76,16 +77,36 @@ def __init__( self.dataroot = self.data_src_cfg["dataroot"] def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: + """ + Creates the work dir to be used by AutoRunner and exports the data source config to the specified filename if + running in AzureML, otherwise do nothing. + + Args + param data_src_cfg: dictionary containing the configuration for the AutoRunner, defaults to None + """ if health_azure.utils.is_running_in_azure_ml(): super()._create_work_dir_and_data_src_cfg(data_src_cfg) else: pass - def set_image_save_transform(self, kwargs): + def set_image_save_transform(self, kwargs) -> Optional[SaveImage]: + """ + Set the ensemble output transform if running in AzureML, otherwise do nothing. + + Args: + kwargs: image writing parameters for the ensemble inference. The kwargs format follows SaveImage + transform. For more information, check https://docs.monai.io/en/stable/transforms.html#saveimage . + + """ if health_azure.utils.is_running_in_azure_ml(): return super().set_image_save_transform(kwargs) + else: + pass - def export_cache(self, **kwargs): + def export_cache(self, **kwargs) -> None: + """ + Export cache to the AzureML job working dir if running in AzureML, otherwise do nothing. + """ if health_azure.utils.is_running_in_azure_ml(): return super().export_cache(**kwargs) else: From 468f893e2c74949a596276975b9fc8037e4365f7 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Mar 2023 09:34:07 +0100 Subject: [PATCH 43/58] DCO Remediation Commit for Peter Hessey I, Peter Hessey , hereby add my Signed-off-by to this commit: 354fde159ace4dc53f579d84695bfaa7e3bbdf72 I, Peter Hessey , hereby add my Signed-off-by to this commit: 3a60f6d10adce2d5ae170f39831213fafb12332d I, Peter Hessey , hereby add my Signed-off-by to this commit: a800423b290164946b7dfe920771cb27cd375086 Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/azureml_auto_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 92bb4071ae..96fb177b0a 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -79,7 +79,7 @@ def __init__( def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None = None) -> None: """ Creates the work dir to be used by AutoRunner and exports the data source config to the specified filename if - running in AzureML, otherwise do nothing. + running in AzureML, do nothing otherwise. Args param data_src_cfg: dictionary containing the configuration for the AutoRunner, defaults to None From 45e482f7b8d5aabb6ac66f3b0d92dfb9ad37cd1a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 28 Mar 2023 09:53:20 +0000 Subject: [PATCH 44/58] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/apps/auto3dseg/azureml_auto_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 96fb177b0a..63c0462d70 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -11,7 +11,7 @@ from __future__ import annotations -from typing import Any, Optional +from typing import Any from monai.apps.auto3dseg import AutoRunner from monai.apps.auto3dseg.utils import AZUREML_CONFIG_KEY, submit_auto3dseg_module_to_azureml_if_needed @@ -89,7 +89,7 @@ def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None else: pass - def set_image_save_transform(self, kwargs) -> Optional[SaveImage]: + def set_image_save_transform(self, kwargs) -> SaveImage | None: """ Set the ensemble output transform if running in AzureML, otherwise do nothing. From 838abb7a5b1d67ec10ac5ed13420d0a81e27aa38 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 28 Mar 2023 10:53:46 +0100 Subject: [PATCH 45/58] Styling fixes Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/auto_runner.py | 5 +---- monai/apps/auto3dseg/azureml_auto_runner.py | 3 +-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index ca5d779e77..782fdbd286 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -30,10 +30,7 @@ AlgoEnsembleBuilder, ) from monai.apps.auto3dseg.hpo_gen import NNIGen -from monai.apps.auto3dseg.utils import ( - export_bundle_algo_history, - import_bundle_algo_history, -) +from monai.apps.auto3dseg.utils import export_bundle_algo_history, import_bundle_algo_history from monai.apps.utils import get_logger from monai.auto3dseg.utils import algo_to_pickle from monai.bundle import ConfigParser diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 96fb177b0a..b294868077 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -16,8 +16,8 @@ from monai.apps.auto3dseg import AutoRunner from monai.apps.auto3dseg.utils import AZUREML_CONFIG_KEY, submit_auto3dseg_module_to_azureml_if_needed from monai.apps.utils import get_logger -from monai.utils.module import optional_import from monai.transforms import SaveImage +from monai.utils.module import optional_import logger = get_logger(module_name=__name__) @@ -49,7 +49,6 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): - work_dir = "outputs" super().__init__( work_dir=work_dir, From a073b7659445a681a7b5133d47f4c58504eaaec3 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 29 Mar 2023 10:34:15 +0000 Subject: [PATCH 46/58] Remove num_fold arg Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/auto_runner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 782fdbd286..3ece19065a 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -217,7 +217,6 @@ def __init__( algo_gen: bool | None = None, train: bool | None = None, training_params: dict[str, Any] | None = None, - num_fold: int = 5, hpo: bool = False, hpo_backend: str = "nni", ensemble: bool = True, @@ -258,7 +257,6 @@ def __init__( self.ensemble = ensemble # last step, no need to check # intermediate variables - self.num_fold = num_fold self.ensemble_method_name = "AlgoEnsembleBestByFold" self.set_training_params(training_params) self.set_prediction_params() From 229a48e1e3007c8da05cfc5d07fb8ef98a3491e2 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Wed, 29 Mar 2023 11:48:19 +0100 Subject: [PATCH 47/58] Remove need for --azureml flag Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 554805a855..a7f61ea483 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -87,6 +87,7 @@ def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> h "snapshot_root_directory": os.getcwd(), "conda_environment_file": "environment-azureml.yml", "entry_script": "-m monai.apps.auto3dseg", + "submit_to_azureml": True, "strictly_aml_v1": False, } azureml_args.update(azure_cfg) From a43bd9bc1933314604d320c414ca15246fc24345 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 31 Mar 2023 10:23:30 +0100 Subject: [PATCH 48/58] =?UTF-8?q?=F0=9F=9A=A7=20Reduce=20AzureML=20environ?= =?UTF-8?q?ment=20complexity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environment-azureml.yml | 39 +---------------------------------- monai/apps/auto3dseg/utils.py | 11 +++++++++- 2 files changed, 11 insertions(+), 39 deletions(-) diff --git a/environment-azureml.yml b/environment-azureml.yml index 683c922549..a92e4eeb4b 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -1,46 +1,9 @@ name: monai-azureml channels: - - pytorch - defaults - - nvidia - - conda-forge dependencies: - - numpy>=1.20 - - pytorch>=1.8 - - torchvision - - pytorch-cuda=11.6 - pip - python>=3.8 - pip: - - argcomplete<2.0.0 - - importlib-metadata<5,>=0.23 - - hi-ml-azure>=0.2.18 - git+https://github.com/peterhessey/MONAI.git@6054-integrate-azureml-for-autorunner-training - - pytorch-ignite==0.4.10 - - gdown>=4.4.0 - - scipy - - itk>=5.2 - - nibabel - - pillow!=8.3.0 # https://github.com/python-pillow/Pillow/issues/5571 - - tensorboard>=2.6 # https://github.com/Project-MONAI/MONAI/issues/5776 - - scikit-image>=0.19.0 - - tqdm>=4.47.0 - - lmdb - - ninja - - torchvision - - psutil - - cucim==22.8.1; platform_system == "Linux" - - openslide-python==1.1.2 - - pandas - - requests - - einops - - transformers<4.22 # https://github.com/Project-MONAI/MONAI/issues/5157 - - mlflow>=1.28.0 - - fire - - pydicom - - h5py - - nni - - optuna - - parameterized - - torch>=1.8 - - numpy>=1.20 + - hi-ml-azure>=0.2.19 diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index a7f61ea483..33967c7036 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -91,7 +91,16 @@ def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> h "strictly_aml_v1": False, } azureml_args.update(azure_cfg) - + config_datasets_key = "input_dataset" + himl_datasets_key = "input_datasets" + + if isinstance(azureml_args[config_datasets_key], str): + azureml_args[himl_datasets_key] = [azureml_args[config_datasets_key]] + azureml_args.pop(config_datasets_key) + else: + raise ValueError( + f"Invalid type for {config_datasets_key} in azureml_args, must be str not {type(azureml_args[config_datasets_key])}" + ) needed_keys = {"compute_cluster_name", "default_datastore"} missing_keys = needed_keys.difference(azureml_args.keys()) if len(missing_keys) > 0: From fe49bee25427dee3b8c0002fdc3500e95c4649dc Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 31 Mar 2023 11:57:54 +0100 Subject: [PATCH 49/58] Update env again Signed-off-by: Peter Hessey --- environment-azureml.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/environment-azureml.yml b/environment-azureml.yml index a92e4eeb4b..6dd1b3e096 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -1,7 +1,11 @@ name: monai-azureml channels: + - pytorch - defaults dependencies: + - numpy>=1.20 + - pytorch>=1.9 + - torchvision - pip - python>=3.8 - pip: From 55d2aaf223a2ffd8c0632908c24fc3a4195ac45a Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 31 Mar 2023 11:58:51 +0100 Subject: [PATCH 50/58] DCO Remediation Commit for Peter Hessey I, Peter Hessey , hereby add my Signed-off-by to this commit: a43bd9bc1933314604d320c414ca15246fc24345 Signed-off-by: Peter Hessey --- environment-azureml.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-azureml.yml b/environment-azureml.yml index 6dd1b3e096..aef63a2046 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -1,3 +1,4 @@ +# environment for running in AzureML name: monai-azureml channels: - pytorch From 2b642a5ed1423a5f4efdb1b670a5368cb135bf07 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 31 Mar 2023 12:15:49 +0100 Subject: [PATCH 51/58] Add fire to AzureML environment Signed-off-by: Peter Hessey --- environment-azureml.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment-azureml.yml b/environment-azureml.yml index aef63a2046..83880112e8 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -12,3 +12,4 @@ dependencies: - pip: - git+https://github.com/peterhessey/MONAI.git@6054-integrate-azureml-for-autorunner-training - hi-ml-azure>=0.2.19 + - fire From d66c65f51982aa6f69e7f507f06e81d3820a19da Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 31 Mar 2023 13:44:15 +0100 Subject: [PATCH 52/58] Remove num_fold arg from AMLAutoRunner Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/azureml_auto_runner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index c857403b0e..a2a04fde72 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -41,7 +41,6 @@ def __init__( algo_gen: bool | None = None, train: bool | None = None, training_params: dict[str, Any] | None = None, - num_fold: int = 5, hpo: bool = False, hpo_backend: str = "nni", ensemble: bool = True, @@ -58,7 +57,6 @@ def __init__( algo_gen=algo_gen, train=train, training_params=training_params, - num_fold=num_fold, hpo=hpo, hpo_backend=hpo_backend, ensemble=ensemble, From 0967115f0d29370ca99d38fc51f22a9a706c45b7 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Fri, 7 Apr 2023 10:06:09 +0100 Subject: [PATCH 53/58] =?UTF-8?q?=F0=9F=9A=A7=20=F0=9F=90=9B=20Bugfix=20an?= =?UTF-8?q?d=20env=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- environment-azureml.yml | 2 ++ monai/apps/auto3dseg/utils.py | 6 +++++- tests/test_auto3dseg_azureml.py | 1 - 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/environment-azureml.yml b/environment-azureml.yml index 83880112e8..db672b9786 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -6,6 +6,7 @@ channels: dependencies: - numpy>=1.20 - pytorch>=1.9 + - pytorch-cuda=11.6 - torchvision - pip - python>=3.8 @@ -13,3 +14,4 @@ dependencies: - git+https://github.com/peterhessey/MONAI.git@6054-integrate-azureml-for-autorunner-training - hi-ml-azure>=0.2.19 - fire + - nibabel diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index 33967c7036..cc204498dc 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -89,13 +89,17 @@ def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> h "entry_script": "-m monai.apps.auto3dseg", "submit_to_azureml": True, "strictly_aml_v1": False, + "input_dataset": "", } azureml_args.update(azure_cfg) config_datasets_key = "input_dataset" himl_datasets_key = "input_datasets" if isinstance(azureml_args[config_datasets_key], str): - azureml_args[himl_datasets_key] = [azureml_args[config_datasets_key]] + if azureml_args[config_datasets_key] == "": + azureml_args[himl_datasets_key] = [] + else: + azureml_args[himl_datasets_key] = [azureml_args[config_datasets_key]] azureml_args.pop(config_datasets_key) else: raise ValueError( diff --git a/tests/test_auto3dseg_azureml.py b/tests/test_auto3dseg_azureml.py index 6b480230f0..63793459f4 100644 --- a/tests/test_auto3dseg_azureml.py +++ b/tests/test_auto3dseg_azureml.py @@ -72,7 +72,6 @@ def test_submit_autorunner_job_to_azureml(self) -> None: "run", f"--input={self.test_task_yaml}", f"--training_params={test_params}", - f"--num_fold={test_num_fold}", "--azureml", ], ): From 345c191268c4d83e6eb39bd3b58a6173338b416d Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Mon, 10 Apr 2023 12:53:32 +0000 Subject: [PATCH 54/58] DCO Remediation Commit for Peter Hessey I, Peter Hessey , hereby add my Signed-off-by to this commit: 0967115f0d29370ca99d38fc51f22a9a706c45b7 Signed-off-by: Peter Hessey --- environment-azureml.yml | 248 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 236 insertions(+), 12 deletions(-) diff --git a/environment-azureml.yml b/environment-azureml.yml index db672b9786..050837b560 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -1,17 +1,241 @@ -# environment for running in AzureML -name: monai-azureml +name: monai-azureml-new channels: - pytorch + - nvidia - defaults dependencies: - - numpy>=1.20 - - pytorch>=1.9 - - pytorch-cuda=11.6 - - torchvision - - pip - - python>=3.8 + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - blas=1.0=mkl + - brotlipy=0.7.0=py38h27cfd23_1003 + - bzip2=1.0.8=h7b6447c_0 + - ca-certificates=2023.01.10=h06a4308_0 + - certifi=2022.12.7=py38h06a4308_0 + - cffi=1.15.1=py38h5eee18b_3 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - cryptography=39.0.1=py38h9ce1e76_0 + - cuda-cudart=11.7.99=0 + - cuda-cupti=11.7.101=0 + - cuda-libraries=11.7.1=0 + - cuda-nvrtc=11.7.99=0 + - cuda-nvtx=11.7.91=0 + - cuda-runtime=11.7.1=0 + - ffmpeg=4.3=hf484d3e_0 + - filelock=3.9.0=py38h06a4308_0 + - flit-core=3.8.0=py38h06a4308_0 + - freetype=2.12.1=h4a9f257_0 + - giflib=5.2.1=h5eee18b_3 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py38heeb90bb_0 + - gnutls=3.6.15=he1e5248_0 + - idna=3.4=py38h06a4308_0 + - intel-openmp=2021.4.0=h06a4308_3561 + - jinja2=3.1.2=py38h06a4308_0 + - jpeg=9e=h5eee18b_1 + - lame=3.100=h7b6447c_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libcublas=11.10.3.66=0 + - libcufft=10.7.2.124=h4fbf590_0 + - libcufile=1.6.0.25=0 + - libcurand=10.3.2.56=0 + - libcusolver=11.4.0.1=0 + - libcusparse=11.7.4.91=0 + - libdeflate=1.17=h5eee18b_0 + - libffi=3.4.2=h6a678d5_6 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.2=h7f8727e_0 + - libnpp=11.7.4.75=0 + - libnvjpeg=11.8.0.2=0 + - libpng=1.6.39=h5eee18b_0 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtasn1=4.19.0=h5eee18b_0 + - libtiff=4.5.0=h6a678d5_2 + - libunistring=0.9.10=h27cfd23_0 + - libwebp=1.2.4=h11a3e52_1 + - libwebp-base=1.2.4=h5eee18b_1 + - lz4-c=1.9.4=h6a678d5_0 + - markupsafe=2.1.1=py38h7f8727e_0 + - mkl=2021.4.0=h06a4308_640 + - mkl-service=2.4.0=py38h7f8727e_0 + - mkl_fft=1.3.1=py38hd3c417c_0 + - mkl_random=1.2.2=py38h51133e4_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.2.1=py38h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nettle=3.7.3=hbbd107a_1 + - networkx=2.8.4=py38h06a4308_1 + - numpy=1.23.5=py38h14f4228_0 + - numpy-base=1.23.5=py38h31eccc5_0 + - openh264=2.1.1=h4ff587b_0 + - openssl=1.1.1t=h7f8727e_0 + - pillow=9.4.0=py38h6a678d5_0 + - pip=23.0.1=py38h06a4308_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyopenssl=23.0.0=py38h06a4308_0 + - pysocks=1.7.1=py38h06a4308_0 + - python=3.8.16 + - pytorch=2.0.0=py3.8_cuda11.7_cudnn8.5.0_0 + - pytorch-cuda=11.7=h778d358_3 + - pytorch-mutex=1.0=cuda + - readline=8.2=h5eee18b_0 + - requests=2.28.1=py38h06a4308_1 + - setuptools=65.6.3=py38h06a4308_0 + - six=1.16.0=pyhd3eb1b0_1 + - sqlite=3.41.1=h5eee18b_0 + - sympy=1.11.1=py38h06a4308_0 + - tk=8.6.12=h1ccaba5_0 + - torchaudio=2.0.0=py38_cu117 + - torchtriton=2.0.0=py38 + - torchvision=0.15.0=py38_cu117 + - typing_extensions=4.4.0=py38h06a4308_0 + - urllib3=1.26.15=py38h06a4308_0 + - wheel=0.38.4=py38h06a4308_0 + - xz=5.2.10=h5eee18b_1 + - zlib=1.2.13=h5eee18b_0 + - zstd=1.5.4=hc292b87_0 - pip: - - git+https://github.com/peterhessey/MONAI.git@6054-integrate-azureml-for-autorunner-training - - hi-ml-azure>=0.2.19 - - fire - - nibabel + - absl-py==1.4.0 + - adal==1.2.7 + - alembic==1.10.3 + - applicationinsights==0.11.10 + - argcomplete==2.1.2 + - attrs==22.2.0 + - azure-ai-ml==1.5.0 + - azure-common==1.1.28 + - azure-core==1.26.4 + - azure-graphrbac==0.61.1 + - azure-identity==1.12.0 + - azure-mgmt-authorization==3.0.0 + - azure-mgmt-containerregistry==10.1.0 + - azure-mgmt-core==1.4.0 + - azure-mgmt-keyvault==10.2.1 + - azure-mgmt-resource==21.2.1 + - azure-mgmt-storage==20.1.0 + - azure-storage-blob==12.10.0 + - azure-storage-file-datalake==12.6.0 + - azure-storage-file-share==12.11.1 + - azureml-core==1.49.0 + - azureml-dataprep==4.9.6 + - azureml-dataprep-native==38.0.0 + - azureml-dataprep-rslex==2.16.4 + - azureml-dataset-runtime==1.49.0 + - azureml-mlflow==1.49.0 + - azureml-telemetry==1.49.0 + - azureml-tensorboard==1.49.0 + - azureml-train-core==1.49.0.post1 + - azureml-train-restclients-hyperdrive==1.49.0 + - backports-tempfile==1.0 + - backports-weakref==1.0.post1 + - bcrypt==4.0.1 + - cachetools==5.3.0 + - click==8.1.3 + - cloudpickle==2.2.1 + - colorama==0.4.6 + - conda-merge==0.2.0 + - contextlib2==21.6.0 + - contourpy==1.0.7 + - cycler==0.11.0 + - databricks-cli==0.17.6 + - distro==1.8.0 + - docker==6.0.1 + - dotnetcore2==3.1.23 + - einops==0.6.0 + - entrypoints==0.4 + - fire==0.5.0 + - flask==2.2.3 + - fonttools==4.39.3 + - fusepy==3.0.1 + - gitdb==4.0.10 + - gitpython==3.1.31 + - google-api-core==2.11.0 + - google-auth==2.17.2 + - google-auth-oauthlib==1.0.0 + - googleapis-common-protos==1.59.0 + - greenlet==2.0.2 + - grpcio==1.53.0 + - gunicorn==20.1.0 + - hi-ml-azure==0.2.19 + - humanfriendly==10.0 + - importlib-metadata==6.3.0 + - importlib-resources==5.12.0 + - isodate==0.6.1 + - itsdangerous==2.1.2 + - jeepney==0.8.0 + - jmespath==1.0.1 + - joblib==1.2.0 + - jsonpickle==2.2.0 + - jsonschema==4.17.3 + - kiwisolver==1.4.4 + - knack==0.10.1 + - llvmlite==0.39.1 + - mako==1.2.4 + - markdown==3.4.3 + - marshmallow==3.19.0 + - matplotlib==3.7.1 + - mlflow==2.2.2 + - mlflow-skinny==2.2.2 + - git+https://github.com/peterhessey/MONAI.git@6054-integrate-azureml-for-autorunner-training + - msal==1.21.0 + - msal-extensions==1.0.0 + - msrest==0.7.1 + - msrestazure==0.6.4 + - ndg-httpsclient==0.5.1 + - nibabel==5.1.0 + - numba==0.56.4 + - oauthlib==3.2.2 + - opencensus==0.11.2 + - opencensus-context==0.1.3 + - opencensus-ext-azure==1.1.9 + - packaging==21.3 + - pandas==2.0.0 + - param==1.13.0 + - paramiko==2.12.0 + - pathspec==0.11.1 + - pkginfo==1.9.6 + - pkgutil-resolve-name==1.3.10 + - portalocker==2.7.0 + - protobuf==3.20.3 + - psutil==5.9.4 + - pyarrow==9.0.0 + - pyasn1==0.4.8 + - pyasn1-modules==0.2.8 + - pydash==5.1.2 + - pygments==2.14.0 + - pyjwt==2.6.0 + - pynacl==1.5.0 + - pyparsing==3.0.9 + - pyrsistent==0.19.3 + - python-dateutil==2.8.2 + - pytz==2022.7.1 + - pyyaml==6.0 + - querystring-parser==1.2.4 + - requests-oauthlib==1.3.1 + - rsa==4.9 + - ruamel-yaml==0.17.21 + - ruamel-yaml-clib==0.2.7 + - scikit-learn==1.2.2 + - scipy==1.10.1 + - secretstorage==3.3.3 + - shap==0.41.0 + - slicer==0.0.7 + - smmap==5.0.0 + - sqlalchemy==2.0.9 + - sqlparse==0.4.3 + - strictyaml==1.7.3 + - tabulate==0.9.0 + - tensorboard==2.12.1 + - tensorboard-data-server==0.7.0 + - tensorboard-plugin-wit==1.8.1 + - termcolor==2.2.0 + - threadpoolctl==3.1.0 + - tqdm==4.65.0 + - tzdata==2023.3 + - websocket-client==1.5.1 + - werkzeug==2.2.3 + - zipp==3.15.0 +prefix: /anaconda/envs/monai-azureml-new From ff5c4f420a04b876ba903de84edfb06df97a06b4 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 11 Apr 2023 11:59:05 +0000 Subject: [PATCH 55/58] mypy fixes Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/auto_runner.py | 7 +++---- monai/apps/auto3dseg/azureml_auto_runner.py | 11 ++++------- monai/apps/auto3dseg/utils.py | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 7c8825ae3f..a25b499068 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -16,7 +16,7 @@ import warnings from copy import deepcopy from time import sleep -from typing import Any, cast +from typing import Any, Optional, cast import numpy as np import torch @@ -257,7 +257,6 @@ def __init__( self.ensemble = ensemble # last step, no need to check # intermediate variables - self.ensemble_method_name = "AlgoEnsembleBestByFold" self.set_training_params(training_params) self.set_prediction_params() self.set_analyze_params() @@ -286,7 +285,7 @@ def _set_up_data_src_cfg( input: dict[str, Any] | str | None = None, algos: dict | list | str | None = None, templates_path_or_url: str | None = None, - ): + ) -> None: """Sets up the AutoRunner data source config using the provided input. Args: @@ -590,7 +589,7 @@ def set_nni_search_space(self, search_space): self.search_space = search_space self.hpo_tasks = value_combinations - def set_image_save_transform(self, kwargs) -> SaveImage: + def set_image_save_transform(self, kwargs: Any) -> SaveImage: """ Set the ensemble output transform. diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index a2a04fde72..9fc46f4e1a 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -86,7 +86,7 @@ def _create_work_dir_and_data_src_cfg(self, data_src_cfg: dict[str, Any] | None else: pass - def set_image_save_transform(self, kwargs) -> SaveImage | None: + def set_image_save_transform(self, kwargs: Any) -> Any: """ Set the ensemble output transform if running in AzureML, otherwise do nothing. @@ -97,14 +97,11 @@ def set_image_save_transform(self, kwargs) -> SaveImage | None: """ if health_azure.utils.is_running_in_azure_ml(): return super().set_image_save_transform(kwargs) - else: - pass - def export_cache(self, **kwargs) -> None: + + def export_cache(self, **kwargs: Any) -> None: """ Export cache to the AzureML job working dir if running in AzureML, otherwise do nothing. """ if health_azure.utils.is_running_in_azure_ml(): - return super().export_cache(**kwargs) - else: - pass + super().export_cache(**kwargs) diff --git a/monai/apps/auto3dseg/utils.py b/monai/apps/auto3dseg/utils.py index d14207e12e..eb475ea811 100644 --- a/monai/apps/auto3dseg/utils.py +++ b/monai/apps/auto3dseg/utils.py @@ -83,7 +83,7 @@ def export_bundle_algo_history(history: list[dict[str, BundleAlgo]]) -> None: algo_to_pickle(algo, template_path=algo.template_path) -def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> health_azure.AzureRunInfo: +def submit_auto3dseg_module_to_azureml_if_needed(azure_cfg: dict[str, Any]) -> Any: """ Submit Auto3dSeg modules to run as AzureML jobs if the user has requested it. From 510b141f2aae0095827104bbb97ee02ca5d64ba9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Apr 2023 12:00:51 +0000 Subject: [PATCH 56/58] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/apps/auto3dseg/auto_runner.py | 2 +- monai/apps/auto3dseg/azureml_auto_runner.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/monai/apps/auto3dseg/auto_runner.py b/monai/apps/auto3dseg/auto_runner.py index 44f69a6521..b2da7d8230 100644 --- a/monai/apps/auto3dseg/auto_runner.py +++ b/monai/apps/auto3dseg/auto_runner.py @@ -16,7 +16,7 @@ import warnings from copy import deepcopy from time import sleep -from typing import Any, Optional, cast +from typing import Any, cast import numpy as np import torch diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 9fc46f4e1a..36eaeee762 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -16,7 +16,6 @@ from monai.apps.auto3dseg import AutoRunner from monai.apps.auto3dseg.utils import AZUREML_CONFIG_KEY, submit_auto3dseg_module_to_azureml_if_needed from monai.apps.utils import get_logger -from monai.transforms import SaveImage from monai.utils.module import optional_import logger = get_logger(module_name=__name__) From 643d713e5bab6453a5f8547f503184965e3a37f4 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Tue, 11 Apr 2023 12:10:08 +0000 Subject: [PATCH 57/58] Black fixes Signed-off-by: Peter Hessey --- monai/apps/auto3dseg/azureml_auto_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 36eaeee762..8cacf534c0 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -97,7 +97,6 @@ def set_image_save_transform(self, kwargs: Any) -> Any: if health_azure.utils.is_running_in_azure_ml(): return super().set_image_save_transform(kwargs) - def export_cache(self, **kwargs: Any) -> None: """ Export cache to the AzureML job working dir if running in AzureML, otherwise do nothing. From 4833e131f1d7c6575c6afd154cdce78ddf5c61b5 Mon Sep 17 00:00:00 2001 From: Peter Hessey Date: Thu, 13 Apr 2023 10:06:43 +0000 Subject: [PATCH 58/58] Minor fixes Signed-off-by: Peter Hessey --- environment-azureml.yml | 3 +-- monai/apps/auto3dseg/azureml_auto_runner.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/environment-azureml.yml b/environment-azureml.yml index 050837b560..aa4af1e07a 100644 --- a/environment-azureml.yml +++ b/environment-azureml.yml @@ -1,4 +1,4 @@ -name: monai-azureml-new +name: monai-azureml channels: - pytorch - nvidia @@ -238,4 +238,3 @@ dependencies: - websocket-client==1.5.1 - werkzeug==2.2.3 - zipp==3.15.0 -prefix: /anaconda/envs/monai-azureml-new diff --git a/monai/apps/auto3dseg/azureml_auto_runner.py b/monai/apps/auto3dseg/azureml_auto_runner.py index 8cacf534c0..aaafb7ea5f 100644 --- a/monai/apps/auto3dseg/azureml_auto_runner.py +++ b/monai/apps/auto3dseg/azureml_auto_runner.py @@ -47,6 +47,9 @@ def __init__( templates_path_or_url: str | None = None, **kwargs: Any, ): + if "work_dir" in kwargs: + raise ValueError("work_dir cannot be specified in AzureMLAutoRunner") + work_dir = "outputs" super().__init__( work_dir=work_dir,