From 718cb5eb74a08ff3cea654e3e1785fa64e151bbe Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 29 Nov 2023 13:18:00 -0500 Subject: [PATCH 01/66] port code towards config file-based training --- monai/main.py | 252 ++++++++++++++++++++------------------------------ 1 file changed, 101 insertions(+), 151 deletions(-) diff --git a/monai/main.py b/monai/main.py index dd87afb0..4d01e08a 100644 --- a/monai/main.py +++ b/monai/main.py @@ -25,14 +25,14 @@ # create a "model"-agnostic class with PL to use different models class Model(pl.LightningModule): - def __init__(self, args, data_root, net, loss_function, optimizer_class, exp_id=None, results_path=None): + def __init__(self, config, data_root, net, loss_function, optimizer_class, exp_id=None, results_path=None): super().__init__() - self.args = args + self.cfg = config self.save_hyperparameters(ignore=['net', 'loss_function']) self.root = data_root self.net = net - self.lr = args.learning_rate + self.lr = config["opt"]["lr"] self.loss_function = loss_function self.optimizer_class = optimizer_class self.save_exp_id = exp_id @@ -47,9 +47,8 @@ def __init__(self, args, data_root, net, loss_function, optimizer_class, exp_id= # which could be sub-optimal. # On the other hand, ivadomed used a patch-size that's heavily padded along the R-L direction so that # only the SC is in context. - self.spacing = (1.0, 1.0, 1.0) - self.voxel_cropping_size = self.inference_roi_size = tuple([int(i) for i in args.crop_size.split("x")]) - # self.inference_roi_size = tuple([int(i) for i in args.val_crop_size.split("x")]) + self.spacing = config["preprocessing"]["spacing"] + self.voxel_cropping_size = self.inference_roi_size = config["preprocessing"]["crop_pad_size"] # define post-processing transforms for validation, nothing fancy just making sure that it's a tensor (default) self.val_post_pred = Compose([EnsureType()]) @@ -85,7 +84,7 @@ def forward(self, x): # -------------------------------- def prepare_data(self): # set deterministic training for reproducibility - set_determinism(seed=self.args.seed) + set_determinism(seed=self.cfg["seed"]) # define training and validation transforms transforms_train = train_transforms( @@ -95,7 +94,9 @@ def prepare_data(self): transforms_val = val_transforms(crop_size=self.inference_roi_size, lbl_key='label') # load the dataset - dataset = os.path.join(self.root, f"dataset_{self.args.contrast}_{self.args.label_type}_seed15.json") + dataset = os.path.join(self.root, + f"dataset_{self.cfg['dataset']['contrast']}_{self.cfg['dataset']['label_type']}_seed{self.cfg['seed']}.json" + ) logger.info(f"Loading dataset: {dataset}") train_files = load_decathlon_datalist(dataset, True, "train") val_files = load_decathlon_datalist(dataset, True, "validation") @@ -129,7 +130,7 @@ def prepare_data(self): # DATA LOADERS # -------------------------------- def train_dataloader(self): - return DataLoader(self.train_ds, batch_size=self.args.batch_size, shuffle=True, num_workers=16, + return DataLoader(self.train_ds, batch_size=self.cfg["opt"]["batch_size"], shuffle=True, num_workers=16, pin_memory=True, persistent_workers=True) def val_dataloader(self): @@ -144,13 +145,13 @@ def test_dataloader(self): # OPTIMIZATION # -------------------------------- def configure_optimizers(self): - if self.args.optimizer == "sgd": + if self.cfg["opt"]["name"] == "sgd": optimizer = self.optimizer_class(self.parameters(), lr=self.lr, momentum=0.99, weight_decay=3e-5, nesterov=True) else: optimizer = self.optimizer_class(self.parameters(), lr=self.lr) # scheduler = PolyLRScheduler(optimizer, self.lr, max_steps=self.args.max_epochs) # NOTE: ivadomed using CosineAnnealingLR with T_max = 200 - scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=self.args.max_epochs) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=self.cfg["opt"]["max_epochs"]) return [optimizer], [scheduler] @@ -169,7 +170,7 @@ def training_step(self, batch, batch_idx): output = self.forward(inputs) # logits # print(f"labels.shape: {labels.shape} \t output.shape: {output.shape}") - if self.args.model == "nnunet" and self.args.enable_DS: + if args.model == "nnunet" and self.cfg['model'][args.model]["enable_deep_supervision"]: # calculate dice loss for each output loss, train_soft_dice = 0.0, 0.0 @@ -264,8 +265,7 @@ def validation_step(self, batch, batch_idx): outputs = sliding_window_inference(inputs, self.inference_roi_size, mode="gaussian", sw_batch_size=4, predictor=self.forward, overlap=0.5,) # outputs shape: (B, C, ) - - if self.args.model == "nnunet" and self.args.enable_DS: + if args.model == "nnunet" and self.cfg['model'][args.model]["enable_deep_supervision"]: # we only need the output with the highest resolution outputs = outputs[0] @@ -381,7 +381,7 @@ def test_step(self, batch, batch_idx): pred, label = post_test_out[0]['pred'].cpu(), post_test_out[0]['label'].cpu() # save the prediction and label - if self.args.save_test_preds: + if self.cfg["save_test_preds"]: subject_name = (batch["image_meta_dict"]["filename_or_obj"][0]).split("/")[-1].replace(".nii.gz", "") logger.info(f"Saving subject: {subject_name}") @@ -454,43 +454,21 @@ def on_test_epoch_end(self): # MAIN # -------------------------------- def main(args): + + # load config file + with open(args.config, "r") as f: + config = yaml.load(f, Loader=yaml.FullLoader) + # Setting the seed - pl.seed_everything(args.seed, workers=True) - - # ====================================================================================================== - # Define plans json taken from nnUNet_preprocessed folder - # ====================================================================================================== - nnunet_plans = { - "UNet_class_name": "PlainConvUNet", - "UNet_base_num_features": args.init_filters, - "n_conv_per_stage_encoder": [2, 2, 2, 2, 2, 2], - "n_conv_per_stage_decoder": [2, 2, 2, 2, 2], - "pool_op_kernel_sizes": [ - [1, 1, 1], - [2, 2, 2], - [2, 2, 2], - [2, 2, 2], - [2, 2, 2], - [1, 2, 2] - ], - "conv_kernel_sizes": [ - [3, 3, 3], - [3, 3, 3], - [3, 3, 3], - [3, 3, 3], - [3, 3, 3], - [3, 3, 3] - ], - "unet_max_num_features": 320, - } + pl.seed_everything(config["seed"], workers=True) # define root path for finding datalists - dataset_root = "/home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/datalists/spine-generic/seed15" + dataset_root = config["dataset"]["root_dir"] # define optimizer - if args.optimizer in ["adam", "Adam"]: + if config["opt"]["name"] == "adam": optimizer_class = torch.optim.Adam - elif args.optimizer in ["SGD", "sgd"]: + elif config["opt"]["name"] == "sgd": optimizer_class = torch.optim.SGD # define models @@ -516,23 +494,47 @@ def main(args): f"_fs={args.feature_size}_hs={args.hidden_size}_mlpd={args.mlp_dim}_nh={args.num_heads}" \ f"_CSAdiceL_nspv={args.num_samples_per_volume}_bs={args.batch_size}_{img_size}" \ - elif args.model in ["nnunet"]: - if args.enable_DS: - logger.info(f" Using nnUNet model WITH deep supervision! ") + elif args.model == "nnunet": + + if config["model"]["nnunet"]["enable_deep_supervision"]: + logger.info(f"Using nnUNet model WITH deep supervision ...") else: - logger.info(f" Using nnUNet model WITHOUT deep supervision! ") + logger.info(f"Using nnUNet model WITHOUT deep supervision ...") + + logger.info("Defining plans for nnUNet model ...") + # ========================================================================================= + # Define plans json taken from nnUNet_preprocessed folder + # ========================================================================================= + nnunet_plans = { + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": config["model"]["nnunet"]["base_num_features"], + "n_conv_per_stage_encoder": config["model"]["nnunet"]["n_conv_per_stage_encoder"], + "n_conv_per_stage_decoder": config["model"]["nnunet"]["n_conv_per_stage_decoder"], + "pool_op_kernel_sizes": config["model"]["nnunet"]["pool_op_kernel_sizes"], + "conv_kernel_sizes": [ + [3, 3, 3], + [3, 3, 3], + [3, 3, 3], + [3, 3, 3], + [3, 3, 3], + [3, 3, 3] + ], + "unet_max_num_features": config["model"]["nnunet"]["max_num_features"], + } # define model - net = create_nnunet_from_plans(plans=nnunet_plans, num_input_channels=1, num_classes=1, deep_supervision=args.enable_DS) - patch_size = "64x192x320" - save_exp_id =f"{args.model}_{args.contrast}_{args.label_type}_nf={args.init_filters}" \ - f"_opt={args.optimizer}_lr={args.learning_rate}" \ - f"_AdapW" \ - f"_bs={args.batch_size}_{patch_size}" - # save_exp_id =f"{args.model}_{args.contrast}_{args.label_type}_nf={args.init_filters}" \ - # f"_opt={args.optimizer}_lr={args.learning_rate}" \ - # f"_DiceL" \ - # f"_bs={args.batch_size}_{patch_size}" + net = create_nnunet_from_plans(plans=nnunet_plans, num_input_channels=1, num_classes=1, + deep_supervision=config["model"]["nnunet"]["enable_deep_supervision"]) + # variable for saving patch size in the experiment id (same as crop_pad_size) + patch_size = f"{config['preprocessing']['crop_pad_size'][0]}x" \ + f"{config['preprocessing']['crop_pad_size'][1]}x" \ + f"{config['preprocessing']['crop_pad_size'][2]}" + # save experiment id + save_exp_id = f"{args.model}_seed={config['seed']}_" \ + f"{config['dataset']['contrast']}_{config['dataset']['label_type']}_" \ + f"nf={config['model']['nnunet']['base_num_features']}_" \ + f"opt={config['opt']['name']}_lr={config['opt']['lr']}_AdapW_" \ + f"bs={config['opt']['batch_size']}_{patch_size}" \ if args.debug: save_exp_id = f"DEBUG_{save_exp_id}" @@ -543,7 +545,8 @@ def main(args): save_exp_id = f"{save_exp_id}_{timestamp}" # save output to a log file - logger.add(os.path.join(args.save_path, f"{save_exp_id}", "logs.txt"), rotation="10 MB", level="INFO") + logger.add(os.path.join(config["directories"]["models_dir"], f"{save_exp_id}", "logs.txt"), rotation="10 MB", level="INFO") + # define loss function # loss_func = SoftDiceLoss(p=1, smooth=1.0) @@ -554,27 +557,27 @@ def main(args): logger.info(f"Using AdapWingLoss with theta={loss_func.theta}, omega={loss_func.omega}, alpha={loss_func.alpha}, epsilon={loss_func.epsilon}!") # define callbacks - # early_stopping = pl.callbacks.EarlyStopping(monitor="val_soft_dice", min_delta=0.00, patience=args.patience, - # verbose=False, mode="max") - early_stopping = pl.callbacks.EarlyStopping(monitor="val_loss", min_delta=0.00, patience=args.patience, - verbose=False, mode="min") + early_stopping = pl.callbacks.EarlyStopping( + monitor="val_loss", min_delta=0.00, + patience=config["opt"]["early_stopping_patience"], + verbose=False, mode="min") lr_monitor = pl.callbacks.LearningRateMonitor(logging_interval='epoch') - + # training from scratch if not args.continue_from_checkpoint: # to save the best model on validation - save_path = os.path.join(args.save_path, f"{save_exp_id}") + save_path = os.path.join(config["directories"]["models_dir"], f"{save_exp_id}") if not os.path.exists(save_path): os.makedirs(save_path, exist_ok=True) # to save the results/model predictions - results_path = os.path.join(args.results_dir, f"{save_exp_id}") + results_path = os.path.join(config["directories"]["results_dir"], f"{save_exp_id}") if not os.path.exists(results_path): os.makedirs(results_path, exist_ok=True) # i.e. train by loading weights from scratch - pl_model = Model(args, data_root=dataset_root, + pl_model = Model(config, data_root=dataset_root, optimizer_class=optimizer_class, loss_function=loss_func, net=net, exp_id=save_exp_id, results_path=results_path) @@ -591,15 +594,14 @@ def main(args): logger.info(f" Starting training from scratch! ") # wandb logger - grp = f"monai_ivado_{args.model}" if args.model == "unet" else f"monai_{args.model}" exp_logger = pl.loggers.WandbLogger( name=save_exp_id, - save_dir=args.save_path, - group=grp, + save_dir=config["directories"]["models_dir"], + group=config["dataset"]["name"], log_model=True, # save best model using checkpoint callback project='contrast-agnostic', entity='naga-karthik', - config=args) + config=config) # Saving training script to wandb wandb.save("main.py") @@ -607,14 +609,14 @@ def main(args): # initialise Lightning's trainer. trainer = pl.Trainer( - devices=1, accelerator="gpu", # strategy="ddp", + devices=1, accelerator="gpu", logger=exp_logger, - callbacks=[checkpoint_callback_loss, checkpoint_callback_dice, lr_monitor, early_stopping], - check_val_every_n_epoch=args.check_val_every_n_epochs, - max_epochs=args.max_epochs, - precision=32, # TODO: see if 16-bit precision is stable + callbacks=[checkpoint_callback_loss, lr_monitor, early_stopping], + check_val_every_n_epoch=config["opt"]["check_val_every_n_epochs"], + max_epochs=config["opt"]["max_epochs"], + precision=32, # deterministic=True, - enable_progress_bar=args.enable_progress_bar,) + enable_progress_bar=False) # profiler="simple",) # to profile the training time taken for each step # Train! @@ -625,19 +627,19 @@ def main(args): logger.info(f" Resuming training from the latest checkpoint! ") # check if wandb run folder is provided to resume using the same run - if args.wandb_run_folder is None: + if config["directories"]["wandb_run_folder"] is None: raise ValueError("Please provide the wandb run folder to resume training using the same run on WandB!") else: - wandb_run_folder = os.path.basename(args.wandb_run_folder) + wandb_run_folder = os.path.basename(config["directories"]["wandb_run_folder"]) wandb_run_id = wandb_run_folder.split("-")[-1] - save_exp_id = args.save_path - save_path = os.path.dirname(args.save_path) + save_exp_id = config["directories"]["models_dir"] + save_path = os.path.dirname(config["directories"]["models_dir"]) logger.info(f"save_path: {save_path}") - results_path = args.results_dir + results_path = config["directories"]["results_dir"] - # i.e. train by loading weights from scratch - pl_model = Model(args, data_root=dataset_root, + # i.e. train by loading existing weights + pl_model = Model(config, data_root=dataset_root, optimizer_class=optimizer_class, loss_function=loss_func, net=net, exp_id=save_exp_id, results_path=results_path) @@ -655,7 +657,7 @@ def main(args): grp = f"monai_ivado_{args.model}" if args.model == "unet" else f"monai_{args.model}" exp_logger = pl.loggers.WandbLogger( save_dir=save_path, - group=grp, + group=config["dataset"]["name"], log_model=True, # save best model using checkpoint callback project='contrast-agnostic', entity='naga-karthik', @@ -664,13 +666,13 @@ def main(args): # initialise Lightning's trainer. trainer = pl.Trainer( - devices=1, accelerator="gpu", # strategy="ddp", + devices=1, accelerator="gpu", logger=exp_logger, - callbacks=[checkpoint_callback_loss, checkpoint_callback_dice, lr_monitor, early_stopping], - check_val_every_n_epoch=args.check_val_every_n_epochs, - max_epochs=args.max_epochs, + callbacks=[checkpoint_callback_loss, lr_monitor, early_stopping], + check_val_every_n_epoch=config["opt"]["check_val_every_n_epochs"], + max_epochs=config["opt"]["max_epochs"], precision=32, - enable_progress_bar=args.enable_progress_bar,) + enable_progress_bar=False) # profiler="simple",) # to profile the training time taken for each step # Train! @@ -688,8 +690,12 @@ def main(args): with open(os.path.join(results_path, 'test_metrics.txt'), 'a') as f: print('\n-------------- Test Metrics ----------------', file=f) print(f"\nSeed Used: {args.seed}", file=f) - print(f"\ninitf={args.init_filters}_lr={args.learning_rate}_bs={args.batch_size}_{timestamp}", file=f) - print(f"\npatch_size={pl_model.voxel_cropping_size}", file=f) + print(f"{args.model}_seed={config['seed']}_" \ + f"{config['dataset']['contrast']}_{config['dataset']['label_type']}_" \ + f"nf={config['model']['nnunet']['base_num_features']}_" \ + f"opt={config['opt']['name']}_lr={config['opt']['lr']}_AdapW_" \ + f"bs={config['opt']['batch_size']}_{patch_size}" \ + f"_{timestamp}", file=f) print('\n-------------- Test Hard Dice Scores ----------------', file=f) print("Hard Dice --> Mean: %0.3f, Std: %0.3f" % (pl_model.avg_test_dice_hard, pl_model.std_test_dice_hard), file=f) @@ -707,61 +713,5 @@ def main(args): if __name__ == "__main__": - - parser = argparse.ArgumentParser(description='Script for training custom models for SCI Lesion Segmentation.') - # Arguments for model, data, and training and saving - parser.add_argument('-m', '--model', choices=['unetr', 'nnunet'], - default='unet', type=str, help='Model type to be used') - parser.add_argument('--enable_DS', default=False, action='store_true', help='Enable Deep Supervision') - # dataset - # define args for cropping size - parser.add_argument('-crop', '--crop_size', type=str, default="64x192x320", - help='Center crop size for training/validation. Values correspond to R-L, A-P, I-S axes' - 'of the image after 1mm isotropic resampling. Default: 64x192x320') - parser.add_argument("--contrast", default="t2w", type=str, help="Contrast to use for training", - choices=["t1w", "t2w", "t2star", "mton", "mtoff", "dwi", "all"]) - parser.add_argument('--label-type', default='soft', type=str, help="Type of labels to use for training", - choices=['hard', 'soft']) - - # unet model - parser.add_argument('-initf', '--init_filters', default=16, type=int, help="Number of Filters in Init Layer") - - # unetr model - parser.add_argument('-fs', '--feature_size', default=16, type=int, help="Feature Size") - parser.add_argument('-hs', '--hidden_size', default=768, type=int, help='Dimensionality of hidden embeddings') - parser.add_argument('-mlpd', '--mlp_dim', default=2048, type=int, help='Dimensionality of MLP layer') - parser.add_argument('-nh', '--num_heads', default=12, type=int, help='Number of heads in Multi-head Attention') - - # optimizations - parser.add_argument('-me', '--max_epochs', default=1000, type=int, help='Number of epochs for the training process') - parser.add_argument('-bs', '--batch_size', default=2, type=int, help='Batch size of the training and validation processes') - parser.add_argument('-opt', '--optimizer', - choices=['adam', 'Adam', 'SGD', 'sgd'], - default='adam', type=str, help='Optimizer to use') - parser.add_argument('-lr', '--learning_rate', default=1e-4, type=float, help='Learning rate for training the model') - parser.add_argument('-pat', '--patience', default=25, type=int, - help='number of validation steps (val_every_n_iters) to wait before early stopping') - # NOTE: patience is acutally until (patience * check_val_every_n_epochs) epochs - parser.add_argument('-epb', '--enable_progress_bar', default=False, action='store_true', - help='by default is disabled since it doesnt work in colab') - parser.add_argument('-cve', '--check_val_every_n_epochs', default=1, type=int, help='num of epochs to wait before validation') - # saving - parser.add_argument('-sp', '--save_path', - default=f"/home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/saved_models", - type=str, help='Path to the saved models directory') - parser.add_argument('-se', '--seed', default=42, type=int, help='Set seeds for reproducibility') - parser.add_argument('-debug', default=False, action='store_true', help='if true, results are not logged to wandb') - parser.add_argument('-stp', '--save_test_preds', default=False, action='store_true', - help='if true, test predictions are saved in `save_path`') - parser.add_argument('-c', '--continue_from_checkpoint', default=False, action='store_true', - help='Load model from checkpoint and continue training') - parser.add_argument('-wdb-run', '--wandb-run-folder', default=None, type=str, help='Path to the wandb run folder') - # testing - parser.add_argument('-rd', '--results_dir', - default=f"/home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/results", - type=str, help='Path to the model prediction results directory') - - - args = parser.parse_args() - + args = get_args() main(args) \ No newline at end of file From 35d7093154df6c5877d83786d79e9492e05ff7ad Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 29 Nov 2023 13:27:34 -0500 Subject: [PATCH 02/66] fix/update imports --- monai/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/monai/main.py b/monai/main.py index 4d01e08a..7ab824de 100644 --- a/monai/main.py +++ b/monai/main.py @@ -2,6 +2,7 @@ import argparse from datetime import datetime from loguru import logger +import yaml import numpy as np import wandb @@ -10,16 +11,15 @@ import torch.nn.functional as F import matplotlib.pyplot as plt -from utils import precision_score, recall_score, dice_score, \ - PolyLRScheduler, plot_slices, check_empty_patch -from losses import SoftDiceLoss, AdapWingLoss +from utils import dice_score, PolyLRScheduler, plot_slices, check_empty_patch +from losses import AdapWingLoss from transforms import train_transforms, val_transforms from models import create_nnunet_from_plans from monai.utils import set_determinism from monai.inferers import sliding_window_inference from monai.networks.nets import UNETR -from monai.data import (DataLoader, Dataset, CacheDataset, load_decathlon_datalist, decollate_batch) +from monai.data import (DataLoader, CacheDataset, load_decathlon_datalist, decollate_batch) from monai.transforms import (Compose, EnsureType, EnsureTyped, Invertd, SaveImage) From 893106a0bcc5840118fb348951a27da4b62ef02f Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 29 Nov 2023 13:28:24 -0500 Subject: [PATCH 03/66] create function for argument parser --- monai/main.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/monai/main.py b/monai/main.py index 7ab824de..b9e04dbb 100644 --- a/monai/main.py +++ b/monai/main.py @@ -23,6 +23,24 @@ from monai.transforms import (Compose, EnsureType, EnsureTyped, Invertd, SaveImage) +def get_args(): + parser = argparse.ArgumentParser(description='Script for training contrast-agnositc SC segmentation model.') + + # arguments for model + parser.add_argument('-m', '--model', choices=['unetr', 'nnunet'], default='nnunet', type=str, + help='Model type to be used. Currently only supports nnUNet.') + # path to the config file + parser.add_argument("--config", type=str, default="./config.json", + help="Path to the config file containing all training details.") + # saving + parser.add_argument('--debug', default=False, action='store_true', help='if true, results are not logged to wandb') + parser.add_argument('-c', '--continue_from_checkpoint', default=False, action='store_true', + help='Load model from checkpoint and continue training') + args = parser.parse_args() + + return args + + # create a "model"-agnostic class with PL to use different models class Model(pl.LightningModule): def __init__(self, config, data_root, net, loss_function, optimizer_class, exp_id=None, results_path=None): From c03a962c0b22bb0765ede93313b50ca951f69279 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 29 Nov 2023 13:31:30 -0500 Subject: [PATCH 04/66] minor code improvements --- monai/main.py | 65 +++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 44 deletions(-) diff --git a/monai/main.py b/monai/main.py index b9e04dbb..ed742b60 100644 --- a/monai/main.py +++ b/monai/main.py @@ -69,8 +69,7 @@ def __init__(self, config, data_root, net, loss_function, optimizer_class, exp_i self.voxel_cropping_size = self.inference_roi_size = config["preprocessing"]["crop_pad_size"] # define post-processing transforms for validation, nothing fancy just making sure that it's a tensor (default) - self.val_post_pred = Compose([EnsureType()]) - self.val_post_label = Compose([EnsureType()]) + self.val_post_pred = self.val_post_label = Compose([EnsureType()]) # define evaluation metric self.soft_dice_metric = dice_score @@ -350,7 +349,6 @@ def on_validation_epoch_end(self): f"\nAverage Soft Dice (VAL): {mean_val_soft_dice:.4f}" f"\nAverage Hard Dice (VAL): {mean_val_hard_dice:.4f}" f"\nAverage AdapWing Loss (VAL): {mean_val_loss:.4f}" - # f"\nBest Average Soft Dice: {self.best_val_dice:.4f} at Epoch: {self.best_val_epoch}" f"\nBest Average AdapWing Loss: {self.best_val_loss:.4f} at Epoch: {self.best_val_epoch}" f"\n----------------------------------------------------") @@ -378,12 +376,10 @@ def test_step(self, batch, batch_idx): test_input = batch["image"] # print(batch["label_meta_dict"]["filename_or_obj"][0]) - # print(f"test_input.shape: {test_input.shape} \t test_label.shape: {test_label.shape}") batch["pred"] = sliding_window_inference(test_input, self.inference_roi_size, sw_batch_size=4, predictor=self.forward, overlap=0.5) - # print(f"batch['pred'].shape: {batch['pred'].shape}") - if self.args.model == "nnunet" and self.args.enable_DS: + if args.model == "nnunet" and self.cfg['model'][args.model]["enable_deep_supervision"]: # we only need the output with the highest resolution batch["pred"] = batch["pred"][0] @@ -420,7 +416,8 @@ def test_step(self, batch, batch_idx): # NOTE: Important point from the SoftSeg paper - binarize predictions before computing metrics - # calculate all metrics here + # calculate soft and hard dice here (for quick overview), other metrics can be computed from + # the saved predictions using ANIMA # 1. Dice Score test_soft_dice = self.soft_dice_metric(pred, label) @@ -430,16 +427,10 @@ def test_step(self, batch, batch_idx): # 1.1 Hard Dice Score test_hard_dice = self.soft_dice_metric(pred.numpy(), label.numpy()) - # 2. Precision Score - test_precision = precision_score(pred.numpy(), label.numpy()) - # 3. Recall Score - test_recall = recall_score(pred.numpy(), label.numpy()) metrics_dict = { "test_hard_dice": test_hard_dice, "test_soft_dice": test_soft_dice, - "test_precision": test_precision, - "test_recall": test_recall, } self.test_step_outputs.append(metrics_dict) @@ -451,19 +442,13 @@ def on_test_epoch_end(self): np.stack([x["test_hard_dice"] for x in self.test_step_outputs]).std() avg_soft_dice_test, std_soft_dice_test = np.stack([x["test_soft_dice"] for x in self.test_step_outputs]).mean(), \ np.stack([x["test_soft_dice"] for x in self.test_step_outputs]).std() - avg_precision_test = np.stack([x["test_precision"] for x in self.test_step_outputs]).mean() - avg_recall_test = np.stack([x["test_recall"] for x in self.test_step_outputs]).mean() logger.info(f"Test (Soft) Dice: {avg_soft_dice_test}") logger.info(f"Test (Hard) Dice: {avg_hard_dice_test}") - logger.info(f"Test Precision Score: {avg_precision_test}") - logger.info(f"Test Recall Score: {avg_recall_test}") self.avg_test_dice, self.std_test_dice = avg_soft_dice_test, std_soft_dice_test self.avg_test_dice_hard, self.std_test_dice_hard = avg_hard_dice_test, std_hard_dice_test - self.avg_test_precision = avg_precision_test - self.avg_test_recall = avg_recall_test - + # free up memory self.test_step_outputs.clear() @@ -491,6 +476,7 @@ def main(args): # define models if args.model in ["unetr"]: + # TODO: update if ever using UNETR # define image size to be fed to the model img_size = (160, 224, 96) @@ -557,22 +543,21 @@ def main(args): if args.debug: save_exp_id = f"DEBUG_{save_exp_id}" - - # TODO: move this inside the for loop when using more folds timestamp = datetime.now().strftime(f"%Y%m%d-%H%M") # prints in YYYYMMDD-HHMMSS format save_exp_id = f"{save_exp_id}_{timestamp}" # save output to a log file logger.add(os.path.join(config["directories"]["models_dir"], f"{save_exp_id}", "logs.txt"), rotation="10 MB", level="INFO") + # save config file to the output folder + with open(os.path.join(config["directories"]["models_dir"], f"{save_exp_id}", "config.yaml"), "w") as f: + yaml.dump(config, f) # define loss function - # loss_func = SoftDiceLoss(p=1, smooth=1.0) - # logger.info(f"Using SoftDiceLoss with p={loss_func.p}, smooth={loss_func.smooth}!") loss_func = AdapWingLoss(theta=0.5, omega=8, alpha=2.1, epsilon=1, reduction="sum") # NOTE: tried increasing omega and decreasing epsilon but results marginally worse than the above # loss_func = AdapWingLoss(theta=0.5, omega=12, alpha=2.1, epsilon=0.5, reduction="sum") - logger.info(f"Using AdapWingLoss with theta={loss_func.theta}, omega={loss_func.omega}, alpha={loss_func.alpha}, epsilon={loss_func.epsilon}!") + logger.info(f"Using AdapWingLoss with theta={loss_func.theta}, omega={loss_func.omega}, alpha={loss_func.alpha}, epsilon={loss_func.epsilon} ...") # define callbacks early_stopping = pl.callbacks.EarlyStopping( @@ -602,15 +587,15 @@ def main(args): # saving the best model based on validation loss logger.info(f"Saving best model to {save_path}!") checkpoint_callback_loss = pl.callbacks.ModelCheckpoint( - dirpath=save_path, filename='best_model_loss', monitor='val_loss', + dirpath=save_path, filename='best_model', monitor='val_loss', save_top_k=1, mode="min", save_last=True, save_weights_only=False) - # saving the best model based on soft validation dice score - checkpoint_callback_dice = pl.callbacks.ModelCheckpoint( - dirpath=save_path, filename='best_model_dice', monitor='val_soft_dice', - save_top_k=1, mode="max", save_last=False, save_weights_only=True) + # # saving the best model based on soft validation dice score + # checkpoint_callback_dice = pl.callbacks.ModelCheckpoint( + # dirpath=save_path, filename='best_model_dice', monitor='val_soft_dice', + # save_top_k=1, mode="max", save_last=False, save_weights_only=True) - logger.info(f" Starting training from scratch! ") + logger.info(f"Starting training from scratch ...") # wandb logger exp_logger = pl.loggers.WandbLogger( name=save_exp_id, @@ -663,16 +648,15 @@ def main(args): # saving the best model based on validation CSA loss checkpoint_callback_loss = pl.callbacks.ModelCheckpoint( - dirpath=save_exp_id, filename='best_model_loss', monitor='val_loss', + dirpath=save_exp_id, filename='best_model', monitor='val_loss', save_top_k=1, mode="min", save_last=True, save_weights_only=True) - # saving the best model based on soft validation dice score - checkpoint_callback_dice = pl.callbacks.ModelCheckpoint( - dirpath=save_exp_id, filename='best_model_dice', monitor='val_soft_dice', - save_top_k=1, mode="max", save_last=False, save_weights_only=True) + # # saving the best model based on soft validation dice score + # checkpoint_callback_dice = pl.callbacks.ModelCheckpoint( + # dirpath=save_exp_id, filename='best_model_dice', monitor='val_soft_dice', + # save_top_k=1, mode="max", save_last=False, save_weights_only=True) # wandb logger - grp = f"monai_ivado_{args.model}" if args.model == "unet" else f"monai_{args.model}" exp_logger = pl.loggers.WandbLogger( save_dir=save_path, group=config["dataset"]["name"], @@ -707,7 +691,6 @@ def main(args): # TODO: Figure out saving test metrics to a file with open(os.path.join(results_path, 'test_metrics.txt'), 'a') as f: print('\n-------------- Test Metrics ----------------', file=f) - print(f"\nSeed Used: {args.seed}", file=f) print(f"{args.model}_seed={config['seed']}_" \ f"{config['dataset']['contrast']}_{config['dataset']['label_type']}_" \ f"nf={config['model']['nnunet']['base_num_features']}_" \ @@ -721,12 +704,6 @@ def main(args): print('\n-------------- Test Soft Dice Scores ----------------', file=f) print("Soft Dice --> Mean: %0.3f, Std: %0.3f" % (pl_model.avg_test_dice, pl_model.std_test_dice), file=f) - print('\n-------------- Test Precision Scores ----------------', file=f) - print("Precision --> Mean: %0.3f" % (pl_model.avg_test_precision), file=f) - - print('\n-------------- Test Recall Scores -------------------', file=f) - print("Recall --> Mean: %0.3f" % (pl_model.avg_test_recall), file=f) - print('-------------------------------------------------------', file=f) From 28e99cc32f47fa01e707f5b9ea5d7dde0d40523d Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 29 Nov 2023 13:33:52 -0500 Subject: [PATCH 05/66] add configs folder with soft_all config yaml --- configs/train_soft_all.yaml | 63 +++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 configs/train_soft_all.yaml diff --git a/configs/train_soft_all.yaml b/configs/train_soft_all.yaml new file mode 100644 index 00000000..ef7e9516 --- /dev/null +++ b/configs/train_soft_all.yaml @@ -0,0 +1,63 @@ +seed: 15 +save_test_preds: True + +directories: + # Path to the saved models directory + models_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/saved_models + # Path to the saved results directory + results_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/results + # Path to the saved wandb logs directory + # if None, starts training from scratch. Otherwise, resumes training from the specified wandb run folder + wandb_run_folder: None + +dataset: + # Dataset name (will be used as "group_name" for wandb logging) + name: spine-generic + # Path to the dataset directory containing all datalists (.json files) + root_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/datalists/spine-generic/seed15 + # Type of contrast to be used for training. "all" corresponds to training on all contrasts + contrast: all # choices: ["t1w", "t2w", "t2star", "mton", "mtoff", "dwi", "all"] + # Type of label to be used for training. + label_type: soft # choices: ["hard", "soft"] + +preprocessing: + # Online resampling of images to the specified spacing. + spacing: [1.0, 1.0, 1.0] + # Center crop/pad images to the specified size. (NOTE: done after resampling) + # values correspond to R-L, A-P, I-S axes of the image after 1mm isotropic resampling. + crop_pad_size: [64, 192, 320] + +opt: + name: adam + lr: 0.0001 + max_epochs: 200 + batch_size: 2 + # Interval between validation checks in epochs + check_val_every_n_epochs: 10 + # Early stopping patience (this is until patience * check_val_every_n_epochs) + early_stopping_patience: 20 + + +model: + # Model architecture to be used for training (also to be specified as args in the command line) + nnunet: + # NOTE: these info are typically taken from nnUNetPlans.json (if an nnUNet model is trained) + base_num_features: 8 + max_num_features: 128 + n_conv_per_stage_encoder: [2, 2, 2, 2, 2, 2] + n_conv_per_stage_decoder: [2, 2, 2, 2, 2] + pool_op_kernel_sizes: [ + [1, 1, 1], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + [1, 2, 2] + ] + enable_deep_supervision: True + + unetr: + feature_size: 16 + hidden_size: 768 # dimensionality of hidden embeddings + mlp_dim: 2048 # dimensionality of the MLPs + num_heads: 12 # number of heads in multi-head Attention \ No newline at end of file From 1382696843e2fbf1602225dc13696f0ee66d3aa3 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 24 Nov 2023 10:56:50 -0500 Subject: [PATCH 06/66] add citation bibtex info --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 318a1b68..e7969671 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,18 @@ Official repository for contrast-agnostic spinal cord segmentation project using This repo contains all the code for data preprocessing, training and running inference on other datasets. The code is mainly based on [Spinal Cord Toolbox](https://spinalcordtoolbox.com) and [MONAI](https://github.com/Project-MONAI/MONAI) (PyTorch). +If you find this work and code useful for your research, please cite our paper: + +``` +@article{bedard2023towards, + title={Towards contrast-agnostic soft segmentation of the spinal cord}, + author={B{\'e}dard, Sandrine and Enamundram, Naga Karthik and Tsagkas, Charidimos and Pravat{\`a}, Emanuele and Granziera, Cristina and Smith, Andrew and Weber II, Kenneth Arnold and Cohen-Adad, Julien}, + journal={arXiv preprint arXiv:2310.15402}, + year={2023} + url={https://arxiv.org/abs/2310.15402} +} +``` + ## Table of contents * [1. Main Dependencies](#1-main-dependencies) * [2. Dataset](#2-dataset) From b1b37c3c507f804a78a97f98807e648f5ad97a6e Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 24 Nov 2023 10:57:12 -0500 Subject: [PATCH 07/66] add arxiv badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index e7969671..18ac3cbb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # Towards Contrast-agnostic Soft Segmentation of the Spinal Cord + +[![arXiv](https://img.shields.io/badge/arXiv-2310.15402-b31b1b.svg)](https://arxiv.org/abs/2310.15402) + Official repository for contrast-agnostic spinal cord segmentation project using SoftSeg. This repo contains all the code for data preprocessing, training and running inference on other datasets. The code is mainly based on [Spinal Cord Toolbox](https://spinalcordtoolbox.com) and [MONAI](https://github.com/Project-MONAI/MONAI) (PyTorch). From 44b7b42e4a6feb9a5a8118330608638104674f3b Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 24 Nov 2023 10:59:36 -0500 Subject: [PATCH 08/66] minor edit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18ac3cbb..c25a46b3 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Official repository for contrast-agnostic spinal cord segmentation project using This repo contains all the code for data preprocessing, training and running inference on other datasets. The code is mainly based on [Spinal Cord Toolbox](https://spinalcordtoolbox.com) and [MONAI](https://github.com/Project-MONAI/MONAI) (PyTorch). -If you find this work and code useful for your research, please cite our paper: +**CITATION INFO**: If you find this work and/or code useful for your research, please cite our paper: ``` @article{bedard2023towards, From c8290908a42fe7b2bc88fbf5272e3ca307c0fabb Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Thu, 21 Dec 2023 11:48:46 -0500 Subject: [PATCH 09/66] Fixed typo --- monai/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/monai/README.md b/monai/README.md index 22941c3d..7581b8de 100644 --- a/monai/README.md +++ b/monai/README.md @@ -22,7 +22,6 @@ conda activate venv_monai For CPU-based inference: ```bash -```bash pip install -r requirements_inference.txt ``` From ace417a821190b6e516845f158e63aef469f4242 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Thu, 21 Dec 2023 11:52:53 -0500 Subject: [PATCH 10/66] Fixed wrong file name #92 --- monai/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/README.md b/monai/README.md index 7581b8de..67e5459e 100644 --- a/monai/README.md +++ b/monai/README.md @@ -22,7 +22,7 @@ conda activate venv_monai For CPU-based inference: ```bash -pip install -r requirements_inference.txt +pip install -r requirements_inference_cpu.txt ``` For GPU-based inference: From 1de1e8ae6592523d0b3fd8eb0b38842c8472c857 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 21 Dec 2023 12:29:04 -0500 Subject: [PATCH 11/66] clarify help for checkpoint folder --- monai/run_inference_single_image.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 7d08f377..04271f8f 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -60,7 +60,8 @@ def get_parser(): parser.add_argument("--path-img", type=str, required=True, help="Path to the image to run inference on") - parser.add_argument("--chkp-path", type=str, required=True, help="Path to the checkpoint folder") + parser.add_argument("--chkp-path", type=str, required=True, + help="Path to the checkpoint folder. This folder should contain a file named 'best_model_loss.ckpt") parser.add_argument("--path-out", type=str, required=True, help="Path to the output folder where to store the predictions and associated metrics") parser.add_argument('-crop', '--crop-size', type=str, default="64x192x-1", From ae95bad0e615a5358ccc1d756cb2b8f4c9bd73b2 Mon Sep 17 00:00:00 2001 From: jcohenadad Date: Thu, 21 Dec 2023 13:27:35 -0500 Subject: [PATCH 12/66] Added setup.py --- monai/setup.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 monai/setup.py diff --git a/monai/setup.py b/monai/setup.py new file mode 100644 index 00000000..9ef1560c --- /dev/null +++ b/monai/setup.py @@ -0,0 +1,23 @@ +from setuptools import setup, find_packages + +with open('requirements.txt') as f: + requirements = f.readlines() + +setup( + name='contrast-agnostic-inference', + version='0.1', + author='https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord/graphs/contributors', + author_email='aac@example.com', + packages=find_packages(), + url='https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord', + license='MIT', + description='Inference code for the contrast-agnostic spinal cord segmentation using SoftSeg', + long_description=open('README.md').read(), + install_requires=requirements, + entry_points={ + 'console_scripts': [ + 'run_inference_single_image = run_inference_single_image:main', + ] + } + ) + From 37258e824d71f8bb935ef2f955d3f0050a2be0f4 Mon Sep 17 00:00:00 2001 From: jcohenadad Date: Thu, 21 Dec 2023 13:44:44 -0500 Subject: [PATCH 13/66] Do not install requirements The line "--extra-index-url" makes it tricky to run requirements via setup.py, so the strategy is to simply run pip install once for the requirements, and another time for the package install --- monai/setup.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/monai/setup.py b/monai/setup.py index 9ef1560c..797570c7 100644 --- a/monai/setup.py +++ b/monai/setup.py @@ -1,8 +1,5 @@ from setuptools import setup, find_packages -with open('requirements.txt') as f: - requirements = f.readlines() - setup( name='contrast-agnostic-inference', version='0.1', @@ -13,7 +10,7 @@ license='MIT', description='Inference code for the contrast-agnostic spinal cord segmentation using SoftSeg', long_description=open('README.md').read(), - install_requires=requirements, + install_requires=[], entry_points={ 'console_scripts': [ 'run_inference_single_image = run_inference_single_image:main', From 31372e922d1d292555135e983a7b1b42fde2a889 Mon Sep 17 00:00:00 2001 From: jcohenadad Date: Thu, 21 Dec 2023 13:51:36 -0500 Subject: [PATCH 14/66] Get parser arguments from within main Required by #94 --- monai/run_inference_single_image.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 04271f8f..31f55806 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -225,7 +225,10 @@ def prepare_data(path_image, path_out, crop_size=(64, 160, 320)): # =========================================================================== # Inference method # =========================================================================== -def main(args): +def main(): + + # get parameters + args = get_parser().parse_args() # define device if args.device == "gpu" and not torch.cuda.is_available(): @@ -356,6 +359,4 @@ def main(args): if __name__ == "__main__": - - args = get_parser().parse_args() - main(args) \ No newline at end of file + main() From 40fbaaf1edb35c3a58d4040a2c1a77c2dc004d8d Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Thu, 21 Dec 2023 13:03:26 -0500 Subject: [PATCH 15/66] Instruct to download repos --- monai/README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/monai/README.md b/monai/README.md index 67e5459e..35445f3f 100644 --- a/monai/README.md +++ b/monai/README.md @@ -6,19 +6,26 @@ The following steps are required for using the contrast-agnostic model. The following commands show how to set up the environment. Note that the documentation assumes that the user has `conda` installed on their system. Instructions on installing `conda` can be found [here](https://conda.io/projects/conda/en/latest/user-guide/install/index.html). -1. Create a conda environment with the following command: +Clone this repos: + +```bash +git clone https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord.git +cd contrast-agnostic-softseg-spinalcord +``` + +Create a conda environment with the following command: ```bash conda create -n venv_monai python=3.9 ``` -2. Activate the environment with the following command: +Activate the environment with the following command: ```bash conda activate venv_monai ``` -3. The list of necessary packages can be found in `requirements_inference_.txt`. Use the following commands for installation: +The list of necessary packages can be found in `requirements_inference_.txt`. Use the following commands for installation: For CPU-based inference: ```bash From f7c20a169807451dba880de3a67d3d57e94f5565 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Thu, 21 Dec 2023 13:08:19 -0500 Subject: [PATCH 16/66] Fixed path to install package --- monai/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/README.md b/monai/README.md index 35445f3f..ac861edb 100644 --- a/monai/README.md +++ b/monai/README.md @@ -10,7 +10,7 @@ Clone this repos: ```bash git clone https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord.git -cd contrast-agnostic-softseg-spinalcord +cd contrast-agnostic-softseg-spinalcord/monai ``` Create a conda environment with the following command: From 902fc78ff4414d8766da012c02e9b28f8fe6d787 Mon Sep 17 00:00:00 2001 From: Julien Cohen-Adad Date: Thu, 21 Dec 2023 13:13:07 -0500 Subject: [PATCH 17/66] Added info to download model --- monai/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monai/README.md b/monai/README.md index ac861edb..1df18bc9 100644 --- a/monai/README.md +++ b/monai/README.md @@ -37,6 +37,9 @@ For GPU-based inference: pip install -r requirements_inference_gpu.txt ``` +### Download the model + +All segmentation models can be found under the [release page](https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord/releases). Pick the release you like (we recommend the latest one) and download the file named `model_*.zip`. Then unzip it. ### Method 1: Running inference on a single image From 3405c1805efa1e09f2713f79fdefb9992e8b4c93 Mon Sep 17 00:00:00 2001 From: jcohenadad Date: Thu, 21 Dec 2023 13:56:17 -0500 Subject: [PATCH 18/66] Updated README with install instructions --- monai/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/monai/README.md b/monai/README.md index 1df18bc9..a00d3d91 100644 --- a/monai/README.md +++ b/monai/README.md @@ -25,7 +25,7 @@ Activate the environment with the following command: conda activate venv_monai ``` -The list of necessary packages can be found in `requirements_inference_.txt`. Use the following commands for installation: +Install requirements. Choose your type of installation: For CPU-based inference: ```bash @@ -37,10 +37,19 @@ For GPU-based inference: pip install -r requirements_inference_gpu.txt ``` +Install package to be callable from anywhere: +```bash +pip install -e . +``` + + + ### Download the model All segmentation models can be found under the [release page](https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord/releases). Pick the release you like (we recommend the latest one) and download the file named `model_*.zip`. Then unzip it. + + ### Method 1: Running inference on a single image The script for running inference is `run_inference_single_image.py`. Please run From 1613fb51cc5d79bbccf26cc4f669b20419d266b2 Mon Sep 17 00:00:00 2001 From: jcohenadad Date: Thu, 21 Dec 2023 13:56:50 -0500 Subject: [PATCH 19/66] Updated README with correct syntax --- monai/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/README.md b/monai/README.md index a00d3d91..8fd20744 100644 --- a/monai/README.md +++ b/monai/README.md @@ -54,7 +54,7 @@ All segmentation models can be found under the [release page](https://github.com The script for running inference is `run_inference_single_image.py`. Please run ``` -python run_inference_single_image.py -h +run_inference_single_image -h ``` to get the list of arguments and their descriptions. From 8db546a6d0daf57fe174bb805a2f70b81af37354 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 15 Jan 2024 16:24:06 -0500 Subject: [PATCH 20/66] add code to dump train/val/test subjects into yaml file --- monai/create_msd_data.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/monai/create_msd_data.py b/monai/create_msd_data.py index e7ccbff7..aa1b9630 100644 --- a/monai/create_msd_data.py +++ b/monai/create_msd_data.py @@ -1,7 +1,7 @@ import os import json from tqdm import tqdm -import numpy as np +import yaml import argparse import joblib from utils import FoldGenerator @@ -10,7 +10,7 @@ # root = "/home/GRAMES.POLYMTL.CA/u114716/datasets/spine-generic_uncropped" -parser = argparse.ArgumentParser(description='Code for creating k-fold splits of the spine-generic dataset.') +parser = argparse.ArgumentParser(description='Code for MSD-style JSON datalist for spine-generic dataset.') parser.add_argument('-pd', '--path-data', required=True, type=str, help='Path to the data set directory') parser.add_argument('-pj', '--path-joblib', help='Path to joblib file from ivadomed containing the dataset splits.', @@ -68,6 +68,10 @@ logger.info(f"Number of validation subjects: {len(val_subjects)}") logger.info(f"Number of testing subjects: {len(test_subjects)}") +# dump train/val/test splits into a yaml file +with open(f"data_split_{contrast}_{args.label_type}_seed{seed}.yaml", 'w') as file: + yaml.dump({'train': train_subjects, 'val': val_subjects, 'test': test_subjects}, file, indent=2, sort_keys=True) + # keys to be defined in the dataset_0.json params = {} params["description"] = "spine-generic-uncropped" From eb3f1ff57c644dad7a1ae23713ad6be25457c806 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 15 Jan 2024 16:24:26 -0500 Subject: [PATCH 21/66] add data splits yaml --- monai/data_split_all_soft_seed15.yaml | 263 ++++++++++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 monai/data_split_all_soft_seed15.yaml diff --git a/monai/data_split_all_soft_seed15.yaml b/monai/data_split_all_soft_seed15.yaml new file mode 100644 index 00000000..c826291f --- /dev/null +++ b/monai/data_split_all_soft_seed15.yaml @@ -0,0 +1,263 @@ +test: +- sub-amu02 +- sub-amu05 +- sub-balgrist06 +- sub-barcelona05 +- sub-beijingGE01 +- sub-beijingGE03 +- sub-beijingPrisma02 +- sub-beijingPrisma03 +- sub-brnoUhb04 +- sub-brnoUhb06 +- sub-cardiff01 +- sub-cmrra01 +- sub-cmrra04 +- sub-cmrrb01 +- sub-cmrrb07 +- sub-fslPrisma05 +- sub-geneva03 +- sub-hamburg03 +- sub-mgh02 +- sub-mgh05 +- sub-milan02 +- sub-milan04 +- sub-milan05 +- sub-mniS06 +- sub-mountSinai01 +- sub-nottwil04 +- sub-oxfordFmrib02 +- sub-oxfordOhba02 +- sub-oxfordOhba03 +- sub-oxfordOhba04 +- sub-oxfordOhba05 +- sub-pavia02 +- sub-pavia03 +- sub-pavia05 +- sub-perform05 +- sub-sherbrooke02 +- sub-stanford06 +- sub-strasbourg03 +- sub-strasbourg05 +- sub-tehranS02 +- sub-tehranS04 +- sub-tehranS05 +- sub-tokyo750w03 +- sub-tokyo750w05 +- sub-tokyo750w07 +- sub-tokyoSkyra07 +- sub-ucdavis02 +- sub-ucl03 +- sub-unf05 +- sub-vuiisAchieva01 +- sub-vuiisAchieva02 +- sub-vuiisAchieva06 +- sub-vuiisIngenia04 +train: +- sub-amu01 +- sub-amu03 +- sub-amu04 +- sub-balgrist01 +- sub-balgrist02 +- sub-balgrist04 +- sub-balgrist05 +- sub-barcelona01 +- sub-barcelona02 +- sub-barcelona03 +- sub-beijingGE04 +- sub-beijingPrisma01 +- sub-beijingPrisma05 +- sub-beijingVerio01 +- sub-beijingVerio02 +- sub-brnoCeitec01 +- sub-brnoCeitec02 +- sub-brnoCeitec03 +- sub-brnoCeitec05 +- sub-brnoUhb01 +- sub-brnoUhb05 +- sub-cardiff04 +- sub-cardiff05 +- sub-cardiff06 +- sub-cmrra03 +- sub-cmrra05 +- sub-cmrra06 +- sub-cmrrb02 +- sub-cmrrb03 +- sub-cmrrb04 +- sub-cmrrb05 +- sub-cmrrb06 +- sub-dresden01 +- sub-dresden02 +- sub-fslAchieva01 +- sub-fslAchieva03 +- sub-fslAchieva04 +- sub-fslPrisma01 +- sub-fslPrisma02 +- sub-fslPrisma04 +- sub-geneva01 +- sub-geneva02 +- sub-geneva04 +- sub-hamburg04 +- sub-juntendo750w01 +- sub-juntendo750w03 +- sub-juntendo750w04 +- sub-juntendo750w05 +- sub-juntendo750w06 +- sub-mgh01 +- sub-mgh03 +- sub-mgh04 +- sub-mgh06 +- sub-milan01 +- sub-milan06 +- sub-milan07 +- sub-mniS01 +- sub-mniS03 +- sub-mniS05 +- sub-mniS07 +- sub-mniS08 +- sub-mniS09 +- sub-mountSinai04 +- sub-mountSinai05 +- sub-mountSinai06 +- sub-mpicbs01 +- sub-mpicbs02 +- sub-mpicbs05 +- sub-mpicbs06 +- sub-mpicbs07 +- sub-nottwil02 +- sub-nottwil03 +- sub-nottwil05 +- sub-nwu01 +- sub-nwu02 +- sub-nwu03 +- sub-nwu04 +- sub-nwu05 +- sub-nwu06 +- sub-oxfordFmrib03 +- sub-oxfordFmrib04 +- sub-oxfordFmrib05 +- sub-oxfordFmrib06 +- sub-oxfordFmrib07 +- sub-oxfordFmrib08 +- sub-oxfordFmrib09 +- sub-oxfordFmrib11 +- sub-oxfordOhba01 +- sub-pavia01 +- sub-pavia04 +- sub-perform01 +- sub-perform02 +- sub-perform06 +- sub-queensland02 +- sub-queensland03 +- sub-queensland04 +- sub-queensland05 +- sub-queensland06 +- sub-sherbrooke03 +- sub-sherbrooke04 +- sub-sherbrooke05 +- sub-sherbrooke06 +- sub-stanford01 +- sub-stanford04 +- sub-stanford05 +- sub-strasbourg01 +- sub-strasbourg02 +- sub-strasbourg04 +- sub-strasbourg06 +- sub-tehranS01 +- sub-tehranS03 +- sub-tehranS06 +- sub-tokyo750w01 +- sub-tokyo750w06 +- sub-tokyoIngenia01 +- sub-tokyoIngenia02 +- sub-tokyoIngenia03 +- sub-tokyoIngenia04 +- sub-tokyoIngenia05 +- sub-tokyoIngenia07 +- sub-tokyoSkyra01 +- sub-tokyoSkyra02 +- sub-tokyoSkyra03 +- sub-tokyoSkyra05 +- sub-tokyoSkyra06 +- sub-ubc01 +- sub-ubc02 +- sub-ubc03 +- sub-ubc05 +- sub-ubc06 +- sub-ucdavis01 +- sub-ucdavis03 +- sub-ucdavis04 +- sub-ucdavis05 +- sub-ucdavis06 +- sub-ucl04 +- sub-ucl05 +- sub-ucl06 +- sub-unf01 +- sub-unf03 +- sub-unf04 +- sub-unf06 +- sub-unf07 +- sub-vallHebron01 +- sub-vallHebron02 +- sub-vallHebron03 +- sub-vallHebron04 +- sub-vallHebron07 +- sub-vuiisAchieva03 +- sub-vuiisAchieva04 +- sub-vuiisAchieva05 +- sub-vuiisIngenia01 +- sub-vuiisIngenia03 +- sub-vuiisIngenia05 +- sub-vuiisIngenia06 +val: +- sub-balgrist03 +- sub-barcelona04 +- sub-barcelona06 +- sub-beijingGE02 +- sub-beijingPrisma04 +- sub-beijingVerio03 +- sub-beijingVerio04 +- sub-brnoCeitec04 +- sub-brnoCeitec06 +- sub-cardiff02 +- sub-cardiff03 +- sub-cmrra02 +- sub-fslAchieva02 +- sub-fslAchieva05 +- sub-fslAchieva06 +- sub-fslPrisma03 +- sub-fslPrisma06 +- sub-geneva05 +- sub-geneva06 +- sub-hamburg01 +- sub-hamburg02 +- sub-hamburg05 +- sub-hamburg06 +- sub-juntendo750w02 +- sub-milan03 +- sub-mniPilot1 +- sub-mniS02 +- sub-mniS04 +- sub-mountSinai02 +- sub-mpicbs03 +- sub-nottwil01 +- sub-nottwil06 +- sub-oxfordFmrib10 +- sub-pavia06 +- sub-perform03 +- sub-perform04 +- sub-queensland01 +- sub-sherbrooke01 +- sub-sherbrooke07 +- sub-stanford02 +- sub-stanford03 +- sub-tokyo750w02 +- sub-tokyo750w04 +- sub-tokyoIngenia06 +- sub-tokyoSkyra04 +- sub-ubc04 +- sub-ucl01 +- sub-ucl02 +- sub-unf02 +- sub-vallHebron05 +- sub-vallHebron06 +- sub-vuiisIngenia02 From ceab9a31e4667241d9cd1c2604ba39eb86fc2e36 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 15 Jan 2024 16:27:04 -0500 Subject: [PATCH 22/66] add info about finding train/val/test splits yaml file --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c25a46b3..92fedac8 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,8 @@ The training script expects a datalist file in the Medical Decathlon format cont python monai/create_msd_data.py -pd ~/duke/projects/ivadomed/contrast-agnostic-seg/data_processed_sg_2023-08-08_NO_CROP\data_processed_clean> -po ~/datasets/contrast-agnostic/ --contrast all --label-type soft --seed 42 ``` +The dataset split containing the training, validation, and test subjects can be found in the `monai/data_split_all_soft_seed15.yaml` file. + > **Note** > The output of the above command is just `.json` file pointing to the image-label pairs in the original BIDS dataset. It _does not_ copy the existing data to the output folder. From 2d9899dfd94ddc5efcc6210df5736ea178ce9b7c Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sun, 21 Jan 2024 22:40:50 -0500 Subject: [PATCH 23/66] simplify prepare_data function --- monai/run_inference_single_image.py | 32 ++++------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 31f55806..72b3788b 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -16,7 +16,7 @@ from time import time from monai.inferers import sliding_window_inference -from monai.data import (DataLoader, CacheDataset, load_decathlon_datalist, decollate_batch) +from monai.data import (DataLoader, Dataset, load_decathlon_datalist, decollate_batch) from monai.transforms import (Compose, EnsureTyped, Invertd, SaveImage, Spacingd, LoadImaged, NormalizeIntensityd, EnsureChannelFirstd, DivisiblePadd, Orientationd, ResizeWithPadOrCropd) @@ -177,33 +177,9 @@ def create_nnunet_from_plans(plans, num_input_channels: int, num_classes: int, d # =========================================================================== # Prepare temporary dataset for inference # =========================================================================== -def prepare_data(path_image, path_out, crop_size=(64, 160, 320)): - - # create a temporary datalist containing the image - # boiler plate keys to be defined in the MSD-style datalist - params = {} - params["description"] = "my-awesome-SC-image" - params["labels"] = { - "0": "background", - "1": "soft-sc-seg" - } - params["modality"] = { - "0": "MRI" - } - params["tensorImageSize"] = "3D" - params["test"] = [ - { - "image": path_image - } - ] - - final_json = json.dumps(params, indent=4, sort_keys=True) - jsonFile = open(path_out + "/" + f"temp_msd_datalist.json", "w") - jsonFile.write(final_json) - jsonFile.close() +def prepare_data(path_image, crop_size=(64, 160, 320)): - dataset = os.path.join(path_out, f"temp_msd_datalist.json") - test_files = load_decathlon_datalist(dataset, True, "test") + test_file = [{"image": path_image}] # define test transforms transforms_test = inference_transforms_single_image(crop_size=crop_size) @@ -217,7 +193,7 @@ def prepare_data(path_image, path_out, crop_size=(64, 160, 320)): meta_keys=["pred_meta_dict"], nearest_interp=False, to_tensor=True), ]) - test_ds = CacheDataset(data=test_files, transform=transforms_test, cache_rate=0.75, num_workers=8) + test_ds = Dataset(data=test_file, transform=transforms_test) return test_ds, test_post_pred From 1b6056a8aafd35357f5d5924eb2e896cf33c7083 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sun, 21 Jan 2024 22:41:18 -0500 Subject: [PATCH 24/66] remove unused input arg --- monai/run_inference_single_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 72b3788b..141809e9 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -230,7 +230,7 @@ def main(): inference_roi_size = (64, 192, 320) # define the dataset and dataloader - test_ds, test_post_pred = prepare_data(path_image, results_path, crop_size=crop_size) + test_ds, test_post_pred = prepare_data(path_image, crop_size=crop_size) test_loader = DataLoader(test_ds, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) # define model From c3a9fc0d6573fa9ff9815043358f3c233ce53056 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sun, 21 Jan 2024 22:42:09 -0500 Subject: [PATCH 25/66] binarize preds with 0.5 threshold --- monai/run_inference_single_image.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 141809e9..64766d24 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -281,13 +281,10 @@ def main(): pred = post_test_out[0]['pred'].cpu() - # clip the prediction between 0.5 and 1 - # turns out this sets the background to 0.5 and the SC to 1 (which is not correct) - # details: https://github.com/sct-pipeline/contrast-agnostic-softseg-spinalcord/issues/71 - pred = torch.clamp(pred, 0.5, 1) - # set background values to 0 - pred[pred <= 0.5] = 0 - + # binarize the prediction with a threshold of 0.5 + pred[pred >= 0.5] = 1 + pred[pred < 0.5] = 0 + # get subject name subject_name = (batch["image_meta_dict"]["filename_or_obj"][0]).split("/")[-1].replace(".nii.gz", "") logger.info(f"Saving subject: {subject_name}") @@ -331,7 +328,6 @@ def main(): # free up memory test_step_outputs.clear() test_summary.clear() - os.remove(os.path.join(results_path, "temp_msd_datalist.json")) if __name__ == "__main__": From 998fe5e6e89a0a3d8786b2dd3a6c34c1b6c5d4b1 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sun, 21 Jan 2024 22:57:27 -0500 Subject: [PATCH 26/66] remove unused import --- monai/run_inference_single_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 64766d24..4ce29a98 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -16,7 +16,7 @@ from time import time from monai.inferers import sliding_window_inference -from monai.data import (DataLoader, Dataset, load_decathlon_datalist, decollate_batch) +from monai.data import (DataLoader, Dataset, decollate_batch) from monai.transforms import (Compose, EnsureTyped, Invertd, SaveImage, Spacingd, LoadImaged, NormalizeIntensityd, EnsureChannelFirstd, DivisiblePadd, Orientationd, ResizeWithPadOrCropd) @@ -284,7 +284,7 @@ def main(): # binarize the prediction with a threshold of 0.5 pred[pred >= 0.5] = 1 pred[pred < 0.5] = 0 - + # get subject name subject_name = (batch["image_meta_dict"]["filename_or_obj"][0]).split("/")[-1].replace(".nii.gz", "") logger.info(f"Saving subject: {subject_name}") From 8d19401192007fa38e1e1cce8450645487275a7c Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sun, 28 Jan 2024 02:13:10 -0500 Subject: [PATCH 27/66] fix checkpoint name while loading --- monai/run_inference_single_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 4ce29a98..8ab5c68b 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -216,7 +216,7 @@ def main(): # define root path for finding datalists path_image = args.path_img results_path = args.path_out - chkp_path = os.path.join(args.chkp_path, "best_model_loss.ckpt") + chkp_path = os.path.join(args.chkp_path, "best_model.ckpt") # save terminal outputs to a file logger.add(os.path.join(results_path, "logs.txt"), rotation="10 MB", level="INFO") From 91e57fab094249e1d73ee0cee2fc8f682c0f6404 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 29 Jan 2024 11:23:55 -0500 Subject: [PATCH 28/66] add binrarized soft labels --- monai/main.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/monai/main.py b/monai/main.py index ed742b60..b628a937 100644 --- a/monai/main.py +++ b/monai/main.py @@ -111,9 +111,15 @@ def prepare_data(self): transforms_val = val_transforms(crop_size=self.inference_roi_size, lbl_key='label') # load the dataset - dataset = os.path.join(self.root, - f"dataset_{self.cfg['dataset']['contrast']}_{self.cfg['dataset']['label_type']}_seed{self.cfg['seed']}.json" - ) + if self.cfg["dataset"]["label_type"] == "softBin": + logger.info(f"Training with binarized soft labels (threshold 0.5) ...") + dataset = os.path.join(self.root, + f"dataset_{self.cfg['dataset']['contrast']}_soft_seed{self.cfg['seed']}.json" + ) + else: + dataset = os.path.join(self.root, + f"dataset_{self.cfg['dataset']['contrast']}_{self.cfg['dataset']['label_type']}_seed{self.cfg['seed']}.json" + ) logger.info(f"Loading dataset: {dataset}") train_files = load_decathlon_datalist(dataset, True, "train") val_files = load_decathlon_datalist(dataset, True, "validation") @@ -179,6 +185,10 @@ def training_step(self, batch, batch_idx): inputs, labels = batch["image"], batch["label"] + if self.cfg["dataset"]["label_type"] == "softBin": + # binarize soft labels with threshold 0.5 + labels = (labels > 0.5).float() + # check if any label image patch is empty in the batch if check_empty_patch(labels) is None: # print(f"Empty label patch found. Skipping training step ...") @@ -278,6 +288,10 @@ def validation_step(self, batch, batch_idx): inputs, labels = batch["image"], batch["label"] + if self.cfg["dataset"]["label_type"] == "softBin": + # binarize soft labels with threshold 0.5 + labels = (labels > 0.5).float() + # NOTE: this calculates the loss on the entire image after sliding window outputs = sliding_window_inference(inputs, self.inference_roi_size, mode="gaussian", sw_batch_size=4, predictor=self.forward, overlap=0.5,) @@ -599,7 +613,7 @@ def main(args): # wandb logger exp_logger = pl.loggers.WandbLogger( name=save_exp_id, - save_dir=config["directories"]["models_dir"], + save_dir="/home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/saved_models", group=config["dataset"]["name"], log_model=True, # save best model using checkpoint callback project='contrast-agnostic', From 1b3e351434d39500d95971354d3999e5d3f3ecfd Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 31 Jan 2024 10:30:06 -0500 Subject: [PATCH 29/66] add script for generating binarized soft labels --- .../generate_soft_bin_labels.sh | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 processing_spine_generic/generate_soft_bin_labels.sh diff --git a/processing_spine_generic/generate_soft_bin_labels.sh b/processing_spine_generic/generate_soft_bin_labels.sh new file mode 100644 index 00000000..c3632506 --- /dev/null +++ b/processing_spine_generic/generate_soft_bin_labels.sh @@ -0,0 +1,151 @@ +#!/bin/bash +# +# Compare the CSA of soft GT thresholded at different values on spine-generic test dataset. +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# Uncomment for full verbose +set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 + +echo "SUBJECT: ${SUBJECT}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Copy GT soft segmentation (located under derivatives/labels_softseg) +copy_gt_softseg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels_softseg/${SUBJECT}/${type}/${file}_softseg.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_softseg.nii.gz $PATH_DATA_PROCESSED/derivatives/labels_softseg_bin/${SUBJECT}/${type}/ + rsync -avzh ${FILESEG%.nii.gz}.json $PATH_DATA_PROCESSED/derivatives/labels_softseg_bin/${SUBJECT}/${type}/${file}_softseg_bin.json + else + echo "File ${FILESEG} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +# # Copy source images (used only for generating QC report) +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' +rsync -Ravzh ${PATH_DATA}/./${SUBJECT} . + +mkdir -p ${PATH_DATA_PROCESSED}/derivatives/labels_softseg_bin/ + +# copy GT soft segmentation +rsync -avzh ${PATH_DATA}/derivatives/labels_softseg/${SUBJECT} ${PATH_DATA_PROCESSED}/derivatives/labels_softseg_bin/ + +# ------------------------------------------------------------------------------ +# contrast +# ------------------------------------------------------------------------------ +contrasts="T1w T2w T2star flip-1_mt-on_MTS flip-2_mt-off_MTS rec-average_dwi" +# contrasts="flip-2_mt-off_MTS rec-average_dwi" + +# Loop across contrasts +for contrast in ${contrasts}; do + + if [[ $contrast == "rec-average_dwi" ]]; then + type="dwi" + else + type="anat" + fi + + # Go to the folder where the soft GT are + cd ${PATH_DATA_PROCESSED}/derivatives/labels_softseg_bin/${SUBJECT}/${type}/ + + # Get file name + file="${SUBJECT}_${contrast}" + + # Check if file exists + if [[ ! -e ${file}_softseg.nii.gz ]]; then + echo "File ${file}_softseg.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}_softseg.nii.gz does not exist. Exiting." + exit 1 + fi + +# # Copy GT spinal cord segmentation +# copy_gt_softseg "${file}" "${type}" + + # Binarize the soft GT + FILETHRESH="${file}_softseg_bin" + sct_maths -i ${file}_softseg.nii.gz -bin 0.5 -o ${FILETHRESH}.nii.gz + + # Generate QC report + sct_qc -i ${PATH_DATA_PROCESSED}/${SUBJECT}/${type}/${file}.nii.gz -s ${FILETHRESH}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Rename the json sidecars + mv ${file}_softseg.json ${FILETHRESH}.json + +done + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From dcb42af0428eee754c33a57642d6f9917c78c0d3 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 31 Jan 2024 10:32:59 -0500 Subject: [PATCH 30/66] remove unused function --- .../generate_soft_bin_labels.sh | 27 ------------------- 1 file changed, 27 deletions(-) diff --git a/processing_spine_generic/generate_soft_bin_labels.sh b/processing_spine_generic/generate_soft_bin_labels.sh index c3632506..564d94e1 100644 --- a/processing_spine_generic/generate_soft_bin_labels.sh +++ b/processing_spine_generic/generate_soft_bin_labels.sh @@ -47,30 +47,6 @@ SUBJECT=$1 echo "SUBJECT: ${SUBJECT}" -# ------------------------------------------------------------------------------ -# CONVENIENCE FUNCTIONS -# ------------------------------------------------------------------------------ - -# Copy GT soft segmentation (located under derivatives/labels_softseg) -copy_gt_softseg(){ - local file="$1" - local type="$2" - # Construct file name to GT segmentation located under derivatives/labels - FILESEG="${PATH_DATA}/derivatives/labels_softseg/${SUBJECT}/${type}/${file}_softseg.nii.gz" - echo "" - echo "Looking for manual segmentation: $FILESEG" - if [[ -e $FILESEG ]]; then - echo "Found! Copying ..." - rsync -avzh $FILESEG ${file}_softseg.nii.gz $PATH_DATA_PROCESSED/derivatives/labels_softseg_bin/${SUBJECT}/${type}/ - rsync -avzh ${FILESEG%.nii.gz}.json $PATH_DATA_PROCESSED/derivatives/labels_softseg_bin/${SUBJECT}/${type}/${file}_softseg_bin.json - else - echo "File ${FILESEG} does not exist" >> ${PATH_LOG}/missing_files.log - echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." - exit 1 - fi -} - - # ------------------------------------------------------------------------------ # SCRIPT STARTS HERE # ------------------------------------------------------------------------------ @@ -121,9 +97,6 @@ for contrast in ${contrasts}; do exit 1 fi -# # Copy GT spinal cord segmentation -# copy_gt_softseg "${file}" "${type}" - # Binarize the soft GT FILETHRESH="${file}_softseg_bin" sct_maths -i ${file}_softseg.nii.gz -bin 0.5 -o ${FILETHRESH}.nii.gz From 70938c6f3cb2baa1ce79620bfb7d3b659c5c1c97 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 2 Feb 2024 04:19:28 -0500 Subject: [PATCH 31/66] add option to create datalist for offline binarized soft labels --- monai/create_msd_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/monai/create_msd_data.py b/monai/create_msd_data.py index aa1b9630..9a3f9433 100644 --- a/monai/create_msd_data.py +++ b/monai/create_msd_data.py @@ -19,7 +19,7 @@ parser.add_argument("--contrast", default="t2w", type=str, help="Contrast to use for training", choices=["t1w", "t2w", "t2star", "mton", "mtoff", "dwi", "all"]) parser.add_argument('--label-type', default='soft', type=str, help="Type of labels to use for training", - choices=['hard', 'soft']) + choices=['hard', 'soft', 'soft_bin']) parser.add_argument('--seed', default=42, type=int, help="Seed for reproducibility") args = parser.parse_args() @@ -31,6 +31,10 @@ logger.info("Using SOFT LABELS ...") PATH_DERIVATIVES = os.path.join(root, "derivatives", "labels_softseg") SUFFIX = "softseg" +elif args.label_type == 'soft_bin': + logger.info("Using BINARIZED SOFT LABELS ...") + PATH_DERIVATIVES = os.path.join(root, "derivatives", "labels_softseg_bin") + SUFFIX = "softseg_bin" else: logger.info("Using HARD LABELS ...") PATH_DERIVATIVES = os.path.join(root, "derivatives", "labels") From edae2ee976f440be64eb7f610eee4ab5974a0db9 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 2 Feb 2024 04:20:33 -0500 Subject: [PATCH 32/66] add script to compare csa across training GT --- .../comparison_across_training_labels.sh | 356 ++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh diff --git a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh new file mode 100644 index 00000000..24455889 --- /dev/null +++ b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh @@ -0,0 +1,356 @@ +#!/bin/bash +# +# Compare the CSA of soft GT thresholded at different values on spine-generic test dataset. +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# Uncomment for full verbose +set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 +PATH_NNUNET_SCRIPT=$2 # path to the nnUNet contrast-agnostic run_inference_single_subject.py +PATH_NNUNET_MODEL=$3 # path to the nnUNet contrast-agnostic model +PATH_MONAI_SCRIPT=$4 # path to the MONAI contrast-agnostic run_inference_single_subject.py +PATH_MONAI_MODEL_SOFT=$5 # path to the MONAI contrast-agnostic model trained on soft labels +PATH_MONAI_MODEL_SOFTBIN=$6 # path to the MONAI contrast-agnostic model trained on soft_bin labels + +echo "SUBJECT: ${SUBJECT}" +echo "PATH_NNUNET_SCRIPT: ${PATH_NNUNET_SCRIPT}" +echo "PATH_NNUNET_MODEL: ${PATH_NNUNET_MODEL}" +echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +echo "PATH_MONAI_MODEL_SOFT: ${PATH_MONAI_MODEL_SOFT}" +echo "PATH_MONAI_MODEL_SOFTBIN: ${PATH_MONAI_MODEL_SOFTBIN}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Check if manual label already exists. If it does, copy it locally. +# NOTE: manual disc labels should go from C1-C2 to C7-T1. +label_vertebrae(){ + local file="$1" + local contrast="$2" + + # Update global variable with segmentation file name + FILESEG="${file}_seg-manual" + FILELABEL="${file}_discs" + + # Label vertebral levels + sct_label_utils -i ${file}.nii.gz -disc ${FILELABEL}.nii.gz -o ${FILESEG}_labeled.nii.gz + + # # Run QC + # sct_qc -i ${file}.nii.gz -s ${file_seg}_labeled.nii.gz -p sct_label_vertebrae -qc ${PATH_QC} -qc-subject ${SUBJECT} +} + + +# Copy GT spinal cord disc labels (located under derivatives/labels) +copy_gt_disc_labels(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILEDISCLABELS="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_discs.nii.gz" + echo "" + echo "Looking for manual disc labels: $FILEDISCLABELS" + if [[ -e $FILEDISCLABELS ]]; then + echo "Found! Copying ..." + rsync -avzh $FILEDISCLABELS ${file}_discs.nii.gz + else + echo "File ${FILEDISCLABELS} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Disc Labels ${FILEDISCLABELS} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT segmentation (located under derivatives/labels) +copy_gt_seg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_seg-manual.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_seg-manual.nii.gz + else + echo "File ${FILESEG}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT soft segmentation (located under derivatives/labels_softseg) +copy_gt_softseg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels_softseg/${SUBJECT}/${type}/${file}_softseg.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_softseg.nii.gz + else + echo "File ${FILESEG} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + + +# TODO: Fix the contrast input for deepseg and propseg (i.e. dwi, mton, mtoff won't work) +# Segment spinal cord using methods available in SCT (sct_deepseg_sc or sct_propseg), resample the prediction back to +# native resolution and compute CSA in native space +segment_sc() { + local file="$1" + local contrast="$2" + local method="$3" # deepseg or propseg + local kernel="$4" # 2d or 3d; only relevant for deepseg + local file_gt_vert_label="$5" + local native_res="$6" + + # Segment spinal cord + if [[ $method == 'deepseg' ]];then + FILESEG="${file}_seg_${method}_${kernel}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_deepseg_sc -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -kernel ${kernel} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + elif [[ $method == 'propseg' ]]; then + FILESEG="${file}_seg_${method}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_propseg -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Remove centerline (we don't need it) + rm ${file}_centerline.nii.gz + + fi + + # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + +} + +# Segment spinal cord using the contrast-agnostic nnUNet model +segment_sc_nnUNet(){ + local file="$1" + local kernel="$2" # 2d or 3d + local file_gt_vert_label="$3" + local contrast="$4" # used only for saving output file name + + # FILESEG="${file}_seg_nnunet_${kernel}" + FILESEG="${file%%_*}_${contrast}_seg_nnunet" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_NNUNET_SCRIPT} -i ${file}.nii.gz -o ${FILESEG}.nii.gz -path-model ${PATH_NNUNET_MODEL}/nnUNetTrainer__nnUNetPlans__${kernel} -pred-type sc -use-gpu + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # # Generate QC report + # sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + +} + +# Segment spinal cord using the MONAI contrast-agnostic model +segment_sc_MONAI(){ + local file="$1" + local file_gt_vert_label="$2" + local label_type="$3" # soft or soft_bin + local contrast="$4" # used only for saving output file name + + if [[ $label_type == 'soft' ]]; then + FILESEG="${file%%_*}_${contrast}_seg_monai_soft" + PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFT} + + elif [[ $label_type == 'soft_bin' ]]; then + FILESEG="${file%%_*}_${contrast}_seg_monai_bin" + PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFTBIN} + + fi + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu + # Rename MONAI output + mv ${file}_pred.nii.gz ${FILESEG}.nii.gz + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Binarize MONAI output (which is soft by default); output is overwritten + sct_maths -i ${FILESEG}.nii.gz -bin 0.5 -o ${FILESEG}.nii.gz + + # Generate QC report with soft prediction + sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the soft prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +# Copy source images +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/anat/* . +# copy DWI data +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/dwi/* . + +# ------------------------------------------------------------------------------ +# contrast +# ------------------------------------------------------------------------------ +contrasts="T1w T2w T2star flip-1_mt-on_MTS flip-2_mt-off_MTS rec-average_dwi" +# contrasts="flip-2_mt-off_MTS rec-average_dwi" + +# Loop across contrasts +for contrast in ${contrasts}; do + + if [[ $contrast == "rec-average_dwi" ]]; then + type="dwi" + else + type="anat" + fi + + # go to the folder where the data is + cd ${PATH_DATA_PROCESSED}/${SUBJECT}/${type} + + # Get file name + file="${SUBJECT}_${contrast}" + + # Check if file exists + if [[ ! -e ${file}.nii.gz ]]; then + echo "File ${file}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}.nii.gz does not exist. Exiting." + exit 1 + fi + + # Copy GT spinal cord segmentation + copy_gt_seg "${file}" "${type}" + + # Copy soft GT spinal cord segmentation + copy_gt_softseg "${file}" "${type}" + + # Copy GT disc labels segmentation + copy_gt_disc_labels "${file}" "${type}" + + # Label vertebral levels in the native resolution + label_vertebrae ${file} 't2' + + # rename contrasts + if [[ $contrast == "flip-1_mt-on_MTS" ]]; then + contrast="MTon" + elif [[ $contrast == "flip-2_mt-off_MTS" ]]; then + contrast="MToff" + elif [[ $contrast == "rec-average_dwi" ]]; then + contrast="DWI" + fi + + # 1. Compute (soft) CSA of the original soft GT + # renaming file so that it can be fetched from the CSA csa file later + FILEINPUT="${file%%_*}_${contrast}_softseg" + cp ${file}_softseg.nii.gz ${FILEINPUT}.nii.gz + sct_process_segmentation -i ${FILEINPUT}.nii.gz -vert 2:4 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + + # Threshold the soft GT + FILETHRESH="${file%%_*}_${contrast}_softseg_bin" + sct_maths -i ${file}_softseg.nii.gz -bin 0.5 -o ${FILETHRESH}.nii.gz + + # 2. Compute CSA of the binarized soft GT + sct_process_segmentation -i ${FILETHRESH}.nii.gz -vert 2:4 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + + # 3. Segment SC using different methods, binarize at 0.5 and compute CSA + segment_sc_MONAI ${file} "${file}_seg-manual" 'soft' ${contrast} + segment_sc_MONAI ${file} "${file}_seg-manual" 'soft_bin' ${contrast} + segment_sc_nnUNet ${file} '3d_fullres' "${file}_seg-manual" ${contrast} + # # TODO: run on deep/progseg after fixing the contrasts for those + # segment_sc ${file_res} 't2' 'deepseg' '2d' "${file}_seg-manual" ${native_res} + # segment_sc ${file_res} 't2' 'propseg' '' "${file}_seg-manual" ${native_res} + +done + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From f5ec0f46f36af25293d4a3383956c546df82e1d5 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sat, 3 Feb 2024 12:33:05 -0500 Subject: [PATCH 33/66] change variable name to softseg_soft --- .../comparison_across_training_labels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh index 24455889..3be3cf68 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh @@ -320,7 +320,7 @@ for contrast in ${contrasts}; do # 1. Compute (soft) CSA of the original soft GT # renaming file so that it can be fetched from the CSA csa file later - FILEINPUT="${file%%_*}_${contrast}_softseg" + FILEINPUT="${file%%_*}_${contrast}_softseg_soft" cp ${file}_softseg.nii.gz ${FILEINPUT}.nii.gz sct_process_segmentation -i ${FILEINPUT}.nii.gz -vert 2:4 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 From 5740888ff0fad303bd2c236ff0434ab21495c55c Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sat, 3 Feb 2024 12:35:06 -0500 Subject: [PATCH 34/66] add script to generate csa violin plots for diff training label types --- .../analyse_csa_across_training_labels.py | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 csa_generate_figures/analyse_csa_across_training_labels.py diff --git a/csa_generate_figures/analyse_csa_across_training_labels.py b/csa_generate_figures/analyse_csa_across_training_labels.py new file mode 100644 index 00000000..2747b45c --- /dev/null +++ b/csa_generate_figures/analyse_csa_across_training_labels.py @@ -0,0 +1,296 @@ +""" +Generate violin plot from CSV data across resolutions and methods. + +Usage: + python generate_figure_csa_across_resolutions.py -i /path/to/data.csv +""" + +import os +import argparse +import pandas as pd +import re +import seaborn as sns +import matplotlib.pyplot as plt + +# Setting the hue order as specified +# HUE_ORDER = ["propseg", "deepseg_2d", "nnunet_3d_fullres", "monai"] +HUE_ORDER = ["softseg_soft", "softseg_bin", "nnunet", "monai_soft", "monai_bin"] +HUE_ORDER_ABS_CSA = ["", "nnunet", "monai_soft", "monai_bin"] +CONTRAST_ORDER = ["DWI", "MTon", "MToff", "T1w", "T2star", "T2w"] + + +def fetch_participant_id(filename_path): + """ + Get participant_id from the input BIDS-compatible filename or file path + :return: participant_id: subject ID (e.g., sub-001) + """ + + _, filename = os.path.split(filename_path) # Get just the filename (i.e., remove the path) + participant = re.search('sub-(.*?)[_/]', filename_path) # [_/] slash or underscore + participant_id = participant.group(0)[:-1] if participant else "" # [:-1] removes the last underscore or slash + # REGEX explanation + # \d - digit + # \d? - no or one occurrence of digit + # *? - match the previous element as few times as possible (zero or more times) + # . - any character + + return participant_id + + +# Function to extract method and resolution from the filename +def extract_contrast_and_method(filename): + """ + Extract the segmentation method and resolution from the filename. + The method (e.g., propseg, deepseg_2d, nnunet_3d_fullres, monai) and resolution (e.g., 1mm) + are embedded in the filename. + """ + # pattern = r'.*iso-(\d+mm).*_(propseg|deepseg_2d|nnunet_3d_fullres|monai).*' + pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_soft|softseg_bin|nnunet|monai_soft|monai_bin).*' + match = re.search(pattern, filename) + if match: + return match.group(1), match.group(2) + else: + return 'Unknown', 'Unknown' + + +def generate_figure(data, contrast, file_path): + """ + Generate violinplot across resolutions and methods + :param data: Pandas DataFrame with the data + :param contrast: Contrast (e.g., T1w, T2w, T2star) + :param file_path: Path to the CSV file (will be used to save the figure) + """ + + # Correct labels for the x-axis based on the actual data + # resolution_labels = ['1mm', '125mm', '15mm', '175mm', '2mm'] + resolution_labels = ['1mm', '05mm', '15mm', '3mm', '2mm'] + + # Creating the violin plot + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Resolution', y='MEAN(area)', hue='Method', data=data, order=resolution_labels, + hue_order=HUE_ORDER) + plt.xticks(rotation=45) + plt.xlabel('Resolution') + plt.ylabel('CSA [mm^2]') + plt.title(f'{contrast}: C2-C3 CSA across Resolutions and Methods') + plt.legend(title='Method', loc='lower left') + plt.tight_layout() + + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Update x-axis labels + # plt.gca().set_xticklabels(['1mm', '1.25mm', '1.5mm', '1.75mm', '2mm']) + plt.gca().set_xticklabels(['1x1x1mm', '0.5x0.5x0.5mm', '1.5mm', '3x0.5x0.5mm', '2mm']) + + # Save the figure in 300 DPI as a PNG file + plt.savefig(file_path.replace('.csv', '.png'), dpi=300) + print(f'Figure saved to {file_path.replace(".csv", ".png")}') + + # Display the plot + plt.show() + + +def generate_figure_std(data, file_path): + """ + Generate violinplot showing STD across participants for each method + """ + + # Compute mean and std across contrasts for each method + df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Method', y='std', data=df, order=HUE_ORDER) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Method', y='std', data=df, color='k', order=HUE_ORDER, size=3) + # plt.xticks(rotation=45) + plt.xlabel('Method') + plt.ylabel('STD [mm^2]') + plt.title(f'STD of C2-C4 CSA for each method') + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Draw vertical line between 1st and 2nd violin + plt.axvline(x=0.5, color='k', linestyle='--') + + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for method in df['Method'].unique(): + mean = df[df['Method'] == method]['std'].mean() + std = df[df['Method'] == method]['std'].std() + plt.text(HUE_ORDER.index(method), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + plt.tight_layout() + save_path = os.path.join(os.path.dirname(file_path), "std_csa.png") + plt.savefig(save_path, dpi=300) + print(f'Figure saved to {save_path}') + + +def generate_figure_abs_csa_error(data, file_path): + """ + Generate violinplot showing absolute CSA error across participants for each method + """ + + # Compute mean and std across contrasts for each method + df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + + # Remove "softseg_soft" from the list of methods, if it exists + if 'softseg_soft' in df['Method'].unique(): + df = df[df['Method'] != 'softseg_soft'] + + # Compute the abs error between "sofseg_bin" and all other methods + df['abs_error'] = df.apply(lambda row: abs(row['mean'] - df[(df['Method'] == 'softseg_bin') & (df['Participant'] == row['Participant'])]['mean'].values[0]), axis=1) + + # Remove "softseg_bin" from the list of methods and shift rows by one to match the violinplot + df = df[df['Method'] != 'softseg_bin'] + + plt.figure(figsize=(12, 6)) + # skip the first method (i.e., softseg_bin) + sns.violinplot(x='Method', y='abs_error', data=df, order=HUE_ORDER) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Method', y='abs_error', data=df, color='k', order=HUE_ORDER, size=3) + + # plt.xticks(rotation=45) + plt.xlabel('Method') + plt.ylabel('Absolute CSA error [mm^2]') + plt.title(f'Absolute CSA error between softseg_bin and other methods') + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Draw vertical line between 1st and 2nd violin + plt.axvline(x=0.5, color='k', linestyle='--') + + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for method in df['Method'].unique(): + mean = df[df['Method'] == method]['abs_error'].mean() + std = df[df['Method'] == method]['abs_error'].std() + plt.text(HUE_ORDER.index(method), ymax-0.25, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + plt.tight_layout() + save_path = os.path.join(os.path.dirname(file_path), "abs_csa_error.png") + plt.savefig(save_path, dpi=300) + print(f'Figure saved to {save_path}') + + +def generate_figure_abs_csa_error_per_contrast(data, method, file_path): + """ + Generate violinplot showing absolute CSA error for each contrast for a given method + """ + + # remove "softseg_soft" from the list of methods, if it exists + if 'softseg_soft' in data['Method'].unique(): + data = data[data['Method'] != 'softseg_soft'] + + # create a dataframe with only "softseg_bin" and get the mean CSA for each contrast + df1 = data[data['Method'] == "softseg_bin"].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() + df1 = df1.pivot(index='Participant', columns='Contrast', values='mean').reset_index() + + # create a dataframe with only the given method and get the mean CSA for each contrast + df2 = data[data['Method'] == method].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() + df2 = df2.pivot(index='Participant', columns='Contrast', values='mean').reset_index() + + # compute the absolute error between the two dataframes for each contrast + df = pd.DataFrame() + df['Participant'] = df1['Participant'] + for contrast in CONTRAST_ORDER: + df[contrast] = abs(df1[contrast] - df2[contrast]) + + # reshape the dataframe to have a single column for the contrast and a single column for the absolute error + df = df.melt(id_vars=['Participant'], value_vars=CONTRAST_ORDER, var_name='Contrast', value_name='abs_error') + + # plot the abs error for each contrast in a violinplot + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Contrast', y='abs_error', data=df, order=CONTRAST_ORDER) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Contrast', y='abs_error', data=df, color='k', order=CONTRAST_ORDER, size=3) + plt.xlabel('Contrast') + plt.ylabel('Absolute CSA error [mm^2]') + plt.title(f'Method: {method}; Absolute CSA error for each contrast') + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + # set the y-axis limits + plt.ylim(-2, 10) + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for contrast in CONTRAST_ORDER: + mean = df[df['Contrast'] == contrast]['abs_error'].mean() + std = df[df['Contrast'] == contrast]['abs_error'].std() + plt.text(CONTRAST_ORDER.index(contrast), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + plt.tight_layout() + save_path = os.path.join(os.path.dirname(file_path), f"abs_error_per_contrast_{method}.png") + plt.savefig(save_path, dpi=300) + print(f'Figure saved to {save_path}') + + +def compute_cov(data, file_path): + """ + Compute COV for CSA for each method across resolutions + :param data: Pandas DataFrame with the data + :param file_path: Path to the CSV file (will be used to save the figure) + """ + # Compute COV for CSA ('MEAN(area)' column) for each method across resolutions + df = data.groupby(['Method', 'Resolution'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + for method in df['Method'].unique(): + df.loc[df['Method'] == method, 'COV'] = df.loc[df['Method'] == method, 'std'] / df.loc[ + df['Method'] == method, 'mean'] * 100 + df = df[['Method', 'Resolution', 'COV']].pivot(index='Resolution', columns='Method', values='COV') + # Compute mean +- std across resolutions for each method + df.loc['mean COV'] = df.mean() + df.loc['std COV'] = df.std() + # Keep only two decimals and save as csv + df = df.round(2) + df.to_csv(file_path.replace('.csv', '_COV.csv')) + print(f'COV saved to {file_path.replace(".csv", "_COV.csv")}') + # Print + print(df) + + +def main(file_path): + # Load the CSV file + data = pd.read_csv(file_path) + + # Apply the function to extract method and resolution + data['Contrast'], data['Method'] = zip(*data['Filename'].apply(extract_contrast_and_method)) + # data['Method'] = data['Filename'].apply(extract_method_resolution) + + # Apply the function to extract participant ID + data['Participant'] = data['Filename'].apply(fetch_participant_id) + + # # Fetch contrast (e.g. T1w, T2w, T2star) from the first filename using regex + # contrast = re.search(r'.*_(T1w|T2w|T2star).*', data['Filename'][0]).group(1) + + # # Generate violinplot across resolutions and methods + # generate_figure(data, contrast, file_path) + + # Generate violinplot showing STD across participants for each method + generate_figure_std(data, file_path) + + # Generate violinplot showing absolute CSA error across participants for each method + generate_figure_abs_csa_error(data, file_path) + + # Generate violinplot showing absolute CSA error for each contrast for a given method + generate_figure_abs_csa_error_per_contrast(data, 'monai_bin', file_path) + # generate_figure_abs_csa_error_per_contrast(data, 'monai_soft', file_path) + # generate_figure_abs_csa_error_per_contrast(data, 'nnunet', file_path) + + # # Compute COV for CSA for each method across resolutions + # compute_cov(data, file_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate violin plot from CSV data.') + parser.add_argument('-i', type=str, help='Path to the CSV file') + args = parser.parse_args() + main(args.i) \ No newline at end of file From 1aab0d4e8a7774bd8e4031e4b4d1d1ab66cd9ab3 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 03:12:03 -0500 Subject: [PATCH 35/66] rename training config file --- configs/{train_soft_all.yaml => train_all.yaml} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename configs/{train_soft_all.yaml => train_all.yaml} (92%) diff --git a/configs/train_soft_all.yaml b/configs/train_all.yaml similarity index 92% rename from configs/train_soft_all.yaml rename to configs/train_all.yaml index ef7e9516..77196a07 100644 --- a/configs/train_soft_all.yaml +++ b/configs/train_all.yaml @@ -3,9 +3,9 @@ save_test_preds: True directories: # Path to the saved models directory - models_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/saved_models + models_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/saved_models/followup # Path to the saved results directory - results_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/results + results_dir: /home/GRAMES.POLYMTL.CA/u114716/contrast-agnostic/results/models_followup # Path to the saved wandb logs directory # if None, starts training from scratch. Otherwise, resumes training from the specified wandb run folder wandb_run_folder: None @@ -18,7 +18,7 @@ dataset: # Type of contrast to be used for training. "all" corresponds to training on all contrasts contrast: all # choices: ["t1w", "t2w", "t2star", "mton", "mtoff", "dwi", "all"] # Type of label to be used for training. - label_type: soft # choices: ["hard", "soft"] + label_type: soft_bin # choices: ["hard", "soft", "soft_bin"] preprocessing: # Online resampling of images to the specified spacing. @@ -29,7 +29,7 @@ preprocessing: opt: name: adam - lr: 0.0001 + lr: 0.001 max_epochs: 200 batch_size: 2 # Interval between validation checks in epochs @@ -42,8 +42,8 @@ model: # Model architecture to be used for training (also to be specified as args in the command line) nnunet: # NOTE: these info are typically taken from nnUNetPlans.json (if an nnUNet model is trained) - base_num_features: 8 - max_num_features: 128 + base_num_features: 32 + max_num_features: 320 n_conv_per_stage_encoder: [2, 2, 2, 2, 2, 2] n_conv_per_stage_decoder: [2, 2, 2, 2, 2] pool_op_kernel_sizes: [ From 5960989893d291a8378567cb37a2d2126cd5b54f Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 03:13:09 -0500 Subject: [PATCH 36/66] revert to orignal version; no label binarization --- monai/main.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/monai/main.py b/monai/main.py index b628a937..add717c4 100644 --- a/monai/main.py +++ b/monai/main.py @@ -111,15 +111,10 @@ def prepare_data(self): transforms_val = val_transforms(crop_size=self.inference_roi_size, lbl_key='label') # load the dataset - if self.cfg["dataset"]["label_type"] == "softBin": - logger.info(f"Training with binarized soft labels (threshold 0.5) ...") - dataset = os.path.join(self.root, - f"dataset_{self.cfg['dataset']['contrast']}_soft_seed{self.cfg['seed']}.json" - ) - else: - dataset = os.path.join(self.root, - f"dataset_{self.cfg['dataset']['contrast']}_{self.cfg['dataset']['label_type']}_seed{self.cfg['seed']}.json" - ) + logger.info(f"Training with {self.cfg['dataset']['label_type']} labels ...") + dataset = os.path.join(self.root, + f"dataset_{self.cfg['dataset']['contrast']}_{self.cfg['dataset']['label_type']}_seed{self.cfg['seed']}.json" + ) logger.info(f"Loading dataset: {dataset}") train_files = load_decathlon_datalist(dataset, True, "train") val_files = load_decathlon_datalist(dataset, True, "validation") @@ -185,10 +180,6 @@ def training_step(self, batch, batch_idx): inputs, labels = batch["image"], batch["label"] - if self.cfg["dataset"]["label_type"] == "softBin": - # binarize soft labels with threshold 0.5 - labels = (labels > 0.5).float() - # check if any label image patch is empty in the batch if check_empty_patch(labels) is None: # print(f"Empty label patch found. Skipping training step ...") @@ -288,10 +279,6 @@ def validation_step(self, batch, batch_idx): inputs, labels = batch["image"], batch["label"] - if self.cfg["dataset"]["label_type"] == "softBin": - # binarize soft labels with threshold 0.5 - labels = (labels > 0.5).float() - # NOTE: this calculates the loss on the entire image after sliding window outputs = sliding_window_inference(inputs, self.inference_roi_size, mode="gaussian", sw_batch_size=4, predictor=self.forward, overlap=0.5,) From e78cb740bfef894ad9806ae5e1d9e3bc6b40676a Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 06:11:30 -0500 Subject: [PATCH 37/66] add pycache --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b9a5796b..e4857c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -*.idea \ No newline at end of file +*.idea +*__pycache__/ \ No newline at end of file From 009d207508cce924449c564e13bb74ce687ea0c7 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 06:12:29 -0500 Subject: [PATCH 38/66] add joblib splits file for datalist creation --- monai/split_datasets_all_seed=15.joblib | Bin 0 -> 50675 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 monai/split_datasets_all_seed=15.joblib diff --git a/monai/split_datasets_all_seed=15.joblib b/monai/split_datasets_all_seed=15.joblib new file mode 100644 index 0000000000000000000000000000000000000000..dc477fe1ff7c37cf370cb4f002f58488bb0e71c2 GIT binary patch literal 50675 zcmb7N*^VUFarJ}r2j2JHvH=n^-1TS}-W~)AAniAUsO~Awl-ONORrd@f0|xv7e560g z|KiVZ=824Z&xs|g#V4X7ZmcIFZbs!)|Mw66=l}lqH}KzoegC8He)x1boX+3>$MV0w z`S|wcy|=e_FFyM6i>zW6{s{)3hK=5l`chvVt#c>Cg|P$BF5 z?b{&h57$Evzx?Xq^6H=NFQ=OqKbBK}w32`R_WX1_zk2xe;}_q)_)zH3dH=95e119~ zPA@*r)L*A9&N=|A#qZ3NaAt>li4`2W4;a|_MkLS}N1mSmAd*9x^I=nkQRykSot6{^5^TX5A zx2L<{{O_#77A?njhd@Cz?@vpdgY$p99=SQ5ettT?{@La9cz+01!2QQJ$IH#-;o<9J zFeb(BGfARF{QOl&AzXNMIX=ERo`a!KXNddho14-YMDz2ne|3pYLfif8H$huYQGxw6 z&I$`0d-wKqdaN@COTT)&`}^BBr#0XT{S+7_<1k@gYxVajDgXSB0mTU3TFC$Q<|gE7 zRIwsPYeJoeArw(PnY>Uk@OJ;r>GtL^1P@~09q#V_<+$MH9D<_;sZ4}j3PugK*c>jm z$GeAfjJni@MjWJXj+dn`+y~5!Rujby_+G1NO&^@6R;z}@h%4$~=I!C~>h#rDA*LC} zdK7{QQ>qqmLtN3Q^(eGH9p79I=g)&yklq~bZ`NL`afs-6shRq%6~*`G)8}DoK%%!i zH4?zMIo!QoI?+=Ax6(A?N>eKtq@PN6Vv2$=fJ~JE6^Hw`0n~8I0j7}{0J{kh%qdkn zMnS6uIJPzIe?MMM54)jKk-;@J6oS$7ysK#}0Wlf|n!VxXp~8^EOV5lFQ$z6a+R9Zw zV(h)I@N42*#l@Oa8$hLIr6`#&)>8swEhnDC>Hq>P-A(BOlsCnmk?6v`)ERsVWz7rWvGJ7?pYxMj-;Q)}3mjyeMIs;qGZA zgvhBC+<#T9^iq{b6{3-X>FXnm@(+mU$I#%n=-q5 z)Q(|be3Hd6de~Q8Y1Sy&OtD?Y2lXz4K8_YH_7inPc8LXjoGe&&8AJve<1T~f(Y+-q zkxFS;yd&y}8Fm>&fKbZMQUFBN$0YS;kDi6-nFi=Bc#5Hj6!0$Us$~aa!1QVuIE#ii_`YwYUg;u{Cs8o<>mqDi@I@VnV zk>{OArv@Z?%TptDC!yk$rV&?~W{W|(+M;>i}kzi zGAP&seCl_`NMQ~|Q=w}Jl3fO=g!^NcK?H+|?K0BG%l$$vpDEMNi_4DMKGf*Ps2SO1 zkPYV1ydY_eXSDobzlf#)LQS5oXj-RS+GRwmXa;E(Mx|nxLB~ejsW!@s5~dmMo>oGL zoLa&CSH(&%Rf$v~8mR$S<+qeB5YdmJ!Ee#K+17lO>p`1_~2+ zvF$P_T{--!G1{ZP454~xLQZWopwH6nPU#~6Ct1@E<7J8|9pR!~2Kfz=-&fQQko<%lKe<72z^o%`nPh&@+a%p zY~dXZ<@nQ;Bds>FesXw?Q)ZyttS$=!?rGzF4W+?5N?mdv5naTMw>I?rh+EQ{oL?~` zc0o6JmKbt$Y8L5eDL%lim+D#uV#I(%ea?G#iVH^0e4#^yBI0uoA`1VE-XNIYTIhMp z%4{LYwiMj88;Tn9Yukwk1@Z8Pm;8e2uo;o#U_iv@azq%Q+4mzt$@%Qlk?@4WX;C8b zVH(ZFb+`#b3R~iU2r8x&A4Jce4Qlg!H6p6Dj?%#x)$18g96;oJ~1O8PMPRui9JuNDyCOwKy|X5>fhc}sL=P)D zB}U09wG8(4f-WXAF(Em?G${t0eD5M+1)>T51ERKlk&_t($tdRy#tODNflhu5igAL-nFEQBaM#<`Ko_>Ad_UPiHuN z8&XGH#&vRH`j?O}6?i{L3ScD7&6Kav4{Y}%9*(+Bnkk8iHm!2EQ+0*VCZl*TB>B?b zlV~cu6v~18bjnb$t%W28(B`+HYZyroK=`R#k~3hSt8ij~ey{eVAF&qvBm}@U-xY~# zIZ;V_G%qp^k3mYj|AAh;aN%{Vy<^1zizb~+~Wl-r)PNz&zQXbppYOwD}^f*qKI<~^=Bx&>LMa2yN8c^+7 zQFmj_*O>^_G}`+}t24b`AWXVU@3`fJR)L+F(rbDf!8VtJowP9{)-7SjG)n!~X{bH? z5NNtp^ok=I`iTN}{chf~<(lt#p`7ZeHxs!-p0{md7f4v?ciLVv*yjGRlNNi!vWVpSF%NS6`T+*m~oDeA{exej;pMXnh7o zp{={lPF99z?*mM;?Pn)WL)(iA;pm-!ec>&%PC3elE*(9hMrhVOXeUlerV7o&1Tpm+D) zxY!Vvg=?4LQ9XU|GT5)ULTT$eij@T~WjhlQYTwRwOon%?Y4=&MukBbSmimYlPaJ(b zhFLuenhx)DJL4>3bD7iUe`d_roma9epzmFu;prhdbd`V<=#Epj5 zTSr*xnWhUKzP=rY?DfX+K<-t`5bd>6_vy1WO1qb1@WhU}3o0z~NBU=@xW{H}?0qc2 zNe_oTCv5b#*xQJ_3e@|uKxAx1-9;wDq_;5o(#K~g?uE@V?0vOnG282Nr1d)v25%2> z1SG7wyWk;Zz7pw;1Dii6^lawsY>hqQsgd+dd|kMrlxAI@Q01I)U)E92wS%wZh!IK@;h8# z{^X0#$Ew}PShZJVpocts`DMigA+}<#$N;{+|5>r&$tJu6TJc!{^KW*wT!hn4O3U#@ zi?7pLF@%>7N-^kSv_`KajY!v)q#%7%YKqR+Tsf3Gd@@BoF11=MjNbt3-CMS)I}&{K z)YS&?oY|AS_ z;6k>1Q(kew8rzaroAF8=Y^ICPhh+p)pldb=qnfURiFo^i{*H6D z+^&q)a;`q6$F=4%HrvTkU3U73Ygds){ufyH-e zb;d>)UKiGtno(iTZDX|_1d*m$tt@7?E&#aY_w<@2ms-sSg#_!`EaU@u+ZNJQpkS5X zF>7sLnTzO3N-W1^30<+_HKT0-UD1I?+A3e?sne!%Y6x%zeOg+}9Cc*`#Mj0Eincwj z{1LR*VH;>)x5kx>jGNkCm}1-FN{oiB77<*o4W7N;*C4gsz%dQg8pp%^bu(PqEf}t4 zjDAoX46<#5TLnHKcH%ONtyU8oMQtm%sB_oooV8R-;mc7S{oL1UKDtCb&#jsZS@Ql@ zYl>}TS>0CY7%;9q&*-aD7UNu(uVHlV5^Bp4BVW_DHrt!DJsIoSiae8DN2IvM);?Pu zEKhGW8P(0}*vc4-$lN#75t&)=J>pKbomvG97Wl>QS?H^A6|1)P z7hLodQFBR4s%=7^1fDgl;JvL(EAMl(8;Z60rF7bBD#>NT}ms4$Y2K1C(mIX5f)o+X6*2ns>~h>v^4pQBJigvUcaZZ8=>o>?0-EM8ZZtx2-3thmx8ptcDhpKHpWeDQFWJ*Cmdo-)y)hJ3Ot zBsPedo61B-Vdqt4;=_b4c9n?{c2STllr0uuM!N z=8ZKeSO&@@C&r**%Q4+!lqTLp6;D9kYxWSa0&h=-54t9rXnV4`Ofj(by8U^BwN4TT zzQc%b|CXGVR$1UUEtBBW|A|SwQn^O$Edt#H@XHW zh$clV)XVK*PmnPku=mvotjoN~JLf0G`XxqzK{hGHHr( zl!>oHla!3L#^})t8*?>&jga7ZWA&UIzMp8)0sN<>X!3DH!MCD`L$KdKZ8Vg|YS8Cm zwAQiqk@YY4?8RqRYNIK(n#N$T(HLj|JL|BKBX!s`2EL^=$;y+YVWjI^69BQO31tk@ zo+$6fjX|-aZM)W#;ZY(@ATzG_Y^TgMC1>!D`_^O@;>iow#9`209snD|2lTH(s9`AX z-;L|Tl!lHD64$QDA&bns#W#ta3GvNh6M&4}#?e_!4c?|o?`@4rxIovA1_PtF`Svx% z0uf#NQezRpqv4n~dzhLo5R%iF%oDQFD6_3mMs1==20ULE5{-lb>xK?1{IdEyJT^c> zGO3^&x4}kFLJb@2@}AY$l@Z(E3_+ON*aV-P{Kl$id=rYHuMsmVsPPPXXbW>yPV;Ea zFl>;DK>O2qqX#^GZQkSYn`{#ob8EJ&29s3p4F`|n#vbS*IPRH5j~7isMwvC*$mBf< zb3dCf2z$wpJ$nfmO@OTImuc773vJYS8>5F?zQe7}ru}4NPr6>ze8=y8xTNg#3V;38 zf1%`Or@M!z<%cooFTjH2&jrhmTM)^9CUx1zEk7+nALExp=o-EJ_zY3;YZu!BZu!9p zqTq)==37jv3qmQ1dQ^;De)ol>g5=#zNzqVseE!<6S^_iYm7x$xEqD3F4zdMD=Y3q| z((gB`Oc3nCRe0@>&zb{(Qr?MInV`!aMnxkuj{a9W+D`T5;O4B((%*E`Z3(2-loQY3bfiTTF;I)q!AMj3X=B(Rb|4K z4ZD7fYlLq6)d)$Qh6L8rxKUd`rAkx1D3`I2($qoa@=l*OZTeAZkf>=BX+&#LL?6Qp zZQ4X-$S%$iCDiN^6e2<>fmV--k&A0|5F}5KlA@tYH0j`kvsw}`TZp6-EJaBoaa5W% z5w&SY!f4t=fNS1vGzYnM)XuVLljsN~4R`QERpY2OZ4#WCHqinARHq&+yopUa3Ff9v zM2Vv2t7_VGGJ2|qwQ18at^hp(Q>bavA?PCfD<;dPO{bx9Hf^Fzlxr3xDw>H<07}fj zQ%kvNlNoSNn|7pX((^#I|4_}rK|+!YS4t9FLt7nE}z&h&ZYAKHc7u8Pw|6`ssD=s zra?x4&?U?%xp<-uw|9*)LP!xvB9P3o#oEVPpuy=zyt9h+8$)TTlv#Nwqt_ z1{vG9bnRCjNp%VsTnckUTml1?ak36&#E$@~9_yt!P`ZDx47NF0IyU6jMohmc54dcjsORgo1Ht5Ttz$yy=78(i@=r7L{^p;_=>MfWQbMumIYVsvUy*=F zngzW`ELfKs_OaMGAAtQ_fmb3FCg#3!8l4@ZWU|Lt{27B~71l|Nk?e&6YCc{&aok0E zQE+1p+)fy5lP9u{Gylv=j~$peW^pHRc86Y)WTQt+Xd*phF2KG!SntQ+?p#3lWPiR#-Vap|Ru;Ln>wQeoq(GH8Zu{tiM1aq&l!LF-zW84;M zbtVN8b?6WqnRS&$a8a|X%|-#zjH@L1)oS}HY-LHWPk=`UT;c(==7dXhRA(A-i475w z8JAeBXlUnJ+6;YES{CgjOkA@TRr5GN8lLE0^G1tJmi1Np=FJ&T?AXip&QbcU0VTsB zEDpQGWo?6|UCqJcE^#1?I`0zCe4d<(*=Kpzq%L@8C6KftF=^%{Hng=c^b%7*USpSv zYVd+!6mu`>l*l%D#(>C^FUg9rXv6^!Jo}O$uF^&b?AGa*#KanHY67t;=3kNrvt$4! zmRW!eUL@X4hC`7yT-FiC5KN}P`!oe>wloG4+vvYd=Mlg5c)Zay@i9GP6ec@tO;F6j z#3=}AVu%=}X_!QTjrw6@`*v%+dwn@QJ~gk{AHVy-)A8{M@Al!#(e;hoXB=S>w~ON%t4dmxVRa@wLS4*81)ry)B_qu|ua@ zl}phRL!|=o4c{1WR~ z_b3BJnHuWU_M#A#0V3_BnhEHv7i;B5gty9G{t}Xg5s9Ojp+jQh}*!)5(BkJqXNE*N*3AFIvi*h&bn{Qv{K7g)WDq`dxKx(w4e5(I^8&nU1ba zCyGK;=OA))Z8{0)Y+ai^wp@?-rf@^zv}@B*P_=6l6;Ya1tqK9&@U#&=5Kx zWhJ;eDKr2i^l@RpLr%JYs}O{eySYdJl^c4ddUdT$FH=3c$#^(s&-sZK9b2IxqUrF8 zVxb|Taxqsr&RW$ZxlpXvb3L3hn%o|UNx9C4lko$H>9EB_`@Z-M)Z=@sWYP$ib$>aA zaLKh@J;vEN1LQ1hmXc_7bchEPH%6K0X&0Pp|H2|3Dil$;!Q?T$636h(Z45w)!H0RolLNKG52qBfX#u@ zM-dMDWa?B#hx<32NL>F446?yPtJ4c9df<^^OeC_u?#cJjpsg z(9^b&*71z+uCuLULU?(|b&Qba)2?H}27Zjajtg$mU7|+_zCl@9hY!L|Vkp^OAb>tc z9n?Smq%xCcwDp`@BUbDVdsSVl0{ zI~LsJY{0KyB`?cxc{2}WdvvVG`Nf+0wT;O)hk7LsIO8YDipj2G_`1E;hTbNFyb{A$ z++G&2Z@pp+5XCvbD_I-#Umf0^9+xU_S>}9iS-h>(uEBSo>y?b?yM4JFT_Q2{D?LaA z&+%5?1_p;$4p?IQBz;*>VXW@GROwa>~V0-=t>GI0_78;j8zWLzd8F&hm$(D=f@OtM1NHlhG)9hpfI zc)UCsV3YfMn!JFT5aBYG$7ZrHJJQ0V0iO()pU?M&LH0B3lC{Xw)H^Y`18RDH`d=x9|TyL86?d literal 0 HcmV?d00001 From 97d0a098d852a95b5dd8961e820d6f4b78dcd74a Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 06:46:27 -0500 Subject: [PATCH 39/66] add script for nnunet inference --- nnUnet/run_inference_single_subject.py | 233 +++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 nnUnet/run_inference_single_subject.py diff --git a/nnUnet/run_inference_single_subject.py b/nnUnet/run_inference_single_subject.py new file mode 100644 index 00000000..1644877a --- /dev/null +++ b/nnUnet/run_inference_single_subject.py @@ -0,0 +1,233 @@ +""" +This script is used to run inference on a single subject using a nnUNetV2 model. + +Note: conda environment with nnUNetV2 is required to run this script. +For details how to install nnUNetV2, see: +https://github.com/ivadomed/utilities/blob/main/quick_start_guides/nnU-Net_quick_start_guide.md#installation + +Author: Jan Valosek, Naga Karthik + +Example spinal cord segmentation: + python run_inference_single_subject.py + -i sub-001_T2w.nii.gz + -o sub-001_T2w_seg_nnunet.nii.gz + -path-model /path/to/model + -pred-type sc + -tile-step-size 0.5 + +Example lesion segmentation: + python run_inference_single_subject.py + -i sub-001_T2w.nii.gz + -o sub-001_T2w_lesion_seg_nnunet.nii.gz + -path-model /path/to/model + -pred-type lesion + -tile-step-size 0.5 +""" + + +import os +import shutil +import subprocess +import argparse +import datetime + +import torch +import glob +import time +import tempfile + +# from nnunetv2.inference.predict_from_raw_data import predict_from_raw_data as predictor +from nnunetv2.inference.predict_from_raw_data import nnUNetPredictor +from batchgenerators.utilities.file_and_folder_operations import join + + +def get_parser(): + # parse command line arguments + parser = argparse.ArgumentParser(description='Segment an image using nnUNet model.') + parser.add_argument('-i', help='Input image to segment. Example: sub-001_T2w.nii.gz', required=True) + parser.add_argument('-o', help='Output filename. Example: sub-001_T2w_seg_nnunet.nii.gz', required=True) + parser.add_argument('-path-model', help='Path to the model directory. This folder should contain individual ' + 'folders like fold_0, fold_1, etc. and dataset.json, ' + 'dataset_fingerprint.json and plans.json files.', required=True, type=str) + parser.add_argument('-pred-type', choices=['sc', 'lesion'], + help='Type of prediction to obtain. sc: spinal cord segmentation; lesion: lesion segmentation.', + required=True, type=str) + parser.add_argument('-use-gpu', action='store_true', default=False, + help='Use GPU for inference. Default: False') + parser.add_argument('-use-best-checkpoint', action='store_true', default=False, + help='Use the best checkpoint (instead of the final checkpoint) for prediction. ' + 'NOTE: nnUNet by default uses the final checkpoint. Default: False') + parser.add_argument('-tile-step-size', default=0.5, type=float, + help='Tile step size defining the overlap between images patches during inference. ' + 'Default: 0.5 ' + 'NOTE: changing it from 0.5 to 0.9 makes inference faster but there is a small drop in ' + 'performance.') + return parser + + +def get_orientation(file): + """ + Get the original orientation of an image + :param file: path to the image + :return: orig_orientation: original orientation of the image, e.g. LPI + """ + + # Fetch the original orientation from the output of sct_image + sct_command = "sct_image -i {} -header | grep -E qform_[xyz] | awk '{{printf \"%s\", substr($2, 1, 1)}}'".format( + file) + orig_orientation = subprocess.check_output(sct_command, shell=True).decode('utf-8') + + return orig_orientation + + +def tmp_create(): + """ + Create temporary folder and return its path + """ + prefix = f"sciseg_prediction_{datetime.datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}_" + tmpdir = tempfile.mkdtemp(prefix=prefix) + print(f"Creating temporary folder ({tmpdir})") + return tmpdir + + +def splitext(fname): + """ + Split a fname (folder/file + ext) into a folder/file and extension. + Note: for .nii.gz the extension is understandably .nii.gz, not .gz + (``os.path.splitext()`` would want to do the latter, hence the special case). + Taken (shamelessly) from: https://github.com/spinalcordtoolbox/manual-correction/blob/main/utils.py + """ + dir, filename = os.path.split(fname) + for special_ext in ['.nii.gz', '.tar.gz']: + if filename.endswith(special_ext): + stem, ext = filename[:-len(special_ext)], special_ext + return os.path.join(dir, stem), ext + # If no special case, behaves like the regular splitext + stem, ext = os.path.splitext(filename) + return os.path.join(dir, stem), ext + + +def add_suffix(fname, suffix): + """ + Add suffix between end of file name and extension. Taken (shamelessly) from: + https://github.com/spinalcordtoolbox/manual-correction/blob/main/utils.py + :param fname: absolute or relative file name. Example: t2.nii.gz + :param suffix: suffix. Example: _mean + :return: file name with suffix. Example: t2_mean.nii + Examples: + - add_suffix(t2.nii, _mean) -> t2_mean.nii + - add_suffix(t2.nii.gz, a) -> t2a.nii.gz + """ + stem, ext = splitext(fname) + return os.path.join(stem + suffix + ext) + + +def main(): + parser = get_parser() + args = parser.parse_args() + + fname_file = args.i + fname_file_out = args.o + print(f'Found {fname_file} file.') + + # Create temporary directory in the temp to store the reoriented images + tmpdir = tmp_create() + # Copy the file to the temporary directory using shutil.copyfile + fname_file_tmp = os.path.join(tmpdir, os.path.basename(fname_file)) + shutil.copyfile(fname_file, fname_file_tmp) + print(f'Copied {fname_file} to {fname_file_tmp}') + + # Get the original orientation of the image, for example LPI + orig_orientation = get_orientation(fname_file_tmp) + + # Reorient the image to RPI orientation if not already in RPI + if orig_orientation != 'RPI': + # reorient the image to RPI using SCT + os.system('sct_image -i {} -setorient RPI -o {}'.format(fname_file_tmp, fname_file_tmp)) + + # NOTE: for individual images, the _0000 suffix is not needed. + # BUT, the images should be in a list of lists + fname_file_tmp_list = [[fname_file_tmp]] + + # Use all the folds available in the model folder by default + folds_avail = [int(f.split('_')[-1]) for f in os.listdir(args.path_model) if f.startswith('fold_')] + + # Create directory for nnUNet prediction + tmpdir_nnunet = os.path.join(tmpdir, 'nnUNet_prediction') + fname_prediction = os.path.join(tmpdir_nnunet, os.path.basename(add_suffix(fname_file_tmp, '_pred'))) + os.mkdir(tmpdir_nnunet) + + # Run nnUNet prediction + print('Starting inference...') + start = time.time() + + # instantiate the nnUNetPredictor + predictor = nnUNetPredictor( + tile_step_size=args.tile_step_size, # changing it from 0.5 to 0.9 makes inference faster + use_gaussian=True, # applies gaussian noise and gaussian blur + use_mirroring=False, # test time augmentation by mirroring on all axes + perform_everything_on_gpu=True if args.use_gpu else False, + device=torch.device('cuda') if args.use_gpu else torch.device('cpu'), + verbose=False, + verbose_preprocessing=False, + allow_tqdm=True + ) + print('Running inference on device: {}'.format(predictor.device)) + + # initializes the network architecture, loads the checkpoint + predictor.initialize_from_trained_model_folder( + join(args.path_model), + use_folds=folds_avail, + checkpoint_name='checkpoint_final.pth' if not args.use_best_checkpoint else 'checkpoint_best.pth', + ) + print('Model loaded successfully. Fetching test data...') + + # NOTE: for individual files, the image should be in a list of lists + predictor.predict_from_files( + list_of_lists_or_source_folder=fname_file_tmp_list, + output_folder_or_list_of_truncated_output_files=tmpdir_nnunet, + save_probabilities=False, + overwrite=True, + num_processes_preprocessing=8, + num_processes_segmentation_export=8, + folder_with_segs_from_prev_stage=None, + num_parts=1, + part_id=0 + ) + + end = time.time() + print('Inference done.') + total_time = end - start + print('Total inference time: {} minute(s) {} seconds'.format(int(total_time // 60), int(round(total_time % 60)))) + + # Copy .nii.gz file from tmpdir_nnunet to tmpdir + pred_file = glob.glob(os.path.join(tmpdir_nnunet, '*.nii.gz'))[0] + shutil.copyfile(pred_file, fname_prediction) + + print('Re-orienting the prediction back to original orientation...') + # Reorient the image back to original orientation + # skip if already in RPI + if orig_orientation != 'RPI': + # reorient the image to the original orientation using SCT + os.system('sct_image -i {} -setorient {} -o {}'.format(fname_prediction, orig_orientation, fname_prediction)) + print(f'Reorientation to original orientation {orig_orientation} done.') + + # split the predictions into sc-seg and lesion-seg + if args.pred_type == 'sc': + # keep only the spinal cord segmentation + os.system('sct_maths -i {} -bin 0 -o {}'.format(fname_prediction, fname_file_out)) + elif args.pred_type == 'lesion': + # keep only the lesion segmentation + os.system('sct_maths -i {} -bin 1 -o {}'.format(fname_prediction, fname_file_out)) + + print('Deleting the temporary folder...') + # Delete the temporary folder + shutil.rmtree(tmpdir) + + print('-' * 50) + print(f'Created {fname_file_out}') + print('-' * 50) + + +if __name__ == '__main__': + main() \ No newline at end of file From b26862f0f422990e03064cb48a49b1c708d0c30c Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 06:51:27 -0500 Subject: [PATCH 40/66] add script for reproducing training --- scripts/train.sh | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 scripts/train.sh diff --git a/scripts/train.sh b/scripts/train.sh new file mode 100644 index 00000000..f3f584f9 --- /dev/null +++ b/scripts/train.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# This script does the following: +# 1. Creates a virtual environment and installs the required dependencies +# 2. Generates a MSD-style datalist containing image/label pairs for training +# 3. Trains the contrast-agnostic soft segmentation model +# 4. Evaluates the model on the test set +# +# Usage: +# bash train.sh +# +# Examples: +# 1. Train a model on 'T1w' contrast with 'hard' labels +# bash train.sh /path/to/spine-generic-processed/ t1w hard train.yaml +# 2. Train a model on 'all' contrasts with 'soft' labels +# bash train.sh /path/to/spine-generic-processed/ all soft train.yaml +# +# + + +# Uncomment for full verbose +# set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Set the following variables to the desired values +# Path to the pre-processed spine-generic dataset +PATH_DATA=$1 +CONTRAST=$2 # options: ["t1w", "t2w", "t2star", "mton", "mtoff", "dwi", "all"] +LABEL_TYPE=$3 # options: ["hard", "soft", "soft_bin"] +PATH_TRAIN_YAML=$4 # path to the yaml file containing the training configuration + +PATH_DATALIST_OUT="../datalists/spine-generic/temp/seed15/" +# create folder if doesn't exist +if [ ! -d $PATH_DATALIST_OUT ]; then + mkdir -p $PATH_DATALIST_OUT +fi + +MODEL="nnunet" + +# # Create virtual environment +# conda create -n venv_monai python=3.9 -y + +# NOTE: running conda activate errors out requesting conda init to be run, +# the eval expression here makes it work without conda init. +# source: https://stackoverflow.com/questions/34534513/calling-conda-source-activate-from-bash-script +eval "$(conda shell.bash hook)" +conda activate venv_monai + +# # Install dependencies quietly +# pip install -r monai/requirements.txt --quiet + +# Run the script to generate the datalist JSON file +python monai/create_msd_data.py \ + --path-data $PATH_DATA \ + --path-out $PATH_DATALIST_OUT \ + --path-joblib "monai/split_datasets_all_seed=15.joblib" \ + --contrast $CONTRAST \ + --label-type $LABEL_TYPE \ + --seed 15 + +echo "----------------------------------------" +echo "Training contrast-agnostic '$MODEL' model on '$CONTRAST' contrasts with '$LABEL_TYPE' labels" +echo "----------------------------------------" + +# Train the model +python monai/main.py --model $MODEL --config $PATH_TRAIN_YAML --debug From f2f4f43fced0bbc0e11c7a7b0523216d61ac34f6 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 06:52:05 -0500 Subject: [PATCH 41/66] add script for running inference and generating csa plots --- scripts/analyze_results_across_labels.sh | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 scripts/analyze_results_across_labels.sh diff --git a/scripts/analyze_results_across_labels.sh b/scripts/analyze_results_across_labels.sh new file mode 100644 index 00000000..e1e59720 --- /dev/null +++ b/scripts/analyze_results_across_labels.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# +# This script does the following: +# 1. Runs inference on the test set using the trained monai models across different label types +# 2. Generates CSA violin plots +# +# Usage: +# bash analyze_results_across_labels.sh +# +# Example config file: +# { +# "path_data" : "", +# "path_output" : "", +# "script" : "csa_qc_evaluation_spine_generic/comparison_across_training_labels.py", +# "jobs" : 5, +# "script_args" : "nnUnet/run_inference_single_subject.py monai/run_inference_single_image.py ", +# "include_list": ["sub-amu02", "sub-amu05", "sub-balgrist06", "sub-barcelona05", "sub-beijingPrisma02", "sub-beijingPrisma03", "sub-brnoUhb04", "sub-brnoUhb06", "sub-cardiff01", "sub-cmrra01", "sub-cmrra04", "sub-cmrrb01", "sub-cmrrb07", "sub-fslPrisma05", "sub-geneva03", "sub-hamburg03", "sub-mgh02", "sub-mgh05", "sub-milan02", "sub-milan04", "sub-milan05", "sub-mniS06", "sub-mountSinai01", "sub-nottwil04", "sub-oxfordFmrib02", "sub-oxfordOhba03", "sub-oxfordOhba04", "sub-oxfordOhba05", "sub-pavia02", "sub-pavia03", "sub-pavia05", "sub-perform05", "sub-sherbrooke02", "sub-stanford06", "sub-strasbourg03", "sub-strasbourg05", "sub-tehranS02", "sub-tehranS04", "sub-tehranS05", "sub-tokyo750w03", "sub-tokyo750w05", "sub-tokyo750w07", "sub-tokyoSkyra07", "sub-ucl03", "sub-unf05", "sub-vuiisAchieva01", "sub-vuiisAchieva02", "sub-vuiisAchieva06", "sub-vuiisIngenia04"] +# } + + +# Uncomment for full verbose +# set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Set the following variables to the desired values +PATH_SCT_CONFIG=$1 + +# Run the sct_run_batch script +sct_run_batch -config $PATH_SCT_CONFIG + +# get "path_output" key from the config file +PATH_OUTPUT=$(python -c "import json; print(json.load(open('$PATH_SCT_CONFIG'))['path_output'])") + +# Generate CSA violin plots +python csa_generate_figures/analyse_csa_across_training_labels.py \ + -i ${PATH_OUTPUT}/results/csa_label_types_c24.csv \ No newline at end of file From b7f920e554e1d76e73774a1a265299f47c36d0ef Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 5 Feb 2024 06:52:19 -0500 Subject: [PATCH 42/66] add Readme --- scripts/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..2dbf40c6 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,7 @@ +This folder contains `bash` scripts for reproducing the training and/or various CSA analyses. + +### Training +To train various types of contrast-agnostic soft segmentation models, use the `train.sh` script. Usage examples are provided in the script. + +### Analyzing Results +To analyze the results of the trained models, use the `analyze_results_*` scripts. Usage examples are provided in the script. From cb4c9dea12474f5b1f1ba32984ece6fb455b1e57 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 12 Feb 2024 10:55:30 -0500 Subject: [PATCH 43/66] rename perform_everything_on_device nnUNetPredictor arg --- nnUnet/run_inference_single_subject.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nnUnet/run_inference_single_subject.py b/nnUnet/run_inference_single_subject.py index 1644877a..b417101f 100644 --- a/nnUnet/run_inference_single_subject.py +++ b/nnUnet/run_inference_single_subject.py @@ -166,7 +166,7 @@ def main(): tile_step_size=args.tile_step_size, # changing it from 0.5 to 0.9 makes inference faster use_gaussian=True, # applies gaussian noise and gaussian blur use_mirroring=False, # test time augmentation by mirroring on all axes - perform_everything_on_gpu=True if args.use_gpu else False, + perform_everything_on_device=True if args.use_gpu else False, device=torch.device('cuda') if args.use_gpu else torch.device('cpu'), verbose=False, verbose_preprocessing=False, From d3b0f75084ad00fde7db315d0420de63ef355822 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Mon, 12 Feb 2024 10:56:43 -0500 Subject: [PATCH 44/66] add unified qc generation script --- qc_other_datasets/generate_qc.sh | 297 +++++++++++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 qc_other_datasets/generate_qc.sh diff --git a/qc_other_datasets/generate_qc.sh b/qc_other_datasets/generate_qc.sh new file mode 100644 index 00000000..4506cf9d --- /dev/null +++ b/qc_other_datasets/generate_qc.sh @@ -0,0 +1,297 @@ +#!/bin/bash +# +# Compare the CSA of soft GT thresholded at different values on spine-generic test dataset. +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# # Uncomment for full verbose +# set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 +QC_DATASET=$2 # dataset name to generate QC for +PATH_NNUNET_SCRIPT=$3 # path to the nnUNet contrast-agnostic run_inference_single_subject.py +PATH_NNUNET_MODEL=$4 # path to the nnUNet contrast-agnostic model +# PATH_MONAI_SCRIPT=$3 # path to the MONAI contrast-agnostic run_inference_single_subject.py +# PATH_MONAI_MODEL_SOFT=$4 # path to the MONAI contrast-agnostic model trained on soft labels +# PATH_MONAI_MODEL_SOFTBIN=$5 # path to the MONAI contrast-agnostic model trained on soft_bin labels + +echo "SUBJECT: ${SUBJECT}" +echo "QC_DATASET: ${QC_DATASET}" +echo "PATH_NNUNET_SCRIPT: ${PATH_NNUNET_SCRIPT}" +echo "PATH_NNUNET_MODEL: ${PATH_NNUNET_MODEL}" +# echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +# echo "PATH_MONAI_MODEL_SOFT: ${PATH_MONAI_MODEL_SOFT}" +# echo "PATH_MONAI_MODEL_SOFTBIN: ${PATH_MONAI_MODEL_SOFTBIN}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Get ANIMA binaries path +anima_binaries_path=$(grep "^anima = " ~/.anima/config.txt | sed "s/.* = //" | sed 's/\/$//') + +# Compute ANIMA segmentation performance metrics +compute_anima_metrics(){ + local FILESEG="$1" + local FILEGT="$2" + + # We have to copy qform matrix from seg-manual to the automatically generated segmentation to avoid ITK error: + # "Description: ITK ERROR: SegmentationMeasuresImageFilter(): Inputs do not occupy the same physical space!" + # Related to the following issue : https://github.com/spinalcordtoolbox/spinalcordtoolbox/pull/4135 + sct_image -i ${FILEGT}.nii.gz -copy-header ${FILESEG}.nii.gz -o ${FILESEG}_updated_header.nii.gz + + # Compute ANIMA segmentation performance metrics + # -i : input segmentation + # -r : GT segmentation + # -o : output file + # -d : surface distances evaluation + # -s : compute metrics to evaluate a segmentation + # -X : stores results into a xml file. + ${anima_binaries_path}/animaSegPerfAnalyzer -i ${FILESEG}_updated_header.nii.gz -r ${FILEGT}.nii.gz -o ${PATH_RESULTS}/${FILESEG} -d -s -X + + rm ${FILESEG}_updated_header.nii.gz +} + +# Copy GT segmentation (located under derivatives/labels) +copy_gt_seg(){ + local file="$1" + local label_suffix="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels/${SUBJECT}/anat/${file}_${label_suffix}.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_seg-manual.nii.gz + else + echo "File ${FILESEG}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + +# TODO: Fix the contrast input for deepseg and propseg (i.e. dwi, mton, mtoff won't work) +# Segment spinal cord using methods available in SCT (sct_deepseg_sc or sct_propseg), resample the prediction back to +# native resolution and compute CSA in native space +segment_sc() { + local file="$1" + local contrast="$2" + local method="$3" # deepseg or propseg + local kernel="$4" # 2d or 3d; only relevant for deepseg + local file_gt_vert_label="$5" + local native_res="$6" + + # Segment spinal cord + if [[ $method == 'deepseg' ]];then + FILESEG="${file}_seg_${method}_${kernel}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_deepseg_sc -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -kernel ${kernel} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + elif [[ $method == 'propseg' ]]; then + FILESEG="${file}_seg_${method}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_propseg -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Remove centerline (we don't need it) + rm ${file}_centerline.nii.gz + + fi + + # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + +} + +# Segment spinal cord using the contrast-agnostic nnUNet model +segment_sc_nnUNet(){ + local file="$1" + local kernel="$2" # 2d or 3d + + # FILESEG="${file}_seg_nnunet_${kernel}" + FILESEG="${file}_seg_nnunet" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_NNUNET_SCRIPT} -i ${file}.nii.gz -o ${FILESEG}.nii.gz -path-model ${PATH_NNUNET_MODEL}/nnUNetTrainer__nnUNetPlans__${kernel} -pred-type sc -use-gpu + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Generate QC report + sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # # Compute CSA from the prediction resampled back to native resolution using the GT vertebral labels + # sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + +} + +# Segment spinal cord using the MONAI contrast-agnostic model +segment_sc_MONAI(){ + local file="$1" + local label_type="$2" # soft or soft_bin + + if [[ $label_type == 'soft' ]]; then + FILEPRED="${file}_seg_monai_soft" + PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFT} + + elif [[ $label_type == 'soft_bin' ]]; then + FILEPRED="${file}_seg_monai_bin" + PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFTBIN} + + fi + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu + # Rename MONAI output + mv ${file}_pred.nii.gz ${FILEPRED}.nii.gz + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILEPRED},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Binarize MONAI output (which is soft by default); output is overwritten + sct_maths -i ${FILEPRED}.nii.gz -bin 0.5 -o ${FILEPRED}.nii.gz + + # Generate QC report with soft prediction + sct_qc -i ${file}.nii.gz -s ${FILEPRED}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # compute ANIMA metrics + compute_anima_metrics ${FILEPRED} ${file}_seg-manual + +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' +if [[ $QC_DATASET == "sci-colorado" ]]; then + contrast="T2w" + label_suffix="seg-manual" + +elif [[ $QC_DATASET == "basel-mp2rage-rpi" ]]; then + contrast="UNIT1" + label_suffix="label-SC_seg" + +elif [[ $QC_DATASET == "dcm-zurich" ]]; then + contrast="acq-axial_T2w" + label_suffix="label-SC_mask-manual" + +elif [[ $QC_DATASET == "stanford-epi" ]]; then + contrast="task-rest_desc-mocomean_bold" + label_suffix="spinalcord_mask" +fi +# TODO: add stanford EPI data for QC + +echo "Contrast: ${contrast}" + +# Copy source images +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/anat/${SUBJECT//[\/]/_}*${contrast}.* . + +# Go to the folder where the data is +cd ${PATH_DATA_PROCESSED}/${SUBJECT}/anat + +# Get file name +file="${SUBJECT}_${contrast}" + +# Check if file exists +if [[ ! -e ${file}.nii.gz ]]; then + echo "File ${file}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}.nii.gz does not exist. Exiting." + exit 1 +fi + +# Copy GT spinal cord segmentation +copy_gt_seg "${file}" "${label_suffix}" + +# Segment SC using different methods, binarize at 0.5 and compute QC +# segment_sc_MONAI ${file} 'soft' +# segment_sc_MONAI ${file} 'soft_bin' + +segment_sc_nnUNet ${file} '3d_fullres' +# # TODO: run on deep/progseg after fixing the contrasts for those +# segment_sc ${file_res} 't2' 'deepseg' '2d' "${file}_seg-manual" ${native_res} +# segment_sc ${file_res} 't2' 'propseg' '' "${file}_seg-manual" ${native_res} + + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From ff85f674883a381e82125bbbcaaa50617f90908e Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sat, 17 Feb 2024 02:27:30 -0500 Subject: [PATCH 45/66] add einops requirement for swinunetr --- monai/requirements_inference_cpu.txt | 2 +- monai/requirements_inference_gpu.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/requirements_inference_cpu.txt b/monai/requirements_inference_cpu.txt index 8ef65cc8..2bff77b5 100644 --- a/monai/requirements_inference_cpu.txt +++ b/monai/requirements_inference_cpu.txt @@ -1,7 +1,7 @@ dynamic_network_architectures==0.2 joblib==1.3.0 loguru==0.7.0 -monai[nibabel]==1.3.0 +monai[nibabel,einops]==1.3.0 scipy==1.11.2 numpy==1.24.4 torch==2.0.0 diff --git a/monai/requirements_inference_gpu.txt b/monai/requirements_inference_gpu.txt index 48738232..c6820186 100644 --- a/monai/requirements_inference_gpu.txt +++ b/monai/requirements_inference_gpu.txt @@ -1,7 +1,7 @@ dynamic_network_architectures==0.2 joblib==1.3.0 loguru==0.7.0 -monai[nibabel]==1.3.0 +monai[nibabel,einops]==1.3.0 scipy==1.11.2 numpy==1.24.4 torch==2.0.0 From aaa39716a1ac90574c7cf43d32ebd503715a0e9d Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Sat, 17 Feb 2024 13:42:11 -0500 Subject: [PATCH 46/66] add qc generation script for epi data --- qc_other_datasets/generate_qc_epi.sh | 285 +++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 qc_other_datasets/generate_qc_epi.sh diff --git a/qc_other_datasets/generate_qc_epi.sh b/qc_other_datasets/generate_qc_epi.sh new file mode 100644 index 00000000..7cf7b5d9 --- /dev/null +++ b/qc_other_datasets/generate_qc_epi.sh @@ -0,0 +1,285 @@ +#!/bin/bash +# +# Compare the CSA of soft GT thresholded at different values on spine-generic test dataset. +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# # Uncomment for full verbose +# set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 +QC_DATASET=$2 # dataset name to generate QC for +# PATH_NNUNET_SCRIPT=$2 # path to the nnUNet contrast-agnostic run_inference_single_subject.py +# PATH_NNUNET_MODEL=$3 # path to the nnUNet contrast-agnostic model +PATH_MONAI_SCRIPT=$3 # path to the MONAI contrast-agnostic run_inference_single_subject.py +PATH_MONAI_MODEL_SOFT=$4 # path to the MONAI contrast-agnostic model trained on soft labels +PATH_MONAI_MODEL_SOFTBIN=$5 # path to the MONAI contrast-agnostic model trained on soft_bin labels + +echo "SUBJECT: ${SUBJECT}" +echo "QC_DATASET: ${QC_DATASET}" +# echo "PATH_NNUNET_SCRIPT: ${PATH_NNUNET_SCRIPT}" +# echo "PATH_NNUNET_MODEL: ${PATH_NNUNET_MODEL}" +echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +echo "PATH_MONAI_MODEL_SOFT: ${PATH_MONAI_MODEL_SOFT}" +echo "PATH_MONAI_MODEL_SOFTBIN: ${PATH_MONAI_MODEL_SOFTBIN}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Get ANIMA binaries path +anima_binaries_path=$(grep "^anima = " ~/.anima/config.txt | sed "s/.* = //" | sed 's/\/$//') + +# Compute ANIMA segmentation performance metrics +compute_anima_metrics(){ + local FILESEG="$1" + local FILEGT="$2" + + # We have to copy qform matrix from seg-manual to the automatically generated segmentation to avoid ITK error: + # "Description: ITK ERROR: SegmentationMeasuresImageFilter(): Inputs do not occupy the same physical space!" + # Related to the following issue : https://github.com/spinalcordtoolbox/spinalcordtoolbox/pull/4135 + sct_image -i ${FILEGT}.nii.gz -copy-header ${FILESEG}.nii.gz -o ${FILESEG}_updated_header.nii.gz + + # Compute ANIMA segmentation performance metrics + # -i : input segmentation + # -r : GT segmentation + # -o : output file + # -d : surface distances evaluation + # -s : compute metrics to evaluate a segmentation + # -X : stores results into a xml file. + ${anima_binaries_path}/animaSegPerfAnalyzer -i ${FILESEG}_updated_header.nii.gz -r ${FILEGT}.nii.gz -o ${PATH_RESULTS}/${FILESEG} -d -s -X + + rm ${FILESEG}_updated_header.nii.gz +} + +# Copy GT segmentation (located under derivatives/labels) +copy_gt_seg(){ + local file="$1" + local label_suffix="$2" + # Construct file name to GT segmentation located under derivatives/labels + # FILESEG="${PATH_DATA}/derivatives/label/${SUBJECT}/func/${file/-mocomean_bold/}_${label_suffix}.nii.gz" + FILESEG="${PATH_DATA/moco/label}/${SUBJECT}/func/${file/-mocomean_bold/}-${label_suffix}.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_seg-manual.nii.gz + else + echo "File ${FILESEG}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + +# TODO: Fix the contrast input for deepseg and propseg (i.e. dwi, mton, mtoff won't work) +# Segment spinal cord using methods available in SCT (sct_deepseg_sc or sct_propseg), resample the prediction back to +# native resolution and compute CSA in native space +segment_sc() { + local file="$1" + local contrast="$2" + local method="$3" # deepseg or propseg + local kernel="$4" # 2d or 3d; only relevant for deepseg + local file_gt_vert_label="$5" + local native_res="$6" + + # Segment spinal cord + if [[ $method == 'deepseg' ]];then + FILESEG="${file}_seg_${method}_${kernel}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_deepseg_sc -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -kernel ${kernel} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + elif [[ $method == 'propseg' ]]; then + FILESEG="${file}_seg_${method}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_propseg -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Remove centerline (we don't need it) + rm ${file}_centerline.nii.gz + + fi + + # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + +} + +# Segment spinal cord using the contrast-agnostic nnUNet model +segment_sc_nnUNet(){ + local file="$1" + local kernel="$2" # 2d or 3d + local file_gt_vert_label="$3" + local contrast="$4" # used only for saving output file name + + # FILESEG="${file}_seg_nnunet_${kernel}" + FILESEG="${file%%_*}_${contrast}_seg_nnunet" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_NNUNET_SCRIPT} -i ${file}.nii.gz -o ${FILESEG}.nii.gz -path-model ${PATH_NNUNET_MODEL}/nnUNetTrainer__nnUNetPlans__${kernel} -pred-type sc -use-gpu + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # # Generate QC report + # sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + +} + +# Segment spinal cord using the MONAI contrast-agnostic model +segment_sc_MONAI(){ + local file="$1" + local label_type="$2" # soft or soft_bin + + if [[ $label_type == 'soft' ]]; then + FILEPRED="${file}_seg_monai_soft" + PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFT} + + elif [[ $label_type == 'soft_bin' ]]; then + FILEPRED="${file}_seg_monai_bin" + PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFTBIN} + + fi + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu + # Rename MONAI output + mv ${file}_pred.nii.gz ${FILEPRED}.nii.gz + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILEPRED},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Binarize MONAI output (which is soft by default); output is overwritten + sct_maths -i ${FILEPRED}.nii.gz -bin 0.5 -o ${FILEPRED}.nii.gz + + # Generate QC report with soft prediction + sct_qc -i ${file}.nii.gz -s ${FILEPRED}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # compute ANIMA metrics + compute_anima_metrics ${FILEPRED} ${file}_seg-manual + +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +contrast="task-rest_desc-mocomean_bold" +label_suffix="spinalcord_mask" + +# Copy source images +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' + +# TODO: move from derivatives to the root directory because run_batch don't search in derivatives +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/func/${SUBJECT//[\/]/_}*${contrast}.* . + +# Go to the folder where the data is +cd $PATH_DATA_PROCESSED/./${SUBJECT}/func/ + +# Get file name +file="${SUBJECT}_${contrast}" + +# Check if file exists +if [[ ! -e ${file}.nii.gz ]]; then + echo "File ${file}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}.nii.gz does not exist. Exiting." + exit 1 +fi + +# Copy GT spinal cord segmentation +copy_gt_seg "${file}" "${label_suffix}" + +# Segment SC using different methods, binarize at 0.5 and compute QC +segment_sc_MONAI ${file} 'soft' +segment_sc_MONAI ${file} 'soft_bin' + +# segment_sc_nnUNet ${file} '3d_fullres' "${file}_seg-manual" ${contrast} +# # TODO: run on deep/progseg after fixing the contrasts for those +# segment_sc ${file_res} 't2' 'deepseg' '2d' "${file}_seg-manual" ${native_res} +# segment_sc ${file_res} 't2' 'propseg' '' "${file}_seg-manual" ${native_res} + + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From d1fe4c32917d86ac2e17d114f5ed757d573a428f Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 21 Feb 2024 07:38:37 -0500 Subject: [PATCH 47/66] add script for comparing preds across thresholds --- .../comparison_across_thresholds.sh | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh diff --git a/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh b/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh new file mode 100644 index 00000000..567b0ca9 --- /dev/null +++ b/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh @@ -0,0 +1,270 @@ +#!/bin/bash +# +# Compare the contrast-agnostic model (MONAI and nnUNet versions) with other methods (sct_propseg, sct_deepseg_sc) +# across different resolutions on spine-generic test dataset. +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# Uncomment for full verbose +set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 +PATH_MONAI_SCRIPT=$2 # path to the MONAI contrast-agnostic run_inference_single_subject.py +PATH_MONAI_MODEL=$3 # path to the MONAI contrast-agnostic model + +echo "SUBJECT: ${SUBJECT}" +echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +echo "PATH_MONAI_MODEL: ${PATH_MONAI_MODEL}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Check if manual label already exists. If it does, copy it locally. +# NOTE: manual disc labels should go from C1-C2 to C7-T1. +label_vertebrae(){ + local file="$1" + local contrast="$2" + + # Update global variable with segmentation file name + FILESEG="${file}_seg-manual" + FILELABEL="${file}_discs" + + # Label vertebral levels + sct_label_utils -i ${file}.nii.gz -disc ${FILELABEL}.nii.gz -o ${FILESEG}_labeled.nii.gz + + # # Run QC + # sct_qc -i ${file}.nii.gz -s ${file_seg}_labeled.nii.gz -p sct_label_vertebrae -qc ${PATH_QC} -qc-subject ${SUBJECT} +} + + +# Segment spinal cord using the MONAI contrast-agnostic model, resample the prediction back to native resolution and +# compute CSA in native space +segment_sc_MONAI(){ + local file="$1" + local file_gt_vert_label="$2" + local threshold="$3" + local contrast="$4" + + # Remove the 0. for the file name + thr_e="$(echo $thr | awk -F'.' '{print $2}')" + + # keep on the subject and the re-named contrast in output file name + FILESEG="${file%%_*}_${contrast}_seg_monai_thr_${thr_e}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu --pred-thr ${threshold} --keep-largest + # Rename MONAI output + mv ${file}_pred.nii.gz ${FILESEG}.nii.gz + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # # Generate QC report with soft prediction + # sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the soft prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_softbin_thresh_c24.csv -append 1 + +} + +# Copy GT spinal cord disc labels (located under derivatives/labels) +copy_gt_disc_labels(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILEDISCLABELS="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_discs.nii.gz" + echo "" + echo "Looking for manual disc labels: $FILEDISCLABELS" + if [[ -e $FILEDISCLABELS ]]; then + echo "Found! Copying ..." + rsync -avzh $FILEDISCLABELS ${file}_discs.nii.gz + else + echo "File ${FILEDISCLABELS} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Disc Labels ${FILEDISCLABELS} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT segmentation (located under derivatives/labels) +copy_gt_seg(){ + local file="$1" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels/${SUBJECT}/anat/${file}_seg-manual.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_seg-manual.nii.gz + else + echo "File ${FILESEG}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG}.nii.gz does not exist. Exiting." + exit 1 + fi +} + +# Copy GT soft segmentation (located under derivatives/labels_softseg) +copy_gt_softseg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels_softseg/${SUBJECT}/${type}/${file}_softseg.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_softseg.nii.gz + else + echo "File ${FILESEG} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +# Copy source images +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/anat/* . +# copy DWI data +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/dwi/* . + +# ------------------------------------------------------------------------------ +# contrast +# ------------------------------------------------------------------------------ +contrasts="T1w T2w T2star flip-1_mt-on_MTS flip-2_mt-off_MTS rec-average_dwi" +# contrasts="flip-2_mt-off_MTS rec-average_dwi" + +# Loop across contrasts +for contrast in ${contrasts}; do + + if [[ $contrast == "rec-average_dwi" ]]; then + type="dwi" + else + type="anat" + fi + + # go to the folder where the data is + cd ${PATH_DATA_PROCESSED}/${SUBJECT}/${type} + + # Get file name + file="${SUBJECT}_${contrast}" + + # Check if file exists + if [[ ! -e ${file}.nii.gz ]]; then + echo "File ${file}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}.nii.gz does not exist. Exiting." + exit 1 + fi + + # Copy GT disc labels + copy_gt_disc_labels "${file}" "${type}" + + # Copy GT spinal cord segmentation + copy_gt_softseg "${file}" "${type}" + + # Label vertebral levels in the native resolution + label_vertebrae ${file} 't2' + + # thresholds for thresholding the predictions +# thresholds="0.2 0.15 0.1 0.05 0.01" + thresholds="0.15 0.1 0.05 0.01 0.005" + + # rname contrast if flip 1 or flip 2 + if [[ $contrast == "flip-1_mt-on_MTS" ]]; then + contrast="MTon" + elif [[ $contrast == "flip-2_mt-off_MTS" ]]; then + contrast="MToff" + elif [[ $contrast == "rec-average_dwi" ]]; then + contrast="DWI" + fi + + # Loop across thresholds + for thr in ${thresholds}; do + + echo "Thresholding the soft GT with threshold: ${thr} ..." + + # Remove the 0. for the file name + thr_e="$(echo $thr | awk -F'.' '{print $2}')" + + # Threshold the soft GT + FILETHRESH="${file%%_*}_${contrast}_softseg_thr_${thr_e}" + sct_maths -i ${file}_softseg.nii.gz -thr ${thr} -o ${FILETHRESH}.nii.gz + + # Compute the CSA of the thresholded soft GT + sct_process_segmentation -i ${FILETHRESH}.nii.gz -vert 2:4 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_softbin_thresh_c24.csv -append 1 + + echo "Thresholding MONAI predictions with threshold: ${thr} ..." + + # Segment SC using different thresholds and compute CSA + segment_sc_MONAI ${file} "${file}_seg-manual" ${thr} ${contrast} + + done + +done + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From e04c7e4ceb712c1e4c967f77c75f365d1c494fa8 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Wed, 21 Feb 2024 23:43:40 -0500 Subject: [PATCH 48/66] add unified script to generate csa plots across methods and thresholds --- csa_generate_figures/analyse_csa_across.py | 388 +++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 csa_generate_figures/analyse_csa_across.py diff --git a/csa_generate_figures/analyse_csa_across.py b/csa_generate_figures/analyse_csa_across.py new file mode 100644 index 00000000..39a6d8af --- /dev/null +++ b/csa_generate_figures/analyse_csa_across.py @@ -0,0 +1,388 @@ +""" +Generate violin plot from CSV data across resolutions and methods. + +Usage: + python generate_figure_csa_across_resolutions.py -i /path/to/data.csv +""" + +import os +import argparse +import pandas as pd +import re +import seaborn as sns +import matplotlib.pyplot as plt + +# Setting the hue order as specified +# HUE_ORDER = ["softseg_soft", "softseg_bin", "nnunet", "monai_soft", "monai_bin"] +HUE_ORDER = ["softseg_bin", "deepseg_2d", "nnunet", "monai", "swinunetr", "mednext"] +HUE_ORDER_THR = ["GT", "15", "1", "05", "01", "005"] +CONTRAST_ORDER = ["DWI", "MTon", "MToff", "T1w", "T2star", "T2w"] + + +def save_figure(file_path, save_fname): + plt.tight_layout() + save_path = os.path.join(os.path.dirname(file_path), save_fname) + plt.savefig(save_path, dpi=300) + print(f'Figure saved to {save_path}') + + +def fetch_participant_id(filename_path): + """ + Get participant_id from the input BIDS-compatible filename or file path + :return: participant_id: subject ID (e.g., sub-001) + """ + + _, filename = os.path.split(filename_path) # Get just the filename (i.e., remove the path) + participant = re.search('sub-(.*?)[_/]', filename_path) # [_/] slash or underscore + participant_id = participant.group(0)[:-1] if participant else "" # [:-1] removes the last underscore or slash + # REGEX explanation + # \d - digit + # \d? - no or one occurrence of digit + # *? - match the previous element as few times as possible (zero or more times) + # . - any character + + return participant_id + + +# Function to extract contrast and method from the filename +def extract_contrast_and_details(filename, analysis_type): + """ + Extract the segmentation method and resolution from the filename. + The method (e.g., propseg, deepseg_2d, nnunet_3d_fullres, monai) and resolution (e.g., 1mm) + are embedded in the filename. + """ + # pattern = r'.*iso-(\d+mm).*_(propseg|deepseg_2d|nnunet_3d_fullres|monai).*' + # pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_soft|softseg_bin|nnunet|monai_soft|monai_bin).*' + if analysis_type == "methods": + pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_bin|deepseg_2d|nnunet|monai|swinunetr|mednext).*' + match = re.search(pattern, filename) + if match: + return match.group(1), match.group(2) + else: + return 'Unknown', 'Unknown' + + elif analysis_type == "thresholds": + pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg|monai)_thr_(\d+).*' + match = re.search(pattern, filename) + if match: + return match.group(1), match.group(2), match.group(3) + else: + return 'Unknown', 'Unknown', 'Unknown' + + elif analysis_type == "resolutions": + pattern = r'.*iso-(\d+mm).*_(softseg_bin|deepseg_2d|nnunet|monai|swinunetr|mednext).*' + # TODO + + else: + raise ValueError(f'Unknown analysis type: {analysis_type}. Choices: [methods, resolutions, thresholds].') + + + +def generate_figure(data, contrast, file_path): + """ + Generate violinplot across resolutions and methods + :param data: Pandas DataFrame with the data + :param contrast: Contrast (e.g., T1w, T2w, T2star) + :param file_path: Path to the CSV file (will be used to save the figure) + """ + + # Correct labels for the x-axis based on the actual data + # resolution_labels = ['1mm', '125mm', '15mm', '175mm', '2mm'] + resolution_labels = ['1mm', '05mm', '15mm', '3mm', '2mm'] + + # Creating the violin plot + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Resolution', y='MEAN(area)', hue='Method', data=data, order=resolution_labels, + hue_order=HUE_ORDER) + plt.xticks(rotation=45) + plt.xlabel('Resolution') + plt.ylabel('CSA [mm^2]') + plt.title(f'{contrast}: C2-C3 CSA across Resolutions and Methods') + plt.legend(title='Method', loc='lower left') + plt.tight_layout() + + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Update x-axis labels + # plt.gca().set_xticklabels(['1mm', '1.25mm', '1.5mm', '1.75mm', '2mm']) + plt.gca().set_xticklabels(['1x1x1mm', '0.5x0.5x0.5mm', '1.5mm', '3x0.5x0.5mm', '2mm']) + + # Save the figure in 300 DPI as a PNG file + plt.savefig(file_path.replace('.csv', '.png'), dpi=300) + print(f'Figure saved to {file_path.replace(".csv", ".png")}') + + # Display the plot + plt.show() + + +def generate_figure_std(data, file_path): + """ + Generate violinplot showing STD across participants for each method + """ + + # Compute mean and std across contrasts for each method + df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Method', y='std', data=df, order=HUE_ORDER) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Method', y='std', data=df, color='k', order=HUE_ORDER, size=3) + # plt.xticks(rotation=45) + plt.xlabel('Method') + plt.ylabel('STD [mm^2]') + plt.title(f'STD of C2-C4 CSA for each method') + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Draw vertical line between 1st and 2nd violin + plt.axvline(x=0.5, color='k', linestyle='--') + + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for method in df['Method'].unique(): + mean = df[df['Method'] == method]['std'].mean() + std = df[df['Method'] == method]['std'].std() + plt.text(HUE_ORDER.index(method), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + save_figure(file_path, "std_csa.png") + + +def generate_figure_abs_csa_error(file_path, data, hue_order=None): + """ + Generate violinplot showing absolute CSA error across participants for each method + """ + + # Remove "softseg_soft" from the list of methods, if it exists + if 'softseg_soft' in data['Method'].unique(): + data = data[data['Method'] != 'softseg_soft'] + + # Compute mean and std across contrasts for each method + df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + + # Compute the abs error between "sofseg_bin" and all other methods + df['abs_error'] = df.apply(lambda row: abs(row['mean'] - df[(df['Method'] == 'softseg_bin') & (df['Participant'] == row['Participant'])]['mean'].values[0]), axis=1) + + # Remove "softseg_bin" from the list of methods and shift rows by one to match the violinplot + df = df[df['Method'] != 'softseg_bin'] + + plt.figure(figsize=(12, 6)) + # skip the first method (i.e., softseg_bin) + sns.violinplot(x='Method', y='abs_error', data=df, order=hue_order) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Method', y='abs_error', data=df, color='k', order=hue_order, size=3) + + # plt.xticks(rotation=45) + plt.xlabel('Method') + plt.ylabel('Absolute CSA error [mm^2]') + plt.title(f'Absolute CSA error between softseg_bin and other methods') + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Draw vertical line between 1st and 2nd violin + plt.axvline(x=0.5, color='k', linestyle='--') + + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for method in df['Method'].unique(): + mean = df[df['Method'] == method]['abs_error'].mean() + std = df[df['Method'] == method]['abs_error'].std() + plt.text(hue_order.index(method), ymax-0.25, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + save_figure(file_path, "abs_csa_error.png") + + +def generate_figure_abs_csa_error_threshold(file_path, data, hue_order=None): + """ + Generate violinplot showing absolute CSA error across participants for each threshold value + """ + + # Compute mean and std across thresholds for each method + df = data.groupby(['Method', 'Threshold', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + + # compute abs_error between softseg and monai for each threshold across all contrasts + df['abs_error'] = df.apply(lambda row: abs(row['mean'] - df[(df['Method'] == 'softseg') & (df['Threshold'] == row['Threshold']) & (df['Participant'] == row['Participant'])]['mean'].values[0]), axis=1) + + # remove "softseg" from the list of methods + df = df[df['Method'] != 'softseg'] + + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Threshold', y='abs_error', data=df, order=hue_order) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Threshold', y='abs_error', data=df, color='k', order=hue_order, size=3) + plt.xlabel('Threshold') + plt.ylabel('Absolute CSA error [mm^2]') + plt.title(f'Absolute CSA error between softseg and monai for each threshold') + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Compute the mean +- std across thresholds for each method and place it above the corresponding violin + for threshold in HUE_ORDER_THR: + mean = df[df['Threshold'] == threshold]['abs_error'].mean() + std = df[df['Threshold'] == threshold]['abs_error'].std() + plt.text(HUE_ORDER_THR.index(threshold), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + save_figure(file_path, "abs_csa_error_threshold.png") + + +def generate_figure_abs_csa_error_per_contrast(file_path, data, method=None, threshold=None): + """ + Generate violinplot showing absolute CSA error for each contrast for a given method/threshold + """ + + # remove "softseg_soft" from the list of methods, if it exists + if 'softseg_soft' in data['Method'].unique(): + data = data[data['Method'] != 'softseg_soft'] + + if method is not None and threshold is None: + # create a dataframe with only "softseg_bin" and get the mean CSA for each contrast + df1 = data[data['Method'] == "softseg_bin"].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() + + # create a dataframe with only the given method and get the mean CSA for each contrast + df2 = data[data['Method'] == method].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() + + # define title for the plot + title = f'Method: {method}; Absolute CSA error for each contrast' + + # define save path for the plot + save_fname = f"abs_error_per_contrast_{method}.png" + + elif method is None and threshold is not None: + # create a dataframe for the given threshold + df_thr = data[data['Threshold'] == threshold] + + # create a dataframe with only "softseg" and get the mean CSA for each contrast + df1 = df_thr[df_thr['Method'] == "softseg"].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() + + # create a dataframe with only "monai" and get the mean CSA for each contrast + df2 = df_thr[df_thr['Method'] == "monai"].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() + + # define title for the plot + title = f'Threshold: 0.{threshold}; Absolute CSA error between softseg and monai preds for each contrast' + + # define save path for the plot + save_fname = f"abs_error_per_contrast_{threshold}.png" + + df1 = df1.pivot(index='Participant', columns='Contrast', values='mean').reset_index() + df2 = df2.pivot(index='Participant', columns='Contrast', values='mean').reset_index() + + # compute the absolute error between the two dataframes for each contrast + df = pd.DataFrame() + df['Participant'] = df1['Participant'] + for contrast in CONTRAST_ORDER: + df[contrast] = abs(df1[contrast] - df2[contrast]) + + # reshape the dataframe to have a single column for the contrast and a single column for the absolute error + df = df.melt(id_vars=['Participant'], value_vars=CONTRAST_ORDER, var_name='Contrast', value_name='abs_error') + + # plot the abs error for each contrast in a violinplot + plt.figure(figsize=(12, 6)) + sns.violinplot(x='Contrast', y='abs_error', data=df, order=CONTRAST_ORDER) + # overlay swarm plot on the violin plot to show individual data points + sns.swarmplot(x='Contrast', y='abs_error', data=df, color='k', order=CONTRAST_ORDER, size=3) + plt.xlabel('Contrast') + plt.ylabel('Absolute CSA error [mm^2]') + plt.title(title) + # Add horizontal dashed grid + plt.grid(axis='y', alpha=0.5, linestyle='dashed') + # set the y-axis limits + plt.ylim(-2, 10) + + # Get y-axis limits + ymin, ymax = plt.gca().get_ylim() + + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for contrast in CONTRAST_ORDER: + mean = df[df['Contrast'] == contrast]['abs_error'].mean() + std = df[df['Contrast'] == contrast]['abs_error'].std() + plt.text(CONTRAST_ORDER.index(contrast), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + + # Save the figure in 300 DPI as a PNG file + save_figure(file_path, save_fname) + + + +def compute_cov(data, file_path): + """ + Compute COV for CSA for each method across resolutions + :param data: Pandas DataFrame with the data + :param file_path: Path to the CSV file (will be used to save the figure) + """ + # Compute COV for CSA ('MEAN(area)' column) for each method across resolutions + df = data.groupby(['Method', 'Resolution'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + for method in df['Method'].unique(): + df.loc[df['Method'] == method, 'COV'] = df.loc[df['Method'] == method, 'std'] / df.loc[ + df['Method'] == method, 'mean'] * 100 + df = df[['Method', 'Resolution', 'COV']].pivot(index='Resolution', columns='Method', values='COV') + # Compute mean +- std across resolutions for each method + df.loc['mean COV'] = df.mean() + df.loc['std COV'] = df.std() + # Keep only two decimals and save as csv + df = df.round(2) + df.to_csv(file_path.replace('.csv', '_COV.csv')) + print(f'COV saved to {file_path.replace(".csv", "_COV.csv")}') + # Print + print(df) + + +def main(file_path, analysis_type="methods"): + # Load the CSV file + data = pd.read_csv(file_path) + + # Apply the function to extract participant ID + data['Participant'] = data['Filename'].apply(fetch_participant_id) + + # Apply the function to extract method and the corresponding analysis details + if analysis_type == "methods": + data['Contrast'], data['Method'] = zip( + *data['Filename'].apply(extract_contrast_and_details, analysis_type=analysis_type)) + + # Generate violinplot showing STD across participants for each method + generate_figure_std(data, file_path) + + # Generate violinplot showing absolute CSA error across participants for each method + generate_figure_abs_csa_error(file_path, data, hue_order=HUE_ORDER) + + # Generate violinplot showing absolute CSA error for each contrast for a given method + for method in HUE_ORDER[1:]: + generate_figure_abs_csa_error_per_contrast(file_path, data, method=method, threshold=None) + + elif analysis_type == "thresholds": + data['Contrast'], data['Method'], data['Threshold'] = zip( + *data['Filename'].apply(extract_contrast_and_details, analysis_type=analysis_type)) + + # Generate violinplot showing absolute CSA error across participants for each threshold value + generate_figure_abs_csa_error_threshold(file_path, data, hue_order=HUE_ORDER_THR) + + # Generate violinplot showing absolute CSA error for each contrast for a given threshold + for threshold in HUE_ORDER_THR[1:]: + generate_figure_abs_csa_error_per_contrast(file_path, data, method=None, threshold=threshold) + + elif analysis_type == "resolutions": + # TODO + pass + + else: + raise ValueError(f'Unknown analysis type: {analysis_type}. Choices: [methods, resolutions, thresholds].') + + # # Compute COV for CSA for each method across resolutions + # compute_cov(data, file_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Generate violin plot from CSV data.') + parser.add_argument('-i', type=str, help='Path to the CSV file') + parser.add_argument('-a', type=str, default="methods", + help='Options to analyse CSA across. Choices: [methods, resolutions, thresholds]') + args = parser.parse_args() + main(args.i, analysis_type=args.a) \ No newline at end of file From 953e6f00cefb74beef20e19f238c3aea27f633fd Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 22 Feb 2024 01:04:55 -0500 Subject: [PATCH 49/66] update docstring --- csa_generate_figures/analyse_csa_across.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csa_generate_figures/analyse_csa_across.py b/csa_generate_figures/analyse_csa_across.py index 39a6d8af..8f70dce9 100644 --- a/csa_generate_figures/analyse_csa_across.py +++ b/csa_generate_figures/analyse_csa_across.py @@ -1,8 +1,8 @@ """ -Generate violin plot from CSV data across resolutions and methods. +Generate violin plots from CSV data across different models (methods), thresholds, and resolutions. Usage: - python generate_figure_csa_across_resolutions.py -i /path/to/data.csv + python generate_figure_csa_across_resolutions.py -i /path/to/data.csv -a [methods, resolutions, thresholds] """ import os From 5a6eafb39651825819880987b3d2ea74936037e4 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 22 Feb 2024 01:06:05 -0500 Subject: [PATCH 50/66] remove old analyse_csa script --- .../analyse_csa_across_training_labels.py | 296 ------------------ 1 file changed, 296 deletions(-) delete mode 100644 csa_generate_figures/analyse_csa_across_training_labels.py diff --git a/csa_generate_figures/analyse_csa_across_training_labels.py b/csa_generate_figures/analyse_csa_across_training_labels.py deleted file mode 100644 index 2747b45c..00000000 --- a/csa_generate_figures/analyse_csa_across_training_labels.py +++ /dev/null @@ -1,296 +0,0 @@ -""" -Generate violin plot from CSV data across resolutions and methods. - -Usage: - python generate_figure_csa_across_resolutions.py -i /path/to/data.csv -""" - -import os -import argparse -import pandas as pd -import re -import seaborn as sns -import matplotlib.pyplot as plt - -# Setting the hue order as specified -# HUE_ORDER = ["propseg", "deepseg_2d", "nnunet_3d_fullres", "monai"] -HUE_ORDER = ["softseg_soft", "softseg_bin", "nnunet", "monai_soft", "monai_bin"] -HUE_ORDER_ABS_CSA = ["", "nnunet", "monai_soft", "monai_bin"] -CONTRAST_ORDER = ["DWI", "MTon", "MToff", "T1w", "T2star", "T2w"] - - -def fetch_participant_id(filename_path): - """ - Get participant_id from the input BIDS-compatible filename or file path - :return: participant_id: subject ID (e.g., sub-001) - """ - - _, filename = os.path.split(filename_path) # Get just the filename (i.e., remove the path) - participant = re.search('sub-(.*?)[_/]', filename_path) # [_/] slash or underscore - participant_id = participant.group(0)[:-1] if participant else "" # [:-1] removes the last underscore or slash - # REGEX explanation - # \d - digit - # \d? - no or one occurrence of digit - # *? - match the previous element as few times as possible (zero or more times) - # . - any character - - return participant_id - - -# Function to extract method and resolution from the filename -def extract_contrast_and_method(filename): - """ - Extract the segmentation method and resolution from the filename. - The method (e.g., propseg, deepseg_2d, nnunet_3d_fullres, monai) and resolution (e.g., 1mm) - are embedded in the filename. - """ - # pattern = r'.*iso-(\d+mm).*_(propseg|deepseg_2d|nnunet_3d_fullres|monai).*' - pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_soft|softseg_bin|nnunet|monai_soft|monai_bin).*' - match = re.search(pattern, filename) - if match: - return match.group(1), match.group(2) - else: - return 'Unknown', 'Unknown' - - -def generate_figure(data, contrast, file_path): - """ - Generate violinplot across resolutions and methods - :param data: Pandas DataFrame with the data - :param contrast: Contrast (e.g., T1w, T2w, T2star) - :param file_path: Path to the CSV file (will be used to save the figure) - """ - - # Correct labels for the x-axis based on the actual data - # resolution_labels = ['1mm', '125mm', '15mm', '175mm', '2mm'] - resolution_labels = ['1mm', '05mm', '15mm', '3mm', '2mm'] - - # Creating the violin plot - plt.figure(figsize=(12, 6)) - sns.violinplot(x='Resolution', y='MEAN(area)', hue='Method', data=data, order=resolution_labels, - hue_order=HUE_ORDER) - plt.xticks(rotation=45) - plt.xlabel('Resolution') - plt.ylabel('CSA [mm^2]') - plt.title(f'{contrast}: C2-C3 CSA across Resolutions and Methods') - plt.legend(title='Method', loc='lower left') - plt.tight_layout() - - # Add horizontal dashed grid - plt.grid(axis='y', alpha=0.5, linestyle='dashed') - - # Update x-axis labels - # plt.gca().set_xticklabels(['1mm', '1.25mm', '1.5mm', '1.75mm', '2mm']) - plt.gca().set_xticklabels(['1x1x1mm', '0.5x0.5x0.5mm', '1.5mm', '3x0.5x0.5mm', '2mm']) - - # Save the figure in 300 DPI as a PNG file - plt.savefig(file_path.replace('.csv', '.png'), dpi=300) - print(f'Figure saved to {file_path.replace(".csv", ".png")}') - - # Display the plot - plt.show() - - -def generate_figure_std(data, file_path): - """ - Generate violinplot showing STD across participants for each method - """ - - # Compute mean and std across contrasts for each method - df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() - - plt.figure(figsize=(12, 6)) - sns.violinplot(x='Method', y='std', data=df, order=HUE_ORDER) - # overlay swarm plot on the violin plot to show individual data points - sns.swarmplot(x='Method', y='std', data=df, color='k', order=HUE_ORDER, size=3) - # plt.xticks(rotation=45) - plt.xlabel('Method') - plt.ylabel('STD [mm^2]') - plt.title(f'STD of C2-C4 CSA for each method') - # Add horizontal dashed grid - plt.grid(axis='y', alpha=0.5, linestyle='dashed') - - # Get y-axis limits - ymin, ymax = plt.gca().get_ylim() - - # Draw vertical line between 1st and 2nd violin - plt.axvline(x=0.5, color='k', linestyle='--') - - # Compute the mean +- std across resolutions for each method and place it above the corresponding violin - for method in df['Method'].unique(): - mean = df[df['Method'] == method]['std'].mean() - std = df[df['Method'] == method]['std'].std() - plt.text(HUE_ORDER.index(method), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') - - # Save the figure in 300 DPI as a PNG file - plt.tight_layout() - save_path = os.path.join(os.path.dirname(file_path), "std_csa.png") - plt.savefig(save_path, dpi=300) - print(f'Figure saved to {save_path}') - - -def generate_figure_abs_csa_error(data, file_path): - """ - Generate violinplot showing absolute CSA error across participants for each method - """ - - # Compute mean and std across contrasts for each method - df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() - - # Remove "softseg_soft" from the list of methods, if it exists - if 'softseg_soft' in df['Method'].unique(): - df = df[df['Method'] != 'softseg_soft'] - - # Compute the abs error between "sofseg_bin" and all other methods - df['abs_error'] = df.apply(lambda row: abs(row['mean'] - df[(df['Method'] == 'softseg_bin') & (df['Participant'] == row['Participant'])]['mean'].values[0]), axis=1) - - # Remove "softseg_bin" from the list of methods and shift rows by one to match the violinplot - df = df[df['Method'] != 'softseg_bin'] - - plt.figure(figsize=(12, 6)) - # skip the first method (i.e., softseg_bin) - sns.violinplot(x='Method', y='abs_error', data=df, order=HUE_ORDER) - # overlay swarm plot on the violin plot to show individual data points - sns.swarmplot(x='Method', y='abs_error', data=df, color='k', order=HUE_ORDER, size=3) - - # plt.xticks(rotation=45) - plt.xlabel('Method') - plt.ylabel('Absolute CSA error [mm^2]') - plt.title(f'Absolute CSA error between softseg_bin and other methods') - # Add horizontal dashed grid - plt.grid(axis='y', alpha=0.5, linestyle='dashed') - - # Get y-axis limits - ymin, ymax = plt.gca().get_ylim() - - # Draw vertical line between 1st and 2nd violin - plt.axvline(x=0.5, color='k', linestyle='--') - - # Compute the mean +- std across resolutions for each method and place it above the corresponding violin - for method in df['Method'].unique(): - mean = df[df['Method'] == method]['abs_error'].mean() - std = df[df['Method'] == method]['abs_error'].std() - plt.text(HUE_ORDER.index(method), ymax-0.25, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') - - # Save the figure in 300 DPI as a PNG file - plt.tight_layout() - save_path = os.path.join(os.path.dirname(file_path), "abs_csa_error.png") - plt.savefig(save_path, dpi=300) - print(f'Figure saved to {save_path}') - - -def generate_figure_abs_csa_error_per_contrast(data, method, file_path): - """ - Generate violinplot showing absolute CSA error for each contrast for a given method - """ - - # remove "softseg_soft" from the list of methods, if it exists - if 'softseg_soft' in data['Method'].unique(): - data = data[data['Method'] != 'softseg_soft'] - - # create a dataframe with only "softseg_bin" and get the mean CSA for each contrast - df1 = data[data['Method'] == "softseg_bin"].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() - df1 = df1.pivot(index='Participant', columns='Contrast', values='mean').reset_index() - - # create a dataframe with only the given method and get the mean CSA for each contrast - df2 = data[data['Method'] == method].groupby(['Contrast', 'Participant'])['MEAN(area)'].agg(['mean']).reset_index() - df2 = df2.pivot(index='Participant', columns='Contrast', values='mean').reset_index() - - # compute the absolute error between the two dataframes for each contrast - df = pd.DataFrame() - df['Participant'] = df1['Participant'] - for contrast in CONTRAST_ORDER: - df[contrast] = abs(df1[contrast] - df2[contrast]) - - # reshape the dataframe to have a single column for the contrast and a single column for the absolute error - df = df.melt(id_vars=['Participant'], value_vars=CONTRAST_ORDER, var_name='Contrast', value_name='abs_error') - - # plot the abs error for each contrast in a violinplot - plt.figure(figsize=(12, 6)) - sns.violinplot(x='Contrast', y='abs_error', data=df, order=CONTRAST_ORDER) - # overlay swarm plot on the violin plot to show individual data points - sns.swarmplot(x='Contrast', y='abs_error', data=df, color='k', order=CONTRAST_ORDER, size=3) - plt.xlabel('Contrast') - plt.ylabel('Absolute CSA error [mm^2]') - plt.title(f'Method: {method}; Absolute CSA error for each contrast') - # Add horizontal dashed grid - plt.grid(axis='y', alpha=0.5, linestyle='dashed') - # set the y-axis limits - plt.ylim(-2, 10) - - # Get y-axis limits - ymin, ymax = plt.gca().get_ylim() - - # Compute the mean +- std across resolutions for each method and place it above the corresponding violin - for contrast in CONTRAST_ORDER: - mean = df[df['Contrast'] == contrast]['abs_error'].mean() - std = df[df['Contrast'] == contrast]['abs_error'].std() - plt.text(CONTRAST_ORDER.index(contrast), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') - - # Save the figure in 300 DPI as a PNG file - plt.tight_layout() - save_path = os.path.join(os.path.dirname(file_path), f"abs_error_per_contrast_{method}.png") - plt.savefig(save_path, dpi=300) - print(f'Figure saved to {save_path}') - - -def compute_cov(data, file_path): - """ - Compute COV for CSA for each method across resolutions - :param data: Pandas DataFrame with the data - :param file_path: Path to the CSV file (will be used to save the figure) - """ - # Compute COV for CSA ('MEAN(area)' column) for each method across resolutions - df = data.groupby(['Method', 'Resolution'])['MEAN(area)'].agg(['mean', 'std']).reset_index() - for method in df['Method'].unique(): - df.loc[df['Method'] == method, 'COV'] = df.loc[df['Method'] == method, 'std'] / df.loc[ - df['Method'] == method, 'mean'] * 100 - df = df[['Method', 'Resolution', 'COV']].pivot(index='Resolution', columns='Method', values='COV') - # Compute mean +- std across resolutions for each method - df.loc['mean COV'] = df.mean() - df.loc['std COV'] = df.std() - # Keep only two decimals and save as csv - df = df.round(2) - df.to_csv(file_path.replace('.csv', '_COV.csv')) - print(f'COV saved to {file_path.replace(".csv", "_COV.csv")}') - # Print - print(df) - - -def main(file_path): - # Load the CSV file - data = pd.read_csv(file_path) - - # Apply the function to extract method and resolution - data['Contrast'], data['Method'] = zip(*data['Filename'].apply(extract_contrast_and_method)) - # data['Method'] = data['Filename'].apply(extract_method_resolution) - - # Apply the function to extract participant ID - data['Participant'] = data['Filename'].apply(fetch_participant_id) - - # # Fetch contrast (e.g. T1w, T2w, T2star) from the first filename using regex - # contrast = re.search(r'.*_(T1w|T2w|T2star).*', data['Filename'][0]).group(1) - - # # Generate violinplot across resolutions and methods - # generate_figure(data, contrast, file_path) - - # Generate violinplot showing STD across participants for each method - generate_figure_std(data, file_path) - - # Generate violinplot showing absolute CSA error across participants for each method - generate_figure_abs_csa_error(data, file_path) - - # Generate violinplot showing absolute CSA error for each contrast for a given method - generate_figure_abs_csa_error_per_contrast(data, 'monai_bin', file_path) - # generate_figure_abs_csa_error_per_contrast(data, 'monai_soft', file_path) - # generate_figure_abs_csa_error_per_contrast(data, 'nnunet', file_path) - - # # Compute COV for CSA for each method across resolutions - # compute_cov(data, file_path) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Generate violin plot from CSV data.') - parser.add_argument('-i', type=str, help='Path to the CSV file') - args = parser.parse_args() - main(args.i) \ No newline at end of file From aecb98fc77ec7dba1f55981cb69dc0c7502244ec Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 23 Feb 2024 10:13:40 -0500 Subject: [PATCH 51/66] add v2 inference --- monai/run_inference_single_image_v2.py | 356 +++++++++++++++++++++++++ 1 file changed, 356 insertions(+) create mode 100644 monai/run_inference_single_image_v2.py diff --git a/monai/run_inference_single_image_v2.py b/monai/run_inference_single_image_v2.py new file mode 100644 index 00000000..bba44473 --- /dev/null +++ b/monai/run_inference_single_image_v2.py @@ -0,0 +1,356 @@ +""" +Script to run inference on a MONAI-based model for contrast-agnostic soft segmentation of the spinal cord. + +Author: Naga Karthik + +""" + +import os +import argparse +import numpy as np +from loguru import logger +import torch.nn.functional as F +import torch +import torch.nn as nn +import json +from time import time +from scipy import ndimage + +from monai.inferers import sliding_window_inference +from monai.data import (DataLoader, Dataset, load_decathlon_datalist, decollate_batch) +from monai.transforms import (Compose, EnsureTyped, Invertd, SaveImage, Spacingd, + LoadImaged, NormalizeIntensityd, EnsureChannelFirstd, + DivisiblePadd, Orientationd, ResizeWithPadOrCropd) +from dynamic_network_architectures.architectures.unet import PlainConvUNet, ResidualEncoderUNet +from dynamic_network_architectures.building_blocks.helper import get_matching_instancenorm, convert_dim_to_conv_op +from dynamic_network_architectures.initialization.weight_init import init_last_bn_before_add_to_0 + + +# NNUNET global params +INIT_FILTERS=32 +ENABLE_DS = True + +nnunet_plans = { + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": INIT_FILTERS, + "n_conv_per_stage_encoder": [2, 2, 2, 2, 2, 2], + "n_conv_per_stage_decoder": [2, 2, 2, 2, 2], + "pool_op_kernel_sizes": [ + [1, 1, 1], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + [2, 2, 2], + [1, 2, 2] + ], + "conv_kernel_sizes": [ + [3, 3, 3], + [3, 3, 3], + [3, 3, 3], + [3, 3, 3], + [3, 3, 3], + [3, 3, 3] + ], + "unet_max_num_features": 320, +} + + +def get_parser(): + + parser = argparse.ArgumentParser(description="Run inference on a MONAI-trained model") + + parser.add_argument("--path-img", type=str, required=True, + help="Path to the image to run inference on") + parser.add_argument("--chkp-path", type=str, required=True, + help="Path to the checkpoint folder. This folder should contain a file named 'best_model_loss.ckpt") + parser.add_argument("--path-out", type=str, required=True, + help="Path to the output folder where to store the predictions and associated metrics") + parser.add_argument('-crop', '--crop-size', type=str, default="64x192x-1", + help='Size of the window used to crop the volume before inference (NOTE: Images are resampled to 1mm' + ' isotropic before cropping). The window is centered in the middle of the volume. Dimensions are in the' + ' order R-L, A-P, I-S. Use -1 for no cropping in a specific axis, example: “64x160x-1”.' + ' NOTE: heavy R-L cropping is recommended for positioning the SC at the center of the image.' + ' Default: 64x192x-1') + parser.add_argument('--device', default="gpu", type=str, choices=["gpu", "cpu"], + help='Device to run inference on. Default: cpu') + parser.add_argument('--pred-thr', default=0.5, type=float, help='Threshold to binarize the prediction. Default: 0.5') + parser.add_argument('--keep-largest', action='store_true', help='Keep only the largest connected component in the prediction') + + return parser + + +# ==========================å================================================= +# Test-time Transforms +# =========================================================================== +def inference_transforms_single_image(crop_size): + return Compose([ + LoadImaged(keys=["image"], image_only=False), + EnsureChannelFirstd(keys=["image"]), + Orientationd(keys=["image"], axcodes="RPI"), + Spacingd(keys=["image"], pixdim=(1.0, 1.0, 1.0), mode=(2)), + ResizeWithPadOrCropd(keys=["image"], spatial_size=crop_size,), + DivisiblePadd(keys=["image"], k=2**5), # pad inputs to ensure divisibility by no. of layers nnUNet has (5) + NormalizeIntensityd(keys=["image"], nonzero=False, channel_wise=False), + ]) + + +# =========================================================================== +# Model utils +# =========================================================================== +class InitWeights_He(object): + def __init__(self, neg_slope=1e-2): + self.neg_slope = neg_slope + + def __call__(self, module): + if isinstance(module, nn.Conv3d) or isinstance(module, nn.ConvTranspose3d): + module.weight = nn.init.kaiming_normal_(module.weight, a=self.neg_slope) + if module.bias is not None: + module.bias = nn.init.constant_(module.bias, 0) + +# Copied from ivadomed: +# https://github.com/ivadomed/ivadomed/blob/e101ebea632683d67deab3c50dd6b372207de2a9/ivadomed/postprocessing.py#L101-L116 +def keep_largest_object(predictions): + """Keep the largest connected object from the input array (2D or 3D). + + Args: + predictions (ndarray or nibabel object): Input segmentation. Image could be 2D or 3D. + + Returns: + ndarray or nibabel (same object as the input). + """ + # Find number of closed objects using skimage "label" + labeled_obj, num_obj = ndimage.label(np.copy(predictions)) + # If more than one object is found, keep the largest one + if num_obj > 1: + # Keep the largest object + predictions[np.where(labeled_obj != (np.bincount(labeled_obj.flat)[1:].argmax() + 1))] = 0 + return predictions + + +# ============================================================================ +# Define the network based on nnunet_plans dict +# ============================================================================ +def create_nnunet_from_plans(plans, num_input_channels: int, num_classes: int, deep_supervision: bool = True): + """ + Adapted from nnUNet's source code: + https://github.com/MIC-DKFZ/nnUNet/blob/master/nnunetv2/utilities/get_network_from_plans.py#L9 + + """ + num_stages = len(plans["conv_kernel_sizes"]) + + dim = len(plans["conv_kernel_sizes"][0]) + conv_op = convert_dim_to_conv_op(dim) + + segmentation_network_class_name = plans["UNet_class_name"] + mapping = { + 'PlainConvUNet': PlainConvUNet, + 'ResidualEncoderUNet': ResidualEncoderUNet + } + kwargs = { + 'PlainConvUNet': { + 'conv_bias': True, + 'norm_op': get_matching_instancenorm(conv_op), + 'norm_op_kwargs': {'eps': 1e-5, 'affine': True}, + 'dropout_op': None, 'dropout_op_kwargs': None, + 'nonlin': nn.LeakyReLU, 'nonlin_kwargs': {'inplace': True}, + }, + 'ResidualEncoderUNet': { + 'conv_bias': True, + 'norm_op': get_matching_instancenorm(conv_op), + 'norm_op_kwargs': {'eps': 1e-5, 'affine': True}, + 'dropout_op': None, 'dropout_op_kwargs': None, + 'nonlin': nn.LeakyReLU, 'nonlin_kwargs': {'inplace': True}, + } + } + assert segmentation_network_class_name in mapping.keys(), 'The network architecture specified by the plans file ' \ + 'is non-standard (maybe your own?). Yo\'ll have to dive ' \ + 'into either this ' \ + 'function (get_network_from_plans) or ' \ + 'the init of your nnUNetModule to accomodate that.' + network_class = mapping[segmentation_network_class_name] + + conv_or_blocks_per_stage = { + 'n_conv_per_stage' + if network_class != ResidualEncoderUNet else 'n_blocks_per_stage': plans["n_conv_per_stage_encoder"], + 'n_conv_per_stage_decoder': plans["n_conv_per_stage_decoder"] + } + + # network class name!! + model = network_class( + input_channels=num_input_channels, + n_stages=num_stages, + features_per_stage=[min(plans["UNet_base_num_features"] * 2 ** i, + plans["unet_max_num_features"]) for i in range(num_stages)], + conv_op=conv_op, + kernel_sizes=plans["conv_kernel_sizes"], + strides=plans["pool_op_kernel_sizes"], + num_classes=num_classes, + deep_supervision=deep_supervision, + **conv_or_blocks_per_stage, + **kwargs[segmentation_network_class_name] + ) + model.apply(InitWeights_He(1e-2)) + if network_class == ResidualEncoderUNet: + model.apply(init_last_bn_before_add_to_0) + + return model + + +# =========================================================================== +# Prepare temporary dataset for inference +# =========================================================================== +def prepare_data(path_image, crop_size=(64, 160, 320)): + + test_file = [{"image": path_image}] + + # define test transforms + transforms_test = inference_transforms_single_image(crop_size=crop_size) + + # define post-processing transforms for testing; taken (with explanations) from + # https://github.com/Project-MONAI/tutorials/blob/main/3d_segmentation/torch/unet_inference_dict.py#L66 + test_post_pred = Compose([ + EnsureTyped(keys=["pred"]), + Invertd(keys=["pred"], transform=transforms_test, + orig_keys=["image"], + meta_keys=["pred_meta_dict"], + nearest_interp=False, to_tensor=True), + ]) + test_ds = Dataset(data=test_file, transform=transforms_test) + + + return test_ds, test_post_pred + + +# =========================================================================== +# Inference method +# =========================================================================== +def main(): + + # get parameters + args = get_parser().parse_args() + + # define device + if args.device == "gpu" and not torch.cuda.is_available(): + logger.warning("GPU not available, using CPU instead") + DEVICE = torch.device("cpu") + else: + DEVICE = torch.device("cuda" if torch.cuda.is_available() and args.device == "gpu" else "cpu") + + # define root path for finding datalists + path_image = args.path_img + results_path = args.path_out + chkp_path = os.path.join(args.chkp_path, "best_model.ckpt") + + # save terminal outputs to a file + logger.add(os.path.join(results_path, "logs.txt"), rotation="10 MB", level="INFO") + + logger.info(f"Saving results to: {results_path}") + if not os.path.exists(results_path): + os.makedirs(results_path, exist_ok=True) + + # define inference patch size and center crop size + crop_size = tuple([int(i) for i in args.crop_size.split("x")]) + inference_roi_size = (64, 192, 320) + + # define the dataset and dataloader + test_ds, test_post_pred = prepare_data(path_image, crop_size=crop_size) + test_loader = DataLoader(test_ds, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) + + # define model + net = create_nnunet_from_plans(plans=nnunet_plans, num_input_channels=1, num_classes=1, deep_supervision=ENABLE_DS) + + # define list to collect the test metrics + test_step_outputs = [] + test_summary = {} + + # iterate over the dataset and compute metrics + with torch.no_grad(): + for batch in test_loader: + + # compute time for inference per subject + start_time = time() + + # get the test input + test_input = batch["image"].to(DEVICE) + + # this loop only takes about 0.2s on average on a CPU + checkpoint = torch.load(chkp_path, map_location=torch.device(DEVICE))["state_dict"] + # NOTE: remove the 'net.' prefix from the keys because of how the model was initialized in lightning + # https://discuss.pytorch.org/t/missing-keys-unexpected-keys-in-state-dict-when-loading-self-trained-model/22379/14 + for key in list(checkpoint.keys()): + if 'net.' in key: + checkpoint[key.replace('net.', '')] = checkpoint[key] + del checkpoint[key] + + # load the trained model weights + net.load_state_dict(checkpoint) + net.to(DEVICE) + net.eval() + + # run inference + batch["pred"] = sliding_window_inference(test_input, inference_roi_size, mode="gaussian", + sw_batch_size=4, predictor=net, overlap=0.5, progress=False) + + # take only the highest resolution prediction + batch["pred"] = batch["pred"][0] + + # NOTE: monai's models do not normalize the output, so we need to do it manually + if bool(F.relu(batch["pred"]).max()): + batch["pred"] = F.relu(batch["pred"]) / F.relu(batch["pred"]).max() + else: + batch["pred"] = F.relu(batch["pred"]) + + post_test_out = [test_post_pred(i) for i in decollate_batch(batch)] + + pred = post_test_out[0]['pred'].cpu() + + # threshold the prediction to set all values below pred_thr to 0 + pred[pred < args.pred_thr] = 0 + + if args.keep_largest: + # keep only the largest connected component (to remove tiny blobs after thresholding) + logger.info("Postprocessing: Keeping the largest connected component in the prediction") + pred = keep_largest_object(pred) + + # get subject name + subject_name = (batch["image_meta_dict"]["filename_or_obj"][0]).split("/")[-1].replace(".nii.gz", "") + logger.info(f"Saving subject: {subject_name}") + + # this takes about 0.25s on average on a CPU + # image saver class + pred_saver = SaveImage( + output_dir=results_path, output_postfix="pred", output_ext=".nii.gz", + separate_folder=False, print_log=False) + # save the prediction + pred_saver(pred) + + end_time = time() + metrics_dict = { + "subject_name_and_contrast": subject_name, + "inference_time_in_sec": round((end_time - start_time), 2), + } + test_step_outputs.append(metrics_dict) + + # save the test summary + test_summary["metrics_per_subject"] = test_step_outputs + + # compute the average inference time + avg_inference_time = np.stack([x["inference_time_in_sec"] for x in test_step_outputs]).mean() + + logger.info("========================================================") + logger.info(f" Inference Time per Subject: {avg_inference_time:.2f}s") + logger.info("========================================================") + + + # dump the test summary to a json file + with open(os.path.join(results_path, "test_summary.json"), "w") as f: + json.dump(test_summary, f, indent=4, sort_keys=True) + + # free up memory + test_step_outputs.clear() + test_summary.clear() + # os.remove(os.path.join(results_path, "temp_msd_datalist.json")) + + +if __name__ == "__main__": + main() From c30ca03883336017b7620ba663c5d496872c7890 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Fri, 23 Feb 2024 13:48:03 -0500 Subject: [PATCH 52/66] fix issue with lambda for abs_csa_error plot --- csa_generate_figures/analyse_csa_across.py | 44 ++++++++++++++++------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/csa_generate_figures/analyse_csa_across.py b/csa_generate_figures/analyse_csa_across.py index 8f70dce9..87370ef9 100644 --- a/csa_generate_figures/analyse_csa_across.py +++ b/csa_generate_figures/analyse_csa_across.py @@ -52,8 +52,8 @@ def extract_contrast_and_details(filename, analysis_type): are embedded in the filename. """ # pattern = r'.*iso-(\d+mm).*_(propseg|deepseg_2d|nnunet_3d_fullres|monai).*' - # pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_soft|softseg_bin|nnunet|monai_soft|monai_bin).*' if analysis_type == "methods": + # pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_soft|softseg_bin|nnunet|monai_soft|monai_bin).*' pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_bin|deepseg_2d|nnunet|monai|swinunetr|mednext).*' match = re.search(pattern, filename) if match: @@ -160,20 +160,39 @@ def generate_figure_abs_csa_error(file_path, data, hue_order=None): if 'softseg_soft' in data['Method'].unique(): data = data[data['Method'] != 'softseg_soft'] - # Compute mean and std across contrasts for each method - df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() - - # Compute the abs error between "sofseg_bin" and all other methods - df['abs_error'] = df.apply(lambda row: abs(row['mean'] - df[(df['Method'] == 'softseg_bin') & (df['Participant'] == row['Participant'])]['mean'].values[0]), axis=1) + df = pd.DataFrame() + for method in hue_order[1:]: + df_error_contrast = pd.DataFrame() + for contrast in CONTRAST_ORDER: + df1 = data[(data['Method'] == "softseg_bin") & (data['Contrast'] == contrast)] + df2 = data[(data['Method'] == method) & (data['Contrast'] == contrast)] + + # group by participant and get the mean area for each participant + df1 = df1.groupby('Participant')['MEAN(area)'].mean().reset_index() + df2 = df2.groupby('Participant')['MEAN(area)'].mean().reset_index() + + # compute the absolute error between the two dataframes + df_temp = pd.merge(df1, df2, on='Participant', suffixes=('_gt', '_contrast')) + df_error_contrast[contrast] = abs(df_temp['MEAN(area)_gt'] - df_temp['MEAN(area)_contrast']) + + df_error_contrast['abs_error_mean'] = df_error_contrast.mean(axis=1) + df_error_contrast['abs_error_std'] = df_error_contrast.std(axis=1) + df_error_contrast['Method'] = method + df_error_contrast['Participant'] = df_temp['Participant'] - # Remove "softseg_bin" from the list of methods and shift rows by one to match the violinplot - df = df[df['Method'] != 'softseg_bin'] + df = pd.concat([df, df_error_contrast]) + + # remove the contrasts from the dataframe + df = df.drop(columns=CONTRAST_ORDER) + # # compute the mean and std across contrasts for each method + # df_agg = df.groupby('Method')[['mean_error', 'std_error']].mean().reset_index() + # print(df_agg) plt.figure(figsize=(12, 6)) # skip the first method (i.e., softseg_bin) - sns.violinplot(x='Method', y='abs_error', data=df, order=hue_order) + sns.violinplot(x='Method', y='abs_error_mean', data=df, order=hue_order) # overlay swarm plot on the violin plot to show individual data points - sns.swarmplot(x='Method', y='abs_error', data=df, color='k', order=hue_order, size=3) + sns.swarmplot(x='Method', y='abs_error_mean', data=df, color='k', order=hue_order, size=3) # plt.xticks(rotation=45) plt.xlabel('Method') @@ -190,8 +209,8 @@ def generate_figure_abs_csa_error(file_path, data, hue_order=None): # Compute the mean +- std across resolutions for each method and place it above the corresponding violin for method in df['Method'].unique(): - mean = df[df['Method'] == method]['abs_error'].mean() - std = df[df['Method'] == method]['abs_error'].std() + mean = df[df['Method'] == method]['abs_error_mean'].mean() + std = df[df['Method'] == method]['abs_error_std'].mean() plt.text(hue_order.index(method), ymax-0.25, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') # Save the figure in 300 DPI as a PNG file @@ -206,6 +225,7 @@ def generate_figure_abs_csa_error_threshold(file_path, data, hue_order=None): # Compute mean and std across thresholds for each method df = data.groupby(['Method', 'Threshold', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + # TODO: this is wrong; see update to generate_figure_abs_csa_error function to fix the with incorrect application of `lambda` # compute abs_error between softseg and monai for each threshold across all contrasts df['abs_error'] = df.apply(lambda row: abs(row['mean'] - df[(df['Method'] == 'softseg') & (df['Threshold'] == row['Threshold']) & (df['Participant'] == row['Participant'])]['mean'].values[0]), axis=1) From 1b636857e57c86646121d4b0374fdf5dfa23b673 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 13:06:19 -0500 Subject: [PATCH 53/66] add script for csa comparison across old & new models --- .../comparison_across_models.sh | 371 ++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 csa_qc_evaluation_spine_generic/comparison_across_models.sh diff --git a/csa_qc_evaluation_spine_generic/comparison_across_models.sh b/csa_qc_evaluation_spine_generic/comparison_across_models.sh new file mode 100644 index 00000000..da1a9f2f --- /dev/null +++ b/csa_qc_evaluation_spine_generic/comparison_across_models.sh @@ -0,0 +1,371 @@ +#!/bin/bash +# +# Compare the CSA of different models on the spine-generic test dataset +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# Uncomment for full verbose +set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 +PATH_NNUNET_SCRIPT=$2 # path to the nnUNet contrast-agnostic run_inference_single_subject.py +PATH_NNUNET_MODEL=$3 # path to the nnUNet contrast-agnostic model +PATH_MONAI_SCRIPT=$4 # path to the MONAI contrast-agnostic run_inference_single_subject.py +PATH_MONAI_MODEL=$5 # path to the MONAI contrast-agnostic model trained on soft bin labels +PATH_SWIN_MODEL=$6 +PATH_MEDNEXT_MODEL=$7 + +echo "SUBJECT: ${SUBJECT}" +echo "PATH_NNUNET_SCRIPT: ${PATH_NNUNET_SCRIPT}" +echo "PATH_NNUNET_MODEL: ${PATH_NNUNET_MODEL}" +echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +echo "PATH_MONAI_MODEL: ${PATH_MONAI_MODEL}" +echo "PATH_SWIN_MODEL: ${PATH_SWIN_MODEL}" +echo "PATH_MEDNEXT_MODEL: ${PATH_MEDNEXT_MODEL}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Check if manual label already exists. If it does, copy it locally. +# NOTE: manual disc labels should go from C1-C2 to C7-T1. +label_vertebrae(){ + local file="$1" + local contrast="$2" + + # Update global variable with segmentation file name + FILESEG="${file}_seg-manual" + FILELABEL="${file}_discs" + + # Label vertebral levels + sct_label_utils -i ${file}.nii.gz -disc ${FILELABEL}.nii.gz -o ${FILESEG}_labeled.nii.gz + + # # Run QC + # sct_qc -i ${file}.nii.gz -s ${file_seg}_labeled.nii.gz -p sct_label_vertebrae -qc ${PATH_QC} -qc-subject ${SUBJECT} +} + + +# Copy GT spinal cord disc labels (located under derivatives/labels) +copy_gt_disc_labels(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILEDISCLABELS="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_discs.nii.gz" + echo "" + echo "Looking for manual disc labels: $FILEDISCLABELS" + if [[ -e $FILEDISCLABELS ]]; then + echo "Found! Copying ..." + rsync -avzh $FILEDISCLABELS ${file}_discs.nii.gz + else + echo "File ${FILEDISCLABELS} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Disc Labels ${FILEDISCLABELS} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT segmentation (located under derivatives/labels) +copy_gt_seg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_seg-manual.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_seg-manual.nii.gz + else + echo "File ${FILESEG}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT soft segmentation (located under derivatives/labels_softseg) +copy_gt_softseg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels_softseg/${SUBJECT}/${type}/${file}_softseg.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_softseg.nii.gz + else + echo "File ${FILESEG} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + + +# TODO: Fix the contrast input for deepseg and propseg (i.e. dwi, mton, mtoff won't work) +# Segment spinal cord using methods available in SCT (sct_deepseg_sc or sct_propseg), resample the prediction back to +# native resolution and compute CSA in native space +segment_sc() { + local file="$1" + local file_gt_vert_label="$2" + local method="$3" # deepseg or propseg + local contrast_input="$4" # used for input arg `-c` + local contrast_name="$5" # used only for saving output file name + + # Segment spinal cord + if [[ $method == 'deepseg' ]];then + # FILESEG="${file}_seg_${method}_${kernel}" + FILESEG="${file%%_*}_${contrast_name}_seg_${method}_2d" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_deepseg_sc -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast_input} -kernel 2d -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + elif [[ $method == 'propseg' ]]; then + FILESEG="${file}_seg_${method}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_propseg -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Remove centerline (we don't need it) + rm ${file}_centerline.nii.gz + + fi + + # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:3 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c23.csv -append 1 + +} + +# Segment spinal cord using the contrast-agnostic nnUNet model +segment_sc_nnUNet(){ + local file="$1" + local file_gt_vert_label="$2" + local kernel="$3" # 2d or 3d + local contrast="$4" # used only for saving output file name + + FILESEG="${file%%_*}_${contrast}_seg_nnunet" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_NNUNET_SCRIPT} -i ${file}.nii.gz -o ${FILESEG}.nii.gz -path-model ${PATH_NNUNET_MODEL}/nnUNetTrainer__nnUNetPlans__${kernel} -pred-type sc -use-gpu + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # # Generate QC report + # sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:3 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c23.csv -append 1 + +} + +# Segment spinal cord using the MONAI contrast-agnostic model +segment_sc_MONAI(){ + local file="$1" + local file_gt_vert_label="$2" + local model="$3" # monai or swinunetr or mednext + local contrast="$4" # used only for saving output file name + + if [[ $model == 'monai' ]]; then + FILESEG="${file%%_*}_${contrast}_seg_monai" + PATH_MODEL=${PATH_MONAI_MODEL} + + elif [[ $model == 'swinunetr' ]]; then + FILESEG="${file%%_*}_${contrast}_seg_swinunetr" + PATH_MODEL=${PATH_SWIN_MODEL} + + elif [[ $model == 'mednext' ]]; then + FILESEG="${file%%_*}_${contrast}_seg_mednext" + PATH_MODEL=${PATH_MEDNEXT_MODEL} + + fi + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} + # Rename MONAI output + mv ${file}_pred.nii.gz ${FILESEG}.nii.gz + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Binarize MONAI output (which is soft by default); output is overwritten + sct_maths -i ${FILESEG}.nii.gz -bin 0.5 -o ${FILESEG}.nii.gz + + # Generate QC report with soft prediction + sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the soft prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:3 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c23.csv -append 1 +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +# Copy source images +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/anat/* . +# copy DWI data +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/dwi/* . + +# ------------------------------------------------------------------------------ +# contrast +# ------------------------------------------------------------------------------ +contrasts="T1w T2w T2star flip-1_mt-on_MTS flip-2_mt-off_MTS rec-average_dwi" +# contrasts="flip-2_mt-off_MTS rec-average_dwi" + +# Loop across contrasts +for contrast in ${contrasts}; do + + if [[ $contrast == "rec-average_dwi" ]]; then + type="dwi" + else + type="anat" + fi + + # go to the folder where the data is + cd ${PATH_DATA_PROCESSED}/${SUBJECT}/${type} + + # Get file name + file="${SUBJECT}_${contrast}" + + # Check if file exists + if [[ ! -e ${file}.nii.gz ]]; then + echo "File ${file}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}.nii.gz does not exist. Exiting." + exit 1 + fi + + # Copy GT spinal cord segmentation + copy_gt_seg "${file}" "${type}" + + # Copy soft GT spinal cord segmentation + copy_gt_softseg "${file}" "${type}" + + # Copy GT disc labels segmentation + copy_gt_disc_labels "${file}" "${type}" + + # Label vertebral levels in the native resolution + label_vertebrae ${file} 't2' + + # rename contrasts + if [[ $contrast == "flip-1_mt-on_MTS" ]]; then + contrast="MTon" + deepseg_input_c="t2s" + elif [[ $contrast == "flip-2_mt-off_MTS" ]]; then + contrast="MToff" + deepseg_input_c="t1" + elif [[ $contrast == "rec-average_dwi" ]]; then + contrast="DWI" + deepseg_input_c="dwi" + elif [[ $contrast == "T1w" ]]; then + deepseg_input_c="t1" + elif [[ $contrast == "T2w" ]]; then + deepseg_input_c="t2" + elif [[ $contrast == "T2star" ]]; then + deepseg_input_c="t2s" + fi + + # # 1. Compute (soft) CSA of the original soft GT + # # renaming file so that it can be fetched from the CSA csa file later + # FILEINPUT="${file%%_*}_${contrast}_softseg_soft" + # cp ${file}_softseg.nii.gz ${FILEINPUT}.nii.gz + # sct_process_segmentation -i ${FILEINPUT}.nii.gz -vert 2:4 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + + # Threshold the soft GT + FILETHRESH="${file%%_*}_${contrast}_softseg_bin" + sct_maths -i ${file}_softseg.nii.gz -bin 0.5 -o ${FILETHRESH}.nii.gz + + # 2. Compute CSA of the binarized soft GT + sct_process_segmentation -i ${FILETHRESH}.nii.gz -vert 2:3 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c23.csv -append 1 + + # 3. Segment SC using different methods, binarize at 0.5 and compute CSA + segment_sc_MONAI ${file} "${file}_seg-manual" 'monai' ${contrast} + segment_sc_MONAI ${file} "${file}_seg-manual" 'swinunetr' ${contrast} + segment_sc_MONAI ${file} "${file}_seg-manual" 'mednext' ${contrast} + segment_sc_nnUNet ${file} "${file}_seg-manual" '3d_fullres' ${contrast} + segment_sc ${file} "${file}_seg-manual" 'deepseg' ${deepseg_input_c} ${contrast} + # TODO: run on deep/progseg after fixing the contrasts for those + # segment_sc ${file_res} 't2' 'propseg' '' "${file}_seg-manual" ${native_res} + +done + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From eb90326a78a4a660b3488204360e711b2b59ee82 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 13:06:58 -0500 Subject: [PATCH 54/66] add script for csa comparison across resolutions for old & new models --- .../comparison_across_resolutions.sh | 404 ++++++++++++++++++ 1 file changed, 404 insertions(+) create mode 100644 csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh diff --git a/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh b/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh new file mode 100644 index 00000000..6baae431 --- /dev/null +++ b/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh @@ -0,0 +1,404 @@ +#!/bin/bash +# +# Compare the CSA of different models on the spine-generic test dataset +# +# Adapted from: https://github.com/ivadomed/model_seg_sci/blob/main/baselines/comparison_with_other_methods_sc.sh +# +# Usage: +# sct_run_batch -config config.json +# +# Example of config.json: +# { +# "path_data" : "", +# "path_output" : "_2023-08-18", +# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", +# "jobs" : 8, +# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# } +# +# The following global variables are retrieved from the caller sct_run_batch +# but could be overwritten by uncommenting the lines below: +# PATH_DATA_PROCESSED="~/data_processed" +# PATH_RESULTS="~/results" +# PATH_LOG="~/log" +# PATH_QC="~/qc" +# +# Author: Jan Valosek and Naga Karthik +# + +# Uncomment for full verbose +set -x + +# Immediately exit if error +set -e -o pipefail + +# Exit if user presses CTRL+C (Linux) or CMD+C (OSX) +trap "echo Caught Keyboard Interrupt within script. Exiting now.; exit" INT + +# Print retrieved variables from the sct_run_batch script to the log (to allow easier debug) +echo "Retrieved variables from from the caller sct_run_batch:" +echo "PATH_DATA: ${PATH_DATA}" +echo "PATH_DATA_PROCESSED: ${PATH_DATA_PROCESSED}" +echo "PATH_RESULTS: ${PATH_RESULTS}" +echo "PATH_LOG: ${PATH_LOG}" +echo "PATH_QC: ${PATH_QC}" + +SUBJECT=$1 +PATH_NNUNET_SCRIPT=$2 # path to the nnUNet contrast-agnostic run_inference_single_subject.py +PATH_NNUNET_MODEL=$3 # path to the nnUNet contrast-agnostic model +PATH_MONAI_SCRIPT=$4 # path to the MONAI contrast-agnostic run_inference_single_subject.py +PATH_MONAI_MODEL=$5 # path to the MONAI contrast-agnostic model trained on soft bin labels +PATH_SWIN_MODEL=$6 +PATH_MEDNEXT_MODEL=$7 + +echo "SUBJECT: ${SUBJECT}" +echo "PATH_NNUNET_SCRIPT: ${PATH_NNUNET_SCRIPT}" +echo "PATH_NNUNET_MODEL: ${PATH_NNUNET_MODEL}" +echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +echo "PATH_MONAI_MODEL: ${PATH_MONAI_MODEL}" +echo "PATH_SWIN_MODEL: ${PATH_SWIN_MODEL}" +echo "PATH_MEDNEXT_MODEL: ${PATH_MEDNEXT_MODEL}" + +# ------------------------------------------------------------------------------ +# CONVENIENCE FUNCTIONS +# ------------------------------------------------------------------------------ + +# Check if manual label already exists. If it does, copy it locally. +# NOTE: manual disc labels should go from C1-C2 to C7-T1. +label_vertebrae(){ + local file="$1" + local contrast="$2" + + # Update global variable with segmentation file name + FILESEG="${file}_seg-manual" + FILELABEL="${file}_discs" + + # Label vertebral levels + sct_label_utils -i ${file}.nii.gz -disc ${FILELABEL}.nii.gz -o ${FILESEG}_labeled.nii.gz + + # # Run QC + # sct_qc -i ${file}.nii.gz -s ${file_seg}_labeled.nii.gz -p sct_label_vertebrae -qc ${PATH_QC} -qc-subject ${SUBJECT} +} + + +# Copy GT spinal cord disc labels (located under derivatives/labels) +copy_gt_disc_labels(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILEDISCLABELS="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_discs.nii.gz" + echo "" + echo "Looking for manual disc labels: $FILEDISCLABELS" + if [[ -e $FILEDISCLABELS ]]; then + echo "Found! Copying ..." + rsync -avzh $FILEDISCLABELS ${file}_discs.nii.gz + else + echo "File ${FILEDISCLABELS} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Disc Labels ${FILEDISCLABELS} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT segmentation (located under derivatives/labels) +copy_gt_seg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels/${SUBJECT}/${type}/${file}_seg-manual.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_seg-manual.nii.gz + else + echo "File ${FILESEG}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + +# Copy GT soft segmentation (located under derivatives/labels_softseg) +copy_gt_softseg(){ + local file="$1" + local type="$2" + # Construct file name to GT segmentation located under derivatives/labels + FILESEG="${PATH_DATA}/derivatives/labels_softseg/${SUBJECT}/${type}/${file}_softseg.nii.gz" + echo "" + echo "Looking for manual segmentation: $FILESEG" + if [[ -e $FILESEG ]]; then + echo "Found! Copying ..." + rsync -avzh $FILESEG ${file}_softseg.nii.gz + else + echo "File ${FILESEG} does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: Manual Segmentation ${FILESEG} does not exist. Exiting." + exit 1 + fi +} + + +# TODO: Fix the contrast input for deepseg and propseg (i.e. dwi, mton, mtoff won't work) +# Segment spinal cord using methods available in SCT (sct_deepseg_sc or sct_propseg), resample the prediction back to +# native resolution and compute CSA in native space +segment_sc() { + local file="$1" + local file_gt_vert_label="$2" + local method="$3" # deepseg or propseg + local contrast_input="$4" # used for input arg `-c` + local contrast_name="$5" # used only for saving output file name + local native_res="$6" # native resolution of the image + + # Segment spinal cord + if [[ $method == 'deepseg' ]];then + # FILESEG="${file}_seg_${method}_${kernel}" + # FILESEG="${file%%_*}_${contrast_name}_seg_${method}_2d" + FILESEG="${file}_seg_${method}_2d" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_deepseg_sc -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast_input} -kernel 2d -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + elif [[ $method == 'propseg' ]]; then + FILESEG="${file}_seg_${method}" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + sct_propseg -i ${file}.nii.gz -o ${FILESEG}.nii.gz -c ${contrast} -qc ${PATH_QC} -qc-subject ${SUBJECT} + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Remove centerline (we don't need it) + rm ${file}_centerline.nii.gz + + fi + + # Resample the prediction back to native resolution; output is overwritten + sct_resample -i ${FILESEG}.nii.gz -mm ${native_res} -x linear -o ${FILESEG}.nii.gz + + # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:3 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_models_resolutions_c23.csv -append 1 + +} + +# Segment spinal cord using the contrast-agnostic nnUNet model +segment_sc_nnUNet(){ + local file="$1" + local file_gt_vert_label="$2" + local kernel="$3" # 2d or 3d + local contrast="$4" # used only for saving output file name + local native_res="$5" # native resolution of the image + + # FILESEG="${file%%_*}_${contrast}_seg_nnunet" + FILESEG="${file}_seg_nnunet" + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_NNUNET_SCRIPT} -i ${file}.nii.gz -o ${FILESEG}.nii.gz -path-model ${PATH_NNUNET_MODEL}/nnUNetTrainer__nnUNetPlans__${kernel} -pred-type sc -use-gpu + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # # Generate QC report + # sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Resample the prediction back to native resolution; output is overwritten + sct_resample -i ${FILESEG}.nii.gz -mm ${native_res} -x linear -o ${FILESEG}.nii.gz + + # Compute CSA from the prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:3 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_models_resolutions_c23.csv -append 1 + +} + +# Segment spinal cord using the MONAI contrast-agnostic model +segment_sc_MONAI(){ + local file="$1" + local file_gt_vert_label="$2" + local model="$3" # monai or swinunetr or mednext + local contrast="$4" # used only for saving output file name + local native_res="$5" # native resolution of the image + + if [[ $model == 'monai' ]]; then + # FILESEG="${file%%_*}_${contrast}_seg_monai" + FILESEG="${file}_seg_monai" + PATH_MODEL=${PATH_MONAI_MODEL} + + elif [[ $model == 'swinunetr' ]]; then + FILESEG="${file}_seg_swinunetr" + PATH_MODEL=${PATH_SWIN_MODEL} + + elif [[ $model == 'mednext' ]]; then + FILESEG="${file}_seg_mednext" + PATH_MODEL=${PATH_MEDNEXT_MODEL} + + fi + + # Get the start time + start_time=$(date +%s) + # Run SC segmentation + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} + # Rename MONAI output + mv ${file}_pred.nii.gz ${FILESEG}.nii.gz + # Get the end time + end_time=$(date +%s) + # Calculate the time difference + execution_time=$(python3 -c "print($end_time - $start_time)") + echo "${FILESEG},${execution_time}" >> ${PATH_RESULTS}/execution_time.csv + + # Resample the prediction back to native resolution; output is overwritten + sct_resample -i ${FILESEG}.nii.gz -mm ${native_res} -x linear -o ${FILESEG}.nii.gz + + # Binarize MONAI output (which is soft by default); output is overwritten + sct_maths -i ${FILESEG}.nii.gz -bin 0.5 -o ${FILESEG}.nii.gz + + # Generate QC report with soft prediction + sct_qc -i ${file}.nii.gz -s ${FILESEG}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} + + # Compute CSA from the soft prediction resampled back to native resolution using the GT vertebral labels + sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:3 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_models_resolutions_c23.csv -append 1 +} + + +# ------------------------------------------------------------------------------ +# SCRIPT STARTS HERE +# ------------------------------------------------------------------------------ +# get starting time: +start=`date +%s` + +# Display useful info for the log, such as SCT version, RAM and CPU cores available +sct_check_dependencies -short + +# Go to folder where data will be copied and processed +cd $PATH_DATA_PROCESSED + +# Copy source images +# Note: we use '/./' in order to include the sub-folder 'ses-0X' +# We do a substitution '/' --> '_' in case there is a subfolder 'ses-0X/' +# rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/anat/* . +rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/anat/${SUBJECT//[\/]/_}_*T2w.* . +# # copy DWI data +# rsync -Ravzh ${PATH_DATA}/./${SUBJECT}/dwi/* . + +# ------------------------------------------------------------------------------ +# contrast +# ------------------------------------------------------------------------------ +# contrasts="T1w T2w T2star flip-1_mt-on_MTS flip-2_mt-off_MTS rec-average_dwi" +# contrasts="flip-2_mt-off_MTS rec-average_dwi" +contrasts="T2w" + +# ------------------------------------------------------------------------------ +# resolutions +# ------------------------------------------------------------------------------ +resolutions="1x1x1 0.5x0.5x4 1.5x1.5x1.5 3x0.5x0.5 2x2x2" +# resolutions="0.5x0.5x4 1.5x1.5x1.5" + +# Loop across contrasts +for contrast in ${contrasts}; do + + if [[ $contrast == "rec-average_dwi" ]]; then + type="dwi" + else + type="anat" + fi + + # go to the folder where the data is + cd ${PATH_DATA_PROCESSED}/${SUBJECT}/${type} + + # Get file name + file="${SUBJECT}_${contrast}" + + # Check if file exists + if [[ ! -e ${file}.nii.gz ]]; then + echo "File ${file}.nii.gz does not exist" >> ${PATH_LOG}/missing_files.log + echo "ERROR: File ${file}.nii.gz does not exist. Exiting." + exit 1 + fi + + # Get the native resolution of the image + native_res=$(sct_image -i ${file}.nii.gz -header | grep pixdim | awk -F'[][]' '{split($2, a, ", "); print a[2]"x"a[3]"x"a[4]}') + + # Copy GT spinal cord segmentation + copy_gt_seg "${file}" "${type}" + + # # Copy soft GT spinal cord segmentation + # copy_gt_softseg "${file}" "${type}" + + # Copy GT disc labels segmentation + copy_gt_disc_labels "${file}" "${type}" + + # Label vertebral levels in the native resolution + label_vertebrae ${file} 't2' + + # rename contrasts + if [[ $contrast == "flip-1_mt-on_MTS" ]]; then + contrast="MTon" + deepseg_input_c="t2s" + elif [[ $contrast == "flip-2_mt-off_MTS" ]]; then + contrast="MToff" + deepseg_input_c="t1" + elif [[ $contrast == "rec-average_dwi" ]]; then + contrast="DWI" + deepseg_input_c="dwi" + elif [[ $contrast == "T1w" ]]; then + deepseg_input_c="t1" + elif [[ $contrast == "T2w" ]]; then + deepseg_input_c="t2" + elif [[ $contrast == "T2star" ]]; then + deepseg_input_c="t2s" + fi + + # Loop across resolutions + for res in ${resolutions}; do + + # TODO: should we also resample the GT? + # # Threshold the soft GT + # FILETHRESH="${file%%_*}_${contrast}_softseg_bin" + # sct_maths -i ${file}_softseg.nii.gz -bin 0.5 -o ${FILETHRESH}.nii.gz + + # echo "Resampling softseg to ${res}mm resolution ..." + # file_res="${file}_iso-$(echo "${res}" | awk -Fx '{print $1}' | sed 's/\.//')mm" + + # # 2. Compute CSA of the binarized soft GT + # sct_process_segmentation -i ${FILETHRESH}.nii.gz -vert 2:3 -vertfile ${file}_seg-manual_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c23.csv -append 1 + + echo "Resampling image to ${res}mm resolution ..." + file_res="${file}_res_$(echo "${res}" | awk -Fx '{print $1}' | sed 's/\.//')mm" + sct_resample -i ${file}.nii.gz -mm ${res} -x linear -o ${file_res}.nii.gz + + # 3. Segment SC using different methods, binarize at 0.5 and compute CSA + segment_sc_MONAI ${file_res} "${file}_seg-manual" 'monai' ${contrast} ${native_res} + segment_sc_MONAI ${file_res} "${file}_seg-manual" 'swinunetr' ${contrast} ${native_res} + segment_sc_MONAI ${file_res} "${file}_seg-manual" 'mednext' ${contrast} ${native_res} + segment_sc_nnUNet ${file_res} "${file}_seg-manual" '3d_fullres' ${contrast} ${native_res} + segment_sc ${file_res} "${file}_seg-manual" 'deepseg' ${deepseg_input_c} ${contrast} ${native_res} + # TODO: run on deep/progseg after fixing the contrasts for those + # segment_sc ${file_res} 't2' 'propseg' '' "${file}_seg-manual" ${native_res} + + done + +done + +# ------------------------------------------------------------------------------ +# End +# ------------------------------------------------------------------------------ + +# Display results (to easily compare integrity across SCT versions) +end=`date +%s` +runtime=$((end-start)) +echo +echo "~~~" +echo "SCT version: `sct_version`" +echo "Ran on: `uname -nsr`" +echo "Duration: $(($runtime / 3600))hrs $((($runtime / 60) % 60))min $(($runtime % 60))sec" +echo "~~~" From 2dc4cd3106cb2836d360ca5784e66ea7ee655153 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 13:11:59 -0500 Subject: [PATCH 55/66] update by adding support for analyzing csa across resolutions --- csa_generate_figures/analyse_csa_across.py | 94 +++++++++++++++------- 1 file changed, 66 insertions(+), 28 deletions(-) diff --git a/csa_generate_figures/analyse_csa_across.py b/csa_generate_figures/analyse_csa_across.py index 87370ef9..5c6ee31a 100644 --- a/csa_generate_figures/analyse_csa_across.py +++ b/csa_generate_figures/analyse_csa_across.py @@ -16,6 +16,7 @@ # HUE_ORDER = ["softseg_soft", "softseg_bin", "nnunet", "monai_soft", "monai_bin"] HUE_ORDER = ["softseg_bin", "deepseg_2d", "nnunet", "monai", "swinunetr", "mednext"] HUE_ORDER_THR = ["GT", "15", "1", "05", "01", "005"] +HUE_ORDER_RES = ["1mm", "05mm", "15mm", "3mm", "2mm"] CONTRAST_ORDER = ["DWI", "MTon", "MToff", "T1w", "T2star", "T2w"] @@ -45,14 +46,14 @@ def fetch_participant_id(filename_path): # Function to extract contrast and method from the filename -def extract_contrast_and_details(filename, analysis_type): +def extract_contrast_and_details(filename, across="Method"): """ Extract the segmentation method and resolution from the filename. The method (e.g., propseg, deepseg_2d, nnunet_3d_fullres, monai) and resolution (e.g., 1mm) are embedded in the filename. """ # pattern = r'.*iso-(\d+mm).*_(propseg|deepseg_2d|nnunet_3d_fullres|monai).*' - if analysis_type == "methods": + if across == "Method": # pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_soft|softseg_bin|nnunet|monai_soft|monai_bin).*' pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg_bin|deepseg_2d|nnunet|monai|swinunetr|mednext).*' match = re.search(pattern, filename) @@ -61,7 +62,7 @@ def extract_contrast_and_details(filename, analysis_type): else: return 'Unknown', 'Unknown' - elif analysis_type == "thresholds": + elif across == "Threshold": pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w).*_(softseg|monai)_thr_(\d+).*' match = re.search(pattern, filename) if match: @@ -69,12 +70,16 @@ def extract_contrast_and_details(filename, analysis_type): else: return 'Unknown', 'Unknown', 'Unknown' - elif analysis_type == "resolutions": - pattern = r'.*iso-(\d+mm).*_(softseg_bin|deepseg_2d|nnunet|monai|swinunetr|mednext).*' - # TODO + elif across == "Resolution": + pattern = r'.*_(DWI|MTon|MToff|T1w|T2star|T2w)_res_(\d+mm).*_(deepseg_2d|nnunet|monai|swinunetr|mednext).*' + match = re.search(pattern, filename) + if match: + return match.group(1), match.group(3), match.group(2) + else: + return 'Unknown', 'Unknown', 'Unknown' else: - raise ValueError(f'Unknown analysis type: {analysis_type}. Choices: [methods, resolutions, thresholds].') + raise ValueError(f'Unknown analysis type: {across}. Choices: [Method, Resolution, Threshold].') @@ -116,22 +121,29 @@ def generate_figure(data, contrast, file_path): plt.show() -def generate_figure_std(data, file_path): +def generate_figure_std(data, file_path, across="Method", hue_order=HUE_ORDER): """ Generate violinplot showing STD across participants for each method """ + if across == "Threshold": + # create a dataframe with only "monai" + data = data[data['Method'] == "monai"] + elif across == "Resolution": + # create a dataframe with only the specified model + model = "monai" + data = data[data['Method'] == model] # Compute mean and std across contrasts for each method - df = data.groupby(['Method', 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() + df = data.groupby([across, 'Participant'])['MEAN(area)'].agg(['mean', 'std']).reset_index() plt.figure(figsize=(12, 6)) - sns.violinplot(x='Method', y='std', data=df, order=HUE_ORDER) + sns.violinplot(x=across, y='std', data=df, order=hue_order) # overlay swarm plot on the violin plot to show individual data points - sns.swarmplot(x='Method', y='std', data=df, color='k', order=HUE_ORDER, size=3) + sns.swarmplot(x=across, y='std', data=df, color='k', order=hue_order, size=3) # plt.xticks(rotation=45) - plt.xlabel('Method') + plt.xlabel(across) plt.ylabel('STD [mm^2]') - plt.title(f'STD of C2-C4 CSA for each method') + plt.title(f'STD of C2-C3 CSA for each {across}') # Add horizontal dashed grid plt.grid(axis='y', alpha=0.5, linestyle='dashed') @@ -141,14 +153,32 @@ def generate_figure_std(data, file_path): # Draw vertical line between 1st and 2nd violin plt.axvline(x=0.5, color='k', linestyle='--') - # Compute the mean +- std across resolutions for each method and place it above the corresponding violin - for method in df['Method'].unique(): - mean = df[df['Method'] == method]['std'].mean() - std = df[df['Method'] == method]['std'].std() - plt.text(HUE_ORDER.index(method), ymax-1, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + if across == "Method": + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for method in df['Method'].unique(): + mean = df[df['Method'] == method]['std'].mean() + std = df[df['Method'] == method]['std'].std() + plt.text(hue_order.index(method), ymax-0.5, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + elif across == "Threshold": + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for thr in df['Threshold'].unique(): + mean = df[df['Threshold'] == thr]['std'].mean() + std = df[df['Threshold'] == thr]['std'].std() + plt.text(hue_order.index(thr), ymax-0.5, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + elif across == "Resolution": + # Compute the mean +- std across resolutions for each method and place it above the corresponding violin + for res in df['Resolution'].unique(): + mean = df[df['Resolution'] == res]['std'].mean() + std = df[df['Resolution'] == res]['std'].std() + plt.text(hue_order.index(res), ymax-0.5, f'{mean:.2f} +- {std:.2f}', ha='center', va='bottom', color='k') + else: + raise ValueError(f'Unknown analysis type: {across}. Choices: [Method, Resolution, Threshold].') # Save the figure in 300 DPI as a PNG file - save_figure(file_path, "std_csa.png") + if across == "Resolution": + save_figure(file_path, f"std_csa_{across.lower()}_{model}.png") + else: + save_figure(file_path, f"std_csa_{across.lower()}.png") def generate_figure_abs_csa_error(file_path, data, hue_order=None): @@ -365,7 +395,7 @@ def main(file_path, analysis_type="methods"): # Apply the function to extract method and the corresponding analysis details if analysis_type == "methods": data['Contrast'], data['Method'] = zip( - *data['Filename'].apply(extract_contrast_and_details, analysis_type=analysis_type)) + *data['Filename'].apply(extract_contrast_and_details, across="Method")) # Generate violinplot showing STD across participants for each method generate_figure_std(data, file_path) @@ -379,18 +409,26 @@ def main(file_path, analysis_type="methods"): elif analysis_type == "thresholds": data['Contrast'], data['Method'], data['Threshold'] = zip( - *data['Filename'].apply(extract_contrast_and_details, analysis_type=analysis_type)) + *data['Filename'].apply(extract_contrast_and_details, across="Threshold")) + + # Generate violinplot showing STD across participants for each threshold + # generate_figure_std_threshold(data, file_path, analysis_type="Threshold") + generate_figure_std(data, file_path, across="Threshold", hue_order=HUE_ORDER_THR) - # Generate violinplot showing absolute CSA error across participants for each threshold value - generate_figure_abs_csa_error_threshold(file_path, data, hue_order=HUE_ORDER_THR) + # # Generate violinplot showing absolute CSA error across participants for each threshold value + # generate_figure_abs_csa_error_threshold(file_path, data, hue_order=HUE_ORDER_THR) - # Generate violinplot showing absolute CSA error for each contrast for a given threshold - for threshold in HUE_ORDER_THR[1:]: - generate_figure_abs_csa_error_per_contrast(file_path, data, method=None, threshold=threshold) + # # Generate violinplot showing absolute CSA error for each contrast for a given threshold + # for threshold in HUE_ORDER_THR[1:]: + # generate_figure_abs_csa_error_per_contrast(file_path, data, method=None, threshold=threshold) elif analysis_type == "resolutions": - # TODO - pass + data['Contrast'], data['Method'], data['Resolution'] = zip( + *data['Filename'].apply(extract_contrast_and_details, across="Resolution")) + + # Generate violinplot showing STD across participants for each resolution + # generate_figure_std(data, file_path, across="Resolution", hue_order=HUE_ORDER_RES) + generate_figure_std(data, file_path, across="Method", hue_order=HUE_ORDER[1:]) else: raise ValueError(f'Unknown analysis type: {analysis_type}. Choices: [methods, resolutions, thresholds].') From 5425989464e0bb025819c7b91b09a2bf6743cc8d Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 13:16:51 -0500 Subject: [PATCH 56/66] add feature for training swinunetr & mednext models --- monai/main.py | 90 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 27 deletions(-) diff --git a/monai/main.py b/monai/main.py index add717c4..dd448154 100644 --- a/monai/main.py +++ b/monai/main.py @@ -18,17 +18,20 @@ from monai.utils import set_determinism from monai.inferers import sliding_window_inference -from monai.networks.nets import UNETR +from monai.networks.nets import UNETR, SwinUNETR from monai.data import (DataLoader, CacheDataset, load_decathlon_datalist, decollate_batch) from monai.transforms import (Compose, EnsureType, EnsureTyped, Invertd, SaveImage) +# mednext +from nnunet_mednext import MedNeXt def get_args(): parser = argparse.ArgumentParser(description='Script for training contrast-agnositc SC segmentation model.') # arguments for model - parser.add_argument('-m', '--model', choices=['unetr', 'nnunet'], default='nnunet', type=str, - help='Model type to be used. Currently only supports nnUNet.') + parser.add_argument('-m', '--model', choices=['nnunet', 'mednext', 'swinunetr'], + default='nnunet', type=str, + help='Model type to be used. Options: nnunet, mednext, swinunetr.') # path to the config file parser.add_argument("--config", type=str, default="./config.json", help="Path to the config file containing all training details.") @@ -188,7 +191,7 @@ def training_step(self, batch, batch_idx): output = self.forward(inputs) # logits # print(f"labels.shape: {labels.shape} \t output.shape: {output.shape}") - if args.model == "nnunet" and self.cfg['model'][args.model]["enable_deep_supervision"]: + if args.model in ["nnunet", "mednext"] and self.cfg['model'][args.model]["enable_deep_supervision"]: # calculate dice loss for each output loss, train_soft_dice = 0.0, 0.0 @@ -283,7 +286,7 @@ def validation_step(self, batch, batch_idx): outputs = sliding_window_inference(inputs, self.inference_roi_size, mode="gaussian", sw_batch_size=4, predictor=self.forward, overlap=0.5,) # outputs shape: (B, C, ) - if args.model == "nnunet" and self.cfg['model'][args.model]["enable_deep_supervision"]: + if args.model in ["nnunet", "mednext"] and self.cfg['model'][args.model]["enable_deep_supervision"]: # we only need the output with the highest resolution outputs = outputs[0] @@ -380,7 +383,7 @@ def test_step(self, batch, batch_idx): batch["pred"] = sliding_window_inference(test_input, self.inference_roi_size, sw_batch_size=4, predictor=self.forward, overlap=0.5) - if args.model == "nnunet" and self.cfg['model'][args.model]["enable_deep_supervision"]: + if args.model in ["nnunet", "mednext"] and self.cfg['model'][args.model]["enable_deep_supervision"]: # we only need the output with the highest resolution batch["pred"] = batch["pred"][0] @@ -476,28 +479,28 @@ def main(args): optimizer_class = torch.optim.SGD # define models - if args.model in ["unetr"]: - # TODO: update if ever using UNETR - # define image size to be fed to the model - img_size = (160, 224, 96) + if args.model in ["swinunetr"]: + # # define image size to be fed to the model # define model - net = UNETR(spatial_dims=3, - in_channels=1, out_channels=1, - img_size=img_size, - feature_size=args.feature_size, - hidden_size=args.hidden_size, - mlp_dim=args.mlp_dim, - num_heads=args.num_heads, - pos_embed="conv", - norm_name="instance", - res_block=True, - dropout_rate=0.2, - ) - img_size = f"{img_size[0]}x{img_size[1]}x{img_size[2]}" - save_exp_id = f"{args.model}_opt={args.optimizer}_lr={args.learning_rate}" \ - f"_fs={args.feature_size}_hs={args.hidden_size}_mlpd={args.mlp_dim}_nh={args.num_heads}" \ - f"_CSAdiceL_nspv={args.num_samples_per_volume}_bs={args.batch_size}_{img_size}" \ + net = SwinUNETR(spatial_dims=config["model"]["swinunetr"]["spatial_dims"], + in_channels=1, out_channels=1, + img_size=config["preprocessing"]["crop_pad_size"], + depths=config["model"]["swinunetr"]["depths"], + feature_size=config["model"]["swinunetr"]["feature_size"], + num_heads=config["model"]["swinunetr"]["num_heads"], + ) + patch_size = f"{config['preprocessing']['crop_pad_size'][0]}x" \ + f"{config['preprocessing']['crop_pad_size'][1]}x" \ + f"{config['preprocessing']['crop_pad_size'][2]}" + + save_exp_id = f"{args.model}_seed={config['seed']}_" \ + f"{config['dataset']['contrast']}_{config['dataset']['label_type']}_" \ + f"d={config['model']['swinunetr']['depths'][0]}_" \ + f"nf={config['model']['swinunetr']['feature_size']}_" \ + f"opt={config['opt']['name']}_lr={config['opt']['lr']}_AdapW_" \ + f"bs={config['opt']['batch_size']}_{patch_size}" \ + # save_exp_id = f"_CSAdiceL_nspv={args.num_samples_per_volume}_bs={args.batch_size}_{img_size}" \ elif args.model == "nnunet": @@ -543,6 +546,39 @@ def main(args): if args.debug: save_exp_id = f"DEBUG_{save_exp_id}" + + elif args.model == "mednext": + # NOTE: the S, B models in the paper don't fit as-is for this data, gpu + # hence tweaking the models + logger.info(f"Using MedNext model tweaked ...") + net = MedNeXt( + in_channels=config["model"]["mednext"]["num_input_channels"], + n_channels=config["model"]["mednext"]["base_num_features"], + n_classes=config["model"]["mednext"]["num_classes"], + exp_r=2, + kernel_size=config["model"]["mednext"]["kernel_size"], + deep_supervision=config["model"]["mednext"]["enable_deep_supervision"], + do_res=True, + do_res_up_down=True, + checkpoint_style="outside_block", + block_counts=config["model"]["mednext"]["block_counts"], + ) + + # variable for saving patch size in the experiment id (same as crop_pad_size) + patch_size = f"{config['preprocessing']['crop_pad_size'][0]}x" \ + f"{config['preprocessing']['crop_pad_size'][1]}x" \ + f"{config['preprocessing']['crop_pad_size'][2]}" + # count number of 2s in the block_counts list + num_two_blocks = config["model"]["mednext"]["block_counts"].count(2) + # save experiment id + save_exp_id = f"{args.model}_seed={config['seed']}_" \ + f"{config['dataset']['contrast']}_{config['dataset']['label_type']}_" \ + f"nf={config['model']['mednext']['base_num_features']}_bcs={num_two_blocks}_" \ + f"opt={config['opt']['name']}_lr={config['opt']['lr']}_AdapW_" \ + f"bs={config['opt']['batch_size']}_{patch_size}" \ + + if args.debug: + save_exp_id = f"DEBUG_{save_exp_id}" timestamp = datetime.now().strftime(f"%Y%m%d-%H%M") # prints in YYYYMMDD-HHMMSS format save_exp_id = f"{save_exp_id}_{timestamp}" @@ -675,7 +711,7 @@ def main(args): check_val_every_n_epoch=config["opt"]["check_val_every_n_epochs"], max_epochs=config["opt"]["max_epochs"], precision=32, - enable_progress_bar=False) + enable_progress_bar=True) # profiler="simple",) # to profile the training time taken for each step # Train! From 47edbbd043c38707a82832b33c0fae744707e701 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 13:52:22 -0500 Subject: [PATCH 57/66] add support for running inference with new models --- monai/run_inference_single_image.py | 50 +++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 8ab5c68b..1a09dfae 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -14,9 +14,11 @@ import torch.nn as nn import json from time import time +import yaml from monai.inferers import sliding_window_inference from monai.data import (DataLoader, Dataset, decollate_batch) +from monai.networks.nets import SwinUNETR from monai.transforms import (Compose, EnsureTyped, Invertd, SaveImage, Spacingd, LoadImaged, NormalizeIntensityd, EnsureChannelFirstd, DivisiblePadd, Orientationd, ResizeWithPadOrCropd) @@ -24,6 +26,7 @@ from dynamic_network_architectures.building_blocks.helper import get_matching_instancenorm, convert_dim_to_conv_op from dynamic_network_architectures.initialization.weight_init import init_last_bn_before_add_to_0 +from nnunet_mednext import MedNeXt # NNUNET global params INIT_FILTERS=32 @@ -72,6 +75,8 @@ def get_parser(): ' Default: 64x192x-1') parser.add_argument('--device', default="gpu", type=str, choices=["gpu", "cpu"], help='Device to run inference on. Default: cpu') + parser.add_argument('--model', default="nnunet", type=str, choices=["monai", "swinunetr", "mednext"], + help='Model to use for inference. Default: nnunet') return parser @@ -234,7 +239,44 @@ def main(): test_loader = DataLoader(test_ds, batch_size=1, shuffle=False, num_workers=8, pin_memory=True) # define model - net = create_nnunet_from_plans(plans=nnunet_plans, num_input_channels=1, num_classes=1, deep_supervision=ENABLE_DS) + if args.model == "monai": + net = create_nnunet_from_plans(plans=nnunet_plans, + num_input_channels=1, num_classes=1, deep_supervision=ENABLE_DS) + + elif args.model == "swinunetr": + # load config file + config_path = os.path.join(args.chkp_path, "config.yaml") + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + net = SwinUNETR( + spatial_dims=config["model"]["swinunetr"]["spatial_dims"], + in_channels=1, out_channels=1, + img_size=config["preprocessing"]["crop_pad_size"], + depths=config["model"]["swinunetr"]["depths"], + feature_size=config["model"]["swinunetr"]["feature_size"], + num_heads=config["model"]["swinunetr"]["num_heads"]) + + elif args.model == "mednext": + config_path = os.path.join(args.chkp_path, "config.yaml") + with open(config_path, "r") as f: + config = yaml.safe_load(f) + + net = MedNeXt( + in_channels=config["model"]["mednext"]["num_input_channels"], + n_channels=config["model"]["mednext"]["base_num_features"], + n_classes=config["model"]["mednext"]["num_classes"], + exp_r=2, + kernel_size=config["model"]["mednext"]["kernel_size"], + deep_supervision=config["model"]["mednext"]["enable_deep_supervision"], + do_res=True, + do_res_up_down=True, + checkpoint_style="outside_block", + block_counts=config["model"]["mednext"]["block_counts"],) + + else: + raise ValueError("Model not recognized. Please choose from: nnunet, swinunetr, mednext") + # define list to collect the test metrics test_step_outputs = [] @@ -268,8 +310,10 @@ def main(): batch["pred"] = sliding_window_inference(test_input, inference_roi_size, mode="gaussian", sw_batch_size=4, predictor=net, overlap=0.5, progress=False) - # take only the highest resolution prediction - batch["pred"] = batch["pred"][0] + if args.model in ["monai", "mednext"]: + # take only the highest resolution prediction + # NOTE: both these models use Deep Supervision, so only the highest resolution prediction is taken + batch["pred"] = batch["pred"][0] # NOTE: monai's models do not normalize the output, so we need to do it manually if bool(F.relu(batch["pred"]).max()): From 8cafaab3711f917a70651e9cfe17323fe8fdbd54 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 14:06:27 -0500 Subject: [PATCH 58/66] add arg to output soft/hard sc seg masks --- monai/run_inference_single_image.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index 1a09dfae..f18defe1 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -77,6 +77,9 @@ def get_parser(): help='Device to run inference on. Default: cpu') parser.add_argument('--model', default="nnunet", type=str, choices=["monai", "swinunetr", "mednext"], help='Model to use for inference. Default: nnunet') + parser.add_argument('--pred-type', default="soft", type=str, choices=["soft", "hard"], + help='Type of prediction to output/save. `soft` outputs soft segmentation masks with a threshold of 0.1' + '`hard` outputs binarized masks thresholded at 0.5 Default: hard') return parser @@ -325,9 +328,11 @@ def main(): pred = post_test_out[0]['pred'].cpu() - # binarize the prediction with a threshold of 0.5 - pred[pred >= 0.5] = 1 - pred[pred < 0.5] = 0 + # threshold or binarize the output based on the pred_type + if args.pred_type == "soft": + pred[pred < 0.1] = 0 + elif args.pred_type == "hard": + pred = torch.where(pred > 0.5, 1, 0) # get subject name subject_name = (batch["image_meta_dict"]["filename_or_obj"][0]).split("/")[-1].replace(".nii.gz", "") From 8911c97fd93f84487137598f3c57d449b14a16d1 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 14:10:11 -0500 Subject: [PATCH 59/66] add --pred-type soft input in segment_sc_MONAI function --- csa_qc_evaluation_spine_generic/comparison_across_models.sh | 2 +- .../comparison_across_resolutions.sh | 2 +- .../comparison_across_training_labels.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/csa_qc_evaluation_spine_generic/comparison_across_models.sh b/csa_qc_evaluation_spine_generic/comparison_across_models.sh index da1a9f2f..1ea059a4 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_models.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_models.sh @@ -235,7 +235,7 @@ segment_sc_MONAI(){ # Get the start time start_time=$(date +%s) # Run SC segmentation - python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} --pred-type soft # Rename MONAI output mv ${file}_pred.nii.gz ${FILESEG}.nii.gz # Get the end time diff --git a/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh b/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh index 6baae431..9e66d849 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh @@ -247,7 +247,7 @@ segment_sc_MONAI(){ # Get the start time start_time=$(date +%s) # Run SC segmentation - python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} --pred-type soft # Rename MONAI output mv ${file}_pred.nii.gz ${FILESEG}.nii.gz # Get the end time diff --git a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh index 3be3cf68..23637d18 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh @@ -230,7 +230,7 @@ segment_sc_MONAI(){ # Get the start time start_time=$(date +%s) # Run SC segmentation - python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu --pred-type soft # Rename MONAI output mv ${file}_pred.nii.gz ${FILESEG}.nii.gz # Get the end time From d96c3a4cfa988e3f2ea27fca1ac4879b8440df15 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Tue, 5 Mar 2024 14:11:14 -0500 Subject: [PATCH 60/66] add keys for mednext and swinunetr models in training yaml --- configs/train_all.yaml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/configs/train_all.yaml b/configs/train_all.yaml index 77196a07..4e4a4882 100644 --- a/configs/train_all.yaml +++ b/configs/train_all.yaml @@ -33,7 +33,7 @@ opt: max_epochs: 200 batch_size: 2 # Interval between validation checks in epochs - check_val_every_n_epochs: 10 + check_val_every_n_epochs: 5 # Early stopping patience (this is until patience * check_val_every_n_epochs) early_stopping_patience: 20 @@ -56,8 +56,16 @@ model: ] enable_deep_supervision: True - unetr: - feature_size: 16 - hidden_size: 768 # dimensionality of hidden embeddings - mlp_dim: 2048 # dimensionality of the MLPs - num_heads: 12 # number of heads in multi-head Attention \ No newline at end of file + mednext: + num_input_channels: 1 + base_num_features: 32 + num_classes: 1 + kernel_size: 3 # 3x3x3 and 5x5x5 were tested in publication + block_counts: [2,2,2,2,1,1,1,1,1] # number of blocks in each layer + enable_deep_supervision: True + + swinunetr: + spatial_dims: 3 + depths: [2, 2, 2, 2] + num_heads: [3, 6, 12, 24] # number of heads in multi-head Attention + feature_size: 36 \ No newline at end of file From dadb66650caace97625c6c5a90cc61bc0221d415 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 7 Mar 2024 09:54:15 -0500 Subject: [PATCH 61/66] add keep_largest_object postprocessing util --- monai/run_inference_single_image.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index f18defe1..bb3ce2bb 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -15,6 +15,7 @@ import json from time import time import yaml +from scipy import ndimage from monai.inferers import sliding_window_inference from monai.data import (DataLoader, Dataset, decollate_batch) @@ -206,6 +207,30 @@ def prepare_data(path_image, crop_size=(64, 160, 320)): return test_ds, test_post_pred +# =========================================================================== +# Post-processing +# =========================================================================== +def keep_largest_object(predictions): + """Keep the largest connected object from the input array (2D or 3D). + + Taken from: + https://github.com/ivadomed/ivadomed/blob/e101ebea632683d67deab3c50dd6b372207de2a9/ivadomed/postprocessing.py#L101-L116 + + Args: + predictions (ndarray or nibabel object): Input segmentation. Image could be 2D or 3D. + + Returns: + ndarray or nibabel (same object as the input). + """ + # Find number of closed objects using skimage "label" + labeled_obj, num_obj = ndimage.label(np.copy(predictions)) + # If more than one object is found, keep the largest one + if num_obj > 1: + # Keep the largest object + predictions[np.where(labeled_obj != (np.bincount(labeled_obj.flat)[1:].argmax() + 1))] = 0 + return predictions + + # =========================================================================== # Inference method # =========================================================================== @@ -334,6 +359,9 @@ def main(): elif args.pred_type == "hard": pred = torch.where(pred > 0.5, 1, 0) + # keep the largest connected object + pred = keep_largest_object(pred) + # get subject name subject_name = (batch["image_meta_dict"]["filename_or_obj"][0]).split("/")[-1].replace(".nii.gz", "") logger.info(f"Saving subject: {subject_name}") From 894db9cf92ec43d11cc2b09a860badc5e4c6e802 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 7 Mar 2024 10:00:53 -0500 Subject: [PATCH 62/66] update script to generate qc for all models --- qc_other_datasets/generate_qc.sh | 79 ++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/qc_other_datasets/generate_qc.sh b/qc_other_datasets/generate_qc.sh index 4506cf9d..fb8dbf6e 100644 --- a/qc_other_datasets/generate_qc.sh +++ b/qc_other_datasets/generate_qc.sh @@ -10,10 +10,10 @@ # Example of config.json: # { # "path_data" : "", -# "path_output" : "_2023-08-18", -# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", -# "jobs" : 8, -# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# "path_output" : "/results_qc_other_datasets/qc-reports", +# "script" : "/qc_other_datasets/generate_qc.sh", +# "jobs" : 5, +# "script_args" : " /nnUnet/run_inference_single_subject.py /monai/run_inference_single_image.py " # } # # The following global variables are retrieved from the caller sct_run_batch @@ -47,6 +47,10 @@ SUBJECT=$1 QC_DATASET=$2 # dataset name to generate QC for PATH_NNUNET_SCRIPT=$3 # path to the nnUNet contrast-agnostic run_inference_single_subject.py PATH_NNUNET_MODEL=$4 # path to the nnUNet contrast-agnostic model +PATH_MONAI_SCRIPT=$5 # path to the MONAI contrast-agnostic run_inference_single_subject.py +PATH_MONAI_MODEL=$6 # path to the MONAI contrast-agnostic model trained on soft bin labels +PATH_SWIN_MODEL=$7 +PATH_MEDNEXT_MODEL=$8 # PATH_MONAI_SCRIPT=$3 # path to the MONAI contrast-agnostic run_inference_single_subject.py # PATH_MONAI_MODEL_SOFT=$4 # path to the MONAI contrast-agnostic model trained on soft labels # PATH_MONAI_MODEL_SOFTBIN=$5 # path to the MONAI contrast-agnostic model trained on soft_bin labels @@ -55,6 +59,10 @@ echo "SUBJECT: ${SUBJECT}" echo "QC_DATASET: ${QC_DATASET}" echo "PATH_NNUNET_SCRIPT: ${PATH_NNUNET_SCRIPT}" echo "PATH_NNUNET_MODEL: ${PATH_NNUNET_MODEL}" +echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" +echo "PATH_MONAI_MODEL: ${PATH_MONAI_MODEL}" +echo "PATH_SWIN_MODEL: ${PATH_SWIN_MODEL}" +echo "PATH_MEDNEXT_MODEL: ${PATH_MEDNEXT_MODEL}" # echo "PATH_MONAI_SCRIPT: ${PATH_MONAI_SCRIPT}" # echo "PATH_MONAI_MODEL_SOFT: ${PATH_MONAI_MODEL_SOFT}" # echo "PATH_MONAI_MODEL_SOFTBIN: ${PATH_MONAI_MODEL_SOFTBIN}" @@ -106,16 +114,13 @@ copy_gt_seg(){ fi } -# TODO: Fix the contrast input for deepseg and propseg (i.e. dwi, mton, mtoff won't work) # Segment spinal cord using methods available in SCT (sct_deepseg_sc or sct_propseg), resample the prediction back to # native resolution and compute CSA in native space segment_sc() { local file="$1" - local contrast="$2" - local method="$3" # deepseg or propseg - local kernel="$4" # 2d or 3d; only relevant for deepseg - local file_gt_vert_label="$5" - local native_res="$6" + local method="$2" # deepseg or propseg + local contrast="$3" # used for input arg `-c` + local kernel="2d" # 2d or 3d; only relevant for deepseg # Segment spinal cord if [[ $method == 'deepseg' ]];then @@ -149,8 +154,8 @@ segment_sc() { fi - # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels - sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 + # # Compute CSA from the the SC segmentation resampled back to native resolution using the GT vertebral labels + # sct_process_segmentation -i ${FILESEG}.nii.gz -vert 2:4 -vertfile ${file_gt_vert_label}_labeled.nii.gz -o $PATH_RESULTS/csa_label_types_c24.csv -append 1 } @@ -183,22 +188,36 @@ segment_sc_nnUNet(){ # Segment spinal cord using the MONAI contrast-agnostic model segment_sc_MONAI(){ local file="$1" - local label_type="$2" # soft or soft_bin + # local label_type="$2" # soft or soft_bin + local model="$2" # monai, swinunetr, mednext - if [[ $label_type == 'soft' ]]; then - FILEPRED="${file}_seg_monai_soft" - PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFT} + # if [[ $label_type == 'soft' ]]; then + # FILEPRED="${file}_seg_monai_soft" + # PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFT} + + # elif [[ $label_type == 'soft_bin' ]]; then + # FILEPRED="${file}_seg_monai_bin" + # PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFTBIN} - elif [[ $label_type == 'soft_bin' ]]; then - FILEPRED="${file}_seg_monai_bin" - PATH_MONAI_MODEL=${PATH_MONAI_MODEL_SOFTBIN} + # fi + if [[ $model == 'monai' ]]; then + FILEPRED="${file}_seg_monai" + PATH_MODEL=${PATH_MONAI_MODEL} + + elif [[ $model == 'swinunetr' ]]; then + FILEPRED="${file}_seg_swinunetr" + PATH_MODEL=${PATH_SWIN_MODEL} + + elif [[ $model == 'mednext' ]]; then + FILEPRED="${file}_seg_mednext" + PATH_MODEL=${PATH_MEDNEXT_MODEL} fi # Get the start time start_time=$(date +%s) # Run SC segmentation - python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MONAI_MODEL} --device gpu + python ${PATH_MONAI_SCRIPT} --path-img ${file}.nii.gz --path-out . --chkp-path ${PATH_MODEL} --device gpu --model ${model} # Rename MONAI output mv ${file}_pred.nii.gz ${FILEPRED}.nii.gz # Get the end time @@ -210,7 +229,7 @@ segment_sc_MONAI(){ # Binarize MONAI output (which is soft by default); output is overwritten sct_maths -i ${FILEPRED}.nii.gz -bin 0.5 -o ${FILEPRED}.nii.gz - # Generate QC report with soft prediction + # Generate QC report sct_qc -i ${file}.nii.gz -s ${FILEPRED}.nii.gz -p sct_deepseg_sc -qc ${PATH_QC} -qc-subject ${SUBJECT} # compute ANIMA metrics @@ -236,20 +255,19 @@ cd $PATH_DATA_PROCESSED if [[ $QC_DATASET == "sci-colorado" ]]; then contrast="T2w" label_suffix="seg-manual" + deepseg_input_c="t2" -elif [[ $QC_DATASET == "basel-mp2rage-rpi" ]]; then +elif [[ $QC_DATASET == "basel-mp2rage" ]]; then contrast="UNIT1" label_suffix="label-SC_seg" + deepseg_input_c="t1" elif [[ $QC_DATASET == "dcm-zurich" ]]; then contrast="acq-axial_T2w" label_suffix="label-SC_mask-manual" + deepseg_input_c="t2" -elif [[ $QC_DATASET == "stanford-epi" ]]; then - contrast="task-rest_desc-mocomean_bold" - label_suffix="spinalcord_mask" fi -# TODO: add stanford EPI data for QC echo "Contrast: ${contrast}" @@ -275,11 +293,12 @@ copy_gt_seg "${file}" "${label_suffix}" # Segment SC using different methods, binarize at 0.5 and compute QC # segment_sc_MONAI ${file} 'soft' # segment_sc_MONAI ${file} 'soft_bin' +segment_sc_MONAI ${file} 'monai' +# segment_sc_MONAI ${file} 'swinunetr' +# segment_sc_MONAI ${file} 'mednext' -segment_sc_nnUNet ${file} '3d_fullres' -# # TODO: run on deep/progseg after fixing the contrasts for those -# segment_sc ${file_res} 't2' 'deepseg' '2d' "${file}_seg-manual" ${native_res} -# segment_sc ${file_res} 't2' 'propseg' '' "${file}_seg-manual" ${native_res} +# segment_sc_nnUNet ${file} '3d_fullres' +# segment_sc ${file} 'deepseg' ${deepseg_input_c} # ------------------------------------------------------------------------------ From 041ba27636d4dfc14b03f37a41bbebc2d6c80627 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 7 Mar 2024 10:12:31 -0500 Subject: [PATCH 63/66] update usage examples in docstring --- .../comparison_across_models.sh | 8 ++++---- .../comparison_across_resolutions.sh | 8 ++++---- .../comparison_across_thresholds.sh | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/csa_qc_evaluation_spine_generic/comparison_across_models.sh b/csa_qc_evaluation_spine_generic/comparison_across_models.sh index 1ea059a4..3f70a4df 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_models.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_models.sh @@ -10,10 +10,10 @@ # Example of config.json: # { # "path_data" : "", -# "path_output" : "_2023-08-18", -# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", -# "jobs" : 8, -# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# "path_output" : "/results_csa/across_models/csa_c2c3_20240226", +# "script" : "/csa_qc_evaluation_spine_generic/comparison_across_models.sh", +# "jobs" : 5, +# "script_args" : "/nnUnet/run_inference_single_subject.py /monai/run_inference_single_image.py " # } # # The following global variables are retrieved from the caller sct_run_batch diff --git a/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh b/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh index 9e66d849..78641b55 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh @@ -10,10 +10,10 @@ # Example of config.json: # { # "path_data" : "", -# "path_output" : "_2023-08-18", -# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", -# "jobs" : 8, -# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# "path_output" : "/results_csa/across_resolutions/csa_models_resolutions_t2w_20240228", +# "script" : "/csa_qc_evaluation_spine_generic/comparison_across_resolutions.sh", +# "jobs" : 5, +# "script_args" : "/nnUnet/run_inference_single_subject.py /monai/run_inference_single_image.py " # } # # The following global variables are retrieved from the caller sct_run_batch diff --git a/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh b/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh index 567b0ca9..1b5192f3 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh @@ -11,10 +11,10 @@ # Example of config.json: # { # "path_data" : "", -# "path_output" : "_2023-08-18", -# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", -# "jobs" : 8, -# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# "path_output" : "/results_csa/across_thresholds/csa_softbin_thr_20240218", +# "script" : "/csa_qc_evaluation_spine_generic/comparison_across_thresholds.sh", +# "jobs" : 5, +# "script_args" : "/monai/run_inference_single_image.py " # } # # The following global variables are retrieved from the caller sct_run_batch From d37f25524701052bab7fe94739b5e1fa63e908ef Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 7 Mar 2024 10:14:45 -0500 Subject: [PATCH 64/66] incorporate suggestions --- scripts/train.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/train.sh b/scripts/train.sh index f3f584f9..9a44b17c 100644 --- a/scripts/train.sh +++ b/scripts/train.sh @@ -1,7 +1,7 @@ #!/bin/bash # # This script does the following: -# 1. Creates a virtual environment and installs the required dependencies +# 1. Creates a virtual environment `venv_monai` and installs the required dependencies # 2. Generates a MSD-style datalist containing image/label pairs for training # 3. Trains the contrast-agnostic soft segmentation model # 4. Evaluates the model on the test set @@ -69,3 +69,6 @@ echo "----------------------------------------" # Train the model python monai/main.py --model $MODEL --config $PATH_TRAIN_YAML --debug + +# Deactivate the virtual environment +conda deactivate From b728ee7390c80659f52e85d15228e4c7e9f3727e Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 7 Mar 2024 10:20:48 -0500 Subject: [PATCH 65/66] update docstring usage example --- .../comparison_across_training_labels.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh index 23637d18..a3f004b1 100644 --- a/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh +++ b/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh @@ -10,10 +10,10 @@ # Example of config.json: # { # "path_data" : "", -# "path_output" : "_2023-08-18", -# "script" : "/model_seg_sci/baselines/comparison_with_other_methods.sh", -# "jobs" : 8, -# "script_args" : "/model_seg_sci/packaging/run_inference_single_subject.py /sci-multisite-model /monai/run_inference_single_image.py " +# "path_output" : "/results_csa/across_labels/csa_training_labels_20240203", +# "script" : "/csa_qc_evaluation_spine_generic/comparison_across_training_labels.sh", +# "jobs" : 5, +# "script_args" : "/nnUnet/run_inference_single_subject.py /monai/run_inference_single_image.py " # } # # The following global variables are retrieved from the caller sct_run_batch From c1c4e8099021975b3c47dfbfa2e994cf3d92e450 Mon Sep 17 00:00:00 2001 From: Naga Karthik Date: Thu, 7 Mar 2024 10:42:44 -0500 Subject: [PATCH 66/66] fix minor bug in --model args --- monai/run_inference_single_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/run_inference_single_image.py b/monai/run_inference_single_image.py index bb3ce2bb..a7eda9c7 100644 --- a/monai/run_inference_single_image.py +++ b/monai/run_inference_single_image.py @@ -76,8 +76,8 @@ def get_parser(): ' Default: 64x192x-1') parser.add_argument('--device', default="gpu", type=str, choices=["gpu", "cpu"], help='Device to run inference on. Default: cpu') - parser.add_argument('--model', default="nnunet", type=str, choices=["monai", "swinunetr", "mednext"], - help='Model to use for inference. Default: nnunet') + parser.add_argument('--model', default="monai", type=str, choices=["monai", "swinunetr", "mednext"], + help='Model to use for inference. Default: monai') parser.add_argument('--pred-type', default="soft", type=str, choices=["soft", "hard"], help='Type of prediction to output/save. `soft` outputs soft segmentation masks with a threshold of 0.1' '`hard` outputs binarized masks thresholded at 0.5 Default: hard')