diff --git a/docs/source/conf.py b/docs/source/conf.py index 5036df6..a8ae57f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -78,6 +78,41 @@ # The master toctree document. master_doc = 'index' +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +todo_include_todos = False + +# Resolve function for the linkcode extension. +def linkcode_resolve(domain, info): + def find_source(): + # try to find the file and line number, based on code from numpy: + # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286 + obj = sys.modules[info['module']] + for part in info['fullname'].split('.'): + obj = getattr(obj, part) + import inspect + import os + fn = inspect.getsourcefile(obj) + fn = os.path.relpath(fn, start=os.path.dirname(torchcp.__file__)) + source, lineno = inspect.getsourcelines(obj) + return fn, lineno, lineno + len(source) - 1 + + if domain != 'py' or not info['module']: + return None + try: + filename = 'torchcp/%s#L%d-L%d' % find_source() + except Exception: + filename = info['module'].replace('.', '/') + '.py' + tag = 'master' + url = "https://github.com/ml-stat-Sustech/TorchCP/blob/%s/%s" + return url % (tag, filename) + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -95,3 +130,5 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True + + diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 586374d..80c4157 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -9,7 +9,7 @@ We developed TorchCP under Python 3.9 and PyTorch 2.0.1. To install TorchCP, sim .. code-block:: bash - pip install --index-url https://test.pypi.org/simple/ --no-deps torchcp + pip install torchcp or clone the repo and run diff --git a/docs/source/torchcp.classification.rst b/docs/source/torchcp.classification.rst index e06fd95..2189d4e 100644 --- a/docs/source/torchcp.classification.rst +++ b/docs/source/torchcp.classification.rst @@ -26,6 +26,14 @@ Predictors ClusterPredictor WeightedPredictor +.. automodule:: torchcp.classification.loss +Loss functions +------- + +.. autosummary:: + :nosignatures: + + ConfTr Detailed description -------------------- @@ -57,4 +65,8 @@ Detailed description :members: .. autoclass:: WeightedPredictor - :members: \ No newline at end of file + :members: + +.. autoclass:: ConfTr + :members: + diff --git a/docs/source/torchcp.regression.rst b/docs/source/torchcp.regression.rst index 39b08d3..d03e063 100644 --- a/docs/source/torchcp.regression.rst +++ b/docs/source/torchcp.regression.rst @@ -12,6 +12,15 @@ Predictors cqr ACI +.. automodule:: torchcp.regression.loss +Loss functions +------- + +.. autosummary:: + :nosignatures: + + QuantileLoss + Detailed description -------------------- @@ -24,3 +33,7 @@ Detailed description .. autoclass:: ACI :members: + +.. autoclass:: QuantileLoss + :members: + diff --git a/examples/conformal_training.py b/examples/conformal_training.py index d06c0c7..f3840c6 100644 --- a/examples/conformal_training.py +++ b/examples/conformal_training.py @@ -17,6 +17,7 @@ import argparse +import itertools import torch import torch.nn as nn @@ -29,27 +30,9 @@ from torchcp.classification.scores import THR, APS, SAPS, RAPS from torchcp.utils import fix_randomness -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Covariate shift') - parser.add_argument('--seed', default=0, type=int) - parser.add_argument('--predictor', default="Standard", help="Standard") - parser.add_argument('--score', default="THR", help="THR") - parser.add_argument('--loss', default="CE", help="CE | ConfTr") - args = parser.parse_args() - res = {'Coverage_rate': 0, 'Average_size': 0} - num_trials = 1 - for seed in range(num_trials): - fix_randomness(seed=seed) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - ################################## - # Invalid prediction sets - ################################## - train_dataset = build_dataset("mnist") - train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) - class Net(nn.Module): +class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.fc1 = nn.Linear(28 * 28, 500) @@ -60,77 +43,54 @@ def forward(self, x): x = F.relu(self.fc1(x)) x = self.fc2(x) return x + +def train(model, device, train_loader,criterion, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + - - model = Net().to(device) - - if args.loss == "CE": - criterion = nn.CrossEntropyLoss() - elif args.loss == "ConfTr": - predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) - criterion = ConfTr(weights=0.01, - predictor=predictor, - alpha=0.05, - device=device, - fraction=0.5, - loss_types="valid", - base_loss_fn=nn.CrossEntropyLoss()) - else: - raise NotImplementedError - - optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) - - - def train(model, device, train_loader, optimizer, epoch): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - data, target = data.to(device), target.to(device) - optimizer.zero_grad() - output = model(data) - loss = criterion(output, target) - loss.backward() - optimizer.step() - if batch_idx % 10 == 0: - print( - f'Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100. * batch_idx / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}') - - - checkpoint_path = f'.cache/conformal_training_model_checkpoint_{args.loss}_seed={seed}.pth' - # if os.path.exists(checkpoint_path): - # checkpoint = torch.load(checkpoint_path) - # model.load_state_dict(checkpoint['model_state_dict']) - # else: - for epoch in range(1, 10): - train(model, device, train_data_loader, optimizer, epoch) - - torch.save({'model_state_dict': model.state_dict(), }, checkpoint_path) - - test_dataset = build_dataset("mnist", mode='test') - cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) - cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) - test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) - - if args.score == "THR": - score_function = THR() - elif args.score == "APS": - score_function = APS() - elif args.score == "RAPS": - score_function = RAPS(args.penalty, args.kreg) - elif args.score == "SAPS": - score_function = SAPS(weight=args.weight) - - alpha = 0.01 - if args.predictor == "Standard": - predictor = SplitPredictor(score_function, model) - elif args.predictor == "ClassWise": - predictor = ClassWisePredictor(score_function, model) - elif args.predictor == "Cluster": - predictor = ClusterPredictor(score_function, model, args.seed) - predictor.calibrate(cal_data_loader, alpha) - - # test examples - tmp_res = predictor.evaluate(test_data_loader) - res['Coverage_rate'] += tmp_res['Coverage_rate'] / num_trials - res['Average_size'] += tmp_res['Average_size'] / num_trials - - print(res) +if __name__ == '__main__': + alpha = 0.01 + num_trials = 5 + loss = "ConfTr" + result = {} + print(f"############################## {loss} #########################") + + predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) + criterion = ConfTr(weight=0.01, + predictor=predictor, + alpha=0.05, + fraction=0.5, + loss_type="valid", + base_loss_fn=nn.CrossEntropyLoss()) + + fix_randomness(seed=0) + ################################## + # Training a pytorch model + ################################## + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_dataset = build_dataset("mnist") + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) + test_dataset = build_dataset("mnist", mode='test') + cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) + + model = Net().to(device) + optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) + for epoch in range(1, 10): + train(model, device, train_data_loader, criterion, optimizer, epoch) + + + score_function = THR() + + predictor = SplitPredictor(score_function, model) + predictor.calibrate(cal_data_loader, alpha) + result = predictor.evaluate(test_data_loader) + print(f"Result--Coverage_rate: {result['Coverage_rate']}, Average_size: {result['Average_size']}") \ No newline at end of file diff --git a/examples/covariate_shift.py b/examples/covariate_shift.py index 6ce746c..ecb6e18 100644 --- a/examples/covariate_shift.py +++ b/examples/covariate_shift.py @@ -19,7 +19,7 @@ from torchcp.utils import fix_randomness if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Covariate shift') + parser = argparse.ArgumentParser(description='Coveriate shift') parser.add_argument('--seed', default=0, type=int) args = parser.parse_args() diff --git a/examples/dataset.py b/examples/dataset.py index 3706a51..8fd83d8 100644 --- a/examples/dataset.py +++ b/examples/dataset.py @@ -13,7 +13,7 @@ def build_dataset(dataset_name, transform=None, mode="train"): data_dir = os.path.join(usr_dir, "data") if dataset_name == 'imagenet': - if transform == None: + if transform is None: transform = trn.Compose([ trn.Resize(256), trn.CenterCrop(224), @@ -25,7 +25,7 @@ def build_dataset(dataset_name, transform=None, mode="train"): dataset = dset.ImageFolder(data_dir + "/imagenet/val", transform) elif dataset_name == 'imagenetv2': - if transform == None: + if transform is None: transform = trn.Compose([ trn.Resize(256), trn.CenterCrop(224), @@ -38,7 +38,7 @@ def build_dataset(dataset_name, transform=None, mode="train"): transform) elif dataset_name == 'mnist': - if transform == None: + if transform is None: transform = trn.Compose([ trn.ToTensor(), trn.Normalize((0.1307,), (0.3081,)) diff --git a/examples/imagenet_example.py b/examples/imagenet_example.py index 1e911b0..4d63a47 100644 --- a/examples/imagenet_example.py +++ b/examples/imagenet_example.py @@ -24,18 +24,14 @@ parser = argparse.ArgumentParser(description='') parser.add_argument('--seed', default=0, type=int) parser.add_argument('--alpha', default=0.1, type=float) - parser.add_argument('--predictor', default="Standard", help="Standard | ClassWise | Cluster") - parser.add_argument('--score', default="THR", help="THR | APS | SAPS") - parser.add_argument('--penalty', default=1, type=float) - parser.add_argument('--kreg', default=0, type=int) - parser.add_argument('--weight', default=0.2, type=int) - parser.add_argument('--split', default="random", type=str, help="proportional | doubledip | random") args = parser.parse_args() fix_randomness(seed=args.seed) + ####################################### + # Loading ImageNet dataset and a pytorch model + ####################################### model_name = 'ResNet101' - # load model model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) model_device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(model_device) @@ -55,47 +51,14 @@ cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1024, shuffle=False, pin_memory=True) test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1024, shuffle=False, pin_memory=True) + + ####################################### + # A standard process of conformal prediction + ####################################### alpha = args.alpha - print( - f"Experiment--Data : ImageNet, Model : {model_name}, Score : {args.score}, Predictor : {args.predictor}, Alpha : {alpha}") - num_classes = 1000 - if args.score == "THR": - score_function = THR() - elif args.score == "APS": - score_function = APS() - elif args.score == "RAPS": - score_function = RAPS(args.penalty, args.kreg) - elif args.score == "SAPS": - score_function = SAPS(weight=args.weight) - else: - raise NotImplementedError - - if args.predictor == "Standard": - predictor = SplitPredictor(score_function, model) - elif args.predictor == "ClassWise": - predictor = ClassWisePredictor(score_function, model) - elif args.predictor == "Cluster": - predictor = ClusterPredictor(score_function, model, args.seed) - else: - raise NotImplementedError + print(f"Experiment--Data : ImageNet, Model : {model_name}, Score : THR, Predictor : SplitPredictor, Alpha : {alpha}") + score_function = THR() + predictor = SplitPredictor(score_function, model) print(f"The size of calibration set is {len(cal_dataset)}.") predictor.calibrate(cal_data_loader, alpha) - # predictor.evaluate(test_data_loader) - - # test examples - print("Testing examples...") - prediction_sets = [] - labels_list = [] - with torch.no_grad(): - for examples in tqdm(test_data_loader): - tmp_x, tmp_label = examples[0], examples[1] - prediction_sets_batch = predictor.predict(tmp_x) - prediction_sets.extend(prediction_sets_batch) - labels_list.append(tmp_label) - test_labels = torch.cat(labels_list) - - metrics = Metrics() - print("Etestuating prediction sets...") - print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") - print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") - print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") + predictor.evaluate(test_data_loader) diff --git a/examples/imagenet_example_logits.py b/examples/imagenet_example_logits.py deleted file mode 100644 index ab92016..0000000 --- a/examples/imagenet_example_logits.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2023-present, SUSTech-ML. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -# - - -import argparse -import os -import pickle - -import torch -import torchvision -import torchvision.datasets as dset -import torchvision.transforms as trn -from tqdm import tqdm - -from torchcp.classification.predictors import SplitPredictor, ClusterPredictor, ClassWisePredictor -from torchcp.classification.scores import THR, APS, SAPS, RAPS, Margin -from torchcp.classification.utils.metrics import Metrics -from torchcp.utils import fix_randomness - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='') - parser.add_argument('--seed', default=0, type=int) - parser.add_argument('--alpha', default=0.1, type=float) - parser.add_argument('--predictor', default="Standard", help="Standard | ClassWise | Cluster") - parser.add_argument('--score', default="THR", help="THR | APS | SAPS") - parser.add_argument('--penalty', default=1, type=float) - parser.add_argument('--kreg', default=0, type=int) - parser.add_argument('--weight', default=0.2, type=int) - parser.add_argument('--split', default="random", type=str, help="proportional | doubledip | random") - args = parser.parse_args() - - fix_randomness(seed=args.seed) - - model_name = 'ResNet101' - fname = ".cache/" + model_name + ".pkl" - if os.path.exists(fname): - with open(fname, 'rb') as handle: - dataset = pickle.load(handle) - - else: - # load dataset - transform = trn.Compose([trn.Resize(256), - trn.CenterCrop(224), - trn.ToTensor(), - trn.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - ]) - usr_dir = os.path.expanduser('~') - data_dir = os.path.join(usr_dir, "data") - dataset = dset.ImageFolder(data_dir + "/imagenet/val", - transform) - data_loader = torch.utils.data.DataLoader(dataset, batch_size=320, shuffle=False, pin_memory=True) - - # load model - model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) - - logits_list = [] - labels_list = [] - with torch.no_grad(): - for examples in tqdm(data_loader): - tmp_x, tmp_label = examples[0], examples[1] - tmp_logits = model(tmp_x) - logits_list.append(tmp_logits) - labels_list.append(tmp_label) - logits = torch.cat(logits_list) - labels = torch.cat(labels_list) - dataset = torch.utils.data.TensorDataset(logits, labels.long()) - with open(fname, 'wb') as handle: - pickle.dump(dataset, handle, protocol=pickle.HIGHEST_PROTOCOL) - - cal_data, val_data = torch.utils.data.random_split(dataset, [25000, 25000]) - cal_logits = torch.stack([sample[0] for sample in cal_data]) - cal_labels = torch.stack([sample[1] for sample in cal_data]) - - test_logits = torch.stack([sample[0] for sample in val_data]) - test_labels = torch.stack([sample[1] for sample in val_data]) - - num_classes = 1000 - alpha = args.alpha - print( - f"Experiment--Data : ImageNet, Model : {model_name}, Score : {args.score}, Predictor : {args.predictor}, Alpha : {alpha}") - if args.score == "THR": - score_function = THR() - elif args.score == "APS": - score_function = APS() - elif args.score == "RAPS": - score_function = RAPS(args.penalty, args.kreg) - elif args.score == "SAPS": - score_function = SAPS(weight=args.weight) - elif args.score == "Margin": - score_function = Margin() - else: - raise NotImplementedError - - if args.predictor == "Standard": - predictor = SplitPredictor(score_function, model=None) - elif args.predictor == "ClassWise": - predictor = ClassWisePredictor(score_function, model=None) - elif args.predictor == "Cluster": - predictor = ClusterPredictor(score_function, split=args.split, model=None) - print(f"The size of calibration set is {cal_labels.shape[0]}.") - predictor.calculate_threshold(cal_logits, cal_labels, alpha) - - # print("Testing examples...") - # prediction_sets = [] - # for index, ele in enumerate(test_logits): - # prediction_set = predictor.predict_with_logits(ele) - # prediction_sets.append(prediction_set) - - prediction_sets = predictor.predict_with_logits(test_logits) - - metrics = Metrics() - print("Evaluating prediction sets...") - print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") - print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") - print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") diff --git a/examples/reg_CQR.py b/examples/reg_CQR.py deleted file mode 100644 index 44af2b5..0000000 --- a/examples/reg_CQR.py +++ /dev/null @@ -1,74 +0,0 @@ -import numpy as np -import torch -from torch.utils.data import TensorDataset -from tqdm import tqdm - -from torchcp.regression.loss import QuantileLoss -from torchcp.regression.predictors import CQR -from torchcp.regression.utils.metrics import Metrics -from torchcp.utils import fix_randomness -from utils import build_reg_data, build_regression_model - -device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") -fix_randomness(seed=2) - -X, y = build_reg_data() -indices = np.arange(X.shape[0]) -np.random.shuffle(indices) -split_index1 = int(len(indices) * 0.4) -split_index2 = int(len(indices) * 0.6) -part1, part2, part3 = np.split(indices, [split_index1, split_index2]) - -from sklearn.preprocessing import StandardScaler - -scalerX = StandardScaler() -scalerX = scalerX.fit(X[part1, :]) - -train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) -cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) -test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) - -train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) -cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) -test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) - -alpha = 0.1 -quantiles = [alpha / 2, 1 - alpha / 2] -model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) -criterion = QuantileLoss(quantiles) -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -# Train Model -epochs = 100 -for epoch in tqdm(range(epochs)): - for index, (tmp_x, tmp_y) in enumerate(train_data_loader): - outputs = model(tmp_x.to(device)) - loss = criterion(outputs, tmp_y.to(device)) - optimizer.zero_grad() - loss.backward() - optimizer.step() - -predictor = CQR(model) -predictor.calibrate(cal_data_loader, alpha) - -y_list = [] -x_list = [] -predict_list = [] -with torch.no_grad(): - for examples in test_data_loader: - tmp_x, tmp_y = examples[0].to(device), examples[1] - tmp_prediction_intervals = predictor.predict(tmp_x) - y_list.append(tmp_y) - x_list.append(tmp_x) - predict_list.append(tmp_prediction_intervals) - -predicts = torch.cat(predict_list).float().cpu() -test_y = torch.cat(y_list) -x = torch.cat(x_list).float() - -metrics = Metrics() -print("Etestuating prediction sets...") -print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}.") -print(f"Average_size: {metrics('average_size')(predicts)}.") - -# print(predictors.evaluate(test_data_loader)) diff --git a/examples/reg_aci.py b/examples/reg_aci.py deleted file mode 100644 index d103079..0000000 --- a/examples/reg_aci.py +++ /dev/null @@ -1,56 +0,0 @@ -import torch -from torch.utils.data import TensorDataset -from tqdm import tqdm - -from torchcp.regression import Metrics -from torchcp.regression.loss import QuantileLoss -from torchcp.regression.predictors import ACI -from torchcp.utils import fix_randomness -from utils import build_reg_data, build_regression_model - -device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") -fix_randomness(seed=2) - -X, y = build_reg_data(data_name="synthetic") -num_examples = X.shape[0] - -T0 = int(num_examples * 0.4) - -train_dataset = TensorDataset(torch.from_numpy(X[:T0, :]), torch.from_numpy(y[:T0])) -train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) - -alpha = 0.1 -quantiles = [alpha / 2, 1 - alpha / 2] -model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) -criterion = QuantileLoss(quantiles) -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -epochs = 10 -for epoch in tqdm(range(epochs)): - for index, (tmp_x, tmp_y) in enumerate(train_data_loader): - outputs = model(tmp_x.to(device)) - loss = criterion(outputs, tmp_y.to(device)) - optimizer.zero_grad() - loss.backward() - optimizer.step() - -predictor = ACI(model, 0.0001) - -test_y = torch.from_numpy(y[T0:num_examples]).to(device) -predicts = torch.zeros((num_examples - T0, 2)).to(device) -for i in range(num_examples - T0): - with torch.no_grad(): - cal_dataset = TensorDataset(torch.from_numpy(X[i:(T0 + i), :]), torch.from_numpy(y[i:(T0 + i)])) - cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) - predictor.calibrate(cal_data_loader, alpha) - tmp_x = torch.from_numpy(X[(T0 + i), :]) - if i == 0: - tmp_prediction_intervals = predictor.predict(tmp_x) - else: - tmp_prediction_intervals = predictor.predict(tmp_x, test_y[i - 1], predicts[i - 1]) - predicts[i, :] = tmp_prediction_intervals - -metrics = Metrics() -print("Etestuating prediction sets...") -print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}.") -print(f"Average_size: {metrics('average_size')(predicts)}.") diff --git a/examples/reg_splitCP.py b/examples/reg_splitCP.py deleted file mode 100644 index 0cb26cf..0000000 --- a/examples/reg_splitCP.py +++ /dev/null @@ -1,78 +0,0 @@ -import numpy as np -import torch -import torch.nn as nn -from torch.utils.data import TensorDataset -from tqdm import tqdm - -from torchcp.regression.predictors import SplitPredictor -from torchcp.utils import fix_randomness -from utils import build_reg_data, build_regression_model - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -fix_randomness(seed=2) - -X, y = build_reg_data() -indices = np.arange(X.shape[0]) -np.random.shuffle(indices) -split_index1 = int(len(indices) * 0.4) -split_index2 = int(len(indices) * 0.6) -part1, part2, part3 = np.split(indices, [split_index1, split_index2]) - -from sklearn.preprocessing import StandardScaler - -scalerX = StandardScaler() -scalerX = scalerX.fit(X[part1, :]) -train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) -cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) -test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) - -train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) -cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) -test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) - -model = build_regression_model("NonLinearNet")(X.shape[1], 1, 64, 0.5).to(device) -criterion = nn.MSELoss() -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -# Train Model -epochs = 200 -for epoch in tqdm(range(epochs)): - for index, (tmp_x, tmp_y) in enumerate(train_data_loader): - outputs = model(tmp_x.to(device)) - loss = criterion(outputs.reshape(-1), tmp_y.to(device)) - optimizer.zero_grad() - loss.backward() - optimizer.step() - -alpha = 0.1 -model.eval() -predictor = SplitPredictor(model) -predictor.calibrate(cal_data_loader, alpha) - -############################################ -# First method to evaluate test instances -############################################ -print(predictor.evaluate(test_data_loader)) - -############################################ -# Second method to evaluate test instances -############################################ -# y_list = [] -# x_list = [] -# predict_list = [] -# with torch.no_grad(): -# for examples in test_data_loader: -# tmp_x, tmp_y = examples[0].to(device), examples[1] -# tmp_prediction_intervals = predictors.predict(tmp_x) -# y_list.append(tmp_y) -# x_list.append(tmp_x) -# predict_list.append(tmp_prediction_intervals) - -# predicts = torch.cat(predict_list).float().cpu() -# test_y = torch.cat(y_list) -# x = torch.cat(x_list).float() - -# metrics = Metrics() -# print("Etestuating prediction sets...") -# print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}.") -# print(f"Average_size: {metrics('average_size')(predicts)}.") diff --git a/examples/regression.py b/examples/regression.py new file mode 100644 index 0000000..592c80d --- /dev/null +++ b/examples/regression.py @@ -0,0 +1,81 @@ +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import TensorDataset +from tqdm import tqdm +from sklearn.preprocessing import StandardScaler + + +from torchcp.regression.predictors import SplitPredictor,CQR +from torchcp.regression.loss import QuantileLoss +from torchcp.utils import fix_randomness +from utils import build_reg_data, build_regression_model + + +def train(model, device, epoch, train_data_loader, criterion, optimizer): + for index, (tmp_x, tmp_y) in enumerate(train_data_loader): + outputs = model(tmp_x.to(device)) + loss = criterion(outputs, tmp_y.unsqueeze(dim=1).to(device)) + optimizer.zero_grad() + loss.backward() + optimizer.step() + +if __name__ == '__main__': + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=1) + X, y = build_reg_data() + indices = np.arange(X.shape[0]) + np.random.shuffle(indices) + split_index1 = int(len(indices) * 0.4) + split_index2 = int(len(indices) * 0.6) + part1, part2, part3 = np.split(indices, [split_index1, split_index2]) + scalerX = StandardScaler() + scalerX = scalerX.fit(X[part1, :]) + train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) + cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) + test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) + + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + + epochs = 100 + alpha = 0.1 + + ################################## + # Split Conformal Prediction + ################################## + print("########################## SplitPredictor ###########################") + model = build_regression_model("NonLinearNet")(X.shape[1], 1, 64, 0.5).to(device) + criterion = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + predictor = SplitPredictor(model) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + ################################## + # Conformal Quantile Regression + ################################## + print("########################## CQR ###########################") + + + quantiles = [alpha / 2, 1 - alpha / 2] + model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) + criterion = QuantileLoss(quantiles) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + predictor = CQR(model) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) diff --git a/examples/run_test_logits.sh b/examples/run_test_logits.sh deleted file mode 100644 index 0bf3cb0..0000000 --- a/examples/run_test_logits.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -predictors=("Standard" "ClassWise" "Cluster") -scores=("THR" "APS" "RAPS" "SAPS") - -for predictor in "${predictors[@]}" -do - for score in "${scores[@]}" - do - python examples/imagenet_example_logits.py \ - --predictor ${predictor} \ - --score ${score} \ - --seed 0 - done -done \ No newline at end of file diff --git a/examples/time_series.py b/examples/time_series.py new file mode 100644 index 0000000..6856d9f --- /dev/null +++ b/examples/time_series.py @@ -0,0 +1,77 @@ +import torch +from torch.utils.data import TensorDataset +from tqdm import tqdm + +from torchcp.regression import Metrics +from torchcp.regression.loss import QuantileLoss +from torchcp.regression.predictors import ACI, CQR +from torchcp.utils import fix_randomness +from utils import build_reg_data, build_regression_model +from regression import train + + + +if __name__ == '__main__': + + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=2) + X, y = build_reg_data(data_name="synthetic") + num_examples = X.shape[0] + T0 = int(num_examples * 0.4) + train_dataset = TensorDataset(torch.from_numpy(X[:T0, :]), torch.from_numpy(y[:T0])) + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + + + + + + alpha = 0.1 + quantiles = [alpha / 2, 1 - alpha / 2] + model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) + criterion = QuantileLoss(quantiles) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + epochs = 10 + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + ################################## + # Conformal Quantile Regression + ################################## + print("########################## CQR ###########################") + + predictor = CQR(model) + cal_dataset = TensorDataset(torch.from_numpy(X[0:T0, :]), torch.from_numpy(y[0:T0])) + test_dataset = TensorDataset(torch.from_numpy(X[T0:, :]), torch.from_numpy(y[T0:])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + ################################## + # Adaptive Conformal Inference, + ################################## + print("########################## ACI ###########################") + predictor = ACI(model, 0.0001) + test_y = torch.from_numpy(y[T0:num_examples]).to(device) + predicts = torch.zeros((num_examples - T0, 2)).to(device) + for i in range(num_examples - T0): + with torch.no_grad(): + cal_dataset = TensorDataset(torch.from_numpy(X[i:(T0 + i), :]), torch.from_numpy(y[i:(T0 + i)])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + tmp_x = torch.from_numpy(X[(T0 + i), :]) + if i == 0: + tmp_prediction_intervals = predictor.predict(tmp_x) + else: + tmp_prediction_intervals = predictor.predict(tmp_x, test_y[i - 1], predicts[i - 1]) + predicts[i, :] = tmp_prediction_intervals + + metrics = Metrics() + print("Evaluating prediction sets...") + print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}") + print(f"Average_size: {metrics('average_size')(predicts)}") diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..316f4fd --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = + tests diff --git a/tests/dataset.py b/tests/dataset.py new file mode 100644 index 0000000..3706a51 --- /dev/null +++ b/tests/dataset.py @@ -0,0 +1,70 @@ +import os +import pathlib + +import torchvision.datasets as dset +import torchvision.transforms as trn +from PIL import Image +from torch.utils.data import Dataset + + +def build_dataset(dataset_name, transform=None, mode="train"): + # path of usr + usr_dir = os.path.expanduser('~') + data_dir = os.path.join(usr_dir, "data") + + if dataset_name == 'imagenet': + if transform == None: + transform = trn.Compose([ + trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + + dataset = dset.ImageFolder(data_dir + "/imagenet/val", + transform) + elif dataset_name == 'imagenetv2': + if transform == None: + transform = trn.Compose([ + trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + + dataset = ImageNetV2Dataset(os.path.join(data_dir, "imagenetv2/imagenetv2-matched-frequency-format-val"), + transform) + + elif dataset_name == 'mnist': + if transform == None: + transform = trn.Compose([ + trn.ToTensor(), + trn.Normalize((0.1307,), (0.3081,)) + ]) + if mode == "train": + dataset = dset.MNIST(data_dir, train=True, download=True, transform=transform) + elif mode == "test": + dataset = dset.MNIST(data_dir, train=False, download=True, transform=transform) + + else: + raise NotImplementedError + + return dataset + + +class ImageNetV2Dataset(Dataset): + def __init__(self, root, transform=None): + self.dataset_root = pathlib.Path(root) + self.fnames = list(self.dataset_root.glob("**/*.jpeg")) + self.transform = transform + + def __len__(self): + return len(self.fnames) + + def __getitem__(self, i): + img, label = Image.open(self.fnames[i]), int(self.fnames[i].parent.name) + if self.transform is not None: + img = self.transform(img) + return img, label diff --git a/tests/test_classification.py b/tests/test_classification.py new file mode 100644 index 0000000..096a85d --- /dev/null +++ b/tests/test_classification.py @@ -0,0 +1,134 @@ +# Copyright (c) 2023-present, SUSTech-ML. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + + +import argparse +import os +import pickle + +import torch +import torchvision +import torchvision.datasets as dset +import torchvision.transforms as trn +from tqdm import tqdm + +from torchcp.classification.predictors import SplitPredictor, ClusterPredictor, ClassWisePredictor +from torchcp.classification.scores import THR, APS, SAPS, RAPS, Margin +from torchcp.classification.utils.metrics import Metrics +from torchcp.utils import fix_randomness + + +transform = trn.Compose([trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + +def test_imagenet_logits(): + ####################################### + # Loading ImageNet dataset and a pytorch model + ####################################### + fix_randomness(seed=0) + model_name = 'ResNet101' + fname = ".cache/" + model_name + ".pkl" + if os.path.exists(fname): + with open(fname, 'rb') as handle: + dataset = pickle.load(handle) + + else: + usr_dir = os.path.expanduser('~') + data_dir = os.path.join(usr_dir, "data") + dataset = dset.ImageFolder(data_dir + "/imagenet/val", + transform) + data_loader = torch.utils.data.DataLoader(dataset, batch_size=320, shuffle=False, pin_memory=True) + + # load model + model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) + + logits_list = [] + labels_list = [] + with torch.no_grad(): + for examples in tqdm(data_loader): + tmp_x, tmp_label = examples[0], examples[1] + tmp_logits = model(tmp_x) + logits_list.append(tmp_logits) + labels_list.append(tmp_label) + logits = torch.cat(logits_list) + labels = torch.cat(labels_list) + dataset = torch.utils.data.TensorDataset(logits, labels.long()) + with open(fname, 'wb') as handle: + pickle.dump(dataset, handle, protocol=pickle.HIGHEST_PROTOCOL) + + cal_data, val_data = torch.utils.data.random_split(dataset, [25000, 25000]) + cal_logits = torch.stack([sample[0] for sample in cal_data]) + cal_labels = torch.stack([sample[1] for sample in cal_data]) + + test_logits = torch.stack([sample[0] for sample in val_data]) + test_labels = torch.stack([sample[1] for sample in val_data]) + + num_classes = 1000 + + ####################################### + # A standard process of conformal prediction + ####################################### + alpha = 0.1 + predictors = [SplitPredictor, ClassWisePredictor, ClusterPredictor] + score_functions = [THR(), APS(), RAPS(1, 0), SAPS(0.2), Margin()] + for score in score_functions: + for class_predictor in predictors: + predictor = class_predictor(score) + predictor.calculate_threshold(cal_logits, cal_labels, alpha) + print(f"Experiment--Data : ImageNet, Model : {model_name}, Score : {score.__class__.__name__}, Predictor : {predictor.__class__.__name__}, Alpha : {alpha}") + # print("Testing examples...") + # prediction_sets = [] + # for index, ele in enumerate(test_logits): + # prediction_set = predictor.predict_with_logits(ele) + # prediction_sets.append(prediction_set) + prediction_sets = predictor.predict_with_logits(test_logits) + + metrics = Metrics() + print("Evaluating prediction sets...") + print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") + print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") + print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") + + + + + +def test_imagenet(): + fix_randomness(seed=0) + ####################################### + # Loading ImageNet dataset and a pytorch model + ####################################### + model_name = 'ResNet101' + model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + model.to(device) + usr_dir = os.path.expanduser('~') + data_dir = os.path.join(usr_dir, "data") + dataset = dset.ImageFolder(data_dir + "/imagenet/val", transform) + + cal_dataset, test_dataset = torch.utils.data.random_split(dataset, [25000, 25000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1024, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1024, shuffle=False, pin_memory=True) + + ####################################### + # A standard process of conformal prediction + ####################################### + alpha = 0.1 + predictors = [SplitPredictor, ClassWisePredictor, ClusterPredictor] + score_functions = [THR(), APS(), RAPS(1, 0), SAPS(0.2), Margin()] + for score in score_functions: + for class_predictor in predictors: + predictor = class_predictor(score, model) + predictor.calibrate(cal_data_loader, alpha) + print(f"Experiment--Data : ImageNet, Model : {model_name}, Score : {score.__class__.__name__}, Predictor : {predictor.__class__.__name__}, Alpha : {alpha}") + print(predictor.evaluate(test_data_loader)) + + diff --git a/tests/test_conformal_training.py b/tests/test_conformal_training.py new file mode 100644 index 0000000..3ff5de7 --- /dev/null +++ b/tests/test_conformal_training.py @@ -0,0 +1,115 @@ +# Copyright (c) 2023-present, SUSTech-ML. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +# @Time : 13/12/2023 21:13 + + +# Copyright (c) 2023-present, SUSTech-ML. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + + +import argparse +import itertools + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +from dataset import build_dataset +from torchcp.classification.loss import ConfTr +from torchcp.classification.predictors import SplitPredictor, ClusterPredictor, ClassWisePredictor +from torchcp.classification.scores import THR, APS, SAPS, RAPS +from torchcp.utils import fix_randomness + + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = nn.Linear(28 * 28, 500) + self.fc2 = nn.Linear(500, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = self.fc2(x) + return x + +def train(model, device, train_loader,criterion, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + + +def test_training(): + alpha = 0.01 + num_trials = 5 + result = {} + for loss in ["CE", "ConfTr"]: + print(f"############################## {loss} #########################") + result[loss] = {} + if loss == "CE": + criterion = nn.CrossEntropyLoss() + elif loss == "ConfTr": + predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) + criterion = ConfTr(weight=0.01, + predictor=predictor, + alpha=0.05, + fraction=0.5, + loss_type="valid", + base_loss_fn=nn.CrossEntropyLoss()) + else: + raise NotImplementedError + for seed in range(num_trials): + fix_randomness(seed=seed) + ################################## + # Training a pytorch model + ################################## + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_dataset = build_dataset("mnist") + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) + test_dataset = build_dataset("mnist", mode='test') + cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) + + model = Net().to(device) + optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) + for epoch in range(1, 10): + train(model, device, train_data_loader, criterion, optimizer, epoch) + + for score in ["THR", "APS", "RAPS", "SAPS"]: + if score == "THR": + score_function = THR() + elif score == "APS": + score_function = APS() + elif score == "RAPS": + score_function = RAPS(1, 0) + elif score == "SAPS": + score_function = SAPS(weight=0.2) + if score not in result[loss]: + result[loss][score] = {} + result[loss][score]['Coverage_rate'] = 0 + result[loss][score]['Average_size'] = 0 + predictor = SplitPredictor(score_function, model) + predictor.calibrate(cal_data_loader, alpha) + tmp_res = predictor.evaluate(test_data_loader) + result[loss][score]['Coverage_rate'] += tmp_res['Coverage_rate'] / num_trials + result[loss][score]['Average_size'] += tmp_res['Average_size'] / num_trials + + for score in ["THR", "APS", "RAPS", "SAPS"]: + print(f"Score: {score}. Result is {result[loss][score]}") diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 0000000..1d990a1 --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,128 @@ +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import TensorDataset +from tqdm import tqdm +from sklearn.preprocessing import StandardScaler + + +from torchcp.regression.predictors import SplitPredictor,CQR,ACI +from torchcp.regression.loss import QuantileLoss +from torchcp.regression import Metrics +from torchcp.utils import fix_randomness +from utils import build_reg_data, build_regression_model + + +def train(model, device, epoch, train_data_loader, criterion, optimizer): + for index, (tmp_x, tmp_y) in enumerate(train_data_loader): + outputs = model(tmp_x.to(device)) + loss = criterion(outputs, tmp_y.unsqueeze(dim=1).to(device)) + optimizer.zero_grad() + loss.backward() + optimizer.step() + +def test_SplitPredictor(): + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=1) + X, y = build_reg_data() + indices = np.arange(X.shape[0]) + np.random.shuffle(indices) + split_index1 = int(len(indices) * 0.4) + split_index2 = int(len(indices) * 0.6) + part1, part2, part3 = np.split(indices, [split_index1, split_index2]) + scalerX = StandardScaler() + scalerX = scalerX.fit(X[part1, :]) + train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) + cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) + test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) + + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + + epochs = 100 + alpha = 0.1 + + ################################## + # Split Conformal Prediction + ################################## + print("########################## SplitPredictor ###########################") + model = build_regression_model("NonLinearNet")(X.shape[1], 1, 64, 0.5).to(device) + criterion = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + predictor = SplitPredictor(model) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + +def test_time_series(): + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=2) + X, y = build_reg_data(data_name="synthetic") + num_examples = X.shape[0] + T0 = int(num_examples * 0.4) + train_dataset = TensorDataset(torch.from_numpy(X[:T0, :]), torch.from_numpy(y[:T0])) + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + + + + + + alpha = 0.1 + quantiles = [alpha / 2, 1 - alpha / 2] + model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) + criterion = QuantileLoss(quantiles) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + epochs = 10 + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + ################################## + # Conformal Quantile Regression + ################################## + print("########################## CQR ###########################") + + predictor = CQR(model) + cal_dataset = TensorDataset(torch.from_numpy(X[0:T0, :]), torch.from_numpy(y[0:T0])) + test_dataset = TensorDataset(torch.from_numpy(X[T0:, :]), torch.from_numpy(y[T0:])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + ################################## + # Adaptive Conformal Inference, + ################################## + print("########################## ACI ###########################") + predictor = ACI(model, 0.0001) + test_y = torch.from_numpy(y[T0:num_examples]).to(device) + predicts = torch.zeros((num_examples - T0, 2)).to(device) + for i in range(num_examples - T0): + with torch.no_grad(): + cal_dataset = TensorDataset(torch.from_numpy(X[i:(T0 + i), :]), torch.from_numpy(y[i:(T0 + i)])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + tmp_x = torch.from_numpy(X[(T0 + i), :]) + if i == 0: + tmp_prediction_intervals = predictor.predict(tmp_x) + else: + tmp_prediction_intervals = predictor.predict(tmp_x, test_y[i - 1], predicts[i - 1]) + predicts[i, :] = tmp_prediction_intervals + + metrics = Metrics() + print("Evaluating prediction sets...") + print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}") + print(f"Average_size: {metrics('average_size')(predicts)}") diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..104787d --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,85 @@ +import numpy as np +import pandas as pd +import torch.nn as nn + +base_path = ".cache/data/" + + +def build_reg_data(data_name="community"): + if data_name == "community": + # https://github.com/vbordalo/Communities-Crime/blob/master/Crime_v1.ipynb + attrib = pd.read_csv(base_path + 'communities_attributes.csv', delim_whitespace=True) + data = pd.read_csv(base_path + 'communities.data', names=attrib['attributes']) + data = data.drop(columns=['state', 'county', + 'community', 'communityname', + 'fold'], axis=1) + + data = data.replace('?', np.nan) + + # Impute mean values for samples with missing values + + # imputer = SimpleImputer(missing_values = 'NaN', strategy = 'mean') + + # imputer = imputer.fit(data[['OtherPerCap']]) + # data[['OtherPerCap']] = imputer.transform(data[['OtherPerCap']]) + data['OtherPerCap'] = data['OtherPerCap'].astype("float") + mean_value = data['OtherPerCap'].mean() + data['OtherPerCap'].fillna(value=mean_value, inplace=True) + data = data.dropna(axis=1) + X = data.iloc[:, 0:100].values + y = data.iloc[:, 100].values + + # imputer = SimpleImputer(missing_values = 'NaN', strategy = 'mean') + + # imputer = imputer.fit(data[['OtherPerCap']]) + # data[['OtherPerCap']] = imputer.transform(data[['OtherPerCap']]) + data['OtherPerCap'] = data['OtherPerCap'].astype("float") + mean_value = data['OtherPerCap'].mean() + data['OtherPerCap'].fillna(value=mean_value, inplace=True) + data = data.dropna(axis=1) + X = data.iloc[:, 0:100].values + y = data.iloc[:, 100].values + elif data_name == "synthetic": + X = np.random.rand(500, 5) + y_wo_noise = 10 * np.sin(X[:, 0] * X[:, 1] * np.pi) + 20 * (X[:, 2] - 0.5) ** 2 + 10 * X[:, 3] + 5 * X[:, 4] + eplison = np.zeros(500) + phi = theta = 0.8 + delta_t_1 = np.random.randn() + for i in range(1, 500): + delta_t = np.random.randn() + eplison[i] = phi * eplison[i - 1] + delta_t_1 + theta * delta_t + delta_t_1 = delta_t + + y = y_wo_noise + eplison + + X = X.astype(np.float32) + y = y.astype(np.float32) + + return X, y + + +def build_regression_model(model_name="NonLinearNet"): + if model_name == "NonLinearNet": + class NonLinearNet(nn.Module): + def __init__(self, in_shape, out_shape, hidden_size, dropout): + super(NonLinearNet, self).__init__() + self.hidden_size = hidden_size + self.in_shape = in_shape + self.out_shape = out_shape + self.dropout = dropout + self.base_model = nn.Sequential( + nn.Linear(self.in_shape, self.hidden_size), + nn.ReLU(), + nn.Dropout(self.dropout), + nn.Linear(self.hidden_size, self.hidden_size), + nn.ReLU(), + nn.Dropout(self.dropout), + nn.Linear(self.hidden_size, self.out_shape), + ) + + def forward(self, x): + return self.base_model(x) + + return NonLinearNet + else: + raise NotImplementedError diff --git a/torchcp/classification/loss/conftr.py b/torchcp/classification/loss/conftr.py index 9978302..0380699 100644 --- a/torchcp/classification/loss/conftr.py +++ b/torchcp/classification/loss/conftr.py @@ -4,6 +4,8 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. # +__all__ = ["ConfTr"] + import torch @@ -12,48 +14,50 @@ class ConfTr(nn.Module): - def __init__(self, weights, predictor, alpha, fraction, loss_types="valid", target_size=1, + """ + Conformal Training (Stutz et al., 2021). + Paper: https://arxiv.org/abs/2110.09192 + + :param weight: the weight of each loss function + :param predictor: the CP predictors + :param alpha: the significance level for each training batch + :param fraction: the fraction of the calibration set in each training batch + :param loss_type: the selected (multi-selected) loss functions, which can be "valid", "classification", "probs", "coverage". + :param target_size: Optional: 0 | 1. + :param loss_transform: a transform for loss + :param base_loss_fn: a base loss function. For example, cross entropy in classification. + """ + def __init__(self, weight, predictor, alpha, fraction, loss_type="valid", target_size=1, loss_transform="square", base_loss_fn=None): - """ - :param weights: the weight of each loss function - :param predictor: the CP predictors - :param alpha: the significance level for each training batch - :param fraction: the fraction of the calibration set in each training batch - :param loss_types: the selected (multi-selected) loss functions, which can be "valid", "classification", "probs", "coverage". - :param target_size: - :param loss_transform: a transform for loss - :param base_loss_fn: a base loss function, such as cross entropy for classification - """ + super(ConfTr, self).__init__() - self.weight = weights + assert weight>0, "weight must be greater than 0." + assert (0 < fraction < 1), "fraction should be a value in (0,1)." + assert loss_type in ["valid", "classification", "probs", "coverage"], ('loss_type should be a value in [' + '"valid", "classification", "probs", ' + '"coverage"].') + assert target_size==0 or target_size ==1, "target_size should be 0 or 1." + assert loss_transform in ["square", "abs", "log"], ('loss_transform should be a value in ["square", "abs", ' + '"log"].') + self.weight = weight self.predictor = predictor self.alpha = alpha self.fraction = fraction - self.base_loss_fn = base_loss_fn - + self.loss_type = loss_type self.target_size = target_size + self.base_loss_fn = base_loss_fn + if loss_transform == "square": self.transform = torch.square elif loss_transform == "abs": self.transform = torch.abs elif loss_transform == "log": self.transform = torch.log - else: - raise NotImplementedError self.loss_functions_dict = {"valid": self.__compute_hinge_size_loss, "probs": self.__compute_probabilistic_size_loss, "coverage": self.__compute_coverage_loss, - "classification": self.__compute_classification_loss} - - if type(loss_types) == set: - if type(weights) != set: - raise TypeError("weights must be a set.") - elif type(loss_types) == str: - if type(weights) != float and type(weights) != int: - raise TypeError("weights must be a float or a int.") - else: - raise TypeError("types must be a set or a string.") - self.loss_types = loss_types + "classification": self.__compute_classification_loss + } def forward(self, logits, labels): # Compute Size Loss @@ -65,15 +69,10 @@ def forward(self, logits, labels): self.predictor.calculate_threshold(cal_logits.detach(), cal_labels.detach(), self.alpha) tau = self.predictor.q_hat - test_scores = self.predictor.score_function.predict(test_logits) + test_scores = self.predictor.score_function(test_logits) + # Computing the probability of each label contained in the prediction set. pred_sets = torch.sigmoid(tau - test_scores) - - if type(self.loss_types) == set: - loss = torch.tensor(0).to(logits.device) - for i in range(len(self.loss_types)): - loss += self.weight[i] * self.loss_functions_dict[self.loss_types[i]](pred_sets, test_labels) - else: - loss = self.weight * self.loss_functions_dict[self.loss_types](pred_sets, test_labels) + loss = self.weight * self.loss_functions_dict[self.loss_type](pred_sets, test_labels) if self.base_loss_fn is not None: loss += self.base_loss_fn(logits, labels).float() @@ -109,13 +108,13 @@ def __compute_coverage_loss(self, pred_sets, labels): def __compute_classification_loss(self, pred_sets, labels): # Convert labels to one-hot encoding one_hot_labels = F.one_hot(labels, num_classes=pred_sets.shape[1]).float() - loss_matrix = torch.eye(pred_sets.shape[1]).to(pred_sets.device) + loss_matrix = torch.eye(pred_sets.shape[1], device=pred_sets.device) # Calculate l1 and l2 losses l1 = (1 - pred_sets) * one_hot_labels * loss_matrix[labels] l2 = pred_sets * (1 - one_hot_labels) * loss_matrix[labels] # Calculate the total loss - loss = torch.sum(torch.maximum(l1 + l2, torch.zeros_like(l1).to(pred_sets.device)), dim=1) + loss = torch.sum(torch.maximum(l1 + l2, torch.zeros_like(l1, device=pred_sets.device)), dim=1) # Return the mean loss return torch.mean(loss) diff --git a/torchcp/classification/predictors/base.py b/torchcp/classification/predictors/base.py index ae954c0..378a1f7 100644 --- a/torchcp/classification/predictors/base.py +++ b/torchcp/classification/predictors/base.py @@ -17,16 +17,16 @@ class BasePredictor(object): """ - Abstract base class for all predictors classes. + Abstract base class for all conformal predictors. + + :param score_function: non-conformity score function. + :param model: a pytorch model. + :param temperature: the temperature of Temperature Scaling. """ __metaclass__ = ABCMeta def __init__(self, score_function, model=None, temperature=1): - """ - :param score_function: non-conformity score function. - :param model: a deep learning model. - """ self.score_function = score_function self._model = model @@ -47,7 +47,7 @@ def calibrate(self, cal_dataloader, alpha): @abstractmethod def predict(self, x_batch): """ - Generate prediction sets for test examples. + Generate prediction sets for the test examples. :param x_batch: a batch of input. """ diff --git a/torchcp/classification/predictors/classwise.py b/torchcp/classification/predictors/classwise.py index f2b9026..e415acf 100644 --- a/torchcp/classification/predictors/classwise.py +++ b/torchcp/classification/predictors/classwise.py @@ -14,6 +14,10 @@ class ClassWisePredictor(SplitPredictor): Applications of Class-Conditional Conformal Predictor in Multi-Class Classification (Shi et al., 2013) paper: https://ieeexplore.ieee.org/document/6784618 + + + :param score_function: non-conformity score function. + :param model: a pytorch model. """ def __init__(self, score_function, model=None): diff --git a/torchcp/classification/predictors/cluster.py b/torchcp/classification/predictors/cluster.py index df6b884..4d36daf 100644 --- a/torchcp/classification/predictors/cluster.py +++ b/torchcp/classification/predictors/cluster.py @@ -17,19 +17,17 @@ class ClusterPredictor(SplitPredictor): """ Class-Conditional Conformal Prediction with Many Classes (Ding et al., 2023). - paper: https://arxiv.org/abs/2306.09335 + paper: https://arxiv.org/abs/2306.09335. + + :param score_function: a non-conformity score function. + :param model: a pytorch model. + :param ratio_clustering: the ratio of examples in the calibration dataset used to cluster classes. + :param num_clusters: the number of clusters. If ratio_clustering is "auto", the number of clusters is automatically computed. + :param split: the method to split the dataset into clustering dataset and calibration set. Options are 'proportional' (sample proportional to distribution such that rarest class has n_clustering example), 'doubledip' (don't split and use all data for both steps, or 'random' (each example is assigned to clustering step with some fixed probability). """ def __init__(self, score_function, model=None, ratio_clustering="auto", num_clusters="auto", split='random', temperature=1): - """ - - :param score_function: score functions of CP - :param model: a deep learning model - :param ratio_clustering: The ratio of examples in the calibration dataset used to cluster classes - :param num_clusters: The number of clusters. If cluster_ratio is "auto", the number of clusters is automatically computed. - :param split: The method to split the dataset into clustering dataset and calibration set. split: How to split data between clustering step and calibration step. Options are 'proportional' (sample proportional to distribution such that rarest class has n_clustering example), 'doubledip' (don't split and use all data for both steps, or 'random' (each example is assigned to clustering step with some fixed probability) - """ super(ClusterPredictor, self).__init__(score_function, model, temperature) self.__ratio_clustering = ratio_clustering diff --git a/torchcp/classification/predictors/split.py b/torchcp/classification/predictors/split.py index 2ecd529..b273853 100644 --- a/torchcp/classification/predictors/split.py +++ b/torchcp/classification/predictors/split.py @@ -12,6 +12,14 @@ class SplitPredictor(BasePredictor): + """ + Split Conformal Prediction (Vovk et a., 2005). + Book: https://link.springer.com/book/10.1007/978-3-031-06649-8. + + :param score_function: non-conformity score function. + :param model: a pytorch model. + :param temperature: the temperature of Temperature Scaling. + """ def __init__(self, score_function, model=None, temperature=1): super().__init__(score_function, model, temperature) @@ -110,7 +118,6 @@ def evaluate(self, val_dataloader): labels_list.append(tmp_label) val_labels = torch.cat(labels_list) - res_dict = {} - res_dict["Coverage_rate"] = self._metric('coverage_rate')(prediction_sets, val_labels) - res_dict["Average_size"] = self._metric('average_size')(prediction_sets, val_labels) + res_dict = {"Coverage_rate": self._metric('coverage_rate')(prediction_sets, val_labels), + "Average_size": self._metric('average_size')(prediction_sets, val_labels)} return res_dict diff --git a/torchcp/classification/predictors/weight.py b/torchcp/classification/predictors/weight.py index 7a3c0f6..b628483 100644 --- a/torchcp/classification/predictors/weight.py +++ b/torchcp/classification/predictors/weight.py @@ -14,6 +14,12 @@ class WeightedPredictor(SplitPredictor): """ Conformal Prediction Under Covariate Shift (Tibshirani et al., 2019) paper : https://arxiv.org/abs/1904.06019 + + :param score_function: non-conformity score function. + :param model: a pytorch model. + :param image_encoder: a pytorch model to generate the embedding feature of a input image. + :param domain_classifier: a pytorch model (a binary classifier ) to predict the probability that a embedding feature comes from the source domain. + :param temperature: the temperature of Temperature Scaling. """ def __init__(self, score_function, model, image_encoder, domain_classifier=None, temperature=1): @@ -133,7 +139,6 @@ def evaluate(self, val_dataloader): labels_list.append(tmp_label) val_labels = torch.cat(labels_list) - res_dict = {} - res_dict["Coverage_rate"] = self._metric('coverage_rate')(prediction_sets, val_labels) - res_dict["Average_size"] = self._metric('average_size')(prediction_sets, val_labels) - return res_dict + result_dict = {"Coverage_rate": self._metric('coverage_rate')(prediction_sets, val_labels), + "Average_size": self._metric('average_size')(prediction_sets, val_labels)} + return result_dict diff --git a/torchcp/classification/scores/raps.py b/torchcp/classification/scores/raps.py index ee14501..814452a 100644 --- a/torchcp/classification/scores/raps.py +++ b/torchcp/classification/scores/raps.py @@ -17,14 +17,13 @@ class RAPS(APS): """ Regularized Adaptive Prediction Sets (Angelopoulos et al., 2020) paper : https://arxiv.org/abs/2009.14193 + + :param penalty: the weight of regularization. When penalty = 0, RAPS=APS. + :param kreg: the rank of regularization which is an integer in [0,labels_num]. """ def __init__(self, penalty, kreg=0): - """ - when penalty = 0, RAPS=APS. - - :param kreg : the rank of regularization which is an integer in [0,labels_num]. - """ + if penalty <= 0: raise ValueError("The parameter 'penalty' must be a positive value.") if kreg < 0: @@ -44,7 +43,7 @@ def _calculate_all_label(self, probs): _, sorted_indices = torch.sort(indices, descending=False, dim=-1) scores = ordered_scores.gather(dim=-1, index=sorted_indices) return scores - + def _calculate_single_label(self, probs, label): indices, ordered, cumsum = self._sort_sum(probs) U = torch.rand(indices.shape[0], device=probs.device) diff --git a/torchcp/classification/scores/saps.py b/torchcp/classification/scores/saps.py index 5983fbf..0b36eac 100644 --- a/torchcp/classification/scores/saps.py +++ b/torchcp/classification/scores/saps.py @@ -15,12 +15,12 @@ class SAPS(APS): """ Sorted Adaptive Prediction Sets (Huang et al., 2023) paper: https://arxiv.org/abs/2310.06430 + + :param weight: the weight of label ranking. """ def __init__(self, weight): - """ - :param weight: the weigth of label ranking. - """ + super(SAPS, self).__init__() if weight <= 0: raise ValueError("The parameter 'weight' must be a positive value.") diff --git a/torchcp/classification/scores/thr.py b/torchcp/classification/scores/thr.py index ae4a87f..705c3ee 100644 --- a/torchcp/classification/scores/thr.py +++ b/torchcp/classification/scores/thr.py @@ -11,14 +11,14 @@ class THR(BaseScore): """ - Threshold conformal predictors (Sadinle et al., 2016) - paper : https://arxiv.org/abs/1609.00451 + Threshold conformal predictors (Sadinle et al., 2016). + paper : https://arxiv.org/abs/1609.00451. + + :param score_type: a transformation on logits. Default: "softmax". Optional: "softmax", "Identity", "log_softmax" or "log". """ def __init__(self, score_type="softmax") -> None: - """ - param score_type: either "softmax" "Identity", "log_softmax" or "log". Default: "softmax". A transformation for logits. - """ + super().__init__() self.score_type = score_type if score_type == "Identity": @@ -28,7 +28,7 @@ def __init__(self, score_type="softmax") -> None: elif score_type == "log_softmax": self.transform = lambda x: torch.log_softmax(x, dim=-1) elif score_type == "log": - self.transform = lambda x: torch.log(x, dim=-1) + self.transform = lambda x: torch.log(x) else: raise NotImplementedError diff --git a/torchcp/regression/loss/quantile.py b/torchcp/regression/loss/quantile.py index 76bd2e4..31140a7 100644 --- a/torchcp/regression/loss/quantile.py +++ b/torchcp/regression/loss/quantile.py @@ -1,25 +1,30 @@ import torch import torch.nn as nn +__all__ = ["QuantileLoss"] + class QuantileLoss(nn.Module): - """ Pinball loss function + """ + Pinball loss function (Romano et al., 2019). + Paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf + + :param quantiles: a list of quantiles, such as :math: [alpha/2, 1-alpha/2]. """ def __init__(self, quantiles): """ - :param quantiles: quantile levels of predictions, each in the range (0,1) """ super().__init__() self.quantiles = quantiles def forward(self, preds, target): """ - Compute the pinball loss + Compute the pinball loss. - :param preds: the alpha/2 and 1-alpha/2 predictions of the model - :param target: the truth values + :param preds: the alpha/2 and 1-alpha/2 predictions of the model. The shape is batch x 2. + :param target: the truth values. The shape is batch x 1. """ assert not target.requires_grad if preds.size(0) != target.size(0): @@ -27,7 +32,7 @@ def forward(self, preds, target): losses = preds.new_zeros(len(self.quantiles)) for i, q in enumerate(self.quantiles): - errors = target - preds[:, i] - losses[i] = torch.sum(torch.max((q - 1) * errors, q * errors).unsqueeze(1)) + errors = target - preds[:, i:i + 1] + losses[i] = torch.sum(torch.max((q - 1) * errors, q * errors).squeeze(1)) loss = torch.mean(losses) return loss diff --git a/torchcp/regression/predictors/aci.py b/torchcp/regression/predictors/aci.py index c1ddf2e..df1b3ae 100644 --- a/torchcp/regression/predictors/aci.py +++ b/torchcp/regression/predictors/aci.py @@ -15,7 +15,8 @@ class ACI(SplitPredictor): Adaptive conformal inference (Gibbs et al., 2021) paper: https://arxiv.org/abs/2106.00170 - :param model: a deep learning model that can output alpha/2 and 1-alpha/2 quantile regression. + :param model: a pytorch model that can output the values of different quantiles. + :param gamma: a step size parameter. """ def __init__(self, model, gamma): diff --git a/torchcp/regression/predictors/cqr.py b/torchcp/regression/predictors/cqr.py index c7c7084..3b55d20 100644 --- a/torchcp/regression/predictors/cqr.py +++ b/torchcp/regression/predictors/cqr.py @@ -16,7 +16,7 @@ class CQR(SplitPredictor): Conformalized Quantile Regression (Romano et al., 2019) paper: https://arxiv.org/abs/1905.03222 - :param model: a deep learning model that can output alpha/2 and 1-alpha/2 quantile regression. + :param model: a pytorch model that can output alpha/2 and 1-alpha/2 quantile regression. """ def __init__(self, model): diff --git a/torchcp/regression/predictors/split.py b/torchcp/regression/predictors/split.py index 94b7a11..017181e 100644 --- a/torchcp/regression/predictors/split.py +++ b/torchcp/regression/predictors/split.py @@ -17,6 +17,8 @@ class SplitPredictor(object): """ Distribution-Free Predictive Inference For Regression (Lei et al., 2017) paper: https://arxiv.org/abs/1604.04173 + + :param model: a pytorch model for regression. """ def __init__(self, model):