Skip to content
Snippets Groups Projects
experiments.py 2.69 KiB
Newer Older
import sys
import torch
import numpy as np
from typing import Tuple, List

from src.detectors import Detector, CorrelatedDetector, YOLOv8
Bruce Cyusa Mukama's avatar
Bruce Cyusa Mukama committed
from src.data import Dataset, FilteredKitti, Split, split_dataset
from src.reporting import UQEvalResults
from src.conformalizers import (
    Conformalizer,
    BonferroniAdditiveConformalizer,
    MaxAdditiveConformalizer,
    ImprovedIndependentConformalizer,
    ImprovedGumbelConformalizer,
    ImprovedEmpiricalConformalizer,
)


VERSION = "___release"
EPSILONS = np.array([0.01, 0.05, 0.1, 0.2]) #, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])


def setup_experiment(epsilons: np.ndarray = EPSILONS) -> Tuple:
Bruce Cyusa Mukama's avatar
Bruce Cyusa Mukama committed
    UQers = [
        BonferroniAdditiveConformalizer(),
        MaxAdditiveConformalizer(),
        ImprovedIndependentConformalizer(),
        ImprovedGumbelConformalizer(),
        ImprovedEmpiricalConformalizer(),
    ]
    parameters = [
        {'epsilon': eps,
         'k_folds': 10,
         'cal_ratio': 0.15,
         'val_size': 100,
         'iou_threshold': 0.3,
Bruce Cyusa Mukama's avatar
Bruce Cyusa Mukama committed
         'confidence_threshold': 0.3,
         'epochs': 100,
         'from_scratch': False,
         'device': torch.device('cuda:0')
         } for eps in epsilons
    ]
Bruce Cyusa Mukama's avatar
Bruce Cyusa Mukama committed
    dataset = FilteredKitti('datasets', download=True)
    return UQers, parameters, dataset


def run_experiment(UQers: List[Conformalizer], parameters: List,
                   split: Split, detector: Detector,
                   res: UQEvalResults, nn_perfs: str) -> UQEvalResults:
    for conformalizer in UQers:
        for params in parameters:
            conformalizer.set_parameters(**params)
            cal_res = conformalizer.calibrate(split.cal_pairs)
            test_res = conformalizer.conformalize(split.test_pairs)
            res.add_results(
                split, conformalizer, detector,
                params, nn_perfs, cal_res, test_res, 
                conformalizer.dissimilarities)
def experiment_with_detector(detector: Detector, res: UQEvalResults) -> None:
    UQers, parameters, dataset = setup_experiment()
    splits = split_dataset(dataset, **parameters[0])
    for split in splits:
        nn_perfs = detector.train(split, **parameters[0])
        split.gen_data_pairs(detector)
        run_experiment(UQers, parameters, split,
                       detector, res, nn_perfs)


if __name__ == "__main__":
    if not sys.warnoptions:
        import warnings
        warnings.simplefilter("ignore")
    detectors = [
        CorrelatedDetector(),
        CorrelatedDetector(cov_matrix=None),
        YOLOv8()
    res = UQEvalResults()
    label = f'UQ_experiments{VERSION}'
    for detector in detectors:
        experiment_with_detector(detector, res)
    res.save(dest_folder='results', label=label, time=False)