experiments.py

import sys
import torch
import numpy as np
from typing import Tuple, List

from src.detectors import Detector, CorrelatedDetector, YOLOv8
from src.data import Dataset, FilteredKitti, Split, split_dataset
from src.reporting import UQEvalResults
from src.conformalizers import (
    Conformalizer,
    BonferroniAdditiveConformalizer,
    MaxAdditiveConformalizer,
    ImprovedIndependentConformalizer,
    ImprovedGumbelConformalizer,
    ImprovedEmpiricalConformalizer,
)


VERSION = "___release"
EPSILONS = np.array([0.01, 0.05, 0.1, 0.2]) #, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])


def setup_experiment(epsilons: np.ndarray = EPSILONS) -> Tuple:
    UQers = [
        BonferroniAdditiveConformalizer(),
        MaxAdditiveConformalizer(),
        ImprovedIndependentConformalizer(),
        ImprovedGumbelConformalizer(),
        ImprovedEmpiricalConformalizer(),
    ]
    parameters = [
        {'epsilon': eps,
         'k_folds': 10,
         'cal_ratio': 0.15,
         'val_size': 100,
         'iou_threshold': 0.3,
         'confidence_threshold': 0.3,
         'epochs': 100,
         'from_scratch': False,
         'device': torch.device('cuda:0')
         } for eps in epsilons
    ]
    dataset = FilteredKitti('datasets', download=True)
    return UQers, parameters, dataset


def run_experiment(UQers: List[Conformalizer], parameters: List,
                   split: Split, detector: Detector,
                   res: UQEvalResults, nn_perfs: str) -> UQEvalResults:
    for conformalizer in UQers:
        for params in parameters:
            conformalizer.set_parameters(**params)
            cal_res = conformalizer.calibrate(split.cal_pairs)
            test_res = conformalizer.conformalize(split.test_pairs)
            res.add_results(
                split, conformalizer, detector,
                params, nn_perfs, cal_res, test_res, 
                conformalizer.dissimilarities)


def experiment_with_detector(detector: Detector, res: UQEvalResults) -> None:
    UQers, parameters, dataset = setup_experiment()
    splits = split_dataset(dataset, **parameters[0])
    for split in splits:
        nn_perfs = detector.train(split, **parameters[0])
        split.gen_data_pairs(detector)
        run_experiment(UQers, parameters, split,
                       detector, res, nn_perfs)


if __name__ == "__main__":
    if not sys.warnoptions:
        import warnings
        warnings.simplefilter("ignore")
    detectors = [
        CorrelatedDetector(),
        CorrelatedDetector(cov_matrix=None),
        YOLOv8()
    ]
    res = UQEvalResults()
    label = f'UQ_experiments{VERSION}'
    for detector in detectors:
        experiment_with_detector(detector, res)
    res.save(dest_folder='results', label=label, time=False)