Newer
Older
import torch
import numpy as np
from typing import Tuple, List
from src.detectors import Detector, CorrelatedDetector, YOLOv8
from src.data import Dataset, FilteredKitti, Split, split_dataset
from src.reporting import UQEvalResults
from src.conformalizers import (
Conformalizer,
BonferroniAdditiveConformalizer,
MaxAdditiveConformalizer,
ImprovedIndependentConformalizer,
ImprovedGumbelConformalizer,
ImprovedEmpiricalConformalizer,
)
EPSILONS = np.array([0.01, 0.05, 0.1, 0.2]) #, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])
def setup_experiment(epsilons: np.ndarray = EPSILONS) -> Tuple:
BonferroniAdditiveConformalizer(),
MaxAdditiveConformalizer(),
ImprovedIndependentConformalizer(),
ImprovedGumbelConformalizer(),
ImprovedEmpiricalConformalizer(),
]
parameters = [
{'epsilon': eps,
'val_size': 100,
'iou_threshold': 0.3,
'from_scratch': False,
'device': torch.device('cuda:0')
} for eps in epsilons
]
dataset = FilteredKitti('datasets', download=True)
return UQers, parameters, dataset
def run_experiment(UQers: List[Conformalizer], parameters: List,
split: Split, detector: Detector,
res: UQEvalResults, nn_perfs: str) -> UQEvalResults:
for conformalizer in UQers:
for params in parameters:
conformalizer.set_parameters(**params)
cal_res = conformalizer.calibrate(split.cal_pairs)
test_res = conformalizer.conformalize(split.test_pairs)
params, nn_perfs, cal_res, test_res,
conformalizer.dissimilarities)
def experiment_with_detector(detector: Detector, res: UQEvalResults) -> None:
UQers, parameters, dataset = setup_experiment()
splits = split_dataset(dataset, **parameters[0])
for split in splits:
nn_perfs = detector.train(split, **parameters[0])
split.gen_data_pairs(detector)
run_experiment(UQers, parameters, split,
if not sys.warnoptions:
import warnings
warnings.simplefilter("ignore")
CorrelatedDetector(cov_matrix=None),
YOLOv8()
res = UQEvalResults()
label = f'UQ_experiments{VERSION}'
for detector in detectors:
experiment_with_detector(detector, res)
res.save(dest_folder='results', label=label, time=False)