信頼度校正ベンチマーク（シンライドコウセイベンチマーク）

def compute_ece(confidences, correctness, n_bins=15):
    bin_boundaries = np.linspace(0, 1, n_bins + 1)
    ece = 0.0
    for i in range(n_bins):
        mask = (confidences > bin_boundaries[i]) & (confidences <= bin_boundaries[i+1])
        if mask.sum() == 0:
            continue
        bin_acc = correctness[mask].mean()
        bin_conf = confidences[mask].mean()
        ece += (mask.sum() / len(confidences)) * abs(bin_acc - bin_conf)
    return ece

Brier Score = (1/N) Σ (p_i - y_i)^2

Brier Score = Calibration + Refinement + Uncertainty

import json
from dataclasses import dataclass

@dataclass
class CalibrationResult:
    model: str
    dataset: str
    ece: float
    ace: float
    brier: float
    auroc: float
    coverage_at_95: float
    n_samples: int

def run_calibration_benchmark(model_client, dataset, calibrator=None):
    confidences = []
    correctness = []
    for item in dataset:
        response = model_client.generate(item["question"])
        conf = model_client.get_confidence(item["question"], response)
        if calibrator:
            conf = calibrator.transform(conf)
        is_correct = evaluate_answer(response, item["answer"])
        confidences.append(conf)
        correctness.append(is_correct)

    confs = np.array(confidences)
    corrs = np.array(correctness)
    return CalibrationResult(
        model=model_client.name,
        dataset=dataset.name,
        ece=compute_ece(confs, corrs),
        ace=compute_ace(confs, corrs),
        brier=compute_brier(confs, corrs),
        auroc=compute_auroc(confs, corrs),
        coverage_at_95=compute_coverage_at_accuracy(confs, corrs, target_acc=0.95),
        n_samples=len(confidences)
    )

import matplotlib.pyplot as plt

def plot_reliability_diagram(confidences, accuracies, n_bins=10, title="Reliability Diagram"):
    bin_boundaries = np.linspace(0, 1, n_bins + 1)
    bin_accs = []
    bin_confs = []
    bin_sizes = []
    for i in range(n_bins):
        mask = (confidences > bin_boundaries[i]) & (confidences <= bin_boundaries[i+1])
        if mask.sum() > 0:
            bin_accs.append(accuracies[mask].mean())
            bin_confs.append(confidences[mask].mean())
            bin_sizes.append(mask.sum())
        else:
            bin_accs.append(0)
            bin_confs.append((bin_boundaries[i] + bin_boundaries[i+1]) / 2)
            bin_sizes.append(0)
    fig, ax = plt.subplots(1, 1, figsize=(8, 6))
    ax.bar(bin_confs, bin_accs, width=1.0/n_bins, alpha=0.7, label="Model")
    ax.plot([0, 1], [0, 1], "r--", label="Perfect Calibration")
    ax.set_xlabel("Confidence")
    ax.set_ylabel("Accuracy")
    ax.set_title(title)
    ax.legend()
    return fig