ML Evaluation Metrics: The Complete Guide

import numpy as np
from sklearn.metrics import precision_recall_curve, roc_curve, f1_score

def find_optimal_threshold(y_true, y_proba, method="f1", beta=1.0,
                           tp_value=500.0, fp_cost=50.0):
    """
    Find operating threshold using different criteria.
    
    method options:
      "f1"       — maximize F-beta score
      "youden"   — maximize TPR-FPR (good for balanced)
      "ev"       — maximize expected value (use tp_value, fp_cost)
      "prec_at_recall" — max precision subject to recall >= target
    """
    thresholds = np.linspace(0, 1, 300)
    
    if method == "f1":
        scores = []
        for t in thresholds:
            y_pred = (y_proba >= t).astype(int)
            scores.append(f1_score(y_true, y_pred, beta=beta, zero_division=0))
        best_t = thresholds[np.argmax(scores)]
        
    elif method == "youden":
        fpr, tpr, roc_thresh = roc_curve(y_true, y_proba)
        j = tpr - fpr
        best_t = roc_thresh[np.argmax(j)]
        
    elif method == "ev":
        n_pos = y_true.sum()
        n_neg = len(y_true) - n_pos
        ev_scores = []
        for t in thresholds:
            y_pred = (y_proba >= t).astype(int)
            tp = ((y_pred == 1) & (y_true == 1)).sum()
            fp = ((y_pred == 1) & (y_true == 0)).sum()
            ev = tp * tp_value - fp * fp_cost
            ev_scores.append(ev)
        best_t = thresholds[np.argmax(ev_scores)]
        
    elif method == "prec_at_recall":
        # max precision where recall >= target
        target_recall = 0.8
        prec, rec, pr_thresh = precision_recall_curve(y_true, y_proba)
        valid = rec >= target_recall
        if valid.any():
            best_t = pr_thresh[valid[:-1].argmax()]  # highest precision in valid range
        else:
            best_t = 0.5
            
    return best_t

import numpy as np

def dcg_at_k(relevances: list, k: int) -> float:
    """
    Discounted Cumulative Gain at K.
    relevances: list of relevance scores (0, 1, 2, ...) in ranked order
    """
    relevances = np.array(relevances[:k], dtype=float)
    if len(relevances) == 0:
        return 0.0
    # Position 1-indexed: positions 1, 2, ..., k
    positions = np.arange(1, len(relevances) + 1)
    discounts = np.log2(positions + 1)  # log2(2)=1, log2(3)≈1.58, ...
    gains = (2 ** relevances - 1) / discounts
    return gains.sum()

def ndcg_at_k(relevances: list, k: int) -> float:
    """
    Normalized DCG at K: DCG / IDCG.
    IDCG = DCG of the ideal (perfectly sorted) ranking.
    """
    ideal = sorted(relevances, reverse=True)  # best possible ordering
    idcg = dcg_at_k(ideal, k)
    if idcg == 0:
        return 0.0
    return dcg_at_k(relevances, k) / idcg

# Example: search results with graded relevance
# 2=highly relevant, 1=somewhat, 0=not relevant
actual_order    = [2, 0, 1, 0, 2]  # what model returned
ideal_order     = [2, 2, 1, 0, 0]  # best possible order

print(f"NDCG@5: {ndcg_at_k(actual_order, k=5):.4f}")  # ≈ 0.891
print(f"NDCG@3: {ndcg_at_k(actual_order, k=3):.4f}")  # ≈ 0.796

def mean_reciprocal_rank(queries_results: list[list[bool]]) -> float:
    """MRR over a list of query results. Each inner list = [relevant?, ...] in rank order."""
    reciprocal_ranks = []
    for results in queries_results:
        for rank, is_relevant in enumerate(results, 1):
            if is_relevant:
                reciprocal_ranks.append(1.0 / rank)
                break
        else:
            reciprocal_ranks.append(0.0)  # no relevant result found
    return np.mean(reciprocal_ranks)

import numpy as np
from sklearn.calibration import calibration_curve, CalibratedClassifierCV
from sklearn.linear_model import LogisticRegression

def expected_calibration_error(y_true, y_proba, n_bins=10):
    """
    ECE: weighted average gap between predicted confidence and actual accuracy.
    Lower is better. ECE < 0.05 is generally considered well-calibrated.
    """
    bins = np.linspace(0, 1, n_bins + 1)
    ece = 0.0
    n = len(y_true)
    
    for i in range(n_bins):
        mask = (y_proba >= bins[i]) & (y_proba < bins[i + 1])
        if mask.sum() == 0:
            continue
        bin_conf = y_proba[mask].mean()       # mean predicted probability
        bin_acc  = y_true[mask].mean()         # actual fraction positive
        ece += (mask.sum() / n) * abs(bin_conf - bin_acc)
        
    return ece

def temperature_scaling(logits, y_val, n_iter=100, lr=0.01):
    """
    Temperature scaling: calibrate deep network by finding scalar T
    that minimizes NLL on a held-out validation set.
    
    Intuition: T > 1 → soften probabilities (overconfident model)
                T < 1 → sharpen probabilities (underconfident model)
    """
    T = 1.0
    for _ in range(n_iter):
        p = np.exp(logits / T) / np.exp(logits / T).sum(axis=1, keepdims=True)
        # gradient of NLL w.r.t. T
        nll_grad = -np.mean(
            (y_val - p) * logits / (T ** 2)
        )
        T -= lr * nll_grad
        T = max(0.1, T)  # prevent T from going negative
    return T

# Using sklearn's built-in calibration
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.calibration import CalibratedClassifierCV

base_model = GradientBoostingClassifier()
# method='sigmoid' = Platt scaling, method='isotonic' = isotonic regression
calibrated_model = CalibratedClassifierCV(base_model, method='sigmoid', cv=5)