bradley_terry.py
bradley_terry.py
"""
Bradley-Terry Model Implementation
Official Chatbot Arena leaderboard calculation method
"""
import numpy as np
import pandas as pd
import math
from typing import Dict
from sklearn.linear_model import LogisticRegression
def compute_mle_elo(df: pd.DataFrame,
SCALE: int = 400,
BASE: int = 10,
INIT_RATING: int = 1000,
calibration_model: str = None,
calibration_rating: int = None) -> pd.Series:
"""
Compute Elo ratings using Bradley-Terry model with Maximum Likelihood Estimation.
This is the official method used by Chatbot Arena for their leaderboard.
It uses sklearn's LogisticRegression to fit a Bradley-Terry model.
Args:
df: DataFrame with columns 'model_a', 'model_b', 'winner'
SCALE: Elo scale parameter (default 400)
BASE: Base for logistic function (default 10)
INIT_RATING: Initial rating (default 1000)
calibration_model: Model name to calibrate ratings to
calibration_rating: Target rating for calibration model
Returns:
Series of Elo ratings indexed by model name
"""
# Create pivot tables for wins
ptbl_a_win = pd.pivot_table(
df[df["winner"] == "model_a"],
index="model_a",
columns="model_b",
aggfunc="size",
fill_value=0,
observed=False
)
# Handle ties (including "tie (bothbad)")
if sum(df["winner"].isin(["tie", "tie (bothbad)"])) == 0:
ptbl_tie = pd.DataFrame(0, index=ptbl_a_win.index, columns=ptbl_a_win.columns)
else:
ptbl_tie = pd.pivot_table(
df[df["winner"].isin(["tie", "tie (bothbad)"])],
index="model_a",
columns="model_b",
aggfunc="size",
fill_value=0,
observed=False
)
ptbl_tie = ptbl_tie + ptbl_tie.T
ptbl_b_win = pd.pivot_table(
df[df["winner"] == "model_b"],
index="model_a",
columns="model_b",
aggfunc="size",
fill_value=0,
observed=False
)
# Compute win matrix (A wins * 2 + B wins * 2 + ties)
ptbl_win = ptbl_a_win * 2 + ptbl_b_win.T * 2 + ptbl_tie
# Map models to indices
models = pd.Series(np.arange(len(ptbl_win.index)), index=ptbl_win.index)
p = len(models)
X = np.zeros([p * (p - 1) * 2, p])
Y = np.zeros(p * (p - 1) * 2)
cur_row = 0
sample_weights = []
for m_a in ptbl_win.index:
for m_b in ptbl_win.columns:
if m_a == m_b:
continue
# Skip if nan
if math.isnan(ptbl_win.loc[m_a, m_b]) or math.isnan(ptbl_win.loc[m_b, m_a]):
continue
X[cur_row, models[m_a]] = +math.log(BASE)
X[cur_row, models[m_b]] = -math.log(BASE)
Y[cur_row] = 1.0
sample_weights.append(ptbl_win.loc[m_a, m_b])
X[cur_row + 1, models[m_a]] = math.log(BASE)
X[cur_row + 1, models[m_b]] = -math.log(BASE)
Y[cur_row + 1] = 0.0
sample_weights.append(ptbl_win.loc[m_b, m_a])
cur_row += 2
X = X[:cur_row]
Y = Y[:cur_row]
# Fit logistic regression
lr = LogisticRegression(fit_intercept=False, penalty=None, tol=1e-6)
lr.fit(X, Y, sample_weight=sample_weights)
# Convert to Elo scores
elo_scores = SCALE * lr.coef_[0] + INIT_RATING
# Calibrate to reference model if provided
if calibration_model and calibration_model in models.index:
elo_scores += calibration_rating - elo_scores[models[calibration_model]]
return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
def predict_win_rate(elo_ratings: Dict[str, float],
SCALE: int = 400,
BASE: int = 10) -> pd.DataFrame:
"""
Predict win rates between all model pairs using Elo ratings.
Args:
elo_ratings: Dictionary of model names to Elo ratings
SCALE: Elo scale parameter
BASE: Base for logistic function
Returns:
DataFrame with predicted win rates (row vs column)
"""
from collections import defaultdict
names = sorted(list(elo_ratings.keys()))
wins = defaultdict(lambda: defaultdict(lambda: 0))
for a in names:
for b in names:
ea = 1 / (1 + BASE ** ((elo_ratings[b] - elo_ratings[a]) / SCALE))
wins[a][b] = ea
wins[b][a] = 1 - ea
data = {
a: [wins[a][b] if a != b else np.NAN for b in names]
for a in names
}
df = pd.DataFrame(data, index=names)
df.index.name = "model_a"
df.columns.name = "model_b"
return df.T
def get_bootstrap_result(battles: pd.DataFrame,
func_compute_elo,
num_round: int = 100) -> pd.DataFrame:
"""
Compute bootstrap confidence intervals for Elo ratings.
Args:
battles: DataFrame with battle data
func_compute_elo: Function to compute Elo ratings
num_round: Number of bootstrap rounds
Returns:
DataFrame with ratings from each bootstrap round
"""
from tqdm import tqdm
rows = []
for i in tqdm(range(num_round), desc="Bootstrap sampling"):
rows.append(func_compute_elo(battles.sample(frac=1.0, replace=True)))
df = pd.DataFrame(rows)
return df[df.median().sort_values(ascending=False).index]
def compute_bradley_terry_leaderboard(df: pd.DataFrame,
bootstrap_rounds: int = 0) -> pd.DataFrame:
"""
Compute leaderboard using Bradley-Terry model (official Chatbot Arena method).
Args:
df: DataFrame with columns 'model_a', 'model_b', 'winner'
bootstrap_rounds: Number of bootstrap rounds for confidence intervals (0 = no bootstrap)
Returns:
DataFrame with model ratings (and confidence intervals if bootstrap > 0)
"""
print(f"Computing Bradley-Terry model ratings...")
# Compute MLE Elo ratings
elo_ratings = compute_mle_elo(df)
if bootstrap_rounds > 0:
print(f"Computing {bootstrap_rounds} bootstrap samples for confidence intervals...")
bootstrap_df = get_bootstrap_result(df, compute_mle_elo, bootstrap_rounds)
# Compute confidence intervals
result = pd.DataFrame({
'rating': bootstrap_df.quantile(0.5),
'lower_ci': bootstrap_df.quantile(0.025),
'upper_ci': bootstrap_df.quantile(0.975)
}).sort_values('rating', ascending=False)
else:
result = pd.DataFrame({
'rating': elo_ratings
}).sort_values('rating', ascending=False)
result.index.name = 'model'
return result.reset_index()