calculate_spearman.py

# Copyright 2026 Fondazione Bruno Kessler
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#   https://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.



# Calculate Spearman rank correlations between GPT-4o and Gemini judges

def spearman_correlation(x, y):
    """Calculate Spearman rank correlation coefficient"""
    n = len(x)
    # Rank the data
    def rank_data(data):
        sorted_indices = sorted(range(len(data)), key=lambda i: data[i], reverse=True)
        ranks = [0] * len(data)
        for rank, idx in enumerate(sorted_indices, 1):
            ranks[idx] = rank
        return ranks
    
    ranks_x = rank_data(x)
    ranks_y = rank_data(y)
    
    # Calculate differences
    d_squared_sum = sum((rx - ry)**2 for rx, ry in zip(ranks_x, ranks_y))
    
    # Spearman correlation formula
    rho = 1 - (6 * d_squared_sum) / (n * (n**2 - 1))
    return rho

# GPT-4o Judge - Overall Mean Scores
gpt4o_overall = {
    'claude-sonnet-4.5': 0.869, 'multi-agent_claude-sonnet-4.5': 0.819,
    'multi-agent_gemma3:27b': 0.776, 'multi-agent_phi4': 0.787,
    'multi-agent_gpt-4o-mini': 0.739, 'gpt-4o-mini': 0.783,
    'gemma3:27b': 0.778, 'phi4': 0.800,
    'multi-agent_qwen2.5:72b': 0.728, 'multi-agent_gemma3:12b': 0.724,
    'multi-agent_qwq': 0.740, 'multi-agent_qwen2.5:32b': 0.747,
    'qwen2.5:72b': 0.752, 'llama3.3': 0.756,
    'gemma3:12b': 0.748, 'multi-agent_granite3.3': 0.742,
    'deepseek-r1:32b': 0.706, 'qwen2.5:32b': 0.736,
    'multi-agent_llama3.3': 0.707, 'granite3.3': 0.742,
    'qwq': 0.701, 'deepseek-r1:14b': 0.627,
    'multi-agent_qwen2.5:14b': 0.678, 'multi-agent_deepseek-r1:14b': 0.646,
    'qwen2.5:14b': 0.645, 'deepseek-r1:7b': 0.627,
    'multi-agent_deepseek-r1:32b': 0.621, 'deepseek-r1:8b': 0.618,
    'multi-agent_deepseek-r1:7b': 0.562, 'multi-agent_deepseek-r1:8b': 0.579,
    'multi-agent_llama3.2': 0.543, 'llama3.2': 0.516,
    'CiscoSec:8B': 0.479, 'llama3.2:1b': 0.457,
    'multi-agent_CiscoSec:8B': 0.454, 'multi-agent_llama3.2:1b': 0.399
}

# Gemini Judge - Overall Mean Scores
gemini_overall = {
    'claude-sonnet-4.5': 0.799, 'multi-agent_claude-sonnet-4.5': 0.830,
    'gemma3:12b': 0.727, 'gemma3:27b': 0.722,
    'gpt-4o-mini': 0.794, 'multi-agent_gemma3:12b': 0.694,
    'multi-agent_gemma3:27b': 0.778, 'multi-agent_gpt-4o-mini': 0.683,
    'multi-agent_phi4': 0.735, 'multi-agent_qwq': 0.682,
    'phi4': 0.711, 'qwq': 0.587,
    'granite3.3': 0.620, 'multi-agent_granite3.3': 0.622,
    'llama3.3': 0.540, 'multi-agent_deepseek-r1:14b': 0.504,
    'multi-agent_deepseek-r1:32b': 0.486, 'multi-agent_llama3.3': 0.534,
    'multi-agent_qwen2.5:14b': 0.520, 'multi-agent_qwen2.5:32b': 0.555,
    'multi-agent_qwen2.5:72b': 0.562, 'deepseek-r1:7b': 0.442,
    'qwen2.5:32b': 0.445, 'deepseek-r1:14b': 0.417,
    'deepseek-r1:32b': 0.431, 'deepseek-r1:8b': 0.385,
    'multi-agent_deepseek-r1:8b': 0.437, 'qwen2.5:14b': 0.426,
    'qwen2.5:72b': 0.427, 'llama3.2': 0.308,
    'multi-agent_llama3.2': 0.262, 'multi-agent_deepseek-r1:7b': 0.240,
    'CiscoSec:8B': 0.163, 'llama3.2:1b': 0.090,
    'multi-agent_CiscoSec:8B': 0.061, 'multi-agent_llama3.2:1b': 0.059
}

# GPT-4o Concordant
gpt4o_concordant = {
    'claude-sonnet-4.5': 0.968, 'multi-agent_claude-sonnet-4.5': 0.939,
    'multi-agent_gemma3:27b': 0.870, 'multi-agent_phi4': 0.866,
    'multi-agent_gpt-4o-mini': 0.858, 'gpt-4o-mini': 0.855,
    'multi-agent_gemma3:12b': 0.846, 'gemma3:27b': 0.837,
    'multi-agent_qwen2.5:72b': 0.838, 'phi4': 0.834,
    'multi-agent_qwq': 0.834, 'multi-agent_qwen2.5:32b': 0.832,
    'multi-agent_granite3.3': 0.829, 'gemma3:12b': 0.814,
    'qwen2.5:72b': 0.802, 'qwen2.5:32b': 0.781,
    'multi-agent_llama3.3': 0.810, 'llama3.3': 0.755,
    'deepseek-r1:32b': 0.717, 'multi-agent_qwen2.5:14b': 0.751,
    'multi-agent_deepseek-r1:14b': 0.751, 'qwq': 0.731,
    'granite3.3': 0.751, 'multi-agent_deepseek-r1:32b': 0.725,
    'deepseek-r1:7b': 0.671, 'qwen2.5:14b': 0.689,
    'deepseek-r1:14b': 0.612, 'deepseek-r1:8b': 0.654,
    'multi-agent_deepseek-r1:8b': 0.655, 'multi-agent_deepseek-r1:7b': 0.631,
    'multi-agent_llama3.2': 0.641, 'llama3.2': 0.584,
    'llama3.2:1b': 0.482, 'CiscoSec:8B': 0.443,
    'multi-agent_llama3.2:1b': 0.382, 'multi-agent_CiscoSec:8B': 0.424
}

# Gemini Concordant
gemini_concordant = {
    'claude-sonnet-4.5': 0.970, 'multi-agent_claude-sonnet-4.5': 0.972,
    'gemma3:12b': 0.815, 'gemma3:27b': 0.800,
    'gpt-4o-mini': 0.881, 'multi-agent_gemma3:12b': 0.874,
    'multi-agent_gemma3:27b': 0.879, 'multi-agent_gpt-4o-mini': 0.832,
    'multi-agent_granite3.3': 0.807, 'multi-agent_phi4': 0.872,
    'multi-agent_qwq': 0.834, 'phi4': 0.760,
    'qwq': 0.651, 'granite3.3': 0.641,
    'llama3.3': 0.577, 'multi-agent_deepseek-r1:14b': 0.655,
    'multi-agent_deepseek-r1:32b': 0.669, 'multi-agent_llama3.3': 0.662,
    'multi-agent_qwen2.5:14b': 0.627, 'multi-agent_qwen2.5:32b': 0.653,
    'multi-agent_qwen2.5:72b': 0.687, 'deepseek-r1:14b': 0.465,
    'deepseek-r1:32b': 0.482, 'deepseek-r1:7b': 0.505,
    'multi-agent_deepseek-r1:8b': 0.543, 'qwen2.5:32b': 0.501,
    'qwen2.5:14b': 0.473, 'deepseek-r1:8b': 0.446,
    'qwen2.5:72b': 0.481, 'llama3.2': 0.367,
    'multi-agent_llama3.2': 0.345, 'multi-agent_deepseek-r1:7b': 0.316,
    'CiscoSec:8B': 0.123, 'llama3.2:1b': 0.100,
    'multi-agent_CiscoSec:8B': 0.030, 'multi-agent_llama3.2:1b': 0.062
}

# GPT-4o Discordant
gpt4o_discordant = {
    'llama3.3': 0.758, 'phi4': 0.734,
    'deepseek-r1:32b': 0.697, 'granite3.3': 0.729,
    'qwq': 0.654, 'deepseek-r1:14b': 0.639,
    'claude-sonnet-4.5': 0.642, 'qwen2.5:72b': 0.632,
    'qwen2.5:32b': 0.607, 'gemma3:27b': 0.637,
    'multi-agent_claude-sonnet-4.5': 0.516, 'deepseek-r1:8b': 0.575,
    'deepseek-r1:7b': 0.550, 'multi-agent_qwen2.5:14b': 0.551,
    'gpt-4o-mini': 0.527, 'qwen2.5:14b': 0.538,
    'multi-agent_granite3.3': 0.542, 'gemma3:12b': 0.580,
    'multi-agent_phi4': 0.525, 'multi-agent_deepseek-r1:7b': 0.476,
    'multi-agent_gemma3:27b': 0.464, 'multi-agent_gemma3:12b': 0.457,
    'CiscoSec:8B': 0.502, 'multi-agent_llama3.3': 0.445,
    'multi-agent_qwq': 0.430, 'multi-agent_qwen2.5:32b': 0.451,
    'llama3.2': 0.411, 'multi-agent_deepseek-r1:32b': 0.447,
    'multi-agent_deepseek-r1:14b': 0.445, 'multi-agent_qwen2.5:72b': 0.383,
    'multi-agent_deepseek-r1:8b': 0.434, 'multi-agent_CiscoSec:8B': 0.481,
    'multi-agent_gpt-4o-mini': 0.387, 'multi-agent_llama3.2:1b': 0.409,
    'multi-agent_llama3.2': 0.337, 'llama3.2:1b': 0.428
}

# Gemini Discordant
gemini_discordant = {
    'granite3.3': 0.593, 'phi4': 0.618,
    'gemma3:12b': 0.504, 'gemma3:27b': 0.534,
    'qwq': 0.484, 'gpt-4o-mini': 0.491,
    'llama3.3': 0.457, 'multi-agent_claude-sonnet-4.5': 0.471,
    'deepseek-r1:14b': 0.380, 'deepseek-r1:32b': 0.389,
    'multi-agent_gemma3:27b': 0.443, 'claude-sonnet-4.5': 0.407,
    'qwen2.5:32b': 0.288, 'deepseek-r1:7b': 0.333,
    'deepseek-r1:8b': 0.311, 'multi-agent_deepseek-r1:8b': 0.235,
    'multi-agent_gemma3:12b': 0.300, 'multi-agent_gpt-4o-mini': 0.240,
    'multi-agent_llama3.3': 0.211, 'multi-agent_phi4': 0.283,
    'multi-agent_qwen2.5:14b': 0.333, 'multi-agent_qwen2.5:32b': 0.209,
    'qwen2.5:14b': 0.314, 'qwen2.5:72b': 0.297,
    'llama3.2': 0.218, 'multi-agent_deepseek-r1:14b': 0.215,
    'multi-agent_deepseek-r1:32b': 0.178, 'multi-agent_granite3.3': 0.197,
    'multi-agent_qwen2.5:72b': 0.171, 'multi-agent_qwq': 0.178,
    'CiscoSec:8B': 0.188, 'llama3.2:1b': 0.078,
    'multi-agent_CiscoSec:8B': 0.088, 'multi-agent_deepseek-r1:7b': 0.145,
    'multi-agent_llama3.2:1b': 0.057, 'multi-agent_llama3.2': 0.087
}

# Calculate correlations
print("=" * 60)
print("SPEARMAN RANK CORRELATIONS: GPT-4o vs Gemini 2.5 Flash")
print("=" * 60)

# 1. Overall
common = sorted(set(gpt4o_overall.keys()) & set(gemini_overall.keys()))
gpt4o_vals = [gpt4o_overall[m] for m in common]
gemini_vals = [gemini_overall[m] for m in common]
rho_overall = spearman_correlation(gpt4o_vals, gemini_vals)
print(f"\n1. Overall (All Cases)")
print(f"   Models: {len(common)}")
print(f"   Spearman ρ = {rho_overall:.3f}")

# 2. Concordant
common_conc = sorted(set(gpt4o_concordant.keys()) & set(gemini_concordant.keys()))
gpt4o_conc = [gpt4o_concordant[m] for m in common_conc]
gemini_conc = [gemini_concordant[m] for m in common_conc]
rho_concordant = spearman_correlation(gpt4o_conc, gemini_conc)
print(f"\n2. Concordant Cases Only")
print(f"   Models: {len(common_conc)}")
print(f"   Spearman ρ = {rho_concordant:.3f}")

# 3. Discordant
common_disc = sorted(set(gpt4o_discordant.keys()) & set(gemini_discordant.keys()))
gpt4o_disc = [gpt4o_discordant[m] for m in common_disc]
gemini_disc = [gemini_discordant[m] for m in common_disc]
rho_discordant = spearman_correlation(gpt4o_disc, gemini_disc)
print(f"\n3. Discordant Cases Only")
print(f"   Models: {len(common_disc)}")
print(f"   Spearman ρ = {rho_discordant:.3f}")

# 4. Single-Agent only
single_agent_models = [m for m in common if not m.startswith('multi-agent_')]
gpt4o_single = [gpt4o_overall[m] for m in single_agent_models]
gemini_single = [gemini_overall[m] for m in single_agent_models]
rho_single = spearman_correlation(gpt4o_single, gemini_single)
print(f"\n4. Single-Agent Models Only")
print(f"   Models: {len(single_agent_models)}")
print(f"   Spearman ρ = {rho_single:.3f}")

# 5. Multi-Agent only
multi_agent_models = [m for m in common if m.startswith('multi-agent_')]
gpt4o_multi = [gpt4o_overall[m] for m in multi_agent_models]
gemini_multi = [gemini_overall[m] for m in multi_agent_models]
rho_multi = spearman_correlation(gpt4o_multi, gemini_multi)
print(f"\n5. Multi-Agent Models Only")
print(f"   Models: {len(multi_agent_models)}")
print(f"   Spearman ρ = {rho_multi:.3f}")

print("\n" + "=" * 60)
print("SUMMARY TABLE FOR LATEX")
print("=" * 60)
print(f"Overall (All Cases)        & {rho_overall:.2f} \\\\")
print(f"Concordant Cases Only      & {rho_concordant:.2f} \\\\")
print(f"Discordant Cases Only      & {rho_discordant:.2f} \\\\")
print(f"Single-Agent Models        & {rho_single:.2f} \\\\")
print(f"Multi-Agent Models         & {rho_multi:.2f} \\\\")
print("=" * 60)