diff --git a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py index 5c78317197b43..2ae893cf82333 100644 --- a/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py +++ b/.automation_scripts/pytorch-unit-test-scripts/generate_summary.py @@ -114,10 +114,10 @@ def test_config_stats_keys(s1_name, s2_name, has_set2=True): f'TOTAL {s1}', ] return [ - f'SKIPPED (on {s1_name}, but not on {s2_name})', + f'SKIPPED (on {s1_name}, PASSED on {s2_name})', f'SKIPPED (on {s1_name})', f'SKIPPED (on {s2_name})', - f'MISSED (MISSED on {s1_name}, NOT SKIPPED on {s2_name})', + f'MISSED (on {s1_name}, PASSED on {s2_name})', f'{s1}ONLY (PASSED on {s1}, NOT PASSED on {s2})', s2, s1, @@ -141,15 +141,18 @@ def compute_test_config_stats(rows, s1_col, s2_col, s1_name, s2_name, has_set2=T vals[keys[4]] = sum(1 for r in rows if r[s1_col].strip()) return vals + # Count a ROCm SKIPPED/MISSED test as a disagreement only when CUDA actually + # PASSED it. This excludes parametrization variants CUDA never runs (CUDA + # MISSED) or also skips, which aren't real ROCm-vs-CUDA coverage gaps. s1_skip_not_s2 = sum( 1 for r in rows - if r[s1_col] == 'SKIPPED' and r[s2_col] != 'SKIPPED' + if r[s1_col] == 'SKIPPED' and r[s2_col] == 'PASSED' ) s1_skip = sum(1 for r in rows if r[s1_col] == 'SKIPPED') s2_skip = sum(1 for r in rows if r[s2_col] == 'SKIPPED') s1_miss_not_s2_skip = sum( 1 for r in rows - if r[s1_col] == 'MISSED' and r[s2_col] != 'SKIPPED' + if r[s1_col] == 'MISSED' and r[s2_col] == 'PASSED' ) only_s1 = sum( 1 for r in rows @@ -233,11 +236,11 @@ def safe_float(v): wf_rows = [r for r in rows if r['test_config'] == wf] s1_skip_not_s2 = sum( 1 for r in wf_rows - if r[s1_col] == 'SKIPPED' and r[s2_col] != 'SKIPPED' + if r[s1_col] == 'SKIPPED' and r[s2_col] == 'PASSED' ) s1_miss_not_s2_skip = sum( 1 for r in wf_rows - if r[s1_col] == 'MISSED' and r[s2_col] != 'SKIPPED' + if r[s1_col] == 'MISSED' and r[s2_col] == 'PASSED' ) total_disagree += s1_skip_not_s2 + s1_miss_not_s2_skip total_s2 += sum(1 for r in wf_rows if r[s2_col].strip() and r[s2_col].strip() != 'MISSED')