Skip to content

[CI] Parity: carry commit SHA labels through log + running-time parsing #3362

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ethanwee1 wants to merge 2 commits into
base: develop
Choose a base branch
from
+74 −34
Open

[CI] Parity: carry commit SHA labels through log + running-time parsing #3362

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a3d95bd
parity: carry commit SHA labels through log + running-time parsing
ethanwee1 Jun 23, 2026
bfaa5c6
summarize_xml: make printed summary labels follow set1/set2 naming
ethanwee1 Jun 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions .automation_scripts/pytorch-unit-test-scripts/detect_log_failures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,23 @@


def classify_log_file(filename):
"""Return (platform, test_config, shard_num) from a log filename like rocm3.txt."""
"""Return (platform, test_config, shard_num) from a log filename like rocm3.txt.

Commit-vs-commit parity prefixes log files with the short commit SHA
(for example, 09e0c59b_rocm3.txt). In that mode the SHA label is the
platform name used by generate_summary.py, so preserve it here.
"""
stem = Path(filename).stem
label = None
m = re.match(r"(?P<label>[0-9a-f]{8,40})_(?P<stem>.+)", stem)
if m:
label = m.group("label")[:8]
stem = m.group("stem")
for prefix, (platform, test_config) in sorted(LOG_FILE_MAP.items(), key=lambda x: -len(x[0])):
if stem.startswith(prefix):
remainder = stem[len(prefix):]
if remainder.isdigit():
return platform, test_config, int(remainder)
return label or platform, test_config, int(remainder)
return None, None, None


Expand Down
94 changes: 62 additions & 32 deletions .automation_scripts/pytorch-unit-test-scripts/summarize_xml_testreports.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,10 +318,15 @@ def summarize_xml_files(args):
# test file level running time: ROCm and CUDA
test_file_level_ROCm: Dict[Tuple[str], float] = {}
test_file_level_CUDA: Dict[Tuple[str], float] = {}
test_file_shards_ROCm: Dict[Tuple[str], set] = {}
test_file_shards_CUDA: Dict[Tuple[str], set] = {}
for (k,v) in list(test_cases_set1_running_time.items()):
test_file_name = k[0]
test_config_name = k[2]
tar_tup_rocm = (test_file_name, test_config_name,)
test_file_shards_ROCm.setdefault(tar_tup_rocm, set())
if v.get("shard"):
test_file_shards_ROCm[tar_tup_rocm].add(v["shard"])
if test_file_level_ROCm.get(tar_tup_rocm) == None:
test_file_level_ROCm[ ( test_file_name, test_config_name ) ] = v["running_time_xml"]
else:
Expand All @@ -330,6 +335,9 @@ def summarize_xml_files(args):
test_file_name = k[0]
test_config_name = k[2]
tar_tup_cuda = (test_file_name, test_config_name)
test_file_shards_CUDA.setdefault(tar_tup_cuda, set())
if v.get("shard"):
test_file_shards_CUDA[tar_tup_cuda].add(v["shard"])
if test_file_level_CUDA.get(tar_tup_cuda) == None:
test_file_level_CUDA[ ( test_file_name, test_config_name ) ] = v["running_time_xml"]
else:
Expand Down Expand Up @@ -615,43 +623,56 @@ def sorting_key(e):

# write test file running time to file
test_file_running_time_for_csv = {}
set1_running_time_col = f"{set1_name}_running_time"
set2_running_time_col = f"{set2_name}_running_time"
set1_tests_run_col = f"{set1_name}_tests_run"
set2_tests_run_col = f"{set2_name}_tests_run"
set1_test_shards_col = f"{set1_name}_test_shards"
set2_test_shards_col = f"{set2_name}_test_shards"
set1_passed_col = f"{set1_name}_passed"
set1_skipped_col = f"{set1_name}_skipped"
set1_missed_col = f"{set1_name}_missed"

@jithunnair-amd jithunnair-amd Jun 24, 2026 •
edited
Loading

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know how far you want to take this "secularizing" of set1/set2 naming instead of ROCm/CUDA, but there are still many occurrences in the file that refer to ROCm/CUDA. Just something to keep in mind if you really want to make everything consistent. Not everything in this PR probably.

@ethanwee1 ethanwee1 Jun 24, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed — I've kept this PR scoped to the user-facing output: the running-time CSV columns and the printed summary now both follow set1/set2. The remaining ROCm/CUDA occurrences are internal variable names (test_file_level_ROCm, *_DEFAULT counters, etc.); I left those untouched to keep the diff focused, and we can do a broader internal rename as a separate cleanup if you'd prefer.

for key_rocm in test_file_level_ROCm.keys():
item_values = {}
item_values["test_file"] = key_rocm[0]
item_values["test_config"] = key_rocm[1]
item_values["rocm_running_time"] = test_file_level_ROCm[key_rocm]
item_values["cuda_running_time"] = 0.0
item_values[set1_running_time_col] = test_file_level_ROCm[key_rocm]
item_values[set2_running_time_col] = 0.0
if key_rocm in test_file_level_CUDA.keys():
item_values["cuda_running_time"] = test_file_level_CUDA[key_rocm]
item_values["abs_time_diff"] = item_values["rocm_running_time"] - item_values["cuda_running_time"]
item_values[set2_running_time_col] = test_file_level_CUDA[key_rocm]
item_values["abs_time_diff"] = item_values[set1_running_time_col] - item_values[set2_running_time_col]
item_values["relative_time_diff"] = 0.0
if item_values["cuda_running_time"] != 0.0:
item_values["relative_time_diff"] = 100 * (item_values["rocm_running_time"] - item_values["cuda_running_time"]) / item_values["cuda_running_time"]
if item_values[set2_running_time_col] != 0.0:
item_values["relative_time_diff"] = 100 * (item_values[set1_running_time_col] - item_values[set2_running_time_col]) / item_values[set2_running_time_col]
# Add test counts
item_values["rocm_tests_run"] = test_file_counts_ROCm.get(key_rocm, {}).get('tests_run', 0)
item_values["cuda_tests_run"] = test_file_counts_CUDA.get(key_rocm, 0)
item_values["rocm_passed"] = test_file_counts_ROCm.get(key_rocm, {}).get('passed', 0)
item_values["rocm_skipped"] = test_file_counts_ROCm.get(key_rocm, {}).get('skipped', 0)
item_values["rocm_missed"] = test_file_counts_ROCm.get(key_rocm, {}).get('missed', 0)
item_values[set1_tests_run_col] = test_file_counts_ROCm.get(key_rocm, {}).get('tests_run', 0)
item_values[set2_tests_run_col] = test_file_counts_CUDA.get(key_rocm, 0)
item_values[set1_test_shards_col] = len(test_file_shards_ROCm.get(key_rocm, set()))
item_values[set2_test_shards_col] = len(test_file_shards_CUDA.get(key_rocm, set()))
item_values[set1_passed_col] = test_file_counts_ROCm.get(key_rocm, {}).get('passed', 0)
item_values[set1_skipped_col] = test_file_counts_ROCm.get(key_rocm, {}).get('skipped', 0)
item_values[set1_missed_col] = test_file_counts_ROCm.get(key_rocm, {}).get('missed', 0)
test_file_running_time_for_csv[key_rocm] = item_values

for key_cuda in test_file_level_CUDA.keys():
if not key_cuda in test_file_level_ROCm.keys():
item_values = {}
item_values["test_file"] = key_cuda[0]
item_values["test_config"] = key_cuda[1]
item_values["rocm_running_time"] = 0.0
item_values["cuda_running_time"] = test_file_level_CUDA[key_cuda]
item_values["abs_time_diff"] = item_values["rocm_running_time"] - item_values["cuda_running_time"]
item_values[set1_running_time_col] = 0.0
item_values[set2_running_time_col] = test_file_level_CUDA[key_cuda]
item_values["abs_time_diff"] = item_values[set1_running_time_col] - item_values[set2_running_time_col]
item_values["relative_time_diff"] = 0.0
if item_values["cuda_running_time"] != 0.0:
item_values["relative_time_diff"] = 100 * (item_values["rocm_running_time"] - item_values["cuda_running_time"]) / item_values["cuda_running_time"]
if item_values[set2_running_time_col] != 0.0:
item_values["relative_time_diff"] = 100 * (item_values[set1_running_time_col] - item_values[set2_running_time_col]) / item_values[set2_running_time_col]
# Add test counts
item_values["rocm_tests_run"] = test_file_counts_ROCm.get(key_cuda, {}).get('tests_run', 0)
item_values["cuda_tests_run"] = test_file_counts_CUDA.get(key_cuda, 0)
item_values["rocm_passed"] = test_file_counts_ROCm.get(key_cuda, {}).get('passed', 0)
item_values["rocm_skipped"] = test_file_counts_ROCm.get(key_cuda, {}).get('skipped', 0)
item_values["rocm_missed"] = test_file_counts_ROCm.get(key_cuda, {}).get('missed', 0)
item_values[set1_tests_run_col] = test_file_counts_ROCm.get(key_cuda, {}).get('tests_run', 0)
item_values[set2_tests_run_col] = test_file_counts_CUDA.get(key_cuda, 0)
item_values[set1_test_shards_col] = len(test_file_shards_ROCm.get(key_cuda, set()))
item_values[set2_test_shards_col] = len(test_file_shards_CUDA.get(key_cuda, set()))
item_values[set1_passed_col] = test_file_counts_ROCm.get(key_cuda, {}).get('passed', 0)
item_values[set1_skipped_col] = test_file_counts_ROCm.get(key_cuda, {}).get('skipped', 0)
item_values[set1_missed_col] = test_file_counts_ROCm.get(key_cuda, {}).get('missed', 0)
test_file_running_time_for_csv[key_cuda] = item_values

test_file_running_time_for_csv = dict(sorted(test_file_running_time_for_csv.items()))
Expand All @@ -661,24 +682,28 @@ def sorting_key_running_time(e):
return 0
elif e == "test_config":
return 1
elif e == "rocm_running_time":
elif e == set1_running_time_col:
return 2
elif e == "cuda_running_time":
elif e == set2_running_time_col:
return 3
elif e == "abs_time_diff":
return 4
elif e == "relative_time_diff":
return 5
elif e == "rocm_tests_run":
elif e == set1_tests_run_col:
return 6
elif e == "cuda_tests_run":
elif e == set2_tests_run_col:
return 7
elif e == "rocm_passed":
elif e == set1_test_shards_col:
return 8
elif e == "rocm_skipped":
elif e == set2_test_shards_col:
return 9
elif e == "rocm_missed":
elif e == set1_passed_col:
return 10
elif e == set1_skipped_col:
return 11
elif e == set1_missed_col:
return 12
else:
return 100

Expand All @@ -690,20 +715,25 @@ def sorting_key_running_time(e):
writer.writerows(test_file_running_time_for_csv.values())

# print summary
# User-facing labels follow the same set1/set2 naming as the CSV columns
# (set1_name/set2_name default to rocm/cuda, or are commit SHAs in
# commit-vs-commit mode) so the printed summary stays consistent with them.
set1_disp = set1_name.upper()
set2_disp = set2_name.upper()
print( " " )
print( "_____________________________________" )
print( "Test-results" )
print( " " )
print( "=====Single GPU Number=====" )
print( "SKIPPED_DEFAULT, MISSED_DEFAULT, ROCMONLY_DEFAULT, CUDA_DEFAULT, ROCM_DEFAULT" )
print( f"SKIPPED_DEFAULT, MISSED_DEFAULT, {set1_disp}ONLY_DEFAULT, {set2_disp}_DEFAULT, {set1_disp}_DEFAULT" )
print( str(SKIPPED_DEFAULT) + ", " + str(MISSED_DEFAULT) + ", " + str(ROCMONLY_DEFAULT) + ", " + str(CUDA_DEFAULT) + ", " + str(ROCM_DEFAULT) )
print( " " )
print( "=====Distributed GPU Number=====" )
print( "SKIPPED_DISTRIBUTED, MISSED_DISTRIBUTED, ROCMONLY_DISTRIBUTED, CUDA_DISTRIBUTED, ROCM_DISTRIBUTED" )
print( f"SKIPPED_DISTRIBUTED, MISSED_DISTRIBUTED, {set1_disp}ONLY_DISTRIBUTED, {set2_disp}_DISTRIBUTED, {set1_disp}_DISTRIBUTED" )
print( str(SKIPPED_DISTRIBUTED) + ", " + str(MISSED_DISTRIBUTED) + ", " + str(ROCMONLY_DISTRIBUTED) + ", " + str(CUDA_DISTRIBUTED) + ", " + str(ROCM_DISTRIBUTED) )
print( " " )
print( "=====Inductor GPU Number=====" )
print( "SKIPPED_INDUCTOR, MISSED_INDUCTOR, ROCMONLY_INDUCTOR, CUDA_INDUCTOR, ROCM_INDUCTOR" )
print( f"SKIPPED_INDUCTOR, MISSED_INDUCTOR, {set1_disp}ONLY_INDUCTOR, {set2_disp}_INDUCTOR, {set1_disp}_INDUCTOR" )
print( str(SKIPPED_INDUCTOR) + ", " + str(MISSED_INDUCTOR) + ", " + str(ROCMONLY_INDUCTOR) + ", " + str(CUDA_INDUCTOR) + ", " + str(ROCM_INDUCTOR) )
print( " " )
print( "SELECTED CAUSES SUMMARY" )
Expand All @@ -728,7 +758,7 @@ def sorting_key_running_time(e):
print( " " )
print( "=====================" )
print( "Time statistics" )
print( "ROCM_RUNNING_TIME, CUDA_RUNNING_TIME" )
print( f"{set1_disp}_RUNNING_TIME, {set2_disp}_RUNNING_TIME" )
print( str(TOTAL_ROCM_RUNNING_TIME) + ", " + str(TOTAL_CUDA_RUNNING_TIME) )
#print( "ROCm test file level time statistics" )
#for (k,v) in list(test_file_level_ROCm.items()):
Expand Down