diff --git a/src/helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py b/src/helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py index acbc80b277..a5cdb1e2d2 100644 --- a/src/helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py +++ b/src/helm/benchmark/metrics/codeinsights_code_evaluation_metrics.py @@ -88,7 +88,11 @@ def calculate_ast_distance(self, code1: str, code2: str) -> float: nodes1: List[str] = self._extract_ast_features(tu1.cursor) nodes2: List[str] = self._extract_ast_features(tu2.cursor) - return levenshtein_distance_ratio(nodes1, nodes2) + # `Levenshtein.ratio` is a similarity in [0, 1] (1.0 = identical), + # while the docstring, the public stat name "ast_distance", and the + # error-fallback `return 1.0` below all assume *distance* semantics + # (0 = identical, 1 = completely different). Convert to a distance. + return 1.0 - levenshtein_distance_ratio(nodes1, nodes2) except Exception: # any parse error or clang error → max distance @@ -232,7 +236,10 @@ def evaluate_generation( if gen_asm == "" or truth_asm == "": asm_distance = 1.0 else: - asm_distance = levenshtein_distance_ratio(gen_asm, truth_asm) + # `Levenshtein.ratio` returns a similarity in [0, 1]; convert to + # distance so the value matches the "asm_distance" stat name and + # the compile-failure fallback above (1.0 = max distance). + asm_distance = 1.0 - levenshtein_distance_ratio(gen_asm, truth_asm) # Create assembly-based statistics stats.extend(self._create_asm_stats(asm_distance)) diff --git a/src/helm/benchmark/metrics/test_codeinsights_code_evaluation_metrics.py b/src/helm/benchmark/metrics/test_codeinsights_code_evaluation_metrics.py new file mode 100644 index 0000000000..7f3e6299c9 --- /dev/null +++ b/src/helm/benchmark/metrics/test_codeinsights_code_evaluation_metrics.py @@ -0,0 +1,64 @@ +"""Tests for codeinsights_code_evaluation_metrics distance semantics. + +`Levenshtein.ratio` returns a *similarity* in [0, 1] (1.0 = identical), but the +metric is reported as ``ast_distance`` / ``asm_distance`` and its error +fallback returns 1.0 for "max distance". Computing similarity directly meant a +perfect match was reported as 1.0 — indistinguishable from a parse failure. +These tests pin the distance semantics: identical inputs return 0.0 and a +totally-different input returns 1.0. +""" + +import pytest + + +# The module imports clang.cindex and Levenshtein at top level (both ship in +# the codeinsights extras). Skip cleanly when they're absent. +clang_cindex = pytest.importorskip("clang.cindex") +pytest.importorskip("Levenshtein") + + +def _patch_clang_index(monkeypatch): + """Replace clang.cindex.Index.create so ASTAnalyzer.__init__ doesn't need + libclang installed; ``parse`` round-trips the source text through .cursor + so the stubbed _extract_ast_features can read it back.""" + + class _DummyTU: + def __init__(self, code: str) -> None: + self.cursor = code + + class _DummyIndex: + def parse(self, path, args, unsaved_files): + return _DummyTU(unsaved_files[0][1]) + + monkeypatch.setattr(clang_cindex.Index, "create", staticmethod(lambda: _DummyIndex())) + + +def test_calculate_ast_distance_identical_returns_zero(monkeypatch): + _patch_clang_index(monkeypatch) + from helm.benchmark.metrics.codeinsights_code_evaluation_metrics import ASTAnalyzer + + analyzer = ASTAnalyzer() + analyzer._extract_ast_features = lambda cursor: list(cursor) + + assert analyzer.calculate_ast_distance("int x = 1;", "int x = 1;") == 0.0 + + +def test_calculate_ast_distance_completely_different_returns_one(monkeypatch): + _patch_clang_index(monkeypatch) + from helm.benchmark.metrics.codeinsights_code_evaluation_metrics import ASTAnalyzer + + analyzer = ASTAnalyzer() + analyzer._extract_ast_features = lambda cursor: list(cursor) + + assert analyzer.calculate_ast_distance("aaaa", "zzzz") == 1.0 + + +def test_calculate_ast_distance_partial_overlap_is_small_positive(monkeypatch): + _patch_clang_index(monkeypatch) + from helm.benchmark.metrics.codeinsights_code_evaluation_metrics import ASTAnalyzer + + analyzer = ASTAnalyzer() + analyzer._extract_ast_features = lambda cursor: list(cursor) + + distance = analyzer.calculate_ast_distance("int x = 1;", "int y = 2;") + assert 0.0 < distance < 1.0