Validate schema has no unreferenced metrics or metric groups

src/helm/benchmark/presentation/schema.py

@@ -382,7 +382,7 @@ def validate_schema(
     schema_path: Optional[str] = None,
     strict: bool = True,
     check_parent_child_partition: bool = False,
-    check_orphan_children: bool = False,
+    check_orphan_children: bool = True,
 ) -> List[SchemaValidationMessage]:
     """Validate a Schema object and return a list of validation messages.
 
@@ -591,6 +591,29 @@ def add_warning(message: str, location: Optional[str] = None) -> None:
                     location=f"run_groups[{run_group.name}]",
                 )
 
+        referenced_metric_groups: Set[str] = set()
+        for run_group in run_groups:
+            referenced_metric_groups |= set(run_group.metric_groups)
+        for metric_group in metric_groups:
+            if metric_group.name not in referenced_metric_groups:
+                add_warning(
+                    f"Metric group '{metric_group.name}' is not referenced by any run group",
+                    location=f"metric_groups[{metric_group.name}]",
+                )
+
+        referenced_metrics: Set[str] = set()
+        for metric_group in metric_groups:
+            for metric_matcher in metric_group.metrics:
+                referenced_metrics.add(metric_matcher.name)
+        for run_group in run_groups:
+            referenced_metrics |= set(run_group.environment.values())
+        for metric in metrics:
+            if metric.name not in referenced_metrics:
+                add_warning(
+                    f"Metric '{metric.name}' is not referenced by any metric group",
+                    location=f"metrics[{metric.name}]",
+                )
+
     if strict:
         error_messages = [msg for msg in messages if msg.severity == ValidationSeverity.ERROR]
         if error_messages:

src/helm/benchmark/presentation/summarize.py

@@ -31,7 +31,7 @@
     unique_simplification,
 )
 from helm.common.codec import from_json
-from helm.common.hierarchical_logger import hlog, htrack, htrack_block, hwarn, setup_default_logging
+from helm.common.hierarchical_logger import hlog, htrack, htrack_block, hwarn, herror, setup_default_logging
 from helm.benchmark.scenarios.scenario import Scenario, ScenarioMetadata, ScenarioSpec, create_scenario
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.metrics.metric_name import MetricName
@@ -1377,10 +1377,16 @@ def _run_schema_validation(self) -> None:
             hwarn(f"Schema validation warning: {msg}")
 
         for msg in errors:
-            hlog(f"Schema validation error: {msg}")
+            herror(f"Schema validation error: {msg}")
 
         if messages:
-            hlog(f"Schema validation complete: {len(errors)} error(s), {len(warnings)} warning(s)")
+            schema_validation_message = (
+                f"Schema validation complete: {len(errors)} error(s), {len(warnings)} warning(s)"
+            )
+            if errors:
+                herror(schema_validation_message)
+            elif warnings:
+                hwarn(schema_validation_message)
 
         # Raise error if in "error" mode and there are errors
         if self.schema_validation == "error" and errors: