Support disabling code probes in test harness

contrib/Joshua/scripts/correctnessTest.sh

@@ -99,6 +99,10 @@
 #                                Maps to: --long-running
 #                                Values: true/false (default: false)
 #
+# TH_DISABLE_CODE_PROBES        Disable code probe collection and ensemble coverage checks
+#                                Maps to: --disable-code-probes
+#                                Values: true/false (default: false)
+#
 # TH_RANDOM_SEED                Force specific random seed for debugging
 #                                Maps to: --random-seed
 #                                Default: auto-generated

contrib/TestHarness2/README.md

@@ -76,6 +76,7 @@ TestHarness2 supports environment variables for configuration. Most variables fo
 - **`TH_BUGGIFY`**: Buggify mode (`on`, `off`, or `random`, default: `random`)
 - **`TH_USE_VALGRIND`**: Run tests under valgrind (`true`/`false`, default: `false`)
 - **`TH_LONG_RUNNING`**: Enable long-running test mode (`true`/`false`, default: `false`)
+- **`TH_DISABLE_CODE_PROBES`**: Disable code probe collection and ensemble coverage checks (`true`/`false`, default: `false`)
 - **`TH_FDBSERVER_MEMORY`**: Pass `--memory SIZE` to `fdbserver` (for example `12288MiB`)
 
 **Optional Configuration:**

contrib/TestHarness2/test_harness/config.py

@@ -216,6 +216,11 @@ def __init__(self):
         }
         self.print_coverage = False
         self.print_coverage_args = {"action": "store_true"}
+        self.disable_code_probes: bool = False
+        self.disable_code_probes_args = {
+            "action": "store_true",
+            "help": "Disable code probe collection and ensemble coverage checks",
+        }
         self.binary = Path("bin") / (
             "fdbserver.exe" if os.name == "nt" else "fdbserver"
         )
@@ -358,7 +363,23 @@ def _read_env(self):
                 # Use the env var to supply the default value, so that if the
                 # environment variable is set and the corresponding command line
                 # flag is not, the environment variable has an effect.
-                self._config_map[attr].kwargs["default"] = attr_type(e)
+                self._config_map[attr].kwargs["default"] = self._parse_env_value(
+                    attr_type, env_name, e
+                )
+
+    def _parse_env_value(self, attr_type: type, env_name: str, value: str):
+        if attr_type is bool:
+            normalized = value.lower()
+            if normalized in ("true", "1", "yes", "on"):
+                return True
+            if normalized in ("false", "0", "no", "off"):
+                return False
+            raise ValueError(
+                "Invalid boolean value {} for {} -- use true or false".format(
+                    value, env_name
+                )
+            )
+        return attr_type(value)
 
     def build_arguments(self, parser: argparse.ArgumentParser):
         for val in self._config_map.values():

contrib/TestHarness2/test_harness/results.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import collections
 import io
 import json
 import re
@@ -28,9 +29,14 @@ def __init__(self, cluster_file: str | None, ensemble_id: str):
         self.fdb_path = ("joshua", "ensembles", "results", "application", ensemble_id)
         self.coverage_path = self.fdb_path + ("coverage",)
         self.statistics = test_harness.fdb.Statistics(cluster_file, self.fdb_path)
-        coverage_dict: OrderedDict[Coverage, int] = test_harness.fdb.read_coverage(
-            cluster_file, self.coverage_path
-        )
+        self.code_probe_tracking_enabled = not config.disable_code_probes
+        coverage_dict: OrderedDict[Coverage, int]
+        if self.code_probe_tracking_enabled:
+            coverage_dict = test_harness.fdb.read_coverage(
+                cluster_file, self.coverage_path
+            )
+        else:
+            coverage_dict = collections.OrderedDict()
         self.coverage: List[Tuple[Coverage, int]] = []
         self.min_coverage_hit: int | None = None
         self.ratio = self.global_statistics.total_test_runs / config.hit_per_runs_ratio
@@ -54,7 +60,9 @@ def __init__(self, cluster_file: str | None, ensemble_id: str):
             self.global_statistics.total_cpu_time += v.runtime
             self.stats.append((k, v.runtime, v.run_count))
         self.stats.sort(key=lambda x: x[1], reverse=True)
-        if self.min_coverage_hit is not None:
+        if not self.code_probe_tracking_enabled:
+            self.coverage_ok = True
+        elif self.min_coverage_hit is not None:
             self.coverage_ok = self.min_coverage_hit > self.ratio
         else:
             self.coverage_ok = False
@@ -64,6 +72,9 @@ def dump(self, prefix: str):
         out = SummaryTree("EnsembleResults")
         out.attributes["TotalRuntime"] = str(self.global_statistics.total_cpu_time)
         out.attributes["TotalTestRuns"] = str(self.global_statistics.total_test_runs)
+        out.attributes["CodeProbeTrackingEnabled"] = (
+            "1" if self.code_probe_tracking_enabled else "0"
+        )
         out.attributes["TotalProbesHit"] = str(self.global_statistics.total_probes_hit)
         out.attributes["MinProbeHit"] = str(self.min_coverage_hit)
         out.attributes["TotalProbes"] = str(len(self.coverage))

contrib/TestHarness2/test_harness/summarize.py

@@ -390,7 +390,11 @@ def summarize(self, trace_dir: Path, command: str):
             return
         self.summarize_files(trace_files[0])
         # Skip write_coverage for old binaries in restarting tests
-        if config.joshua_dir is not None and not self.is_old_binary:
+        if (
+            config.joshua_dir is not None
+            and not self.is_old_binary
+            and not config.disable_code_probes
+        ):
             import test_harness.fdb
 
             test_harness.fdb.write_coverage(
@@ -424,7 +428,7 @@ def ok(self):
         return (not self.error) != self.is_negative_test
 
     def done(self):
-        if config.print_coverage:
+        if config.print_coverage and not config.disable_code_probes:
             for k, v in self.coverage.items():
                 child = SummaryTree("CodeCoverage")
                 child.attributes["File"] = k.file
@@ -702,6 +706,8 @@ def parse_error(attrs: Dict[str, str]):
         self.handler.add_handler(("Severity", "40"), parse_error)
 
         def coverage(attrs: Dict[str, str]):
+            if config.disable_code_probes:
+                return
             covered = True
             if "Covered" in attrs:
                 covered = int(attrs["Covered"]) != 0