[hls-fuzzer] Implement statistics for II

tools/hls-fuzzer/AbstractWorker.h

@@ -49,6 +49,9 @@ class AbstractWorker {
   /// own thread; implementations must therefore be thread-safe with respect to
   /// the worker's 'generate' and 'verify' methods. The default implementation
   /// returns no statistics.
+  ///
+  /// It is the workers responsibility to only return requested statistics.
+  /// See 'isStatisticEnabled'.
   virtual std::vector<Statistic> getStatistics() const { return {}; }
 
 protected:

tools/hls-fuzzer/CMakeLists.txt

@@ -29,6 +29,8 @@ add_llvm_executable(hls-fuzzer
         OptionsParser.cpp
         TargetRegistry.cpp
         statistics/ASTStatistic.cpp
+        statistics/IIReport.cpp
+        statistics/IIStatistic.cpp
         targets/BitwidthOptimizationsTarget.cpp
         targets/BitwidthTypeSystem.cpp
         targets/DynamaticTypeSystem.cpp

tools/hls-fuzzer/statistics/Histogram.h

@@ -0,0 +1,126 @@
+#ifndef DYNAMATIC_HLS_FUZZER_STATISTICS_HISTOGRAM
+#define DYNAMATIC_HLS_FUZZER_STATISTICS_HISTOGRAM
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/JSON.h"
+
+#include <cstddef>
+#include <cstdint>
+#include <map>
+#include <type_traits>
+
+namespace dynamatic {
+
+/// A frequency histogram mapping observed values of type 'T' to the number of
+/// times they were observed. Values are kept ordered so the median and any
+/// rendering come out sorted. Formatting is intentionally left to callers (it
+/// differs per statistic); iterate the histogram with 'begin()'/'end()' to
+/// print it.
+template <typename T>
+class Histogram {
+public:
+  /// Records 'count' additional observations of 'value'.
+  void add(T value, std::size_t count = 1) { counts[value] += count; }
+
+  /// Merges the observations of 'rhs' into this histogram.
+  void merge(const Histogram &rhs) {
+    for (const auto &[value, count] : rhs.counts)
+      counts[value] += count;
+  }
+
+  /// Total number of observations across all values.
+  std::size_t total() const {
+    std::size_t sum = 0;
+    for (const auto &[value, count] : counts)
+      sum += count;
+    return sum;
+  }
+
+  bool empty() const { return counts.empty(); }
+
+  /// The median of all observations. For an even number of observations the
+  /// mean of the two central values is returned; an empty histogram has median
+  /// 0.
+  double median() const {
+    std::size_t n = total();
+    if (n == 0)
+      return 0.0;
+
+    double median = 0.0;
+    std::size_t seen = 0;
+    std::size_t lowerHalf = n / 2;
+    for (const auto &[value, count] : counts) {
+      // The median sits at index 'lowerHalf' (or, for an even count, between
+      // 'lowerHalf - 1' and 'lowerHalf'). Capture both samples as they are
+      // crossed.
+      if (n % 2 == 0 && seen <= lowerHalf - 1 && seen + count > lowerHalf - 1)
+        median += static_cast<double>(value) / 2.0;
+      if (seen <= lowerHalf && seen + count > lowerHalf)
+        median += n % 2 == 0 ? static_cast<double>(value) / 2.0
+                             : static_cast<double>(value);
+      seen += count;
+    }
+    return median;
+  }
+
+  /// Ordered iteration over '{value, count}' pairs, for rendering.
+  auto begin() const { return counts.begin(); }
+  auto end() const { return counts.end(); }
+
+private:
+  std::map<T, std::size_t> counts;
+};
+
+/// The type a histogram value of type 'T' is parsed as before being narrowed
+/// back to 'T'. 'llvm::json' only parses the widest integer and floating-point
+/// types, so a 'Histogram<unsigned>' has to go through 'uint64_t'.
+template <typename T>
+using HistogramParseType = std::conditional_t<
+    std::is_floating_point_v<T>, double,
+    std::conditional_t<std::is_signed_v<T>, int64_t, uint64_t>>;
+
+/// Serializes 'histogram' as an array of '{"value": ..., "count": ...}'
+/// objects, ordered by value.
+///
+/// Counts are written raw rather than as a share of the total, so that a
+/// histogram read back by 'fromJSON' is indistinguishable from the original and
+/// can still be merged into another one.
+template <typename T>
+llvm::json::Value toJSON(const Histogram<T> &histogram) {
+  llvm::json::Array entries;
+  for (const auto &[value, count] : histogram)
+    entries.push_back(llvm::json::Object{
+        {"value", value},
+        {"count", static_cast<uint64_t>(count)},
+    });
+  return entries;
+}
+
+/// Parses a histogram written by 'toJSON', replacing the contents of
+/// 'histogram'. Returns false if 'value' is not a valid representation,
+/// reporting why to 'path'.
+template <typename T>
+bool fromJSON(const llvm::json::Value &value, Histogram<T> &histogram,
+              llvm::json::Path path) {
+  const llvm::json::Array *entries = value.getAsArray();
+  if (!entries) {
+    path.report("expected array");
+    return false;
+  }
+
+  histogram = Histogram<T>();
+  for (const auto &[index, entry] : llvm::enumerate(*entries)) {
+    HistogramParseType<T> entryValue;
+    uint64_t count;
+    llvm::json::ObjectMapper mapper(entry, path.index(index));
+    if (!mapper || !mapper.map("value", entryValue) ||
+        !mapper.map("count", count))
+      return false;
+
+    histogram.add(static_cast<T>(entryValue), static_cast<std::size_t>(count));
+  }
+  return true;
+}
+
+} // namespace dynamatic
+#endif

tools/hls-fuzzer/statistics/IIReport.cpp

@@ -0,0 +1,90 @@
+#include "IIReport.h"
+
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/LineIterator.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace dynamatic;
+
+std::optional<double> LoopIIReport::getMedianII() const {
+  if (intervals.empty())
+    return std::nullopt;
+  return intervals.median();
+}
+
+llvm::SmallVector<LoopIIReport>
+dynamatic::parseIIReport(const std::filesystem::path &outputDir) {
+  llvm::SmallVector<LoopIIReport> reports;
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+      llvm::MemoryBuffer::getFile((outputDir / "sim" / "report.txt").string());
+  if (!buffer)
+    return reports;
+
+  // Index into 'reports', so that a loop's lines -- which the simulator
+  // interleaves with every other loop's -- all land in the same entry while the
+  // loops keep the order they were first seen in.
+  llvm::StringMap<unsigned> indexByLoop;
+  for (llvm::line_iterator it(**buffer, /*SkipBlanks=*/true); !it.is_at_eof();
+       ++it) {
+    llvm::StringRef ref = *it;
+    size_t pos = ref.find("II_INSTRUMENT:");
+    if (pos == llvm::StringRef::npos)
+      continue;
+
+    // The simulator surrounds the report with a severity prefix and, depending
+    // on the backend, a trailing time stamp; the object itself is what lies
+    // between the outermost braces of the line.
+    ref = ref.drop_front(pos);
+    size_t objStart = ref.find('{'), objEnd = ref.rfind('}');
+    if (objStart == llvm::StringRef::npos || objEnd == llvm::StringRef::npos ||
+        objEnd < objStart)
+      continue;
+
+    llvm::Expected<llvm::json::Value> value =
+        llvm::json::parse(ref.slice(objStart, objEnd + 1));
+    if (!value) {
+      llvm::consumeError(value.takeError());
+      continue;
+    }
+    const llvm::json::Object *report = value->getAsObject();
+    if (!report)
+      continue;
+
+    std::optional<llvm::StringRef> loop = report->getString("loop");
+    std::optional<int64_t> depth = report->getInteger("depth");
+    std::optional<int64_t> maxDepth = report->getInteger("max_depth");
+    std::optional<int64_t> iter = report->getInteger("iter");
+    if (!loop || !depth || !maxDepth || !iter)
+      continue;
+
+    auto [entry, inserted] = indexByLoop.try_emplace(*loop, reports.size());
+    if (inserted) {
+      LoopIIReport &newReport = reports.emplace_back();
+      newReport.loop = loop->str();
+      newReport.depth = static_cast<unsigned>(*depth);
+      newReport.maxDepth = static_cast<unsigned>(*maxDepth);
+    }
+    LoopIIReport &loopReport = reports[entry->second];
+
+    if (*iter == 0) {
+      // A fresh activation of the loop. Its interval, if any, spans the gap
+      // since the previous activation and is dropped.
+      loopReport.iterationsPerActivation.push_back(1);
+      continue;
+    }
+    if (loopReport.iterationsPerActivation.empty())
+      // An iteration of an activation the simulation did not record the start
+      // of, which cannot happen but would corrupt the iteration counts.
+      continue;
+
+    ++loopReport.iterationsPerActivation.back();
+    // 'null' on the very first line of the whole run, which has no previous
+    // iteration to measure against.
+    if (std::optional<int64_t> interval = report->getInteger("interval"))
+      loopReport.intervals.add(static_cast<unsigned>(*interval));
+  }
+  return reports;
+}

tools/hls-fuzzer/statistics/IIReport.h

@@ -0,0 +1,65 @@
+#ifndef DYNAMATIC_HLS_FUZZER_STATISTICS_IIREPORT
+#define DYNAMATIC_HLS_FUZZER_STATISTICS_IIREPORT
+
+#include "Histogram.h"
+
+#include "llvm/ADT/SmallVector.h"
+
+#include <filesystem>
+#include <optional>
+#include <string>
+
+namespace dynamatic {
+
+/// Everything one II monitor reported about its loop over a simulation,
+/// gathered from the per-iteration lines it prints (see 'parseIIReport').
+struct LoopIIReport {
+  /// Hierarchical instance path of the monitor, which identifies the loop
+  /// within the program.
+  std::string loop;
+  /// Nesting depth of the loop (1 for a top-level loop) and the deepest depth
+  /// reachable in its nest. The loop is innermost exactly when the two are
+  /// equal.
+  unsigned depth = 0;
+  unsigned maxDepth = 0;
+
+  /// Intervals, in cycles, between an iteration and the one before it within
+  /// the same activation.
+  Histogram<unsigned> intervals;
+
+  /// Sequential history of number of iterations.
+  /// Every fresh activation adds a new entry which contains the number of
+  /// iterations that activation lasted.
+  llvm::SmallVector<unsigned> iterationsPerActivation;
+
+  /// Whether the loop is an innermost one, i.e. has no loop nested inside it.
+  bool isInnermost() const { return depth == maxDepth; }
+
+  /// The loop's depth measured from the innermost loop of its nest outwards: 0
+  /// for an innermost loop, 1 for a loop directly enclosing one, and so on.
+  unsigned depthFromInnermost() const { return maxDepth - depth; }
+
+  /// The loop's achieved median II over all intervals.
+  /// Empty if the loop never ran more than one iteration per activation.
+  std::optional<double> getMedianII() const;
+};
+
+/// Parses the II instrumentation's reports out of the simulation log in
+/// 'outputDir' (the output directory dynamatic was run with, compiled with
+/// '--instrument-ii') and returns one entry per loop. Returns an empty vector
+/// if the log is missing or holds no report.
+///
+/// Every monitor prints one line per iteration of its loop, as the loop takes
+/// that iteration in:
+///   'II_INSTRUMENT: {"loop": <path>, "depth": <d>, "max_depth": <m>,
+///                    "iter": <i>, "interval": <n>}'
+/// where 'iter' is the iteration's index within its activation (so 'iter == 0'
+/// opens a fresh activation) and 'interval' the number of cycles since the
+/// previous iteration was taken in ('null' on the very first line of the run).
+/// The monitor deliberately leaves how these are aggregated to its consumers;
+/// this is where the fuzzer's statistics decide on it.
+llvm::SmallVector<LoopIIReport>
+parseIIReport(const std::filesystem::path &outputDir);
+
+} // namespace dynamatic
+#endif