From 8a84db9f485d76165ff931d0eb53136835859662 Mon Sep 17 00:00:00 2001 From: Chris Harris Date: Fri, 15 May 2026 17:58:50 +0100 Subject: [PATCH] Post Processing: Add a timeseries report generator Implements the code to: - Parse FIO time-series data into a common intermediate format for time-series data. - Generate a plot of the time series data: - IOPS over time - bandwidth over time - latency over time - create a timeseries report in github markdown and pdf format A common intermediate format for time-series data is analogous to the common intemediate format for hockey-stick curves prevously implemented. It allows for a single plotter to be able to handle time-series data from sevear benchmark tools. The code uses the existing design patterns, with different axis plotter sub-classes for each plot. This allows for adding multiple data lines to a single plot if required. Additional unit tests have been added to cover the changed code, and to extend coverage for files that were touched Ruff, pylint, mypy and black checks all pass on the changed files IBM Bob v1.02 and v1.03 was used to help with these changes. Signed-off-by: Chris Harris --- command/fio_command.py | 2 - include/timeseries_report.tex | 7 + post_processing/common.py | 244 ++++- post_processing/formatter/base_formatter.py | 128 +++ .../formatter/common_output_formatter.py | 192 ++-- .../formatter/time_series_output_formatter.py | 142 +++ post_processing/log_configuration.py | 4 - post_processing/parsers/__init__.py | 0 post_processing/parsers/fio_log_parser.py | 279 ++++++ .../parsers/fio_time_series_parser.py | 446 +++++++++ post_processing/parsers/timestamp_aligner.py | 187 ++++ .../plotter/common_format_plotter.py | 6 +- post_processing/plotter/cpu_plotter.py | 2 +- .../plotter/directory_comparison_plotter.py | 101 +- post_processing/plotter/io_plotter.py | 2 +- post_processing/plotter/simple_plotter.py | 33 +- .../plotter/time_series_bandwidth_plotter.py | 54 + .../plotter/time_series_iops_plotter.py | 34 + .../plotter/time_series_latency_plotter.py | 192 ++++ .../plotter/time_series_metric_plotter.py | 110 ++ .../plotter/time_series_plotter.py | 421 ++++++++ post_processing/post_processing_types.py | 114 ++- post_processing/report.py | 90 +- .../reports/comparison_report_generator.py | 133 ++- post_processing/reports/report_generator.py | 260 ++++- .../reports/simple_report_generator.py | 92 +- .../reports/time_series_report_generator.py | 591 +++++++++++ .../{benchmarks => }/benchmark_result.py | 42 +- post_processing/run_results/benchmarks/fio.py | 452 ++++++++- post_processing/run_results/rbdfio.py | 189 +++- .../{resources => }/resource_result.py | 8 +- .../run_results/resources/fio_resource.py | 2 +- post_processing/run_results/run_result.py | 370 ++++++- .../run_results/run_result_factory.py | 9 +- requirements.txt | 3 +- tests/test_base_formatter.py | 118 +++ tests/test_benchmark_result.py | 18 +- tests/test_common_output_formatter.py | 33 +- tests/test_comparison_report_generator.py | 70 +- tests/test_cpu_plotter.py | 2 +- tests/test_directory_comparison_plotter.py | 36 +- tests/test_fio_benchmarks.py | 947 ++++++++++++++++++ tests/test_fio_log_parser.py | 294 ++++++ tests/test_io_plotter.py | 2 +- tests/test_log_configuration.py | 17 +- tests/test_post_processing_types.py | 28 +- tests/test_rbdfio.py | 407 ++++++++ tests/test_report.py | 113 ++- tests/test_report_generator.py | 59 +- tests/test_resource_result.py | 2 +- tests/test_run_result.py | 462 +++++++++ tests/test_run_result_directory_detection.py | 179 ++++ tests/test_run_result_factory.py | 265 +++++ tests/test_simple_plotter.py | 10 +- tests/test_simple_report_generator.py | 76 +- tests/test_time_series_bandwidth_plotter.py | 97 ++ tests/test_time_series_formatter.py | 388 +++++++ tests/test_time_series_iops_plotter.py | 90 ++ tests/test_time_series_latency_plotter.py | 158 +++ tests/test_time_series_output_formatter.py | 248 +++++ tests/test_time_series_plotter.py | 536 ++++++++++ tests/test_time_series_report_generator.py | 418 ++++++++ tests/test_timestamp_aligner.py | 354 +++++++ tests/test_visualisation_directory_helpers.py | 196 ++++ tools/fio_common_output_wrapper.py | 26 +- .../generate_timeseries_performance_report.py | 131 +++ 66 files changed, 10103 insertions(+), 618 deletions(-) create mode 100644 include/timeseries_report.tex create mode 100644 post_processing/formatter/base_formatter.py create mode 100644 post_processing/formatter/time_series_output_formatter.py create mode 100644 post_processing/parsers/__init__.py create mode 100644 post_processing/parsers/fio_log_parser.py create mode 100644 post_processing/parsers/fio_time_series_parser.py create mode 100644 post_processing/parsers/timestamp_aligner.py create mode 100644 post_processing/plotter/time_series_bandwidth_plotter.py create mode 100644 post_processing/plotter/time_series_iops_plotter.py create mode 100644 post_processing/plotter/time_series_latency_plotter.py create mode 100644 post_processing/plotter/time_series_metric_plotter.py create mode 100644 post_processing/plotter/time_series_plotter.py create mode 100644 post_processing/reports/time_series_report_generator.py rename post_processing/run_results/{benchmarks => }/benchmark_result.py (74%) rename post_processing/run_results/{resources => }/resource_result.py (96%) create mode 100644 tests/test_base_formatter.py create mode 100644 tests/test_fio_benchmarks.py create mode 100644 tests/test_fio_log_parser.py create mode 100644 tests/test_rbdfio.py create mode 100644 tests/test_run_result.py create mode 100644 tests/test_run_result_directory_detection.py create mode 100644 tests/test_run_result_factory.py create mode 100644 tests/test_time_series_bandwidth_plotter.py create mode 100644 tests/test_time_series_formatter.py create mode 100644 tests/test_time_series_iops_plotter.py create mode 100644 tests/test_time_series_latency_plotter.py create mode 100644 tests/test_time_series_output_formatter.py create mode 100644 tests/test_time_series_plotter.py create mode 100644 tests/test_time_series_report_generator.py create mode 100644 tests/test_timestamp_aligner.py create mode 100644 tests/test_visualisation_directory_helpers.py create mode 100755 tools/generate_timeseries_performance_report.py diff --git a/command/fio_command.py b/command/fio_command.py index f87a79b9..7552e83e 100644 --- a/command/fio_command.py +++ b/command/fio_command.py @@ -143,8 +143,6 @@ def _generate_output_directory_path(self) -> str: if total_iodepth was used in the options, otherwise: numjobs-/iodepth- """ - # TODO Need to output the benchmark type in this directory structure, - # before the numjobs bit output_path: str = ( f"{self._workload_output_directory}/{self.benchmark}/numjobs-{int(str(self._options['numjobs'])):03d}/" ) diff --git a/include/timeseries_report.tex b/include/timeseries_report.tex new file mode 100644 index 00000000..ff8066b3 --- /dev/null +++ b/include/timeseries_report.tex @@ -0,0 +1,7 @@ +% Additional LaTeX commands for timeseries reports +% This file is used in addition to performance_report.tex +% Scale all images to 50% of their natural size for timeseries reports +\let\oldincludegraphics\includegraphics +\renewcommand{\includegraphics}[2][]{\oldincludegraphics[#1,scale=0.5]{#2}} + +% Made with Bob diff --git a/post_processing/common.py b/post_processing/common.py index 639e6942..88be6c32 100644 --- a/post_processing/common.py +++ b/post_processing/common.py @@ -10,7 +10,7 @@ from math import sqrt from pathlib import Path from re import Pattern -from typing import Any, Optional, Union +from typing import Any, Callable, Optional, Union, cast from post_processing.post_processing_types import CommonFormatDataType @@ -33,6 +33,9 @@ PLOT_FILE_EXTENSION_WITH_DOT: str = f".{PLOT_FILE_EXTENSION}" DATA_FILE_EXTENSION_WITH_DOT: str = f".{DATA_FILE_EXTENSION}" +# Common conversion factors +KB_CONVERSION_FACTOR: int = 1024 + # Regex patterns for stripping confidential data _IPV4_PATTERN: Pattern[str] = re.compile(r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b") _IPV6_PATTERN: Pattern[str] = re.compile( @@ -100,6 +103,39 @@ def read_intermediate_file(file_path: str) -> CommonFormatDataType: return data +def _extract_metrics_from_intermediate_file( + file_path: Path, metric_keys: list[str], format_function: Optional[list[Callable[[str], str]]] = None +) -> tuple[str, ...]: + """ + Generic function to extract metrics from an intermediate format file. + + Args: + file_path: Path to the intermediate format data file + metric_keys: List of keys to extract from the data + format_function: Optional list of formatting functions to apply to each metric. + If None, values are returned as-is. List must match length of metric_keys. + + Returns: + Tuple of extracted and formatted metric values as strings + """ + data: CommonFormatDataType = read_intermediate_file(file_path=f"{file_path}") + + results: list[str] = [] + for index, key in enumerate(metric_keys): + value = data.get(key, "0") + assert isinstance(value, str) + + # Apply formatting function if provided + if format_function and index < len(format_function): + formatted_value = format_function[index](value) + else: + formatted_value = value + + results.append(formatted_value) + + return tuple(results) + + def get_latency_throughput_from_file(file_path: Path) -> tuple[str, str]: """ Reads the data stored in the intermediate file format and returns the @@ -145,11 +181,21 @@ def get_resource_details_from_file(file_path: Path) -> tuple[str, str]: Returns: A tuple of (max_cpu, max_memory) as formatted strings """ - data: CommonFormatDataType = read_intermediate_file(file_path=f"{file_path}") - max_cpu: float = float(f"{data.get('maximum_cpu_usage', '0')}") - max_memory: float = float(f"{data.get('maximum_memory_usage', '0')}") - return f"{max_cpu:.2f}", f"{max_memory:.2f}" + # Use the generic extraction function with formatting + def format_cpu(value: str) -> str: + return f"{float(value):.2f}" + + def format_memory(value: str) -> str: + return f"{float(value):.2f}" + + result = _extract_metrics_from_intermediate_file( + file_path=file_path, + metric_keys=["maximum_cpu_usage", "maximum_memory_usage"], + format_function=[format_cpu, format_memory], + ) + # Cast to the specific tuple type for type safety + return cast(tuple[str, str], result) def strip_confidential_data_from_yaml(yaml_data: str) -> str: @@ -193,17 +239,54 @@ def replace_hostname(match: re.Match[str]) -> str: def find_common_data_file_names(directories: list[Path]) -> list[str]: """ - Find a list of file names that are common to all directories in - a list of directories. + Find a list of file names from the provided directories. + + For a single archive (multiple subdirectories from the same parent), + returns all unique file names found across all subdirectories. + For multiple archives (comparison), returns only files that + exist in ALL provided directories. + + Note: Excludes time-series files (*_timeseries.json) as they have + a different data structure and are handled by TimeSeriesReportGenerator. """ - common_files: set[str] = set(path.parts[-1] for path in directories[0].glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}")) + if not directories: + return [] + + # Collect all file names from all directories, excluding time-series files + all_files: set[str] = set() + for directory in directories: + files = set( + path.parts[-1] + for path in directory.glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}") + if not path.stem.endswith("_timeseries") + ) + all_files.update(files) - # first find all the common paths between all the directories - for index in range(1, (len(directories))): - files: set[str] = set(path.parts[-1] for path in directories[index].glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}")) - common_files = common_files.intersection(files) + # If we only have one directory, return all files found + if len(directories) == 1: + return sorted(list(all_files)) + + # Check if all directories share a common ancestor (same archive) + # by checking if they all have the same great-great-grandparent + # (archive_dir/results/00000000/id-xxx/workload/visualisation) + try: + ancestors = [directory.parents[3] for directory in directories if len(directory.parents) > 3] + if ancestors and all(ancestor == ancestors[0] for ancestor in ancestors): + # All from same archive - return all unique files + return sorted(list(all_files)) + except (IndexError, AttributeError): + pass - return list(common_files) + # For multiple archives, check if files exist in ALL directories + common_files: set[str] = set() + for file_name in all_files: + # Check if this file exists in all of the directories + found_count = sum(1 for directory in directories if (directory / file_name).exists()) + # Only include if found in ALL directories + if found_count == len(directories): + common_files.add(file_name) + + return sorted(list(common_files)) def calculate_percent_difference_to_baseline(baseline: str, comparison: str) -> str: @@ -347,3 +430,138 @@ def sum_mean_values(latencies: list[float], num_ops: list[int], total_ios: int) combined_mean_latency: float = weighted_latency / total_ios return combined_mean_latency + + +def _find_visualisation_directories_with_filter( + archive_directory: Path, + filter_function: Callable[[Path], bool], + sort_function: Callable[[list[Path]], list[Path]] = lambda paths: sorted(paths, key=str), +) -> list[Path]: + """ + Generic function to find visualisation directories based on a filter function. + + Args: + archive_directory: Root directory to search for visualisation directories + filter_function: Function that takes a Path and returns True if it should be included + sort_function: Function to sort the resulting paths (default: sort by string representation) + + Returns: + List of Path objects for visualisation directories that match the filter + """ + visualisation_directories: list[Path] = [] + + # Search for all visualisation directories recursively + for visualisation_directory in archive_directory.glob("**/visualisation"): + if filter_function(visualisation_directory): + visualisation_directories.append(visualisation_directory) + + return sort_function(visualisation_directories) + + +def find_hockey_stick_visualisation_directories(archive_directory: Path) -> list[Path]: + """ + Find visualisation directories containing hockey-stick (common format) JSON data. + + Hockey-stick data is written at the operation level, e.g.: + - archive_directory/randread/visualisation/ (new structure) + - archive_directory/randwrite/visualisation/ (new structure) + - archive_directory/visualisation/ (legacy structure with JSON files) + - archive_directory/**/visualisation/ (deeply nested legacy CBT structure) + + Only returns directories that actually contain .json files. + + Args: + archive_directory: Root directory to search for visualisation directories + + Returns: + List of Path objects for visualisation directories that contain JSON files + """ + # First check for legacy structure: visualisation directly under archive directory + legacy_visualisation_directory = archive_directory / "visualisation" + if legacy_visualisation_directory.exists() and legacy_visualisation_directory.is_dir(): + # Only include if there are non-timeseries JSON files + json_files = [ + filename + for filename in legacy_visualisation_directory.glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}") + if not filename.stem.endswith("_timeseries") + ] + if json_files: + # Legacy structure with data - use only this + return [legacy_visualisation_directory] + + # Look for new structure: visualisation directories under operation directories + # Operation directories are direct children of the archive directory + visualisation_directories: list[Path] = [] + for operation_dir in archive_directory.iterdir(): + if operation_dir.is_dir() and not operation_dir.name.startswith("."): + visualisation_directory = operation_dir / "visualisation" + if visualisation_directory.exists() and visualisation_directory.is_dir(): + # Only include if there are non-timeseries JSON files + json_files = [ + filename + for filename in visualisation_directory.glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}") + if not filename.stem.endswith("_timeseries") + ] + if json_files: + visualisation_directories.append(visualisation_directory) + + # If we found visualisation directories at top level, return them + if visualisation_directories: + return sorted(visualisation_directories, key=str) + + # If no visualisation directories found at top level, search recursively + # This handles deeply nested CBT structures + def has_json_files(visualisation_directory: Path) -> bool: + """Filter function: check if directory contains non-timeseries JSON files and is not the legacy dir.""" + if visualisation_directory == legacy_visualisation_directory: + return False + # Only count non-timeseries JSON files + json_files = [ + filename + for filename in visualisation_directory.glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}") + if not filename.stem.endswith("_timeseries") + ] + return len(json_files) > 0 + + return _find_visualisation_directories_with_filter( + archive_directory=archive_directory, + filter_function=has_json_files, + sort_function=lambda paths: sorted(paths, key=str), + ) + + +def find_timeseries_visualisation_directories(archive_directory: Path) -> list[Path]: + """ + Find visualisation directories containing timeseries data. + + Timeseries data is written at the iodepth/total_iodepth level, e.g.: + - archive_directory/randread/total_iodepth-256/visualisation/ + - archive_directory/randread/iodepth-32/visualisation/ + + Args: + archive_directory: Root directory to search for visualisation directories + + Returns: + List of Path objects for iodepth-level visualisation directories + """ + + def is_iodepth_directory(visualisation_directory: Path) -> bool: + """Filter function: check if parent is an iodepth or total_iodepth directory.""" + parent_name = visualisation_directory.parent.name + return parent_name.startswith(("iodepth-", "total_iodepth-")) + + def sort_by_priority(paths: list[Path]) -> list[Path]: + """Sort by priority: total_iodepth > iodepth, then by path.""" + + def sort_key(path: Path) -> tuple[int, str]: + parent_name = path.parent.name + if parent_name.startswith("total_iodepth"): + return (0, str(path)) + # iodepth + return (1, str(path)) + + return sorted(paths, key=sort_key) + + return _find_visualisation_directories_with_filter( + archive_directory=archive_directory, filter_function=is_iodepth_directory, sort_function=sort_by_priority + ) diff --git a/post_processing/formatter/base_formatter.py b/post_processing/formatter/base_formatter.py new file mode 100644 index 00000000..aeeeb9ac --- /dev/null +++ b/post_processing/formatter/base_formatter.py @@ -0,0 +1,128 @@ +""" +Base class for all formatters. + +This module provides the BaseFormatter abstract class that defines +the common API for all formatter implementations. +""" + +from abc import ABC, abstractmethod +from logging import Logger, getLogger +from pathlib import Path + + +class BaseFormatter(ABC): + """ + Base class for all formatters. + + Formatters convert benchmark output into intermediate formats + for report generation. With the memory-efficient approach, + data is written during process() to avoid accumulating large + datasets in memory. + """ + + def __init__(self, archive_directory: str) -> None: + """ + Initialize formatter. + + Args: + archive_directory: Directory containing benchmark results + """ + self._directory = archive_directory + self._log: Logger = getLogger("formatter") + self._all_test_run_ids: set[str] = set() + + @property + def log(self) -> Logger: + """Get the logger instance""" + return self._log + + @property + def path(self) -> Path: + """ + Get Path object for archive directory. + + Returns: + Path object representing the archive directory + """ + return Path(self._directory) + + def _ensure_output_directory(self, directory: Path) -> None: + """ + Ensure output directory exists, creating it if necessary. + + This method creates the directory and any necessary parent directories. + If the directory already exists, no action is taken. + + Args: + directory: Path object for directory to create + """ + if not directory.exists(): + self.log.debug("Creating output directory: %s", directory) + directory.mkdir(parents=True, exist_ok=True) + + def _get_testrun_directories(self, testrun_id: str) -> list[Path]: + """ + Get the directories for a specific test run ID. + + When calling from CBT itself, the archive dir already includes the testrun + directory, so we handle this case by checking if 'id-' is in the path. + + Args: + testrun_id: The test run identifier to find directories for + + Returns: + List of Path objects for directories matching the test run ID + """ + if "id-" in f"{self.path}": + return [self.path] + return list(self.path.glob(f"**/{testrun_id}")) + + def _find_all_testrun_ids(self, file_list: list[Path]) -> None: + """ + Find all the unique test run IDs from a list of file paths. + + Populates self._all_test_run_ids with unique test run ID strings. + + Args: + file_list: List of Path objects for result files + + Note: This may only work for fio output runs in cbt, and we will need + separate sub-classes for each benchmark type to be able to find and + parse the required data + """ + for file_path in file_list: + # We know the format of the output dir is something like + # /00000000/id-ab40819c//output.x + # + # We know that the output files reside in the directory structure with the + # test run ID, so splitting up the path gives the filename as the + # last element (-1) and the test run id directory somewhere higher up + # the directory structure. + # This should allow us to get test run IDs when any point in the + # archive directory tree is passed as the archive directory + potential_ids: list[str] = [part for part in file_path.parts if "id-" in part] + # There is a possibility that there could be more than one id-xxxxxx string in the + # file path, and we want only one. We choose to always take the first one. + # If there are none then just return the directory name above the file + if len(potential_ids) > 0: + testrun_id: str = potential_ids[0] + else: + # if we get no matches then just use the directory directly above + # the output file + testrun_id = file_path.parts[-2] + + self._all_test_run_ids.add(testrun_id) + + @abstractmethod + def process(self) -> None: + """ + Process input data and convert to intermediate format. + + This method reads benchmark output files and converts them + to the formatter's target format. With the memory-efficient + approach, data is written immediately during processing to + avoid accumulating large datasets in memory. + """ + + +# Made with Bob diff --git a/post_processing/formatter/common_output_formatter.py b/post_processing/formatter/common_output_formatter.py index 35516aac..bee8b642 100644 --- a/post_processing/formatter/common_output_formatter.py +++ b/post_processing/formatter/common_output_formatter.py @@ -44,18 +44,15 @@ import json import re -from logging import Logger, getLogger from pathlib import Path from typing import Optional -from common import pdsh # pylint: disable=[no-name-in-module] +from post_processing.formatter.base_formatter import BaseFormatter from post_processing.post_processing_types import CommonFormatDataType, InternalFormattedOutputType from post_processing.run_results.run_result_factory import get_run_result_from_directory_name -log: Logger = getLogger(name="formatter") - -class CommonOutputFormatter: +class CommonOutputFormatter(BaseFormatter): """ This class contains all the common code for converting an output file in json format to the format we want to use to draw iops and latency @@ -69,17 +66,10 @@ class CommonOutputFormatter: DEFAULT_OUTPUT_FILE_PART: str = "json_output" def __init__(self, archive_directory: str, filename_root: Optional[str] = None) -> None: - self._directory: str = archive_directory + super().__init__(archive_directory) self._filename_root: str = filename_root if filename_root else self.DEFAULT_OUTPUT_FILE_PART self._formatted_output: InternalFormattedOutputType = {} - self._all_test_run_ids: set[str] = set() - # Note that we use a set here as it does not allow duplicate entries, - # and we do not care about ordering. It would be possible to use a List - # and manually check for duplicates, but that seems more untidy - - self._path: Path = Path(self._directory) - self._file_list: list[Path] = [] self._benchmark_types: dict[str, str] = {} # Track benchmark type per operation def _merge_results(self, processed_results: InternalFormattedOutputType) -> None: @@ -112,23 +102,6 @@ def _merge_results(self, processed_results: InternalFormattedOutputType) -> None # Update existing blocksize data self._formatted_output[run_type][numjobs][blocksize].update(blocksize_data) - def _get_testrun_directories(self, testrun_id: str) -> list[Path]: - """ - Get the directories for a specific test run ID. - - When calling from CBT itself, the archive dir already includes the testrun - directory, so we handle this case by checking if 'id-' is in the path. - - Args: - testrun_id: The test run identifier to find directories for - - Returns: - List of Path objects for directories matching the test run ID - """ - if "id-" in f"{self._path}": - return [self._path] - return list(self._path.glob(f"**/{testrun_id}")) - def _process_compatibility_mode(self) -> str: """ Process test run using compatibility method for legacy directory structures. @@ -152,7 +125,7 @@ def _process_single_testrun(self, testrun_directory: Path) -> str: Process a single test run directory and all its IO pattern subdirectories. This method iterates through all subdirectories in the test run directory, - processes each one, and merges the results into the formatted output. + processes each one, and writes results immediately to reduce memory usage. Args: testrun_directory: Path to the test run directory @@ -170,7 +143,7 @@ def _process_single_testrun(self, testrun_directory: Path) -> str: and not f"{directory.name}".startswith(".") and "visualisation" not in f"{directory.name}" ]: - log.debug("Looking at results for directory %s", io_pattern_directory) + self.log.debug("Looking at results for directory %s", io_pattern_directory) # Use factory method to get the correct results type based on directory name results = get_run_result_from_directory_name( directory=io_pattern_directory, file_name_root=self._filename_root @@ -178,7 +151,11 @@ def _process_single_testrun(self, testrun_directory: Path) -> str: results.process() processed_results = results.get() - self._merge_results(processed_results) + + # Memory-efficient approach: write results for this operation immediately + # instead of accumulating everything + self._write_operation_results(io_pattern_directory, processed_results, results.type) + benchmark_type = results.type return benchmark_type @@ -215,6 +192,8 @@ def _add_peak_metrics(self) -> None: for operation_type, operation_data in self._formatted_output.items(): for _, numjobs_data in operation_data.items(): for _, blocksize_data in numjobs_data.items(): + # Cast to CommonFormatDataType since blocksize_data contains both + # string metadata and dict iodepth entries max_bandwidth, max_bandwidth_latency, max_iops, max_iops_latency = ( self._find_maximum_bandwidth_and_iops_with_latency(blocksize_data) ) @@ -227,104 +206,97 @@ def _add_peak_metrics(self) -> None: blocksize_data["maximum_memory_usage"] = max_memory blocksize_data["benchmark"] = self._benchmark_types.get(operation_type, "unknown") - def convert_all_files(self) -> None: + def _write_operation_results( # pylint: disable=too-many-locals + self, operation_directory: Path, processed_results: InternalFormattedOutputType, benchmark_type: str + ) -> None: """ - Convert all files in a given directory to our internal format and then - write out the intermediate file that can then be used to produce a graph + Write results for a single operation directory immediately to reduce memory usage. + + Args: + operation_directory: Path to the operation directory (e.g., 256krandomwrite.../rbdfio/) + processed_results: Processed results for this operation + benchmark_type: Type of benchmark (e.g., "rbdfio", "fio") """ - log.info("Converting all files with name %s in directory %s", self._filename_root, self._directory) - self._find_all_results_files_in_directory() - self._find_all_testrun_ids() + if not processed_results: + return - for testrun_id in self._all_test_run_ids: - log.debug("Looking at test run with id %s", testrun_id) + # Create visualisation directory at operation level + output_dir = operation_directory / "visualisation" + output_dir.mkdir(parents=True, exist_ok=True) - testrun_directories = self._get_testrun_directories(testrun_id) + self.log.info("Writing hockey-stick data to %s", output_dir) - if len(testrun_directories) > 1: - log.debug( - "We have more than one directory for test run %s so using the compatibility method", testrun_id - ) - benchmark_type = self._process_compatibility_mode() - else: - benchmark_type = self._process_single_testrun(testrun_directories[0]) + # Process each operation type in the results + for operation_type, operation_data in processed_results.items(): + # Track benchmark type + self._benchmark_types[operation_type] = benchmark_type - # Track benchmark type for all operations processed in this test run - for operation_type, _ in self._formatted_output.items(): - self._benchmark_types[operation_type] = benchmark_type + for number_of_jobs, numjobs_data in operation_data.items(): + for blocksize, blocksize_data in numjobs_data.items(): + # Add metadata + blocksize_data["benchmark"] = benchmark_type + blocksize_data["number_of_jobs"] = number_of_jobs - # Add common metadata fields (benchmark type, number_of_jobs) to the test run data - self._add_common_metadata() - # Add the maximum values (max bandwidth, IOPS, resource usage) to the test run data - self._add_peak_metrics() + # Add peak metrics + max_bandwidth, max_bandwidth_latency, max_iops, max_iops_latency = ( + self._find_maximum_bandwidth_and_iops_with_latency(blocksize_data) + ) + max_cpu, max_memory = self._find_max_resource_usage(blocksize_data) + blocksize_data["maximum_bandwidth"] = max_bandwidth + blocksize_data["latency_at_max_bandwidth"] = max_bandwidth_latency + blocksize_data["maximum_iops"] = max_iops + blocksize_data["latency_at_max_iops"] = max_iops_latency + blocksize_data["maximum_cpu_usage"] = max_cpu + blocksize_data["maximum_memory_usage"] = max_memory - def write_output_file(self) -> None: - """ - Write the formatted output to the output file in JSON format - """ + # Write file + filename = output_dir / f"{blocksize}_{number_of_jobs}_{operation_type}.json" + self.log.debug("Writing hockey-stick data to %s", filename) - destination_directory: str = f"{self._directory}/visualisation/" - log.info("writing new format files to %s", destination_directory) + try: + with filename.open("w", encoding="utf8") as f: + json.dump(blocksize_data, f, indent=4, sort_keys=True) + except OSError as e: + self.log.error("Failed to write hockey-stick file %s: %s", filename, e) - if not Path(destination_directory).is_dir(): - pdsh("localhost", f"mkdir -p {destination_directory}").communicate() # type: ignore[no-untyped-call] + def process(self) -> None: + """ + Process input data and convert to intermediate format. - for operation_type, operation_data in self._formatted_output.items(): - for number_of_jobs, numbjob_data in operation_data.items(): - for blocksize, blocksize_data in numbjob_data.items(): - destination_filename: str = ( - f"{self._directory}/visualisation/{blocksize}_{number_of_jobs}_{operation_type}.json" - ) - log.info("Writing formatted results to destination file %s", destination_filename) - with open(destination_filename, "w", encoding="utf8") as output: - json.dump(blocksize_data, output, indent=4, sort_keys=True) + Convert all files in a given directory to our internal format and then + prepare the intermediate data that can be used to produce a graph. - def _find_all_results_files_in_directory(self) -> None: - """ - find the files of interest in the archive directory we have been given + Note: With memory-efficient mode, results are written immediately during + processing rather than accumulated in memory. """ - log.debug("Finding all %s* files from %s", self._filename_root, self._directory) + self.log.info("Converting all files with name %s in directory %s", self._filename_root, self._directory) - # this gives a generator where each contained object is a Path of format: - # /results///json_output. - self._file_list = [ + # Find all result files + self.log.debug("Finding all %s* files from %s", self._filename_root, self._directory) + file_list = [ path - for path in self._path.glob(f"**/{self._filename_root}.*") + for path in self.path.glob(f"**/{self._filename_root}.*") if re.search(rf"{self._filename_root}.\d+$", f"{path}") ] - def _find_all_testrun_ids(self) -> None: - """ - Find all the unique test run IDs in the output directory. We will need - these to allow us to collect the data we require from a test run + self._find_all_testrun_ids(file_list) - Note: This may only work for fio output runs in cbt, and we will need - separate sub-classes for each benchmark type to be able to find and - parse the required data - """ + for testrun_id in self._all_test_run_ids: + self.log.debug("Looking at test run with id %s", testrun_id) - for file_path in self._file_list: - # We know the format of the output dir is something like - # /00000000/id-ab40819c//output.x - # - # We know that the output files reside in the directory structure with the - # test run ID, so splitting up the path gives the filename as the - # last element (-1) and the test run id directory somewhere higher up - # the directory structure. - # This should allow us to get test run IDs when any point in the - # archive directory tree is passed as the archive directory - potential_ids: list[str] = [part for part in file_path.parts if "id-" in part] - # There is a possibility that there could be more than one id-xxxxxx string in the - # file path, and we want only one. We choose to always take the fist one. - # If there are none then just return the directory name above the file - if len(potential_ids) > 0: - testrun_id: str = potential_ids[0] - else: - # if we get no matches then just use the directory directly above - # the output file - testrun_id = file_path.parts[-2] + testrun_directories = self._get_testrun_directories(testrun_id) - self._all_test_run_ids.add(testrun_id) + if len(testrun_directories) > 1: + self.log.debug( + "We have more than one directory for test run %s so using the compatibility method", testrun_id + ) + self._process_compatibility_mode() + # For compatibility mode, still need to add metadata and write + self._add_common_metadata() + self._add_peak_metrics() + else: + # Memory-efficient mode: results written during _process_single_testrun + self._process_single_testrun(testrun_directories[0]) def _find_maximum_bandwidth_and_iops_with_latency( self, test_run_data: CommonFormatDataType diff --git a/post_processing/formatter/time_series_output_formatter.py b/post_processing/formatter/time_series_output_formatter.py new file mode 100644 index 00000000..c8c8fb01 --- /dev/null +++ b/post_processing/formatter/time_series_output_formatter.py @@ -0,0 +1,142 @@ +""" +Formatter for converting time-series log files to plotting format. + +This module provides the TimeSeriesOutputFormatter class which automatically +discovers and processes time-series log files using the same factory pattern +as CommonOutputFormatter. +""" + +from pathlib import Path +from typing import Optional + +from post_processing.formatter.base_formatter import BaseFormatter +from post_processing.post_processing_types import TimeSeriesFormatType +from post_processing.run_results.run_result_factory import get_run_result_from_directory_name + + +class TimeSeriesOutputFormatter(BaseFormatter): + """ + Automatically discover and process time-series log files. + + This class follows the same pattern as CommonOutputFormatter: + - Uses factory pattern to get appropriate RunResult subclass + - Delegates time-series processing to BenchmarkResult classes + - Provides simple process() and write_output() interface + """ + + # To make sure that we read files that only contain JSON data we will look + # at the json_output* files by default + DEFAULT_OUTPUT_FILE_PART: str = "json_output" + + def __init__(self, archive_directory: str, filename_root: Optional[str] = None) -> None: + """ + Initialize the time series output formatter. + + Args: + archive_directory: Directory containing benchmark results + filename_root: Root name for output files (default: "json_output") + """ + super().__init__(archive_directory) + self._filename_root: str = filename_root if filename_root else self.DEFAULT_OUTPUT_FILE_PART + self._timeseries_data: dict[str, TimeSeriesFormatType] = {} + self._benchmark_types: dict[str, str] = {} # Track benchmark type per operation + + def _process_compatibility_mode(self) -> str: + """ + Process test run using compatibility method for legacy directory structures. + + This handles cases where there are multiple directories for a single test run, + which requires using the directory-level processing approach. + + Returns: + The benchmark type identifier (e.g., "rbdfio", "fio") + """ + results = get_run_result_from_directory_name( + Path(self._directory), self._filename_root, include_timeseries=True + ) + results.process() + + # Note: Timeseries data is now written directly to disk during RunResult.process() + # to reduce memory usage. We no longer accumulate it here. + + return results.type + + def _process_single_testrun(self, testrun_directory: Path) -> str: + """ + Process a single test run directory and all its IO pattern subdirectories. + + This method iterates through all subdirectories in the test run directory, + processes each one with time-series enabled, and collects the results. + + Args: + testrun_directory: Path to the test run directory + + Returns: + The benchmark type identifier from the last processed result + """ + benchmark_type: str = "unknown" + + for io_pattern_directory in [ + directory + for directory in testrun_directory.iterdir() + if directory.is_dir() + and not f"{directory.name}".startswith(".") + and "visualisation" not in f"{directory.name}" + ]: + self.log.debug("Looking at results for directory %s", io_pattern_directory) + + # Use factory method to get the correct results type based on directory name + # Enable time-series processing + results = get_run_result_from_directory_name( + directory=io_pattern_directory, file_name_root=self._filename_root, include_timeseries=True + ) + + results.process() + + # Note: Timeseries data is now written directly to disk during RunResult.process() + # to reduce memory usage. We no longer accumulate it here. + + benchmark_type = results.type + + return benchmark_type + + def process(self) -> None: + """ + Process input data and convert to intermediate format. + + Discovers all test runs in the archive directory, processes them using + the factory pattern to get appropriate RunResult subclasses, and extracts + time-series data from each. + """ + self.log.info("Processing time-series data from %s", self._directory) + + # Find all result files + file_list = [ + path for path in self.path.glob(f"**/{self._filename_root}.*") if path.name.split(".")[-1].isdigit() + ] + self._find_all_testrun_ids(file_list) + + if not self._all_test_run_ids: + self.log.warning("No test runs found in %s", self._directory) + return + + for testrun_id in self._all_test_run_ids: + self.log.debug("Looking at test run with id %s", testrun_id) + + testrun_directories = self._get_testrun_directories(testrun_id) + + if len(testrun_directories) > 1: + self.log.debug( + "We have more than one directory for test run %s so using the compatibility method", testrun_id + ) + benchmark_type = self._process_compatibility_mode() + else: + benchmark_type = self._process_single_testrun(testrun_directories[0]) + + # Track benchmark type for all operations processed in this test run + for key in self._timeseries_data: + if key not in self._benchmark_types: + self._benchmark_types[key] = benchmark_type + + +# Made with Bob diff --git a/post_processing/log_configuration.py b/post_processing/log_configuration.py index 2e892192..71402d2d 100644 --- a/post_processing/log_configuration.py +++ b/post_processing/log_configuration.py @@ -4,7 +4,6 @@ import logging.config import os -from logging import Logger, getLogger from typing import Any from post_processing.post_processing_types import HandlerType @@ -23,9 +22,6 @@ def setup_logging() -> None: """ os.makedirs(f"{LOGFILE_LOCATION}/cbt/", exist_ok=True) logging.config.dictConfig(_get_configuration()) - log: Logger = getLogger("formatter") - - log.info("=== Starting Post Processing of CBT results ===") def _get_handlers_configuration() -> HandlerType: diff --git a/post_processing/parsers/__init__.py b/post_processing/parsers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/post_processing/parsers/fio_log_parser.py b/post_processing/parsers/fio_log_parser.py new file mode 100644 index 00000000..cb584589 --- /dev/null +++ b/post_processing/parsers/fio_log_parser.py @@ -0,0 +1,279 @@ +""" +Parser for FIO time-series log files. + +This module provides functionality to parse FIO's time-series log files +(_iops, _clat, _bw, _lat) into pandas DataFrames for further processing. +""" + +from logging import Logger, getLogger +from pathlib import Path +from typing import Callable, Optional + +import pandas as pd +from pandas.core.frame import DataFrame + +log: Logger = getLogger("parser") + + +class FIOLogParser: + """ + Parse FIO time-series log files into pandas DataFrames. + + FIO generates time-series logs when the --write_iops_log, --write_bw_log, + and --write_lat_log options are used. These logs have a consistent format: + timestamp_ms, value, direction, block_offset + + This parser converts these logs into DataFrames with appropriate units + (seconds for time, milliseconds for latency, bytes for bandwidth). + """ + + def parse_iops_log(self, file_path: Path) -> Optional[pd.DataFrame]: + """ + Parse FIO _iops.log file. + + Format: timestamp_ms, iops, direction, block_size, offset + + Args: + file_path: Path to the _iops.log file + + Returns: + DataFrame with columns [timestamp_sec, iops, direction] or None if error + """ + try: + data_framef = pd.read_csv( + file_path, + names=["timestamp_ms", "iops", "direction", "block_size", "offset"], + skipinitialspace=True, + ) + # Handle empty file - return empty DataFrame + if data_framef.empty: + log.debug("Empty IOPS log file %s", file_path) + return pd.DataFrame(columns=["timestamp_sec", "iops", "direction"]) + # Validate numeric columns + data_framef["timestamp_ms"] = pd.to_numeric(data_framef["timestamp_ms"], errors="coerce") + data_framef["iops"] = pd.to_numeric(data_framef["iops"], errors="coerce") + # Check if we have any valid data after coercion + if data_framef["timestamp_ms"].isna().all() or data_framef["iops"].isna().all(): + log.error("No valid numeric data in IOPS log %s", file_path) + return None + data_framef["timestamp_sec"] = data_framef["timestamp_ms"] / 1000.0 + result: DataFrame = data_framef[["timestamp_sec", "iops", "direction"]].copy() + log.debug("Parsed %d IOPS samples from %s", len(result), file_path) + return result + except (FileNotFoundError, pd.errors.ParserError, ValueError, TypeError) as e: + log.error("Error parsing IOPS log %s: %s", file_path, e) + return None + + def parse_clat_log(self, file_path: Path) -> Optional[DataFrame]: + """ + Parse FIO _clat.log file (completion latency). + + Format: timestamp_ms, latency_ns, direction, block_size, offset + + Args: + file_path: Path to the _clat.log file + + Returns: + DataFrame with columns [timestamp_sec, latency_ms, direction] or None if error + """ + try: + df: DataFrame = pd.read_csv( + file_path, + names=["timestamp_ms", "latency_ns", "direction", "block_size", "offset"], + skipinitialspace=True, + ) + # Handle empty file - return empty DataFrame + if df.empty: + log.debug("Empty clat log file %s", file_path) + return pd.DataFrame(columns=["timestamp_sec", "latency_ms", "direction"]) + # Validate numeric columns + df["timestamp_ms"] = pd.to_numeric(df["timestamp_ms"], errors="coerce") + df["latency_ns"] = pd.to_numeric(df["latency_ns"], errors="coerce") + # Check if we have any valid data after coercion + if df["timestamp_ms"].isna().all() or df["latency_ns"].isna().all(): + log.error("No valid numeric data in clat log %s", file_path) + return None + df["timestamp_sec"] = df["timestamp_ms"] / 1000.0 + df["latency_ms"] = df["latency_ns"] / 1_000_000.0 + result: DataFrame = df[["timestamp_sec", "latency_ms", "direction"]].copy() + log.debug("Parsed %d latency samples from %s", len(result), file_path) + return result + except (FileNotFoundError, pd.errors.ParserError, ValueError, TypeError) as e: + log.error("Error parsing clat log %s: %s", file_path, e) + return None + + def parse_lat_log(self, file_path: Path) -> Optional[pd.DataFrame]: + """ + Parse FIO _lat.log file (total latency). + + Format: timestamp_ms, latency_ns, direction, block_size, offset + + Args: + file_path: Path to the _lat.log file + + Returns: + DataFrame with columns [timestamp_sec, latency_ms, direction] or None if error + """ + # Same format as clat + return self.parse_clat_log(file_path) + + def parse_bw_log(self, file_path: Path) -> Optional[DataFrame]: + """ + Parse FIO _bw.log file (bandwidth). + + Format: timestamp_ms, bandwidth_kb, direction, block_size, offset + + Args: + file_path: Path to the _bw.log file + + Returns: + DataFrame with columns [timestamp_sec, bandwidth_bytes, direction] or None if error + """ + try: + dataframe: DataFrame = pd.read_csv( + file_path, + names=["timestamp_ms", "bandwidth_kb", "direction", "block_size", "offset"], + skipinitialspace=True, + ) + # Handle empty file - return empty DataFrame + if dataframe.empty: + log.debug("Empty bandwidth log file %s", file_path) + return pd.DataFrame(columns=["timestamp_sec", "bandwidth_bytes", "direction"]) + # Validate numeric columns + dataframe["timestamp_ms"] = pd.to_numeric(dataframe["timestamp_ms"], errors="coerce") + dataframe["bandwidth_kb"] = pd.to_numeric(dataframe["bandwidth_kb"], errors="coerce") + # Check if we have any valid data after coercion + if dataframe["timestamp_ms"].isna().all() or dataframe["bandwidth_kb"].isna().all(): + log.error("No valid numeric data in bandwidth log %s", file_path) + return None + dataframe["timestamp_sec"] = dataframe["timestamp_ms"] / 1000.0 + dataframe["bandwidth_bytes"] = dataframe["bandwidth_kb"] * 1024 + result: DataFrame = dataframe[["timestamp_sec", "bandwidth_bytes", "direction"]].copy() + log.debug("Parsed %d bandwidth samples from %s", len(result), file_path) + return result + except (FileNotFoundError, pd.errors.ParserError, ValueError, TypeError) as e: + log.error("Error parsing bandwidth log %s: %s", file_path, e) + return None + + def parse_slat_log(self, file_path: Path) -> Optional[DataFrame]: + """ + Parse FIO _slat.log file (submission latency). + + Format: timestamp_ms, latency_ns, direction, block_size, offset + + Args: + file_path: Path to the _slat.log file + + Returns: + DataFrame with columns [timestamp_sec, latency_ms, direction] or None if error + """ + # Same format as clat + return self.parse_clat_log(file_path) + + def parse_and_combine_logs( # pylint: disable=too-many-locals + self, directory: Path, log_type: str, pattern: str = "*" + ) -> Optional[DataFrame]: + """ + Parse and combine multiple FIO log files from a directory. + + This method finds all matching log files in a directory (e.g., output.0_clat.1.log, + output.1_clat.1.log, etc.), parses each one, and combines them by summing values + at matching timestamps. + + Args: + directory: Directory containing the log files + log_type: Type of log to parse ('iops', 'clat', 'lat', 'bw', 'slat') + pattern: Glob pattern to match files (default: "*") + + Returns: + Combined DataFrame with aggregated values, or None if error + + Example: + parser = FIOLogParser() + # Combine all clat logs matching output.*_clat.1.log + combined = parser.parse_and_combine_logs( + Path("/path/to/logs"), "clat", "output.*_clat.1.log" + ) + """ + # Map log type to parser method + parser_methods: dict[str, Callable[..., Optional[DataFrame]]] = { + "iops": self.parse_iops_log, + "clat": self.parse_clat_log, + "lat": self.parse_lat_log, + "bw": self.parse_bw_log, + "slat": self.parse_slat_log, + } + + if log_type not in parser_methods: + log.error("Invalid log_type '%s'. Must be one of: %s", log_type, list(parser_methods.keys())) + return None + + parser_method: Callable[..., Optional[DataFrame]] = parser_methods[log_type] + + # Find all matching log files + try: + log_files = sorted(directory.glob(pattern)) + except OSError as e: + log.error("Error finding log files in %s with pattern %s: %s", directory, pattern, e) + return None + + if not log_files: + log.warning("No log files found in %s matching pattern %s", directory, pattern) + return None + + log.info("Found %d log files to combine: %s", len(log_files), [f.name for f in log_files]) + + # Determine the value column name based on log type + value_columns: dict[str, str] = { + "iops": "iops", + "clat": "latency_ms", + "lat": "latency_ms", + "bw": "bandwidth_bytes", + "slat": "latency_ms", + } + value_col = value_columns[log_type] + + # Parse all files and aggregate each one individually first + # This handles duplicate timestamps within a single file + aggregated_dataframes: list[pd.DataFrame] = [] + for log_file in log_files: + dataframe: Optional[DataFrame] = parser_method(log_file) + if dataframe is not None and not dataframe.empty: + # Aggregate duplicate timestamps within this file first + # Group by timestamp and direction, sum the values + aggregated_dataframe = dataframe.groupby(["timestamp_sec", "direction"], as_index=False)[ + [value_col] + ].sum() + aggregated_dataframes.append(aggregated_dataframe) + log.debug( + "Aggregated %d samples to %d unique timestamps in %s", + len(dataframe), + len(aggregated_dataframe), + log_file.name, + ) + else: + log.warning("Skipping empty or invalid log file: %s", log_file) + + if not aggregated_dataframes: + log.error("No valid data found in any log files") + return None + + # Now combine the pre-aggregated dataframes across files + # Each file now has unique timestamps, so we can safely sum across files + combined: DataFrame = pd.concat(aggregated_dataframes, ignore_index=True) + + # Group by timestamp and direction again to combine across files + result: DataFrame = combined.groupby(["timestamp_sec", "direction"], as_index=False)[[value_col]].sum() + result = result.sort_values(by=["timestamp_sec"]).reset_index(drop=True) + + log.info( + "Combined %d log files into %d aggregated samples for %s", + len(aggregated_dataframes), + len(result), + log_type, + ) + + return result + + +# Made with Bob diff --git a/post_processing/parsers/fio_time_series_parser.py b/post_processing/parsers/fio_time_series_parser.py new file mode 100644 index 00000000..8978667b --- /dev/null +++ b/post_processing/parsers/fio_time_series_parser.py @@ -0,0 +1,446 @@ +""" +Parser for converting aligned time-series data to plotting format. + +This module provides the FIOTimeSeriesParser class which converts +pandas DataFrames from FIOLogParser into the TimeSeriesFormatType +intermediate format suitable for plotting. +""" + +from collections.abc import Mapping +from logging import Logger, getLogger +from typing import Optional, Union, cast + +import pandas as pd + +from post_processing.post_processing_types import ( + TimeSeriesDataPoint, + TimeSeriesFormatType, + TimeSeriesMetadata, +) + +log: Logger = getLogger("parser") + + +class FIOTimeSeriesParser: # pylint: disable=too-many-instance-attributes + """ + Parse aligned time-series data for plotting. + + Takes pandas DataFrames from FIOLogParser and converts them to + the TimeSeriesFormatType intermediate format. This format includes + all metrics (IOPS, bandwidth, latency) at each timestamp along with + metadata about the test run. + """ + + def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments,too-many-locals + self, + archive_directory: str, + benchmark: str = "fio", + operation: str = "unknown", + blocksize: str = "unknown", + numjobs: str = "1", + iodepth: str = "1", + iops_df: Optional[pd.DataFrame] = None, + bandwidth_df: Optional[pd.DataFrame] = None, + mean_latency_df: Optional[pd.DataFrame] = None, + max_latency_df: Optional[pd.DataFrame] = None, + p50_latency_df: Optional[pd.DataFrame] = None, + p95_latency_df: Optional[pd.DataFrame] = None, + p99_latency_df: Optional[pd.DataFrame] = None, + num_volumes: int = 1, + log_avg_msec: int = 1000, + ) -> None: + """ + Initialize the parser with all required data. + + Args: + archive_directory: Directory containing benchmark results + benchmark: Name of the benchmark (default: "fio") + operation: Operation type (e.g., "randread", "randwrite", "randrw") + blocksize: Block size used in test (e.g., "4k", "128k") + numjobs: Number of jobs used in test (default: "1") + iodepth: IO depth (total_iodepth if exists, otherwise iodepth) (default: "1") + iops_df: DataFrame with columns [timestamp_sec, iops] + bandwidth_df: DataFrame with columns [timestamp_sec, bandwidth_bytes] + mean_latency_df: DataFrame with columns [timestamp_sec, latency_ms] + max_latency_df: DataFrame with columns [timestamp_sec, latency_ms] + p50_latency_df: DataFrame with columns [timestamp_sec, latency_ms] + p95_latency_df: DataFrame with columns [timestamp_sec, latency_ms] + p99_latency_df: DataFrame with columns [timestamp_sec, latency_ms] + num_volumes: Number of volumes in the test + log_avg_msec: FIO log averaging interval in milliseconds + """ + self._directory = archive_directory + self._benchmark = benchmark + self._operation = operation + self._blocksize = blocksize + self._numjobs = numjobs + self._iodepth = iodepth + self._formatted_output: Optional[TimeSeriesFormatType] = None + + # Store input data directly + self._dataframes = { + "iops": iops_df, + "bandwidth": bandwidth_df, + "mean_latency": mean_latency_df, + "max_latency": max_latency_df, + "p50_latency": p50_latency_df, + "p95_latency": p95_latency_df, + "p99_latency": p99_latency_df, + } + self._num_volumes = num_volumes + self._log_avg_msec = log_avg_msec + + def process(self) -> None: + """ + Process input data and convert to intermediate format. + + This method processes the stored input data and converts it to + the TimeSeriesFormatType format. Results are stored internally + for aggregation at RunResult level (needed for RBDFIO with multiple volumes). + """ + self._formatted_output = self._format_time_series( + dataframes=self._dataframes, + num_volumes=self._num_volumes, + log_avg_msec=self._log_avg_msec, + ) + + def get_formatted_output(self) -> Optional[TimeSeriesFormatType]: + """ + Get the formatted time-series output. + + Returns: + The formatted time-series data, or None if process() hasn't been called + or if no valid data was available. + """ + return self._formatted_output + + def _format_time_series( + self, + dataframes: Mapping[str, Optional[pd.DataFrame]], + num_volumes: int = 1, + log_avg_msec: int = 1000, + ) -> Optional[TimeSeriesFormatType]: + """ + Format aligned time-series data into TimeSeriesFormatType. + + Args: + dataframes: Dictionary with keys: 'iops', 'bandwidth', 'mean_latency', + 'max_latency', 'p50_latency', 'p95_latency', 'p99_latency'. + Each value is an optional DataFrame with appropriate columns. + num_volumes: Number of volumes in the test + log_avg_msec: FIO log averaging interval in milliseconds + + Returns: + TimeSeriesFormatType with all metrics, or None if no valid data + """ + # Merge all DataFrames on timestamp + merged_df = self._merge_dataframes(dataframes) + + if merged_df is None or merged_df.empty: + log.warning( + "No valid time-series data to format for %s %s %s", + self._benchmark, + self._operation, + self._blocksize, + ) + return None + + # Create metadata + metadata = self._create_metadata(merged_df, num_volumes, log_avg_msec) + + # Convert DataFrame rows to TimeSeriesDataPoint list + timeseries = self._create_timeseries_points(merged_df) + + # Calculate maximum values for report generation + maximum_values = self._calculate_maximum_values(timeseries) + + result: TimeSeriesFormatType = { + "benchmark": self._benchmark, + "operation": self._operation, + "blocksize": self._blocksize, + "numjobs": self._numjobs, + "iodepth": self._iodepth, + "metadata": metadata, + "timeseries": timeseries, + "maximum_iops": maximum_values["maximum_iops"], + "maximum_bandwidth": maximum_values["maximum_bandwidth"], + "latency_at_max_iops": maximum_values["latency_at_max_iops"], + "latency_at_max_bandwidth": maximum_values["latency_at_max_bandwidth"], + "timestamp_at_max_iops": maximum_values["timestamp_at_max_iops"], + "timestamp_at_max_bandwidth": maximum_values["timestamp_at_max_bandwidth"], + "maximum_latency": maximum_values["maximum_latency"], + "timestamp_at_max_latency": maximum_values["timestamp_at_max_latency"], + "maximum_cpu_usage": maximum_values["maximum_cpu_usage"], + "maximum_memory_usage": maximum_values["maximum_memory_usage"], + } + + log.info( + "Formatted %d time-series data points for %s %s %s", + len(timeseries), + self._benchmark, + self._operation, + self._blocksize, + ) + + return result + + def _merge_dataframes( # pylint: disable=too-many-branches + self, + dataframes: Mapping[str, Optional[pd.DataFrame]], + ) -> Optional[pd.DataFrame]: + """ + Merge all metric DataFrames on timestamp. + + Args: + dataframes: Dictionary with keys: 'iops', 'bandwidth', 'mean_latency', + 'max_latency', 'p50_latency', 'p95_latency', 'p99_latency' + + Returns: + Merged DataFrame with all metrics, or None if no data + """ + # Early validation + if not dataframes: + log.error("Empty dataframes dictionary provided") + return None + + # Metric configuration: (dict_key, source_column, target_column) + metric_configs = [ + ("iops", "iops", "iops"), + ("bandwidth", "bandwidth_bytes", "bandwidth_bytes"), + ("mean_latency", "latency_ms", "mean_latency_ms"), + ("max_latency", "latency_ms", "max_latency_ms"), + ("p50_latency", "latency_ms", "p50_latency_ms"), + ("p95_latency", "latency_ms", "p95_latency_ms"), + ("p99_latency", "latency_ms", "p99_latency_ms"), + ] + + # Collect valid DataFrames with column validation + valid_dataframes: list[tuple[str, pd.DataFrame, str, str]] = [] + + for key, source_column, target_column in metric_configs: + dataframe = dataframes.get(key) + if dataframe is not None and not dataframe.empty: + # Validate required columns exist + if "timestamp_sec" not in dataframe.columns or source_column not in dataframe.columns: + log.warning( + "DataFrame for '%s' missing required columns (timestamp_sec, %s), skipping", + key, + source_column, + ) + continue + valid_dataframes.append((key, dataframe, source_column, target_column)) + + if not valid_dataframes: + log.error("No valid data in any DataFrame") + return None + + # Initialize with first valid DataFrame + key, dataframe, source_column, target_column = valid_dataframes[0] + + # Aggregate across directions if direction column exists + if "direction" in dataframe.columns: + base_dataframe = dataframe.groupby("timestamp_sec", as_index=False)[[source_column]].sum() + log.debug("Aggregated '%s' across directions: %d rows", key, len(base_dataframe)) + else: + base_dataframe = dataframe[["timestamp_sec", source_column]].copy() + + if source_column != target_column: + base_dataframe.rename(columns={source_column: target_column}, inplace=True) + + log.debug("Starting merge with '%s' as base (%d rows)", key, len(base_dataframe)) + + # Merge remaining DataFrames + # Strategy: Use outer join but track which metrics have data at each timestamp + for key, dataframe, source_column, target_column in valid_dataframes[1:]: + try: + # Aggregate across directions if direction column exists + if "direction" in dataframe.columns: + temp_df = dataframe.groupby("timestamp_sec", as_index=False)[[source_column]].sum() + log.debug("Aggregated '%s' across directions: %d rows", key, len(temp_df)) + else: + temp_df = dataframe[["timestamp_sec", source_column]].copy() + + if source_column != target_column: + temp_df.rename(columns={source_column: target_column}, inplace=True) + + # Use outer join to preserve all timestamps + base_dataframe = base_dataframe.merge(temp_df, on="timestamp_sec", how="outer", validate="one_to_one") + + log.debug( + "Merged '%s' (%d rows), result now has %d rows", + key, + len(temp_df), + len(base_dataframe), + ) + except pd.errors.MergeError as merge_error: + log.error( + "Failed to merge DataFrame for '%s': %s. Skipping this metric.", + key, + str(merge_error), + ) + continue + + # Sort by timestamp for chronological order + base_dataframe.sort_values("timestamp_sec", inplace=True) + base_dataframe.reset_index(drop=True, inplace=True) + + # Identify throughput columns (IOPS and bandwidth) - these are the primary metrics + metric_columns = [column for column in base_dataframe.columns if column != "timestamp_sec"] + throughput_columns = [column for column in metric_columns if column in ["iops", "bandwidth_bytes"]] + + if metric_columns: + # Strategy: Remove rows where throughput metrics (IOPS/bandwidth) are missing or zero + # This prevents artificial zero spikes in plots caused by timestamp misalignment + # between different log files (e.g., _iops.log vs _clat.log) + + if throughput_columns: + # Drop rows where ALL throughput columns are NaN + base_dataframe.dropna(subset=throughput_columns, how="all", inplace=True) + + # Fill remaining NaN values in throughput columns with 0.0 + base_dataframe[throughput_columns] = base_dataframe[throughput_columns].fillna(0.0) + + # Now drop rows where ALL throughput columns are actually zero + # These are timestamps from latency logs that don't have corresponding IOPS/bandwidth data + throughput_mask = (base_dataframe[throughput_columns] != 0).any(axis=1) + base_dataframe = base_dataframe[throughput_mask].copy() + + log.debug( + "Removed rows with zero/missing throughput, %d rows remain", + len(base_dataframe), + ) + + # For remaining rows, fill any NaN values in other metrics with 0.0 + base_dataframe[metric_columns] = base_dataframe[metric_columns].fillna(0.0) + + base_dataframe.reset_index(drop=True, inplace=True) + + log.info( + "Successfully merged %d DataFrames into %d rows with %d metrics (filtered zero throughput)", + len(valid_dataframes), + len(base_dataframe), + len(metric_columns), + ) + + return base_dataframe + + def _create_metadata(self, df: pd.DataFrame, num_volumes: int, log_avg_msec: int) -> TimeSeriesMetadata: + """ + Create metadata from the DataFrame. + + Args: + df: Merged DataFrame with timestamp_sec column + num_volumes: Number of volumes in test + log_avg_msec: FIO log averaging interval + + Returns: + TimeSeriesMetadata with test information + """ + start_time = float(df["timestamp_sec"].min()) + end_time = float(df["timestamp_sec"].max()) + duration = end_time - start_time + + metadata: TimeSeriesMetadata = { + "start_time_epoch": start_time, + "end_time_epoch": end_time, + "duration_seconds": duration, + "num_volumes": num_volumes, + "sampling_interval_ms": log_avg_msec, + "log_avg_msec": log_avg_msec, + } + + return metadata + + def _create_timeseries_points(self, df: pd.DataFrame) -> list[TimeSeriesDataPoint]: + """ + Convert DataFrame rows to TimeSeriesDataPoint list. + + Args: + df: Merged DataFrame with all metrics + + Returns: + List of TimeSeriesDataPoint dictionaries + """ + timeseries: list[TimeSeriesDataPoint] = [] + + for _, row in df.iterrows(): + row_dict = cast(dict[str, Union[float, int]], row.to_dict()) + point: TimeSeriesDataPoint = { + "timestamp_sec": float(row_dict.get("timestamp_sec", 0.0)), + "iops": float(row_dict.get("iops", 0.0)), + "bandwidth_bytes": float(row_dict.get("bandwidth_bytes", 0.0)), + "mean_latency_ms": float(row_dict.get("mean_latency_ms", 0.0)), + "max_latency_ms": float(row_dict.get("max_latency_ms", 0.0)), + "p50_latency_ms": float(row_dict.get("p50_latency_ms", 0.0)), + "p95_latency_ms": float(row_dict.get("p95_latency_ms", 0.0)), + "p99_latency_ms": float(row_dict.get("p99_latency_ms", 0.0)), + "num_samples": int(row_dict.get("num_samples", 1)), + } + timeseries.append(point) + + return timeseries + + def _calculate_maximum_values(self, timeseries: list[TimeSeriesDataPoint]) -> dict[str, str]: + """ + Calculate maximum values from time-series data points. + + This follows the same pattern as the hockey-stick intermediate format, + pre-calculating maximum values for efficient report generation. + + Args: + timeseries: List of time-series data points + + Returns: + Dictionary with maximum values as strings, including timestamps + """ + if not timeseries: + return { + "maximum_iops": "0", + "maximum_bandwidth": "0", + "latency_at_max_iops": "0.0", + "latency_at_max_bandwidth": "0.0", + "timestamp_at_max_iops": "0.0", + "timestamp_at_max_bandwidth": "0.0", + "maximum_latency": "0.0", + "timestamp_at_max_latency": "0.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + } + + # Find maximum IOPS and corresponding latency and timestamp + maximum_iops_point = max(timeseries, key=lambda p: p["iops"]) + maximum_iops = maximum_iops_point["iops"] + latency_at_max_iops = maximum_iops_point["mean_latency_ms"] + timestamp_at_max_iops = maximum_iops_point["timestamp_sec"] + + # Find maximum bandwidth and corresponding latency and timestamp + maximum_bandwidth_point = max(timeseries, key=lambda p: p["bandwidth_bytes"]) + maximum_bandwidth = maximum_bandwidth_point["bandwidth_bytes"] + latency_at_max_bandwidth = maximum_bandwidth_point["mean_latency_ms"] + timestamp_at_max_bandwidth = maximum_bandwidth_point["timestamp_sec"] + + # Find maximum latency and its timestamp + # Use max_latency_ms which represents the maximum latency observed in each time window + maximum_latency_point = max(timeseries, key=lambda p: p["max_latency_ms"]) + maximum_latency = maximum_latency_point["max_latency_ms"] + timestamp_at_max_latency = maximum_latency_point["timestamp_sec"] + + # CPU and memory are not yet available in time-series data + maximum_cpu_usage = 0.0 + maximum_memory_usage = 0.0 + + return { + "maximum_iops": f"{maximum_iops:.0f}", + "maximum_bandwidth": f"{maximum_bandwidth:.0f}", + "latency_at_max_iops": f"{latency_at_max_iops:.6f}", + "latency_at_max_bandwidth": f"{latency_at_max_bandwidth:.6f}", + "timestamp_at_max_iops": f"{timestamp_at_max_iops:.1f}", + "timestamp_at_max_bandwidth": f"{timestamp_at_max_bandwidth:.1f}", + "maximum_latency": f"{maximum_latency:.6f}", + "timestamp_at_max_latency": f"{timestamp_at_max_latency:.1f}", + "maximum_cpu_usage": f"{maximum_cpu_usage:.2f}", + "maximum_memory_usage": f"{maximum_memory_usage:.2f}", + } + + +# Made with Bob diff --git a/post_processing/parsers/timestamp_aligner.py b/post_processing/parsers/timestamp_aligner.py new file mode 100644 index 00000000..bd54dd8b --- /dev/null +++ b/post_processing/parsers/timestamp_aligner.py @@ -0,0 +1,187 @@ +""" +Timestamp alignment and aggregation for time-series data. + +This module provides functionality to align timestamps from multiple volumes +using time-window binning and aggregate metrics appropriately. +""" + +from logging import Logger, getLogger +from typing import Literal, Optional + +import numpy as np +import pandas as pd +from pandas.core.frame import DataFrame + +log: Logger = getLogger("parser") + + +class TimestampAligner: + """ + Align and aggregate time-series data from multiple volumes. + + When testing multiple volumes, timestamps may not align exactly due to + slight timing differences in when each volume starts logging. This class + uses time-window binning to group data points into fixed time intervals + and aggregate them appropriately. + + For example, with a 1-second window: + - All data points between 0.0-1.0 seconds are grouped together + - IOPS values are summed across volumes + - Latency values are averaged or max'd as appropriate + """ + + def __init__(self, window_size_ms: int = 1000) -> None: + """ + Initialize the timestamp aligner. + + Args: + window_size_sec: Size of time windows in seconds (default: 1000ms = 1s) + """ + self._window_size_sec = window_size_ms / 1000.0 + log.debug("Initialized TimestampAligner with window size %dms", window_size_ms) + + def align_and_aggregate( + self, + dataframes: list[Optional[pd.DataFrame]], + metric_column: str, + aggregation: Literal["sum", "mean", "max"] = "sum", + ) -> pd.DataFrame: + """ + Align timestamps using time windows and aggregate metric values. + + Args: + dataframes: List of DataFrames from different volumes, each with + columns [timestamp_sec, , ...] + metric_column: Name of the column to aggregate (e.g., 'iops', 'latency_ms') + aggregation: How to aggregate values: + - 'sum': Add values (use for IOPS, bandwidth) + - 'mean': Average values (use for mean latency) + - 'max': Take maximum (use for max latency) + + Returns: + DataFrame with columns [timestamp_sec, , num_samples] + where timestamp_sec is the start of each time window + """ + # Validate aggregation method early + valid_aggregations = {"sum", "mean", "max"} + if aggregation not in valid_aggregations: + raise ValueError(f"Unknown aggregation method: {aggregation}. Must be one of {valid_aggregations}") + + if not dataframes: + log.warning("No dataframes provided for alignment") + return pd.DataFrame(columns=["timestamp_sec", metric_column, "num_samples"]) + + # Filter out None dataframes + valid_dfs: list[DataFrame] = [df for df in dataframes if df is not None and not df.empty] + if not valid_dfs: + log.warning("All dataframes are None or empty") + return pd.DataFrame(columns=["timestamp_sec", metric_column, "num_samples"]) + + # Combine all dataframes + combined = pd.concat(valid_dfs, ignore_index=True) + log.debug("Combined %d dataframes into %d total samples", len(valid_dfs), len(combined)) + + # Create time windows using vectorized floor division + combined["time_window"] = (combined["timestamp_sec"] // self._window_size_sec) * self._window_size_sec + + # Aggregate by time window and count samples in single operation + grouped = combined.groupby("time_window") + result = grouped[metric_column].agg(aggregation) + counts = grouped.size() + + # Build result dataframe + result_df = pd.DataFrame( + {"timestamp_sec": result.index, metric_column: result.values, "num_samples": counts.values} + ).reset_index(drop=True) + + log.debug("Aggregated into %d time windows using %s", len(result_df), aggregation) + return result_df + + def calculate_percentiles( + self, dataframes: list[pd.DataFrame], percentiles: Optional[list[float]] = None + ) -> pd.DataFrame: + """ + Calculate latency percentiles per time window. + + Args: + dataframes: List of DataFrames with latency data, each with + columns [timestamp_sec, latency_ms, ...] + percentiles: List of percentiles to calculate (e.g., [50, 95, 99]) + Defaults to [50, 95, 99] if not provided + + Returns: + DataFrame with columns [timestamp_sec, p50_latency_ms, p95_latency_ms, ...] + """ + if percentiles is None: + percentiles = [50, 95, 99] + + if not dataframes: + log.warning("No dataframes provided for percentile calculation") + return pd.DataFrame(columns=["timestamp_sec"]) + + # Filter out None dataframes + valid_dfs = [df for df in dataframes if df is not None and not df.empty] + if not valid_dfs: + log.warning("All dataframes are None or empty") + return pd.DataFrame(columns=["timestamp_sec"]) + + # Combine all dataframes + combined = pd.concat(valid_dfs, ignore_index=True) + + # Create time windows using vectorized floor division (consistent with align_and_aggregate) + combined["time_window"] = (combined["timestamp_sec"] // self._window_size_sec) * self._window_size_sec + + # Calculate percentiles per window + quantiles = [p / 100.0 for p in percentiles] + result = ( + combined.groupby("time_window")["latency_ms"] + .quantile(quantiles) # type: ignore[arg-type] + .unstack(fill_value=np.nan) # type: ignore[arg-type] + ) + + # Rename columns and reset index with timestamp + result.columns = [f"p{int(q * 100)}_latency_ms" for q in result.columns] + result_df = result.reset_index(names="timestamp_sec") + + log.debug("Calculated percentiles %s for %d time windows", percentiles, len(result_df)) + return result_df + + def merge_metrics(self, *dataframes: pd.DataFrame) -> pd.DataFrame: + """ + Merge multiple metric dataframes on timestamp_sec. + + All dataframes should have a 'timestamp_sec' column. This method + performs an outer join to include all timestamps from all dataframes. + + Args: + *dataframes: Variable number of DataFrames to merge + + Returns: + Merged DataFrame with all metrics aligned by timestamp + """ + if not dataframes: + return pd.DataFrame() + + # Filter out None and empty dataframes + valid_dfs = [df for df in dataframes if df is not None and not df.empty] + if not valid_dfs: + return pd.DataFrame() + + # Start with the first dataframe + result = valid_dfs[0].copy() + + # Merge each subsequent dataframe + for df in valid_dfs[1:]: + result = pd.merge(result, df, on="timestamp_sec", how="outer", suffixes=("", "_dup")) + + # Remove duplicate columns (keep the first occurrence) + result = result.loc[:, ~result.columns.str.endswith("_dup")] + + # Sort by timestamp + result = result.sort_values("timestamp_sec").reset_index(drop=True) + + log.debug("Merged %d dataframes into %d rows", len(valid_dfs), len(result)) + return result + + +# Made with Bob diff --git a/post_processing/plotter/common_format_plotter.py b/post_processing/plotter/common_format_plotter.py index 445e77d0..ddbf8f4b 100644 --- a/post_processing/plotter/common_format_plotter.py +++ b/post_processing/plotter/common_format_plotter.py @@ -9,13 +9,14 @@ # the ModuleType does exists in the types module, so no idea why pylint is # flagging this -from types import ModuleType # pylint: disable=[no-name-in-module] +from types import ModuleType from typing import Optional from matplotlib.axes import Axes from post_processing.common import ( DATA_FILE_EXTENSION_WITH_DOT, + KB_CONVERSION_FACTOR, PLOT_FILE_EXTENSION, get_blocksize_percentage_operation_numjobs_from_file_name, ) @@ -32,7 +33,6 @@ BYTES_TO_MB_DIVISOR = 1024 * 1024 # Using 1024 for MiB NANOSECONDS_TO_MS_DIVISOR = 1_000_000 ERROR_BAR_CAP_SIZE = 3 -KB_CONVERSION_FACTOR = 1024 # pylint: disable=too-few-public-methods, too-many-locals @@ -41,7 +41,7 @@ class CommonFormatPlotter(ABC): The base class for plotting results curves """ - def __init__(self, plotter: ModuleType) -> None: + def __init__(self, plotter: ModuleType): self._plotter = plotter @abstractmethod diff --git a/post_processing/plotter/cpu_plotter.py b/post_processing/plotter/cpu_plotter.py index 37d95f09..56b3fcfe 100644 --- a/post_processing/plotter/cpu_plotter.py +++ b/post_processing/plotter/cpu_plotter.py @@ -9,7 +9,7 @@ log: Logger = getLogger("plotter") -CPU_PLOT_DEFAULT_COLOUR: str = "#5ca904" +CPU_PLOT_DEFAULT_COLOUR: str = "xkcd:leaf green" # Leaf green from xkcd color survey CPU_Y_LABEL: str = "System CPU use (%)" CPU_PLOT_LABEL: str = "CPU use" diff --git a/post_processing/plotter/directory_comparison_plotter.py b/post_processing/plotter/directory_comparison_plotter.py index cc99e8fd..d51ef7db 100644 --- a/post_processing/plotter/directory_comparison_plotter.py +++ b/post_processing/plotter/directory_comparison_plotter.py @@ -14,6 +14,7 @@ from post_processing.common import ( PLOT_FILE_EXTENSION_WITH_DOT, find_common_data_file_names, + find_hockey_stick_visualisation_directories, read_intermediate_file, ) from post_processing.plotter.common_format_plotter import CommonFormatPlotter @@ -32,46 +33,78 @@ class DirectoryComparisonPlotter(CommonFormatPlotter): def __init__(self, output_directory: str, directories: list[str]) -> None: self._output_directory: str = f"{output_directory}" - self._comparison_directories: list[Path] = [Path(f"{directory}/visualisation") for directory in directories] + # Store the archive directories - we'll find visualisation dirs per operation + self._archive_directories: list[Path] = [Path(d) for d in directories] super().__init__(plotter) def draw_and_save(self) -> None: - # output_file_path: str = self._generate_output_file_name(files=self._comparison_directories) - - # We will only compare data for files with the same name, so find all - # the file names that are common across all directories. Not sure this - # is the right way though - common_file_names: list[str] = find_common_data_file_names(self._comparison_directories) - - for file_name in common_file_names: - output_file_path: str = self._generate_output_file_name(files=[Path(file_name)]) - - figure: Figure - io_axis: Axes - figure, io_axis = self._plotter.subplots() - - for directory in self._comparison_directories: - file_data: CommonFormatDataType = read_intermediate_file(f"{directory}/{file_name}") - # we choose the last directory name for the label to apply to the data - # figure = self._add_single_file_data_with_optional_errorbars( - self._add_single_file_data_with_optional_errorbars( - file_data=file_data, - main_axes=io_axis, - label=f"{directory.parts[-2]}", - plot_error_bars=False, - plot_resource_usage=False, + # For each archive directory, find all visualisation directories + # Group them by operation type (e.g., all 'randread/visualisation' together) + operation_vis_dirs: dict[str, list[Path]] = {} + + for archive_dir in self._archive_directories: + vis_dirs = find_hockey_stick_visualisation_directories(archive_dir) + for vis_dir in vis_dirs: + # Extract operation name (parent directory of visualisation) + operation = vis_dir.parent.name if vis_dir.parent != archive_dir else "legacy" + if operation not in operation_vis_dirs: + operation_vis_dirs[operation] = [] + operation_vis_dirs[operation].append(vis_dir) + + # For each operation type, find common files and create comparison plots + for operation, vis_directories in operation_vis_dirs.items(): + # Only create plots if we have data from multiple archives for this operation + if len(vis_directories) < len(self._archive_directories): + log.warning( + "Skipping operation '%s' - not present in all archives (found in %d of %d)", + operation, + len(vis_directories), + len(self._archive_directories), + ) + continue + + # Find files common to all archives for this operation + common_file_names: list[str] = find_common_data_file_names(vis_directories) + + if not common_file_names: + log.warning("No common files found for operation '%s'", operation) + continue + + for file_name in common_file_names: + output_file_path: str = self._generate_output_file_name(files=[Path(file_name)]) + + figure: Figure + io_axis: Axes + figure, io_axis = self._plotter.subplots() + + for vis_dir in vis_directories: + file_path = vis_dir / file_name + if not file_path.exists(): + continue + + file_data: CommonFormatDataType = read_intermediate_file(f"{file_path}") + # Use the archive directory name (2 levels up from visualisation) for the label + archive_name = ( + vis_dir.parent.parent.name if vis_dir.parent.name != "visualisation" else vis_dir.parent.name + ) + self._add_single_file_data_with_optional_errorbars( + file_data=file_data, + main_axes=io_axis, + label=archive_name, + plot_error_bars=False, + plot_resource_usage=False, + ) + + # make sure we add the legend to the plot + figure.legend( # pyright: ignore[reportUnknownMemberType, reportPossiblyUnboundVariable] + bbox_to_anchor=(0.5, -0.1), loc="upper center", ncol=2 ) - # make sure we add the legend to the plot - figure.legend( # pyright: ignore[reportUnknownMemberType, reportPossiblyUnboundVariable] - bbox_to_anchor=(0.5, -0.1), loc="upper center", ncol=2 - ) - - self._add_title(source_files=[Path(file_name)]) - self._set_axis() + self._add_title(source_files=[Path(file_name)]) + self._set_axis() - self._save_plot(file_path=output_file_path) - self._clear_plot() + self._save_plot(file_path=output_file_path) + self._clear_plot() def _generate_output_file_name(self, files: list[Path]) -> str: # we know we will only ever be passed a single file name diff --git a/post_processing/plotter/io_plotter.py b/post_processing/plotter/io_plotter.py index 10ba92f0..f3b976fa 100644 --- a/post_processing/plotter/io_plotter.py +++ b/post_processing/plotter/io_plotter.py @@ -9,7 +9,7 @@ log: Logger = getLogger("plotter") -IO_PLOT_DEFAULT_COLOUR: str = "#5ca904" +IO_PLOT_DEFAULT_COLOUR: str = "xkcd:leaf green" # Leaf green from xkcd color survey IO_Y_LABEL: str = "Latency (ms)" IO_PLOT_LABEL: str = "IO Details" diff --git a/post_processing/plotter/simple_plotter.py b/post_processing/plotter/simple_plotter.py index 9dc2bf01..5a202460 100644 --- a/post_processing/plotter/simple_plotter.py +++ b/post_processing/plotter/simple_plotter.py @@ -11,7 +11,6 @@ from matplotlib.figure import Figure from post_processing.common import ( - DATA_FILE_EXTENSION, DATA_FILE_EXTENSION_WITH_DOT, PLOT_FILE_EXTENSION, read_intermediate_file, @@ -28,14 +27,25 @@ class SimplePlotter(CommonFormatPlotter): """ def __init__(self, archive_directory: str, plot_error_bars: bool, plot_resources: bool) -> None: - # A Path object for the directory where the data files are stored - self._path: Path = Path(f"{archive_directory}/visualisation") + # Archive directory is the root directory for the test run + self._archive_directory: Path = Path(archive_directory) + # SVG files are saved to top-level visualisation directory for easy access by report generator + self._svg_output_path: Path = self._archive_directory / "visualisation" + self._svg_output_path.mkdir(parents=True, exist_ok=True) self._plot_error_bars: bool = plot_error_bars self._plot_resources: bool = plot_resources super().__init__(plotter) def draw_and_save(self) -> None: - for file_path in self._path.glob(f"*{DATA_FILE_EXTENSION_WITH_DOT}"): + # Search recursively for JSON data files in all visualisation directories + # This supports both legacy (archive/visualisation/) and new (archive/operation/visualisation/) structures + json_files = list(self._archive_directory.glob(f"**/visualisation/*{DATA_FILE_EXTENSION_WITH_DOT}")) + + for file_path in json_files: + # Skip timeseries JSON files (they have their own plotter) + if "_timeseries" in file_path.stem: + continue + file_data: CommonFormatDataType = read_intermediate_file(f"{file_path}") output_file_path: str = self._generate_output_file_name(files=[file_path]) @@ -63,5 +73,16 @@ def draw_and_save(self) -> None: self._clear_plot() def _generate_output_file_name(self, files: list[Path]) -> str: - # we know we will only ever be passed a single file name - return f"{str(files[0])[: -len(DATA_FILE_EXTENSION)]}{PLOT_FILE_EXTENSION}" + """ + Generate output filename for SVG plot. + + SVG files are saved to the top-level visualisation directory + (archive_directory/visualisation/) regardless of where the JSON + data file is located. This ensures all plots are in one place + for the report generator to find them. + """ + # Extract just the filename without path and extension + json_filename = files[0].stem # e.g., "4k_1_randread" + svg_filename = f"{json_filename}.{PLOT_FILE_EXTENSION}" + # Save to top-level visualisation directory + return str(self._svg_output_path / svg_filename) diff --git a/post_processing/plotter/time_series_bandwidth_plotter.py b/post_processing/plotter/time_series_bandwidth_plotter.py new file mode 100644 index 00000000..40c11d0a --- /dev/null +++ b/post_processing/plotter/time_series_bandwidth_plotter.py @@ -0,0 +1,54 @@ +""" +A file containing the TimeSeriesBandwidthPlotter class for plotting bandwidth over time. +""" + +from logging import Logger, getLogger + +from post_processing.plotter.time_series_metric_plotter import TimeSeriesMetricPlotter + +log: Logger = getLogger("plotter") + +BANDWIDTH_PLOT_DEFAULT_COLOUR: str = "xkcd:purple" # Purple from xkcd color survey +BANDWIDTH_Y_LABEL: str = "Bandwidth (MB/s)" +BANDWIDTH_PLOT_LABEL: str = "Bandwidth" +BYTES_TO_MB_DIVISOR: int = 1024 * 1024 + + +class TimeSeriesBandwidthPlotter(TimeSeriesMetricPlotter): + """ + A class to plot bandwidth data over time on a time-series plot. + Converts bandwidth from bytes to MB/s. + """ + + def _get_default_color(self) -> str: + """ + Return the default color for bandwidth plots. + """ + return BANDWIDTH_PLOT_DEFAULT_COLOUR + + def _get_y_label(self) -> str: + """ + Return the y-axis label for bandwidth plots. + """ + return BANDWIDTH_Y_LABEL + + def _get_plot_label(self) -> str: + """ + Return the plot label for bandwidth plots. + """ + return BANDWIDTH_PLOT_LABEL + + def _convert_value(self, value: float) -> float: + """ + Convert bandwidth from bytes to MB/s. + + Args: + value: Bandwidth value in bytes + + Returns: + Bandwidth value in MB/s + """ + return value / BYTES_TO_MB_DIVISOR + + +# Made with Bob diff --git a/post_processing/plotter/time_series_iops_plotter.py b/post_processing/plotter/time_series_iops_plotter.py new file mode 100644 index 00000000..887e5e27 --- /dev/null +++ b/post_processing/plotter/time_series_iops_plotter.py @@ -0,0 +1,34 @@ +""" +A file containing the TimeSeriesIOPSPlotter class for plotting IOPS over time. +""" + +from logging import Logger, getLogger + +from post_processing.plotter.time_series_metric_plotter import TimeSeriesMetricPlotter + +log: Logger = getLogger("plotter") + +IOPS_PLOT_DEFAULT_COLOUR: str = "xkcd:blue" # Blue from xkcd color survey +IOPS_Y_LABEL: str = "IOPS (ops/s)" +IOPS_PLOT_LABEL: str = "IOPS" + + +class TimeSeriesIOPSPlotter(TimeSeriesMetricPlotter): + """ + A class to plot IOPS data over time on a time-series plot. + """ + + def _get_default_color(self) -> str: + """Return the default color for IOPS plots.""" + return IOPS_PLOT_DEFAULT_COLOUR + + def _get_y_label(self) -> str: + """Return the y-axis label for IOPS plots.""" + return IOPS_Y_LABEL + + def _get_plot_label(self) -> str: + """Return the plot label for IOPS plots.""" + return IOPS_PLOT_LABEL + + +# Made with Bob diff --git a/post_processing/plotter/time_series_latency_plotter.py b/post_processing/plotter/time_series_latency_plotter.py new file mode 100644 index 00000000..e3a0773d --- /dev/null +++ b/post_processing/plotter/time_series_latency_plotter.py @@ -0,0 +1,192 @@ +""" +A file containing the TimeSeriesLatencyPlotter class for plotting latency over time. +""" + +from logging import Logger, getLogger + +from matplotlib.axes import Axes + +from post_processing.plotter.time_series_metric_plotter import ( + DEFAULT_ALPHA, + DEFAULT_LINE_WIDTH, + TimeSeriesMetricPlotter, +) + +log: Logger = getLogger("plotter") + +LATENCY_MEAN_COLOR: str = "xkcd:orange" # Orange from xkcd color survey +LATENCY_P50_COLOR: str = "xkcd:green" # Green from xkcd color survey +LATENCY_P95_COLOR: str = "xkcd:red" # Red from xkcd color survey +LATENCY_P99_COLOR: str = "xkcd:dark red" # Dark red from xkcd color survey +LATENCY_MAX_COLOR: str = "xkcd:dark grey" # Dark grey from xkcd color survey +LATENCY_Y_LABEL: str = "Latency (ms)" +LATENCY_PLOT_LABEL: str = "Mean Latency" +PERCENTILE_BAND_ALPHA: float = 0.2 + + +class TimeSeriesLatencyPlotter(TimeSeriesMetricPlotter): + """ + A class to plot latency data over time on a time-series plot. + Supports mean, percentile (P50, P95, P99), and max latency values. + """ + + def __init__(self, main_axis: Axes) -> None: + """ + Initialize the TimeSeriesLatencyPlotter with a matplotlib Axes object. + + Args: + main_axis: The main matplotlib Axes object for this plot + """ + super().__init__(main_axis) + self._p50_data: list[float] = [] + self._p95_data: list[float] = [] + self._p99_data: list[float] = [] + self._max_data: list[float] = [] + + def _get_default_color(self) -> str: + """Return the default color for latency plots.""" + return LATENCY_MEAN_COLOR + + def _get_y_label(self) -> str: + """Return the y-axis label for latency plots.""" + return LATENCY_Y_LABEL + + def _get_plot_label(self) -> str: + """Return the plot label for latency plots.""" + return LATENCY_PLOT_LABEL + + def add_p50_data(self, data_value: str) -> None: + """ + Add a point of P50 latency data. + + Args: + data_value: A single P50 latency value in milliseconds as a string. + """ + self._p50_data.append(float(data_value)) + + def add_p95_data(self, data_value: str) -> None: + """ + Add a point of P95 latency data. + + Args: + data_value: A single P95 latency value in milliseconds as a string. + """ + self._p95_data.append(float(data_value)) + + def add_p99_data(self, data_value: str) -> None: + """ + Add a point of P99 latency data. + + Args: + data_value: A single P99 latency value in milliseconds as a string. + """ + self._p99_data.append(float(data_value)) + + def add_max_data(self, data_value: str) -> None: + """ + Add a point of max latency data. + + Args: + data_value: A single max latency value in milliseconds as a string. + """ + self._max_data.append(float(data_value)) + + def plot(self, x_data: list[float], colour: str = "") -> None: + """ + Plot latency data on the main axes with percentile bands. + + Creates a plot showing mean latency with shaded regions for + P50-P95 and P95-P99 percentile ranges, plus lines for specific percentiles. + + Args: + x_data: The data for the x-axis (timestamps) + colour: The colour for the plot line (optional, not used for latency plots) + """ + latency_axis = self._main_axes + self._label = self._get_plot_label() + self._y_label = self._get_y_label() + + latency_axis.set_ylabel(self._y_label) # pyright: ignore[reportUnknownMemberType] + + # Plot percentile bands (from bottom to top) + # P50-P95 band + if self._p50_data and self._p95_data and any(self._p50_data) and any(self._p95_data): + latency_axis.fill_between( # pyright: ignore[reportUnknownMemberType] + x_data, + self._p50_data, + self._p95_data, + color=LATENCY_P50_COLOR, + alpha=PERCENTILE_BAND_ALPHA, + label="P50-P95 Range", + ) + + # P95-P99 band + if self._p95_data and self._p99_data and any(self._p95_data) and any(self._p99_data): + latency_axis.fill_between( # pyright: ignore[reportUnknownMemberType] + x_data, + self._p95_data, + self._p99_data, + color=LATENCY_P95_COLOR, + alpha=PERCENTILE_BAND_ALPHA, + label="P95-P99 Range", + ) + + # Plot lines for specific percentiles + if self._p50_data and any(self._p50_data): + latency_axis.plot( # pyright: ignore[reportUnknownMemberType] + x_data, + self._p50_data, + color=LATENCY_P50_COLOR, + linewidth=DEFAULT_LINE_WIDTH, + alpha=DEFAULT_ALPHA, + label="P50 Latency", + linestyle="--", + ) + + if self._p95_data and any(self._p95_data): + latency_axis.plot( # pyright: ignore[reportUnknownMemberType] + x_data, + self._p95_data, + color=LATENCY_P95_COLOR, + linewidth=DEFAULT_LINE_WIDTH, + alpha=DEFAULT_ALPHA, + label="P95 Latency", + linestyle="--", + ) + + if self._p99_data and any(self._p99_data): + latency_axis.plot( # pyright: ignore[reportUnknownMemberType] + x_data, + self._p99_data, + color=LATENCY_P99_COLOR, + linewidth=DEFAULT_LINE_WIDTH, + alpha=DEFAULT_ALPHA, + label="P99 Latency", + linestyle=":", + ) + + # Plot mean latency as main line + if self._y_data and any(self._y_data): + latency_axis.plot( # pyright: ignore[reportUnknownMemberType] + x_data, + self._y_data, + color=LATENCY_MEAN_COLOR, + linewidth=DEFAULT_LINE_WIDTH + 0.5, + alpha=DEFAULT_ALPHA, + label=self._label, + ) + + # Optionally plot max latency + if self._max_data and any(self._max_data) and max(self._max_data) > 0: + latency_axis.plot( # pyright: ignore[reportUnknownMemberType] + x_data, + self._max_data, + color=LATENCY_MAX_COLOR, + linewidth=DEFAULT_LINE_WIDTH - 0.5, + alpha=0.5, + label="Max Latency", + linestyle=":", + ) + + +# Made with Bob diff --git a/post_processing/plotter/time_series_metric_plotter.py b/post_processing/plotter/time_series_metric_plotter.py new file mode 100644 index 00000000..0c103423 --- /dev/null +++ b/post_processing/plotter/time_series_metric_plotter.py @@ -0,0 +1,110 @@ +""" +Base class for time-series metric plotters. + +This module provides the TimeSeriesMetricPlotter base class that contains +common plotting logic for time-series metrics (bandwidth, IOPS, latency). +Subclasses only need to override specific attributes and conversion logic. +""" + +from abc import ABC, abstractmethod +from logging import Logger, getLogger + +from post_processing.plotter.axis_plotter import AxisPlotter + +log: Logger = getLogger("plotter") + +# Default plotting parameters +DEFAULT_LINE_WIDTH: float = 1.5 +DEFAULT_ALPHA: float = 0.8 + + +class TimeSeriesMetricPlotter(AxisPlotter, ABC): + """ + Base class for plotting time-series metrics. + + Provides common plotting functionality for metrics like bandwidth, IOPS, and latency. + Subclasses should override: + - _get_default_color(): Return the default color for this metric + - _get_y_label(): Return the y-axis label + - _get_plot_label(): Return the plot label for the legend + - _convert_value(value): Convert raw value to display units (optional) + """ + + @abstractmethod + def _get_default_color(self) -> str: + """ + Get the default color for this metric. + + Returns: + Color string (e.g., "xkcd:purple", "#FF0000") + """ + + @abstractmethod + def _get_y_label(self) -> str: + """ + Get the y-axis label for this metric. + + Returns: + Y-axis label string (e.g., "Bandwidth (MB/s)") + """ + + @abstractmethod + def _get_plot_label(self) -> str: + """ + Get the plot label for the legend. + + Returns: + Plot label string (e.g., "Bandwidth") + """ + + def _convert_value(self, value: float) -> float: + """ + Convert raw value to display units. + + Default implementation returns value unchanged. + Override in subclasses for unit conversion (e.g., bytes to MB). + + Args: + value: Raw value from data + + Returns: + Converted value for display + """ + return value + + def add_y_data(self, data_value: str) -> None: + """ + Add a point of data for this plot. + + Args: + data_value: A single value as a string. + Will be converted using _convert_value() internally. + """ + converted_value = self._convert_value(float(data_value)) + self._y_data.append(converted_value) + + def plot(self, x_data: list[float], colour: str = "") -> None: + """ + Plot data on the main axes. + + Args: + x_data: The data for the x-axis (timestamps) + colour: The colour for the plot line (optional, uses default if not provided) + """ + axis = self._main_axes + self._label = self._get_plot_label() + self._y_label = self._get_y_label() + plot_colour = colour if colour else self._get_default_color() + + axis.set_ylabel(self._y_label) # pyright: ignore[reportUnknownMemberType] + axis.plot( # pyright: ignore[reportUnknownMemberType] + x_data, + self._y_data, + color=plot_colour, + linewidth=DEFAULT_LINE_WIDTH, + alpha=DEFAULT_ALPHA, + label=self._label, + ) + + +# Made with Bob diff --git a/post_processing/plotter/time_series_plotter.py b/post_processing/plotter/time_series_plotter.py new file mode 100644 index 00000000..f0215cca --- /dev/null +++ b/post_processing/plotter/time_series_plotter.py @@ -0,0 +1,421 @@ +""" +Time-series plotter for FIO benchmark results. + +This module provides the TimeSeriesPlotter class which generates time-series +plots for IOPS, bandwidth, and latency metrics from FIO benchmark runs. + +Unlike CommonFormatPlotter which handles hockey-stick plots, this plotter +is designed specifically for time-series data visualization. + +This plotter now uses the AxisPlotter pattern for consistency with other +plotters in the codebase, enabling easy addition of resource usage metrics. +""" + +import json +from logging import Logger, getLogger +from pathlib import Path +from types import ModuleType +from typing import Optional + +from matplotlib.axes import Axes +from matplotlib.figure import Figure + +from post_processing.common import KB_CONVERSION_FACTOR, PLOT_FILE_EXTENSION, TITLE_CONVERSION +from post_processing.plotter.cpu_plotter import CPUPlotter +from post_processing.plotter.time_series_bandwidth_plotter import TimeSeriesBandwidthPlotter +from post_processing.plotter.time_series_iops_plotter import TimeSeriesIOPSPlotter +from post_processing.plotter.time_series_latency_plotter import TimeSeriesLatencyPlotter +from post_processing.post_processing_types import TimeSeriesDataPoint, TimeSeriesFormatType + +log: Logger = getLogger("plotter") + +# Plot styling constants +DEFAULT_FIGURE_SIZE = (12, 3) +DEFAULT_DPI = 100 + + +class TimeSeriesPlotter: + """ + Generate time-series plots from FIO benchmark data. + + This class creates line plots showing how performance metrics change + over time during a benchmark run. It supports: + - IOPS over time + - Bandwidth over time + - Latency over time (with percentile bands) + - Optional CPU usage overlay on any metric + + Unlike CommonFormatPlotter which is designed for hockey-stick plots, + this plotter works with TimeSeriesFormatType data and uses the + AxisPlotter pattern for consistency and extensibility. + """ + + def __init__( + self, + archive_directory: str, + plotter: ModuleType, + figure_size: tuple[int, int] = DEFAULT_FIGURE_SIZE, + dpi: int = DEFAULT_DPI, + ) -> None: + """ + Initialize the time-series plotter. + + Args: + archive_directory: Directory containing benchmark results + plotter: Matplotlib pyplot module for plotting + figure_size: Figure size in inches (width, height) + dpi: Dots per inch for plot resolution + """ + self._archive_directory = Path(archive_directory) + self._plotter = plotter + self._figure_size = figure_size + self._dpi = dpi + self._output_dir = self._archive_directory / "visualisation" + + def draw_and_save(self) -> None: + """ + Generate plots for all time-series JSON files in the archive directory. + + This method follows the same pattern as other plotters in the codebase, + scanning for time-series data files and generating plots for each. + """ + self.plot_all_in_directory() + + def plot_time_series(self, data: TimeSeriesFormatType, plot_cpu: bool = False) -> None: + """ + Generate all time-series plots for the given data. + + Creates separate plots for IOPS, bandwidth, and latency metrics. + Optionally overlays CPU usage on each plot. + + Args: + data: Time-series data in TimeSeriesFormatType format + plot_cpu: Whether to overlay CPU usage on the plots + """ + if not data.get("timeseries"): + log.warning("No time-series data to plot") + return + + # Ensure output directory exists + self._output_dir.mkdir(parents=True, exist_ok=True) + + # Generate individual plots + self._plot_iops(data, plot_cpu) + self._plot_bandwidth(data, plot_cpu) + self._plot_latency(data, plot_cpu) + + log.debug( + "Generated time-series plots for %s %s %s", + data["benchmark"], + data["operation"], + data["blocksize"], + ) + + def _plot_iops(self, data: TimeSeriesFormatType, plot_cpu: bool = False) -> None: + """ + Plot IOPS over time using TimeSeriesIOPSPlotter. + + Args: + data: Time-series data containing IOPS measurements + plot_cpu: Whether to overlay CPU usage on the plot + """ + timeseries = data["timeseries"] + timestamps = [point["timestamp_sec"] for point in timeseries] + + # Skip if no IOPS data + iops_values = [point["iops"] for point in timeseries] + if not any(iops_values): + log.debug("No IOPS data to plot") + return + + fig, ax = self._plotter.subplots(figsize=self._figure_size, dpi=self._dpi) + + # Initialize IOPS plotter + iops_plotter = TimeSeriesIOPSPlotter(main_axis=ax) + + # Collect IOPS data + for point in timeseries: + iops_plotter.add_y_data(str(point["iops"])) + + # Plot IOPS + iops_plotter.plot(x_data=timestamps) + + # Optionally add CPU overlay + if plot_cpu and self._has_cpu_data(timeseries): + cpu_plotter = CPUPlotter(main_axis=ax) + for point in timeseries: + cpu_value = point.get("cpu") + if cpu_value is not None: + cpu_plotter.add_y_data(str(cpu_value)) + cpu_plotter.plot(x_data=timestamps) + + self._configure_axes( + ax=ax, + title=self._generate_title(data, "IOPS"), + xlabel="Time (seconds)", + ) + + ax.legend(loc="best") # pyright: ignore[reportUnknownMemberType] + ax.grid(True, alpha=0.3) # pyright: ignore[reportUnknownMemberType] + + output_path = self._generate_output_path(data, "iops") + self._save_plot(fig, output_path) + self._plotter.close(fig) + + def _plot_bandwidth(self, data: TimeSeriesFormatType, plot_cpu: bool = False) -> None: + """ + Plot bandwidth over time using TimeSeriesBandwidthPlotter. + + Args: + data: Time-series data containing bandwidth measurements + plot_cpu: Whether to overlay CPU usage on the plot + """ + timeseries = data["timeseries"] + timestamps = [point["timestamp_sec"] for point in timeseries] + + # Skip if no bandwidth data + bandwidth_values = [point["bandwidth_bytes"] for point in timeseries] + if not any(bandwidth_values): + log.debug("No bandwidth data to plot") + return + + fig, ax = self._plotter.subplots(figsize=self._figure_size, dpi=self._dpi) + + # Initialize bandwidth plotter + bandwidth_plotter = TimeSeriesBandwidthPlotter(main_axis=ax) + + # Collect bandwidth data + for point in timeseries: + bandwidth_plotter.add_y_data(str(point["bandwidth_bytes"])) + + # Plot bandwidth + bandwidth_plotter.plot(x_data=timestamps) + + # Optionally add CPU overlay + if plot_cpu and self._has_cpu_data(timeseries): + cpu_plotter = CPUPlotter(main_axis=ax) + for point in timeseries: + cpu_value = point.get("cpu") + if cpu_value is not None: + cpu_plotter.add_y_data(str(cpu_value)) + cpu_plotter.plot(x_data=timestamps) + + self._configure_axes( + ax=ax, + title=self._generate_title(data, "Bandwidth"), + xlabel="Time (seconds)", + ) + + ax.legend(loc="best") # pyright: ignore[reportUnknownMemberType] + ax.grid(True, alpha=0.3) # pyright: ignore[reportUnknownMemberType] + + output_path = self._generate_output_path(data, "bandwidth") + self._save_plot(fig, output_path) + self._plotter.close(fig) + + def _plot_latency(self, data: TimeSeriesFormatType, plot_cpu: bool = False) -> None: + """ + Plot latency over time with percentile bands using TimeSeriesLatencyPlotter. + + Creates a plot showing mean latency with shaded regions for + P50-P95 and P95-P99 percentile ranges. + + Args: + data: Time-series data containing latency measurements + plot_cpu: Whether to overlay CPU usage on the plot + """ + timeseries = data["timeseries"] + timestamps = [point["timestamp_sec"] for point in timeseries] + + # Skip if no latency data + mean_latency = [point["mean_latency_ms"] for point in timeseries] + if not any(mean_latency): + log.debug("No latency data to plot") + return + + fig, ax = self._plotter.subplots(figsize=self._figure_size, dpi=self._dpi) + + # Initialize latency plotter + latency_plotter = TimeSeriesLatencyPlotter(main_axis=ax) + + # Collect latency data + for point in timeseries: + latency_plotter.add_y_data(str(point["mean_latency_ms"])) + latency_plotter.add_p50_data(str(point["p50_latency_ms"])) + latency_plotter.add_p95_data(str(point["p95_latency_ms"])) + latency_plotter.add_p99_data(str(point["p99_latency_ms"])) + latency_plotter.add_max_data(str(point["max_latency_ms"])) + + # Plot latency + latency_plotter.plot(x_data=timestamps) + + # Optionally add CPU overlay + if plot_cpu and self._has_cpu_data(timeseries): + cpu_plotter = CPUPlotter(main_axis=ax) + for point in timeseries: + cpu_value = point.get("cpu") + if cpu_value is not None: + cpu_plotter.add_y_data(str(cpu_value)) + cpu_plotter.plot(x_data=timestamps) + + self._configure_axes( + ax=ax, + title=self._generate_title(data, "Latency"), + xlabel="Time (seconds)", + ) + + ax.legend(loc="best", fontsize="small") # pyright: ignore[reportUnknownMemberType] + ax.grid(True, alpha=0.3) # pyright: ignore[reportUnknownMemberType] + + output_path = self._generate_output_path(data, "latency") + self._save_plot(fig, output_path) + self._plotter.close(fig) + + def _has_cpu_data(self, timeseries: list[TimeSeriesDataPoint]) -> bool: + """ + Check if CPU data is available in the time series. + + Args: + timeseries: List of time series data points + + Returns: + True if CPU data is present, False otherwise + """ + return any("cpu" in point for point in timeseries) + + def _generate_title(self, data: TimeSeriesFormatType, metric: str) -> str: + """ + Generate a plot title for time-series data. + + Follows the same pattern as CommonFormatPlotter for consistency: + - Converts blocksize from bytes to KB (e.g., "4096" -> "4K") + - Converts operation to human-readable format (e.g., "randwrite" -> "Random Write") + - Includes iodepth to distinguish plots with different iodepth values + - Excludes numjobs since plots are sorted by numjobs in reports + + Args: + data: Time-series data containing benchmark metadata + metric: Metric name (e.g., "IOPS", "Bandwidth", "Latency") + + Returns: + Formatted title string (e.g., "IOPS Over Time - 4K Random Write (iodepth=8)") + """ + # Convert blocksize from bytes to KB + blocksize_bytes = int(data["blocksize"]) + blocksize_kb = int(blocksize_bytes / KB_CONVERSION_FACTOR) + blocksize_str = f"{blocksize_kb}K" + + # Convert operation to human-readable format + operation = data["operation"] + operation_str = TITLE_CONVERSION.get(operation, operation) + + # Get iodepth value + iodepth = data.get("iodepth", "1") + + return f"{metric} Over Time - {blocksize_str} {operation_str} (iodepth={iodepth})" + + def _configure_axes( + self, + ax: Axes, + title: str, + xlabel: str, + ) -> None: + """ + Configure plot axes with labels and title. + + Args: + ax: Matplotlib axes object + title: Plot title + xlabel: X-axis label + """ + ax.set_title(title, fontsize=14, fontweight="bold") # pyright: ignore[reportUnknownMemberType] + ax.set_xlabel(xlabel, fontsize=12) # pyright: ignore[reportUnknownMemberType] + + # Start y-axis at 0 for better visualization + ax.set_ylim(bottom=0) # pyright: ignore[reportUnknownMemberType] + + def _generate_output_path(self, data: TimeSeriesFormatType, metric: str) -> Path: + """ + Generate output file path for a plot. + + Args: + data: Time-series data containing benchmark metadata + metric: Metric name (e.g., "iops", "bandwidth", "latency") + + Returns: + Path object for the output file + """ + iodepth = data.get("iodepth", "1") + filename = ( + f"{data['blocksize']}_{data['numjobs']}_{data['operation']}_{iodepth}_{metric}_timeseries." + f"{PLOT_FILE_EXTENSION}" + ) + return self._output_dir / filename + + def _save_plot(self, fig: Figure, output_path: Path) -> None: + """ + Save plot to disk. + + Args: + fig: Matplotlib figure object + output_path: Path where plot should be saved + """ + fig.savefig( + output_path, + format=PLOT_FILE_EXTENSION, + bbox_inches="tight", + dpi=self._dpi, + ) + log.debug("Saved plot to %s", output_path) + + def plot_from_file(self, json_file_path: str, plot_cpu: bool = False) -> None: + """ + Load time-series data from JSON file and generate plots. + + Args: + json_file_path: Path to JSON file containing TimeSeriesFormatType data + plot_cpu: Whether to overlay CPU usage on the plots + """ + file_path = Path(json_file_path) + if not file_path.exists(): + log.error("Time-series data file not found: %s", json_file_path) + return + + try: + with file_path.open("r", encoding="utf8") as f: + data: TimeSeriesFormatType = json.load(f) + + self.plot_time_series(data, plot_cpu) + + except (json.JSONDecodeError, KeyError) as e: + log.error("Failed to load time-series data from %s: %s", json_file_path, e) + + def plot_all_in_directory(self, directory: Optional[str] = None, plot_cpu: bool = False) -> None: + """ + Generate plots for all time-series JSON files in a directory. + + Args: + directory: Directory to search for JSON files. If None, uses + the archive directory's visualisation subdirectory. + plot_cpu: Whether to overlay CPU usage on the plots + """ + search_dir = Path(directory) if directory else self._output_dir + + if not search_dir.exists(): + log.warning("Directory not found: %s", search_dir) + return + + json_files = list(search_dir.glob("*_timeseries.json")) + + if not json_files: + log.info("No time-series JSON files found in %s", search_dir) + return + + log.info("Generating plots for %d time-series files", len(json_files)) + + for json_file in json_files: + log.debug("Processing %s", json_file.name) + self.plot_from_file(str(json_file), plot_cpu) + + +# Made with Bob diff --git a/post_processing/post_processing_types.py b/post_processing/post_processing_types.py index 1bdeb874..844930dc 100644 --- a/post_processing/post_processing_types.py +++ b/post_processing/post_processing_types.py @@ -1,9 +1,31 @@ """ -A file to contain common type definitions for use in the post-processing +Type definitions for post-processing data structures. + +This module defines TypedDict types for various data formats used in the +post-processing pipeline, including the common intermediate format and +time-series format. """ from enum import Enum, auto -from typing import NamedTuple, Union +from typing import NamedTuple, TypedDict, Union + +# Log setup types +HandlerType = dict[str, dict[str, str]] + +# FIO json file data types +JobsDataType = list[dict[str, Union[str, dict[str, Union[int, float, dict[str, Union[int, float]]]]]]] + +# Common formatter data types +IodepthDataType = dict[str, str] +CommonFormatDataType = dict[str, Union[str, IodepthDataType]] + +# Common formatter internal data types +InternalBlocksizeDataType = dict[str, dict[str, Union[str, IodepthDataType]]] +InternalNumJobsDataType = dict[str, InternalBlocksizeDataType] +InternalFormattedOutputType = dict[str, InternalNumJobsDataType] + +# Plotter types +PlotDataType = dict[str, dict[str, str]] class CPUPlotType(Enum): @@ -18,6 +40,16 @@ class CPUPlotType(Enum): NODES = auto() +class ReportType(Enum): + """ + The different types of reports that can be generated. + """ + + SIMPLE = auto() + COMPARISON = auto() + TIMESERIES = auto() + + class ReportOptions(NamedTuple): """ This class is used to store the options required to create a report. @@ -29,24 +61,74 @@ class ReportOptions(NamedTuple): create_pdf: bool force_refresh: bool no_error_bars: bool - comparison: bool + report_type: ReportType plot_resources: bool -# Log setup types -HandlerType = dict[str, dict[str, str]] +# New types for time-series format +class TimeSeriesDataPoint(TypedDict): + """ + A single data point in a time-series. -# FIO json file data types -JobsDataType = list[dict[str, Union[str, dict[str, Union[int, float, dict[str, Union[int, float]]]]]]] + Represents aggregated metrics at a specific timestamp, typically + aggregated across multiple volumes using time-window binning. + """ -# Common formatter data types -IodepthDataType = dict[str, str] -CommonFormatDataType = dict[str, Union[str, IodepthDataType]] + timestamp_sec: float + iops: float + bandwidth_bytes: float + mean_latency_ms: float + max_latency_ms: float + p50_latency_ms: float + p95_latency_ms: float + p99_latency_ms: float + num_samples: int -# Common formatter internal data types -InternalBlocksizeDataType = dict[str, dict[str, Union[str, IodepthDataType]]] -InternalNumJobsDataType = dict[str, InternalBlocksizeDataType] -InternalFormattedOutputType = dict[str, InternalNumJobsDataType] -# Plotter types -PlotDataType = dict[str, dict[str, str]] +class TimeSeriesMetadata(TypedDict): + """ + Metadata about a time-series test run. + + Contains information about the test duration, sampling configuration, + and number of volumes tested. + """ + + start_time_epoch: float + end_time_epoch: float + duration_seconds: float + num_volumes: int + sampling_interval_ms: int + log_avg_msec: int + + +class TimeSeriesFormatType(TypedDict): + """ + Complete time-series intermediate format. + + This format is used to store time-indexed performance data from + benchmark runs. It is designed to be benchmark-agnostic, allowing + different benchmarks (FIO, elbencho, etc.) to produce data in this + format for consistent plotting. + + Includes pre-calculated maximum values for efficient report generation, + following the same pattern as CommonFormatDataType. + """ + + benchmark: str + operation: str + blocksize: str + numjobs: str + iodepth: str # total_iodepth if exists, otherwise iodepth + metadata: TimeSeriesMetadata + timeseries: list[TimeSeriesDataPoint] + # Pre-calculated maximum values (same as CommonFormatDataType) + maximum_iops: str + maximum_bandwidth: str + latency_at_max_iops: str + latency_at_max_bandwidth: str + timestamp_at_max_iops: str + timestamp_at_max_bandwidth: str + maximum_latency: str + timestamp_at_max_latency: str + maximum_cpu_usage: str + maximum_memory_usage: str diff --git a/post_processing/report.py b/post_processing/report.py index 627b54a8..73e3dff5 100644 --- a/post_processing/report.py +++ b/post_processing/report.py @@ -11,19 +11,31 @@ import traceback from argparse import Namespace from logging import Logger, getLogger +from pathlib import Path +from post_processing.formatter.base_formatter import BaseFormatter from post_processing.formatter.common_output_formatter import CommonOutputFormatter +from post_processing.formatter.time_series_output_formatter import ( + TimeSeriesOutputFormatter, +) from post_processing.log_configuration import setup_logging -from post_processing.post_processing_types import ReportOptions +from post_processing.post_processing_types import ReportOptions, ReportType from post_processing.reports.comparison_report_generator import ComparisonReportGenerator from post_processing.reports.report_generator import ReportGenerator from post_processing.reports.simple_report_generator import SimpleReportGenerator +from post_processing.reports.time_series_report_generator import ( + TimeSeriesReportGenerator, +) setup_logging() log: Logger = getLogger(name="reports") -def parse_namespace_to_options(arguments: Namespace, comparison_report: bool = False) -> ReportOptions: +def parse_namespace_to_options( + arguments: Namespace, + comparison_report: bool = False, + timeseries_report: bool = False, +) -> ReportOptions: """ Parse a namespace as used by argparse into our internal NamedTuple representation """ @@ -32,11 +44,17 @@ def parse_namespace_to_options(arguments: Namespace, comparison_report: bool = F archives: list[str] = [] output_directory: str = arguments.output_directory - if comparison_report: + # Determine report type + if timeseries_report: + report_type = ReportType.TIMESERIES + archives.append(arguments.archive) + elif comparison_report: + report_type = ReportType.COMPARISON archives.append(arguments.baseline) for directory in arguments.archives.split(","): archives.append(directory) else: + report_type = ReportType.SIMPLE archives.append(arguments.archive) if hasattr(arguments, "no_error_bars"): @@ -52,7 +70,7 @@ def parse_namespace_to_options(arguments: Namespace, comparison_report: bool = F results_file_root=arguments.results_file_root, force_refresh=arguments.force_refresh, no_error_bars=no_error_bars, - comparison=comparison_report, + report_type=report_type, plot_resources=plot_resources, ) @@ -94,13 +112,20 @@ def generate(self, throw_exception: bool = False) -> None: self._generate_intermediate_files() report_generator: ReportGenerator - if self._options.comparison: + if self._options.report_type == ReportType.COMPARISON: report_generator = ComparisonReportGenerator( archive_directories=self._options.archives, output_directory=self._options.output_directory, force_refresh=self._options.force_refresh, ) - else: + elif self._options.report_type == ReportType.TIMESERIES: + report_generator = TimeSeriesReportGenerator( + archive_directories=self._options.archives, + output_directory=self._options.output_directory, + force_refresh=self._options.force_refresh, + plot_resources=self._options.plot_resources, + ) + else: # ReportType.SIMPLE report_generator = SimpleReportGenerator( archive_directories=self._options.archives, output_directory=self._options.output_directory, @@ -128,6 +153,31 @@ def generate(self, throw_exception: bool = False) -> None: if throw_exception: raise e + def _archive_has_intermediate_files(self, directory: str) -> bool: + """ + Check whether an archive already contains the intermediate files needed + for the selected report type. + + For time-series reports, intermediate files are expected in the new + nested operation/visualisation/ structure with *_timeseries.json names. + + For simple/comparison reports, accept both: + - new nested operation/visualisation/*.json layout + - legacy top-level visualisation/*.json layout + + For non-time-series reports, *_timeseries.json files do not count as + hockey-stick intermediate data. + """ + archive_path = Path(directory) + + if self._options.report_type == ReportType.TIMESERIES: + return any(archive_path.glob("**/visualisation/*_timeseries.json")) + + return any( + not file_path.name.endswith("_timeseries.json") + for file_path in archive_path.glob("**/visualisation/*.json") + ) + def _generate_intermediate_files(self) -> None: """ If the raw fio results have not yet been post-processed then we need to do @@ -135,22 +185,26 @@ def _generate_intermediate_files(self) -> None: """ for directory in self._options.archives: - output_directory: str = f"{directory}/visualisation/" + if not self._archive_has_intermediate_files(directory) or self._options.force_refresh: + log.debug("Preparing to generate intermediate files for %s", directory) + os.makedirs(name=f"{directory}/visualisation/", exist_ok=True) - if not os.path.exists(output_directory) or not os.listdir(output_directory) or self._options.force_refresh: - # Either the directory doesn't exists, or is empty, or the user has told us to regenerate the files + log.info("Generating intermediate files for %s", directory) - log.debug("Creating directory %s", output_directory) - os.makedirs(name=output_directory, exist_ok=True) - - log.info("Generating intermediate files for %s in directory %s", directory, output_directory) - formatter: CommonOutputFormatter = CommonOutputFormatter( - archive_directory=directory, filename_root=self._options.results_file_root - ) + # Use the appropriate formatter based on report type + formatter: BaseFormatter + if self._options.report_type == ReportType.TIMESERIES: + formatter = TimeSeriesOutputFormatter( + archive_directory=directory, filename_root=self._options.results_file_root + ) + else: + formatter = CommonOutputFormatter( + archive_directory=directory, filename_root=self._options.results_file_root + ) try: - formatter.convert_all_files() - formatter.write_output_file() + # With memory-efficient approach, data is written during process() + formatter.process() except Exception as e: # pylint: disable=[broad-exception-caught] # Generating the intermediate files can raise a broad range of exceptions from # the different sub-modules called. Therefore we want to catch them all and raise diff --git a/post_processing/reports/comparison_report_generator.py b/post_processing/reports/comparison_report_generator.py index a3dfbf22..4a90585a 100644 --- a/post_processing/reports/comparison_report_generator.py +++ b/post_processing/reports/comparison_report_generator.py @@ -42,56 +42,28 @@ class ComparisonReportGenerator(ReportGenerator): containing several individual FIO runs """ + # No need to override __init__ anymore - base class handles everything + def _generate_report_title(self) -> str: title: str = f"Comparitive Performance Report for {' vs '.join(self._build_strings)}" return title - def _add_plots(self) -> None: # pylint: disable=[too-many-locals] - self._report.new_header(level=1, title="Response Curves") - empty_table_header: list[str] = ["", ""] - - for number_of_jobs in self._get_all_number_of_jobs_values(): - self._report.new_header(level=2, title=f"Number of Jobs {number_of_jobs}") - image_tables: dict[str, list[str]] = {} - - for _, operation in TITLE_CONVERSION.items(): - image_tables[operation] = empty_table_header.copy() - - for image_file in self._plot_files: - # The comparison plot files have a different name format: - # Comparison____ - # so we need to split the Comparison_ from the front before calling the common method - (blocksize, percent, operation, numjobs) = get_blocksize_percentage_operation_numjobs_from_file_name( - image_file.stem[len("Comparison_") :] - ) - if numjobs == number_of_jobs: - title: str = f"{blocksize} {percent} {operation}" - - image_line: str = self._report.new_inline_image( - text=title, path=f"{self._plots_directory.parts[-1]}/{image_file.parts[-1]}" - ) - anchor: str = f'' - - image_line = f"{anchor}{image_line}" - - image_tables[operation].append(image_line) + def _get_plot_file_stem(self, image_file: Path) -> str: + """ + Get the file stem for a comparison report plot file. - # Create the correct sections and add a table for each section to the report + For comparison reports, removes the "Comparison_" prefix from the stem. - for section, data in image_tables.items(): - # We don't want to display a section if it doesn't contain any plots - if len(data) > len(empty_table_header): - self._report.new_header(level=3, title=section) - table_images = data + Args: + image_file: Path to the plot file - # We need to calculate the number of rows, but new_table() requires the - # exact number of items to fill the table, so we may need to add a dummy - # entry at the end - number_of_rows: int = len(table_images) // 2 - if len(table_images) % 2 > 0: - number_of_rows += 1 - table_images.append("") - self._report.new_table(columns=2, rows=number_of_rows, text=table_images, text_align="center") + Returns: + The file stem without the "Comparison_" prefix + """ + stem = image_file.stem + if stem.startswith("Comparison_"): + return stem[len("Comparison_") :] + return stem def _add_summary_table(self) -> None: self._report.new_header(level=1, title=f"Comparison summary for {' vs '.join(self._build_strings)}") @@ -111,10 +83,9 @@ def _add_summary_table(self) -> None: if data: self._report.new_line(text=f"|{operation}|{table_header}") self._report.new_line(text=table_justfication_string) - - for line in data: - self._report.new_line(text=line) - self._report.new_line() + for line in data: + self._report.new_line(text=line) + self._report.new_line() def _add_configuration_yaml_files(self) -> None: self._report.new_header(level=1, title="Configuration yaml files") @@ -166,33 +137,21 @@ def _find_and_sort_file_paths(self, paths: list[Path], search_pattern: str, inde def _find_and_sort_plot_files(self) -> list[Path]: """ - Find all the plot files in the directory. That is any file that - has the .svg file extension + Find all the comparison plot files in the directory. This overrides the one in the ReportGenerator as the comparison plot files have a different naming convention: Comparison_B___.svg + + Filters to only include files starting with "Comparison_" to avoid + accidentally including simple or time-series plot files. """ - return self._find_and_sort_file_paths( + all_plots = self._find_and_sort_file_paths( paths=[self._plots_directory], search_pattern=f"*{PLOT_FILE_EXTENSION_WITH_DOT}", index=1 ) - - def _find_configuration_yaml_files(self) -> list[Path]: - file_paths: list[Path] = [] - - for directory in self._archive_directories: - # Because this can either be called during a run, or separately afterwards the - # archive directory passed may be at a different point in the directory tree. - # We therefore need to search both above and below the current directory for - # the config yaml - paths: list[Path] = [path for path in directory.parents if f"{path}".endswith("/results")] - if len(paths) == 0: - paths = [path for path in directory.iterdir() if f"{path}".endswith("/results")] - - for path in paths: - file_paths.extend(path.glob("**/cbt_config.yaml")) - - return file_paths + # Filter to only include comparison plots (those starting with "Comparison_") + comparison_plots = [plot for plot in all_plots if plot.stem.startswith("Comparison_")] + return comparison_plots def _create_comparison_plots(self) -> None: """ @@ -211,17 +170,21 @@ def _generate_table_headers(self) -> tuple[str, str]: """ Generate the header lines for the table """ - # The first directory is always the baseline, so we want to get it out of the way first - table_header: str = f"numjobs|{self._data_directories.pop(0).parts[-2]}|" + # Use archive directories (not data directories which now contain multiple subdirs per archive) + # The first archive is always the baseline + archive_dirs = self._archive_directories.copy() + baseline_dir = archive_dirs.pop(0) + + table_header: str = f"numjobs|{baseline_dir.parts[-1]}|" table_justfication_string: str = "| :--- | ---: | ---: |" - if len(self._data_directories) < 2: - for directory in self._data_directories: - table_header += f"{directory.parts[-2]}|%change throughput|%change latency|" + if len(archive_dirs) < 2: + for directory in archive_dirs: + table_header += f"{directory.parts[-1]}|%change throughput|%change latency|" table_justfication_string += " ---: | ---: | ---: |" else: - for directory in self._data_directories: - table_header += f"{directory.parts[-2]}|%change|" + for directory in archive_dirs: + table_header += f"{directory.parts[-1]}|%change|" table_justfication_string += " ---: | ---: |" return (table_header, table_justfication_string) @@ -275,12 +238,28 @@ def _yaml_file_has_more_that_20_differences(self, base_file: Path, comparison_fi If there are more that 20 differences between base and comparison then return True, otherwise False """ - diff_command: str = f"/usr/bin/env diff -wy --suppress-common-lines {base_file!s} {comparison_file!s} | wc -l" + # Use shell=False for security - pass command as list + diff_command: list[str] = [ + "/usr/bin/env", + "diff", + "-wy", + "--suppress-common-lines", + str(base_file), + str(comparison_file), + ] + wc_command: list[str] = ["/usr/bin/env", "wc", "-l"] output: bytes try: - output = subprocess.check_output(diff_command, shell=True) - except subprocess.CalledProcessError: + # Use two subprocess calls with pipe to avoid shell=True + with subprocess.Popen(diff_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as diff_process: + with subprocess.Popen( + wc_command, stdin=diff_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) as wc_process: + if diff_process.stdout: + diff_process.stdout.close() # Allow diff_process to receive SIGPIPE if wc_process exits + output, _ = wc_process.communicate() + except (subprocess.CalledProcessError, OSError): return False output_as_string: str = output.decode().strip() diff --git a/post_processing/reports/report_generator.py b/post_processing/reports/report_generator.py index 883789a2..f20b5033 100644 --- a/post_processing/reports/report_generator.py +++ b/post_processing/reports/report_generator.py @@ -18,7 +18,9 @@ from post_processing.common import ( DATA_FILE_EXTENSION_WITH_DOT, PLOT_FILE_EXTENSION_WITH_DOT, + TITLE_CONVERSION, find_common_data_file_names, + find_hockey_stick_visualisation_directories, get_blocksize_percentage_operation_numjobs_from_file_name, get_date_time_string, ) @@ -59,13 +61,42 @@ def __init__( # pylint: disable=[too-many-arguments, too-many-positional-argume for archive_directory in archive_directories: archive_path: Path = Path(archive_directory) self._archive_directories.append(archive_path) - data_directory: Path = Path(f"{archive_directory}/visualisation") - self._data_directories.append(data_directory) + + # Find visualisation directories (type depends on subclass) + visualisation_directories = self._find_visualisation_directories(archive_path) + + # Reject legacy top-level visualisation directories unless force_refresh was requested. + # force_refresh regenerates intermediate files before report generator construction, + # so the legacy directory may still exist alongside the new nested structure. + if ( + len(visualisation_directories) == 1 + and visualisation_directories[0].name == "visualisation" + and visualisation_directories[0].parent == archive_path + and not self._force_refresh + ): + error_msg = ( + f"\nError: Legacy visualisation directory structure detected in '{archive_path}'.\n" + f"Reports require the new nested structure (operation/visualisation/).\n" + f"Please re-run with the --force_refresh flag to regenerate the intermediate format files.\n" + ) + log.error(error_msg) + raise ValueError(error_msg) + + # Check if no visualisation directories were found + if not visualisation_directories: + error_msg = ( + f"\nError: No visualisation directories with data found in '{archive_path}'.\n" + f"Please ensure the data has been processed, or use --force_refresh to regenerate files.\n" + ) + log.error(error_msg) + raise ValueError(error_msg) + + self._data_directories.extend(visualisation_directories) + # We need to replace all _ characters in the build string as pandoc conversion # breaks if there are _ characters in the file anywhere self._build_strings.append(f"{archive_path.parts[-1]}".replace("_", "-")) - # self._data_directories = self._data_directories self._data_files = self._find_and_sort_data_files() self._output_directory: Path = Path(output_directory) @@ -106,6 +137,20 @@ def create_report(self) -> None: # outupt directory self._save_report() + def _get_additional_header_files(self) -> list[str]: + """ + Get additional LaTeX header files to include during PDF conversion. + + Subclasses can override this method to add report-specific headers. + These will be included in addition to the base header file. + + Default implementation returns an empty list (no additional headers). + + Returns: + List of paths to additional header files + """ + return [] + def save_as_pdf(self) -> int: """ Convert a report in markdown format and save it as a pdf file. @@ -114,24 +159,40 @@ def save_as_pdf(self) -> int: # We need to change directory so we can include a relative reference to # the plot files chdir(self._output_directory) - header_file = self._create_header_files() + header_files = self._create_header_files() pdf_file_path: Path = Path(f"{self._output_directory}/{self._report_path.stem}.{self.PDF_FILE_EXTENSION}") - pandoc_command: str = ( - f"/usr/bin/env pandoc -H {header_file} " - + "-f markdown-implicit_figures --columns=10 " - + "-V papersize=A4 -V documentclass=report --top-level-division=chapter " - + f"-o {pdf_file_path} {self._report_path}" + pandoc_command: list[str] = ["/usr/bin/env", "pandoc"] + + for header_file in header_files: + pandoc_command.extend(["-H", str(header_file)]) + + pandoc_command.extend( + [ + "-f", + "markdown-implicit_figures", + "--columns=10", + "-V", + "papersize=A4", + "-V", + "documentclass=report", + "--top-level-division=chapter", + "-o", + str(pdf_file_path), + str(self._report_path), + ] ) return_code: int = subprocess.call( - pandoc_command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT + pandoc_command, shell=False, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT ) if return_code != 0: log.error("Unable to convert %s to pdf format", f"{self._report_path}") else: - header_file.unlink() + # Clean up all header files + for header_file in header_files: + header_file.unlink() return return_code @@ -177,10 +238,18 @@ def _add_summary_table(self) -> None: """ @abstractmethod - def _add_plots(self) -> None: + def _get_plot_file_stem(self, image_file: Path) -> str: """ - Add the plots to the report. - We are using a table to get multiple images on a single line + Get the file stem for a plot file, removing any prefixes. + + For simple reports: returns the stem as-is + For comparison reports: removes "Comparison_" prefix + + Args: + image_file: Path to the plot file + + Returns: + The file stem without any prefix """ @abstractmethod @@ -197,13 +266,6 @@ def _find_and_sort_file_paths(self, paths: list[Path], search_pattern: str, inde name """ - @abstractmethod - def _find_configuration_yaml_files(self) -> list[Path]: - """ - Find the path to the configuration yaml files in the archive - directories - """ - @abstractmethod def _copy_images(self) -> None: """ @@ -211,6 +273,108 @@ def _copy_images(self) -> None: so the markdown can link to them using a known relative path """ + def _find_visualisation_directories(self, archive_path: Path) -> list[Path]: + """ + Find visualisation directories for this report type. + + Default implementation finds hockey-stick visualisation directories. + Subclasses can override to find different types (e.g., timeseries). + + Args: + archive_path: Path to the archive directory + + Returns: + List of visualisation directory paths + """ + return find_hockey_stick_visualisation_directories(archive_path) + + def _add_plots(self) -> None: # pylint: disable=too-many-locals + """ + Add the plots to the report. + We are using a table to get multiple images on a single line. + + This is a template method that calls helper methods to be implemented by subclasses. + + Note: This method has many local variables (17) because it consolidates plotting logic + that was previously duplicated across SimpleReportGenerator and ComparisonReportGenerator. + The complexity is justified as it eliminates ~90 lines of duplicate code and provides + a single source of truth for plot generation. Further refactoring would require breaking + this into smaller methods, which would reduce readability for this template method pattern. + """ + self._report.new_header(level=1, title="Response Curves") + empty_table_header: list[str] = ["", ""] + + for number_of_jobs in self._get_all_number_of_jobs_values(): + self._report.new_header(level=2, title=f"Number of Jobs {number_of_jobs}") + image_tables: dict[str, list[str]] = {} + + for _, operation in TITLE_CONVERSION.items(): + image_tables[operation] = empty_table_header.copy() + + for image_file in self._plot_files: + # Get the file stem without any prefix (e.g., "Comparison_") + file_stem = self._get_plot_file_stem(image_file) + (blocksize, percent, operation, numjobs) = get_blocksize_percentage_operation_numjobs_from_file_name( + file_stem + ) + if numjobs == number_of_jobs: + title: str = f"{blocksize} {percent} {operation}" + + image_line: str = self._report.new_inline_image( + text=title, path=f"{self._plots_directory.parts[-1]}/{image_file.parts[-1]}" + ) + anchor: str = f'' + + image_line = f"{anchor}{image_line}" + + image_tables[operation].append(image_line) + + # Create the correct sections and add a table for each section to the report + for section, data in image_tables.items(): + # We don't want to display a section if it doesn't contain any plots + if len(data) > len(empty_table_header): + self._report.new_header(level=3, title=section) + table_images = data + + # We need to calculate the number of rows, but new_table() requires the + # exact number of items to fill the table, so we may need to add a dummy + # entry at the end + number_of_rows: int = len(table_images) // 2 + if len(table_images) % 2 > 0: + number_of_rows += 1 + table_images.append("") + self._report.new_table(columns=2, rows=number_of_rows, text=table_images, text_align="center") + + def _find_configuration_yaml_files(self) -> list[Path]: + """ + Find the path to the configuration yaml files in the archive directories. + + This method searches for cbt_config.yaml files in the results directories + associated with each archive directory. It handles both cases where the + archive directory is within the results tree or contains a results subdirectory. + + Returns: + List of Path objects for cbt_config.yaml files found + """ + file_paths: list[Path] = [] + + for directory in self._archive_directories: + # Search both above and below the current directory for the config yaml + # This handles different calling contexts (during run vs. post-processing) + + # First, try to find results directory in parent paths + paths: list[Path] = [path for path in directory.parents if f"{path}".endswith("/results")] + + # If not found in parents, look in subdirectories + if len(paths) == 0: + paths = [path for path in directory.iterdir() if f"{path}".endswith("/results")] + + # Search for yaml files in the results directories + for path in paths: + file_paths.extend(path.glob("**/cbt_config.yaml")) + + return file_paths + def _all_plot_files_exist(self) -> bool: """ return true if plot files exist for all the data files. If we have @@ -259,7 +423,7 @@ def _create_plots_results_directory(self) -> None: """ Create the plots sub-directory in the output directory """ - subprocess.call(f"mkdir -p {self._plots_directory}", shell=True) + self._plots_directory.mkdir(exist_ok=True) def _sort_list_of_paths(self, paths: list[Path], index: int) -> list[Path]: """ @@ -279,15 +443,17 @@ def _find_files_with_filename(self, file_name: str) -> list[Path]: return file_paths - def _create_header_files(self) -> Path: + def _create_header_files(self) -> list[Path]: """ - Create the header file in tex format that is used to provide + Create the header files in tex format that are used to provide headers and footers when the report is created in pdf format. - Replace any placeholders with the correct values. + Copies all header files (base + additional) to the output directory. + Replaces BUILD placeholder with the build string in all files. + + Returns: + List of paths to the copied header files in the output directory """ - # Note: This currently only enters the build string, but it could be - # expanded in the future to give more detals current_file_path: Path = Path(__file__) cbt_directory: str = "" @@ -296,22 +462,28 @@ def _create_header_files(self) -> Path: if part == "cbt": break - base_header_file: Path = Path(f"{cbt_directory}/{self.BASE_HEADER_FILE_PATH}") - - tex_output_path: Path = Path(f"{self._output_directory}/perf_report.tex") + # Combine base header and any additional headers + all_header_paths = [self.BASE_HEADER_FILE_PATH, *self._get_additional_header_files()] build_string: str = " vs ".join(self._build_strings) + output_header_files: list[Path] = [] + + for header_path in all_header_paths: + source_path = Path(f"{cbt_directory}/{header_path}") + dest_path = Path(f"{self._output_directory}/{source_path.name}") - try: - tex_contents: str = base_header_file.read_text(encoding="utf-8") - tex_contents = tex_contents.replace("BUILD", build_string) - tex_output_path.write_text(tex_contents, encoding="utf-8") - except FileNotFoundError: - log.error("Unable to read from %s", base_header_file) - except PermissionError: - log.error("Unable to write to %s", tex_output_path) + try: + tex_contents: str = source_path.read_text(encoding="utf-8") + # Replace BUILD placeholder if present + tex_contents = tex_contents.replace("BUILD", build_string) + dest_path.write_text(tex_contents, encoding="utf-8") + output_header_files.append(dest_path) + except FileNotFoundError: + log.error("Unable to read from %s", source_path) + except PermissionError: + log.error("Unable to write to %s", dest_path) - return tex_output_path + return output_header_files def _generate_plot_directory_name(self) -> str: """ @@ -326,17 +498,17 @@ def _generate_plot_directory_name(self) -> str: def _get_all_number_of_jobs_values(self) -> list[str]: """ Get all the possible number_of_jobs values for this set of data. - Works for both simple and comparison plot file naming conventions. + Works for both simple, comparison and timeseries plot file naming conventions. Simple format: ___.svg Comparison format: Comparison____.svg + Time-series format: ____timeseries.svg """ numbers_of_jobs: set[str] = set() for image_file in self._plot_files: - # Handle both naming conventions by stripping "Comparison_" prefix if present - stem = image_file.stem - if stem.startswith("Comparison_"): - stem = stem[len("Comparison_") :] + # Use the subclass-specific method to get the proper stem + # This handles different naming conventions (simple, comparison, time-series) + stem = self._get_plot_file_stem(image_file) (_, _, _, number_of_jobs) = get_blocksize_percentage_operation_numjobs_from_file_name(stem) numbers_of_jobs.add(number_of_jobs) diff --git a/post_processing/reports/simple_report_generator.py b/post_processing/reports/simple_report_generator.py index 798fd135..691cf570 100644 --- a/post_processing/reports/simple_report_generator.py +++ b/post_processing/reports/simple_report_generator.py @@ -92,56 +92,24 @@ def _add_summary_table(self) -> None: # pylint: disable=[too-many-locals] data_tables[operation].append(data) - # for operation in data_tables.keys(): + # Add all data rows to the table for _, operation_data in data_tables.items(): for line in operation_data: self._report.new_line(text=line) - def _add_plots(self) -> None: # pylint: disable=[too-many-locals] - self._report.new_header(level=1, title="Response Curves") + def _get_plot_file_stem(self, image_file: Path) -> str: + """ + Get the file stem for a simple report plot file. - empty_table_header: list[str] = ["", ""] - image_tables: dict[str, dict[str, list[str]]] = {} + For simple reports, the stem is used as-is without any prefix removal. - for number_of_jobs in self._get_all_number_of_jobs_values(): - self._report.new_header(level=2, title=f"Number of Jobs {number_of_jobs}") - image_tables[number_of_jobs] = {} - for _, operation in TITLE_CONVERSION.items(): - image_tables[number_of_jobs][operation] = empty_table_header.copy() + Args: + image_file: Path to the plot file - for image_file in self._plot_files: - (blocksize, percent, operation, numjobs) = get_blocksize_percentage_operation_numjobs_from_file_name( - image_file.stem - ) - if numjobs == number_of_jobs: - title: str = f"{blocksize} {percent} {operation}" - - image_line: str = self._report.new_inline_image( - text=title, path=f"{self._plots_directory.parts[-1]}/{image_file.parts[-1]}" - ) - anchor: str = f'' - - image_line = f"{anchor}{image_line}" - - image_tables[number_of_jobs][operation].append(image_line) - - # Create the correct sections and add a table for each section to the report - - # for section in image_tables.keys(): - for section_name, section_data in image_tables[number_of_jobs].items(): - # We don't want to display a section if it doesn't contain any plots - if len(section_data) > len(empty_table_header): - self._report.new_header(level=3, title=section_name) - table_images = section_data - - # We need to calculate the number of rows, but new_table() requires the - # exact number of items to fill the table, so we may need to add a dummy - # entry at the end - number_of_rows: int = len(table_images) // 2 - if len(table_images) % 2 > 0: - number_of_rows += 1 - table_images.append("") - self._report.new_table(columns=2, rows=number_of_rows, text=table_images, text_align="center") + Returns: + The file stem without extension + """ + return image_file.stem def _add_configuration_yaml_files(self) -> None: self._report.new_header(level=1, title="Configuration yaml") @@ -156,35 +124,29 @@ def _add_configuration_yaml_files(self) -> None: def _copy_images(self) -> None: plot_files: list[Path] = [] for directory in self._data_directories: - plot_files.extend(list(directory.glob(f"*{PLOT_FILE_EXTENSION_WITH_DOT}"))) + # Check for existing plot files in this directory + existing_plots = list(directory.glob(f"*{PLOT_FILE_EXTENSION_WITH_DOT}")) - # If there are no plotfiles in the directory then we should create them - if len(plot_files) == 0 or self._force_refresh: + # If there are no plot files in the directory or force_refresh is set, create them + if len(existing_plots) == 0 or self._force_refresh: + log.info("Generating plots for %s", directory.parent) plotter = SimplePlotter(str(directory.parent), self._plot_error_bars, self._plot_resources) plotter.draw_and_save() + # Collect the newly generated plot files plot_files.extend(list(directory.glob(f"*{PLOT_FILE_EXTENSION_WITH_DOT}"))) + else: + # Use existing plots + plot_files.extend(existing_plots) - for plot_file in plot_files: - subprocess.call(f"cp {plot_file} {self._plots_directory}/", shell=True) + # Filter out time-series files (they have _timeseries suffix before the extension) + # Simple reports should only include standard hockey-stick plots + filtered_plot_files = [plot_file for plot_file in plot_files if "_timeseries" not in plot_file.stem] + + for plot_file in filtered_plot_files: + # Use shell=False for security - pass command as list + subprocess.call(["/usr/bin/env", "cp", str(plot_file), f"{self._plots_directory}/"]) def _find_and_sort_file_paths(self, paths: list[Path], search_pattern: str, index: Optional[int] = 0) -> list[Path]: unsorted_paths: list[Path] = list(paths[0].glob(search_pattern)) assert index is not None return self._sort_list_of_paths(unsorted_paths, index) - - def _find_configuration_yaml_files(self) -> list[Path]: - file_paths: list[Path] = [ - path for path in self._archive_directories[0].parents if f"{path}".endswith("/results") - ] - # Because this can either be called during a run, or separately afterwards the - # archive directory passed may be at a different point in the directory tree. - # We therefore need to search both above and below the current directory for - # the config yaml - if len(file_paths) == 0: - file_paths = [path for path in self._archive_directories[0].iterdir() if f"{path}".endswith("/results")] - - yaml_file_path: list[Path] = list(file_paths[0].glob("**/cbt_config.yaml")) - - # This should only ever return a single path as each archive directory - # should only ever contain a single cbt_config.yaml file - return yaml_file_path diff --git a/post_processing/reports/time_series_report_generator.py b/post_processing/reports/time_series_report_generator.py new file mode 100644 index 00000000..db969e22 --- /dev/null +++ b/post_processing/reports/time_series_report_generator.py @@ -0,0 +1,591 @@ +""" +Code to automatically generate a time-series report from a directory containing +time-series performance results. + +The report will be generated in markdown format using the create_report() +method and the resulting file saved to the specified output directory. + +Optionally the markdown report can be converted to a pdf using pandoc by +calling save_as_pdf(). +The pdf file will have the same name as the markdown file, and be saved in +the same output directory. + +For blocksizes < 64K: displays IOPS and latency time-series plots +For blocksizes >= 64K: displays bandwidth and latency time-series plots +""" + +import subprocess +from datetime import datetime +from logging import Logger, getLogger +from pathlib import Path +from typing import Optional + +import matplotlib.pyplot as plotter + +from post_processing.common import ( + DATA_FILE_EXTENSION_WITH_DOT, + PLOT_FILE_EXTENSION_WITH_DOT, + TITLE_CONVERSION, + find_timeseries_visualisation_directories, + get_blocksize_percentage_operation_numjobs_from_file_name, + get_resource_details_from_file, + read_intermediate_file, + strip_confidential_data_from_yaml, +) +from post_processing.plotter.time_series_plotter import TimeSeriesPlotter +from post_processing.post_processing_types import CommonFormatDataType +from post_processing.reports.report_generator import ReportGenerator + +log: Logger = getLogger("reports") + + +class TimeSeriesReportGenerator(ReportGenerator): + """ + The class responsible for generating a report from a single time-series FIO run. + + Shows time-series plots (IOPS, bandwidth, latency over time) instead of + hockey-stick curves. + + For blocksizes < 64K: displays IOPS and latency + For blocksizes >= 64K: displays bandwidth and latency + """ + + # Threshold for switching between IOPS and bandwidth display + BLOCKSIZE_THRESHOLD_BYTES: int = 65536 # 64K in bytes + + # Additional LaTeX header file for timeseries-specific formatting + TIMESERIES_HEADER_FILE_PATH: str = "include/timeseries_report.tex" + + def _get_additional_header_files(self) -> list[str]: + """ + Get additional LaTeX header files for timeseries reports. + + Returns the timeseries-specific header that scales images to 50% height. + + Returns: + List containing the path to the timeseries header file + """ + return [self.TIMESERIES_HEADER_FILE_PATH] + + def _generate_report_title(self) -> str: + """ + Generate the title for the report. + + Any _ must be converted to a - otherwise the pandoc conversion to PDF + will fail + """ + title: str = f"Time-Series Performance Report for {''.join(self._build_strings)}" + return title + + def _generate_report_name(self) -> str: + """ + The report name is of the format: + timeseries_performance_report_yymmdd_hhmmss.md + """ + current_datetime: datetime = datetime.now() + + # Convert to string + datetime_string: str = current_datetime.strftime("%y%m%d_%H%M%S") + output_file_name: str = f"timeseries_performance_report_{datetime_string}.{self.MARKDOWN_FILE_EXTENSION}" + return output_file_name + + def _find_visualisation_directories(self, archive_path: Path) -> list[Path]: + """ + Find timeseries visualisation directories. + + Overrides base class to find directories containing timeseries data + (under iodepth-X or total_iodepth-X subdirectories). + + Args: + archive_path: Path to the archive directory + + Returns: + List of timeseries visualisation directory paths + """ + return find_timeseries_visualisation_directories(archive_path) + + def _get_plot_file_stem(self, image_file: Path) -> str: + """ + Get the file stem for a time-series report plot file. + + For time-series reports, removes the metric suffix, iodepth, and "_timeseries" suffix. + Example: "4k_1_randread_8_iops_timeseries.svg" -> "4k_1_randread" + + Args: + image_file: Path to the plot file + + Returns: + The file stem without metric, iodepth, and timeseries suffixes + """ + stem = image_file.stem + # Remove _timeseries suffix if present + if stem.endswith("_timeseries"): + stem = stem[: -len("_timeseries")] + # Remove metric suffix (_iops, _bandwidth, _latency) + for metric in ["_iops", "_bandwidth", "_latency"]: + if stem.endswith(metric): + stem = stem[: -len(metric)] + break + # Remove iodepth (last part after removing metric) + # Format at this point: {blocksize}_{numjobs}_{operation}_{iodepth} + parts = stem.split("_") + if len(parts) >= 4: + # Remove the last part (iodepth) to get back to expected format + stem = "_".join(parts[:-1]) + return stem + + def get_latency_throughput_from_file(self, file_path: Path) -> tuple[str, str]: + """ + Reads the data stored in the time-series intermediate file format and returns the + maximum throughput in either iops or MB/s, and the latency in ms recorded for that throughput. + + Time-series format has pre-calculated maximum values at the top level. + Overrides the base class method to handle time-series specific format. + """ + data: CommonFormatDataType = read_intermediate_file(file_path=f"{file_path}") + + # Get blocksize to determine whether to use IOPS or bandwidth + blocksize_str = data.get("blocksize", "0") + assert isinstance(blocksize_str, str) + blocksize: int = int(int(blocksize_str) / 1024) + + throughput_key: str = "maximum_iops" + latency_key: str = "latency_at_max_iops" + throughput_type: str = "IOps" + + if blocksize >= 64: + throughput_key = "maximum_bandwidth" + latency_key = "latency_at_max_bandwidth" + throughput_type = "MB/s" + + # Get the pre-calculated maximum values + throughput = data[throughput_key] + assert isinstance(throughput, str) + max_throughput: float = float(throughput) + + # Convert bandwidth from bytes to MB/s if needed + if blocksize >= 64: + max_throughput = max_throughput / (1000 * 1000) + + latency = data[latency_key] + assert isinstance(latency, str) + latency_at_maximum_throughput: float = float(latency) + + return (f"{max_throughput:.0f} {throughput_type}", f"{latency_at_maximum_throughput:.1f}") + + def _add_summary_table(self) -> None: # pylint: disable=[too-many-locals] + """ + Add a table that contains a summary of the time-series results. + + The table format is: + |Workload Name|Number of Jobs|Iodepth|Maximum Throughput|Maximum Latency| + + - Iodepth: total_iodepth if exists, otherwise iodepth + - Maximum Throughput: bandwidth (MB/s) for blocksize >= 64K, IOPS for < 64K + - Values are in format: @ + - Rows are sorted by workload name, then by iodepth + """ + self._report.new_header(level=1, title=f"Summary of results for {''.join(self._build_strings)}") + + log.info("Generating summary table for time-series data") + + # Single table format with iodepth and timestamp columns + headers: str = "|Workload Name|Number of Jobs|Iodepth|Maximum Throughput|Maximum Latency|" + alignment: str = "| :--- | :---: | :---: | ---: | ---: |" + + if self._plot_resources: + alignment += " ---: |" + headers += "System CPU (%)|" + + # Collect data for the table with sorting information + # Structure: list of tuples (workload_name, iodepth_int, data_row) + table_rows: list[tuple[str, int, str]] = [] + + # Process each time-series data file + for _, file_data in self._data_files.items(): + for file_path in file_data: + log.debug("Looking at file %s", file_path) + + # Extract iodepth from file path + iodepth = self._extract_iodepth_from_path(file_path) + + # Get maximum throughput and latency with timestamps + (max_throughput_with_ts, max_latency_with_ts) = self._get_max_metrics_with_timestamps(file_path) + (cpu_usage, _) = get_resource_details_from_file(file_path) + + # Remove _timeseries suffix from filename before parsing + # Filename format: {blocksize}_{numjobs}_{operation}_{iodepth}_timeseries + file_stem = file_path.stem + if file_stem.endswith("_timeseries"): + file_stem = file_stem[: -len("_timeseries")] + + # Remove iodepth (last part) before parsing with the common function + # which expects format: {blocksize}_{numjobs}_{operation} + file_parts = file_stem.split("_") + if len(file_parts) >= 4: + # Remove the last part (iodepth) to get back to the expected format + file_stem_without_iodepth = "_".join(file_parts[:-1]) + else: + file_stem_without_iodepth = file_stem + + (blocksize, percent, operation, number_of_jobs) = ( + get_blocksize_percentage_operation_numjobs_from_file_name(file_stem_without_iodepth) + ) + workload_name: str = f"{blocksize} {percent} {operation}" + + # Build the data row + data: str = ( + f"|[{workload_name}](#{file_path.stem.replace('_', '-')})|" + + f"{number_of_jobs}|{iodepth}|{max_throughput_with_ts}|{max_latency_with_ts}|" + ) + + if self._plot_resources: + data += f"{cpu_usage}|" + + # Convert iodepth to int for sorting (handle "N/A" case) + try: + iodepth_int = int(iodepth) + except ValueError: + iodepth_int = 0 # Put N/A entries at the beginning + + table_rows.append((workload_name, iodepth_int, data)) + + # Sort by workload name, then by iodepth + table_rows.sort(key=lambda x: (x[0], x[1])) + + # Output table if we have data + if table_rows: + self._report.new_line(text=headers) + self._report.new_line(text=alignment) + for _, _, data_row in table_rows: + self._report.new_line(text=data_row) + + def _extract_iodepth_from_path(self, file_path: Path) -> str: + """ + Extract iodepth value from the file path. + + Looks for 'total_iodepth-XXX' or 'iodepth-XXX' in the path. + Prefers total_iodepth if both exist. + + Args: + file_path: Path to the data file + + Returns: + The iodepth value as a string, or "N/A" if not found + """ + path_parts = file_path.parts + + # Look for total_iodepth first (higher priority) + for part in path_parts: + if part.startswith("total_iodepth-"): + return part.split("-")[1] + + # If no total_iodepth, look for iodepth + for part in path_parts: + if part.startswith("iodepth-"): + return part.split("-")[1] + + return "N/A" + + def _get_max_metrics_with_timestamps(self, file_path: Path) -> tuple[str, str]: # pylint: disable=too-many-locals + """ + Get maximum throughput and latency with their timestamps from a time-series file. + + Reads the pre-calculated maximum values and timestamps from the intermediate file. + Formats them as: @ + + Args: + file_path: Path to the time-series intermediate file + + Returns: + Tuple of (max_throughput_with_timestamp, max_latency_with_timestamp) + """ + data: CommonFormatDataType = read_intermediate_file(file_path=f"{file_path}") + + # Get blocksize to determine whether to use IOPS or bandwidth + blocksize_str = data.get("blocksize", "0") + assert isinstance(blocksize_str, str) + blocksize: int = int(int(blocksize_str) / 1024) + + # Determine which metrics to use based on blocksize + if blocksize >= 64: + # Use bandwidth for large blocksizes + throughput_key = "maximum_bandwidth" + throughput_timestamp_key = "timestamp_at_max_bandwidth" + throughput_type = "MB/s" + + throughput = data[throughput_key] + assert isinstance(throughput, str) + max_throughput = float(throughput) / (1000 * 1000) # Convert bytes to MB/s + + timestamp = data[throughput_timestamp_key] + assert isinstance(timestamp, str) + throughput_timestamp = float(timestamp) + + max_throughput_str = f"{max_throughput:.0f} {throughput_type}@{throughput_timestamp:.1f}s" + else: + # Use IOPS for small blocksizes + throughput_key = "maximum_iops" + throughput_timestamp_key = "timestamp_at_max_iops" + throughput_type = "IOps" + + throughput = data[throughput_key] + assert isinstance(throughput, str) + max_throughput = float(throughput) + + timestamp = data[throughput_timestamp_key] + assert isinstance(timestamp, str) + throughput_timestamp = float(timestamp) + + max_throughput_str = f"{max_throughput:.0f} {throughput_type}@{throughput_timestamp:.1f}s" + + # Get maximum latency and its timestamp + latency_key = "maximum_latency" + latency_timestamp_key = "timestamp_at_max_latency" + + latency = data[latency_key] + assert isinstance(latency, str) + max_latency = float(latency) + + latency_ts = data[latency_timestamp_key] + assert isinstance(latency_ts, str) + latency_timestamp = float(latency_ts) + + max_latency_str = f"{max_latency:.1f}@{latency_timestamp:.1f}s" + + return (max_throughput_str, max_latency_str) + + def _add_configuration_yaml_files(self) -> None: + """ + Add the configuration yaml file to the report. + + Same as simple report - single YAML file. + """ + self._report.new_header(level=1, title="Configuration yaml") + yaml_file_paths: list[Path] = self._find_configuration_yaml_files() + + if not yaml_file_paths: + log.warning("No configuration YAML files found") + return + + yaml_file_path: Path = yaml_file_paths[0] + + file_contents: str = yaml_file_path.read_text() + safe_contents = strip_confidential_data_from_yaml(file_contents) + markdown_string: str = f"```{safe_contents}```" + + self._report.new_paragraph(markdown_string) + + def _copy_images(self) -> None: + """ + Copy the time-series plot files to a 'plots' subdirectory in the output directory + so the markdown can link to them using a known relative path. + + If plots don't exist or force_refresh is set, generate them using TimeSeriesPlotter. + """ + plot_files: list[Path] = [] + for directory in self._data_directories: + # Check for existing time-series plot files in this directory + existing_plots = list(directory.glob(f"*_timeseries{PLOT_FILE_EXTENSION_WITH_DOT}")) + + # If there are no plot files in the directory or force_refresh is set, create them + if len(existing_plots) == 0 or self._force_refresh: + log.info("Generating time-series plots for %s", directory.parent) + ts_plotter = TimeSeriesPlotter(str(directory.parent), plotter, figure_size=(12, 6), dpi=100) + ts_plotter.draw_and_save() + # Collect the newly generated plot files + plot_files.extend(list(directory.glob(f"*_timeseries{PLOT_FILE_EXTENSION_WITH_DOT}"))) + else: + # Use existing plots + plot_files.extend(existing_plots) + + # Copy all plot files to the plots directory + for plot_file in plot_files: + # Use shell=False for security - pass command as list + subprocess.call(["/usr/bin/env", "cp", str(plot_file), f"{self._plots_directory}/"]) + + log.debug( + "Copied %d time-series plot files to %s", + len(plot_files), + self._plots_directory, + ) + + def _find_and_sort_file_paths(self, paths: list[Path], search_pattern: str, index: Optional[int] = 0) -> list[Path]: + """ + Given the search_pattern find all the files in a Path that match + that pattern, and return them as a list sorted numerically by file name. + + Same implementation as SimpleReportGenerator. + """ + unsorted_paths: list[Path] = list(paths[0].glob(search_pattern)) + assert index is not None + return self._sort_list_of_paths(unsorted_paths, index) + + def _find_and_sort_plot_files(self) -> list[Path]: + """ + Find all the time-series plot files in the directory. + + Overrides base class to only include files with _timeseries suffix, + filtering out simple and comparison plot files. + + Sorts by numjobs (for section grouping), then blocksize, then iodepth. + This ensures plots appear in the correct order within each numjobs section, + matching the summary table sorting. + + Returns: + List of Path objects for time-series plot files only, sorted by + (numjobs, blocksize, iodepth) + """ + all_plots = self._find_and_sort_file_paths( + paths=[self._plots_directory], search_pattern=f"*{PLOT_FILE_EXTENSION_WITH_DOT}" + ) + # Filter to only include time-series plots (those with _timeseries in the stem) + timeseries_plots = [plot for plot in all_plots if "_timeseries" in plot.stem] + + # Sort by numjobs, blocksize, and iodepth + # Format: {blocksize}_{numjobs}_{operation}_{iodepth}_{metric}_timeseries.svg + def get_sort_key(plot_path: Path) -> tuple[int, int, int]: + parts = plot_path.stem.split("_") + blocksize = int(parts[0]) + numjobs = int(parts[1]) + # iodepth is the 4th part (index 3), default to 0 if not present + iodepth = int(parts[3]) if len(parts) > 3 else 0 + return (numjobs, blocksize, iodepth) + + return sorted(timeseries_plots, key=get_sort_key) + + def _find_and_sort_data_files(self) -> dict[str, list[Path]]: + """ + Find and sort all the time-series data files that will be needed for + producing the summary table. + + Overrides base class to look for *_timeseries.json files instead of + regular .json files. + """ + # Find common time-series file names across all data directories + unique_file_names: list[str] = self._find_common_timeseries_file_names() + + # Sort by blocksize and numjobs + sorted_data_file_names: list[str] = sorted( + unique_file_names, + key=lambda file_name: ( + int(file_name.split("_")[0]), + int(file_name.split("_")[1]), + ), + ) + + sorted_data_files: dict[str, list[Path]] = {} + for file_name in sorted_data_file_names: + file_name_without_extension: str = file_name[: -len(DATA_FILE_EXTENSION_WITH_DOT)] + sorted_data_files[file_name_without_extension] = self._find_files_with_filename(file_name_without_extension) + + return sorted_data_files + + def _find_common_timeseries_file_names(self) -> list[str]: + """ + Find common time-series file names across all data directories. + + Returns: + List of time-series JSON file names (with extension) + """ + if not self._data_directories: + return [] + + # Collect all time-series file names from all directories + all_files: set[str] = set() + for directory in self._data_directories: + files = set(path.parts[-1] for path in directory.glob(f"*_timeseries{DATA_FILE_EXTENSION_WITH_DOT}")) + all_files.update(files) + + # For a single archive, return all unique files + if len(self._data_directories) == 1: + return sorted(list(all_files)) + + # Check if all directories share a common ancestor (same archive) + # For time-series reports from a single archive, we want all unique files + # even if they're in different subdirectories (different iodepth values) + try: + # Check if all directories are from the same archive by looking at the archive root + # Path structure: archive/results/00000000/id-xxx/workload/rbdfio/ + # numjobs-xxx/total_iodepth-xxx/visualisation + # We need to check if they all share the same archive root (before /results/) + archive_roots = set() + for d in self._data_directories: + # Find the archive root by looking for the 'results' directory in parents + for i, parent in enumerate(d.parents): + if parent.name == "results" and i > 0: + # The parent before 'results' is the archive root + archive_roots.add(d.parents[i + 1]) + break + + # If all directories are from the same archive, return all unique files + if len(archive_roots) == 1: + log.debug("All directories from same archive, returning %d unique files", len(all_files)) + return sorted(list(all_files)) + except (IndexError, AttributeError): + pass + + # For multiple archives, check if files exist in ALL directories + common_files: set[str] = set() + for file_name in all_files: + # Check if this file exists in all of the directories + found_count = sum(1 for d in self._data_directories if (d / file_name).exists()) + # Only include if found in ALL directories + if found_count == len(self._data_directories): + common_files.add(file_name) + + log.debug("Multiple archives detected, returning %d common files", len(common_files)) + return sorted(list(common_files)) + + def _add_plots(self) -> None: # pylint: disable=too-many-locals + """ + Add the plots to the report in single-column layout. + + Overrides base class to use full-width plots (1 column instead of 2). + Plots are organized by numjobs sections, then by operation, and sorted + by blocksize and iodepth within each section (via _find_and_sort_plot_files). + + Note: This method has many local variables because it consolidates plotting logic. + The complexity is justified as it provides consistent plot organization across reports. + """ + self._report.new_header(level=1, title="Response Curves") + empty_table_header: list[str] = [""] + + for number_of_jobs in self._get_all_number_of_jobs_values(): + self._report.new_header(level=2, title=f"Number of Jobs {number_of_jobs}") + image_tables: dict[str, list[str]] = {} + + for _, operation in TITLE_CONVERSION.items(): + image_tables[operation] = empty_table_header.copy() + + for image_file in self._plot_files: + # Get the file stem without any prefix (e.g., "Comparison_") + file_stem = self._get_plot_file_stem(image_file) + (blocksize, percent, operation, numjobs) = get_blocksize_percentage_operation_numjobs_from_file_name( + file_stem + ) + if numjobs == number_of_jobs: + title: str = f"{blocksize} {percent} {operation}" + + image_line: str = self._report.new_inline_image( + text=title, path=f"{self._plots_directory.parts[-1]}/{image_file.parts[-1]}" + ) + anchor: str = f'' + + image_line = f"{anchor}{image_line}" + + image_tables[operation].append(image_line) + + # Create the correct sections and add a table for each section to the report + for section, data in image_tables.items(): + # We don't want to display a section if it doesn't contain any plots + if len(data) > len(empty_table_header): + self._report.new_header(level=3, title=section) + table_images = data + + # Single column layout - one row per image + number_of_rows: int = len(table_images) + self._report.new_table(columns=1, rows=number_of_rows, text=table_images, text_align="center") + + +# Made with Bob diff --git a/post_processing/run_results/benchmarks/benchmark_result.py b/post_processing/run_results/benchmark_result.py similarity index 74% rename from post_processing/run_results/benchmarks/benchmark_result.py rename to post_processing/run_results/benchmark_result.py index 806244b3..5a05b493 100644 --- a/post_processing/run_results/benchmarks/benchmark_result.py +++ b/post_processing/run_results/benchmark_result.py @@ -8,15 +8,15 @@ from abc import ABC, abstractmethod from logging import Logger, getLogger from pathlib import Path -from typing import Any +from typing import Any, Optional from post_processing.common import file_is_empty, get_blocksize -from post_processing.post_processing_types import IodepthDataType, JobsDataType +from post_processing.post_processing_types import IodepthDataType, JobsDataType, TimeSeriesFormatType log: Logger = getLogger("formatter") -class BenchmarkResult(ABC): # pylint: disable=[too-many-instance-attributes] +class BenchmarkResult(ABC): # pylint: disable=too-many-instance-attributes """ This is the top level class for a benchmark run result. As each benchmark tool produces different output results we will need a @@ -28,8 +28,8 @@ def __init__(self, file_path: Path) -> None: self._data: dict[str, Any] = self._read_results_from_file() if not self._data: raise ValueError(f"File {file_path} is empty") - self._number_of_jobs: str = "" + self._number_of_jobs: str = "" # Will be set by subclass in _get_global_options self._global_options: dict[str, str] = self._get_global_options(self._data["global options"]) self._iodepth = self._get_iodepth(f"{self._data['global options']['iodepth']}") self._io_details: IodepthDataType = self._get_io_details(self._data["jobs"]) @@ -71,14 +71,14 @@ def _get_iodepth(self, iodepth_value: str) -> str: @property def blocksize(self) -> str: """ - Getter for blocksize + Return the blocksize for this benchmark run. """ return get_blocksize(f"{self._data['global options']['bs']}") @property def operation(self) -> str: """ - Getter for the operation (read, write, rw etc) + Return the operation type for this benchmark run. """ operation: str = f"{self._data['global options']['rw']}" if self._global_options.get("percentage_reads", None): @@ -91,31 +91,53 @@ def operation(self) -> str: @property def global_options(self) -> dict[str, str]: """ - Getter for the global options from this run + Return the global options for this benchmark run. """ return self._global_options @property def iodepth(self) -> str: """ - Getter for iodepth value + Return the IO depth for this benchmark run. """ return self._iodepth @property def io_details(self) -> IodepthDataType: """ - Getter for the I/O details + Return the IO details for this benchmark run. """ return self._io_details @property def number_of_jobs(self) -> str: """ - Getter for number of jobs + Return the number of jobs for this benchmark run. """ return self._number_of_jobs + @property + def resource_file_path(self) -> Path: + """ + Return the resource file path for this benchmark run. + """ + return self._resource_file_path + + @abstractmethod + def get_timeseries_data(self) -> Optional[TimeSeriesFormatType]: + """ + Get time-series data if available for this benchmark. + + Subclasses must implement this method to either: + - Return TimeSeriesFormatType with time-indexed metrics if supported + - Return None if time-series data is not available for this benchmark + + This forces explicit consideration of time-series support for each benchmark type. + + Returns: + TimeSeriesFormatType with time-indexed metrics, or None if not supported + """ + def _read_results_from_file(self) -> dict[str, Any]: """ Read the data from the results file and return the results in a dict diff --git a/post_processing/run_results/benchmarks/fio.py b/post_processing/run_results/benchmarks/fio.py index e0de513d..e9fdfd10 100644 --- a/post_processing/run_results/benchmarks/fio.py +++ b/post_processing/run_results/benchmarks/fio.py @@ -3,16 +3,27 @@ """ from logging import Logger, getLogger +from pathlib import Path +from typing import Optional, Union + +import pandas as pd from post_processing.common import get_blocksize, sum_mean_values, sum_standard_deviation_values +from post_processing.parsers.fio_log_parser import FIOLogParser +from post_processing.parsers.fio_time_series_parser import FIOTimeSeriesParser +from post_processing.parsers.timestamp_aligner import TimestampAligner from post_processing.post_processing_types import ( IodepthDataType, JobsDataType, + TimeSeriesFormatType, ) -from post_processing.run_results.benchmarks.benchmark_result import BenchmarkResult +from post_processing.run_results.benchmark_result import BenchmarkResult log: Logger = getLogger("formatter") +# Constants for job types to process +_READ_WRITE_JOBS: frozenset[str] = frozenset(["read", "write"]) + class FIO(BenchmarkResult): """ @@ -45,66 +56,170 @@ def _get_global_options(self, fio_global_options: dict[str, str]) -> dict[str, s return global_options_details + def _extract_int_metric( + self, job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]], key: str, job_type: str + ) -> int: + """ + Safely extract and validate an integer metric from job data. + + Args: + job_data: Dictionary containing job metrics + key: Metric key to extract + job_type: Job type name for error messages + + Returns: + Integer value of the metric + + Raises: + ValueError: If metric is missing or not an integer + """ + value = job_data.get(key) + if not isinstance(value, int): + raise ValueError(f"Missing or invalid '{key}' for {job_type} job: expected int, got {type(value).__name__}") + return value + + def _extract_float_metric( + self, job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]], key: str, job_type: str + ) -> float: + """ + Safely extract and validate a float metric from job data. + + Args: + job_data: Dictionary containing job metrics + key: Metric key to extract + job_type: Job type name for error messages + + Returns: + Float value of the metric + + Raises: + ValueError: If metric is missing or not a float + """ + value = job_data.get(key) + if not isinstance(value, (int, float)): + raise ValueError( + f"Missing or invalid '{key}' for {job_type} job: expected float, got {type(value).__name__}" + ) + return float(value) + # pylint: disable=[too-many-locals] def _get_io_details(self, all_jobs: JobsDataType) -> IodepthDataType: """ - In the fio output the details are split by operation (read, write) so to - get the total IO numbers we need to sum together the details for read and write. - For a single operation e.g. read then the write details will all be 0 so - this still gives the correct values. - - The values of interest are: - io_bytes - bw_bytes - clat_ns/mean - clat_ns/stddev - iops - total_ios - """ - jobs_of_interest: list[str] = ["read", "write"] - io_details: dict[str, str] = {} + Aggregate IO metrics across read and write operations from FIO output. + + FIO splits metrics by operation type (read/write), so this method combines + them to get total IO statistics. For single-operation tests (e.g., read-only), + the unused operation will have zero values, so summation still works correctly. + + The method calculates weighted averages for latency metrics using the formula: + combined_mean = sum(mean_i * num_ops_i) / total_ops + combined_stddev = sqrt((sum((n_i-1)*stddev_i^2 + n_i*mean_i^2) - N*mean_combined^2) / (N-1)) + + Args: + all_jobs: List of job dictionaries from FIO JSON output, each containing + 'read' and 'write' keys with their respective metrics + + Returns: + Dictionary with aggregated metrics: + - io_bytes: Total bytes transferred + - bandwidth_bytes: Total bandwidth in bytes/sec + - iops: Total IO operations per second + - latency: Weighted mean completion latency in nanoseconds + - std_deviation: Combined standard deviation of latency + - total_ios: Total number of IO operations + + Raises: + ValueError: If job data structure is invalid or missing required fields + """ + # Initialize accumulators io_bytes: int = 0 bw_bytes: int = 0 + io_operations_second: float = 0.0 + total_ios: int = 0 + + # Lists for weighted statistical calculations latencies: list[float] = [] operations: list[int] = [] std_deviations: list[float] = [] - io_operations_second: float = 0 - total_ios: int = 0 - for entry in all_jobs: # A single run in the json - for job, job_data in entry.items(): - if job in jobs_of_interest and isinstance(job_data, dict): - assert isinstance(job_data["io_bytes"], int) - io_bytes += job_data["io_bytes"] - assert isinstance(job_data["bw_bytes"], int) - bw_bytes += job_data["bw_bytes"] - assert isinstance(job_data["iops"], float) - io_operations_second += job_data["iops"] - assert isinstance(job_data["total_ios"], int) - num_ops: int = job_data["total_ios"] + # Process each job entry + for job_entry in all_jobs: + for job_type, job_data in job_entry.items(): + # Only process read/write operations + if job_type not in _READ_WRITE_JOBS: + continue + + # Validate job data structure + if not isinstance(job_data, dict): + log.warning( + "Skipping job '%s' in %s: expected dict, got %s", + job_type, + self._resource_file_path, + type(job_data).__name__, + ) + continue + + try: + # Extract and validate metrics with proper error handling + io_bytes += self._extract_int_metric(job_data, "io_bytes", job_type) + bw_bytes += self._extract_int_metric(job_data, "bw_bytes", job_type) + io_operations_second += self._extract_float_metric(job_data, "iops", job_type) + + num_ops = self._extract_int_metric(job_data, "total_ios", job_type) operations.append(num_ops) total_ios += num_ops - assert isinstance(job_data["clat_ns"], dict) - latencies.append(float(job_data["clat_ns"]["mean"])) - std_deviations.append(float(job_data["clat_ns"]["stddev"])) - combined_mean_latency = sum_mean_values(latencies, operations, total_ios) + # Extract latency statistics + clat_ns = job_data.get("clat_ns") + if not isinstance(clat_ns, dict): + raise ValueError( + f"Missing or invalid 'clat_ns' for {job_type} job: " + f"expected dict, got {type(clat_ns).__name__}" + ) + + mean_latency = float(clat_ns.get("mean", 0)) + stddev_latency = float(clat_ns.get("stddev", 0)) + + latencies.append(mean_latency) + std_deviations.append(stddev_latency) + + except (KeyError, ValueError, TypeError) as e: + log.error( + "Error processing %s job in %s: %s", + job_type, + self._resource_file_path, + str(e), + ) + raise ValueError(f"Invalid job data structure for {job_type} in {self._resource_file_path}") from e + # Validate we have data to process + if total_ios == 0: + log.warning("No IO operations found in %s", self._resource_file_path) + return { + "io_bytes": "0", + "bandwidth_bytes": "0", + "iops": "0.0", + "latency": "0.0", + "std_deviation": "0.0", + "total_ios": "0", + } + + # Calculate weighted statistics + combined_mean_latency = sum_mean_values(latencies, operations, total_ios) latency_standard_deviation = sum_standard_deviation_values( std_deviations, operations, latencies, total_ios, combined_mean_latency ) - io_details = { - "io_bytes": f"{io_bytes}", - "bandwidth_bytes": f"{bw_bytes}", - "iops": f"{io_operations_second}", - "latency": f"{combined_mean_latency}", - "std_deviation": f"{latency_standard_deviation}", - "total_ios": f"{total_ios}", + # Return aggregated metrics + return { + "io_bytes": str(io_bytes), + "bandwidth_bytes": str(bw_bytes), + "iops": str(io_operations_second), + "latency": str(combined_mean_latency), + "std_deviation": str(latency_standard_deviation), + "total_ios": str(total_ios), } - return io_details - def _get_iodepth(self, iodepth_value: str) -> str: log.debug("Getting iodepth from %s and %s", iodepth_value, self._resource_file_path) iodepth: int = int(iodepth_value) @@ -137,3 +252,256 @@ def _get_iodepth(self, iodepth_value: str) -> str: log.debug("iodepth value is %s", iodepth) return str(iodepth) + + def _get_log_avg_msec(self) -> int: + """ + Extract and validate log_avg_msec from global options. + + Returns: + The log averaging interval in milliseconds (default: 1000) + """ + default_value = 1000 + raw_value = self._global_options.get("log_avg_msec", default_value) + + try: + log_avg_msec = int(raw_value) + if log_avg_msec <= 0: + log.warning( + "Invalid log_avg_msec value %d (must be positive), using default %d", + log_avg_msec, + default_value, + ) + return default_value + return log_avg_msec + except (ValueError, TypeError) as e: + log.warning( + "Cannot convert log_avg_msec '%s' to int: %s, using default %d", + raw_value, + e, + default_value, + ) + return default_value + + def _validate_log_directory(self, log_directory: Path, base_name: str) -> bool: + """ + Validate that the log directory exists and is accessible. + + Args: + log_directory: Path to the log directory + base_name: Base name for log files + + Returns: + True if valid, False otherwise + """ + if not log_directory.exists(): + log.warning("Log directory does not exist: %s", log_directory) + return False + + if not log_directory.is_dir(): + log.error("Log path is not a directory: %s", log_directory) + return False + + log.debug("Looking for time-series logs for %s in %s", base_name, log_directory) + return True + + def _parse_metric_logs(self, log_directory: Path, base_name: str) -> Optional[dict[str, Optional[pd.DataFrame]]]: + """ + Parse all FIO metric log files. + + Args: + log_directory: Path to the log directory + base_name: Base name for log files + + Returns: + Dictionary of parsed metric data, or None if no logs found + """ + parser = FIOLogParser() + + # Define metric types and their patterns for cleaner iteration + metric_patterns = { + "iops": f"{base_name}_iops.*.log", + "clat": f"{base_name}_clat.*.log", + "bw": f"{base_name}_bw.*.log", + "lat": f"{base_name}_lat.*.log", + "slat": f"{base_name}_slat.*.log", + } + + # Parse all metric types + parsed_data = { + metric: parser.parse_and_combine_logs(log_directory, metric, pattern) + for metric, pattern in metric_patterns.items() + } + + # Early return if no logs found + if all(dataframe is None for dataframe in parsed_data.values()): + log.debug("No time-series log files found for %s", base_name) + return None + + log.debug("Found time-series logs for %s, formatting data", base_name) + return parsed_data + + def _align_throughput_metrics( + self, parsed_data: dict[str, Optional[pd.DataFrame]], aligner: TimestampAligner + ) -> tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]: + """ + Align throughput metrics (IOPS and bandwidth) to common time windows. + + Args: + parsed_data: Dictionary of parsed metric data + aligner: TimestampAligner instance + + Returns: + Tuple of (aligned_iops, aligned_bw) + """ + aligned_iops = None + aligned_bw = None + + if parsed_data["iops"] is not None and not parsed_data["iops"].empty: + aligned_iops = aligner.align_and_aggregate([parsed_data["iops"]], "iops") + log.debug("Aligned IOPS data to %d time windows", len(aligned_iops)) + + if parsed_data["bw"] is not None and not parsed_data["bw"].empty: + aligned_bw = aligner.align_and_aggregate([parsed_data["bw"]], "bandwidth_bytes") + log.debug("Aligned bandwidth data to %d time windows", len(aligned_bw)) + + return aligned_iops, aligned_bw + + def _align_latency_metrics( + self, parsed_data: dict[str, Optional[pd.DataFrame]], aligner: TimestampAligner + ) -> tuple[Optional[pd.DataFrame], Optional[pd.DataFrame]]: + """ + Align latency metrics (mean and max) to common time windows. + + Args: + parsed_data: Dictionary of parsed metric data + aligner: TimestampAligner instance + + Returns: + Tuple of (aligned_mean_latency, aligned_max_latency) + """ + aligned_mean_latency = None + aligned_max_latency = None + + if parsed_data["clat"] is not None and not parsed_data["clat"].empty: + aligned_mean_latency = aligner.align_and_aggregate([parsed_data["clat"]], "latency_ms") + log.debug("Aligned mean latency (clat) data to %d time windows", len(aligned_mean_latency)) + + if parsed_data["lat"] is not None and not parsed_data["lat"].empty: + aligned_max_latency = aligner.align_and_aggregate([parsed_data["lat"]], "latency_ms") + log.debug("Aligned max latency (lat) data to %d time windows", len(aligned_max_latency)) + + return aligned_mean_latency, aligned_max_latency + + def _calculate_latency_percentiles( + self, parsed_data: dict[str, Optional[pd.DataFrame]], aligner: TimestampAligner + ) -> tuple[Optional[pd.DataFrame], Optional[pd.DataFrame], Optional[pd.DataFrame]]: + """ + Calculate latency percentiles from raw clat data. + + Args: + parsed_data: Dictionary of parsed metric data + aligner: TimestampAligner instance + + Returns: + Tuple of (p50_dataframe, p95_dataframe, p99_dataframe) + """ + p50_dataframe = None + p95_dataframe = None + p99_dataframe = None + + if parsed_data["clat"] is not None and not parsed_data["clat"].empty: + log.debug("Calculating latency percentiles from clat data") + percentiles_dataframe = aligner.calculate_percentiles([parsed_data["clat"]], percentiles=[50, 95, 99]) + + if not percentiles_dataframe.empty: + # Split into separate DataFrames for each percentile + if "p50_latency_ms" in percentiles_dataframe.columns: + p50_dataframe = percentiles_dataframe[["timestamp_sec", "p50_latency_ms"]].rename( + columns={"p50_latency_ms": "latency_ms"} + ) + if "p95_latency_ms" in percentiles_dataframe.columns: + p95_dataframe = percentiles_dataframe[["timestamp_sec", "p95_latency_ms"]].rename( + columns={"p95_latency_ms": "latency_ms"} + ) + if "p99_latency_ms" in percentiles_dataframe.columns: + p99_dataframe = percentiles_dataframe[["timestamp_sec", "p99_latency_ms"]].rename( + columns={"p99_latency_ms": "latency_ms"} + ) + log.debug("Calculated percentiles for %d time windows", len(percentiles_dataframe)) + else: + log.warning("Percentile calculation returned empty DataFrame") + else: + log.debug("No clat data available for percentile calculation") + + return p50_dataframe, p95_dataframe, p99_dataframe + + def get_timeseries_data(self) -> Optional[TimeSeriesFormatType]: + """ + Parse FIO time-series logs and return formatted data. + + This method looks for FIO time-series log files matching the current + benchmark output file prefix (for example output.0_*). If found, it parses + that single file's logs into the TimeSeriesFormatType intermediate format. + + Returns: + TimeSeriesFormatType with time-indexed metrics if logs exist, None otherwise + """ + try: + log_directory = self._resource_file_path.parent + # Strip 'json_' prefix from filename to match actual log file names + # e.g., 'json_output.0' -> 'output.0' to match 'output.0_iops.1.log' + base_name = self._resource_file_path.name.replace("json_", "") + + # Validate log directory + if not self._validate_log_directory(log_directory, base_name): + return None + + # Parse metric logs + parsed_data = self._parse_metric_logs(log_directory, base_name) + if parsed_data is None: + return None + + # Align all metrics to common time windows + log_avg_msec = self._get_log_avg_msec() + aligner = TimestampAligner(window_size_ms=log_avg_msec) + + # Align throughput and latency metrics + aligned_iops, aligned_bw = self._align_throughput_metrics(parsed_data, aligner) + aligned_mean_latency, aligned_max_latency = self._align_latency_metrics(parsed_data, aligner) + + # Calculate percentiles + p50_dataframe, p95_dataframe, p99_dataframe = self._calculate_latency_percentiles(parsed_data, aligner) + + # Get iodepth value (prefers total_iodepth if it exists) + iodepth_value = self._get_iodepth(self.iodepth) + + # Initialize parser with aligned data including calculated percentiles + timeseries_parser = FIOTimeSeriesParser( + archive_directory=str(log_directory), + benchmark="fio", + operation=self.operation, + blocksize=self.blocksize, + numjobs=self._number_of_jobs, + iodepth=iodepth_value, + iops_df=aligned_iops, + bandwidth_df=aligned_bw, + mean_latency_df=aligned_mean_latency, + max_latency_df=aligned_max_latency, + p50_latency_df=p50_dataframe, + p95_latency_df=p95_dataframe, + p99_latency_df=p99_dataframe, + num_volumes=1, + log_avg_msec=log_avg_msec, + ) + + timeseries_parser.process() + + # Return the formatted time-series data for aggregation at RunResult level + return timeseries_parser.get_formatted_output() + + except (OSError, PermissionError) as e: + log.error("File system error accessing logs for %s: %s", self._resource_file_path, e) + return None + except (ValueError, KeyError, AttributeError) as e: + log.error("Data parsing error processing time-series data for %s: %s", self._resource_file_path, e) + return None diff --git a/post_processing/run_results/rbdfio.py b/post_processing/run_results/rbdfio.py index 330f736a..73165538 100644 --- a/post_processing/run_results/rbdfio.py +++ b/post_processing/run_results/rbdfio.py @@ -6,17 +6,16 @@ import re from logging import Logger, getLogger from pathlib import Path -from typing import Union +from typing import Literal, Union, cast from post_processing.common import sum_mean_values, sum_standard_deviation_values -from post_processing.post_processing_types import IodepthDataType - -# from post_processing.run_results.benchmarks.benchmark_result import BenchmarkResult -# To be removed when factory is ready +from post_processing.post_processing_types import IodepthDataType, TimeSeriesDataPoint, TimeSeriesFormatType +from post_processing.run_results.benchmark_result import BenchmarkResult +from post_processing.run_results.benchmarks.fio import FIO +from post_processing.run_results.resource_result import ResourceResult +from post_processing.run_results.resources.fio_resource import FIOResource from post_processing.run_results.run_result import RunResult -# from post_processing.run_results.resources.resource_result import ResourceResult - log: Logger = getLogger(name="formatter") @@ -93,3 +92,179 @@ def _sum_io_details( combined_data["std_deviation"] = f"{combined_std_dev}" return combined_data + + def _merge_timeseries_data( # pylint: disable=too-many-locals,too-many-statements + self, + test_config: tuple[str, str, str, str], + new_timeseries: TimeSeriesFormatType, + ) -> TimeSeriesFormatType: + """ + Aggregate time-series data across multiple per-volume FIO result files. + + For matching timestamps: + - Throughput metrics (IOPS, bandwidth) are summed across volumes + - Latency metrics are weighted-averaged by IOPS (more accurate than simple average) + - Maximum latency uses the maximum value across volumes + """ + operation, blocksize, iodepth, _ = test_config + key = f"{operation}_{blocksize}_{iodepth}" + existing_timeseries = self._timeseries_data.get(key) + if not existing_timeseries: + return new_timeseries + + existing_points = {point["timestamp_sec"]: point for point in existing_timeseries["timeseries"]} + new_points = {point["timestamp_sec"]: point for point in new_timeseries["timeseries"]} + all_timestamps = sorted(set(existing_points) | set(new_points)) + + combined_timeseries: list[TimeSeriesDataPoint] = [] + throughput_metrics: list[Literal["iops", "bandwidth_bytes"]] = ["iops", "bandwidth_bytes"] + latency_metrics: list[Literal["mean_latency_ms", "p50_latency_ms", "p95_latency_ms", "p99_latency_ms"]] = [ + "mean_latency_ms", + "p50_latency_ms", + "p95_latency_ms", + "p99_latency_ms", + ] + + for timestamp in all_timestamps: + existing_point = existing_points.get(timestamp) + new_point = new_points.get(timestamp) + + combined_point: TimeSeriesDataPoint = { + "timestamp_sec": timestamp, + "iops": 0.0, + "bandwidth_bytes": 0.0, + "mean_latency_ms": 0.0, + "max_latency_ms": 0.0, + "p50_latency_ms": 0.0, + "p95_latency_ms": 0.0, + "p99_latency_ms": 0.0, + "num_samples": 0, + } + + existing_metric_values = cast(dict[str, float], existing_point) if existing_point else {} + new_metric_values = cast(dict[str, float], new_point) if new_point else {} + + # Sum throughput metrics (IOPS, bandwidth) + for metric in throughput_metrics: + combined_point[metric] = float(existing_metric_values.get(metric, 0.0)) + float( + new_metric_values.get(metric, 0.0) + ) + + # Calculate weighted average for latency metrics (weighted by IOPS) + existing_iops = float(existing_metric_values.get("iops", 0.0)) + new_iops = float(new_metric_values.get("iops", 0.0)) + total_iops = existing_iops + new_iops + + if total_iops > 0: + for latency_metric in latency_metrics: + existing_latency = float(existing_metric_values.get(latency_metric, 0.0)) + new_latency = float(new_metric_values.get(latency_metric, 0.0)) + # Weighted average: (lat1 * iops1 + lat2 * iops2) / (iops1 + iops2) + combined_point[latency_metric] = ( + existing_latency * existing_iops + new_latency * new_iops + ) / total_iops + else: + # If no IOPS, use simple average (fallback) + for latency_metric in latency_metrics: + existing_latency = float(existing_metric_values.get(latency_metric, 0.0)) + new_latency = float(new_metric_values.get(latency_metric, 0.0)) + combined_point[latency_metric] = (existing_latency + new_latency) / 2.0 + + # Maximum latency should be the maximum across volumes + combined_point["max_latency_ms"] = max( + float(existing_metric_values.get("max_latency_ms", 0.0)), + float(new_metric_values.get("max_latency_ms", 0.0)), + ) + + combined_point["num_samples"] = int(existing_point.get("num_samples", 0) if existing_point else 0) + int( + new_point.get("num_samples", 0) if new_point else 0 + ) + + combined_timeseries.append(combined_point) + + start_time = combined_timeseries[0]["timestamp_sec"] if combined_timeseries else 0.0 + end_time = combined_timeseries[-1]["timestamp_sec"] if combined_timeseries else 0.0 + + # Calculate maximum values from the combined timeseries + # (similar to _calculate_maximum_values in FIOTimeSeriesParser) + if combined_timeseries: + max_iops_point = max(combined_timeseries, key=lambda p: p["iops"]) + maximum_iops = max_iops_point["iops"] + latency_at_max_iops = max_iops_point["mean_latency_ms"] + timestamp_at_max_iops = max_iops_point["timestamp_sec"] + + max_bandwidth_point = max(combined_timeseries, key=lambda p: p["bandwidth_bytes"]) + maximum_bandwidth = max_bandwidth_point["bandwidth_bytes"] + latency_at_max_bandwidth = max_bandwidth_point["mean_latency_ms"] + timestamp_at_max_bandwidth = max_bandwidth_point["timestamp_sec"] + + max_latency_point = max(combined_timeseries, key=lambda p: p["mean_latency_ms"]) + maximum_latency = max_latency_point["mean_latency_ms"] + timestamp_at_max_latency = max_latency_point["timestamp_sec"] + else: + maximum_iops = 0.0 + latency_at_max_iops = 0.0 + timestamp_at_max_iops = 0.0 + maximum_bandwidth = 0.0 + latency_at_max_bandwidth = 0.0 + timestamp_at_max_bandwidth = 0.0 + maximum_latency = 0.0 + timestamp_at_max_latency = 0.0 + + return { + "benchmark": new_timeseries["benchmark"], + "operation": new_timeseries["operation"], + "blocksize": new_timeseries["blocksize"], + "numjobs": new_timeseries["numjobs"], + "iodepth": new_timeseries.get("iodepth", iodepth), + "metadata": { + "start_time_epoch": start_time, + "end_time_epoch": end_time, + "duration_seconds": end_time - start_time, + "num_volumes": int(existing_timeseries["metadata"]["num_volumes"]) + + int(new_timeseries["metadata"]["num_volumes"]), + "sampling_interval_ms": int(new_timeseries["metadata"]["sampling_interval_ms"]), + "log_avg_msec": int(new_timeseries["metadata"]["log_avg_msec"]), + }, + "timeseries": combined_timeseries, + "maximum_iops": f"{maximum_iops:.0f}", + "maximum_bandwidth": f"{maximum_bandwidth:.0f}", + "latency_at_max_iops": f"{latency_at_max_iops:.6f}", + "latency_at_max_bandwidth": f"{latency_at_max_bandwidth:.6f}", + "timestamp_at_max_iops": f"{timestamp_at_max_iops:.6f}", + "timestamp_at_max_bandwidth": f"{timestamp_at_max_bandwidth:.6f}", + "maximum_latency": f"{maximum_latency:.6f}", + "timestamp_at_max_latency": f"{timestamp_at_max_latency:.6f}", + "maximum_cpu_usage": "0.00", # TODO: Add CPU/memory tracking + "maximum_memory_usage": "0.00", + } + + def _create_benchmark_result(self, file_path: Path) -> BenchmarkResult: + """ + Factory method to create FIO benchmark result parser. + + RBDFIO uses FIO as the underlying benchmark tool, so this returns + a FIO instance to parse the benchmark output. + + Args: + file_path: Path to the FIO JSON output file + + Returns: + FIO instance for parsing the benchmark results + """ + return FIO(file_path=file_path) + + def _create_resource_result(self, file_path: Path) -> ResourceResult: + """ + Factory method to create FIO resource result parser. + + RBDFIO uses FIO's resource monitoring, so this returns a FIOResource + instance to parse CPU and memory usage from the output. + + Args: + file_path: Path to the FIO JSON output file + + Returns: + FIOResource instance for parsing resource usage statistics + """ + return FIOResource(file_path=file_path) diff --git a/post_processing/run_results/resources/resource_result.py b/post_processing/run_results/resource_result.py similarity index 96% rename from post_processing/run_results/resources/resource_result.py rename to post_processing/run_results/resource_result.py index 80d1570c..01994e81 100644 --- a/post_processing/run_results/resources/resource_result.py +++ b/post_processing/run_results/resource_result.py @@ -55,18 +55,14 @@ def _parse(self, data: dict[str, Any]) -> None: @property def cpu(self) -> str: - """ - getter for the CPU value - """ + """Return the CPU usage as a string.""" if not self._has_been_parsed: self._parse(self._read_results_from_file()) return self._cpu @property def memory(self) -> str: - """ - getter for the memory value - """ + """Return the memory usage as a string.""" if not self._has_been_parsed: self._parse(self._read_results_from_file()) return self._memory diff --git a/post_processing/run_results/resources/fio_resource.py b/post_processing/run_results/resources/fio_resource.py index 37bdd1fc..a5a4cfbf 100644 --- a/post_processing/run_results/resources/fio_resource.py +++ b/post_processing/run_results/resources/fio_resource.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Any -from post_processing.run_results.resources.resource_result import ResourceResult +from post_processing.run_results.resource_result import ResourceResult log: Logger = getLogger("formatter") diff --git a/post_processing/run_results/run_result.py b/post_processing/run_results/run_result.py index a2ac1709..cabb4cc8 100644 --- a/post_processing/run_results/run_result.py +++ b/post_processing/run_results/run_result.py @@ -3,10 +3,11 @@ common data format that can be plotted """ +import json from abc import ABC, abstractmethod from logging import Logger, getLogger from pathlib import Path -from typing import Union +from typing import Optional, Union, cast from post_processing.common import file_is_empty, file_is_precondition from post_processing.post_processing_types import ( @@ -14,12 +15,10 @@ InternalFormattedOutputType, InternalNumJobsDataType, IodepthDataType, + TimeSeriesFormatType, ) - -# from post_processing.run_results.benchmarks.benchmark_result import BenchmarkResult -# To be removed when factory is ready -from post_processing.run_results.benchmarks.fio import FIO -from post_processing.run_results.resources.fio_resource import FIOResource +from post_processing.run_results.benchmark_result import BenchmarkResult +from post_processing.run_results.resource_result import ResourceResult # from post_processing.run_results.resources.resource_result import ResourceResult @@ -31,12 +30,15 @@ class RunResult(ABC): A result run file that needs processing """ - def __init__(self, directory: Path, file_name_root: str) -> None: + def __init__(self, directory: Path, file_name_root: str, include_timeseries: bool = False) -> None: self._path: Path = directory self._has_been_processed: bool = False + self._include_timeseries: bool = include_timeseries self._files: list[Path] = self._find_files_for_testrun(file_name_root=file_name_root) self._processed_data: InternalFormattedOutputType = {} + self._timeseries_data: dict[str, TimeSeriesFormatType] = {} + self._timeseries_by_directory: dict[Path, dict[str, TimeSeriesFormatType]] = {} @abstractmethod def _find_files_for_testrun(self, file_name_root: str) -> list[Path]: @@ -64,15 +66,77 @@ def type(self) -> str: The benchmark type identifier (e.g., "rbdfio", "fio") """ + @abstractmethod + def _create_benchmark_result(self, file_path: Path) -> BenchmarkResult: + """ + Factory method to create the appropriate BenchmarkResult subclass. + + Subclasses should implement this to return the correct benchmark result + parser based on the benchmark type (e.g., FIO, CosBench, etc.). + + Args: + file_path: Path to the benchmark output file + + Returns: + BenchmarkResult subclass instance for parsing this benchmark type + """ + + @abstractmethod + def _create_resource_result(self, file_path: Path) -> ResourceResult: + """ + Factory method to create the appropriate ResourceResult subclass. + + Subclasses should implement this to return the correct resource result + parser based on the benchmark type (e.g., FIOResource, etc.). + + Args: + file_path: Path to the benchmark output file + + Returns: + ResourceResult subclass instance for parsing resource usage + """ + + def _merge_timeseries_data( + self, + test_config: tuple[str, str, str, str], + new_timeseries: TimeSeriesFormatType, + ) -> TimeSeriesFormatType: + """ + Merge new time-series data with existing data for the same test configuration. + + Default behavior is replacement. Subclasses such as RBDFIO can override + this to aggregate time-series data across multiple files/volumes. + + Args: + test_config: Tuple of (operation, blocksize, iodepth, number_of_jobs) + new_timeseries: Newly parsed time-series data + + Returns: + TimeSeriesFormatType to store for this configuration + """ + operation, blocksize, iodepth, _ = test_config + key = f"{operation}_{blocksize}_{iodepth}" + existing_timeseries = self._timeseries_data.get(key) + if existing_timeseries: + log.debug("Replacing existing time-series data for %s", key) + return new_timeseries + def process(self) -> None: """ Convert the results data from all the individual files that make up this - result into the standard intermediate format + result into the standard intermediate format. + + With memory-efficient approach, timeseries data is aggregated then written + immediately after processing all files to avoid holding data in memory longer than needed. """ number_of_volumes_for_test_run: int = len(self._files) if number_of_volumes_for_test_run > 0: self._process_test_run_files() + # Write timeseries data immediately after processing all files + # Group by aggregation directory and write each group separately + if self._include_timeseries and self._timeseries_by_directory: + self._write_and_clear_timeseries_by_directory() else: log.warning("test run with directory %s has no files - not doing any conversion", self._path) @@ -105,6 +169,221 @@ def _process_test_run_files(self) -> None: else: log.warning("Cannot process file %s as it is empty", file_path) + def _extract_test_configuration(self, benchmark_result: BenchmarkResult) -> tuple[str, str, str, str]: + """ + Extract test configuration parameters from benchmark result. + + Args: + benchmark_result: Parsed benchmark result object + + Returns: + Tuple of (operation, blocksize, iodepth, number_of_jobs) + """ + return ( + benchmark_result.operation, + benchmark_result.blocksize, + benchmark_result.iodepth, + benchmark_result.number_of_jobs, + ) + + def _merge_io_details( + self, test_config: tuple[str, str, str, str], new_io_details: IodepthDataType + ) -> IodepthDataType: + """ + Merge new IO details with existing data for the same test configuration. + + This handles cases where multiple volumes produce results for the same + test parameters, requiring aggregation of metrics. + + Args: + test_config: Tuple of (operation, blocksize, iodepth, number_of_jobs) + new_io_details: New IO details to merge + + Returns: + Merged IO details (either summed with existing or new details) + """ + operation, blocksize, iodepth, number_of_jobs = test_config + + # Check if we have existing data for this configuration + existing_data = self._processed_data.get(operation, {}).get(number_of_jobs, {}).get(blocksize, {}).get(iodepth) + + if existing_data: + log.debug("We have details for iodepth %s so using them", iodepth) + return self._sum_io_details(existing_data, new_io_details) + + return new_io_details + + def _build_test_result_data( + self, + test_config: tuple[str, str, str, str], + io_details: IodepthDataType, + global_details: dict[str, str], + resource_data: dict[str, str], + ) -> InternalNumJobsDataType: + """ + Build the complete nested data structure for a test result. + + Args: + test_config: Tuple of (operation, blocksize, iodepth, number_of_jobs) + io_details: Merged IO performance details + global_details: Global benchmark options + resource_data: Resource usage statistics + + Returns: + Nested dictionary structure: {numjobs: {blocksize: {iodepth: data}}} + """ + _, blocksize, iodepth, number_of_jobs = test_config + + # Build from innermost to outermost level + iodepth_data = {**global_details, **io_details, **resource_data} + iodepth_details = {iodepth: iodepth_data} + blocksize_details = cast(InternalBlocksizeDataType, {blocksize: iodepth_details}) + numjobs_details = cast(InternalNumJobsDataType, {number_of_jobs: blocksize_details}) + + return numjobs_details + + def _update_processed_data( + self, + test_config: tuple[str, str, str, str], + numjobs_details: InternalNumJobsDataType, + ) -> None: + """ + Update the internal processed data structure with new test results. + + Args: + test_config: Tuple of (operation, blocksize, iodepth, number_of_jobs) + numjobs_details: Complete test result data to merge + """ + operation, blocksize, _, number_of_jobs = test_config + + # Extract nested structures for updating + blocksize_details = numjobs_details[number_of_jobs] + iodepth_details = blocksize_details[blocksize] + + # Update at the appropriate nesting level + if operation not in self._processed_data: + self._processed_data[operation] = numjobs_details + elif number_of_jobs not in self._processed_data[operation]: + self._processed_data[operation][number_of_jobs] = blocksize_details + elif blocksize not in self._processed_data[operation][number_of_jobs]: + self._processed_data[operation][number_of_jobs][blocksize] = iodepth_details + else: + self._processed_data[operation][number_of_jobs][blocksize].update(iodepth_details) + + def _process_timeseries_data( + self, test_config: tuple[str, str, str, str], benchmark_result: BenchmarkResult + ) -> None: + """ + Extract and store time-series data if available, grouped by aggregation directory. + + Args: + test_config: Tuple of (operation, blocksize, iodepth, number_of_jobs) + benchmark_result: Parsed benchmark result object + """ + operation, blocksize, iodepth, _ = test_config + + ts_data: Optional[TimeSeriesFormatType] = benchmark_result.get_timeseries_data() + if ts_data: + key = f"{operation}_{blocksize}_{iodepth}" + + # Determine aggregation directory from the file path + file_path = benchmark_result.resource_file_path + aggregation_directory = self._determine_aggregation_directory_from_file(file_path) + + # Initialize directory dict if needed + if aggregation_directory not in self._timeseries_by_directory: + self._timeseries_by_directory[aggregation_directory] = {} + + # Store existing data temporarily in _timeseries_data for merge to find it + existing_data = self._timeseries_by_directory[aggregation_directory].get(key) + if existing_data: + self._timeseries_data[key] = existing_data + + # Now merge will find the existing data + merged_ts_data = self._merge_timeseries_data(test_config, ts_data) + + # Clear temporary storage + if key in self._timeseries_data: + del self._timeseries_data[key] + + # Store merged result + self._timeseries_by_directory[aggregation_directory][key] = merged_ts_data + + log.debug("Stored time-series data for %s at %s", key, aggregation_directory) + else: + log.debug("No time-series data available for %s %s %s", operation, blocksize, iodepth) + + def _determine_aggregation_directory_from_file(self, file_path: Path) -> Path: + """ + Determine the correct aggregation directory from a file path. + + Looks for 'total_iodepth' or 'iodepth' in the file's parent directories. + Prefers total_iodepth if it exists (for aggregation), otherwise uses iodepth. + + Args: + file_path: Path to a result file + + Returns: + Path object for the aggregation directory + """ + path_parts = file_path.parts + + # Look for total_iodepth first (higher priority for aggregation) + for index, part in enumerate(path_parts): + if part.startswith("total_iodepth"): + return Path(*path_parts[: index + 1]) + + # If no total_iodepth, look for iodepth + for index, part in enumerate(path_parts): + if part.startswith("iodepth"): + return Path(*path_parts[: index + 1]) + + # If neither found, use the file's parent directory + return file_path.parent + + def _write_and_clear_timeseries_by_directory(self) -> None: + """ + Write timeseries data grouped by aggregation directory and clear memory. + + This method writes timeseries data at the correct aggregation level + (typically total_iodepth for aggregated results, or iodepth if no + total_iodepth exists) for each group and then clears memory. + + The aggregation of data from multiple files happens during processing + via _merge_timeseries_data() in subclasses like RBDFIO. + """ + if not self._timeseries_by_directory: + return + + total_files = sum(len(ts_dict) for ts_dict in self._timeseries_by_directory.values()) + log.debug("Writing %d timeseries files across %d directories", total_files, len(self._timeseries_by_directory)) + + for aggregation_dir, timeseries_dict in self._timeseries_by_directory.items(): + output_dir = aggregation_dir / "visualisation" + output_dir.mkdir(parents=True, exist_ok=True) + + log.debug("Writing %d timeseries files to %s", len(timeseries_dict), output_dir) + + for _, ts_data in timeseries_dict.items(): + # Extract configuration from the TimeSeriesFormatType data + operation = ts_data.get("operation", "unknown") + blocksize = ts_data.get("blocksize", "unknown") + numjobs = ts_data.get("numjobs", "1") + iodepth = ts_data.get("iodepth", "1") + + filename = output_dir / f"{blocksize}_{numjobs}_{operation}_{iodepth}_timeseries.json" + log.debug("Writing timeseries data to %s", filename) + + try: + with filename.open("w", encoding="utf8") as f: + json.dump(ts_data, f, indent=4, sort_keys=True) + except OSError as e: + log.error("Failed to write timeseries file %s: %s", filename, e) + + # Clear timeseries data from memory after writing + self._timeseries_by_directory.clear() + log.debug("Cleared timeseries data from memory") + def _convert_file(self, file_path: Path) -> None: """ Convert an individual benchmark result file to the common intermediate format. @@ -113,51 +392,38 @@ def _convert_file(self, file_path: Path) -> None: statistics, and stores them in the internal data structure organized by operation type, blocksize, and IO depth. + If include_timeseries is True, also extracts time-series data from log files. + Args: file_path: Path to the benchmark result file to process + + Raises: + ValueError: If benchmark or resource result creation fails + KeyError: If required data fields are missing from results """ + try: + # Use factory methods to get the correct classes + io: BenchmarkResult = self._create_benchmark_result(file_path) + resource: ResourceResult = self._create_resource_result(file_path) - # call the factory methods here to get the correct classes - io: FIO = FIO(file_path=file_path) - resource: FIOResource = FIOResource(file_path=file_path) - - iodepth = io.iodepth - blocksize: str = io.blocksize - operation: str = io.operation - number_of_jobs: str = io.number_of_jobs - global_details: dict[str, str] = io.global_options - - blocksize_details: InternalBlocksizeDataType = {blocksize: {}} - iodepth_details: dict[str, dict[str, str]] = {iodepth: global_details} - numjobs_details: InternalNumJobsDataType = {number_of_jobs: blocksize_details} - - io_details: IodepthDataType = {} - - if self._processed_data.get(operation, None): - if self._processed_data[operation].get(number_of_jobs, None): - if self._processed_data[operation][number_of_jobs].get(blocksize, None): - if self._processed_data[operation][number_of_jobs][blocksize].get(iodepth, None): - # we already have data here, so use it - log.debug("We have details for iodepth %s so using them", iodepth) - io_details = self._sum_io_details( - self._processed_data[operation][number_of_jobs][blocksize][iodepth], io.io_details - ) - - if not io_details: - io_details = io.io_details - - iodepth_details[iodepth].update(io_details) - iodepth_details[iodepth].update(resource.get()) - blocksize_details[blocksize].update(iodepth_details) - numjobs_details[number_of_jobs].update(blocksize_details) - - if self._processed_data.get(operation, None): - if self._processed_data[operation].get(number_of_jobs, None): - if self._processed_data[operation][number_of_jobs].get(blocksize, None): - self._processed_data[operation][number_of_jobs][blocksize].update(iodepth_details) - else: - self._processed_data[operation][number_of_jobs].update(blocksize_details) - else: - self._processed_data[operation].update(numjobs_details) - else: - self._processed_data.update({operation: numjobs_details}) + test_config = self._extract_test_configuration(io) + + # Merge IO details with existing data if present + io_details = self._merge_io_details(test_config, io.io_details) + + # Build complete test result data structure + numjobs_details = self._build_test_result_data(test_config, io_details, io.global_options, resource.get()) + + # Update internal processed data + self._update_processed_data(test_config, numjobs_details) + + # Process time-series data if requested + if self._include_timeseries: + self._process_timeseries_data(test_config, io) + + except (ValueError, KeyError) as e: + log.error("Failed to convert file %s: %s", file_path, e) + raise + except Exception as e: + log.exception("Unexpected error converting file %s: %s", file_path, e) + raise diff --git a/post_processing/run_results/run_result_factory.py b/post_processing/run_results/run_result_factory.py index fa9168ab..d63ed55c 100644 --- a/post_processing/run_results/run_result_factory.py +++ b/post_processing/run_results/run_result_factory.py @@ -30,7 +30,9 @@ } -def get_run_result_from_directory_name(directory: Path, file_name_root: str) -> RunResult: +def get_run_result_from_directory_name( + directory: Path, file_name_root: str, include_timeseries: bool = False +) -> RunResult: """ Create the appropriate RunResult subclass based on the directory name. @@ -41,6 +43,7 @@ def get_run_result_from_directory_name(directory: Path, file_name_root: str) -> Args: directory: Path to the directory containing benchmark results file_name_root: Root name of the result files to process + include_timeseries: Whether to process time-series data (default: False) Returns: An instance of the appropriate RunResult subclass @@ -62,7 +65,9 @@ def get_run_result_from_directory_name(directory: Path, file_name_root: str) -> for benchmark_type, result_class in BENCHMARK_TYPE_MAP.items(): if benchmark_type in directory_str: log.debug("Creating %s result processor for directory %s", result_class.__name__, directory) - return result_class(directory=directory, file_name_root=file_name_root) + return result_class( + directory=directory, file_name_root=file_name_root, include_timeseries=include_timeseries + ) raise NotImplementedError(f"Could not determine benchmark type from directory {directory}, ") diff --git a/requirements.txt b/requirements.txt index 390fd1d6..dd782544 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ pyyaml lxml matplotlib -mdutils \ No newline at end of file +mdutils +pandas \ No newline at end of file diff --git a/tests/test_base_formatter.py b/tests/test_base_formatter.py new file mode 100644 index 00000000..9eafd989 --- /dev/null +++ b/tests/test_base_formatter.py @@ -0,0 +1,118 @@ +""" +Unit tests for the BaseFormatter class +""" + +# pyright: strict, reportPrivateUsage=false +# +# We are OK to ignore private use in unit tests as the whole point of the tests +# is to validate the functions contained in the module + +import shutil +import tempfile +import unittest +from pathlib import Path + +from post_processing.formatter.base_formatter import BaseFormatter + + +class ConcreteFormatter(BaseFormatter): + """Concrete implementation of BaseFormatter for testing""" + + def process(self) -> None: + """Dummy implementation""" + + def write_output(self) -> None: + """Dummy implementation""" + + +class TestBaseFormatter(unittest.TestCase): + """ + Unit tests for BaseFormatter class methods + """ + + def setUp(self) -> None: + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.formatter = ConcreteFormatter(self.temp_dir) + + def tearDown(self) -> None: + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_initialization(self) -> None: + """Test that BaseFormatter initializes correctly""" + self.assertEqual(self.formatter._directory, self.temp_dir) # pylint: disable=protected-access + self.assertIsNotNone(self.formatter.log) + + def test_log_property(self) -> None: + """Test that log property returns a logger""" + logger = self.formatter.log + self.assertIsNotNone(logger) + self.assertEqual(logger.name, "formatter") + + def test_path_property(self) -> None: + """Test that path property returns a Path object""" + path = self.formatter.path + self.assertIsInstance(path, Path) + self.assertEqual(str(path), self.temp_dir) + + def test_path_property_with_nested_directory(self) -> None: + """Test path property with nested directory structure""" + nested_dir = Path(self.temp_dir) / "nested" / "directory" + nested_dir.mkdir(parents=True, exist_ok=True) + + formatter = ConcreteFormatter(str(nested_dir)) + path = formatter.path + + self.assertIsInstance(path, Path) + self.assertEqual(path, nested_dir) + self.assertTrue(path.exists()) + + def test_path_property_consistency(self) -> None: + """Test that path property returns consistent results""" + path1 = self.formatter.path + path2 = self.formatter.path + + self.assertEqual(path1, path2) + self.assertIsInstance(path1, Path) + self.assertIsInstance(path2, Path) + + def test_ensure_output_directory_creates_directory(self) -> None: + """Test that _ensure_output_directory creates a new directory""" + new_dir = Path(self.temp_dir) / "output" / "nested" + self.assertFalse(new_dir.exists()) + + self.formatter._ensure_output_directory(new_dir) # pylint: disable=protected-access + + self.assertTrue(new_dir.exists()) + self.assertTrue(new_dir.is_dir()) + + def test_ensure_output_directory_with_existing_directory(self) -> None: + """Test that _ensure_output_directory handles existing directories""" + existing_dir = Path(self.temp_dir) / "existing" + existing_dir.mkdir(parents=True, exist_ok=True) + self.assertTrue(existing_dir.exists()) + + # Should not raise an error + self.formatter._ensure_output_directory(existing_dir) # pylint: disable=protected-access + + self.assertTrue(existing_dir.exists()) + self.assertTrue(existing_dir.is_dir()) + + def test_ensure_output_directory_creates_parent_directories(self) -> None: + """Test that _ensure_output_directory creates parent directories""" + nested_dir = Path(self.temp_dir) / "level1" / "level2" / "level3" + self.assertFalse(nested_dir.exists()) + self.assertFalse(nested_dir.parent.exists()) + + self.formatter._ensure_output_directory(nested_dir) # pylint: disable=protected-access + + self.assertTrue(nested_dir.exists()) + self.assertTrue(nested_dir.parent.exists()) + self.assertTrue(nested_dir.parent.parent.exists()) + + +if __name__ == "__main__": + unittest.main() + +# Made with Bob diff --git a/tests/test_benchmark_result.py b/tests/test_benchmark_result.py index 975a2999..96dd9c64 100644 --- a/tests/test_benchmark_result.py +++ b/tests/test_benchmark_result.py @@ -12,9 +12,10 @@ import tempfile import unittest from pathlib import Path -from typing import Any +from typing import Any, Optional -from post_processing.run_results.benchmarks.benchmark_result import BenchmarkResult +from post_processing.post_processing_types import TimeSeriesFormatType +from post_processing.run_results.benchmark_result import BenchmarkResult class ConcreteBenchmarkResult(BenchmarkResult): @@ -25,7 +26,8 @@ def source(self) -> str: return "test_benchmark" def _get_global_options(self, fio_global_options: dict[str, str]) -> dict[str, str]: - return {"test_option": "test_value"} + self._number_of_jobs = fio_global_options.get("numjobs", "1") + return {"test_option": "test_value", "number_of_jobs": self._number_of_jobs} def _get_io_details(self, all_jobs: list[dict[str, Any]]) -> dict[str, str]: return {"test_io": "test_value"} @@ -33,6 +35,10 @@ def _get_io_details(self, all_jobs: list[dict[str, Any]]) -> dict[str, str]: def _get_iodepth(self, iodepth_value: str) -> str: return iodepth_value + def get_timeseries_data(self) -> Optional[TimeSeriesFormatType]: + """Test implementation returns None (no time-series support)""" + return None + class TestBenchmarkResult(unittest.TestCase): """Test cases for BenchmarkResult base class""" @@ -138,6 +144,12 @@ def test_io_details_property(self) -> None: self.assertIsInstance(result.io_details, dict) self.assertEqual(result.io_details["test_io"], "test_value") + def test_number_of_jobs_property(self) -> None: + """Test number_of_jobs property""" + result = ConcreteBenchmarkResult(self.test_file) + + self.assertEqual(result.number_of_jobs, "1") + def test_read_results_from_empty_file(self) -> None: """Test reading from empty file raises KeyError""" empty_file = Path(self.temp_dir) / "empty.json" diff --git a/tests/test_common_output_formatter.py b/tests/test_common_output_formatter.py index 40d36b59..336448a0 100644 --- a/tests/test_common_output_formatter.py +++ b/tests/test_common_output_formatter.py @@ -7,6 +7,7 @@ # We are OK to ignore private use in unit tests as the whole point of the tests # is to validate the functions contained in the module +import re import shutil import tempfile import unittest @@ -133,13 +134,19 @@ def test_find_all_results_files_in_directory(self) -> None: (test_dir / "other_file.0").touch() formatter = CommonOutputFormatter(str(test_dir)) - formatter._find_all_results_files_in_directory() + + # Simulate the file finding logic from process() + file_list = [ + path + for path in formatter.path.glob(f"**/{formatter._filename_root}.*") + if re.search(rf"{formatter._filename_root}.\d+$", f"{path}") + ] # Should find exactly 3 files - self.assertEqual(len(formatter._file_list), 3) + self.assertEqual(len(file_list), 3) # All files should match the pattern - for file_path in formatter._file_list: + for file_path in file_list: self.assertTrue(file_path.name.startswith("json_output.")) self.assertTrue(file_path.name.split(".")[-1].isdigit()) @@ -158,8 +165,14 @@ def test_find_all_testrun_ids(self) -> None: (run2_dir / "json_output.0").touch() formatter = CommonOutputFormatter(str(test_dir)) - formatter._find_all_results_files_in_directory() - formatter._find_all_testrun_ids() + + # Simulate the file finding logic from process() + file_list = [ + path + for path in formatter.path.glob(f"**/{formatter._filename_root}.*") + if re.search(rf"{formatter._filename_root}.\d+$", f"{path}") + ] + formatter._find_all_testrun_ids(file_list) # Should find 2 unique test run IDs self.assertEqual(len(formatter._all_test_run_ids), 2) @@ -176,8 +189,14 @@ def test_find_all_testrun_ids_without_id_prefix(self) -> None: (run_dir / "json_output.0").touch() formatter = CommonOutputFormatter(str(test_dir)) - formatter._find_all_results_files_in_directory() - formatter._find_all_testrun_ids() + + # Simulate the file finding logic from process() + file_list = [ + path + for path in formatter.path.glob(f"**/{formatter._filename_root}.*") + if re.search(rf"{formatter._filename_root}.\d+$", f"{path}") + ] + formatter._find_all_testrun_ids(file_list) # Should use the directory name above the file self.assertEqual(len(formatter._all_test_run_ids), 1) diff --git a/tests/test_comparison_report_generator.py b/tests/test_comparison_report_generator.py index e160fd6f..e6ca2265 100644 --- a/tests/test_comparison_report_generator.py +++ b/tests/test_comparison_report_generator.py @@ -23,12 +23,13 @@ def setUp(self) -> None: """Set up test fixtures""" self.temp_dir = tempfile.mkdtemp() - # Create two archive directories for comparison + # Create two archive directories for comparison using new nested structure self.archive1 = Path(self.temp_dir) / "baseline" self.archive2 = Path(self.temp_dir) / "comparison" - self.vis1 = self.archive1 / "visualisation" - self.vis2 = self.archive2 / "visualisation" + # Use new nested structure: operation/visualisation/ + self.vis1 = self.archive1 / "read" / "visualisation" + self.vis2 = self.archive2 / "read" / "visualisation" self.vis1.mkdir(parents=True) self.vis2.mkdir(parents=True) @@ -86,10 +87,16 @@ def test_generate_report_name(self) -> None: def test_find_and_sort_file_paths_multiple_directories(self) -> None: """Test finding files across multiple directories""" + # Create additional operation directories with visualisation subdirectories + vis1_write = self.archive1 / "write" / "visualisation" + vis2_write = self.archive2 / "write" / "visualisation" + vis1_write.mkdir(parents=True) + vis2_write.mkdir(parents=True) + # Create additional files # Format: {blocksize}_{numjobs}_{operation}.json - (self.vis1 / "8192_1_write.json").touch() - (self.vis2 / "8192_1_write.json").touch() + (vis1_write / "8192_1_write.json").touch() + (vis2_write / "8192_1_write.json").touch() output_dir = f"{self.temp_dir}/output" @@ -98,9 +105,10 @@ def test_find_and_sort_file_paths_multiple_directories(self) -> None: output_directory=output_dir, ) - paths = generator._find_and_sort_file_paths(paths=[self.vis1, self.vis2], search_pattern="*.json", index=0) + # Now we have 2 visualisation directories per archive (read and write) + paths = generator._find_and_sort_file_paths(paths=[self.vis1, self.vis2, vis1_write, vis2_write], search_pattern="*.json", index=0) - # Should find files from both directories + # Should find files from all directories self.assertEqual(len(paths), 4) @patch("post_processing.reports.comparison_report_generator.DirectoryComparisonPlotter") @@ -135,9 +143,8 @@ def test_generate_table_headers_two_directories(self) -> None: # Should include numjobs column self.assertIn("numjobs", header) - # Should include baseline directory name + # Should have archive directory names self.assertIn("baseline", header) - # Should include comparison directory name self.assertIn("comparison", header) # Should have percentage change columns self.assertIn("%change", header) @@ -150,7 +157,8 @@ def test_generate_table_headers_two_directories(self) -> None: def test_generate_table_headers_multiple_directories(self) -> None: """Test generating table headers for more than two directories""" archive3 = Path(self.temp_dir) / "comparison2" - vis3 = archive3 / "visualisation" + # Use new nested structure + vis3 = archive3 / "read" / "visualisation" vis3.mkdir(parents=True) # Format: {blocksize}_{numjobs}_{operation}.json (vis3 / "4096_1_read.json").touch() @@ -166,7 +174,7 @@ def test_generate_table_headers_multiple_directories(self) -> None: # Should include numjobs column self.assertIn("numjobs", header) - # Should have all directory names + # Should have all archive directory names self.assertIn("baseline", header) self.assertIn("comparison", header) self.assertIn("comparison2", header) @@ -176,11 +184,23 @@ def test_generate_table_headers_multiple_directories(self) -> None: # (left-aligned operation, right-aligned numjobs, baseline, then comparison + %change for each comparison dir) self.assertEqual(justification, "| :--- | ---: | ---: | ---: | ---: | ---: | ---: |") - @patch("subprocess.check_output") - def test_yaml_file_has_more_than_20_differences_true(self, mock_check_output: MagicMock) -> None: + @patch("subprocess.Popen") + def test_yaml_file_has_more_than_20_differences_true(self, mock_popen: MagicMock) -> None: """Test detecting significant differences between yaml files""" - # Mock diff output showing 25 differences - mock_check_output.return_value = b"25\n" + # Mock the wc process to return 25 differences + mock_wc_process = MagicMock() + mock_wc_process.communicate.return_value = (b"25\n", b"") + mock_wc_process.__enter__ = MagicMock(return_value=mock_wc_process) + mock_wc_process.__exit__ = MagicMock(return_value=False) + + # Mock the diff process + mock_diff_process = MagicMock() + mock_diff_process.stdout = MagicMock() + mock_diff_process.__enter__ = MagicMock(return_value=mock_diff_process) + mock_diff_process.__exit__ = MagicMock(return_value=False) + + # Configure Popen to return diff process first, then wc process + mock_popen.side_effect = [mock_diff_process, mock_wc_process] output_dir = f"{self.temp_dir}/output" @@ -198,11 +218,23 @@ def test_yaml_file_has_more_than_20_differences_true(self, mock_check_output: Ma self.assertTrue(result) - @patch("subprocess.check_output") - def test_yaml_file_has_more_than_20_differences_false(self, mock_check_output: MagicMock) -> None: + @patch("subprocess.Popen") + def test_yaml_file_has_more_than_20_differences_false(self, mock_popen: MagicMock) -> None: """Test detecting minor differences between yaml files""" - # Mock diff output showing 10 differences - mock_check_output.return_value = b"10\n" + # Mock the wc process to return 10 differences + mock_wc_process = MagicMock() + mock_wc_process.communicate.return_value = (b"10\n", b"") + mock_wc_process.__enter__ = MagicMock(return_value=mock_wc_process) + mock_wc_process.__exit__ = MagicMock(return_value=False) + + # Mock the diff process + mock_diff_process = MagicMock() + mock_diff_process.stdout = MagicMock() + mock_diff_process.__enter__ = MagicMock(return_value=mock_diff_process) + mock_diff_process.__exit__ = MagicMock(return_value=False) + + # Configure Popen to return diff process first, then wc process + mock_popen.side_effect = [mock_diff_process, mock_wc_process] output_dir = f"{self.temp_dir}/output" diff --git a/tests/test_cpu_plotter.py b/tests/test_cpu_plotter.py index 186026c4..93602038 100644 --- a/tests/test_cpu_plotter.py +++ b/tests/test_cpu_plotter.py @@ -65,7 +65,7 @@ def test_plot(self) -> None: def test_cpu_constants(self) -> None: """Test CPU plotter constants""" - self.assertEqual(CPU_PLOT_DEFAULT_COLOUR, "#5ca904") + self.assertEqual(CPU_PLOT_DEFAULT_COLOUR, "xkcd:leaf green") self.assertEqual(CPU_Y_LABEL, "System CPU use (%)") self.assertEqual(CPU_PLOT_LABEL, "CPU use") diff --git a/tests/test_directory_comparison_plotter.py b/tests/test_directory_comparison_plotter.py index bd30789b..986a63fe 100644 --- a/tests/test_directory_comparison_plotter.py +++ b/tests/test_directory_comparison_plotter.py @@ -2,6 +2,7 @@ Unit tests for the DirectoryComparisonPlotter class """ +import json import shutil import tempfile import unittest @@ -44,9 +45,9 @@ def test_initialization(self) -> None: plotter = DirectoryComparisonPlotter(output_directory=str(self.output_dir), directories=directories) assert plotter._output_directory == str(self.output_dir) - assert len(plotter._comparison_directories) == 2 - assert plotter._comparison_directories[0] == self.vis_dir1 - assert plotter._comparison_directories[1] == self.vis_dir2 + assert len(plotter._archive_directories) == 2 + assert plotter._archive_directories[0] == self.dir1 + assert plotter._archive_directories[1] == self.dir2 def test_generate_output_file_name(self) -> None: """Test generating output file name""" @@ -134,6 +135,12 @@ def test_draw_and_save_multiple_common_files( mock_find_common: MagicMock, ) -> None: """Test draw_and_save with multiple common files""" + # Create the JSON files BEFORE instantiating plotter so find_hockey_stick_visualisation_directories finds them + (self.vis_dir1 / "4096_100_read_1.json").touch() + (self.vis_dir1 / "8192_100_write_1.json").touch() + (self.vis_dir2 / "4096_100_read_1.json").touch() + (self.vis_dir2 / "8192_100_write_1.json").touch() + directories = [str(self.dir1), str(self.dir2)] plotter = DirectoryComparisonPlotter(output_directory=str(self.output_dir), directories=directories) @@ -281,20 +288,29 @@ def test_draw_and_save_no_common_files( def test_uses_directory_name_as_label(self) -> None: """Test that directory names are used as labels in the plot""" - # Create directories with meaningful names + # Create directories with meaningful names using new nested structure baseline_dir = Path(self.temp_dir) / "baseline" optimized_dir = Path(self.temp_dir) / "optimized" - baseline_vis = baseline_dir / "visualisation" - optimized_vis = optimized_dir / "visualisation" + + # Create operation-level visualisation directories (new structure) + baseline_vis = baseline_dir / "randread" / "visualisation" + optimized_vis = optimized_dir / "randread" / "visualisation" baseline_vis.mkdir(parents=True) optimized_vis.mkdir(parents=True) + # Create JSON files so directories are found + test_data = {"data": [{"x": 1, "y": 100}]} + with open(baseline_vis / "test.json", "w") as f: + json.dump(test_data, f) + with open(optimized_vis / "test.json", "w") as f: + json.dump(test_data, f) + directories = [str(baseline_dir), str(optimized_dir)] plotter = DirectoryComparisonPlotter(output_directory=str(self.output_dir), directories=directories) - # Verify the comparison directories are set correctly - assert plotter._comparison_directories[0] == baseline_vis - assert plotter._comparison_directories[1] == optimized_vis + # Verify the archive directories are set correctly + assert plotter._archive_directories[0] == baseline_dir + assert plotter._archive_directories[1] == optimized_dir -# Made with Bob \ No newline at end of file +# Made with Bob diff --git a/tests/test_fio_benchmarks.py b/tests/test_fio_benchmarks.py new file mode 100644 index 00000000..6a5275de --- /dev/null +++ b/tests/test_fio_benchmarks.py @@ -0,0 +1,947 @@ +""" +Comprehensive tests for post_processing/run_results/benchmarks/fio.py + +Tests cover error handling, edge cases, and time-series data processing. +""" + +import json +import tempfile +from pathlib import Path +from typing import Any, Union +from unittest.mock import Mock, patch + +import pytest + +from post_processing.run_results.benchmarks.fio import FIO + + +class TestFIOExtractMetrics: + """Test metric extraction helper methods.""" + + def test_extract_int_metric_success(self) -> None: + """Test successful integer metric extraction.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]] = {"io_bytes": 1000, "total_ios": 50} + + result = fio._extract_int_metric(job_data, "io_bytes", "read") + assert result == 1000 + + def test_extract_int_metric_missing(self) -> None: + """Test integer metric extraction with missing key.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]] = {"io_bytes": 1000} + + with pytest.raises(ValueError, match="Missing or invalid 'total_ios'"): + fio._extract_int_metric(job_data, "total_ios", "read") + + def test_extract_int_metric_wrong_type(self) -> None: + """Test integer metric extraction with wrong type.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]] = {"io_bytes": "not_an_int"} # type: ignore[dict-item] + + with pytest.raises(ValueError, match="expected int, got str"): + fio._extract_int_metric(job_data, "io_bytes", "read") + + def test_extract_float_metric_success(self) -> None: + """Test successful float metric extraction.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.5, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]] = {"iops": 10.5} + + result = fio._extract_float_metric(job_data, "iops", "read") + assert result == 10.5 + + def test_extract_float_metric_from_int(self) -> None: + """Test float metric extraction accepts integers.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]] = {"iops": 10} + + result = fio._extract_float_metric(job_data, "iops", "read") + assert result == 10.0 + + def test_extract_float_metric_wrong_type(self) -> None: + """Test float metric extraction with wrong type.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + job_data: dict[str, Union[int, float, dict[str, Union[int, float]]]] = {"iops": "not_a_float"} # type: ignore[dict-item] + + with pytest.raises(ValueError, match="expected float, got str"): + fio._extract_float_metric(job_data, "iops", "read") + + +class TestFIOGetIODetails: + """Test _get_io_details method with various scenarios.""" + + def test_get_io_details_with_invalid_job_data(self) -> None: + """Test IO details extraction with invalid job data structure.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": "not_a_dict", # Invalid: should be dict + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + # Should handle invalid data gracefully and return zero values + io_details = fio.io_details + assert io_details["total_ios"] == "0" + + def test_get_io_details_missing_clat_ns(self) -> None: + """Test IO details extraction with missing clat_ns.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + # Missing clat_ns + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + with pytest.raises(ValueError, match="Invalid job data structure"): + FIO(test_file) + + def test_get_io_details_zero_total_ios(self) -> None: + """Test IO details when total_ios is zero.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + io_details = fio.io_details + + assert io_details["total_ios"] == "0" + assert io_details["iops"] == "0.0" + assert io_details["latency"] == "0.0" + + def test_get_io_details_with_extra_job_keys(self) -> None: + """Test IO details extraction ignores non-read/write keys.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "jobname": "test_job", # Should be ignored + "sys_cpu": 5.5, # Should be ignored + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + io_details = fio.io_details + + # Should process successfully, ignoring extra keys + assert io_details["total_ios"] == "50" + + +class TestFIOGetLogAvgMsec: + """Test _get_log_avg_msec method.""" + + def test_get_log_avg_msec_default(self) -> None: + """Test default log_avg_msec value.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + log_avg_msec = fio._get_log_avg_msec() + + assert log_avg_msec == 1000 + + def test_get_log_avg_msec_custom_value(self) -> None: + """Test custom log_avg_msec value.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + "log_avg_msec": "500", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + # Manually add log_avg_msec to _global_options since it's not copied by _get_global_options + fio._global_options["log_avg_msec"] = "500" + log_avg_msec = fio._get_log_avg_msec() + + assert log_avg_msec == 500 + + def test_get_log_avg_msec_invalid_negative(self) -> None: + """Test log_avg_msec with negative value.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + "log_avg_msec": "-100", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + log_avg_msec = fio._get_log_avg_msec() + + # Should return default value + assert log_avg_msec == 1000 + + def test_get_log_avg_msec_invalid_string(self) -> None: + """Test log_avg_msec with non-numeric string.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + "log_avg_msec": "invalid", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + log_avg_msec = fio._get_log_avg_msec() + + # Should return default value + assert log_avg_msec == 1000 + + +class TestFIOGetTimeseriesData: + """Test get_timeseries_data method.""" + + def test_get_timeseries_data_no_logs(self) -> None: + """Test timeseries data when no log files exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + result = fio.get_timeseries_data() + + # Should return None when no logs found + assert result is None + + def test_get_timeseries_data_directory_not_exists(self) -> None: + """Test timeseries data when directory doesn't exist.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.json" + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + fio = FIO(test_file) + # Modify the path to non-existent directory + fio._resource_file_path = Path("/nonexistent/path/test.json") + + result = fio.get_timeseries_data() + + # Should return None gracefully + assert result is None + + @pytest.mark.parametrize( + "test_file_name", + [ + "output.0", + "json_output.0", + ], + ) + @patch("post_processing.run_results.benchmarks.fio.FIOLogParser") + @patch("post_processing.run_results.benchmarks.fio.FIOTimeSeriesParser") + def test_get_timeseries_data_with_parameterized_file_patterns( + self, mock_parser_ts_class: Any, mock_parser_class: Any, test_file_name: str + ) -> None: + """Test timeseries data processing for supported result file naming patterns.""" + with tempfile.TemporaryDirectory() as tmpdir: + log_dir = Path(tmpdir) + test_file = log_dir / test_file_name + + # Create mock log files with standard naming: output.X_*.log + (log_dir / "output.0_iops.1.log").touch() + (log_dir / "output.0_clat.1.log").touch() + (log_dir / "output.0_bw.1.log").touch() + + test_data = { + "global options": { + "bs": "4K", + "rw": "randread", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + # Setup mocks + mock_parser = Mock() + mock_parser.parse_and_combine_logs.return_value = Mock() # Return non-None DataFrame + mock_parser_class.return_value = mock_parser + + mock_timeseries_parser = Mock() + mock_formatted_output: Any = {"test": "data"} + mock_timeseries_parser.get_formatted_output.return_value = mock_formatted_output + mock_parser_ts_class.return_value = mock_timeseries_parser + + fio = FIO(test_file) + result = fio.get_timeseries_data() + + # Should process and return the formatted output + assert result == mock_formatted_output + mock_timeseries_parser.process.assert_called_once() + parser_call_patterns = [call.args[2] for call in mock_parser.parse_and_combine_logs.call_args_list] + assert "output.0_iops.*.log" in parser_call_patterns + assert "output.0_clat.*.log" in parser_call_patterns + assert "output.0_bw.*.log" in parser_call_patterns + assert all("output.1_" not in pattern for pattern in parser_call_patterns) + + if test_file_name.startswith("json_"): + assert all("json_output" not in pattern for pattern in parser_call_patterns) + + @patch("post_processing.run_results.benchmarks.fio.TimestampAligner") + @patch("post_processing.run_results.benchmarks.fio.FIOLogParser") + @patch("post_processing.run_results.benchmarks.fio.FIOTimeSeriesParser") + def test_get_timeseries_data_with_percentile_calculation( + self, mock_parser_ts_class: Any, mock_parser_class: Any, mock_aligner_class: Any + ) -> None: + """Test that percentiles are calculated from clat data.""" + import pandas as pd + + with tempfile.TemporaryDirectory() as tmpdir: + log_dir = Path(tmpdir) + test_file = log_dir / "output.0" + + # Create mock log files + (log_dir / "output.0_iops.1.log").touch() + (log_dir / "output.0_clat.1.log").touch() + (log_dir / "output.0_bw.1.log").touch() + + test_data = { + "global options": { + "bs": "4K", + "rw": "randwrite", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + "log_avg_msec": "1000", + }, + "jobs": [ + { + "read": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "write": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + # Setup mock parser to return clat data + mock_parser = Mock() + mock_clat_df = pd.DataFrame( + {"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [10.0, 15.0, 20.0], "direction": [1, 1, 1]} + ) + mock_parser.parse_and_combine_logs.side_effect = lambda dir, metric, pattern: ( + mock_clat_df if metric == "clat" else None + ) + mock_parser_class.return_value = mock_parser + + # Setup mock aligner to return aligned data and percentile data + mock_aligner = Mock() + + # Mock aligned data (returned by align_and_aggregate) + mock_aligned_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [10.0, 15.0, 20.0]}) + mock_aligner.align_and_aggregate.return_value = mock_aligned_df + + # Mock percentile data (returned by calculate_percentiles) + mock_percentiles_df = pd.DataFrame( + { + "timestamp_sec": [1.0, 2.0, 3.0], + "p50_latency_ms": [12.0, 16.0, 21.0], + "p95_latency_ms": [18.0, 22.0, 28.0], + "p99_latency_ms": [19.0, 23.0, 29.0], + } + ) + mock_aligner.calculate_percentiles.return_value = mock_percentiles_df + mock_aligner_class.return_value = mock_aligner + + # Setup mock timeseries parser + mock_timeseries_parser = Mock() + mock_formatted_output: Any = {"test": "data"} + mock_timeseries_parser.get_formatted_output.return_value = mock_formatted_output + mock_parser_ts_class.return_value = mock_timeseries_parser + + fio = FIO(test_file) + result = fio.get_timeseries_data() + + # Verify TimestampAligner was created with correct window size + mock_aligner_class.assert_called_once_with(window_size_ms=1000) + + # Verify calculate_percentiles was called with clat data + mock_aligner.calculate_percentiles.assert_called_once() + call_args = mock_aligner.calculate_percentiles.call_args + assert len(call_args[0][0]) == 1 # One dataframe in list + pd.testing.assert_frame_equal(call_args[0][0][0], mock_clat_df) + assert call_args[1]["percentiles"] == [50, 95, 99] + + # Verify FIOTimeSeriesParser was called with percentile dataframes + parser_init_call = mock_parser_ts_class.call_args + assert parser_init_call[1]["p50_latency_df"] is not None + assert parser_init_call[1]["p95_latency_df"] is not None + assert parser_init_call[1]["p99_latency_df"] is not None + + # Verify the percentile dataframes have correct structure + p50_df = parser_init_call[1]["p50_latency_df"] + assert "timestamp_sec" in p50_df.columns + assert "latency_ms" in p50_df.columns + + assert result == mock_formatted_output + + @patch("post_processing.run_results.benchmarks.fio.FIOLogParser") + @patch("post_processing.run_results.benchmarks.fio.FIOTimeSeriesParser") + def test_get_timeseries_data_no_clat_data_for_percentiles( + self, mock_parser_ts_class: Any, mock_parser_class: Any + ) -> None: + """Test that percentiles are None when no clat data available.""" + with tempfile.TemporaryDirectory() as tmpdir: + log_dir = Path(tmpdir) + test_file = log_dir / "output.0" + + # Create mock log files (but no clat) + (log_dir / "output.0_iops.1.log").touch() + (log_dir / "output.0_bw.1.log").touch() + + test_data = { + "global options": { + "bs": "4K", + "rw": "randwrite", + "iodepth": "4", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "write": { + "io_bytes": 1000, + "bw_bytes": 100, + "iops": 10.0, + "total_ios": 50, + "clat_ns": {"mean": 5000.0, "stddev": 500.0}, + }, + } + ], + } + + with open(test_file, "w") as f: + json.dump(test_data, f) + + # Setup mock parser to return None for clat + mock_parser = Mock() + mock_parser.parse_and_combine_logs.side_effect = lambda dir, metric, pattern: ( + None if metric == "clat" else Mock() + ) + mock_parser_class.return_value = mock_parser + + # Setup mock timeseries parser + mock_timeseries_parser = Mock() + mock_formatted_output: Any = {"test": "data"} + mock_timeseries_parser.get_formatted_output.return_value = mock_formatted_output + mock_parser_ts_class.return_value = mock_timeseries_parser + + fio = FIO(test_file) + result = fio.get_timeseries_data() + + # Verify FIOTimeSeriesParser was called with None percentile dataframes + parser_init_call = mock_parser_ts_class.call_args + assert parser_init_call[1]["p50_latency_df"] is None + assert parser_init_call[1]["p95_latency_df"] is None + assert parser_init_call[1]["p99_latency_df"] is None + + assert result == mock_formatted_output + + +# Made with Bob diff --git a/tests/test_fio_log_parser.py b/tests/test_fio_log_parser.py new file mode 100644 index 00000000..990da009 --- /dev/null +++ b/tests/test_fio_log_parser.py @@ -0,0 +1,294 @@ +""" +Unit tests for the FIO log parser module. +""" + +# pyright: strict, reportPrivateUsage=false + +import shutil +import tempfile +import unittest +from pathlib import Path + +from post_processing.parsers.fio_log_parser import FIOLogParser + + +class TestFIOLogParser(unittest.TestCase): + """Test cases for FIOLogParser class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.parser = FIOLogParser() + + def tearDown(self) -> None: + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_parse_iops_log(self) -> None: + """Test parsing IOPS log file""" + # Create test IOPS log file + iops_file = Path(self.temp_dir) / "test_iops.log" + with iops_file.open("w", encoding="utf-8") as f: + f.write("1000, 15234, 0, 0\n") + f.write("2000, 15456, 0, 4096\n") + f.write("3000, 15123, 1, 8192\n") + + result = self.parser.parse_iops_log(iops_file) + + self.assertIsNotNone(result) + assert result is not None # Type narrowing for mypy + self.assertEqual(len(result), 3) + self.assertIn("timestamp_sec", result.columns) + self.assertIn("iops", result.columns) + self.assertIn("direction", result.columns) + + # Check timestamp conversion (ms to seconds) + self.assertAlmostEqual(result.iloc[0]["timestamp_sec"], 1.0, places=3) + self.assertAlmostEqual(result.iloc[1]["timestamp_sec"], 2.0, places=3) + + # Check IOPS values + self.assertAlmostEqual(result.iloc[0]["iops"], 15234.0, places=1) + self.assertAlmostEqual(result.iloc[1]["iops"], 15456.0, places=1) + + def test_parse_clat_log(self) -> None: + """Test parsing completion latency log file""" + # Create test clat log file + clat_file = Path(self.temp_dir) / "test_clat.log" + with clat_file.open("w", encoding="utf-8") as f: + f.write("1000, 2340000, 0, 0\n") # 2.34ms in nanoseconds + f.write("2000, 2310000, 0, 4096\n") # 2.31ms + f.write("3000, 15670000, 1, 8192\n") # 15.67ms + + result = self.parser.parse_clat_log(clat_file) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 3) + self.assertIn("timestamp_sec", result.columns) + self.assertIn("latency_ms", result.columns) + self.assertIn("direction", result.columns) + + # Check latency conversion (ns to ms) + self.assertAlmostEqual(result.iloc[0]["latency_ms"], 2.34, places=2) + self.assertAlmostEqual(result.iloc[1]["latency_ms"], 2.31, places=2) + self.assertAlmostEqual(result.iloc[2]["latency_ms"], 15.67, places=2) + + def test_parse_bw_log(self) -> None: + """Test parsing bandwidth log file""" + # Create test bandwidth log file + bw_file = Path(self.temp_dir) / "test_bw.log" + with bw_file.open("w", encoding="utf-8") as f: + f.write("1000, 60892, 0, 0\n") # KB/s + f.write("2000, 61769, 0, 4096\n") + f.write("3000, 58901, 1, 8192\n") + + result = self.parser.parse_bw_log(bw_file) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 3) + self.assertIn("timestamp_sec", result.columns) + self.assertIn("bandwidth_bytes", result.columns) + self.assertIn("direction", result.columns) + + # Check bandwidth conversion (KB to bytes) + self.assertAlmostEqual(result.iloc[0]["bandwidth_bytes"], 60892 * 1024, places=0) + self.assertAlmostEqual(result.iloc[1]["bandwidth_bytes"], 61769 * 1024, places=0) + + def test_parse_lat_log(self) -> None: + """Test parsing total latency log file (same format as clat)""" + lat_file = Path(self.temp_dir) / "test_lat.log" + with lat_file.open("w", encoding="utf-8") as f: + f.write("1000, 2500000, 0, 0\n") + + result = self.parser.parse_lat_log(lat_file) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 1) + self.assertAlmostEqual(result.iloc[0]["latency_ms"], 2.5, places=2) + + def test_parse_slat_log(self) -> None: + """Test parsing submission latency log file (same format as clat)""" + slat_file = Path(self.temp_dir) / "test_slat.log" + with slat_file.open("w", encoding="utf-8") as f: + f.write("1000, 100000, 0, 0\n") # 0.1ms + + result = self.parser.parse_slat_log(slat_file) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 1) + self.assertAlmostEqual(result.iloc[0]["latency_ms"], 0.1, places=2) + + def test_parse_empty_file(self) -> None: + """Test parsing an empty log file""" + empty_file = Path(self.temp_dir) / "empty.log" + empty_file.touch() + + result = self.parser.parse_iops_log(empty_file) + + # Should return empty DataFrame, not None + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 0) + + def test_parse_nonexistent_file(self) -> None: + """Test parsing a file that doesn't exist""" + nonexistent = Path(self.temp_dir) / "nonexistent.log" + + result = self.parser.parse_iops_log(nonexistent) + + # Should return None on error + self.assertIsNone(result) + + def test_parse_malformed_file(self) -> None: + """Test parsing a malformed log file""" + malformed_file = Path(self.temp_dir) / "malformed.log" + with malformed_file.open("w", encoding="utf-8") as f: + f.write("not,valid,data\n") + f.write("1000, abc, 0, 0\n") # Invalid IOPS value + + result = self.parser.parse_iops_log(malformed_file) + + # Should return None on parsing error + self.assertIsNone(result) + + def test_parse_with_whitespace(self) -> None: + """Test parsing log file with extra whitespace""" + iops_file = Path(self.temp_dir) / "whitespace.log" + with iops_file.open("w", encoding="utf-8") as f: + f.write(" 1000 , 15234 , 0 , 0 \n") + f.write(" 2000, 15456, 0, 4096 \n") + + result = self.parser.parse_iops_log(iops_file) + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 2) + self.assertAlmostEqual(result.iloc[0]["iops"], 15234.0, places=1) + + def test_parse_and_combine_logs_iops(self) -> None: + """Test combining multiple IOPS log files""" + # Create multiple IOPS log files + iops_file1 = Path(self.temp_dir) / "output.0_iops.1.log" + with iops_file1.open("w", encoding="utf-8") as f: + f.write("1000, 15000, 0, 0\n") + f.write("2000, 16000, 0, 4096\n") + + iops_file2 = Path(self.temp_dir) / "output.1_iops.1.log" + with iops_file2.open("w", encoding="utf-8") as f: + f.write("1000, 14000, 0, 0\n") + f.write("2000, 15000, 0, 4096\n") + + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "iops", "output.*_iops.1.log") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 2) + # Values should be summed: 15000 + 14000 = 29000 + self.assertAlmostEqual(result.iloc[0]["iops"], 29000.0, places=1) + self.assertAlmostEqual(result.iloc[1]["iops"], 31000.0, places=1) + + def test_parse_and_combine_logs_clat(self) -> None: + """Test combining multiple completion latency log files""" + clat_file1 = Path(self.temp_dir) / "output.0_clat.1.log" + with clat_file1.open("w", encoding="utf-8") as f: + f.write("1000, 2000000, 0, 0\n") # 2ms + f.write("2000, 3000000, 0, 4096\n") # 3ms + + clat_file2 = Path(self.temp_dir) / "output.1_clat.1.log" + with clat_file2.open("w", encoding="utf-8") as f: + f.write("1000, 2500000, 0, 0\n") # 2.5ms + f.write("2000, 3500000, 0, 4096\n") # 3.5ms + + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "clat", "output.*_clat.1.log") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 2) + # Latencies should be summed: 2.0 + 2.5 = 4.5ms + self.assertAlmostEqual(result.iloc[0]["latency_ms"], 4.5, places=1) + self.assertAlmostEqual(result.iloc[1]["latency_ms"], 6.5, places=1) + + def test_parse_and_combine_logs_bw(self) -> None: + """Test combining multiple bandwidth log files""" + bw_file1 = Path(self.temp_dir) / "output.0_bw.1.log" + with bw_file1.open("w", encoding="utf-8") as f: + f.write("1000, 50000, 0, 0\n") # 50MB/s in KB/s + + bw_file2 = Path(self.temp_dir) / "output.1_bw.1.log" + with bw_file2.open("w", encoding="utf-8") as f: + f.write("1000, 30000, 0, 0\n") # 30MB/s in KB/s + + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "bw", "output.*_bw.1.log") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 1) + # Bandwidth should be summed: (50000 + 30000) * 1024 = 81920000 bytes + self.assertAlmostEqual(result.iloc[0]["bandwidth_bytes"], 81920000.0, places=1) + + def test_parse_and_combine_logs_no_files(self) -> None: + """Test combining when no matching files exist""" + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "iops", "nonexistent_*.log") + + self.assertIsNone(result) + + def test_parse_and_combine_logs_invalid_type(self) -> None: + """Test combining with invalid log type""" + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "invalid_type", "*.log") + + self.assertIsNone(result) + + def test_parse_and_combine_logs_empty_files(self) -> None: + """Test combining when all files are empty""" + empty_file1 = Path(self.temp_dir) / "output.0_iops.1.log" + empty_file1.open("w", encoding="utf-8").close() + + empty_file2 = Path(self.temp_dir) / "output.1_iops.1.log" + empty_file2.open("w", encoding="utf-8").close() + + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "iops", "output.*_iops.1.log") + + self.assertIsNone(result) + + def test_parse_and_combine_logs_mixed_directions(self) -> None: + """Test combining logs with different I/O directions""" + iops_file1 = Path(self.temp_dir) / "output.0_iops.1.log" + with iops_file1.open("w", encoding="utf-8") as f: + f.write("1000, 10000, 0, 0\n") # Read + f.write("1000, 5000, 1, 0\n") # Write + + iops_file2 = Path(self.temp_dir) / "output.1_iops.1.log" + with iops_file2.open("w", encoding="utf-8") as f: + f.write("1000, 12000, 0, 0\n") # Read + f.write("1000, 6000, 1, 0\n") # Write + + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "iops", "output.*_iops.1.log") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 2) + # Should have separate entries for read (direction=0) and write (direction=1) + read_row = result[result["direction"] == 0].iloc[0] + write_row = result[result["direction"] == 1].iloc[0] + self.assertAlmostEqual(read_row["iops"], 22000.0, places=1) + self.assertAlmostEqual(write_row["iops"], 11000.0, places=1) + + def test_parse_and_combine_logs_single_file(self) -> None: + """Test combining with only one file (should still work)""" + iops_file = Path(self.temp_dir) / "output.0_iops.1.log" + with iops_file.open("w", encoding="utf-8") as f: + f.write("1000, 15000, 0, 0\n") + f.write("2000, 16000, 0, 4096\n") + + result = self.parser.parse_and_combine_logs(Path(self.temp_dir), "iops", "output.0_iops.1.log") + + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(len(result), 2) + self.assertAlmostEqual(result.iloc[0]["iops"], 15000.0, places=1) + + +# Made with Bob diff --git a/tests/test_io_plotter.py b/tests/test_io_plotter.py index 5e5e6264..289f5ca5 100644 --- a/tests/test_io_plotter.py +++ b/tests/test_io_plotter.py @@ -87,7 +87,7 @@ def test_plot_with_error_bars_no_caps(self) -> None: def test_io_constants(self) -> None: """Test IO plotter constants""" - self.assertEqual(IO_PLOT_DEFAULT_COLOUR, "#5ca904") + self.assertEqual(IO_PLOT_DEFAULT_COLOUR, "xkcd:leaf green") self.assertEqual(IO_Y_LABEL, "Latency (ms)") self.assertEqual(IO_PLOT_LABEL, "IO Details") diff --git a/tests/test_log_configuration.py b/tests/test_log_configuration.py index bd6b9b07..0848373d 100644 --- a/tests/test_log_configuration.py +++ b/tests/test_log_configuration.py @@ -1,6 +1,7 @@ """ Unit tests for the post_processing/log_configuration.py module """ + # pyright: strict, reportPrivateUsage=false # # We are OK to ignore private use in unit tests as the whole point of the tests @@ -130,14 +131,8 @@ def test_get_configuration_handlers(self) -> None: @patch("post_processing.log_configuration.os.makedirs") @patch("post_processing.log_configuration.logging.config.dictConfig") - @patch("post_processing.log_configuration.getLogger") - def test_setup_logging( - self, mock_get_logger: MagicMock, mock_dict_config: MagicMock, mock_makedirs: MagicMock - ) -> None: + def test_setup_logging(self, mock_dict_config: MagicMock, mock_makedirs: MagicMock) -> None: """Test setup_logging function""" - mock_logger = MagicMock() - mock_get_logger.return_value = mock_logger - setup_logging() # Should create log directory @@ -148,14 +143,6 @@ def test_setup_logging( # Should configure logging mock_dict_config.assert_called_once() - # Should get formatter logger - mock_get_logger.assert_called_once_with("formatter") - - # Should log startup message - mock_logger.info.assert_called_once() - call_args = mock_logger.info.call_args[0][0] - self.assertIn("Starting Post Processing", call_args) - @patch.dict(os.environ, {"CBT_PP_LOGFILE_LOCATION": "/custom/path"}) def test_logfile_location_from_env(self) -> None: """Test that LOGFILE_LOCATION can be set via environment variable""" diff --git a/tests/test_post_processing_types.py b/tests/test_post_processing_types.py index bb549ca6..3f76b6b0 100644 --- a/tests/test_post_processing_types.py +++ b/tests/test_post_processing_types.py @@ -14,6 +14,7 @@ JobsDataType, PlotDataType, ReportOptions, + ReportType, ) @@ -67,7 +68,7 @@ def test_report_options_creation(self) -> None: create_pdf=True, force_refresh=False, no_error_bars=True, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=True, ) @@ -77,7 +78,7 @@ def test_report_options_creation(self) -> None: self.assertTrue(options.create_pdf) self.assertFalse(options.force_refresh) self.assertTrue(options.no_error_bars) - self.assertFalse(options.comparison) + self.assertEqual(options.report_type, ReportType.SIMPLE) self.assertTrue(options.plot_resources) def test_report_options_immutable(self) -> None: @@ -89,7 +90,7 @@ def test_report_options_immutable(self) -> None: create_pdf=True, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) @@ -105,7 +106,7 @@ def test_report_options_access_by_index(self) -> None: create_pdf=True, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) @@ -123,13 +124,20 @@ def test_report_options_unpack(self) -> None: create_pdf=True, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) - (archives, output_dir, results_root, create_pdf, force_refresh, no_error_bars, comparison, plot_resources) = ( - options - ) + ( + archives, + output_dir, + results_root, + create_pdf, + _force_refresh, + _no_error_bars, + _report_type, + _plot_resources, + ) = options self.assertEqual(archives, ["archive1"]) self.assertEqual(output_dir, "/output") @@ -145,7 +153,7 @@ def test_report_options_as_dict(self) -> None: create_pdf=True, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) @@ -165,7 +173,7 @@ def test_report_options_replace(self) -> None: create_pdf=True, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) diff --git a/tests/test_rbdfio.py b/tests/test_rbdfio.py new file mode 100644 index 00000000..d347f2bf --- /dev/null +++ b/tests/test_rbdfio.py @@ -0,0 +1,407 @@ +""" +Unit tests for the RBDFIO class +""" + +# pyright: strict, reportPrivateUsage=false + +import json +import shutil +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from post_processing.run_results.rbdfio import RBDFIO + + +class TestRBDFIO(unittest.TestCase): + """Test cases for RBDFIO class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.test_path = Path(self.temp_dir) + + # Create test FIO output data for multiple volumes + self.test_data: dict[str, Any] = { + "global options": { + "bs": "4096", + "rw": "randread", + "iodepth": "32", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [ + { + "read": { + "io_bytes": 1000000000, + "bw_bytes": 16666666, + "iops": 4000.0, + "total_ios": 244140, + "clat_ns": {"mean": 8000000.0, "stddev": 500000.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "sys_cpu": 5.5, + "usr_cpu": 10.2, + } + ], + } + + # Create multiple test files simulating multiple volumes + for i in range(3): + test_file = self.test_path / f"json_output.{i}" + with open(test_file, "w") as f: + json.dump(self.test_data, f) + + def tearDown(self) -> None: + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_initialization(self) -> None: + """Test RBDFIO initialization""" + rbdfio = RBDFIO(self.test_path, "json_output") + + self.assertEqual(rbdfio._path, self.test_path) + self.assertFalse(rbdfio._has_been_processed) + self.assertEqual(len(rbdfio._files), 3) + + def test_type_property(self) -> None: + """Test type property returns correct value""" + rbdfio = RBDFIO(self.test_path, "json_output") + + self.assertEqual(rbdfio.type, "rbdfio") + + def test_find_files_for_testrun(self) -> None: + """Test finding files matching the pattern""" + rbdfio = RBDFIO(self.test_path, "json_output") + + files = rbdfio._find_files_for_testrun("json_output") + + self.assertEqual(len(files), 3) + for i, file_path in enumerate(sorted(files)): + self.assertTrue(file_path.name.startswith("json_output.")) + self.assertTrue(file_path.name.endswith(str(i))) + + def test_find_files_no_matches(self) -> None: + """Test finding files when no matches exist""" + rbdfio = RBDFIO(self.test_path, "json_output") + + files = rbdfio._find_files_for_testrun("nonexistent") + + self.assertEqual(len(files), 0) + + def test_sum_io_details(self) -> None: + """Test summing IO details from multiple volumes""" + rbdfio = RBDFIO(self.test_path, "json_output") + + existing_values = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "244140", + "latency": "8.0", + "std_deviation": "0.5", + } + + new_values = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "244140", + "latency": "8.0", + "std_deviation": "0.5", + } + + result = rbdfio._sum_io_details(existing_values, new_values) + + # Check summed values + self.assertAlmostEqual(float(result["io_bytes"]), 2000000000.0, places=0) + self.assertAlmostEqual(float(result["iops"]), 8000.0, places=1) + self.assertAlmostEqual(float(result["bandwidth_bytes"]), 33333332.0, places=0) + self.assertEqual(int(result["total_ios"]), 488280) + + # Latency should be weighted average (same values = same result) + self.assertAlmostEqual(float(result["latency"]), 8.0, places=1) + + def test_sum_io_details_different_latencies(self) -> None: + """Test summing IO details with different latencies""" + rbdfio = RBDFIO(self.test_path, "json_output") + + existing_values = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "100000", + "latency": "5.0", + "std_deviation": "0.5", + } + + new_values = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "200000", + "latency": "10.0", + "std_deviation": "1.0", + } + + result = rbdfio._sum_io_details(existing_values, new_values) + + # Weighted average: (5*100000 + 10*200000) / 300000 = 8.333... + self.assertAlmostEqual(float(result["latency"]), 8.333, places=2) + + def test_create_benchmark_result(self) -> None: + """Test creating benchmark result returns FIO instance""" + rbdfio = RBDFIO(self.test_path, "json_output") + test_file = self.test_path / "json_output.0" + + result = rbdfio._create_benchmark_result(test_file) + + self.assertEqual(result.source, "fio") + self.assertEqual(result.blocksize, "4096") + self.assertEqual(result.operation, "randread") + self.assertEqual(result.iodepth, "32") + self.assertEqual(result.number_of_jobs, "1") + + def test_create_resource_result(self) -> None: + """Test creating resource result returns FIOResource instance""" + rbdfio = RBDFIO(self.test_path, "json_output") + test_file = self.test_path / "json_output.0" + + result = rbdfio._create_resource_result(test_file) + + # FIOResource should have source property + self.assertEqual(result.source, "fio") + + def test_process_results(self) -> None: + """Test processing results aggregates data correctly""" + rbdfio = RBDFIO(self.test_path, "json_output") + + # Process the results + rbdfio.process() + + self.assertTrue(rbdfio._has_been_processed) + self.assertIsNotNone(rbdfio._processed_data) + + # Check that data was aggregated + # Should have operation -> numjobs -> blocksize -> iodepth structure + self.assertIn("randread", rbdfio._processed_data) + + @patch("post_processing.run_results.benchmarks.fio.FIO.get_timeseries_data") + def test_process_results_with_timeseries(self, mock_get_timeseries_data: Any) -> None: + """Test processing results with time-series data enabled""" + mock_get_timeseries_data.side_effect = [ + { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4096", + "numjobs": "1", + "iodepth": "32", + "metadata": { + "start_time_epoch": 1.0, + "end_time_epoch": 2.0, + "duration_seconds": 1.0, + "num_volumes": 1, + "sampling_interval_ms": 1000, + "log_avg_msec": 1000, + }, + "timeseries": [ + { + "timestamp_sec": 1.0, + "iops": 100.0, + "bandwidth_bytes": 4096.0, + "mean_latency_ms": 2.0, + "max_latency_ms": 3.0, + "p50_latency_ms": 1.5, + "p95_latency_ms": 2.5, + "p99_latency_ms": 2.8, + "num_samples": 1, + } + ], + "maximum_iops": "100", + "maximum_bandwidth": "4096", + "latency_at_max_iops": "2.0", + "latency_at_max_bandwidth": "2.0", + "timestamp_at_max_iops": "1.0", + "timestamp_at_max_bandwidth": "1.0", + "maximum_latency": "3.0", + "timestamp_at_max_latency": "1.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + }, + { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4096", + "numjobs": "1", + "iodepth": "32", + "metadata": { + "start_time_epoch": 1.0, + "end_time_epoch": 2.0, + "duration_seconds": 1.0, + "num_volumes": 1, + "sampling_interval_ms": 1000, + "log_avg_msec": 1000, + }, + "timeseries": [ + { + "timestamp_sec": 1.0, + "iops": 200.0, + "bandwidth_bytes": 8192.0, + "mean_latency_ms": 4.0, + "max_latency_ms": 5.0, + "p50_latency_ms": 3.5, + "p95_latency_ms": 4.5, + "p99_latency_ms": 4.8, + "num_samples": 1, + } + ], + "maximum_iops": "200", + "maximum_bandwidth": "8192", + "latency_at_max_iops": "4.0", + "latency_at_max_bandwidth": "4.0", + "timestamp_at_max_iops": "1.0", + "timestamp_at_max_bandwidth": "1.0", + "maximum_latency": "5.0", + "timestamp_at_max_latency": "1.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + }, + { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4096", + "numjobs": "1", + "iodepth": "32", + "metadata": { + "start_time_epoch": 1.0, + "end_time_epoch": 2.0, + "duration_seconds": 1.0, + "num_volumes": 1, + "sampling_interval_ms": 1000, + "log_avg_msec": 1000, + }, + "timeseries": [ + { + "timestamp_sec": 1.0, + "iops": 300.0, + "bandwidth_bytes": 12288.0, + "mean_latency_ms": 6.0, + "max_latency_ms": 7.0, + "p50_latency_ms": 5.5, + "p95_latency_ms": 6.5, + "p99_latency_ms": 6.8, + "num_samples": 1, + } + ], + "maximum_iops": "300", + "maximum_bandwidth": "12288", + "latency_at_max_iops": "6.0", + "latency_at_max_bandwidth": "6.0", + "timestamp_at_max_iops": "1.0", + "timestamp_at_max_bandwidth": "1.0", + "maximum_latency": "7.0", + "timestamp_at_max_latency": "1.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + }, + ] + + rbdfio = RBDFIO(self.test_path, "json_output", include_timeseries=True) + + # Process the results + rbdfio.process() + + self.assertTrue(rbdfio._has_been_processed) + + # Verify that timeseries data was written to disk + # Check the visualisation directory was created + vis_dir = self.test_path / "visualisation" + self.assertTrue(vis_dir.exists(), "Visualisation directory should be created") + + # Find the timeseries JSON file + ts_files = list(vis_dir.glob("*_timeseries.json")) + self.assertEqual(len(ts_files), 1, "Should have exactly one timeseries file") + + # Load and verify the aggregated data + with open(ts_files[0]) as f: + aggregated_data = json.load(f) + + # CRITICAL TEST: Verify aggregation across all 3 volumes + # This test would have FAILED before the fix because only the last volume's + # data would be present (300 IOPS instead of 600) + self.assertEqual(aggregated_data["metadata"]["num_volumes"], 3, "Should aggregate data from all 3 volumes") + + # Verify IOPS are summed: 100 + 200 + 300 = 600 + first_point = aggregated_data["timeseries"][0] + self.assertAlmostEqual( + first_point["iops"], 600.0, places=1, msg="IOPS should be summed across all volumes (100+200+300=600)" + ) + + # Verify bandwidth is summed: 4096 + 8192 + 12288 = 24576 + self.assertAlmostEqual( + first_point["bandwidth_bytes"], 24576.0, places=1, msg="Bandwidth should be summed across all volumes" + ) + + # Verify latency is weighted-averaged by IOPS (correct statistical approach) + # Weighted avg = (100*2.0 + 200*4.0 + 300*6.0) / (100+200+300) + # = (200 + 800 + 1800) / 600 = 2800/600 = 4.667ms + self.assertAlmostEqual( + first_point["mean_latency_ms"], 4.667, places=2, msg="Mean latency should be weighted average by IOPS" + ) + + # Verify max_latency takes the maximum value (worst case across all volumes) + # max(3.0, 5.0, 7.0) = 7.0 + self.assertAlmostEqual( + first_point["max_latency_ms"], 7.0, places=2, msg="Max latency should be the maximum across all volumes" + ) + + # Verify other latency percentiles are weighted-averaged by IOPS + # p50: (100*1.5 + 200*3.5 + 300*5.5) / 600 = 4.167 + self.assertAlmostEqual( + first_point["p50_latency_ms"], 4.167, places=2, msg="P50 latency should be weighted average" + ) + # p95: (100*2.5 + 200*4.5 + 300*6.5) / 600 = 5.167 + self.assertAlmostEqual( + first_point["p95_latency_ms"], 5.167, places=2, msg="P95 latency should be weighted average" + ) + # p99: (100*2.8 + 200*4.8 + 300*6.8) / 600 = 5.467 + self.assertAlmostEqual( + first_point["p99_latency_ms"], 5.467, places=2, msg="P99 latency should be weighted average" + ) + + # After processing with memory-efficient mode, data should be written and cleared + self.assertEqual( + len(rbdfio._timeseries_data), 0, "Timeseries data should be empty after memory-efficient write" + ) + + def test_get_results_without_processing(self) -> None: + """Test getting results automatically processes if not done""" + rbdfio = RBDFIO(self.test_path, "json_output") + + # get() should automatically call process() if not already processed + results = rbdfio.get() + + self.assertTrue(rbdfio._has_been_processed) + self.assertIsNotNone(results) + self.assertIsInstance(results, dict) + + def test_get_results_after_processing(self) -> None: + """Test getting results after processing returns data""" + rbdfio = RBDFIO(self.test_path, "json_output") + + rbdfio.process() + results = rbdfio.get() + + self.assertIsNotNone(results) + self.assertIsInstance(results, dict) + + +# Made with Bob diff --git a/tests/test_report.py b/tests/test_report.py index eb66e556..c643b174 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -7,6 +7,7 @@ # We are OK to ignore private use in unit tests as the whole point of the tests # is to validate the functions contained in the module +import json import shutil import tempfile import unittest @@ -14,7 +15,7 @@ from pathlib import Path from unittest.mock import MagicMock, patch -from post_processing.post_processing_types import ReportOptions +from post_processing.post_processing_types import ReportOptions, ReportType from post_processing.report import Report, parse_namespace_to_options @@ -39,7 +40,7 @@ def test_parse_namespace_simple_report(self) -> None: self.assertTrue(options.create_pdf) self.assertFalse(options.force_refresh) self.assertFalse(options.no_error_bars) - self.assertFalse(options.comparison) + self.assertEqual(options.report_type, ReportType.SIMPLE) self.assertFalse(options.plot_resources) def test_parse_namespace_comparison_report(self) -> None: @@ -59,7 +60,7 @@ def test_parse_namespace_comparison_report(self) -> None: self.assertEqual(options.archives[0], "/path/to/baseline") self.assertEqual(options.archives[1], "/path/to/archive1") self.assertEqual(options.archives[2], "/path/to/archive2") - self.assertTrue(options.comparison) + self.assertEqual(options.report_type, ReportType.COMPARISON) self.assertTrue(options.force_refresh) def test_parse_namespace_with_no_error_bars(self) -> None: @@ -122,7 +123,7 @@ def setUp(self) -> None: create_pdf=False, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) @@ -216,9 +217,9 @@ def test_generate_with_intermediate_file_creation( report.generate() # Should create formatter and convert files + # With memory-efficient approach, data is written during process() mock_formatter.assert_called_once() - mock_formatter_instance.convert_all_files.assert_called_once() - mock_formatter_instance.write_output_file.assert_called_once() + mock_formatter_instance.process.assert_called_once() @patch("post_processing.report.os.makedirs") @patch("post_processing.report.CommonOutputFormatter") @@ -237,7 +238,7 @@ def test_generate_comparison_report( create_pdf=False, force_refresh=False, no_error_bars=False, - comparison=True, + report_type=ReportType.COMPARISON, plot_resources=False, ) @@ -274,7 +275,7 @@ def test_generate_with_pdf_creation( create_pdf=True, force_refresh=False, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) @@ -369,7 +370,7 @@ def test_generate_with_force_refresh( create_pdf=False, force_refresh=True, no_error_bars=False, - comparison=False, + report_type=ReportType.SIMPLE, plot_resources=False, ) @@ -387,9 +388,9 @@ def test_generate_with_force_refresh( report.generate() # Should still create formatter and regenerate files + # With memory-efficient approach, data is written during process() mock_formatter.assert_called_once() - mock_formatter_instance.convert_all_files.assert_called_once() - mock_formatter_instance.write_output_file.assert_called_once() + mock_formatter_instance.process.assert_called_once() @patch("post_processing.report.os.makedirs") @patch("post_processing.report.os.path.exists") @@ -407,14 +408,100 @@ def test_generate_intermediate_files_exception( # Make formatter raise exception mock_formatter_instance = MagicMock() - mock_formatter_instance.convert_all_files.side_effect = Exception("Conversion error") + mock_formatter_instance.process.side_effect = RuntimeError("Conversion error") mock_formatter.return_value = mock_formatter_instance report = Report(self.options) # Should catch and re-raise exception - with self.assertRaises(Exception): + with self.assertRaises(RuntimeError): report.generate(throw_exception=True) + @patch("post_processing.report.os.makedirs") + @patch("post_processing.report.TimeSeriesOutputFormatter") + @patch("post_processing.report.TimeSeriesReportGenerator") + def test_generate_timeseries_report_skips_regeneration_with_nested_timeseries_files( + self, + mock_timeseries_generator: MagicMock, + mock_formatter: MagicMock, + mock_makedirs: MagicMock, + ) -> None: + """Test time-series report skips regeneration when nested time-series files exist""" + timeseries_options = ReportOptions( + archives=[self.temp_dir], + output_directory=f"{self.temp_dir}/output", + results_file_root="test_results", + create_pdf=False, + force_refresh=False, + no_error_bars=False, + report_type=ReportType.TIMESERIES, + plot_resources=False, + ) + + nested_vis_dir = Path(self.temp_dir) / "read" / "visualisation" + nested_vis_dir.mkdir(parents=True) + with open(nested_vis_dir / "4096_1_read_timeseries.json", "w", encoding="utf-8") as file_handle: + json.dump({"metadata": {}, "timeseries": []}, file_handle) + + mock_generator_instance = MagicMock() + mock_timeseries_generator.return_value = mock_generator_instance + + report = Report(timeseries_options) + report.generate() + + mock_formatter.assert_not_called() + mock_timeseries_generator.assert_called_once() + mock_generator_instance.create_report.assert_called_once() + + @patch("post_processing.report.os.makedirs") + @patch("post_processing.report.CommonOutputFormatter") + @patch("post_processing.report.SimpleReportGenerator") + def test_generate_simple_report_skips_regeneration_with_nested_hockey_stick_files( + self, + mock_simple_generator: MagicMock, + mock_formatter: MagicMock, + mock_makedirs: MagicMock, + ) -> None: + """Test simple report skips regeneration when nested non-timeseries files exist""" + nested_vis_dir = Path(self.temp_dir) / "read" / "visualisation" + nested_vis_dir.mkdir(parents=True) + with open(nested_vis_dir / "4096_1_read.json", "w", encoding="utf-8") as file_handle: + json.dump({"data": []}, file_handle) + + mock_generator_instance = MagicMock() + mock_simple_generator.return_value = mock_generator_instance + + report = Report(self.options) + report.generate() + + mock_formatter.assert_not_called() + mock_simple_generator.assert_called_once() + mock_generator_instance.create_report.assert_called_once() + + @patch("post_processing.report.os.makedirs") + @patch("post_processing.report.CommonOutputFormatter") + @patch("post_processing.report.SimpleReportGenerator") + def test_generate_simple_report_skips_regeneration_with_legacy_visualisation_files( + self, + mock_simple_generator: MagicMock, + mock_formatter: MagicMock, + mock_makedirs: MagicMock, + ) -> None: + """Test simple report still skips regeneration with legacy top-level files""" + legacy_vis_dir = Path(self.temp_dir) / "visualisation" + legacy_vis_dir.mkdir(parents=True) + with open(legacy_vis_dir / "4096_1_read.json", "w", encoding="utf-8") as file_handle: + json.dump({"data": []}, file_handle) + + mock_generator_instance = MagicMock() + mock_simple_generator.return_value = mock_generator_instance + + report = Report(self.options) + report.generate() + + mock_formatter.assert_not_called() + mock_simple_generator.assert_called_once() + mock_generator_instance.create_report.assert_called_once() + # Made with Bob diff --git a/tests/test_report_generator.py b/tests/test_report_generator.py index 42102ded..3d7d5f16 100644 --- a/tests/test_report_generator.py +++ b/tests/test_report_generator.py @@ -20,16 +20,24 @@ class TestReportGenerator(unittest.TestCase): """Test cases for ReportGenerator base class""" def setUp(self) -> None: - """Set up test fixtures""" + """Set up test fixtures with new nested structure""" self.temp_dir = tempfile.mkdtemp() self.archive_dir = Path(self.temp_dir) / "archive" + + # Create new nested structure: operation/visualisation/ + read_vis_dir = self.archive_dir / "read" / "visualisation" + write_vis_dir = self.archive_dir / "write" / "visualisation" + read_vis_dir.mkdir(parents=True) + write_vis_dir.mkdir(parents=True) + + # Also create top-level visualisation for SVG files self.vis_dir = self.archive_dir / "visualisation" self.vis_dir.mkdir(parents=True) - # Create some test data files + # Create some test data files in nested structure # Format: {blocksize}_{numjobs}_{operation}.json - (self.vis_dir / "4096_1_read.json").touch() - (self.vis_dir / "8192_1_write.json").touch() + (read_vis_dir / "4096_1_read.json").touch() + (write_vis_dir / "8192_1_write.json").touch() def tearDown(self) -> None: """Clean up test fixtures""" @@ -51,7 +59,8 @@ def test_initialization(self) -> None: self.assertFalse(generator._force_refresh) self.assertFalse(generator._plot_resources) self.assertEqual(len(generator._archive_directories), 1) - self.assertEqual(len(generator._data_directories), 1) + # With new nested structure, we have 2 operation directories (read and write) + self.assertEqual(len(generator._data_directories), 2) def test_initialization_with_no_error_bars(self) -> None: """Test initialization with no_error_bars=True""" @@ -84,10 +93,16 @@ def test_initialization_with_plot_resources(self) -> None: def test_build_strings_replace_underscores(self) -> None: """Test that build strings replace underscores with hyphens""" archive_with_underscores = Path(self.temp_dir) / "test_archive_name" - vis_dir = archive_with_underscores / "visualisation" - vis_dir.mkdir(parents=True) + + # Create new nested structure + read_vis_dir = archive_with_underscores / "read" / "visualisation" + read_vis_dir.mkdir(parents=True) # Format: {blocksize}_{numjobs}_{operation}.json - (vis_dir / "4096_1_read.json").touch() + (read_vis_dir / "4096_1_read.json").touch() + + # Also create top-level visualisation for SVG files + top_vis_dir = archive_with_underscores / "visualisation" + top_vis_dir.mkdir(parents=True) output_dir = f"{self.temp_dir}/output" @@ -115,10 +130,12 @@ def test_find_files_with_filename(self) -> None: def test_sort_list_of_paths(self) -> None: """Test sorting paths by numeric blocksize""" - # Create files with different blocksizes + # Create files with different blocksizes in new nested structure # Format: {blocksize}_{numjobs}_{operation}.json - (self.vis_dir / "16384_1_read.json").touch() - (self.vis_dir / "1024_1_read.json").touch() + read_vis_dir = self.archive_dir / "read" / "visualisation" + read_vis_dir.mkdir(parents=True, exist_ok=True) + (read_vis_dir / "16384_1_read.json").touch() + (read_vis_dir / "1024_1_read.json").touch() output_dir = f"{self.temp_dir}/output" @@ -127,7 +144,7 @@ def test_sort_list_of_paths(self) -> None: output_directory=output_dir, ) - paths = list(self.vis_dir.glob("*.json")) + paths = list(read_vis_dir.glob("*.json")) sorted_paths = generator._sort_list_of_paths(paths, index=0) # Should be sorted by blocksize: 1024, 4096, 8192, 16384 @@ -149,6 +166,24 @@ def test_generate_plot_directory_name(self) -> None: # Should have timestamp appended self.assertGreater(len(plot_dir_name), len(f"{output_dir}/plots.")) + def test_force_refresh_allows_legacy_visualisation_directory(self) -> None: + """Test that force_refresh bypasses legacy visualisation directory rejection""" + legacy_archive = Path(self.temp_dir) / "legacy_archive" + legacy_vis_dir = legacy_archive / "visualisation" + legacy_vis_dir.mkdir(parents=True) + (legacy_vis_dir / "4096_1_read.json").touch() + + output_dir = f"{self.temp_dir}/output" + + generator = SimpleReportGenerator( + archive_directories=[str(legacy_archive)], + output_directory=output_dir, + force_refresh=True, + ) + + self.assertTrue(generator._force_refresh) + self.assertEqual(generator._data_directories, [legacy_vis_dir]) + def test_constants(self) -> None: """Test ReportGenerator constants""" self.assertEqual(ReportGenerator.MARKDOWN_FILE_EXTENSION, "md") diff --git a/tests/test_resource_result.py b/tests/test_resource_result.py index f30b1535..8bdfb5e1 100644 --- a/tests/test_resource_result.py +++ b/tests/test_resource_result.py @@ -14,7 +14,7 @@ from pathlib import Path from typing import Any -from post_processing.run_results.resources.resource_result import ResourceResult +from post_processing.run_results.resource_result import ResourceResult class ConcreteResourceResult(ResourceResult): diff --git a/tests/test_run_result.py b/tests/test_run_result.py new file mode 100644 index 00000000..7292780d --- /dev/null +++ b/tests/test_run_result.py @@ -0,0 +1,462 @@ +""" +Tests for the RunResult base class. + +This module tests the abstract base class functionality and helper methods +that are inherited by concrete implementations like RBDFIO. +""" + +import json +import tempfile +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from post_processing.post_processing_types import IodepthDataType +from post_processing.run_results.benchmark_result import BenchmarkResult +from post_processing.run_results.rbdfio import RBDFIO + + +class TestRunResultInitialization: + """Test RunResult initialization through concrete subclass.""" + + def test_initialization_basic(self): + """Test basic initialization of RunResult.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + assert result._path == path + assert result._has_been_processed is False + assert result._include_timeseries is False + assert isinstance(result._files, list) + assert isinstance(result._processed_data, dict) + assert isinstance(result._timeseries_data, dict) + + def test_initialization_with_timeseries(self): + """Test initialization with timeseries enabled.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output", include_timeseries=True) + + assert result._include_timeseries is True + + +class TestProcessMethod: + """Test the process() method.""" + + def test_process_with_no_files(self): + """Test process() when no files are found.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + # No files created, so _files should be empty + result.process() + + assert result._has_been_processed is True + assert len(result._processed_data) == 0 + + def test_process_with_files(self): + """Test process() with valid files.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create test data + test_data = { + "global options": { + "bs": "4096", + "rw": "randread", + "iodepth": "32", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [{ + "read": { + "io_bytes": 1000000000, + "bw_bytes": 16666666, + "iops": 4000.0, + "total_ios": 244140, + "clat_ns": {"mean": 8000000.0, "stddev": 500000.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "sys_cpu": 5.5, + "usr_cpu": 10.2, + }], + } + + # Create test file + test_file = path / "json_output.0" + with open(test_file, "w") as f: + json.dump(test_data, f) + + result = RBDFIO(path, "json_output") + result.process() + + assert result._has_been_processed is True + assert len(result._processed_data) > 0 + + +class TestGetMethods: + """Test get() and get_timeseries() methods.""" + + def test_get_without_processing(self): + """Test get() automatically processes if not done.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + assert result._has_been_processed is False + + data = result.get() + + assert result._has_been_processed is True + assert isinstance(data, dict) + + def test_get_after_processing(self): + """Test get() returns data after processing.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + result.process() + data = result.get() + + assert isinstance(data, dict) + + # Tests for get_timeseries() removed - with memory-efficient approach, + # timeseries data is written immediately during process() and not stored in memory + + +class TestProcessTestRunFiles: + """Test _process_test_run_files() method.""" + + def test_process_empty_file(self): + """Test processing skips empty files.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create empty file + empty_file = path / "json_output.0" + empty_file.touch() + + result = RBDFIO(path, "json_output") + result.process() + + # Should complete without error, but no data processed + assert result._has_been_processed is True + + def test_process_precondition_file(self): + """Test processing skips precondition files.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create test data + test_data = { + "global options": { + "bs": "4096", + "rw": "randread", + "iodepth": "32", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [{ + "jobname": "precondition", + "read": { + "io_bytes": 1000000000, + "bw_bytes": 16666666, + "iops": 4000.0, + "total_ios": 244140, + "clat_ns": {"mean": 8000000.0, "stddev": 500000.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "sys_cpu": 5.5, + "usr_cpu": 10.2, + }], + } + + # Create precondition file + precond_file = path / "json_output.0" + with open(precond_file, "w") as f: + json.dump(test_data, f) + + result = RBDFIO(path, "json_output") + result.process() + + # Should complete without error + assert result._has_been_processed is True + + +class TestExtractTestConfiguration: + """Test _extract_test_configuration() method.""" + + def test_extract_configuration(self): + """Test extracting configuration from benchmark result.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + # Create mock benchmark result + mock_benchmark = Mock(spec=BenchmarkResult) + mock_benchmark.operation = "randread" + mock_benchmark.blocksize = "4096" + mock_benchmark.iodepth = "32" + mock_benchmark.number_of_jobs = "1" + + config = result._extract_test_configuration(mock_benchmark) + + assert config == ("randread", "4096", "32", "1") + + +class TestMergeIODetails: + """Test _merge_io_details() method.""" + + def test_merge_with_no_existing_data(self): + """Test merge when no existing data exists.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + test_config = ("randread", "4096", "32", "1") + new_io_details: IodepthDataType = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "244140", + "latency": "8.0", + "std_deviation": "0.5", + } + + merged = result._merge_io_details(test_config, new_io_details) + + # Should return new_io_details unchanged + assert merged == new_io_details + + def test_merge_with_existing_data(self): + """Test merge when existing data exists.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + result = RBDFIO(path, "json_output") + + # Set up existing data + test_config = ("randread", "4096", "32", "1") + existing_io: IodepthDataType = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "244140", + "latency": "8.0", + "std_deviation": "0.5", + } + + result._processed_data = { + "randread": { + "1": { + "4096": { + "32": existing_io + } + } + } + } + + new_io_details: IodepthDataType = { + "io_bytes": "1000000000", + "iops": "4000.0", + "bandwidth_bytes": "16666666", + "total_ios": "244140", + "latency": "8.0", + "std_deviation": "0.5", + } + + merged = result._merge_io_details(test_config, new_io_details) + + # Should sum the values + assert float(merged["io_bytes"]) == 2000000000.0 + assert float(merged["iops"]) == 8000.0 + + +class TestConvertFile: + """Test _convert_file() method.""" + + def test_convert_file_success(self): + """Test successful file conversion.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create test data + test_data = { + "global options": { + "bs": "4096", + "rw": "randread", + "iodepth": "32", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [{ + "read": { + "io_bytes": 1000000000, + "bw_bytes": 16666666, + "iops": 4000.0, + "total_ios": 244140, + "clat_ns": {"mean": 8000000.0, "stddev": 500000.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "sys_cpu": 5.5, + "usr_cpu": 10.2, + }], + } + + test_file = path / "json_output.0" + with open(test_file, "w") as f: + json.dump(test_data, f) + + result = RBDFIO(path, "json_output") + result._convert_file(test_file) + + # Should have processed data + assert len(result._processed_data) > 0 + + def test_convert_file_with_error(self): + """Test file conversion with error handling.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create invalid JSON file + test_file = path / "json_output.0" + with open(test_file, "w") as f: + f.write("invalid json") + + result = RBDFIO(path, "json_output") + + # Should raise an exception + with pytest.raises(Exception): + result._convert_file(test_file) + + +class TestIntegration: + """Integration tests for RunResult.""" + + def test_full_workflow_single_volume(self): + """Test complete workflow with single volume.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create test data + test_data = { + "global options": { + "bs": "4096", + "rw": "randread", + "iodepth": "32", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [{ + "read": { + "io_bytes": 1000000000, + "bw_bytes": 16666666, + "iops": 4000.0, + "total_ios": 244140, + "clat_ns": {"mean": 8000000.0, "stddev": 500000.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "sys_cpu": 5.5, + "usr_cpu": 10.2, + }], + } + + test_file = path / "json_output.0" + with open(test_file, "w") as f: + json.dump(test_data, f) + + result = RBDFIO(path, "json_output") + data = result.get() + + assert "randread" in data + assert "1" in data["randread"] + assert "4096" in data["randread"]["1"] + assert "32" in data["randread"]["1"]["4096"] + + def test_full_workflow_multiple_volumes(self): + """Test complete workflow with multiple volumes.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = Path(tmpdir) + + # Create test data for multiple volumes + test_data = { + "global options": { + "bs": "4096", + "rw": "randread", + "iodepth": "32", + "numjobs": "1", + "runtime": "60", + }, + "jobs": [{ + "read": { + "io_bytes": 1000000000, + "bw_bytes": 16666666, + "iops": 4000.0, + "total_ios": 244140, + "clat_ns": {"mean": 8000000.0, "stddev": 500000.0}, + }, + "write": { + "io_bytes": 0, + "bw_bytes": 0, + "iops": 0.0, + "total_ios": 0, + "clat_ns": {"mean": 0.0, "stddev": 0.0}, + }, + "sys_cpu": 5.5, + "usr_cpu": 10.2, + }], + } + + # Create multiple volume files + for i in range(3): + test_file = path / f"json_output.{i}" + with open(test_file, "w") as f: + json.dump(test_data, f) + + result = RBDFIO(path, "json_output") + data = result.get() + + # Data should be aggregated from all volumes + assert "randread" in data + # Navigate through nested structure + randread_data = data["randread"] + assert isinstance(randread_data, dict) + numjobs_data = randread_data["1"] + assert isinstance(numjobs_data, dict) + blocksize_data = numjobs_data["4096"] + assert isinstance(blocksize_data, dict) + iodepth_data = blocksize_data["32"] + assert isinstance(iodepth_data, dict) + iops_value = float(iodepth_data["iops"]) + # Should be sum of 3 volumes: 4000 * 3 = 12000 + assert iops_value == pytest.approx(12000.0, rel=0.01) + + +# Made with Bob diff --git a/tests/test_run_result_directory_detection.py b/tests/test_run_result_directory_detection.py new file mode 100644 index 00000000..de4266a2 --- /dev/null +++ b/tests/test_run_result_directory_detection.py @@ -0,0 +1,179 @@ +""" +Unit tests for RunResult directory detection and timeseries writing. + +Tests verify that timeseries and hockey-stick data are written at the +correct directory levels in the hierarchy. +""" + +import unittest +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +from post_processing.post_processing_types import TimeSeriesFormatType +from post_processing.run_results.run_result import RunResult + + +class TestRunResultDirectoryDetection(unittest.TestCase): + """Test directory detection for timeseries output""" + + def setUp(self): + """Set up test fixtures""" + # Create a mock RunResult subclass + self.mock_run_result = MagicMock(spec=RunResult) + self.mock_run_result._determine_aggregation_directory_from_file = ( + RunResult._determine_aggregation_directory_from_file.__get__(self.mock_run_result) + ) + + def test_determine_aggregation_directory_from_file_with_total_iodepth(self): + """Test detection of total_iodepth directory from file path""" + file_path = Path( + "/home/user/results/00000000/id-806228fa/seq8kwrite.host.com/" + "rbdfio/numjobs-001/total_iodepth-256/iodepth-000032/json_output.0" + ) + + result = self.mock_run_result._determine_aggregation_directory_from_file(file_path) + + expected = Path( + "/home/user/results/00000000/id-806228fa/seq8kwrite.host.com/" + "rbdfio/numjobs-001/total_iodepth-256" + ) + self.assertEqual(result, expected) + + def test_determine_aggregation_directory_from_file_with_iodepth_only(self): + """Test detection of iodepth directory when no total_iodepth exists""" + file_path = Path( + "/home/user/results/00000000/id-123/test/rbdfio/" + "numjobs-001/iodepth-032/json_output.0" + ) + + result = self.mock_run_result._determine_aggregation_directory_from_file(file_path) + + expected = Path("/home/user/results/00000000/id-123/test/rbdfio/numjobs-001/iodepth-032") + self.assertEqual(result, expected) + + def test_determine_aggregation_directory_from_file_no_depth_dirs(self): + """Test fallback to parent directory when no depth directories exist""" + file_path = Path("/home/user/results/00000000/id-123/test/rbdfio/numjobs-001/json_output.0") + + result = self.mock_run_result._determine_aggregation_directory_from_file(file_path) + + expected = Path("/home/user/results/00000000/id-123/test/rbdfio/numjobs-001") + self.assertEqual(result, expected) + + def test_determine_aggregation_directory_from_file_prefers_total_iodepth(self): + """Test that total_iodepth is preferred over iodepth""" + # Path with both total_iodepth and iodepth + file_path = Path( + "/home/user/results/rbdfio/numjobs-001/total_iodepth-256/" + "iodepth-000032/json_output.0" + ) + + result = self.mock_run_result._determine_aggregation_directory_from_file(file_path) + + # Should stop at total_iodepth, not continue to iodepth + expected = Path("/home/user/results/rbdfio/numjobs-001/total_iodepth-256") + self.assertEqual(result, expected) + + +class TestRunResultTimeseriesWriting(unittest.TestCase): + """Test timeseries data writing at correct directory levels""" + + def setUp(self): + """Set up test fixtures""" + self.test_dir = Path("/tmp/test_run_result") + + @patch("json.dump") + @patch("post_processing.run_results.run_result.Path.open") + @patch("post_processing.run_results.run_result.Path.mkdir") + def test_write_timeseries_by_directory_groups_correctly(self, mock_mkdir, mock_open, mock_json_dump): + """Test that timeseries data is grouped and written by directory""" + # Create a mock RunResult + mock_result = MagicMock(spec=RunResult) + mock_result._timeseries_by_directory = { + Path("/test/total_iodepth-256"): { + "randread_4k_256": { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4k", + "numjobs": "1", + "metadata": {}, + "timeseries": [], + } + }, + Path("/test/total_iodepth-128"): { + "randwrite_4k_128": { + "benchmark": "fio", + "operation": "randwrite", + "blocksize": "4k", + "numjobs": "1", + "metadata": {}, + "timeseries": [], + } + }, + } + + # Call the actual method + RunResult._write_and_clear_timeseries_by_directory(mock_result) + + # Verify mkdir was called for each directory + self.assertEqual(mock_mkdir.call_count, 2) + + # Verify json.dump was called twice (once per file) + self.assertEqual(mock_json_dump.call_count, 2) + + # Verify data was cleared + self.assertEqual(len(mock_result._timeseries_by_directory), 0) + + @patch("json.dump") + @patch("builtins.open") + def test_write_timeseries_creates_visualisation_subdirectory(self, mock_open, mock_json_dump): + """Test that visualisation subdirectory is created at aggregation level""" + import tempfile + import shutil + + # Use a real temporary directory for this test + test_dir = Path(tempfile.mkdtemp()) + try: + agg_dir = test_dir / "total_iodepth-256" + agg_dir.mkdir(parents=True) + + mock_result = MagicMock(spec=RunResult) + mock_result._timeseries_by_directory = { + agg_dir: { + "test_key": { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4k", + "numjobs": "1", + "metadata": {}, + "timeseries": [], + } + } + } + + RunResult._write_and_clear_timeseries_by_directory(mock_result) + + # Verify visualisation subdirectory was created + expected_dir = agg_dir / "visualisation" + self.assertTrue(expected_dir.exists(), f"Expected {expected_dir} to exist") + self.assertTrue(expected_dir.is_dir(), f"Expected {expected_dir} to be a directory") + finally: + # Clean up + shutil.rmtree(test_dir, ignore_errors=True) + + +class TestRunResultIntegration(unittest.TestCase): + """Integration tests for directory detection and writing""" + + def test_timeseries_grouping_by_file_location(self): + """Test that files from different directories are grouped separately""" + # This would be an integration test that processes actual files + # and verifies they're written to the correct locations + # Placeholder for now - would need actual test data + pass + + +if __name__ == "__main__": + unittest.main() + +# Made with Bob diff --git a/tests/test_run_result_factory.py b/tests/test_run_result_factory.py new file mode 100644 index 00000000..30a2c868 --- /dev/null +++ b/tests/test_run_result_factory.py @@ -0,0 +1,265 @@ +""" +Tests for the run_result_factory module. + +This module tests the factory function that creates appropriate RunResult +subclasses based on benchmark type. +""" + +import tempfile +from pathlib import Path + +import pytest + +from post_processing.run_results.rbdfio import RBDFIO +from post_processing.run_results.run_result_factory import ( + BENCHMARK_TYPE_MAP, + get_run_result_from_directory_name, +) + + +class TestBenchmarkTypeMap: + """Test the BENCHMARK_TYPE_MAP constant.""" + + def test_map_contains_rbdfio(self): + """Test that map contains rbdfio.""" + assert "rbdfio" in BENCHMARK_TYPE_MAP + assert BENCHMARK_TYPE_MAP["rbdfio"] == RBDFIO + + def test_map_contains_librbdfio(self): + """Test that map contains librbdfio.""" + assert "librbdfio" in BENCHMARK_TYPE_MAP + assert BENCHMARK_TYPE_MAP["librbdfio"] == RBDFIO + + def test_map_contains_kvmrbdfio(self): + """Test that map contains kvmrbdfio.""" + assert "kvmrbdfio" in BENCHMARK_TYPE_MAP + assert BENCHMARK_TYPE_MAP["kvmrbdfio"] == RBDFIO + + def test_map_contains_rawfio(self): + """Test that map contains rawfio.""" + assert "rawfio" in BENCHMARK_TYPE_MAP + assert BENCHMARK_TYPE_MAP["rawfio"] == RBDFIO + + def test_all_map_to_same_class(self): + """Test that all FIO variants map to RBDFIO class.""" + fio_variants = ["rbdfio", "librbdfio", "kvmrbdfio", "rawfio"] + for variant in fio_variants: + assert BENCHMARK_TYPE_MAP[variant] == RBDFIO + + +class TestGetRunResultFromDirectoryName: + """Test the get_run_result_from_directory_name factory function.""" + + def test_rbdfio_directory(self): + """Test creating RunResult for rbdfio directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rbdfio_dir = parent_dir / "rbdfio" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + assert result.type == "rbdfio" + + def test_librbdfio_directory(self): + """Test creating RunResult for librbdfio directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + librbdfio_dir = parent_dir / "librbdfio" + librbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + assert result.type == "rbdfio" + + def test_kvmrbdfio_directory(self): + """Test creating RunResult for kvmrbdfio directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + kvmrbdfio_dir = parent_dir / "kvmrbdfio" + kvmrbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + assert result.type == "rbdfio" + + def test_rawfio_directory(self): + """Test creating RunResult for rawfio directory.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rawfio_dir = parent_dir / "rawfio" + rawfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + assert result.type == "rbdfio" + + def test_case_insensitive_matching(self): + """Test that directory matching is case-insensitive.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + # Create directory with mixed case + rbdfio_dir = parent_dir / "RbdFio" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + + def test_directory_with_prefix(self): + """Test matching when benchmark type is part of longer directory name.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + # Directory name contains 'rbdfio' as substring + rbdfio_dir = parent_dir / "test_rbdfio_results" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + + def test_directory_with_suffix(self): + """Test matching when benchmark type has suffix.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rbdfio_dir = parent_dir / "rbdfio_test" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + assert isinstance(result, RBDFIO) + + def test_unknown_benchmark_type_raises_error(self): + """Test that unknown benchmark type raises NotImplementedError.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + unknown_dir = parent_dir / "unknown_benchmark" + unknown_dir.mkdir() + + with pytest.raises(NotImplementedError, match="Could not determine benchmark type"): + get_run_result_from_directory_name(parent_dir, "json_output") + + def test_no_subdirectories_raises_error(self): + """Test that directory with no subdirectories raises error.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + # Create a file instead of directory + (parent_dir / "test.txt").touch() + + with pytest.raises(NotImplementedError, match="Could not determine benchmark type"): + get_run_result_from_directory_name(parent_dir, "json_output") + + def test_multiple_subdirectories_uses_first(self): + """Test that when multiple subdirectories exist, first one found is checked.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + # Create subdirectory with known benchmark type + # The function will use the first directory it finds via iterdir() + rbdfio_dir = parent_dir / "rbdfio" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + # Should find rbdfio since it's a known type + assert isinstance(result, RBDFIO) + + def test_custom_filename_root(self): + """Test that custom filename root is passed to RunResult.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rbdfio_dir = parent_dir / "rbdfio" + rbdfio_dir.mkdir() + + custom_root = "custom_output" + result = get_run_result_from_directory_name(parent_dir, custom_root) + + assert isinstance(result, RBDFIO) + # Verify the object was created successfully with custom root + assert result.type == "rbdfio" + + def test_nested_directory_structure(self): + """Test with nested directory structure.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + # Create nested structure: parent/results/rbdfio + results_dir = parent_dir / "results" + results_dir.mkdir() + rbdfio_dir = results_dir / "rbdfio" + rbdfio_dir.mkdir() + + # Pass results_dir as the directory + result = get_run_result_from_directory_name(results_dir, "json_output") + + assert isinstance(result, RBDFIO) + + def test_directory_path_object(self): + """Test that function works with Path objects.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rbdfio_dir = parent_dir / "rbdfio" + rbdfio_dir.mkdir() + + # Explicitly pass as Path object + result = get_run_result_from_directory_name(Path(parent_dir), "json_output") + + assert isinstance(result, RBDFIO) + + def test_empty_directory(self): + """Test with empty directory (no subdirectories).""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + + with pytest.raises(NotImplementedError): + get_run_result_from_directory_name(parent_dir, "json_output") + + def test_priority_when_multiple_types_match(self): + """Test behavior when directory name could match multiple types.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + # Create directory that contains multiple benchmark type names + # This tests the iteration order in BENCHMARK_TYPE_MAP + mixed_dir = parent_dir / "rbdfio_and_librbdfio" + mixed_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + # Should match one of them (whichever is found first in iteration) + assert isinstance(result, RBDFIO) + + +class TestFactoryIntegration: + """Integration tests for the factory function.""" + + def test_factory_creates_functional_object(self): + """Test that factory creates a functional RunResult object.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rbdfio_dir = parent_dir / "rbdfio" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + # Verify the object has expected methods + assert hasattr(result, "process") + assert hasattr(result, "get") + assert callable(result.process) + assert callable(result.get) + + def test_factory_preserves_directory_path(self): + """Test that factory preserves the directory path in the result object.""" + with tempfile.TemporaryDirectory() as tmpdir: + parent_dir = Path(tmpdir) + rbdfio_dir = parent_dir / "rbdfio" + rbdfio_dir.mkdir() + + result = get_run_result_from_directory_name(parent_dir, "json_output") + + # The result should store the parent directory + assert result._path == parent_dir + + +# Made with Bob diff --git a/tests/test_simple_plotter.py b/tests/test_simple_plotter.py index ef25ea5d..871ac427 100644 --- a/tests/test_simple_plotter.py +++ b/tests/test_simple_plotter.py @@ -34,7 +34,9 @@ def test_initialization(self) -> None: """Test SimplePlotter initialization""" plotter = SimplePlotter(archive_directory=str(self.archive_dir), plot_error_bars=True, plot_resources=False) - self.assertEqual(plotter._path, self.vis_dir) + # Check that archive directory and SVG output path are set correctly + self.assertEqual(plotter._archive_directory, self.archive_dir) + self.assertEqual(plotter._svg_output_path, self.vis_dir) self.assertTrue(plotter._plot_error_bars) self.assertFalse(plotter._plot_resources) @@ -46,13 +48,15 @@ def test_initialization_with_different_options(self) -> None: self.assertTrue(plotter._plot_resources) def test_generate_output_file_name(self) -> None: - """Test generating output file name""" + """Test generating output file name - SVGs are saved to top-level visualisation directory""" plotter = SimplePlotter(archive_directory=str(self.archive_dir), plot_error_bars=True, plot_resources=False) input_file = Path("/path/to/4096_read.json") output_name = plotter._generate_output_file_name([input_file]) - self.assertEqual(output_name, "/path/to/4096_read.svg") + # SVG files are now saved to archive_directory/visualisation/ regardless of JSON location + expected_output = str(self.archive_dir / "visualisation" / "4096_read.svg") + self.assertEqual(output_name, expected_output) @patch("post_processing.plotter.simple_plotter.read_intermediate_file") @patch("matplotlib.pyplot.subplots") diff --git a/tests/test_simple_report_generator.py b/tests/test_simple_report_generator.py index 865466a4..0f98a3b9 100644 --- a/tests/test_simple_report_generator.py +++ b/tests/test_simple_report_generator.py @@ -7,6 +7,7 @@ # We are OK to ignore private use in unit tests as the whole point of the tests # is to validate the functions contained in the module +import json import shutil import tempfile import unittest @@ -23,12 +24,16 @@ def setUp(self) -> None: """Set up test fixtures""" self.temp_dir = tempfile.mkdtemp() self.archive_dir = Path(self.temp_dir) / "test_archive" - self.vis_dir = self.archive_dir / "visualisation" + + # Use new nested structure: operation/visualisation/ + self.vis_dir = self.archive_dir / "read" / "visualisation" self.vis_dir.mkdir(parents=True) - # Create test data files + # Create test data files with actual JSON content # Format: {blocksize}_{numjobs}_{operation}.json - (self.vis_dir / "4096_1_read.json").touch() + test_data = {"data": [{"x": 1, "y": 100}]} + with open(self.vis_dir / "4096_1_read.json", "w") as f: + json.dump(test_data, f) def tearDown(self) -> None: """Clean up test fixtures""" @@ -103,5 +108,70 @@ def test_find_and_sort_file_paths(self) -> None: # Should be sorted by blocksize self.assertTrue(str(paths[0]).endswith("4096_1_read.json")) + def test_new_structure_with_operation_directories(self) -> None: + """Test report generation with new directory structure (operation/visualisation)""" + # Clean up legacy structure + shutil.rmtree(self.vis_dir, ignore_errors=True) + + # Create new structure: archive/operation/visualisation/ + randread_vis = self.archive_dir / "randread" / "visualisation" + randread_vis.mkdir(parents=True) + test_data = {"data": [{"x": 1, "y": 100}]} + with open(randread_vis / "4096_1_randread.json", "w") as f: + json.dump(test_data, f) + + randwrite_vis = self.archive_dir / "randwrite" / "visualisation" + randwrite_vis.mkdir(parents=True) + with open(randwrite_vis / "8192_1_randwrite.json", "w") as f: + json.dump(test_data, f) + + output_dir = f"{self.temp_dir}/output" + + generator = SimpleReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + # With new nested structure, _data_directories contains one entry per operation + # So with 2 operations (randread, randwrite), we have 2 data directories + self.assertEqual(len(generator._data_directories), 2) + self.assertIn(randread_vis, generator._data_directories) + self.assertIn(randwrite_vis, generator._data_directories) + + @patch("post_processing.reports.simple_report_generator.SimplePlotter") + def test_new_structure_creates_plots_per_operation(self, mock_plotter_class: MagicMock) -> None: + """Test that plots are created for new structure with multiple operations""" + # Clean up legacy structure + shutil.rmtree(self.vis_dir, ignore_errors=True) + + # Create new structure with multiple operations + randread_vis = self.archive_dir / "randread" / "visualisation" + randread_vis.mkdir(parents=True) + test_data = {"data": [{"x": 1, "y": 100}]} + with open(randread_vis / "4096_1_randread.json", "w") as f: + json.dump(test_data, f) + + randwrite_vis = self.archive_dir / "randwrite" / "visualisation" + randwrite_vis.mkdir(parents=True) + with open(randwrite_vis / "8192_1_randwrite.json", "w") as f: + json.dump(test_data, f) + + output_dir = f"{self.temp_dir}/output" + + mock_plotter = MagicMock() + mock_plotter_class.return_value = mock_plotter + + generator = SimpleReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + generator._copy_images() + + # With new nested structure, SimplePlotter is called once per operation directory + # So with 2 operations (randread, randwrite), it should be called twice + self.assertEqual(mock_plotter_class.call_count, 2) + self.assertEqual(mock_plotter.draw_and_save.call_count, 2) + # Made with Bob diff --git a/tests/test_time_series_bandwidth_plotter.py b/tests/test_time_series_bandwidth_plotter.py new file mode 100644 index 00000000..abf51bce --- /dev/null +++ b/tests/test_time_series_bandwidth_plotter.py @@ -0,0 +1,97 @@ +""" +Unit tests for the post_processing/plotter time_series_bandwidth_plotter module class +""" + +# pyright: strict, reportPrivateUsage=false +# pylint: disable=protected-access +# +# We are OK to ignore private use in unit tests as the whole point of the tests +# is to validate the functions contained in the module + +import unittest +from unittest.mock import MagicMock + +from matplotlib.axes import Axes + +from post_processing.plotter.time_series_bandwidth_plotter import ( + BANDWIDTH_PLOT_DEFAULT_COLOUR, + BANDWIDTH_PLOT_LABEL, + BANDWIDTH_Y_LABEL, + BYTES_TO_MB_DIVISOR, + TimeSeriesBandwidthPlotter, +) + + +class TestTimeSeriesBandwidthPlotter(unittest.TestCase): + """Test cases for TimeSeriesBandwidthPlotter class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.mock_axes = MagicMock(spec=Axes) + self.plotter = TimeSeriesBandwidthPlotter(self.mock_axes) + + def test_initialization(self) -> None: + """Test TimeSeriesBandwidthPlotter initialization""" + self.assertEqual(self.plotter._main_axes, self.mock_axes) + self.assertEqual(self.plotter._y_data, []) + + def test_add_y_data_converts_bytes_to_mb(self) -> None: + """Test that add_y_data converts bytes to MB/s""" + # 1048576 bytes = 1 MB + self.plotter.add_y_data("1048576") + self.assertAlmostEqual(self.plotter._y_data[0], 1.0) + + # 2097152 bytes = 2 MB + self.plotter.add_y_data("2097152") + self.assertAlmostEqual(self.plotter._y_data[1], 2.0) + + # 5242880 bytes = 5 MB + self.plotter.add_y_data("5242880") + self.assertAlmostEqual(self.plotter._y_data[2], 5.0) + + def test_plot_with_default_colour(self) -> None: + """Test plotting bandwidth data with default colour""" + self.plotter.add_y_data("1048576") # 1 MB + self.plotter.add_y_data("2097152") # 2 MB + + x_data = [1.0, 2.0] + self.plotter.plot(x_data) + + # Should set label and y_label + self.assertEqual(self.plotter._label, BANDWIDTH_PLOT_LABEL) + self.assertEqual(self.plotter._y_label, BANDWIDTH_Y_LABEL) + + # Should call plot on main axes with default colour + self.mock_axes.set_ylabel.assert_called_once_with(BANDWIDTH_Y_LABEL) + self.mock_axes.plot.assert_called_once() + + # Verify plot was called with correct parameters + call_args = self.mock_axes.plot.call_args + self.assertEqual(list(call_args[0][0]), x_data) + self.assertEqual(list(call_args[0][1]), [1.0, 2.0]) + self.assertEqual(call_args[1]["color"], BANDWIDTH_PLOT_DEFAULT_COLOUR) + self.assertEqual(call_args[1]["linewidth"], 1.5) + self.assertEqual(call_args[1]["alpha"], 0.8) + self.assertEqual(call_args[1]["label"], BANDWIDTH_PLOT_LABEL) + + def test_plot_with_custom_colour(self) -> None: + """Test plotting bandwidth data with custom colour""" + self.plotter.add_y_data("1048576") + + x_data = [1.0] + custom_colour = "#00FF00" + self.plotter.plot(x_data, colour=custom_colour) + + # Verify plot was called with custom colour + call_args = self.mock_axes.plot.call_args + self.assertEqual(call_args[1]["color"], custom_colour) + + def test_bandwidth_constants(self) -> None: + """Test bandwidth plotter constants""" + self.assertEqual(BANDWIDTH_PLOT_DEFAULT_COLOUR, "xkcd:purple") + self.assertEqual(BANDWIDTH_Y_LABEL, "Bandwidth (MB/s)") + self.assertEqual(BANDWIDTH_PLOT_LABEL, "Bandwidth") + self.assertEqual(BYTES_TO_MB_DIVISOR, 1024 * 1024) + + +# Made with Bob diff --git a/tests/test_time_series_formatter.py b/tests/test_time_series_formatter.py new file mode 100644 index 00000000..8f5ffc70 --- /dev/null +++ b/tests/test_time_series_formatter.py @@ -0,0 +1,388 @@ +""" +Unit tests for the time-series parser module. +""" + +# pyright: strict, reportPrivateUsage=false + +import unittest + +import pandas as pd + +from post_processing.parsers.fio_time_series_parser import FIOTimeSeriesParser + + +class TestFIOTimeSeriesParser(unittest.TestCase): + """Test cases for FIOTimeSeriesParser class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.formatter = FIOTimeSeriesParser( + archive_directory="/tmp/test", benchmark="fio", operation="randread", blocksize="4k", numjobs="1" + ) + + def test_format_with_all_metrics(self) -> None: + """Test formatting with all metrics present""" + # Create sample DataFrames + iops_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "iops": [1000.0, 1100.0, 1050.0]}) + + bandwidth_df = pd.DataFrame( + { + "timestamp_sec": [1.0, 2.0, 3.0], + "bandwidth_bytes": [4096000.0, 4505600.0, 4300800.0], + } + ) + + mean_latency_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [2.5, 2.3, 2.4]}) + + max_latency_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [10.0, 9.5, 9.8]}) + + p50_latency_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [2.0, 1.9, 2.1]}) + + p95_latency_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [5.0, 4.8, 4.9]}) + + p99_latency_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "latency_ms": [8.0, 7.5, 7.8]}) + + dataframes = { + "iops": iops_df, + "bandwidth": bandwidth_df, + "mean_latency": mean_latency_df, + "max_latency": max_latency_df, + "p50_latency": p50_latency_df, + "p95_latency": p95_latency_df, + "p99_latency": p99_latency_df, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=2, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None # Type narrowing + + # Check metadata + self.assertEqual(result["benchmark"], "fio") + self.assertEqual(result["operation"], "randread") + self.assertEqual(result["blocksize"], "4k") + self.assertEqual(result["numjobs"], "1") + self.assertEqual(result["metadata"]["num_volumes"], 2) + self.assertEqual(result["metadata"]["log_avg_msec"], 1000) + self.assertAlmostEqual(result["metadata"]["duration_seconds"], 2.0, places=1) + + # Check timeseries data + self.assertEqual(len(result["timeseries"]), 3) + + # Check first data point + point = result["timeseries"][0] + self.assertAlmostEqual(point["timestamp_sec"], 1.0, places=1) + self.assertAlmostEqual(point["iops"], 1000.0, places=1) + self.assertAlmostEqual(point["bandwidth_bytes"], 4096000.0, places=1) + self.assertAlmostEqual(point["mean_latency_ms"], 2.5, places=2) + self.assertAlmostEqual(point["max_latency_ms"], 10.0, places=2) + self.assertAlmostEqual(point["p50_latency_ms"], 2.0, places=2) + self.assertAlmostEqual(point["p95_latency_ms"], 5.0, places=2) + self.assertAlmostEqual(point["p99_latency_ms"], 8.0, places=2) + + def test_format_with_only_iops(self) -> None: + """Test formatting with only IOPS data""" + iops_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0], "iops": [1000.0, 1100.0]}) + + dataframes = { + "iops": iops_df, + "bandwidth": None, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None + + self.assertEqual(len(result["timeseries"]), 2) + + # Check that missing metrics are filled with 0 + point = result["timeseries"][0] + self.assertAlmostEqual(point["iops"], 1000.0, places=1) + self.assertAlmostEqual(point["bandwidth_bytes"], 0.0, places=1) + self.assertAlmostEqual(point["mean_latency_ms"], 0.0, places=2) + + def test_format_with_only_latency(self) -> None: + """ + Test formatting with only latency data. + + When there are no throughput metrics at all, the data is returned as-is + with IOPS/bandwidth filled as 0. This is an edge case that shouldn't + occur in normal FIO runs (which always produce IOPS/bandwidth logs). + """ + mean_latency_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0], "latency_ms": [2.5, 2.3]}) + + dataframes = { + "iops": None, + "bandwidth": None, + "mean_latency": mean_latency_df, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None + + self.assertEqual(len(result["timeseries"]), 2) + + # Check that IOPS/BW are filled with 0 (edge case behavior) + point = result["timeseries"][0] + self.assertAlmostEqual(point["iops"], 0.0, places=1) + self.assertAlmostEqual(point["bandwidth_bytes"], 0.0, places=1) + self.assertAlmostEqual(point["mean_latency_ms"], 2.5, places=2) + + def test_format_with_misaligned_timestamps(self) -> None: + """ + Test formatting with different timestamps in different metrics. + + With the zero-throughput filtering, timestamps that have zero values + for ALL throughput metrics (IOPS and bandwidth) are removed. + """ + iops_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0], "iops": [1000.0, 1100.0, 1050.0]}) + + # Bandwidth has different timestamps + bandwidth_df = pd.DataFrame( + {"timestamp_sec": [1.5, 2.5, 3.5], "bandwidth_bytes": [4096000.0, 4505600.0, 4300800.0]} + ) + + dataframes = { + "iops": iops_df, + "bandwidth": bandwidth_df, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None + + # Should have 6 timestamps (all have at least one throughput metric) + self.assertEqual(len(result["timeseries"]), 6) + + # Verify no timestamps have BOTH IOPS and bandwidth as zero + for point in result["timeseries"]: + has_throughput = point["iops"] > 0 or point["bandwidth_bytes"] > 0 + self.assertTrue(has_throughput, f"Timestamp {point['timestamp_sec']} has zero throughput") + + # Check specific points + # First point should have IOPS but no bandwidth + point = result["timeseries"][0] + self.assertAlmostEqual(point["timestamp_sec"], 1.0, places=1) + self.assertAlmostEqual(point["iops"], 1000.0, places=1) + self.assertAlmostEqual(point["bandwidth_bytes"], 0.0, places=1) + + def test_format_filters_zero_throughput_timestamps(self) -> None: + """ + Test that timestamps with zero throughput are filtered out. + + This test simulates the real-world scenario where IOPS/bandwidth logs + and latency logs have different timestamps due to FIO's logging behavior. + The parser should only keep timestamps that have actual throughput data. + """ + # IOPS data at specific timestamps (simulating _iops.log) + iops_df = pd.DataFrame({"timestamp_sec": [0.999, 1.999, 2.999, 3.999], "iops": [670.0, 675.0, 680.0, 549.0]}) + + # Latency data at different timestamps (simulating _clat.log) + mean_latency_df = pd.DataFrame( + {"timestamp_sec": [1.009, 2.001, 3.000, 4.000], "latency_ms": [23.97, 23.22, 23.54, 28.04]} + ) + + dataframes = { + "iops": iops_df, + "bandwidth": None, + "mean_latency": mean_latency_df, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None + + # Should only have 4 timestamps (from IOPS data) + # Timestamps from latency-only data should be filtered out + self.assertEqual(len(result["timeseries"]), 4) + + # Verify no zero IOPS values exist + for point in result["timeseries"]: + self.assertGreater(point["iops"], 0.0, f"Found zero IOPS at timestamp {point['timestamp_sec']}") + + # Verify we have the IOPS timestamps + timestamps = [point["timestamp_sec"] for point in result["timeseries"]] + self.assertIn(0.999, timestamps) + self.assertIn(1.999, timestamps) + self.assertIn(2.999, timestamps) + self.assertIn(3.999, timestamps) + + # Verify latency-only timestamps are NOT present + self.assertNotIn(1.009, timestamps) + self.assertNotIn(2.001, timestamps) + + def test_format_with_empty_dataframes(self) -> None: + """Test formatting with empty DataFrames""" + empty_df = pd.DataFrame({"timestamp_sec": [], "iops": []}) + + dataframes = { + "iops": empty_df, + "bandwidth": None, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + # Should return None for empty data + self.assertIsNone(result) + + def test_format_with_all_none(self) -> None: + """Test formatting with all None DataFrames""" + dataframes = { + "iops": None, + "bandwidth": None, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + # Should return None when no data + self.assertIsNone(result) + + def test_metadata_calculation(self) -> None: + """Test metadata calculation with specific timestamps""" + iops_df = pd.DataFrame({"timestamp_sec": [10.0, 20.0, 30.0], "iops": [1000.0, 1100.0, 1050.0]}) + + dataframes = { + "iops": iops_df, + "bandwidth": None, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=3, + log_avg_msec=2000, + ) + + self.assertIsNotNone(result) + assert result is not None + + metadata = result["metadata"] + self.assertAlmostEqual(metadata["start_time_epoch"], 10.0, places=1) + self.assertAlmostEqual(metadata["end_time_epoch"], 30.0, places=1) + self.assertAlmostEqual(metadata["duration_seconds"], 20.0, places=1) + self.assertEqual(metadata["num_volumes"], 3) + self.assertEqual(metadata["sampling_interval_ms"], 2000) + + def test_custom_benchmark_info(self) -> None: + """Test parser with custom benchmark information""" + formatter = FIOTimeSeriesParser( + archive_directory="/tmp/test", benchmark="custom_bench", operation="seqwrite", blocksize="128k", numjobs="4" + ) + + iops_df = pd.DataFrame({"timestamp_sec": [1.0, 2.0], "iops": [500.0, 550.0]}) + + dataframes = { + "iops": iops_df, + "bandwidth": None, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None + + self.assertEqual(result["benchmark"], "custom_bench") + self.assertEqual(result["operation"], "seqwrite") + self.assertEqual(result["blocksize"], "128k") + self.assertEqual(result["numjobs"], "4") + + def test_timeseries_sorted_by_timestamp(self) -> None: + """Test that timeseries data is sorted by timestamp""" + # Create unsorted data + iops_df = pd.DataFrame({"timestamp_sec": [3.0, 1.0, 2.0], "iops": [1050.0, 1000.0, 1100.0]}) + + dataframes = { + "iops": iops_df, + "bandwidth": None, + "mean_latency": None, + "max_latency": None, + "p50_latency": None, + "p95_latency": None, + "p99_latency": None, + } + result = self.formatter._format_time_series( + dataframes=dataframes, + num_volumes=1, + log_avg_msec=1000, + ) + + self.assertIsNotNone(result) + assert result is not None + + # Check that timestamps are sorted + timestamps = [point["timestamp_sec"] for point in result["timeseries"]] + self.assertEqual(timestamps, sorted(timestamps)) + + # Check that values match sorted order + self.assertAlmostEqual(result["timeseries"][0]["iops"], 1000.0, places=1) + self.assertAlmostEqual(result["timeseries"][1]["iops"], 1100.0, places=1) + self.assertAlmostEqual(result["timeseries"][2]["iops"], 1050.0, places=1) + + +# Made with Bob diff --git a/tests/test_time_series_iops_plotter.py b/tests/test_time_series_iops_plotter.py new file mode 100644 index 00000000..0706aef3 --- /dev/null +++ b/tests/test_time_series_iops_plotter.py @@ -0,0 +1,90 @@ +""" +Unit tests for the post_processing/plotter time_series_iops_plotter module class +""" + +# pyright: strict, reportPrivateUsage=false +# pylint: disable=protected-access +# +# We are OK to ignore private use in unit tests as the whole point of the tests +# is to validate the functions contained in the module + +import unittest +from unittest.mock import MagicMock + +from matplotlib.axes import Axes + +from post_processing.plotter.time_series_iops_plotter import ( + IOPS_PLOT_DEFAULT_COLOUR, + IOPS_PLOT_LABEL, + IOPS_Y_LABEL, + TimeSeriesIOPSPlotter, +) + + +class TestTimeSeriesIOPSPlotter(unittest.TestCase): + """Test cases for TimeSeriesIOPSPlotter class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.mock_axes = MagicMock(spec=Axes) + self.plotter = TimeSeriesIOPSPlotter(self.mock_axes) + + def test_initialization(self) -> None: + """Test TimeSeriesIOPSPlotter initialization""" + self.assertEqual(self.plotter._main_axes, self.mock_axes) + self.assertEqual(self.plotter._y_data, []) + + def test_add_y_data(self) -> None: + """Test adding IOPS data""" + self.plotter.add_y_data("1000.5") + self.plotter.add_y_data("2500.75") + + self.assertEqual(len(self.plotter._y_data), 2) + self.assertAlmostEqual(self.plotter._y_data[0], 1000.5) + self.assertAlmostEqual(self.plotter._y_data[1], 2500.75) + + def test_plot_with_default_colour(self) -> None: + """Test plotting IOPS data with default colour""" + self.plotter.add_y_data("1000") + self.plotter.add_y_data("2000") + + x_data = [1.0, 2.0] + self.plotter.plot(x_data) + + # Should set label and y_label + self.assertEqual(self.plotter._label, IOPS_PLOT_LABEL) + self.assertEqual(self.plotter._y_label, IOPS_Y_LABEL) + + # Should call plot on main axes with default colour + self.mock_axes.set_ylabel.assert_called_once_with(IOPS_Y_LABEL) + self.mock_axes.plot.assert_called_once() + + # Verify plot was called with correct parameters + call_args = self.mock_axes.plot.call_args + self.assertEqual(list(call_args[0][0]), x_data) + self.assertEqual(list(call_args[0][1]), [1000.0, 2000.0]) + self.assertEqual(call_args[1]["color"], IOPS_PLOT_DEFAULT_COLOUR) + self.assertEqual(call_args[1]["linewidth"], 1.5) + self.assertEqual(call_args[1]["alpha"], 0.8) + self.assertEqual(call_args[1]["label"], IOPS_PLOT_LABEL) + + def test_plot_with_custom_colour(self) -> None: + """Test plotting IOPS data with custom colour""" + self.plotter.add_y_data("1000") + + x_data = [1.0] + custom_colour = "#FF0000" + self.plotter.plot(x_data, colour=custom_colour) + + # Verify plot was called with custom colour + call_args = self.mock_axes.plot.call_args + self.assertEqual(call_args[1]["color"], custom_colour) + + def test_iops_constants(self) -> None: + """Test IOPS plotter constants""" + self.assertEqual(IOPS_PLOT_DEFAULT_COLOUR, "xkcd:blue") + self.assertEqual(IOPS_Y_LABEL, "IOPS (ops/s)") + self.assertEqual(IOPS_PLOT_LABEL, "IOPS") + + +# Made with Bob diff --git a/tests/test_time_series_latency_plotter.py b/tests/test_time_series_latency_plotter.py new file mode 100644 index 00000000..4966694f --- /dev/null +++ b/tests/test_time_series_latency_plotter.py @@ -0,0 +1,158 @@ +""" +Unit tests for the post_processing/plotter time_series_latency_plotter module class +""" + +# pyright: strict, reportPrivateUsage=false +# pylint: disable=protected-access +# +# We are OK to ignore private use in unit tests as the whole point of the tests +# is to validate the functions contained in the module + +import unittest +from unittest.mock import MagicMock + +from matplotlib.axes import Axes + +from post_processing.plotter.time_series_latency_plotter import ( + LATENCY_MEAN_COLOR, + LATENCY_P50_COLOR, + LATENCY_P95_COLOR, + LATENCY_P99_COLOR, + LATENCY_PLOT_LABEL, + LATENCY_Y_LABEL, + TimeSeriesLatencyPlotter, +) + + +class TestTimeSeriesLatencyPlotter(unittest.TestCase): + """Test cases for TimeSeriesLatencyPlotter class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.mock_axes = MagicMock(spec=Axes) + self.plotter = TimeSeriesLatencyPlotter(self.mock_axes) + + def test_initialization(self) -> None: + """Test TimeSeriesLatencyPlotter initialization""" + self.assertEqual(self.plotter._main_axes, self.mock_axes) + self.assertEqual(self.plotter._y_data, []) + self.assertEqual(self.plotter._p50_data, []) + self.assertEqual(self.plotter._p95_data, []) + self.assertEqual(self.plotter._p99_data, []) + self.assertEqual(self.plotter._max_data, []) + + def test_add_y_data(self) -> None: + """Test adding mean latency data""" + self.plotter.add_y_data("5.5") + self.plotter.add_y_data("10.3") + + self.assertEqual(len(self.plotter._y_data), 2) + self.assertAlmostEqual(self.plotter._y_data[0], 5.5) + self.assertAlmostEqual(self.plotter._y_data[1], 10.3) + + def test_add_p50_data(self) -> None: + """Test adding P50 latency data""" + self.plotter.add_p50_data("3.0") + self.plotter.add_p50_data("4.5") + + self.assertEqual(len(self.plotter._p50_data), 2) + self.assertAlmostEqual(self.plotter._p50_data[0], 3.0) + self.assertAlmostEqual(self.plotter._p50_data[1], 4.5) + + def test_add_p95_data(self) -> None: + """Test adding P95 latency data""" + self.plotter.add_p95_data("8.0") + self.plotter.add_p95_data("12.5") + + self.assertEqual(len(self.plotter._p95_data), 2) + self.assertAlmostEqual(self.plotter._p95_data[0], 8.0) + self.assertAlmostEqual(self.plotter._p95_data[1], 12.5) + + def test_add_p99_data(self) -> None: + """Test adding P99 latency data""" + self.plotter.add_p99_data("15.0") + self.plotter.add_p99_data("20.5") + + self.assertEqual(len(self.plotter._p99_data), 2) + self.assertAlmostEqual(self.plotter._p99_data[0], 15.0) + self.assertAlmostEqual(self.plotter._p99_data[1], 20.5) + + def test_add_max_data(self) -> None: + """Test adding max latency data""" + self.plotter.add_max_data("25.0") + self.plotter.add_max_data("30.5") + + self.assertEqual(len(self.plotter._max_data), 2) + self.assertAlmostEqual(self.plotter._max_data[0], 25.0) + self.assertAlmostEqual(self.plotter._max_data[1], 30.5) + + def test_plot_with_all_data(self) -> None: + """Test plotting latency data with all percentiles""" + # Add data for all metrics + self.plotter.add_y_data("5.0") + self.plotter.add_y_data("6.0") + self.plotter.add_p50_data("3.0") + self.plotter.add_p50_data("4.0") + self.plotter.add_p95_data("8.0") + self.plotter.add_p95_data("9.0") + self.plotter.add_p99_data("12.0") + self.plotter.add_p99_data("13.0") + self.plotter.add_max_data("20.0") + self.plotter.add_max_data("21.0") + + x_data = [1.0, 2.0] + self.plotter.plot(x_data) + + # Should set label and y_label + self.assertEqual(self.plotter._label, LATENCY_PLOT_LABEL) + self.assertEqual(self.plotter._y_label, LATENCY_Y_LABEL) + + # Should call set_ylabel + self.mock_axes.set_ylabel.assert_called_once_with(LATENCY_Y_LABEL) + + # Should call fill_between for percentile bands (2 times: P50-P95, P95-P99) + self.assertEqual(self.mock_axes.fill_between.call_count, 2) + + # Should call plot for percentile lines and mean (5 times: P50, P95, P99, mean, max) + self.assertEqual(self.mock_axes.plot.call_count, 5) + + def test_plot_with_minimal_data(self) -> None: + """Test plotting with only mean latency data""" + self.plotter.add_y_data("5.0") + self.plotter.add_y_data("6.0") + + x_data = [1.0, 2.0] + self.plotter.plot(x_data) + + # Should still set labels + self.assertEqual(self.plotter._label, LATENCY_PLOT_LABEL) + self.assertEqual(self.plotter._y_label, LATENCY_Y_LABEL) + + # Should not call fill_between (no percentile data) + self.mock_axes.fill_between.assert_not_called() + + # Should call plot once for mean latency + self.assertEqual(self.mock_axes.plot.call_count, 1) + + def test_plot_skips_max_when_zero(self) -> None: + """Test that max latency is not plotted when all values are zero""" + self.plotter.add_y_data("5.0") + self.plotter.add_max_data("0.0") + + x_data = [1.0] + self.plotter.plot(x_data) + + # Should only plot mean (not max since it's zero) + self.assertEqual(self.mock_axes.plot.call_count, 1) + + def test_latency_constants(self) -> None: + """Test latency plotter constants""" + self.assertEqual(LATENCY_MEAN_COLOR, "xkcd:orange") + self.assertEqual(LATENCY_P50_COLOR, "xkcd:green") + self.assertEqual(LATENCY_P95_COLOR, "xkcd:red") + self.assertEqual(LATENCY_P99_COLOR, "xkcd:dark red") + self.assertEqual(LATENCY_Y_LABEL, "Latency (ms)") + self.assertEqual(LATENCY_PLOT_LABEL, "Mean Latency") + + +# Made with Bob diff --git a/tests/test_time_series_output_formatter.py b/tests/test_time_series_output_formatter.py new file mode 100644 index 00000000..0ec9c405 --- /dev/null +++ b/tests/test_time_series_output_formatter.py @@ -0,0 +1,248 @@ +""" +Unit tests for the TimeSeriesOutputFormatter class. +""" + +# pyright: strict, reportPrivateUsage=false + +import unittest +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch + +from post_processing.formatter.time_series_output_formatter import ( + TimeSeriesOutputFormatter, +) + + +class TestTimeSeriesOutputFormatterInitialization(unittest.TestCase): + """Test TimeSeriesOutputFormatter initialization""" + + def test_initialization_with_defaults(self) -> None: + """Test initialization with default filename root""" + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + + self.assertEqual(formatter._directory, "/tmp/test") + self.assertEqual(formatter._filename_root, "json_output") + self.assertEqual(formatter._timeseries_data, {}) + self.assertEqual(formatter._all_test_run_ids, set()) + self.assertEqual(formatter._benchmark_types, {}) + + def test_initialization_with_custom_filename_root(self) -> None: + """Test initialization with custom filename root""" + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test", filename_root="custom_output") + + self.assertEqual(formatter._filename_root, "custom_output") + + +class TestGetTestrunDirectories(unittest.TestCase): + """Test _get_testrun_directories method""" + + def test_with_id_in_path(self) -> None: + """Test when 'id-' is in the archive directory path""" + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test/id-12345") + + directories = formatter._get_testrun_directories("id-12345") + + self.assertEqual(len(directories), 1) + self.assertEqual(directories[0], Path("/tmp/test/id-12345")) + + @patch("pathlib.Path.glob") + def test_without_id_in_path(self, mock_glob: Mock) -> None: + """Test when 'id-' is not in the archive directory path""" + mock_glob.return_value = [Path("/tmp/test/results/id-12345")] + + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + directories = formatter._get_testrun_directories("id-12345") + + mock_glob.assert_called_once_with("**/id-12345") + self.assertEqual(len(directories), 1) + + +class TestFindAllTestrunIds(unittest.TestCase): + """Test _find_all_testrun_ids method""" + + def test_find_ids_with_id_prefix(self) -> None: + """Test finding test run IDs with 'id-' prefix""" + file_list = [ + Path("/tmp/test/results/id-12345/rbdfio/json_output.0"), + Path("/tmp/test/results/id-12345/rbdfio/json_output.1"), + Path("/tmp/test/results/id-67890/rbdfio/json_output.0"), + ] + + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + formatter._find_all_testrun_ids(file_list) + + self.assertEqual(len(formatter._all_test_run_ids), 2) + self.assertIn("id-12345", formatter._all_test_run_ids) + self.assertIn("id-67890", formatter._all_test_run_ids) + + def test_find_ids_without_id_prefix(self) -> None: + """Test finding test run IDs without 'id-' prefix""" + file_list = [ + Path("/tmp/test/some_directory/workload1/json_output.0"), + Path("/tmp/test/some_directory/workload2/json_output.0"), + ] + + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + formatter._find_all_testrun_ids(file_list) + + # When no 'id-' prefix, uses directory directly above the file + self.assertEqual(len(formatter._all_test_run_ids), 2) + self.assertIn("workload1", formatter._all_test_run_ids) + self.assertIn("workload2", formatter._all_test_run_ids) + + +class TestProcessCompatibilityMode(unittest.TestCase): + """Test _process_compatibility_mode method""" + + @patch("post_processing.formatter.time_series_output_formatter.get_run_result_from_directory_name") + def test_process_compatibility_mode(self, mock_factory: Mock) -> None: + """Test processing in compatibility mode""" + # Create mock RunResult + mock_result = MagicMock() + mock_result.type = "rbdfio" + # With new memory-efficient approach, timeseries data is written during process() + # and get_timeseries() returns empty dict + mock_result.get_timeseries.return_value = {} + mock_factory.return_value = mock_result + + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + benchmark_type = formatter._process_compatibility_mode() + + # Verify factory was called with include_timeseries=True + mock_factory.assert_called_once_with(Path("/tmp/test"), "json_output", include_timeseries=True) + mock_result.process.assert_called_once() + # get_timeseries() is no longer called in the new memory-efficient approach + + self.assertEqual(benchmark_type, "rbdfio") + # With new approach, timeseries_data should be empty (written during process()) + self.assertEqual(len(formatter._timeseries_data), 0) + + +class TestProcessSingleTestrun(unittest.TestCase): + """Test _process_single_testrun method""" + + @patch("post_processing.formatter.time_series_output_formatter.get_run_result_from_directory_name") + @patch("pathlib.Path.iterdir") + def test_process_single_testrun(self, mock_iterdir: Mock, mock_factory: Mock) -> None: + """Test processing a single test run""" + # Create mock directories + mock_dir1 = MagicMock() + mock_dir1.is_dir.return_value = True + mock_dir1.name = "randread_4k" + + mock_dir2 = MagicMock() + mock_dir2.is_dir.return_value = True + mock_dir2.name = "randwrite_4k" + + mock_iterdir.return_value = [mock_dir1, mock_dir2] + + # Create mock RunResult + mock_result = MagicMock() + mock_result.type = "rbdfio" + mock_result.get_timeseries.return_value = {"test_key": {"operation": "randread", "blocksize": "4k"}} + mock_factory.return_value = mock_result + + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + testrun_dir = Path("/tmp/test/id-12345") + + benchmark_type = formatter._process_single_testrun(testrun_dir) + + # Verify factory was called twice (once for each directory) + self.assertEqual(mock_factory.call_count, 2) + self.assertEqual(mock_result.process.call_count, 2) + self.assertEqual(benchmark_type, "rbdfio") + + +class TestProcess(unittest.TestCase): + """Test process method""" + + @patch.object(TimeSeriesOutputFormatter, "_process_single_testrun") + @patch.object(TimeSeriesOutputFormatter, "_get_testrun_directories") + @patch.object(TimeSeriesOutputFormatter, "_find_all_testrun_ids") + def test_process_with_test_runs( + self, + mock_find_ids: Mock, + mock_get_dirs: Mock, + mock_process_testrun: Mock, + ) -> None: + """Test processing with test runs found""" + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + + # Mock finding test run IDs + def set_test_run_ids(file_list: list[Path]) -> None: + formatter._all_test_run_ids = {"id-12345"} + + mock_find_ids.side_effect = set_test_run_ids + + # Mock getting directories + mock_get_dirs.return_value = [Path("/tmp/test/id-12345")] + + # Mock processing + mock_process_testrun.return_value = "rbdfio" + + formatter.process() + + mock_find_ids.assert_called_once() + mock_get_dirs.assert_called_once_with("id-12345") + mock_process_testrun.assert_called_once() + + @patch.object(TimeSeriesOutputFormatter, "_find_all_testrun_ids") + def test_process_with_no_test_runs(self, mock_find_ids: Mock) -> None: + """Test processing when no test runs are found""" + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + + # Mock finding no test run IDs + def set_empty_test_run_ids(file_list: list[Path]) -> None: + formatter._all_test_run_ids = set() + + mock_find_ids.side_effect = set_empty_test_run_ids + + formatter.process() + + mock_find_ids.assert_called_once() + # Should return early without processing + + +class TestIntegration(unittest.TestCase): + """Integration tests for TimeSeriesOutputFormatter""" + + @patch("post_processing.formatter.time_series_output_formatter.get_run_result_from_directory_name") + @patch("pathlib.Path.glob") + @patch("pathlib.Path.iterdir") + def test_full_workflow( + self, + mock_iterdir: Mock, + mock_glob: Mock, + mock_factory: Mock, + ) -> None: + """Test complete workflow with memory-efficient processing""" + # Setup mocks + mock_glob.return_value = [ + Path("/tmp/test/id-12345/rbdfio/json_output.0"), + ] + + mock_dir = MagicMock() + mock_dir.is_dir.return_value = True + mock_dir.name = "randread_4k" + mock_iterdir.return_value = [mock_dir] + + mock_result = MagicMock() + mock_result.type = "rbdfio" + # With new memory-efficient approach, timeseries data is written during process() + mock_factory.return_value = mock_result + + # Run workflow - data is written during process() + formatter = TimeSeriesOutputFormatter(archive_directory="/tmp/test") + formatter.process() + + # Verify + mock_factory.assert_called() + mock_result.process.assert_called() + # With new approach, timeseries_data should be empty (written during process()) + self.assertEqual(len(formatter._timeseries_data), 0) + + +if __name__ == "__main__": + unittest.main() + +# Made with Bob diff --git a/tests/test_time_series_plotter.py b/tests/test_time_series_plotter.py new file mode 100644 index 00000000..a5ce91f2 --- /dev/null +++ b/tests/test_time_series_plotter.py @@ -0,0 +1,536 @@ +""" +Unit tests for the time-series plotter module. +""" + +# pyright: strict, reportPrivateUsage=false +# pylint: disable=protected-access, unused-argument, too-many-public-methods + +import json +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import matplotlib.pyplot as plt + +from post_processing.plotter.time_series_plotter import TimeSeriesPlotter +from post_processing.post_processing_types import ( + TimeSeriesDataPoint, + TimeSeriesFormatType, + TimeSeriesMetadata, +) + + +class TestTimeSeriesPlotter(unittest.TestCase): + """Test cases for TimeSeriesPlotter class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.archive_dir = Path(self.temp_dir) / "archive" + self.archive_dir.mkdir(parents=True) + + # Create sample time-series data + self.sample_metadata: TimeSeriesMetadata = { + "start_time_epoch": 0.0, + "end_time_epoch": 10.0, + "duration_seconds": 10.0, + "num_volumes": 2, + "sampling_interval_ms": 1000, + "log_avg_msec": 1000, + } + + self.sample_timeseries: list[TimeSeriesDataPoint] = [ + { + "timestamp_sec": 1.0, + "iops": 1000.0, + "bandwidth_bytes": 4096000.0, + "mean_latency_ms": 2.5, + "max_latency_ms": 10.0, + "p50_latency_ms": 2.0, + "p95_latency_ms": 5.0, + "p99_latency_ms": 8.0, + "num_samples": 100, + }, + { + "timestamp_sec": 2.0, + "iops": 1100.0, + "bandwidth_bytes": 4505600.0, + "mean_latency_ms": 2.3, + "max_latency_ms": 9.5, + "p50_latency_ms": 1.9, + "p95_latency_ms": 4.8, + "p99_latency_ms": 7.5, + "num_samples": 110, + }, + { + "timestamp_sec": 3.0, + "iops": 1050.0, + "bandwidth_bytes": 4300800.0, + "mean_latency_ms": 2.4, + "max_latency_ms": 9.8, + "p50_latency_ms": 2.1, + "p95_latency_ms": 4.9, + "p99_latency_ms": 7.8, + "num_samples": 105, + }, + ] + + self.sample_data: TimeSeriesFormatType = { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4096", + "numjobs": "1", + "iodepth": "1", + "metadata": self.sample_metadata, + "timeseries": self.sample_timeseries, + "maximum_iops": "1100.0", + "maximum_bandwidth": "4.30", + "latency_at_max_iops": "2.3", + "latency_at_max_bandwidth": "2.4", + "timestamp_at_max_iops": "1001.0", + "timestamp_at_max_bandwidth": "1002.0", + "maximum_latency": "2.4", + "timestamp_at_max_latency": "1002.0", + "maximum_cpu_usage": "50.0", + "maximum_memory_usage": "1024.0", + } + + def tearDown(self) -> None: + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_initialization(self) -> None: + """Test TimeSeriesPlotter initialization""" + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + self.assertEqual(plotter._archive_directory, self.archive_dir) + self.assertEqual(plotter._plotter, plt) + self.assertEqual(plotter._figure_size, (12, 3)) + self.assertEqual(plotter._dpi, 100) + self.assertEqual(plotter._output_dir, self.archive_dir / "visualisation") + + def test_initialization_with_custom_params(self) -> None: + """Test TimeSeriesPlotter initialization with custom parameters""" + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt, figure_size=(10, 5), dpi=150) + + self.assertEqual(plotter._figure_size, (10, 5)) + self.assertEqual(plotter._dpi, 150) + + def test_generate_output_path(self) -> None: + """Test output path generation""" + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + output_path = plotter._generate_output_path(self.sample_data, "iops") + + expected_path = self.archive_dir / "visualisation" / "4096_1_randread_1_iops_timeseries.svg" + self.assertEqual(output_path, expected_path) + + def test_generate_output_path_different_metrics(self) -> None: + """Test output path generation for different metrics""" + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + iops_path = plotter._generate_output_path(self.sample_data, "iops") + bandwidth_path = plotter._generate_output_path(self.sample_data, "bandwidth") + latency_path = plotter._generate_output_path(self.sample_data, "latency") + + self.assertIn("1_iops_timeseries.svg", str(iops_path)) + self.assertIn("1_bandwidth_timeseries.svg", str(bandwidth_path)) + self.assertIn("1_latency_timeseries.svg", str(latency_path)) + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_iops(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test IOPS plotting""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_iops(self.sample_data) + + # Should create subplots + mock_subplots.assert_called_once() + + # Should plot data + mock_axes.plot.assert_called_once() + + # Should configure axes + mock_axes.set_title.assert_called_once() + mock_axes.set_xlabel.assert_called_once() + mock_axes.set_ylabel.assert_called_once() + mock_axes.legend.assert_called_once() + mock_axes.grid.assert_called_once() + + # Should save figure + mock_figure.savefig.assert_called_once() + + # Should close figure + mock_close.assert_called_once() + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_bandwidth(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test bandwidth plotting""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_bandwidth(self.sample_data) + + # Should create subplots + mock_subplots.assert_called_once() + + # Should plot data + mock_axes.plot.assert_called_once() + + # Should save figure + mock_figure.savefig.assert_called_once() + + # Should close figure + mock_close.assert_called_once() + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_latency(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test latency plotting with percentile bands""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_latency(self.sample_data) + + # Should create subplots + mock_subplots.assert_called_once() + + # Should plot multiple lines (mean, p50, p95, p99, max) + # and fill_between for bands + self.assertGreater(mock_axes.plot.call_count, 0) + self.assertGreater(mock_axes.fill_between.call_count, 0) + + # Should save figure + mock_figure.savefig.assert_called_once() + + # Should close figure + mock_close.assert_called_once() + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_time_series_creates_all_plots(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test that plot_time_series creates all three plot types""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter.plot_time_series(self.sample_data) + + # Should create 3 plots (IOPS, bandwidth, latency) + self.assertEqual(mock_subplots.call_count, 3) + self.assertEqual(mock_figure.savefig.call_count, 3) + self.assertEqual(mock_close.call_count, 3) + + def test_plot_time_series_with_empty_data(self) -> None: + """Test plotting with empty timeseries data""" + empty_data: TimeSeriesFormatType = { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4k", + "numjobs": "1", + "iodepth": "1", + "metadata": self.sample_metadata, + "timeseries": [], + "maximum_iops": "0.0", + "maximum_bandwidth": "0.0", + "latency_at_max_iops": "0.0", + "latency_at_max_bandwidth": "0.0", + "timestamp_at_max_iops": "0.0", + "timestamp_at_max_bandwidth": "0.0", + "maximum_latency": "0.0", + "timestamp_at_max_latency": "0.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + } + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + # Should not raise an error + plotter.plot_time_series(empty_data) + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_iops_with_zero_values(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test IOPS plotting with all zero values""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + zero_data: TimeSeriesFormatType = { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4k", + "numjobs": "1", + "iodepth": "1", + "metadata": self.sample_metadata, + "timeseries": [ + { + "timestamp_sec": 1.0, + "iops": 0.0, + "bandwidth_bytes": 0.0, + "mean_latency_ms": 0.0, + "max_latency_ms": 0.0, + "p50_latency_ms": 0.0, + "p95_latency_ms": 0.0, + "p99_latency_ms": 0.0, + "num_samples": 0, + } + ], + "maximum_iops": "0.0", + "maximum_bandwidth": "0.0", + "latency_at_max_iops": "0.0", + "latency_at_max_bandwidth": "0.0", + "timestamp_at_max_iops": "0.0", + "timestamp_at_max_bandwidth": "0.0", + "maximum_latency": "0.0", + "timestamp_at_max_latency": "0.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + } + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_iops(zero_data) + + # Should not create plot for zero data + mock_subplots.assert_not_called() + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_bandwidth_with_zero_values(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test bandwidth plotting with all zero values""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + zero_data: TimeSeriesFormatType = { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4k", + "numjobs": "1", + "iodepth": "1", + "metadata": self.sample_metadata, + "timeseries": [ + { + "timestamp_sec": 1.0, + "iops": 0.0, + "bandwidth_bytes": 0.0, + "mean_latency_ms": 0.0, + "max_latency_ms": 0.0, + "p50_latency_ms": 0.0, + "p95_latency_ms": 0.0, + "p99_latency_ms": 0.0, + "num_samples": 0, + } + ], + "maximum_iops": "0.0", + "maximum_bandwidth": "0.0", + "latency_at_max_iops": "0.0", + "latency_at_max_bandwidth": "0.0", + "timestamp_at_max_iops": "0.0", + "timestamp_at_max_bandwidth": "0.0", + "maximum_latency": "0.0", + "timestamp_at_max_latency": "0.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + } + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_bandwidth(zero_data) + + # Should not create plot for zero data + mock_subplots.assert_not_called() + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_plot_latency_with_zero_values(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test latency plotting with all zero values""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + zero_data: TimeSeriesFormatType = { + "benchmark": "fio", + "operation": "randread", + "blocksize": "4k", + "numjobs": "1", + "iodepth": "1", + "metadata": self.sample_metadata, + "timeseries": [ + { + "timestamp_sec": 1.0, + "iops": 0.0, + "bandwidth_bytes": 0.0, + "mean_latency_ms": 0.0, + "max_latency_ms": 0.0, + "p50_latency_ms": 0.0, + "p95_latency_ms": 0.0, + "p99_latency_ms": 0.0, + "num_samples": 0, + } + ], + "maximum_iops": "0.0", + "maximum_bandwidth": "0.0", + "latency_at_max_iops": "0.0", + "latency_at_max_bandwidth": "0.0", + "timestamp_at_max_iops": "0.0", + "timestamp_at_max_bandwidth": "0.0", + "maximum_latency": "0.0", + "timestamp_at_max_latency": "0.0", + "maximum_cpu_usage": "0.0", + "maximum_memory_usage": "0.0", + } + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_latency(zero_data) + + # Should not create plot for zero data + mock_subplots.assert_not_called() + + def test_plot_from_file(self) -> None: + """Test plotting from JSON file""" + # Create a JSON file with sample data + vis_dir = self.archive_dir / "visualisation" + vis_dir.mkdir(parents=True, exist_ok=True) + + json_file = vis_dir / "4k_randread_timeseries.json" + with json_file.open("w", encoding="utf8") as f: + json.dump(self.sample_data, f) + + with patch("matplotlib.pyplot.subplots") as mock_subplots, patch("matplotlib.pyplot.close"): + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter.plot_from_file(str(json_file)) + + # Should create 3 plots + self.assertEqual(mock_subplots.call_count, 3) + + def test_plot_from_file_not_found(self) -> None: + """Test plotting from non-existent file""" + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + # Should not raise an error + plotter.plot_from_file("/nonexistent/file.json") + + def test_plot_from_file_invalid_json(self) -> None: + """Test plotting from file with invalid JSON""" + vis_dir = self.archive_dir / "visualisation" + vis_dir.mkdir(parents=True, exist_ok=True) + + json_file = vis_dir / "invalid.json" + with json_file.open("w", encoding="utf8") as f: + f.write("invalid json content") + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + # Should not raise an error + plotter.plot_from_file(str(json_file)) + + def test_plot_all_in_directory(self) -> None: + """Test plotting all JSON files in a directory""" + vis_dir = self.archive_dir / "visualisation" + vis_dir.mkdir(parents=True, exist_ok=True) + + # Create multiple JSON files + for i in range(3): + json_file = vis_dir / f"test_{i}_timeseries.json" + with json_file.open("w", encoding="utf8") as f: + json.dump(self.sample_data, f) + + with patch("matplotlib.pyplot.subplots") as mock_subplots, patch("matplotlib.pyplot.close"): + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter.plot_all_in_directory() + + # Should create 9 plots (3 files x 3 plot types each) + self.assertEqual(mock_subplots.call_count, 9) + + def test_plot_all_in_directory_no_files(self) -> None: + """Test plotting all files when directory has no JSON files""" + vis_dir = self.archive_dir / "visualisation" + vis_dir.mkdir(parents=True, exist_ok=True) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + # Should not raise an error + plotter.plot_all_in_directory() + + def test_plot_all_in_directory_not_exists(self) -> None: + """Test plotting all files when directory doesn't exist""" + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + + # Should not raise an error + plotter.plot_all_in_directory() + + def test_configure_axes(self) -> None: + """Test axes configuration""" + mock_axes = MagicMock() + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._configure_axes( + ax=mock_axes, + title="Test Title", + xlabel="X Label", + ) + + mock_axes.set_title.assert_called_once_with("Test Title", fontsize=14, fontweight="bold") + mock_axes.set_xlabel.assert_called_once_with("X Label", fontsize=12) + mock_axes.set_ylim.assert_called_once_with(bottom=0) + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_bandwidth_conversion_to_mb(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test that bandwidth is correctly converted from bytes to MB/s""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + plotter = TimeSeriesPlotter(archive_directory=str(self.archive_dir), plotter=plt) + plotter._plot_bandwidth(self.sample_data) + + # Get the plot call arguments + plot_call_args = mock_axes.plot.call_args + + # The second argument should be the bandwidth values in MB/s + bandwidth_values = plot_call_args[0][1] + + # Check that values are converted (4096000 bytes = ~3.906 MB) + expected_first_value = 4096000.0 / (1024 * 1024) + self.assertAlmostEqual(bandwidth_values[0], expected_first_value, places=2) + + @patch("matplotlib.pyplot.subplots") + @patch("matplotlib.pyplot.close") + def test_output_directory_created(self, mock_close: MagicMock, mock_subplots: MagicMock) -> None: + """Test that output directory is created if it doesn't exist""" + mock_figure = MagicMock() + mock_axes = MagicMock() + mock_subplots.return_value = (mock_figure, mock_axes) + + # Use a new directory that doesn't exist + new_archive = self.archive_dir / "new_test" + plotter = TimeSeriesPlotter(archive_directory=str(new_archive), plotter=plt) + + plotter.plot_time_series(self.sample_data) + + # Output directory should be created + self.assertTrue((new_archive / "visualisation").exists()) + + +# Made with Bob diff --git a/tests/test_time_series_report_generator.py b/tests/test_time_series_report_generator.py new file mode 100644 index 00000000..faad430b --- /dev/null +++ b/tests/test_time_series_report_generator.py @@ -0,0 +1,418 @@ +""" +Unit tests for the TimeSeriesReportGenerator class +""" + +# pyright: strict, reportPrivateUsage=false +# +# We are OK to ignore private use in unit tests as the whole point of the tests +# is to validate the functions contained in the module + +import json +import shutil +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from post_processing.reports.time_series_report_generator import ( + TimeSeriesReportGenerator, +) + + +class TestTimeSeriesReportGenerator(unittest.TestCase): + """Test cases for TimeSeriesReportGenerator class""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.archive_dir = Path(self.temp_dir) / "test_archive" + + # Use timeseries nested structure: operation/total_iodepth-X/visualisation/ + # This matches the structure expected by find_timeseries_visualisation_directories() + self.vis_dir = self.archive_dir / "read" / "total_iodepth-128" / "visualisation" + self.vis_dir.mkdir(parents=True) + + # Create test time-series data files with actual JSON content + # Format: {blocksize}_{numjobs}_{operation}_{iodepth}_timeseries.json + self._create_test_timeseries_file("4096_1_read_1_timeseries.json", 4096, "1") + self._create_test_timeseries_file("65536_1_read_1_timeseries.json", 65536, "1") + # Add test file with different iodepth to test per-iodepth plotting + self._create_test_timeseries_file("4096_1_read_128_timeseries.json", 4096, "128") + + def _create_test_timeseries_file(self, filename: str, blocksize: int, iodepth: str) -> None: + """Create a test time-series JSON file with realistic data""" + test_data = { + "benchmark": "fio", + "operation": "read", + "blocksize": f"{blocksize}", + "numjobs": "1", + "iodepth": iodepth, + "metadata": { + "start_time_epoch": 1000.0, + "end_time_epoch": 1100.0, + "duration_seconds": 100.0, + "num_volumes": 1, + "sampling_interval_ms": 1000, + "log_avg_msec": 1000, + }, + "timeseries": [ + { + "timestamp_sec": 1000.0, + "iops": 1000.0, + "bandwidth_bytes": 4096000.0, + "mean_latency_ms": 1.5, + "max_latency_ms": 5.0, + "p50_latency_ms": 1.2, + "p95_latency_ms": 2.5, + "p99_latency_ms": 3.5, + "num_samples": 100, + }, + { + "timestamp_sec": 1001.0, + "iops": 1200.0, + "bandwidth_bytes": 4915200.0, + "mean_latency_ms": 1.3, + "max_latency_ms": 4.5, + "p50_latency_ms": 1.1, + "p95_latency_ms": 2.3, + "p99_latency_ms": 3.2, + "num_samples": 100, + }, + ], + # Pre-calculated maximum values with timestamps + "maximum_iops": "1200", + "maximum_bandwidth": "4915200", + "latency_at_max_iops": "1.300000", + "latency_at_max_bandwidth": "1.300000", + "timestamp_at_max_iops": "1001.0", + "timestamp_at_max_bandwidth": "1001.0", + "maximum_latency": "1.500000", + "timestamp_at_max_latency": "1000.0", + "maximum_cpu_usage": "0.00", + "maximum_memory_usage": "0.00", + } + with open(self.vis_dir / filename, "w", encoding="utf-8") as f: + json.dump(test_data, f) + + def tearDown(self) -> None: + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_generate_report_title(self) -> None: + """Test generating report title""" + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + title = generator._generate_report_title() + + self.assertIn("Time-Series Performance Report", title) + self.assertIn("test-archive", title) + + def test_generate_report_name(self) -> None: + """Test generating report name with timestamp""" + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + report_name = generator._generate_report_name() + + self.assertTrue(report_name.startswith("timeseries_performance_report_")) + self.assertTrue(report_name.endswith(".md")) + # Should contain timestamp in format YYMMDD_HHMMSS + self.assertIn("_", report_name) + + def test_get_plot_file_stem(self) -> None: + """Test extracting plot file stem from time-series plot filename""" + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + # Test IOPS plot + iops_file = Path("4096_1_read_iops_timeseries.svg") + stem = generator._get_plot_file_stem(iops_file) + self.assertEqual(stem, "4096_1_read") + + # Test bandwidth plot + bw_file = Path("65536_1_read_bandwidth_timeseries.svg") + stem = generator._get_plot_file_stem(bw_file) + self.assertEqual(stem, "65536_1_read") + + # Test latency plot + lat_file = Path("4096_1_read_latency_timeseries.svg") + stem = generator._get_plot_file_stem(lat_file) + self.assertEqual(stem, "4096_1_read") + + @patch("post_processing.reports.time_series_report_generator.TimeSeriesPlotter") + def test_copy_images_creates_plots_if_missing(self, mock_plotter_class: MagicMock) -> None: + """Test that _copy_images creates plots if they don't exist""" + output_dir = f"{self.temp_dir}/output" + + mock_plotter = MagicMock() + mock_plotter_class.return_value = mock_plotter + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + generator._copy_images() + + # Should create plotter and call draw_and_save + mock_plotter_class.assert_called_once() + mock_plotter.draw_and_save.assert_called_once() + + def test_find_common_timeseries_file_names(self) -> None: + """Test finding common time-series file names""" + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + file_names = generator._find_common_timeseries_file_names() + + # Should find both time-series files + self.assertEqual(len(file_names), 3) + self.assertIn("4096_1_read_1_timeseries.json", file_names) + self.assertIn("4096_1_read_128_timeseries.json", file_names) + self.assertIn("65536_1_read_1_timeseries.json", file_names) + + def test_find_and_sort_data_files(self) -> None: + """Test finding and sorting time-series data files""" + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + data_files = generator._find_and_sort_data_files() + + # Should find all three files (including different iodepth), sorted by blocksize then numjobs + # Note: Files are sorted alphabetically, so "128" comes before "1" in string sorting + self.assertEqual(len(data_files), 3) + keys = list(data_files.keys()) + # Verify all expected files are present + self.assertIn("4096_1_read_1_timeseries", keys) + self.assertIn("4096_1_read_128_timeseries", keys) + self.assertIn("65536_1_read_1_timeseries", keys) + # Verify 4K files come before 64K files + idx_4k_1 = keys.index("4096_1_read_1_timeseries") + idx_4k_128 = keys.index("4096_1_read_128_timeseries") + idx_64k = keys.index("65536_1_read_1_timeseries") + self.assertLess(idx_4k_1, idx_64k) + self.assertLess(idx_4k_128, idx_64k) + + def test_find_and_sort_file_paths(self) -> None: + """Test finding and sorting file paths""" + # Create multiple time-series files + self._create_test_timeseries_file("8192_1_write_1_timeseries.json", 8192, "1") + self._create_test_timeseries_file("16384_1_read_1_timeseries.json", 16384, "1") + + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + paths = generator._find_and_sort_file_paths( + paths=[self.vis_dir], + search_pattern="*_timeseries.json", + index=0, + ) + + # Should find all 5 files (3 from setUp + 2 created here), sorted by blocksize + self.assertEqual(len(paths), 5) + # First file should be 4096 + self.assertTrue(paths[0].name.startswith("4096")) + + def test_add_configuration_yaml_files(self) -> None: + """Test adding configuration YAML files to report""" + # Create a mock YAML file + results_dir = self.archive_dir / "results" + results_dir.mkdir(parents=True) + yaml_file = results_dir / "cbt_config.yaml" + yaml_file.write_text("test: config\nkey: value\n") + + output_dir = f"{self.temp_dir}/output" + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + # Mock the report object + generator._report = MagicMock() + + generator._add_configuration_yaml_files() + + # Should add header and paragraph + generator._report.new_header.assert_called_once() + generator._report.new_paragraph.assert_called_once() + + def test_get_all_number_of_jobs_values_with_timeseries_plots(self) -> None: + """ + Test that _get_all_number_of_jobs_values correctly handles time-series plot filenames. + + This test ensures the fix for the KeyError when parsing time-series plot files + (e.g., "4k_1_randread_iops_timeseries.svg") works correctly. + + The method should use _get_plot_file_stem() to strip the metric and timeseries + suffixes before parsing, preventing KeyError: 'timeseries' in TITLE_CONVERSION. + """ + output_dir = f"{self.temp_dir}/output" + + # Create time-series plot files with different metrics and numjobs + plots_dir = self.vis_dir + plots_dir.mkdir(parents=True, exist_ok=True) + + # Create plot files with various metrics and numjobs values + plot_files = [ + "4096_1_read_iops_timeseries.svg", + "4096_1_read_bandwidth_timeseries.svg", + "4096_1_read_latency_timeseries.svg", + "4096_2_read_iops_timeseries.svg", + "65536_4_write_bandwidth_timeseries.svg", + "65536_4_write_latency_timeseries.svg", + ] + + for plot_file in plot_files: + (plots_dir / plot_file).touch() + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + # Manually set plot files to our test files + generator._plot_files = [plots_dir / f for f in plot_files] + + # This should not raise KeyError: 'timeseries' + try: + numjobs_values = generator._get_all_number_of_jobs_values() + except KeyError as e: + self.fail(f"_get_all_number_of_jobs_values raised KeyError: {e}") + + # Should extract unique numjobs values: 1, 2, 4 + self.assertEqual(len(numjobs_values), 3) + self.assertIn("1", numjobs_values) + self.assertIn("2", numjobs_values) + self.assertIn("4", numjobs_values) + + # Should be sorted + self.assertEqual(numjobs_values, ["1", "2", "4"]) + + def test_find_and_sort_plot_files_by_numjobs_blocksize_iodepth(self) -> None: + """Test that plots are sorted by numjobs, then blocksize, then iodepth""" + output_dir = f"{self.temp_dir}/output" + + # Create plot files with different numjobs, blocksizes, and iodepths + # Format: {blocksize}_{numjobs}_{operation}_{iodepth}_{metric}_timeseries.svg + plots_dir = Path(output_dir) / "plots" + plots_dir.mkdir(parents=True, exist_ok=True) + + plot_files = [ + # numjobs=1, various blocksizes and iodepths + "4096_1_read_128_iops_timeseries.svg", # bs=4096, nj=1, iod=128 + "4096_1_read_1_iops_timeseries.svg", # bs=4096, nj=1, iod=1 + "65536_1_read_1_bandwidth_timeseries.svg", # bs=65536, nj=1, iod=1 + "4096_1_read_8_iops_timeseries.svg", # bs=4096, nj=1, iod=8 + # numjobs=2, various blocksizes and iodepths + "8192_2_write_16_iops_timeseries.svg", # bs=8192, nj=2, iod=16 + "4096_2_read_1_iops_timeseries.svg", # bs=4096, nj=2, iod=1 + "8192_2_write_8_iops_timeseries.svg", # bs=8192, nj=2, iod=8 + ] + + for plot_file in plot_files: + (plots_dir / plot_file).touch() + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + # Manually set plots directory to our test directory + generator._plots_directory = plots_dir + + # Get sorted plot files + sorted_plots = generator._find_and_sort_plot_files() + + # Extract filenames for easier assertion + sorted_names = [p.name for p in sorted_plots] + + # Expected order: sorted by (numjobs, blocksize, iodepth) + expected_order = [ + # numjobs=1 group + "4096_1_read_1_iops_timeseries.svg", # nj=1, bs=4096, iod=1 + "4096_1_read_8_iops_timeseries.svg", # nj=1, bs=4096, iod=8 + "4096_1_read_128_iops_timeseries.svg", # nj=1, bs=4096, iod=128 + "65536_1_read_1_bandwidth_timeseries.svg", # nj=1, bs=65536, iod=1 + # numjobs=2 group + "4096_2_read_1_iops_timeseries.svg", # nj=2, bs=4096, iod=1 + "8192_2_write_8_iops_timeseries.svg", # nj=2, bs=8192, iod=8 + "8192_2_write_16_iops_timeseries.svg", # nj=2, bs=8192, iod=16 + ] + + self.assertEqual(sorted_names, expected_order) + + def test_add_plots_single_column_layout(self) -> None: + """Test that _add_plots creates single-column tables (columns=1)""" + output_dir = f"{self.temp_dir}/output" + + # Create plot files + plots_dir = Path(output_dir) / "plots" + plots_dir.mkdir(parents=True, exist_ok=True) + + plot_files = [ + "4096_1_read_1_iops_timeseries.svg", + "4096_1_read_1_latency_timeseries.svg", + ] + + for plot_file in plot_files: + (plots_dir / plot_file).touch() + + generator = TimeSeriesReportGenerator( + archive_directories=[str(self.archive_dir)], + output_directory=output_dir, + ) + + # Set up the generator with our test plots + generator._plots_directory = plots_dir + generator._plot_files = [plots_dir / f for f in plot_files] + + # Mock the report object to capture new_table calls + generator._report = MagicMock() + + # Call _add_plots + generator._add_plots() + + # Verify that new_table was called with columns=1 (not columns=2) + # Get all calls to new_table + table_calls = list(generator._report.new_table.call_args_list) + + # Should have at least one table call + self.assertGreater(len(table_calls), 0) + + # All table calls should use columns=1 + for call in table_calls: + # call is a tuple of (args, kwargs) + kwargs = call[1] + self.assertEqual(kwargs.get("columns"), 1, "Table should use single column layout") + + +if __name__ == "__main__": + unittest.main() + +# Made with Bob diff --git a/tests/test_timestamp_aligner.py b/tests/test_timestamp_aligner.py new file mode 100644 index 00000000..cfbc5d10 --- /dev/null +++ b/tests/test_timestamp_aligner.py @@ -0,0 +1,354 @@ +""" +Tests for the TimestampAligner class. + +This module tests timestamp alignment and aggregation functionality for +time-series data from multiple volumes. +""" + +import numpy as np +import pandas as pd +import pytest + +from post_processing.parsers.timestamp_aligner import TimestampAligner + + +class TestTimestampAlignerInitialization: + """Test TimestampAligner initialization.""" + + def test_default_window_size(self): + """Test default window size is 1000ms (1 second).""" + aligner = TimestampAligner() + assert aligner._window_size_sec == 1.0 + + def test_custom_window_size(self): + """Test custom window size.""" + aligner = TimestampAligner(window_size_ms=500) + assert aligner._window_size_sec == 0.5 + + def test_large_window_size(self): + """Test large window size.""" + aligner = TimestampAligner(window_size_ms=5000) + assert aligner._window_size_sec == 5.0 + + +class TestAlignAndAggregate: + """Test align_and_aggregate method.""" + + def test_empty_dataframes_list(self): + """Test with empty list of dataframes.""" + aligner = TimestampAligner() + result = aligner.align_and_aggregate([], "iops", "sum") + assert result.empty + assert list(result.columns) == ["timestamp_sec", "iops", "num_samples"] + + def test_all_none_dataframes(self): + """Test with all None dataframes.""" + aligner = TimestampAligner() + result = aligner.align_and_aggregate([None, None], "iops", "sum") + assert result.empty + assert list(result.columns) == ["timestamp_sec", "iops", "num_samples"] + + def test_all_empty_dataframes(self): + """Test with all empty dataframes.""" + aligner = TimestampAligner() + df1 = pd.DataFrame(columns=["timestamp_sec", "iops"]) + df2 = pd.DataFrame(columns=["timestamp_sec", "iops"]) + result = aligner.align_and_aggregate([df1, df2], "iops", "sum") + assert result.empty + + def test_single_dataframe_sum(self): + """Test with single dataframe using sum aggregation.""" + aligner = TimestampAligner(window_size_ms=1000) + df = pd.DataFrame({"timestamp_sec": [0.5, 1.5, 2.5], "iops": [100, 200, 300]}) + result = aligner.align_and_aggregate([df], "iops", "sum") + + assert len(result) == 3 + assert list(result.columns) == ["timestamp_sec", "iops", "num_samples"] + assert result["iops"].tolist() == [100, 200, 300] + assert result["num_samples"].tolist() == [1, 1, 1] + + def test_multiple_dataframes_sum(self): + """Test with multiple dataframes using sum aggregation.""" + aligner = TimestampAligner(window_size_ms=1000) + df1 = pd.DataFrame({"timestamp_sec": [0.5, 1.5], "iops": [100, 200]}) + df2 = pd.DataFrame({"timestamp_sec": [0.7, 1.3], "iops": [50, 150]}) + result = aligner.align_and_aggregate([df1, df2], "iops", "sum") + + assert len(result) == 2 + # Both 0.5 and 0.7 fall in window 0.0, both 1.5 and 1.3 fall in window 1.0 + assert result["timestamp_sec"].tolist() == [0.0, 1.0] + assert result["iops"].tolist() == [150, 350] # 100+50, 200+150 + assert result["num_samples"].tolist() == [2, 2] + + def test_aggregation_mean(self): + """Test mean aggregation.""" + aligner = TimestampAligner(window_size_ms=1000) + df1 = pd.DataFrame({"timestamp_sec": [0.5, 1.5], "latency_ms": [10.0, 20.0]}) + df2 = pd.DataFrame({"timestamp_sec": [0.7, 1.3], "latency_ms": [20.0, 30.0]}) + result = aligner.align_and_aggregate([df1, df2], "latency_ms", "mean") + + assert len(result) == 2 + assert result["latency_ms"].tolist() == [15.0, 25.0] # (10+20)/2, (20+30)/2 + + def test_aggregation_max(self): + """Test max aggregation.""" + aligner = TimestampAligner(window_size_ms=1000) + df1 = pd.DataFrame({"timestamp_sec": [0.5, 1.5], "latency_ms": [10.0, 20.0]}) + df2 = pd.DataFrame({"timestamp_sec": [0.7, 1.3], "latency_ms": [20.0, 15.0]}) + result = aligner.align_and_aggregate([df1, df2], "latency_ms", "max") + + assert len(result) == 2 + assert result["latency_ms"].tolist() == [20.0, 20.0] # max(10,20), max(20,15) + + def test_invalid_aggregation_method(self): + """Test with invalid aggregation method.""" + aligner = TimestampAligner() + df = pd.DataFrame({"timestamp_sec": [0.5], "iops": [100]}) + with pytest.raises(ValueError, match="Unknown aggregation method"): + aligner.align_and_aggregate([df], "iops", "invalid") + + def test_mixed_none_and_valid_dataframes(self): + """Test with mix of None and valid dataframes.""" + aligner = TimestampAligner(window_size_ms=1000) + df1 = pd.DataFrame({"timestamp_sec": [0.5], "iops": [100]}) + df2 = None + df3 = pd.DataFrame({"timestamp_sec": [0.7], "iops": [50]}) + result = aligner.align_and_aggregate([df1, df2, df3], "iops", "sum") + + assert len(result) == 1 + assert result["iops"].tolist() == [150] + + def test_small_window_size(self): + """Test with small window size (100ms).""" + aligner = TimestampAligner(window_size_ms=100) + df = pd.DataFrame({"timestamp_sec": [0.05, 0.15, 0.25], "iops": [100, 200, 300]}) + result = aligner.align_and_aggregate([df], "iops", "sum") + + assert len(result) == 3 + assert result["timestamp_sec"].tolist() == [0.0, 0.1, 0.2] + + def test_large_window_size(self): + """Test with large window size (5 seconds).""" + aligner = TimestampAligner(window_size_ms=5000) + df = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 3.0, 6.0], "iops": [100, 200, 300, 400]}) + result = aligner.align_and_aggregate([df], "iops", "sum") + + assert len(result) == 2 + assert result["timestamp_sec"].tolist() == [0.0, 5.0] + assert result["iops"].tolist() == [600, 400] # 100+200+300, 400 + + +class TestCalculatePercentiles: + """Test calculate_percentiles method.""" + + def test_empty_dataframes_list(self): + """Test with empty list of dataframes.""" + aligner = TimestampAligner() + result = aligner.calculate_percentiles([]) + assert result.empty + assert "timestamp_sec" in result.columns + + def test_all_none_dataframes(self): + """Test with all None dataframes.""" + aligner = TimestampAligner() + result = aligner.calculate_percentiles([None, None]) + assert result.empty + + def test_all_empty_dataframes(self): + """Test with all empty dataframes.""" + aligner = TimestampAligner() + df1 = pd.DataFrame(columns=["timestamp_sec", "latency_ms"]) + df2 = pd.DataFrame(columns=["timestamp_sec", "latency_ms"]) + result = aligner.calculate_percentiles([df1, df2]) + assert result.empty + + def test_default_percentiles(self): + """Test with default percentiles (50, 95, 99).""" + aligner = TimestampAligner(window_size_ms=1000) + df = pd.DataFrame({ + "timestamp_sec": [0.5] * 100, + "latency_ms": list(range(100)) + }) + result = aligner.calculate_percentiles([df]) + + assert len(result) == 1 + assert "timestamp_sec" in result.columns + assert "p50_latency_ms" in result.columns + assert "p95_latency_ms" in result.columns + assert "p99_latency_ms" in result.columns + assert result["p50_latency_ms"].iloc[0] == pytest.approx(49.5, rel=0.1) + assert result["p95_latency_ms"].iloc[0] == pytest.approx(94.05, rel=0.1) + assert result["p99_latency_ms"].iloc[0] == pytest.approx(98.01, rel=0.1) + + def test_custom_percentiles(self): + """Test with custom percentiles.""" + aligner = TimestampAligner(window_size_ms=1000) + df = pd.DataFrame({ + "timestamp_sec": [0.5] * 100, + "latency_ms": list(range(100)) + }) + result = aligner.calculate_percentiles([df], percentiles=[25, 75]) + + assert "p25_latency_ms" in result.columns + assert "p75_latency_ms" in result.columns + assert "p50_latency_ms" not in result.columns + + def test_multiple_time_windows(self): + """Test percentiles across multiple time windows.""" + aligner = TimestampAligner(window_size_ms=1000) + df = pd.DataFrame({ + "timestamp_sec": [0.5] * 50 + [1.5] * 50, + "latency_ms": list(range(50)) + list(range(100, 150)) + }) + result = aligner.calculate_percentiles([df]) + + assert len(result) == 2 + assert result["timestamp_sec"].tolist() == [0.0, 1.0] + + def test_multiple_dataframes(self): + """Test percentiles with multiple dataframes.""" + aligner = TimestampAligner(window_size_ms=1000) + df1 = pd.DataFrame({"timestamp_sec": [0.5] * 50, "latency_ms": list(range(50))}) + df2 = pd.DataFrame({"timestamp_sec": [0.7] * 50, "latency_ms": list(range(50, 100))}) + result = aligner.calculate_percentiles([df1, df2]) + + assert len(result) == 1 + # All data falls in the same window (0.0) + + +class TestMergeMetrics: + """Test merge_metrics method.""" + + def test_no_dataframes(self): + """Test with no dataframes.""" + aligner = TimestampAligner() + result = aligner.merge_metrics() + assert result.empty + + def test_all_none_dataframes(self): + """Test with all None dataframes.""" + aligner = TimestampAligner() + result = aligner.merge_metrics(None, None) + assert result.empty + + def test_all_empty_dataframes(self): + """Test with all empty dataframes.""" + aligner = TimestampAligner() + df1 = pd.DataFrame() + df2 = pd.DataFrame() + result = aligner.merge_metrics(df1, df2) + assert result.empty + + def test_single_dataframe(self): + """Test with single dataframe.""" + aligner = TimestampAligner() + df = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200]}) + result = aligner.merge_metrics(df) + + assert len(result) == 2 + assert list(result.columns) == ["timestamp_sec", "iops"] + pd.testing.assert_frame_equal(result, df) + + def test_merge_two_dataframes(self): + """Test merging two dataframes.""" + aligner = TimestampAligner() + df1 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200]}) + df2 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "latency_ms": [10.0, 20.0]}) + result = aligner.merge_metrics(df1, df2) + + assert len(result) == 2 + assert set(result.columns) == {"timestamp_sec", "iops", "latency_ms"} + assert result["iops"].tolist() == [100, 200] + assert result["latency_ms"].tolist() == [10.0, 20.0] + + def test_merge_with_different_timestamps(self): + """Test merging dataframes with different timestamps (outer join).""" + aligner = TimestampAligner() + df1 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200]}) + df2 = pd.DataFrame({"timestamp_sec": [1.0, 2.0], "latency_ms": [20.0, 30.0]}) + result = aligner.merge_metrics(df1, df2) + + assert len(result) == 3 + assert result["timestamp_sec"].tolist() == [0.0, 1.0, 2.0] + assert result["iops"].tolist()[0] == 100 + assert pd.isna(result["iops"].tolist()[2]) # NaN for missing value + + def test_merge_multiple_dataframes(self): + """Test merging multiple dataframes.""" + aligner = TimestampAligner() + df1 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200]}) + df2 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "latency_ms": [10.0, 20.0]}) + df3 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "bandwidth_mb": [50.0, 100.0]}) + result = aligner.merge_metrics(df1, df2, df3) + + assert len(result) == 2 + assert set(result.columns) == {"timestamp_sec", "iops", "latency_ms", "bandwidth_mb"} + + def test_merge_removes_duplicate_columns(self): + """Test that merge removes duplicate columns.""" + aligner = TimestampAligner() + df1 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200], "extra": [1, 2]}) + df2 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200], "latency_ms": [10.0, 20.0]}) + result = aligner.merge_metrics(df1, df2) + + # Should keep first occurrence of 'iops', not create 'iops_dup' + assert "iops_dup" not in result.columns + assert "iops" in result.columns + + def test_merge_sorts_by_timestamp(self): + """Test that merge sorts results by timestamp.""" + aligner = TimestampAligner() + df1 = pd.DataFrame({"timestamp_sec": [2.0, 0.0, 1.0], "iops": [300, 100, 200]}) + df2 = pd.DataFrame({"timestamp_sec": [1.0, 2.0, 0.0], "latency_ms": [20.0, 30.0, 10.0]}) + result = aligner.merge_metrics(df1, df2) + + assert result["timestamp_sec"].tolist() == [0.0, 1.0, 2.0] + + def test_merge_with_none_and_valid_dataframes(self): + """Test merging with mix of None and valid dataframes.""" + aligner = TimestampAligner() + df1 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "iops": [100, 200]}) + df2 = None + df3 = pd.DataFrame({"timestamp_sec": [0.0, 1.0], "latency_ms": [10.0, 20.0]}) + result = aligner.merge_metrics(df1, df2, df3) + + assert len(result) == 2 + assert set(result.columns) == {"timestamp_sec", "iops", "latency_ms"} + + +class TestIntegration: + """Integration tests combining multiple methods.""" + + def test_full_workflow(self): + """Test complete workflow: align, calculate percentiles, merge.""" + aligner = TimestampAligner(window_size_ms=1000) + + # Create sample data from two volumes + df1 = pd.DataFrame({ + "timestamp_sec": [0.5, 1.5, 2.5], + "iops": [100, 200, 300], + "latency_ms": [10.0, 20.0, 30.0] + }) + df2 = pd.DataFrame({ + "timestamp_sec": [0.7, 1.3, 2.8], + "iops": [50, 150, 250], + "latency_ms": [15.0, 25.0, 35.0] + }) + + # Align and aggregate IOPS + iops_result = aligner.align_and_aggregate([df1, df2], "iops", "sum") + + # Calculate latency percentiles + latency_result = aligner.calculate_percentiles([df1, df2], percentiles=[50, 95]) + + # Merge results + final_result = aligner.merge_metrics(iops_result, latency_result) + + assert "timestamp_sec" in final_result.columns + assert "iops" in final_result.columns + assert "p50_latency_ms" in final_result.columns + assert "p95_latency_ms" in final_result.columns + assert len(final_result) == 3 + + +# Made with Bob diff --git a/tests/test_visualisation_directory_helpers.py b/tests/test_visualisation_directory_helpers.py new file mode 100644 index 00000000..ceefa4df --- /dev/null +++ b/tests/test_visualisation_directory_helpers.py @@ -0,0 +1,196 @@ +""" +Unit tests for visualisation directory helper functions in post_processing/common.py +""" + +import shutil +import tempfile +import unittest +from pathlib import Path + +from post_processing.common import ( + find_hockey_stick_visualisation_directories, + find_timeseries_visualisation_directories, +) + + +class TestVisualisationDirectoryHelpers(unittest.TestCase): + """Test cases for visualisation directory helper functions""" + + def setUp(self) -> None: + """Set up test fixtures""" + self.temp_dir = tempfile.mkdtemp() + self.archive_dir = Path(self.temp_dir) / "test_archive" + self.archive_dir.mkdir(parents=True) + + def tearDown(self) -> None: + """Clean up test fixtures""" + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_find_hockey_stick_legacy_structure(self) -> None: + """Test finding hockey-stick directories in legacy structure""" + # Create legacy structure: archive/visualisation/ + legacy_vis = self.archive_dir / "visualisation" + legacy_vis.mkdir(parents=True) + (legacy_vis / "4k_1_randread.json").touch() + + result = find_hockey_stick_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0], legacy_vis) + + def test_find_hockey_stick_new_structure(self) -> None: + """Test finding hockey-stick directories in new structure""" + # Create new structure: archive/operation/visualisation/ + randread_vis = self.archive_dir / "randread" / "visualisation" + randread_vis.mkdir(parents=True) + (randread_vis / "4k_1_randread.json").touch() + + randwrite_vis = self.archive_dir / "randwrite" / "visualisation" + randwrite_vis.mkdir(parents=True) + (randwrite_vis / "4k_1_randwrite.json").touch() + + result = find_hockey_stick_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 2) + self.assertIn(randread_vis, result) + self.assertIn(randwrite_vis, result) + + def test_find_hockey_stick_prefers_legacy(self) -> None: + """Test that legacy structure with data takes precedence if both exist""" + # Create both structures + legacy_vis = self.archive_dir / "visualisation" + legacy_vis.mkdir(parents=True) + + # Add a JSON file to legacy directory + (legacy_vis / "test.json").touch() + + new_vis = self.archive_dir / "randread" / "visualisation" + new_vis.mkdir(parents=True) + # Add JSON file to new structure too + (new_vis / "test2.json").touch() + + result = find_hockey_stick_visualisation_directories(self.archive_dir) + + # Should only return legacy structure when it has data + self.assertEqual(len(result), 1) + self.assertEqual(result[0], legacy_vis) + + def test_find_hockey_stick_empty_legacy_with_new_structure(self) -> None: + """Test that empty legacy directory doesn't prevent finding new structure""" + # Create both structures + legacy_vis = self.archive_dir / "visualisation" + legacy_vis.mkdir(parents=True) + + new_vis = self.archive_dir / "randread" / "visualisation" + new_vis.mkdir(parents=True) + # Add JSON file to new structure + (new_vis / "test.json").touch() + + result = find_hockey_stick_visualisation_directories(self.archive_dir) + + # Should only return new structure since legacy is empty + self.assertEqual(len(result), 1) + self.assertEqual(result[0], new_vis) + + def test_find_hockey_stick_empty_directory(self) -> None: + """Test finding hockey-stick directories in empty archive""" + result = find_hockey_stick_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 0) + + def test_find_hockey_stick_ignores_hidden_dirs(self) -> None: + """Test that hidden directories are ignored""" + # Create hidden directory + hidden_vis = self.archive_dir / ".hidden" / "visualisation" + hidden_vis.mkdir(parents=True) + + result = find_hockey_stick_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 0) + + def test_find_timeseries_with_total_iodepth(self) -> None: + """Test finding timeseries directories under total_iodepth""" + # Create structure: archive/operation/total_iodepth-X/visualisation/ + vis_dir = self.archive_dir / "randread" / "total_iodepth-256" / "visualisation" + vis_dir.mkdir(parents=True) + (vis_dir / "randread_4k_256_timeseries.json").touch() + + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0], vis_dir) + + def test_find_timeseries_with_iodepth(self) -> None: + """Test finding timeseries directories under iodepth""" + # Create structure: archive/operation/iodepth-X/visualisation/ + vis_dir = self.archive_dir / "randread" / "iodepth-32" / "visualisation" + vis_dir.mkdir(parents=True) + (vis_dir / "randread_4k_32_timeseries.json").touch() + + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 1) + self.assertEqual(result[0], vis_dir) + + def test_find_timeseries_prioritizes_total_iodepth(self) -> None: + """Test that total_iodepth directories are prioritized over iodepth""" + # Create both types + total_iodepth_vis = self.archive_dir / "randread" / "total_iodepth-256" / "visualisation" + total_iodepth_vis.mkdir(parents=True) + + iodepth_vis = self.archive_dir / "randread" / "iodepth-32" / "visualisation" + iodepth_vis.mkdir(parents=True) + + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 2) + # total_iodepth should come first + self.assertEqual(result[0], total_iodepth_vis) + self.assertEqual(result[1], iodepth_vis) + + def test_find_timeseries_multiple_operations(self) -> None: + """Test finding timeseries directories across multiple operations""" + # Create multiple operations with timeseries data + randread_vis = self.archive_dir / "randread" / "total_iodepth-256" / "visualisation" + randread_vis.mkdir(parents=True) + + randwrite_vis = self.archive_dir / "randwrite" / "iodepth-32" / "visualisation" + randwrite_vis.mkdir(parents=True) + + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 2) + self.assertIn(randread_vis, result) + self.assertIn(randwrite_vis, result) + + def test_find_timeseries_ignores_operation_level(self) -> None: + """Test that operation-level visualisation directories are ignored""" + # Create operation-level visualisation (for hockey-stick data) + operation_vis = self.archive_dir / "randread" / "visualisation" + operation_vis.mkdir(parents=True) + + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 0) + + def test_find_timeseries_empty_directory(self) -> None: + """Test finding timeseries directories in empty archive""" + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 0) + + def test_find_timeseries_ignores_legacy_structure(self) -> None: + """Test that legacy visualisation directory is ignored for timeseries""" + # Create legacy structure + legacy_vis = self.archive_dir / "visualisation" + legacy_vis.mkdir(parents=True) + + result = find_timeseries_visualisation_directories(self.archive_dir) + + self.assertEqual(len(result), 0) + + +if __name__ == "__main__": + unittest.main() + +# Made with Bob diff --git a/tools/fio_common_output_wrapper.py b/tools/fio_common_output_wrapper.py index 84eb21a7..faa39c64 100755 --- a/tools/fio_common_output_wrapper.py +++ b/tools/fio_common_output_wrapper.py @@ -24,6 +24,7 @@ from logging import Logger, getLogger from post_processing.formatter.common_output_formatter import CommonOutputFormatter +from post_processing.formatter.time_series_output_formatter import TimeSeriesOutputFormatter from post_processing.log_configuration import setup_logging setup_logging() @@ -51,24 +52,39 @@ def main() -> int: args: Namespace = parser.parse_args() output_directory: str = f"{args.archive}/visualisation/" - log.debug("Creating directory %s" % output_directory) + log.debug("Creating directory %s", output_directory) os.makedirs(output_directory, exist_ok=True) - log.info("Generating intermediate files for %s in directory %s" % (args.archive, output_directory)) + log.info("Generating intermediate files for %s in directory %s", args.archive, output_directory) + + # Process common output format formatter: CommonOutputFormatter = CommonOutputFormatter( archive_directory=args.archive, filename_root=args.results_file_root ) try: - formatter.convert_all_files() - formatter.write_output_file() + formatter.process() + # formatter.write_output() except Exception as e: log.error( - "Encountered an error parsing results in directory %s with name %s" % (args.archive, args.results_file_root) + "Encountered an error parsing results in directory %s with name %s", args.archive, args.results_file_root ) log.exception(e) result = 1 + # Process time series format + timeseries_formatter: TimeSeriesOutputFormatter = TimeSeriesOutputFormatter( + archive_directory=args.archive, filename_root=args.results_file_root + ) + + try: + timeseries_formatter.process() + # timeseries_formatter.write_output() + except Exception as e: + log.error("Encountered an error parsing time-series logs in directory %s", args.archive) + log.exception(e) + result = 1 + return result diff --git a/tools/generate_timeseries_performance_report.py b/tools/generate_timeseries_performance_report.py new file mode 100755 index 00000000..f5b27560 --- /dev/null +++ b/tools/generate_timeseries_performance_report.py @@ -0,0 +1,131 @@ +#!/usr/bin/env -S python3 -B +""" +A script to automatically generate a time-series report from a set of performance run data +in the common time-series format. + +The archive should contain the 'visualisation' sub directory where all +the time-series .json and plot files reside. + +tools/fio_common_output_wrapper.py will generate the .json files and the TimeSeriesPlotter +module can be used to generate the plot files + +Usage: + generate_timeseries_performance_report.py --archive= + --output_directory= + --create_pdf + --force_refresh + --plot_resources + + +Input: + --output_directory [Required] The directory to write the time-series report + to. If this does not exists it will be created. + + --archive [Required] The directory that contains the common + time-series format .json files and plot files to include + in the report. + + --create_pdf [Optional] Create a pdf file of the report markdown + file. + This requires pandoc to be installed, + and be on the path. + + --force_refresh [Optional] Generate the intermediate and plot files + from the raw data, even if they already exist + + --plot_resources [Optional] Also draw CPU and memory usage, as recorded + by fio on the plots + +Examples: + + Generate a markdown time-series report file for the results in '/tmp/squid_main' directory + and save it in the '/tmp/main_results' directory: + + generate_timeseries_performance_report.py --archive=/tmp/squid_main + --output_directory=/tmp/main_results + + Additionally generate a pdf report file for the example above: + + generate_timeseries_performance_report.py --archive=/tmp/squid_main + --output_directory=/tmp/main_results + --create_pdf +""" + +from argparse import SUPPRESS, ArgumentParser +from logging import Logger, getLogger + +from post_processing.log_configuration import setup_logging +from post_processing.post_processing_types import ReportOptions +from post_processing.report import Report, parse_namespace_to_options + +setup_logging() +log: Logger = getLogger("reports") +log.info("=== Starting Post Processing of CBT results ===") + + +def main() -> int: + """ + Main routine for the script + """ + + description: str = "Produces a time-series performance report in markdown format \n" + description += "from the time-series json and svg files stored in the visualisation\n" + description += "subdirectory of the directory given by --archive\n" + description += "The resulting report(s) are saved in the specified output directory.\n" + description += "The json files must be in the time-series format with *_timeseries.json naming" + + parser: ArgumentParser = ArgumentParser(description=description) + + parser.add_argument( + "--output_directory", + type=str, + required=True, + help="The directory to store the time-series report file(s)", + ) + parser.add_argument( + "--archive", + type=str, + required=True, + help="The directory that contains the set of time-series json results files and generated plot files" + + "for a particular test run", + ) + parser.add_argument( + "--create_pdf", + action="store_true", + help="Generate a pdf report file in addition to the markdown report", + ) + + parser.add_argument( + "--force_refresh", + action="store_true", + required=False, + help="Regenerate the intermediate files and plots, even if they exist", + ) + + # timeseries reports do not currently produce resource statistics + parser.add_argument( + "--plot_resources", + action="store_true", + required=False, + help=SUPPRESS, + ) + + # the following argument is required to exist for future processing, but is not needed for timeseries reports + parser.add_argument("--results_file_root", type=str, required=False, default="json_output", help=SUPPRESS) + + report_options: ReportOptions = parse_namespace_to_options(arguments=parser.parse_args(), timeseries_report=True) + + report: Report = Report(options=report_options) + + try: + report.generate(throw_exception=True) + except Exception: + log.exception("FAILED: Encountered an error generating the time-series report") + + return report.result_code + + +if __name__ == "__main__": + main() + +# Made with Bob