Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 178 additions & 1 deletion frontend/src/Metrics.tsx
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import { useAtomValue } from "jotai";
import { pivotReportAtom, selectedReportsAtom } from "./atoms.ts";
import {
type HardwareEnvironment,
pivotReportAtom,
selectedReportsAtom,
} from "./atoms.ts";
import { DateTime } from "./DateTime.tsx";
import { Duration } from "./Duration.tsx";
import { Q } from "./Q.tsx";
import { RichCell } from "./RichCell.tsx";
import { Section, SectionTable, SectionTitle } from "./Section.tsx";
import {
Expand All @@ -11,6 +16,132 @@ import {
} from "./Table.tsx";
import { durationTrend } from "./trends.ts";

function formatHardware(hardware: HardwareEnvironment): string {
const parts = [
hardware.cpuModel ?? "unknown CPU",
hardware.cpuCount != null ? `${hardware.cpuCount} cores` : null,
hardware.memoryTotalMb != null
? `${(hardware.memoryTotalMb / 1024).toFixed(0)} GB RAM`
: null,
].filter((p) => p != null);
const origin = hardware.ci ? "CI" : "local";
return `${parts.join(", ")} · ${hardware.os} (${hardware.arch}) · ${origin}`;
}

/**
* Reference specs for CPU models we commonly see, so comparing 2 machines isn't a guessing game.
* Ordered most-specific first: the generic cloud-Xeon fallback at the end only fires when nothing
* more specific matched. Add entries here as new machine types show up (e.g. new GCP zones/generations
* rotate their exact Xeon SKU, and GCP's AMD parts use cloud-only "B"/"9B"-suffixed model numbers that
* don't appear in retail listings).
*/
const KNOWN_CPU_SPECS: { test: (model: string) => boolean; note: string }[] = [
// GitHub Actions runners
{
test: (m) => m.includes("EPYC 7763"),
note: "GitHub Actions Linux x64 runner · ~2.45 GHz base, 3.5 GHz boost",
},
{
test: (m) => m.includes("Ampere Altra"),
note: "GitHub Actions Linux arm64 runner · ~3.0 GHz",
},
// Apple Silicon (local dev machines)
{
test: (m) => m.includes("Apple M1"),
note: "Apple Silicon (2020) · up to 3.2 GHz",
},
{
test: (m) => m.includes("Apple M2"),
note: "Apple Silicon (2022) · up to 3.5 GHz",
},
{
test: (m) => m.includes("Apple M3"),
note: "Apple Silicon (2023) · up to 4.05 GHz",
},
{
test: (m) => m.includes("Apple M4"),
note: "Apple Silicon (2024) · up to 4.4 GHz",
},
// GCP Compute Engine — cloud-only SKUs, don't match any retail part number
{
test: (m) => /EPYC 7B1[23]/.test(m),
note: "GCP N2D/T2D custom AMD EPYC (Rome/Milan class)",
},
{
test: (m) => /EPYC 9B\d{2}/.test(m),
note: "GCP C3D custom AMD EPYC (Genoa class)",
},
{
test: (m) => /Xeon\(R\) Platinum 8481C/.test(m),
note: "GCP C3 custom Intel Xeon (Sapphire Rapids class)",
},
{
test: (m) => /Neoverse-N1/.test(m),
note: "Arm Neoverse N1 server core (GCP T2A / Ampere Altra class) · ~3.0 GHz",
},
// Generic cloud fallback: N1/N2/C2 and most other providers mask the exact SKU in /proc/cpuinfo
// and only report the clock speed, so this is the best we can say without more specific data.
{
test: (m) => /Xeon\(R\) CPU @ [\d.]+\s*GHz/i.test(m),
note: "Cloud VM Intel Xeon · exact model hidden by hypervisor, clock speed as reported",
},
];

function cpuSpecNote(cpuModel: string | null): string | null {
if (cpuModel == null) return null;
return KNOWN_CPU_SPECS.find((spec) => spec.test(cpuModel))?.note ?? null;
}

/** Short, stable, content-derived label so long hardware descriptions don't blow up the table; full text lives in a legend tooltip. */
function hardwareHash(hardware: HardwareEnvironment): string {
const str = JSON.stringify(hardware);
let hash = 0x811c9dc5;
for (let i = 0; i < str.length; i++) {
hash ^= str.charCodeAt(i);
hash = Math.imul(hash, 0x01000193);
}
return (hash >>> 0).toString(16).padStart(8, "0").slice(0, 6);
}

function HardwareLegend({
hardware,
differsFromPivot,
}: {
hardware: HardwareEnvironment[];
differsFromPivot: boolean;
}) {
return (
<Q className="tooltip-left" contentClassName="text-left">
<div className="flex flex-col gap-1.5">
{hardware.map((h) => (
<div key={JSON.stringify(h)}>
<div>
<code>{hardwareHash(h)}</code> — {formatHardware(h)}
</div>
{cpuSpecNote(h.cpuModel) != null && (
<div className="text-xs opacity-70">
{cpuSpecNote(h.cpuModel)}
</div>
)}
</div>
))}
{differsFromPivot && <div>Differs from pivot report's hardware</div>}
</div>
</Q>
);
}

/** Order-insensitive comparison: merged reports can list the same set of environments in different orders. */
function hardwareListsEqual(
a: HardwareEnvironment[],
b: HardwareEnvironment[],
): boolean {
if (a.length !== b.length) return false;
const sortedA = a.map((h) => JSON.stringify(h)).sort();
const sortedB = b.map((h) => JSON.stringify(h)).sort();
return sortedA.every((s, i) => s === sortedB[i]);
}

export function MetricsSection() {
const selectedReports = useAtomValue(selectedReportsAtom);
const pivotReport = useAtomValue(pivotReportAtom);
Expand Down Expand Up @@ -42,6 +173,52 @@ export function MetricsSection() {
title="Created At"
cell={(report) => <DateTime value={report.metrics.createdAt} />}
/>
<ReportTableRow
title="Hardware"
cell={(report) => {
const hardware = report.metrics.hardware;
if (hardware.length === 0) {
return <span className="text-base-content/60">—</span>;
}
const differsFromPivot =
pivotReport != null &&
pivotReport.metrics.hardware.length > 0 &&
!hardwareListsEqual(hardware, pivotReport.metrics.hardware);
if (hardware.length > 1) {
return (
<span className="text-warning">
Mixed ({hardware.map((h) => hardwareHash(h)).join(", ")}){" "}
<HardwareLegend
hardware={hardware}
differsFromPivot={differsFromPivot}
/>
</span>
);
}
const only = hardware[0];
if (only == null) {
return <span className="text-base-content/60">—</span>;
}
return (
<span className={differsFromPivot ? "text-warning" : undefined}>
<code>{hardwareHash(only)}</code>{" "}
<Q className="tooltip-left" contentClassName="text-left">
<div className="flex flex-col gap-1.5">
<div>{formatHardware(only)}</div>
{cpuSpecNote(only.cpuModel) != null && (
<div className="text-xs opacity-70">
{cpuSpecNote(only.cpuModel)}
</div>
)}
{differsFromPivot && (
<div>Differs from pivot report's hardware</div>
)}
</div>
</Q>
</span>
);
}}
/>
</tbody>
<ReportTableSection title="Totals" />
<tbody>
Expand Down
17 changes: 14 additions & 3 deletions frontend/src/Q.tsx
Original file line number Diff line number Diff line change
@@ -1,10 +1,21 @@
import clsx from "clsx";
import type { ReactNode } from "react";
import { VscInfo } from "react-icons/vsc";

export function Q({ children }: { children: ReactNode }) {
export function Q({
children,
className,
contentClassName,
}: {
children: ReactNode;
className?: string;
contentClassName?: string;
}) {
return (
<span className="tooltip cursor-help">
<span className="tooltip-content">{children}</span>
<span className={clsx("tooltip cursor-help", className)}>
<span className={clsx("tooltip-content", contentClassName)}>
{children}
</span>
<VscInfo className="inline text-info" />
</span>
);
Expand Down
10 changes: 10 additions & 0 deletions frontend/src/atoms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ export type ReportTitle = string & {
export type SliceTitle = string & { readonly __nonexistent_tag: unique symbol };
export type TestName = string & { readonly __nonexistent_tag: unique symbol };

export interface HardwareEnvironment {
os: string;
arch: string;
cpuModel: string | null;
cpuCount: number | null;
memoryTotalMb: number | null;
ci: boolean;
}

export interface Metrics {
workspace: string;
scarbVersion: string;
Expand All @@ -21,6 +30,7 @@ export interface Metrics {
createdAt: string;
totalExecutionTime: string;
totalProjects: number;
hardware: HardwareEnvironment[];
meanBuildTime: string | null;
meanLintTime: string | null;
meanTestTime: string | null;
Expand Down
80 changes: 80 additions & 0 deletions src/maat/hardware.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import os
import platform
import re
from typing import Self

from pydantic import BaseModel


class HardwareEnvironment(BaseModel):
os: str
arch: str
cpu_model: str | None
cpu_count: int | None
memory_total_mb: int | None
ci: bool

@classmethod
def capture(cls) -> Self:
try:
return cls(
os=f"{platform.system()} {platform.release()}",
arch=platform.machine(),
cpu_model=_cpu_model(),
cpu_count=os.cpu_count(),
memory_total_mb=_memory_total_mb(),
ci=os.environ.get("GITHUB_ACTIONS") == "true",
)
except Exception:
return cls(
os="unknown",
arch="unknown",
cpu_model=None,
cpu_count=None,
memory_total_mb=None,
ci=False,
)


def _cpu_model() -> str | None:
try:
if platform.system() == "Linux":
with open("/proc/cpuinfo") as f:
for line in f:
if line.lower().startswith("model name"):
return line.split(":", 1)[1].strip()
elif platform.system() == "Darwin":
import subprocess

return (
subprocess.check_output(["sysctl", "-n", "machdep.cpu.brand_string"])
.decode("utf-8")
.strip()
)
except Exception:
pass
return None


def _memory_total_mb() -> int | None:
try:
if platform.system() == "Linux":
with open("/proc/meminfo") as f:
for line in f:
if line.startswith("MemTotal:"):
match = re.search(r"(\d+)", line)
if match is None:
return None
return int(match.group(1)) // 1024
elif platform.system() == "Darwin":
import subprocess

total_bytes = int(
subprocess.check_output(["sysctl", "-n", "hw.memsize"])
.decode("utf-8")
.strip()
)
return total_bytes // (1024 * 1024)
except Exception:
pass
return None
9 changes: 9 additions & 0 deletions src/maat/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
model_validator,
)

from maat.hardware import HardwareEnvironment
from maat.installation import REPO, this_maat_commit
from maat.utils.shell import join_command, inline_env, add_workdir
from maat.utils.smart_sort import smart_sort_key
Expand Down Expand Up @@ -337,6 +338,7 @@ class Report(BaseModel):
created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
total_execution_time: timedelta
tests: list[TestReport] = []
hardware: list[HardwareEnvironment] = []

@property
def by_version_preferring_scarb(self):
Expand All @@ -361,6 +363,12 @@ def merge(cls, reports: list[Self]) -> Self:
):
raise ValueError(f"cannot merge reports with varying '{field}' values")

merged_hardware: list[HardwareEnvironment] = []
for r in reports:
for h in r.hardware:
if h not in merged_hardware:
merged_hardware.append(h)

return Report(
workspace=reports[0].workspace,
scarb=reports[0].scarb,
Expand All @@ -371,6 +379,7 @@ def merge(cls, reports: list[Self]) -> Self:
(r.total_execution_time for r in reports), timedelta()
),
tests=[t for r in reports for t in r.tests],
hardware=merged_hardware,
)

def before_save(self):
Expand Down
4 changes: 4 additions & 0 deletions src/maat/report/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

from pydantic import BaseModel

from maat.hardware import HardwareEnvironment
from maat.model import Report, ReportMeta


Expand All @@ -19,6 +20,8 @@ class Metrics(BaseModel):
total_projects: int
"""Total number of projects tested in the experiment."""

hardware: list[HardwareEnvironment]

mean_build_time: timedelta | None
mean_lint_time: timedelta | None
mean_test_time: timedelta | None
Expand Down Expand Up @@ -92,6 +95,7 @@ def compute(cls, report: Report, meta: ReportMeta) -> Self:
created_at=report.created_at,
total_execution_time=report.total_execution_time,
total_projects=len(report.tests),
hardware=report.hardware,
mean_build_time=mean_build_time,
mean_lint_time=mean_lint_time,
mean_test_time=mean_test_time,
Expand Down
Loading
Loading