Skip to content
Draft
4 changes: 3 additions & 1 deletion configs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,13 @@ benchmarks:
- name: "Benchmark Name"
description: "What this benchmark tests"
variables:
- type: payload|node_type|num_blocks|gas_limit
- type: payload|node_type|num_blocks|gas_limit|target_gps|consensus_timing
value: single-value
values: [array, of, values] # for matrix testing
```

`consensus_timing` can be `prevent-late-fcu` or `base-consensus`. Snapshot load-test runs default to `base-consensus`; other benchmark runs default to `prevent-late-fcu`.

## 🎯 Choosing the Right Configuration

- **Development/Testing**: Use `examples/` configurations for focused testing
Expand Down
20 changes: 20 additions & 0 deletions configs/examples/load-test-config.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
transaction_submission_rpcs:
- "http://localhost:8545"
query_rpc: "http://localhost:8545"
txpool_nodes: []
flashblocks_ws: "ws://localhost:7111"

sender_count: 10
target_gps: 500000000
duration: "60s"
funding_amount: "10000000000000000000"

transactions:
- weight: 70
type: transfer
- weight: 20
type: calldata
max_size: 256
- weight: 10
type: precompile
target: sha256
14 changes: 4 additions & 10 deletions configs/examples/load-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,8 @@ payloads:
- name: Load Test
type: load-test
id: load-test
sender_count: 10
transactions:
- weight: 70
type: transfer
- weight: 20
type: calldata
max_size: 256
- weight: 10
type: precompile
target: sha256
network: devnet
config_file: ./load-test-config.yml

benchmarks:
- variables:
Expand All @@ -27,3 +19,5 @@ benchmarks:
value: 10
- type: gas_limit
value: 1000000000
- type: target_gps
value: 500000000
16 changes: 16 additions & 0 deletions report/src/components/ChartGrid.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,22 @@ function resolveMetricKey(
return key;
}
}

const metricKeys = chartData.flatMap((d) => Object.keys(d.ExecutionMetrics));
for (const key of keys) {
const quantileSuffix = key.match(/(_quantile_\d+(?:_\d+)?)$/)?.[1] ?? "";
const metricPrefix = quantileSuffix
? key.slice(0, -quantileSuffix.length)
: key;
const labeledMetricKey = metricKeys.find(
(metricKey) =>
metricKey.startsWith(`${metricPrefix}_`) &&
(!quantileSuffix || metricKey.endsWith(quantileSuffix)),
);
if (labeledMetricKey) {
return labeledMetricKey;
}
}
return primaryKey;
}

Expand Down
87 changes: 66 additions & 21 deletions report/src/components/ConfigurationTags.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,61 @@ interface ConfigurationTagsProps {
className?: string;
}

const CONFIG_LABELS: Record<string, string> = {
BlockTimeMilliseconds: "Block Time",
ConsensusTimingMode: "Consensus Timing",
GasLimit: "Gas Limit",
NodeType: "Node Type",
TargetGPS: "Target Gas/s",
TransactionPayload: "Transaction Payload",
ValidatorNodeType: "Validator Node Type",
};

const CONFIG_ORDER = [
"TargetGPS",
"GasLimit",
"BlockTimeMilliseconds",
"ConsensusTimingMode",
"NodeType",
"ValidatorNodeType",
"TransactionPayload",
"Roles",
];

const configLabel = (key: string): string =>
CONFIG_LABELS[key] ?? camelToTitleCase(key);

const configValue = (key: string, value: unknown): string => {
if (key === "GasLimit") {
return formatValue(Number(value), "gas");
}
if (key === "TargetGPS") {
return formatValue(Number(value), "gas/s");
}
if (key === "BlockTimeMilliseconds") {
return formatValue(Number(value), "ms");
}
return String(formatLabel(`${value}`));
};

const configEntries = (testConfig: Record<string, unknown>) =>
Object.entries(testConfig || {})
.filter(([key, value]) => key !== "BenchmarkRun" && value !== "")
.sort(([a], [b]) => {
const aIndex = CONFIG_ORDER.indexOf(a);
const bIndex = CONFIG_ORDER.indexOf(b);
if (aIndex === -1 && bIndex === -1) {
return a.localeCompare(b);
}
if (aIndex === -1) {
return 1;
}
if (bIndex === -1) {
return -1;
}
return aIndex - bIndex;
});

const ConfigurationTags = ({
testConfig,
clientVersion,
Expand All @@ -25,28 +80,18 @@ const ConfigurationTags = ({
<span className="font-mono">{clientVersion}</span>
</span>
)}
{Object.entries(testConfig || {})
.filter(([k]) => k !== "BenchmarkRun" && k !== "GasLimit")
.map(([key, value]) => (
<span
key={key}
title={`${camelToTitleCase(key)}: ${value}`}
className="inline-flex items-center rounded-md bg-slate-50 px-2 py-1 text-xs text-slate-700 ring-1 ring-inset ring-slate-500/10"
>
<span className="mr-1.5 text-slate-500 font-normal">
{camelToTitleCase(key)}:
</span>
{key === "GasLimit" ? (
<span className="font-mono">
{formatValue(Number(value), "gas")}
</span>
) : (
<span className="font-mono">
{String(formatLabel(`${value}`))}
</span>
)}
{configEntries(testConfig).map(([key, value]) => (
<span
key={key}
title={`${configLabel(key)}: ${configValue(key, value)}`}
className="inline-flex items-center rounded-md bg-slate-50 px-2 py-1 text-xs text-slate-700 ring-1 ring-inset ring-slate-500/10"
>
<span className="mr-1.5 text-slate-500 font-normal">
{configLabel(key)}:
</span>
))}
<span className="font-mono">{configValue(key, value)}</span>
</span>
))}
</div>
);
};
Expand Down
6 changes: 5 additions & 1 deletion report/src/components/RunList.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -291,9 +291,13 @@ const RunList = ({
const statusCounts = groupBy(section.runs, "status");
const sortedRuns = isExpanded ? sortRuns(section.runs) : section.runs;
const gasLimit = Number(section.runs?.[0]?.testConfig?.GasLimit);
const targetGasPerSecond = Number(
section.runs?.[0]?.testConfig?.TargetGPS,
);
const blockTimeMilliseconds =
Number(section.runs?.[0]?.testConfig?.BlockTimeMilliseconds) || 2000;
const gasPerSecond = gasLimit / (blockTimeMilliseconds / 1000);
const gasPerSecond =
targetGasPerSecond || gasLimit / (blockTimeMilliseconds / 1000);

return (
<div key={section.key} className="mb-4">
Expand Down
177 changes: 177 additions & 0 deletions report/src/metricDefinitions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,29 @@ export const CHART_CONFIG = {
unit: "s",
aliases: ["reth_op_rbuilder_state_root_calculation_duration"],
},
reth_base_builder_state_root_calculation_duration_quantile_0_5: {
type: "line",
title: "Builder State Root Calculation Duration p50",
description: "p50 time taken to calculate the state root",
unit: "s",
},
reth_base_builder_state_root_calculation_duration_quantile_0_9: {
type: "line",
title: "Builder State Root Calculation Duration p90",
description: "p90 time taken to calculate the state root",
unit: "s",
},
reth_base_builder_state_root_calculation_duration_quantile_0_99: {
type: "line",
title: "Builder State Root Calculation Duration p99",
description: "p99 time taken to calculate the state root",
unit: "s",
},
reth_base_builder_state_root_time_per_gas_ratio_quantile_0_9: {
type: "line",
title: "Builder State Root Time per Gas p90",
description: "p90 state-root calculation time divided by gas processed",
},
reth_base_builder_sequencer_tx_duration_avg: {
type: "line",
title: "Builder Sequencer Tx Duration",
Expand Down Expand Up @@ -224,6 +247,135 @@ export const CHART_CONFIG = {
description: "Average gas headroom percentage across flashblocks",
unit: "count",
},
reth_storage_providers_database_save_blocks_total_quantile_0_9: {
type: "line",
title: "Save Blocks Total p90",
description: "p90 total database save-blocks duration",
unit: "s",
},
reth_storage_providers_database_save_blocks_block_count_last: {
type: "line",
title: "Save Blocks Block Count",
description:
"Number of blocks included in the most recent save-blocks operation",
unit: "blocks",
},
reth_storage_providers_database_save_blocks_commit_sf_quantile_0_9: {
type: "line",
title: "Save Blocks Static File Commit p90",
description: "p90 static-file commit duration during save-blocks",
unit: "s",
},
reth_storage_providers_database_save_blocks_commit_mdbx_quantile_0_9: {
type: "line",
title: "Save Blocks MDBX Commit p90",
description: "p90 MDBX commit duration during save-blocks",
unit: "s",
},
reth_storage_providers_database_save_blocks_write_state_quantile_0_9: {
type: "line",
title: "Save Blocks Write State p90",
description: "p90 state write duration during save-blocks",
unit: "s",
},
reth_storage_providers_database_save_blocks_write_hashed_state_quantile_0_9: {
type: "line",
title: "Save Blocks Write Hashed State p90",
description: "p90 hashed-state write duration during save-blocks",
unit: "s",
},
reth_storage_providers_database_save_blocks_write_trie_updates_quantile_0_9: {
type: "line",
title: "Save Blocks Write Trie Updates p90",
description: "p90 trie-update write duration during save-blocks",
unit: "s",
},
reth_storage_providers_database_save_blocks_sf_quantile_0_9: {
type: "line",
title: "Save Blocks Static Files p90",
description: "p90 static-file save-blocks duration",
unit: "s",
},
reth_trie_leaves_added_quantile_0_9: {
type: "line",
title: "Trie Leaves Added p90",
description: "p90 trie leaves added",
unit: "count",
},
reth_trie_branches_added_quantile_0_9: {
type: "line",
title: "Trie Branches Added p90",
description: "p90 trie branches added",
unit: "count",
},
reth_tree_root_sparse_trie_total_duration_histogram_quantile_0_9: {
type: "line",
title: "Sparse Trie Total Duration p90",
description: "p90 sparse-trie total duration",
unit: "s",
aliases: ["reth_tree_root_sparse_trie_total_duration_histogram"],
},
reth_tree_root_sparse_trie_final_update_duration_histogram_quantile_0_9: {
type: "line",
title: "Sparse Trie Final Update Duration p90",
description: "p90 sparse-trie final update duration",
unit: "s",
aliases: ["reth_tree_root_sparse_trie_final_update_duration_histogram"],
},
reth_parallel_sparse_trie_subtrie_hash_update_latency_quantile_0_9: {
type: "line",
title: "Sparse Trie Subtrie Hash Update p90",
description: "p90 subtrie hash update latency",
unit: "s",
},
reth_parallel_sparse_trie_subtrie_upper_hash_latency_quantile_0_9: {
type: "line",
title: "Sparse Trie Subtrie Upper Hash p90",
description: "p90 subtrie upper-hash latency",
unit: "s",
},
reth_trie_proof_task_storage_worker_idle_time_seconds_quantile_0_9: {
type: "line",
title: "Trie Proof Storage Worker Idle p90",
description: "p90 trie-proof storage worker idle time",
unit: "s",
},
reth_trie_proof_task_account_worker_idle_time_seconds_quantile_0_9: {
type: "line",
title: "Trie Proof Account Worker Idle p90",
description: "p90 trie-proof account worker idle time",
unit: "s",
},
reth_trie_proof_task_blinded_storage_nodes_quantile_0_9: {
type: "line",
title: "Trie Proof Blinded Storage Nodes p90",
description: "p90 blinded storage nodes handled by trie proof tasks",
unit: "count",
},
reth_trie_proof_task_blinded_account_nodes_quantile_0_9: {
type: "line",
title: "Trie Proof Blinded Account Nodes p90",
description: "p90 blinded account nodes handled by trie proof tasks",
unit: "count",
},
reth_trie_cursor_overall_duration_quantile_0_9: {
type: "line",
title: "Trie Cursor Overall Duration p90",
description: "p90 trie cursor overall duration",
unit: "s",
},
reth_trie_hashed_cursor_overall_duration_quantile_0_9: {
type: "line",
title: "Trie Hashed Cursor Overall Duration p90",
description: "p90 hashed trie cursor overall duration",
unit: "s",
},
reth_db_freelist: {
type: "line",
title: "MDBX Freelist",
description: "MDBX freelist size",
unit: "count",
},
reth_sync_state_provider_total_storage_fetch_latency_avg: {
type: "line",
title: "Validator Storage Load Latency",
Expand Down Expand Up @@ -268,6 +420,31 @@ const CHART_CONFIG_ORDER: (keyof typeof CHART_CONFIG)[] = [
"chain/storage/commits.50-percentile",
"chain/snapshot/commits.50-percentile",
"chain/triedb/commits.50-percentile",
"reth_base_builder_state_root_calculation_duration_quantile_0_5",
"reth_base_builder_state_root_calculation_duration_quantile_0_9",
"reth_base_builder_state_root_calculation_duration_quantile_0_99",
"reth_base_builder_state_root_time_per_gas_ratio_quantile_0_9",
"reth_storage_providers_database_save_blocks_total_quantile_0_9",
"reth_storage_providers_database_save_blocks_block_count_last",
"reth_storage_providers_database_save_blocks_commit_sf_quantile_0_9",
"reth_storage_providers_database_save_blocks_commit_mdbx_quantile_0_9",
"reth_storage_providers_database_save_blocks_write_state_quantile_0_9",
"reth_storage_providers_database_save_blocks_write_hashed_state_quantile_0_9",
"reth_storage_providers_database_save_blocks_write_trie_updates_quantile_0_9",
"reth_storage_providers_database_save_blocks_sf_quantile_0_9",
"reth_trie_leaves_added_quantile_0_9",
"reth_trie_branches_added_quantile_0_9",
"reth_tree_root_sparse_trie_total_duration_histogram_quantile_0_9",
"reth_tree_root_sparse_trie_final_update_duration_histogram_quantile_0_9",
"reth_parallel_sparse_trie_subtrie_hash_update_latency_quantile_0_9",
"reth_parallel_sparse_trie_subtrie_upper_hash_latency_quantile_0_9",
"reth_trie_proof_task_storage_worker_idle_time_seconds_quantile_0_9",
"reth_trie_proof_task_account_worker_idle_time_seconds_quantile_0_9",
"reth_trie_proof_task_blinded_storage_nodes_quantile_0_9",
"reth_trie_proof_task_blinded_account_nodes_quantile_0_9",
"reth_trie_cursor_overall_duration_quantile_0_9",
"reth_trie_hashed_cursor_overall_duration_quantile_0_9",
"reth_db_freelist",
];

export const SORTED_CHART_CONFIG: [string, ChartConfig][] = Object.entries(
Expand Down
Loading
Loading