diff --git a/.dvc/config b/.dvc/config index a3e24c66755e..a78ffcb90608 100644 --- a/.dvc/config +++ b/.dvc/config @@ -4,3 +4,6 @@ ['remote "storage"'] url = s3://therock-dvc/rocm-libraries allow_anonymous_login = true +['remote "golden-data"'] + url = s3://therock-dvc/rocm-libraries/hipdnn/golden-data + allow_anonymous_login = true diff --git a/dnn-providers/integration-tests/integration_test_bundles/README.md b/dnn-providers/integration-tests/integration_test_bundles/README.md index 9a8670122243..048da0111c34 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/README.md +++ b/dnn-providers/integration-tests/integration_test_bundles/README.md @@ -15,10 +15,72 @@ Binary data is stored in S3 via [DVC](https://dvc.org) — git only tracks small | Key | Value | |----------------|--------------------------------------| -| Remote | `s3://therock-dvc/rocm-libraries` | +| Default remote | `storage` → `s3://therock-dvc/rocm-libraries` (legacy shared store; existing ops) | +| hipDNN remote | `golden-data` → `s3://therock-dvc/rocm-libraries/hipdnn/golden-data` (new ops, per-op subdirs) | | Tracking | Per-bundle or per-sweep-case `.dvc` tracking (`.json` stays in git for PR review) | | Naming spec | [RFC 0011 Section 4.1](../../../projects/hipdnn/docs/rfcs/0011_GoldenReferenceValidation.md) | +### DVC Remote Layout (`golden-data`) + +New hipDNN golden data uses a single `golden-data` remote rooted at +`s3://therock-dvc/rocm-libraries/hipdnn/golden-data`. DVC stores objects +content-addressed under `/files/md5/`, so all ops on this remote share +one store: + +``` +s3://therock-dvc/rocm-libraries/hipdnn/golden-data/files/md5// +``` + +**Per-op organization is in the git tree, not in S3.** Each op is a top-level +folder under `integration_test_bundles/{Tier}/` (e.g. `SdpaFwd`, `BatchnormFwdInference`). +Push/pull is **scoped by repo path** so you operate on one op at a time, even +though the bytes all land in the same content-addressed S3 store: + +```bash +# push only SDPA-forward bundles, to the golden-data remote: +dvc push -r golden-data --recursive \ + dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd \ + dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd +``` + +**Each pointer declares its own remote (so the default `dvc pull` just works).** +A bundle on the `golden-data` remote sets a per-output `remote:` key in its +`.tensors.dvc`: + +```yaml +outs: +- path: Small.tensor0.bin + remote: golden-data # <- DVC fetches/pushes this output from golden-data + md5: 437ad9f6e2b1db28ca900503e09f7a63 + size: 524288 + hash: md5 +``` + +With this key, a **bare `dvc pull` (no `-r`)** pulls golden-data outputs from +golden-data and everything else from the default `storage` remote — in one pass. +This is why **CI needs no change**: the existing `dvc pull` step already fetches +golden-data bundles. The remote travels *with the data pointer*, not in a separate +workflow file every consumer must remember to flag. + +**Why a single remote (not one remote per op):** +- DVC remotes are global names in `.dvc/config`; a `.tensors.dvc` pointer records + md5/size plus an optional per-output `remote:`. The pull/push location is that + per-output remote (falling back to the repo default). +- One remote + the per-output `remote:` key keeps **CI to a single bare `dvc pull`**, + and every new op is just a new folder — no `.dvc/config` edit, no CI change. +- Per-op isolation is **logical** (git folder taxonomy + path-scoped push), not a + separate S3 keyspace. Cross-op dedup is not expected (different ops produce + different tensors), so a shared store under the root is acceptable. +- Trade-off: because all ops share `…/golden-data/files/md5/`, you cannot prune one + op's S3 objects in isolation with `dvc gc --cloud` (gc keys off what the repo + references across the whole remote). If a future op needs independent S3 + lifecycle / gc, promote it to its own remote (`…/golden-data/`) at that time. + +> The legacy `storage` remote (bare `rocm-libraries` root) remains the default. +> Existing ops (e.g. Batchnorm) keep working unchanged; their pointers have no +> `remote:` key so they resolve to `storage`. New hipDNN ops add `remote: golden-data` +> to their pointers (and push with `-r golden-data`). + ## Folder Convention Single-graph bundle: @@ -70,14 +132,18 @@ verification. ```bash # From the repo root: -# Pull all bundles +# Pull everything. A bare pull resolves each output to its own remote: +# golden-data outputs come from golden-data (via the per-output 'remote:' key), +# all other outputs from the default 'storage' remote. dvc pull -# Pull only quick-tier bundles (sufficient for smoke tests) -dvc pull dnn-providers/integration-tests/integration_test_bundles/quick/ +# Pull only one op's quick-tier bundles (sufficient for smoke tests) +dvc pull dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd ``` -The Linux CI workflow (`therock-ci-linux.yml`) runs `dvc pull` automatically. Other pipelines (Windows, superbuild) may need it wired in separately — see backlog. +CI needs no special flag: the existing `dvc pull` step in `therock-ci-linux.yml` +already fetches golden-data bundles, because each golden-data pointer carries its +own `remote: golden-data` key (see the DVC Remote Layout section above). > **Note:** DVC commands must be run from the repo root (`rocm-libraries/`), not from a subdirectory. @@ -93,19 +159,59 @@ cp resnet50_layer3.tensor*.bin dnn-providers/integration-tests/integration_test_ # 3. Author a single per-bundle DVC pointer listing every .bin, then let DVC fill in the hashes. # DVC cannot generate a multi-file pointer itself, so we write the `outs:` list and `dvc commit`. +# For NEW hipDNN ops, tag each output with `remote: golden-data` so a bare +# `dvc pull` (and CI) fetches it from the golden-data remote. BUNDLE=dnn-providers/integration-tests/integration_test_bundles/quick/ConvFwd/nhwc/fp16/resnet50_layer3 -{ echo "outs:"; for f in "$BUNDLE"/*.tensor*.bin; do echo "- path: $(basename "$f")"; done; } > "$BUNDLE/resnet50_layer3.tensors.dvc" +{ echo "outs:"; for f in "$BUNDLE"/*.tensor*.bin; do + echo "- path: $(basename "$f")"; echo " remote: golden-data"; done; } \ + > "$BUNDLE/resnet50_layer3.tensors.dvc" dvc commit -f "$BUNDLE/resnet50_layer3.tensors.dvc" # 4. Git-add the .json (for PR review) and the .tensors.dvc pointer git add "$BUNDLE/resnet50_layer3.json" "$BUNDLE/resnet50_layer3.tensors.dvc" # 5. Commit and push +# NEW hipDNN ops push to the golden-data remote; legacy 'storage' ops omit -r. git commit -m "Add ConvFwd resnet50_layer3 bundle" -dvc push +dvc push -r golden-data # or: dvc push (legacy 'storage' remote) +git push +``` + +### SDPA forward bundles (regen-driven) + +SDPA-forward bundles are produced by a generator, not copied in. Regenerate, +record pointers, and push to `golden-data`: + +```bash +# 1. Regenerate all SDPA-fwd bundles (quick + standard tiers) +cd dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd +bash generate_golden_data.sh # Tier A (bf16/fp16, nomask/causal, hd128/hd192) +GENERATE_TIER_B=1 bash generate_golden_data.sh # + FP8/GROUP (not yet CI-validated) +cd - + +# 2. Record .bin hashes into the per-bundle pointers + cache (from repo root). +# `dvc commit` preserves the existing per-output `remote: golden-data` key, so +# pointers stay routed to golden-data across regenerations. (For a brand-new +# bundle dir, add the key once — see step 3 of "Add a New Bundle".) +dvc commit -f --recursive \ + dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd \ + dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd + +# 3. Push tensor data to the golden-data remote +dvc push -r golden-data --recursive \ + dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd \ + dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd + +# 4. Stage .json + .meta.json + .tensors.dvc (NOT .bin — gitignored), commit, git push +git add dnn-providers/integration-tests/integration_test_bundles/{quick,standard}/SdpaFwd +git commit -m "Update SDPA forward golden bundles" git push ``` +> `dvc push` uploads from the **local cache**, not the working tree — always +> `dvc commit` (or `dvc add`) first so regenerated `.bin` are cached, or push will +> silently skip them. + ## Update an Existing Bundle ```bash diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.json new file mode 100644 index 000000000000..cc20166cc990 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.meta.json new file mode 100644 index 000000000000..f7830312d9ae --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:25Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..6769dcf88303 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch/Small/Small.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: 437ad9f6e2b1db28ca900503e09f7a63 + size: 524288 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: d7ebfe7c68541eb2b0583dec8d626700 + size: 524288 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: 3c6f4153c81aeb2ec5fc0f16f6f5905b + size: 524288 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: 8092b6d5e8f521aa0e063f6e567a0b07 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.json new file mode 100644 index 000000000000..5f06e236a481 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.json @@ -0,0 +1,158 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": 5, + "seq_len_kv_tensor_uid": 6, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 5, + "name": "SeqLenQ", + "dims": [ + 4 + ], + "strides": [ + 1 + ], + "data_type": "int32", + "virtual": false + }, + { + "uid": 6, + "name": "SeqLenKv", + "dims": [ + 4 + ], + "strides": [ + 1 + ], + "data_type": "int32", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.meta.json new file mode 100644 index 000000000000..b657ae3f81c2 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.meta.json @@ -0,0 +1,44 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:40Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 3, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": true, + "seq_lens_q": [ + 256, + 384, + 512 + ], + "seq_lens_kv": [ + 256, + 384, + 512 + ], + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.tensors.dvc new file mode 100644 index 000000000000..7234b40d4ef0 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_causal_group/Small/Small.tensors.dvc @@ -0,0 +1,31 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: e1aab7b90300368b26ea8442792606e0 + size: 1572864 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: a5e4e0008005c7846af3434cbae2ef90 + size: 1572864 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: 2720d2fe20959b0b0cca67cd91847650 + size: 1572864 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: 107ee8861efae26a374e644a79424763 + size: 1572864 + hash: md5 +- path: Small.tensor5.bin + remote: golden-data + md5: b8852f60c69d1ec2ae2bc2218511246f + size: 16 + hash: md5 +- path: Small.tensor6.bin + remote: golden-data + md5: b8852f60c69d1ec2ae2bc2218511246f + size: 16 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.meta.json index ad90b25e7dd4..0d88edfc299d 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "9ba091222729bf4cf25dbf8755b701f9a9da41ef453ac72e14260a8e880767fd", - "generated_at": "2026-06-11T06:08:15Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:15Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.0", + "generator_version": "1.1.0", "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "bf16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.tensors.dvc index ef4ba189b7cd..4db2fb495a0b 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Small/Small.tensors.dvc @@ -1,17 +1,21 @@ outs: - path: Small.tensor0.bin + remote: golden-data md5: 437ad9f6e2b1db28ca900503e09f7a63 size: 524288 hash: md5 - path: Small.tensor1.bin + remote: golden-data md5: d7ebfe7c68541eb2b0583dec8d626700 size: 524288 hash: md5 - path: Small.tensor2.bin + remote: golden-data md5: 3c6f4153c81aeb2ec5fc0f16f6f5905b size: 524288 hash: md5 - path: Small.tensor3.bin + remote: golden-data md5: 7fdc2d4e76c1d67e99776901b1f0c1d1 size: 524288 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json index a575d85ccca8..838d5ff945a1 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "8b4f68d801785e22c616a1b24506cee24461a8270ea17dc3a0a0c8277a113d98", - "generated_at": "2026-06-16T01:29:30Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:17Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "bf16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": true, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc index 29bac4717cde..5f1d13e41e1f 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc @@ -1,21 +1,26 @@ outs: - path: SmallStats.tensor0.bin + remote: golden-data md5: 437ad9f6e2b1db28ca900503e09f7a63 size: 524288 hash: md5 - path: SmallStats.tensor1.bin + remote: golden-data md5: d7ebfe7c68541eb2b0583dec8d626700 size: 524288 hash: md5 - path: SmallStats.tensor2.bin + remote: golden-data md5: 3c6f4153c81aeb2ec5fc0f16f6f5905b size: 524288 hash: md5 - path: SmallStats.tensor3.bin + remote: golden-data md5: 7fdc2d4e76c1d67e99776901b1f0c1d1 size: 524288 hash: md5 - path: SmallStats.tensor4.bin + remote: golden-data md5: c297ba01911c2fba4111ff9d3dff439e size: 8192 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.json new file mode 100644 index 000000000000..e2ddcc43ac5d --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.json @@ -0,0 +1,158 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": 5, + "seq_len_kv_tensor_uid": 6, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 5, + "name": "SeqLenQ", + "dims": [ + 4 + ], + "strides": [ + 1 + ], + "data_type": "int32", + "virtual": false + }, + { + "uid": 6, + "name": "SeqLenKv", + "dims": [ + 4 + ], + "strides": [ + 1 + ], + "data_type": "int32", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.meta.json new file mode 100644 index 000000000000..76438fa9f3d4 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.meta.json @@ -0,0 +1,44 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:37Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 3, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": true, + "seq_lens_q": [ + 256, + 384, + 512 + ], + "seq_lens_kv": [ + 256, + 384, + 512 + ], + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.tensors.dvc new file mode 100644 index 000000000000..e84f1ce72750 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group/Small/Small.tensors.dvc @@ -0,0 +1,31 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: e1aab7b90300368b26ea8442792606e0 + size: 1572864 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: a5e4e0008005c7846af3434cbae2ef90 + size: 1572864 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: 2720d2fe20959b0b0cca67cd91847650 + size: 1572864 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: a747b510be04b0886c7094bee9fd10cb + size: 1572864 + hash: md5 +- path: Small.tensor5.bin + remote: golden-data + md5: b8852f60c69d1ec2ae2bc2218511246f + size: 16 + hash: md5 +- path: Small.tensor6.bin + remote: golden-data + md5: b8852f60c69d1ec2ae2bc2218511246f + size: 16 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.json new file mode 100644 index 000000000000..d301b7188133 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.meta.json new file mode 100644 index 000000000000..6f8ca9b6345b --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:33Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..5ebafaaea69b --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch/Small/Small.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: f88938f4ac9dc5edc1c840a2e95645f7 + size: 786432 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: 14455c9584557700487b6b493dbf6c1e + size: 786432 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: d29ea702048ae1595362d3768830da79 + size: 524288 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: 3a6c80e5a46fda4acfbe9aa6fe2e00fc + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.json new file mode 100644 index 000000000000..fccae7e84d54 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.meta.json new file mode 100644 index 000000000000..eb47f9b3b48a --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:31Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..b84b1ab13c85 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Small/Small.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: f88938f4ac9dc5edc1c840a2e95645f7 + size: 786432 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: 14455c9584557700487b6b493dbf6c1e + size: 786432 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: d29ea702048ae1595362d3768830da79 + size: 524288 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: dbccbde1cc4e0c3d1f6fc40bc63bfa83 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.json new file mode 100644 index 000000000000..4fcf9ea760ae --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.meta.json new file mode 100644 index 000000000000..505aa69826c6 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:28Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..8679c59a8b74 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch/Small/Small.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: 30ab9dcacb268b7148cef8cf4691ea9e + size: 524288 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: d7f12f9b1b4a3d6696c32a47ea5d8831 + size: 524288 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: 16cfe14039fdc67ef2c162a43db7334b + size: 524288 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: be6fdd112f2c8cf6b498d95a0ac8e647 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.meta.json index 6480b7045f1a..fb43fb7a84de 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "cf4a41b52f21e39e18b2ab057d530b720ba775ac3a72fab52b4aa97a584105c1", - "generated_at": "2026-06-16T19:32:57Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:20Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "fp16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.tensors.dvc index 0f8abda5acad..300f27aacb4d 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Small/Small.tensors.dvc @@ -1,17 +1,21 @@ outs: - path: Small.tensor0.bin + remote: golden-data md5: 30ab9dcacb268b7148cef8cf4691ea9e size: 524288 hash: md5 - path: Small.tensor1.bin + remote: golden-data md5: d7f12f9b1b4a3d6696c32a47ea5d8831 size: 524288 hash: md5 - path: Small.tensor2.bin + remote: golden-data md5: 16cfe14039fdc67ef2c162a43db7334b size: 524288 hash: md5 - path: Small.tensor3.bin + remote: golden-data md5: fca0a5e3bdd33be1fe38bee4c1ac638f size: 524288 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json index 3f05041b6f2c..3164a910676b 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "cf4a41b52f21e39e18b2ab057d530b720ba775ac3a72fab52b4aa97a584105c1", - "generated_at": "2026-06-16T19:02:30Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:22Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "fp16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": true, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc index 4233b8191678..4d2aa9f6cc4c 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/SmallStats/SmallStats.tensors.dvc @@ -1,21 +1,26 @@ outs: - path: SmallStats.tensor0.bin + remote: golden-data md5: 30ab9dcacb268b7148cef8cf4691ea9e size: 524288 hash: md5 - path: SmallStats.tensor1.bin + remote: golden-data md5: d7f12f9b1b4a3d6696c32a47ea5d8831 size: 524288 hash: md5 - path: SmallStats.tensor2.bin + remote: golden-data md5: 16cfe14039fdc67ef2c162a43db7334b size: 524288 hash: md5 - path: SmallStats.tensor3.bin + remote: golden-data md5: fca0a5e3bdd33be1fe38bee4c1ac638f size: 524288 hash: md5 - path: SmallStats.tensor4.bin + remote: golden-data md5: 84149c1ab4407a750335896c30b86579 size: 8192 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.json new file mode 100644 index 000000000000..67698f897d74 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.meta.json new file mode 100644 index 000000000000..f8c7652c962f --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:39Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..23b64e404959 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch/Small/Small.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: c0d67c88308d79e1db3b34222fecc75f + size: 786432 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: 92086f42cd07e23c6f0170595fe03dde + size: 786432 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: fbdc182047004d879533b1a1d5779287 + size: 524288 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: 5fdd93b20cbf27cdec98001f3f200239 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.json new file mode 100644 index 000000000000..a9f5a50c377e --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 192 + ], + "strides": [ + 196608, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.meta.json new file mode 100644 index 000000000000..c54759d3e2ce --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:36Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..93cb67712f07 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Small/Small.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: c0d67c88308d79e1db3b34222fecc75f + size: 786432 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: 92086f42cd07e23c6f0170595fe03dde + size: 786432 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: fbdc182047004d879533b1a1d5779287 + size: 524288 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: a35f0d75df812354aeff21ecb313de5a + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.json new file mode 100644 index 000000000000..15173ca38c5a --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.json @@ -0,0 +1,170 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": 7, + "descale_k_tensor_uid": 8, + "descale_v_tensor_uid": 9, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 7, + "name": "DescaleQ", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + }, + { + "uid": 8, + "name": "DescaleK", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + }, + { + "uid": 9, + "name": "DescaleV", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.meta.json new file mode 100644 index 000000000000..f4561fd831cf --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:35Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "fp8: FP8_E4M3 inputs (per-tensor descale), FP32 intermediates, BF16 output", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "fp8", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..c50cb51a2d8f --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch/Small/Small.tensors.dvc @@ -0,0 +1,36 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: 4a5727a988d1494ded6727894fdd2000 + size: 262144 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: 4e3d9083faff8c7c0a7222336a0a11ce + size: 262144 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: ec4b881f2d1f89bb56051be751af2380 + size: 262144 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: fca580c023d2678e2734a24733b8c6bb + size: 524288 + hash: md5 +- path: Small.tensor7.bin + remote: golden-data + md5: 13f6012bf7c8730aecc198273677710f + size: 4 + hash: md5 +- path: Small.tensor8.bin + remote: golden-data + md5: d8eab274550542b422327a87d3b6879e + size: 4 + hash: md5 +- path: Small.tensor9.bin + remote: golden-data + md5: e62c91d01e218b64463410f57e859ca5 + size: 4 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.json new file mode 100644 index 000000000000..dc83da8eacd6 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.json @@ -0,0 +1,194 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": 5, + "seq_len_kv_tensor_uid": 6, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": 7, + "descale_k_tensor_uid": 8, + "descale_v_tensor_uid": 9, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 3, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 5, + "name": "SeqLenQ", + "dims": [ + 4 + ], + "strides": [ + 1 + ], + "data_type": "int32", + "virtual": false + }, + { + "uid": 6, + "name": "SeqLenKv", + "dims": [ + 4 + ], + "strides": [ + 1 + ], + "data_type": "int32", + "virtual": false + }, + { + "uid": 7, + "name": "DescaleQ", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + }, + { + "uid": 8, + "name": "DescaleK", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + }, + { + "uid": 9, + "name": "DescaleV", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.meta.json new file mode 100644 index 000000000000..5ab3f91cce94 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.meta.json @@ -0,0 +1,44 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:43Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "fp8: FP8_E4M3 inputs (per-tensor descale), FP32 intermediates, BF16 output", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 3, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "fp8", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": true, + "seq_lens_q": [ + 256, + 384, + 512 + ], + "seq_lens_kv": [ + 256, + 384, + 512 + ], + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.tensors.dvc new file mode 100644 index 000000000000..4cd0d10a517b --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_causal_group/Small/Small.tensors.dvc @@ -0,0 +1,46 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: d377e62db29e24136bb3de0d3b9b47fd + size: 786432 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: fe45b21a3a2227aef6112f796e7a7f38 + size: 786432 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: 33da9220e8b009d8948f772988e014b4 + size: 786432 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: dd096e705eb4e89a937fd1bf13ebf596 + size: 1572864 + hash: md5 +- path: Small.tensor5.bin + remote: golden-data + md5: b8852f60c69d1ec2ae2bc2218511246f + size: 16 + hash: md5 +- path: Small.tensor6.bin + remote: golden-data + md5: b8852f60c69d1ec2ae2bc2218511246f + size: 16 + hash: md5 +- path: Small.tensor7.bin + remote: golden-data + md5: d8eab274550542b422327a87d3b6879e + size: 4 + hash: md5 +- path: Small.tensor8.bin + remote: golden-data + md5: 2743ed4a5f748a9328600126b5db0a74 + size: 4 + hash: md5 +- path: Small.tensor9.bin + remote: golden-data + md5: 3ebb5d7270c0f49a76387dff9c787602 + size: 4 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.json new file mode 100644 index 000000000000..52acc6095d81 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.json @@ -0,0 +1,170 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": 7, + "descale_k_tensor_uid": 8, + "descale_v_tensor_uid": 9, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "fp8_e4m3", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 256, + 128 + ], + "strides": [ + 131072, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 7, + "name": "DescaleQ", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + }, + { + "uid": 8, + "name": "DescaleK", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + }, + { + "uid": 9, + "name": "DescaleV", + "dims": [ + 1 + ], + "strides": [ + 1 + ], + "data_type": "float", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.meta.json b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.meta.json new file mode 100644 index 000000000000..c8beb462eed1 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:32Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "fp8: FP8_E4M3 inputs (per-tensor descale), FP32 intermediates, BF16 output", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "fp8", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.tensors.dvc new file mode 100644 index 000000000000..01a98b308053 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch/Small/Small.tensors.dvc @@ -0,0 +1,36 @@ +outs: +- path: Small.tensor0.bin + remote: golden-data + md5: 4a5727a988d1494ded6727894fdd2000 + size: 262144 + hash: md5 +- path: Small.tensor1.bin + remote: golden-data + md5: 4e3d9083faff8c7c0a7222336a0a11ce + size: 262144 + hash: md5 +- path: Small.tensor2.bin + remote: golden-data + md5: ec4b881f2d1f89bb56051be751af2380 + size: 262144 + hash: md5 +- path: Small.tensor3.bin + remote: golden-data + md5: a00cb8a5d9404b0d7acde86679b13127 + size: 524288 + hash: md5 +- path: Small.tensor7.bin + remote: golden-data + md5: 13f6012bf7c8730aecc198273677710f + size: 4 + hash: md5 +- path: Small.tensor8.bin + remote: golden-data + md5: d8eab274550542b422327a87d3b6879e + size: 4 + hash: md5 +- path: Small.tensor9.bin + remote: golden-data + md5: e62c91d01e218b64463410f57e859ca5 + size: 4 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/generate_golden_data.sh b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/generate_golden_data.sh index c4919b2eb630..7b8d322fc19b 100755 --- a/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/generate_golden_data.sh +++ b/dnn-providers/integration-tests/integration_test_bundles/quick/SdpaFwd/generate_golden_data.sh @@ -24,12 +24,31 @@ fi TIER="${1:-all}" +# DVC remote that hipDNN SDPA golden bundles live on. Each generated bundle's +# .tensors.dvc pointer is written with a per-output `remote:` key set to this, so +# a bare `dvc pull` (and CI) fetches the data from the right place without any +# `-r` flag or workflow change. See ../../README.md ("DVC Remote Layout"). +DVC_REMOTE="golden-data" + generate_bundle() { local outdir="$1" local name="$2" shift 2 mkdir -p "$outdir/$name" python3 "$GENERATOR" --base-filename "$outdir/$name/$name" "$@" + + # Author the per-bundle DVC pointer with the golden-data remote pinned per + # output. DVC fills in md5/size on `dvc commit`; the `remote:` key persists + # across regenerations. This is what routes the bundle to the golden-data + # remote — do not drop it, or CI will fail to pull the data. + local dvc_file="$outdir/$name/$name.tensors.dvc" + { + echo "outs:" + for bin in "$outdir/$name/$name".tensor*.bin; do + echo "- path: $(basename "$bin")" + echo " remote: $DVC_REMOTE" + done + } > "$dvc_file" } # --- quick tier: Small bundles only --- @@ -52,6 +71,26 @@ if [[ "$TIER" == "all" || "$TIER" == "quick" ]]; then OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats" generate_bundle "$OUTDIR" "SmallStats" --stats --dtype fp16 --q-dims 2 4 256 128 --v-dims 2 4 256 128 --seed 42 + # BF16 hd128 bottom-right causal + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/bf16/hd128_causal_batch" + generate_bundle "$OUTDIR" "Small" --causal bottom_right --q-dims 2 4 256 128 --v-dims 2 4 256 128 --seed 42 + + # FP16 hd128 bottom-right causal + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp16/hd128_causal_batch" + generate_bundle "$OUTDIR" "Small" --causal bottom_right --dtype fp16 --q-dims 2 4 256 128 --v-dims 2 4 256 128 --seed 42 + + # BF16 hd192 (D_qk=192, D_v=128), no mask + causal + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch" + generate_bundle "$OUTDIR" "Small" --q-dims 2 4 256 192 --v-dims 2 4 256 128 --seed 42 + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/bf16/hd192_causal_batch" + generate_bundle "$OUTDIR" "Small" --causal bottom_right --q-dims 2 4 256 192 --v-dims 2 4 256 128 --seed 42 + + # FP16 hd192, no mask + causal + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch" + generate_bundle "$OUTDIR" "Small" --dtype fp16 --q-dims 2 4 256 192 --v-dims 2 4 256 128 --seed 42 + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp16/hd192_causal_batch" + generate_bundle "$OUTDIR" "Small" --causal bottom_right --dtype fp16 --q-dims 2 4 256 192 --v-dims 2 4 256 128 --seed 42 + echo "" fi @@ -79,6 +118,54 @@ if [[ "$TIER" == "all" || "$TIER" == "standard" ]]; then generate_bundle "$OUTDIR" "MediumStats" --stats --dtype fp16 --q-dims 2 4 512 128 --v-dims 2 4 512 128 --seed 42 generate_bundle "$OUTDIR" "GqaStats" --stats --dtype fp16 --q-dims 1 8 256 128 --v-dims 1 2 256 128 --seed 42 + # BF16 hd128 bottom-right causal + OUTDIR="$GOLDEN_ROOT/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch" + generate_bundle "$OUTDIR" "Medium" --causal bottom_right --q-dims 2 4 512 128 --v-dims 2 4 512 128 --seed 42 + generate_bundle "$OUTDIR" "Gqa" --causal bottom_right --q-dims 1 8 256 128 --v-dims 1 2 256 128 --seed 42 + + # FP16 hd128 bottom-right causal + OUTDIR="$GOLDEN_ROOT/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch" + generate_bundle "$OUTDIR" "Medium" --causal bottom_right --dtype fp16 --q-dims 2 4 512 128 --v-dims 2 4 512 128 --seed 42 + generate_bundle "$OUTDIR" "Gqa" --causal bottom_right --dtype fp16 --q-dims 1 8 256 128 --v-dims 1 2 256 128 --seed 42 + + # BF16 hd192, no mask + causal + OUTDIR="$GOLDEN_ROOT/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch" + generate_bundle "$OUTDIR" "Medium" --q-dims 2 4 512 192 --v-dims 2 4 512 128 --seed 42 + generate_bundle "$OUTDIR" "Gqa" --q-dims 1 8 256 192 --v-dims 1 2 256 128 --seed 42 + OUTDIR="$GOLDEN_ROOT/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch" + generate_bundle "$OUTDIR" "Medium" --causal bottom_right --q-dims 2 4 512 192 --v-dims 2 4 512 128 --seed 42 + generate_bundle "$OUTDIR" "Gqa" --causal bottom_right --q-dims 1 8 256 192 --v-dims 1 2 256 128 --seed 42 + + # FP16 hd192, no mask + causal + OUTDIR="$GOLDEN_ROOT/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch" + generate_bundle "$OUTDIR" "Medium" --dtype fp16 --q-dims 2 4 512 192 --v-dims 2 4 512 128 --seed 42 + generate_bundle "$OUTDIR" "Gqa" --dtype fp16 --q-dims 1 8 256 192 --v-dims 1 2 256 128 --seed 42 + OUTDIR="$GOLDEN_ROOT/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch" + generate_bundle "$OUTDIR" "Medium" --causal bottom_right --dtype fp16 --q-dims 2 4 512 192 --v-dims 2 4 512 128 --seed 42 + generate_bundle "$OUTDIR" "Gqa" --causal bottom_right --dtype fp16 --q-dims 1 8 256 192 --v-dims 1 2 256 128 --seed 42 + + echo "" +fi + +# --- Tier B: FP8 and GROUP-mode bundles (generator-complete; not yet validated by +# the CPU reference executor, which does not accept FP8 descale or variable-seq-len +# attributes). Generate with GENERATE_TIER_B=1. --- +if [[ "${GENERATE_TIER_B:-0}" == "1" ]]; then + echo "=== Generating Tier B (fp8 + group) ===" + + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp8/hd128_nomask_batch" + generate_bundle "$OUTDIR" "Small" --dtype fp8 --q-dims 2 4 256 128 --v-dims 2 4 256 128 --seed 42 + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp8/hd128_causal_batch" + generate_bundle "$OUTDIR" "Small" --dtype fp8 --causal bottom_right --q-dims 2 4 256 128 --v-dims 2 4 256 128 --seed 42 + + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/bf16/hd128_nomask_group" + generate_bundle "$OUTDIR" "Small" --variable-seq-lens --seq-lens-q 256 384 512 --seq-lens-kv 256 384 512 --q-dims 3 4 512 128 --v-dims 3 4 512 128 --seed 42 + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/bf16/hd128_causal_group" + generate_bundle "$OUTDIR" "Small" --causal bottom_right --variable-seq-lens --seq-lens-q 256 384 512 --seq-lens-kv 256 384 512 --q-dims 3 4 512 128 --v-dims 3 4 512 128 --seed 42 + + OUTDIR="$GOLDEN_ROOT/quick/SdpaFwd/bhsd/fp8/hd128_causal_group" + generate_bundle "$OUTDIR" "Small" --dtype fp8 --causal bottom_right --variable-seq-lens --seq-lens-q 256 384 512 --seq-lens-kv 256 384 512 --q-dims 3 4 512 128 --v-dims 3 4 512 128 --seed 42 + echo "" fi diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.json new file mode 100644 index 000000000000..c189090e7ef7 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.meta.json new file mode 100644 index 000000000000..8e5fbeb60b25 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:04Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 1, + "num_heads_q": 8, + "num_heads_kv": 2, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 4 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.tensors.dvc new file mode 100644 index 000000000000..5556fb4030ff --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Gqa/Gqa.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Gqa.tensor0.bin + remote: golden-data + md5: 437ad9f6e2b1db28ca900503e09f7a63 + size: 524288 + hash: md5 +- path: Gqa.tensor1.bin + remote: golden-data + md5: 4636dc5e6a3bf843491a10bf2622e1fa + size: 131072 + hash: md5 +- path: Gqa.tensor2.bin + remote: golden-data + md5: e738b64c40ea4c96494d2a17d23c3710 + size: 131072 + hash: md5 +- path: Gqa.tensor3.bin + remote: golden-data + md5: fadbff32856fc945064ab731b077a44a + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.json new file mode 100644 index 000000000000..6b17d6ba38de --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.meta.json new file mode 100644 index 000000000000..7196c4eaeeaa --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:01Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.tensors.dvc new file mode 100644 index 000000000000..24f57fdaeb16 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_causal_batch/Medium/Medium.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Medium.tensor0.bin + remote: golden-data + md5: ee90d59a6dd62519f135136a7f48e33a + size: 1048576 + hash: md5 +- path: Medium.tensor1.bin + remote: golden-data + md5: bddf3c62a61c6d1d3e422b0b98365c5c + size: 1048576 + hash: md5 +- path: Medium.tensor2.bin + remote: golden-data + md5: 0121aba659fd1959bcafa1a8889e748f + size: 1048576 + hash: md5 +- path: Medium.tensor3.bin + remote: golden-data + md5: 369bce4b995069969c659319a86fac12 + size: 1048576 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.meta.json index fac35830c6b9..499e876735b5 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "9ba091222729bf4cf25dbf8755b701f9a9da41ef453ac72e14260a8e880767fd", - "generated_at": "2026-06-11T06:08:20Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:44Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.0", + "generator_version": "1.1.0", "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "bf16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, "scale": 0.08838834764831843, "gqa_ratio": 4 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc index 74e737a30f49..c333c9c11257 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc @@ -1,17 +1,21 @@ outs: - path: Gqa.tensor0.bin + remote: golden-data md5: 437ad9f6e2b1db28ca900503e09f7a63 size: 524288 hash: md5 - path: Gqa.tensor1.bin + remote: golden-data md5: 4636dc5e6a3bf843491a10bf2622e1fa size: 131072 hash: md5 - path: Gqa.tensor2.bin + remote: golden-data md5: e738b64c40ea4c96494d2a17d23c3710 size: 131072 hash: md5 - path: Gqa.tensor3.bin + remote: golden-data md5: e187c8b8a481e39f2ad757a485abba31 size: 524288 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.meta.json index 5b9d817b040e..f6a30c6bb442 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "9ba091222729bf4cf25dbf8755b701f9a9da41ef453ac72e14260a8e880767fd", - "generated_at": "2026-06-11T06:08:18Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:41Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.0", + "generator_version": "1.1.0", "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "bf16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.tensors.dvc index b2e7e05952af..30a8c478a8ab 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch/Medium/Medium.tensors.dvc @@ -1,17 +1,21 @@ outs: - path: Medium.tensor0.bin + remote: golden-data md5: ee90d59a6dd62519f135136a7f48e33a size: 1048576 hash: md5 - path: Medium.tensor1.bin + remote: golden-data md5: bddf3c62a61c6d1d3e422b0b98365c5c size: 1048576 hash: md5 - path: Medium.tensor2.bin + remote: golden-data md5: 0121aba659fd1959bcafa1a8889e748f size: 1048576 hash: md5 - path: Medium.tensor3.bin + remote: golden-data md5: 0cc88cdfb130086190b8a414e0e44218 size: 1048576 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json index 64b483a7174e..ef7983867494 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "8b4f68d801785e22c616a1b24506cee24461a8270ea17dc3a0a0c8277a113d98", - "generated_at": "2026-06-16T01:29:33Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:49Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "bf16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": true, "scale": 0.08838834764831843, "gqa_ratio": 4 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc index e2d6ca8f9f8a..1d745db3ef37 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc @@ -1,21 +1,26 @@ outs: - path: GqaStats.tensor0.bin + remote: golden-data md5: 437ad9f6e2b1db28ca900503e09f7a63 size: 524288 hash: md5 - path: GqaStats.tensor1.bin + remote: golden-data md5: 4636dc5e6a3bf843491a10bf2622e1fa size: 131072 hash: md5 - path: GqaStats.tensor2.bin + remote: golden-data md5: e738b64c40ea4c96494d2a17d23c3710 size: 131072 hash: md5 - path: GqaStats.tensor3.bin + remote: golden-data md5: e187c8b8a481e39f2ad757a485abba31 size: 524288 hash: md5 - path: GqaStats.tensor4.bin + remote: golden-data md5: b1fc79acc036bc714616899ac913312d size: 8192 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json index 619ad18d23a2..4b9c806c1eb6 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "8b4f68d801785e22c616a1b24506cee24461a8270ea17dc3a0a0c8277a113d98", - "generated_at": "2026-06-16T01:36:50Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:47Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "bf16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": true, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc index 2013fd4a6d03..0787f0df38eb 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc @@ -1,21 +1,26 @@ outs: - path: MediumStats.tensor0.bin + remote: golden-data md5: ee90d59a6dd62519f135136a7f48e33a size: 1048576 hash: md5 - path: MediumStats.tensor1.bin + remote: golden-data md5: bddf3c62a61c6d1d3e422b0b98365c5c size: 1048576 hash: md5 - path: MediumStats.tensor2.bin + remote: golden-data md5: 0121aba659fd1959bcafa1a8889e748f size: 1048576 hash: md5 - path: MediumStats.tensor3.bin + remote: golden-data md5: 0cc88cdfb130086190b8a414e0e44218 size: 1048576 hash: md5 - path: MediumStats.tensor4.bin + remote: golden-data md5: 4cbf93127453554a6be2c2c0a1423966 size: 16384 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.json new file mode 100644 index 000000000000..b6e0bc875ee8 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 1, + 8, + 256, + 192 + ], + "strides": [ + 393216, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 1, + 2, + 256, + 192 + ], + "strides": [ + 98304, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.meta.json new file mode 100644 index 000000000000..7d47a6661761 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:20Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 1, + "num_heads_q": 8, + "num_heads_kv": 2, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 4 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.tensors.dvc new file mode 100644 index 000000000000..bca5810f97fa --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Gqa/Gqa.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Gqa.tensor0.bin + remote: golden-data + md5: f88938f4ac9dc5edc1c840a2e95645f7 + size: 786432 + hash: md5 +- path: Gqa.tensor1.bin + remote: golden-data + md5: 55db8bc69bb85b350d757a845c7eaccb + size: 196608 + hash: md5 +- path: Gqa.tensor2.bin + remote: golden-data + md5: a49151bb2e4c4377cf08ea51f40c6ce3 + size: 131072 + hash: md5 +- path: Gqa.tensor3.bin + remote: golden-data + md5: a8fd62b0c37ec50b6729f6fd21c46ee5 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.json new file mode 100644 index 000000000000..b8870c1d7470 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.meta.json new file mode 100644 index 000000000000..43a2cbc35ade --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:17Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.tensors.dvc new file mode 100644 index 000000000000..0e748815a837 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_causal_batch/Medium/Medium.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Medium.tensor0.bin + remote: golden-data + md5: e1aab7b90300368b26ea8442792606e0 + size: 1572864 + hash: md5 +- path: Medium.tensor1.bin + remote: golden-data + md5: a5e4e0008005c7846af3434cbae2ef90 + size: 1572864 + hash: md5 +- path: Medium.tensor2.bin + remote: golden-data + md5: 40b952236771e9710ef339d03f6384c4 + size: 1048576 + hash: md5 +- path: Medium.tensor3.bin + remote: golden-data + md5: f6db8e9bc9eb69232f0002fae0edd229 + size: 1048576 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.json new file mode 100644 index 000000000000..6944255cfdec --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 1, + 8, + 256, + 192 + ], + "strides": [ + 393216, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 1, + 2, + 256, + 192 + ], + "strides": [ + 98304, + 49152, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.meta.json new file mode 100644 index 000000000000..ffbb3b48fd74 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:14Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 1, + "num_heads_q": 8, + "num_heads_kv": 2, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 4 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.tensors.dvc new file mode 100644 index 000000000000..83b2f7ce42ae --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Gqa/Gqa.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Gqa.tensor0.bin + remote: golden-data + md5: f88938f4ac9dc5edc1c840a2e95645f7 + size: 786432 + hash: md5 +- path: Gqa.tensor1.bin + remote: golden-data + md5: 55db8bc69bb85b350d757a845c7eaccb + size: 196608 + hash: md5 +- path: Gqa.tensor2.bin + remote: golden-data + md5: a49151bb2e4c4377cf08ea51f40c6ce3 + size: 131072 + hash: md5 +- path: Gqa.tensor3.bin + remote: golden-data + md5: a73a44f92fd83792c580b8702ca62c7e + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.json new file mode 100644 index 000000000000..ec8fa7b250d6 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "bfloat16", + "virtual": false + } + ], + "io_data_type": "bfloat16", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.meta.json new file mode 100644 index 000000000000..e6423c1b5c81 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:12Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: BF16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "bf16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.tensors.dvc new file mode 100644 index 000000000000..e9029d47d74e --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch/Medium/Medium.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Medium.tensor0.bin + remote: golden-data + md5: e1aab7b90300368b26ea8442792606e0 + size: 1572864 + hash: md5 +- path: Medium.tensor1.bin + remote: golden-data + md5: a5e4e0008005c7846af3434cbae2ef90 + size: 1572864 + hash: md5 +- path: Medium.tensor2.bin + remote: golden-data + md5: 40b952236771e9710ef339d03f6384c4 + size: 1048576 + hash: md5 +- path: Medium.tensor3.bin + remote: golden-data + md5: bb31e042869a662542a3d7870c90ac3c + size: 1048576 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.json new file mode 100644 index 000000000000..73690c65e90c --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.meta.json new file mode 100644 index 000000000000..c93c402f8920 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:09Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 1, + "num_heads_q": 8, + "num_heads_kv": 2, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 4 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.tensors.dvc new file mode 100644 index 000000000000..380f4ec1342a --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Gqa/Gqa.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Gqa.tensor0.bin + remote: golden-data + md5: 30ab9dcacb268b7148cef8cf4691ea9e + size: 524288 + hash: md5 +- path: Gqa.tensor1.bin + remote: golden-data + md5: 35cd85d9f881846c8cb5850526159efe + size: 131072 + hash: md5 +- path: Gqa.tensor2.bin + remote: golden-data + md5: afbf0991921855f2df21eb36468bd32c + size: 131072 + hash: md5 +- path: Gqa.tensor3.bin + remote: golden-data + md5: 0231cf9e729217e4d4019b710d353919 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.json new file mode 100644 index 000000000000..2ff4de702eda --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.08838834764831843, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.meta.json new file mode 100644 index 000000000000..fdf554fc7b11 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:07Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 128, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.08838834764831843, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.tensors.dvc new file mode 100644 index 000000000000..1eaaf5ea3c39 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_causal_batch/Medium/Medium.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Medium.tensor0.bin + remote: golden-data + md5: 72fe60f9d265a84788a2a76e7f580116 + size: 1048576 + hash: md5 +- path: Medium.tensor1.bin + remote: golden-data + md5: 4e5354215d1b43dac8d5b60065d2b109 + size: 1048576 + hash: md5 +- path: Medium.tensor2.bin + remote: golden-data + md5: 4b93615b3a40fc7c9dc1cd9b95e4ff5d + size: 1048576 + hash: md5 +- path: Medium.tensor3.bin + remote: golden-data + md5: 178a0d5ca447825dbc4cd967ada19490 + size: 1048576 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.meta.json index 6f1ba30d2d41..413302aee6a0 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "cf4a41b52f21e39e18b2ab057d530b720ba775ac3a72fab52b4aa97a584105c1", - "generated_at": "2026-06-16T19:33:02Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:54Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "fp16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, "scale": 0.08838834764831843, "gqa_ratio": 4 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc index b4c3bcd3d333..5d38a21dc9dd 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Gqa/Gqa.tensors.dvc @@ -1,17 +1,21 @@ outs: - path: Gqa.tensor0.bin + remote: golden-data md5: 30ab9dcacb268b7148cef8cf4691ea9e size: 524288 hash: md5 - path: Gqa.tensor1.bin + remote: golden-data md5: 35cd85d9f881846c8cb5850526159efe size: 131072 hash: md5 - path: Gqa.tensor2.bin + remote: golden-data md5: afbf0991921855f2df21eb36468bd32c size: 131072 hash: md5 - path: Gqa.tensor3.bin + remote: golden-data md5: 6bf7b4590270a9184b18ed6fdb90d2c9 size: 524288 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.meta.json index ffb61b618348..14b40d7f56c6 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "cf4a41b52f21e39e18b2ab057d530b720ba775ac3a72fab52b4aa97a584105c1", - "generated_at": "2026-06-16T19:32:59Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:52Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "fp16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.tensors.dvc index 66e73518ea4c..56ac2e90d8e7 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch/Medium/Medium.tensors.dvc @@ -1,17 +1,21 @@ outs: - path: Medium.tensor0.bin + remote: golden-data md5: 72fe60f9d265a84788a2a76e7f580116 size: 1048576 hash: md5 - path: Medium.tensor1.bin + remote: golden-data md5: 4e5354215d1b43dac8d5b60065d2b109 size: 1048576 hash: md5 - path: Medium.tensor2.bin + remote: golden-data md5: 4b93615b3a40fc7c9dc1cd9b95e4ff5d size: 1048576 hash: md5 - path: Medium.tensor3.bin + remote: golden-data md5: 60bc1daa264e69ba119ec6af4f52f775 size: 1048576 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json index f433d878d197..8c8a6fd2bcaa 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "cf4a41b52f21e39e18b2ab057d530b720ba775ac3a72fab52b4aa97a584105c1", - "generated_at": "2026-06-16T19:02:32Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:59Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "fp16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": true, "scale": 0.08838834764831843, "gqa_ratio": 4 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc index 78fde4367f0f..13cf7241d268 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/GqaStats/GqaStats.tensors.dvc @@ -1,21 +1,26 @@ outs: - path: GqaStats.tensor0.bin + remote: golden-data md5: 30ab9dcacb268b7148cef8cf4691ea9e size: 524288 hash: md5 - path: GqaStats.tensor1.bin + remote: golden-data md5: 35cd85d9f881846c8cb5850526159efe size: 131072 hash: md5 - path: GqaStats.tensor2.bin + remote: golden-data md5: afbf0991921855f2df21eb36468bd32c size: 131072 hash: md5 - path: GqaStats.tensor3.bin + remote: golden-data md5: 6bf7b4590270a9184b18ed6fdb90d2c9 size: 524288 hash: md5 - path: GqaStats.tensor4.bin + remote: golden-data md5: 942b1463be255c8ea542c588fbc92851 size: 8192 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json index 97ac767f63f8..107103829175 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.meta.json @@ -1,11 +1,11 @@ { "generator": "generate_sdpa_fwd_golden.py", - "generator_sha256": "cf4a41b52f21e39e18b2ab057d530b720ba775ac3a72fab52b4aa97a584105c1", - "generated_at": "2026-06-16T19:02:35Z", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:55:57Z", "reference_source": "PyTorch 2.12.0+rocm7.2", "reference_backend": "pytorch_math_backend", "rocm_version": "7.2", - "generator_version": "1.0.1", + "generator_version": "1.1.0", "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", "direction": "forward", "seed": 42, @@ -23,7 +23,13 @@ "head_dim_qk": 128, "head_dim_v": 128, "dtype": "fp16", - "causal": false, + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": true, "scale": 0.08838834764831843, "gqa_ratio": 1 } diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc index d4d6264818d3..5d91d7373b8f 100644 --- a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats/MediumStats/MediumStats.tensors.dvc @@ -1,21 +1,26 @@ outs: - path: MediumStats.tensor0.bin + remote: golden-data md5: 72fe60f9d265a84788a2a76e7f580116 size: 1048576 hash: md5 - path: MediumStats.tensor1.bin + remote: golden-data md5: 4e5354215d1b43dac8d5b60065d2b109 size: 1048576 hash: md5 - path: MediumStats.tensor2.bin + remote: golden-data md5: 4b93615b3a40fc7c9dc1cd9b95e4ff5d size: 1048576 hash: md5 - path: MediumStats.tensor3.bin + remote: golden-data md5: 60bc1daa264e69ba119ec6af4f52f775 size: 1048576 hash: md5 - path: MediumStats.tensor4.bin + remote: golden-data md5: adcd27776566ead59573b906c28680dc size: 16384 hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.json new file mode 100644 index 000000000000..4adc2b7224c2 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 1, + 8, + 256, + 192 + ], + "strides": [ + 393216, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 1, + 2, + 256, + 192 + ], + "strides": [ + 98304, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.meta.json new file mode 100644 index 000000000000..ba49e2eaffff --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:30Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 1, + "num_heads_q": 8, + "num_heads_kv": 2, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 4 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.tensors.dvc new file mode 100644 index 000000000000..7308c4381420 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Gqa/Gqa.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Gqa.tensor0.bin + remote: golden-data + md5: c0d67c88308d79e1db3b34222fecc75f + size: 786432 + hash: md5 +- path: Gqa.tensor1.bin + remote: golden-data + md5: fb17480c4493320c34c971ec88b1f4f9 + size: 196608 + hash: md5 +- path: Gqa.tensor2.bin + remote: golden-data + md5: a1e1363238873c08472af3130c889b47 + size: 131072 + hash: md5 +- path: Gqa.tensor3.bin + remote: golden-data + md5: 6de34e851bd541237a86324274cf00fd + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.json new file mode 100644 index 000000000000..0d8cefd05ebd --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": -1, + "right_bound": 0, + "max_seq_len_kv": null, + "diagonal_alignment": "BOTTOM_RIGHT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.meta.json new file mode 100644 index 000000000000..0177284dfd0a --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:27Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "bottom_right", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.tensors.dvc new file mode 100644 index 000000000000..c0c378e4d71b --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_causal_batch/Medium/Medium.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Medium.tensor0.bin + remote: golden-data + md5: 196c007d8dc952e0e801d5ae57e3bfc2 + size: 1572864 + hash: md5 +- path: Medium.tensor1.bin + remote: golden-data + md5: 1c2bc04d7e78fa1e92908dea59aa8d4d + size: 1572864 + hash: md5 +- path: Medium.tensor2.bin + remote: golden-data + md5: 3be020719b27e9d43a027d6806825556 + size: 1048576 + hash: md5 +- path: Medium.tensor3.bin + remote: golden-data + md5: fa7b60a3fea21583c5e0dd8b25f90a87 + size: 1048576 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.json new file mode 100644 index 000000000000..5611c79e9683 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 1, + 8, + 256, + 192 + ], + "strides": [ + 393216, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 1, + 2, + 256, + 192 + ], + "strides": [ + 98304, + 49152, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 1, + 2, + 256, + 128 + ], + "strides": [ + 65536, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 1, + 8, + 256, + 128 + ], + "strides": [ + 262144, + 32768, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.meta.json new file mode 100644 index 000000000000..7b2125e1f422 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:25Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 1, + "num_heads_q": 8, + "num_heads_kv": 2, + "seq_q": 256, + "seq_kv": 256, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 4 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.tensors.dvc new file mode 100644 index 000000000000..66c944f20e98 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Gqa/Gqa.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Gqa.tensor0.bin + remote: golden-data + md5: c0d67c88308d79e1db3b34222fecc75f + size: 786432 + hash: md5 +- path: Gqa.tensor1.bin + remote: golden-data + md5: fb17480c4493320c34c971ec88b1f4f9 + size: 196608 + hash: md5 +- path: Gqa.tensor2.bin + remote: golden-data + md5: a1e1363238873c08472af3130c889b47 + size: 131072 + hash: md5 +- path: Gqa.tensor3.bin + remote: golden-data + md5: ab8b879d98212e06f10b7a10ad49f5e1 + size: 524288 + hash: md5 diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.json new file mode 100644 index 000000000000..2d3da8cd3551 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.json @@ -0,0 +1,134 @@ +{ + "nodes": [ + { + "type": "SdpaAttributes", + "compute_data_type": "float", + "name": "", + "inputs": { + "q_tensor_uid": 0, + "k_tensor_uid": 1, + "v_tensor_uid": 2, + "attn_mask_tensor_uid": null, + "scale_tensor_uid": null, + "seq_len_q_tensor_uid": null, + "seq_len_kv_tensor_uid": null, + "seed_tensor_uid": null, + "offset_tensor_uid": null, + "dropout_mask_tensor_uid": null, + "dropout_scale_tensor_uid": null, + "page_table_k_tensor_uid": null, + "page_table_v_tensor_uid": null, + "block_mask_tensor_uid": null, + "sink_token_tensor_uid": null, + "descale_q_tensor_uid": null, + "descale_k_tensor_uid": null, + "descale_v_tensor_uid": null, + "descale_s_tensor_uid": null, + "scale_s_tensor_uid": null, + "scale_o_tensor_uid": null + }, + "outputs": { + "o_tensor_uid": 3, + "stats_tensor_uid": null, + "max_tensor_uid": null, + "sum_exp_tensor_uid": null, + "rng_dump_tensor_uid": null, + "amax_s_tensor_uid": null, + "amax_o_tensor_uid": null + }, + "attributes": { + "generate_stats": null, + "alibi_mask": false, + "padding_mask": false, + "causal_mask": false, + "causal_mask_bottom_right": false, + "dropout_probability": null, + "attn_scale_value": 0.07216878364870323, + "left_bound": null, + "right_bound": null, + "max_seq_len_kv": null, + "diagonal_alignment": "TOP_LEFT", + "mma_core_mode": "float", + "implementation": "AUTO" + } + } + ], + "tensors": [ + { + "uid": 0, + "name": "Q", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 1, + "name": "K", + "dims": [ + 2, + 4, + 512, + 192 + ], + "strides": [ + 393216, + 98304, + 192, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 2, + "name": "V", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + }, + { + "uid": 3, + "name": "O", + "dims": [ + 2, + 4, + 512, + 128 + ], + "strides": [ + 262144, + 65536, + 128, + 1 + ], + "data_type": "half", + "virtual": false + } + ], + "io_data_type": "half", + "compute_data_type": "float", + "intermediate_data_type": "float", + "name": "" +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.meta.json b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.meta.json new file mode 100644 index 000000000000..4fae0234f246 --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.meta.json @@ -0,0 +1,36 @@ +{ + "generator": "generate_sdpa_fwd_golden.py", + "generator_sha256": "eaffaeaf20fd4204529d8b1d130271eb91888775e5d52ccc66d865d568f7f5b9", + "generated_at": "2026-06-26T20:56:22Z", + "reference_source": "PyTorch 2.12.0+rocm7.2", + "reference_backend": "pytorch_math_backend", + "rocm_version": "7.2", + "generator_version": "1.1.0", + "generation_precision": "like-for-like: FP16 inputs, FP32 intermediates", + "direction": "forward", + "seed": 42, + "input_range": [ + -1.0, + 1.0 + ], + "deterministic": true, + "config": { + "batch": 2, + "num_heads_q": 4, + "num_heads_kv": 4, + "seq_q": 512, + "seq_kv": 512, + "head_dim_qk": 192, + "head_dim_v": 128, + "dtype": "fp16", + "causal": "none", + "window_left": -1, + "window_right": -1, + "group_mode": false, + "seq_lens_q": null, + "seq_lens_kv": null, + "stats": false, + "scale": 0.07216878364870323, + "gqa_ratio": 1 + } +} diff --git a/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.tensors.dvc b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.tensors.dvc new file mode 100644 index 000000000000..0556f0bb009c --- /dev/null +++ b/dnn-providers/integration-tests/integration_test_bundles/standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch/Medium/Medium.tensors.dvc @@ -0,0 +1,21 @@ +outs: +- path: Medium.tensor0.bin + remote: golden-data + md5: 196c007d8dc952e0e801d5ae57e3bfc2 + size: 1572864 + hash: md5 +- path: Medium.tensor1.bin + remote: golden-data + md5: 1c2bc04d7e78fa1e92908dea59aa8d4d + size: 1572864 + hash: md5 +- path: Medium.tensor2.bin + remote: golden-data + md5: 3be020719b27e9d43a027d6806825556 + size: 1048576 + hash: md5 +- path: Medium.tensor3.bin + remote: golden-data + md5: 39bf4d4b688fff21701ebbc2bcad6eba + size: 1048576 + hash: md5 diff --git a/dnn-providers/integration-tests/reference_data_scripts/generate_sdpa_fwd_golden.py b/dnn-providers/integration-tests/reference_data_scripts/generate_sdpa_fwd_golden.py index 9890bb899f0a..7616d9ff5534 100644 --- a/dnn-providers/integration-tests/reference_data_scripts/generate_sdpa_fwd_golden.py +++ b/dnn-providers/integration-tests/reference_data_scripts/generate_sdpa_fwd_golden.py @@ -5,19 +5,31 @@ SDPA Forward Golden Reference Bundle Generator Generates pre-computed reference data for SDPA forward kernel validation. -Uses PyTorch's SDPBackend.MATH as the golden reference source. -Supports BF16 and FP16 data types. +Uses PyTorch's SDPBackend.MATH (or an explicit FP32 reference) as the golden +source. Bundles are provider-agnostic: they encode the SDPA *operation* (graph +attributes + reference tensors), not a specific provider's kernel. + +Supported, fully composable operation axes: + - dtype: bf16, fp16, fp8 (FP8_E4M3 inputs, BF16 output, per-tensor descale) + - head dim: hd128, hd192 (via --q-dims / --v-dims) + - causal mask: none, top_left, bottom_right + - sliding window: --window-left / --window-right + - GROUP mode: --variable-seq-lens + --seq-lens-q / --seq-lens-kv + - LSE output: --stats (adds tensor uid=4) + +Any combination of the above may be applied in a single invocation (e.g. fp8 + +bottom_right causal + group + stats), matching the 12 unique AITER forward tuples. Output: {base_filename}.json + {base_filename}.tensor{uid}.bin + {base_filename}.meta.json Usage: python generate_sdpa_fwd_golden.py \ - --base-filename golden_data/SdpaFwd/bf16_hd128_nomask_batch/Small \ + --base-filename golden_data/SdpaFwd/bf16/hd128_nomask_batch/Small \ --q-dims 2 4 256 128 --v-dims 2 4 256 128 --seed 42 python generate_sdpa_fwd_golden.py \ - --base-filename golden_data/SdpaFwd/fp16_hd128_nomask_batch/Small \ - --q-dims 2 4 256 128 --v-dims 2 4 256 128 --dtype fp16 --seed 42 + --base-filename golden_data/SdpaFwd/bf16/hd128_causal_batch/Small \ + --q-dims 2 4 256 128 --v-dims 2 4 256 128 --causal bottom_right --seed 42 """ import argparse @@ -28,7 +40,6 @@ import os import subprocess import sys -from pathlib import Path import torch import torch.nn.functional as F @@ -38,13 +49,35 @@ # (e.g., different reference backend, precision handling, tensor layout) # 1.0.0 — Initial forward generator (Q, K, V, O tensors) # 1.0.1 — Added optional LSE output tensor (uid=4) via --stats flag -GENERATOR_VERSION = "1.0.1" +# 1.1.0 — Added causal/window masking, FP8 inputs, and GROUP (variable-seq-len) mode +GENERATOR_VERSION = "1.1.0" DTYPE_MAP = { "bf16": {"torch": torch.bfloat16, "json": "bfloat16", "bytes": 2}, "fp16": {"torch": torch.float16, "json": "half", "bytes": 2}, } +# FP8 operation: FP8_E4M3 inputs (Q/K/V), BF16 output. AITER's fmha fp8 path uses +# the OCP e4m3 encoding (float8_e4m3fn) with a per-tensor scalar descale; mirror it. +# The "fp8_e4m3" json name matches the flatbuffers DataType enum string map. +FP8_TORCH_DTYPE = torch.float8_e4m3fn +FP8_JSON_DTYPE = "fp8_e4m3" +FP8_OUTPUT_TORCH = torch.bfloat16 +FP8_OUTPUT_JSON = "bfloat16" +FP8_E4M3_MAX = 448.0 # max representable magnitude of float8_e4m3fn + +# Tensor UID assignment (stable contract with the JSON consumer / loader). +UID_Q = 0 +UID_K = 1 +UID_V = 2 +UID_O = 3 +UID_LSE = 4 +UID_SEQ_LEN_Q = 5 +UID_SEQ_LEN_KV = 6 +UID_DESCALE_Q = 7 +UID_DESCALE_K = 8 +UID_DESCALE_V = 9 + def compute_contiguous_strides(dims): strides = [] @@ -56,23 +89,70 @@ def compute_contiguous_strides(dims): return strides -def compute_forward(Q, K, V, scale, H_q, H_kv): - """SDPA forward via PyTorch Math backend (half->FP32->half).""" +def build_mask(s_q, s_kv, causal, window_left, window_right): + """Construct a boolean keep-mask [S_q, S_kv] for the given causal/window config. + + Returns None when no masking applies (the dense attention case), so callers + can take the fast PyTorch path. The mask is True where attention is allowed. + The bound math matches the CPU reference executor's leftBound/rightBound + + topLeftAlignment semantics so golden output is reproducible on the CPU side. + + causal: + - "none": no causal constraint + - "top_left": diagonal anchored at (0,0); keep skv <= sq + - "bottom_right": diagonal anchored bottom-right; keep skv <= sq + (S_kv - S_q) + """ + has_causal = causal != "none" + has_window = window_left >= 0 or window_right >= 0 + if not has_causal and not has_window: + return None + + row = torch.arange(s_q).unsqueeze(1) # [S_q, 1] + col = torch.arange(s_kv).unsqueeze(0) # [1, S_kv] + # Bottom-right alignment shifts the diagonal by (S_kv - S_q). + offset = 0 if causal != "bottom_right" else (s_kv - s_q) + + mask = torch.ones(s_q, s_kv, dtype=torch.bool) + if has_causal: + mask &= col <= (row + offset) + if window_left >= 0: + mask &= (row + offset - col) <= window_left + if window_right >= 0: + mask &= (col - (row + offset)) <= window_right + return mask + + +def _additive_mask(keep_mask): + """Convert a boolean keep-mask to an additive FP32 mask (0 keep, -inf drop).""" + additive = torch.zeros(keep_mask.shape, dtype=torch.float32) + additive.masked_fill_(~keep_mask, float("-inf")) + return additive + + +def compute_forward(Q, K, V, scale, H_q, H_kv, keep_mask=None): + """SDPA forward via PyTorch Math backend (half->FP32->half). + + keep_mask is an optional boolean [S_q, S_kv] (True = attend); applied as an + additive mask so causal/window compose uniformly with GQA. + """ + attn_mask = _additive_mask(keep_mask) if keep_mask is not None else None with sdpa_kernel(SDPBackend.MATH): O = F.scaled_dot_product_attention( Q, K, V, + attn_mask=attn_mask, scale=scale, enable_gqa=(H_q != H_kv), ) return O -def compute_lse(Q, K, scale, H_q, H_kv): +def compute_lse(Q, K, scale, H_q, H_kv, keep_mask=None): """Compute Log-Sum-Exp reference: logsumexp(Q @ K^T * scale, dim=-1). - Returns shape [B, H_q, S_q, 1] in FP32. + Returns shape [B, H_q, S_q, 1] in FP32. For fully-masked rows the LSE is + -inf (log of zero), matching the CPU reference semantics. """ Q_f = Q.float() K_f = K.float() @@ -82,13 +162,37 @@ def compute_lse(Q, K, scale, H_q, H_kv): K_f = K_f.repeat_interleave(gqa_ratio, dim=1) scores = torch.matmul(Q_f, K_f.transpose(-2, -1)) * scale + if keep_mask is not None: + scores = scores + _additive_mask(keep_mask) lse = torch.logsumexp(scores, dim=-1, keepdim=True) # [B, H_q, S_q, 1] return lse +def quantize_fp8_per_tensor(t): + """Quantize a float tensor to FP8_E4M3 with a single per-tensor descale. + + Returns (fp8_tensor, descale_scalar, dequantized_fp32). descale maps FP8 codes + back to the original scale: x ~= fp8_code * descale (AITER convention). + """ + amax = t.abs().max().float() + scale = (amax / FP8_E4M3_MAX).clamp(min=1e-12) + quantized = (t.float() / scale).clamp(-FP8_E4M3_MAX, FP8_E4M3_MAX) + fp8 = quantized.to(FP8_TORCH_DTYPE) + dequant = fp8.float() * scale + return fp8, scale.reshape(1), dequant + + +def cumulative_seqlens(seq_lens): + """Build the cumulative seq-start pointer array [0, s0, s0+s1, ...] (int32).""" + cu = [0] + for s in seq_lens: + cu.append(cu[-1] + s) + return torch.tensor(cu, dtype=torch.int32) + + def save_tensor_bin(tensor, path): t = tensor.contiguous().cpu() - if t.dtype in (torch.bfloat16, torch.float16): + if t.dtype in (torch.bfloat16, torch.float16, FP8_TORCH_DTYPE): raw = t.view(torch.uint8).numpy().tobytes() else: raw = t.numpy().tobytes() @@ -97,15 +201,31 @@ def save_tensor_bin(tensor, path): def build_graph_json( - q_dims, k_dims, v_dims, o_dims, scale, dtype_str="bfloat16", stats=False + q_dims, + k_dims, + v_dims, + o_dims, + scale, + dtype_str="bfloat16", + stats=False, + *, + causal="none", + window_left=-1, + window_right=-1, + group_mode=False, + fp8=False, + o_dtype_str=None, ): B, H_q, S_q = q_dims[0], q_dims[1], q_dims[2] + qkv_dtype = FP8_JSON_DTYPE if fp8 else dtype_str + o_dtype = o_dtype_str if o_dtype_str is not None else dtype_str + tensors = [] - for uid, name, dims in [ - (0, "Q", q_dims), - (1, "K", k_dims), - (2, "V", v_dims), - (3, "O", o_dims), + for uid, name, dims, dt in [ + (UID_Q, "Q", q_dims, qkv_dtype), + (UID_K, "K", k_dims, qkv_dtype), + (UID_V, "V", v_dims, qkv_dtype), + (UID_O, "O", o_dims, o_dtype), ]: tensors.append( { @@ -113,7 +233,7 @@ def build_graph_json( "name": name, "dims": dims, "strides": compute_contiguous_strides(dims), - "data_type": dtype_str, + "data_type": dt, "virtual": False, } ) @@ -122,7 +242,7 @@ def build_graph_json( lse_dims = [B, H_q, S_q, 1] tensors.append( { - "uid": 4, + "uid": UID_LSE, "name": "LSE", "dims": lse_dims, "strides": compute_contiguous_strides(lse_dims), @@ -131,6 +251,48 @@ def build_graph_json( } ) + if group_mode: + for uid, name in [(UID_SEQ_LEN_Q, "SeqLenQ"), (UID_SEQ_LEN_KV, "SeqLenKv")]: + tensors.append( + { + "uid": uid, + "name": name, + "dims": [B + 1], + "strides": [1], + "data_type": "int32", + "virtual": False, + } + ) + + if fp8: + for uid, name in [ + (UID_DESCALE_Q, "DescaleQ"), + (UID_DESCALE_K, "DescaleK"), + (UID_DESCALE_V, "DescaleV"), + ]: + tensors.append( + { + "uid": uid, + "name": name, + "dims": [1], + "strides": [1], + "data_type": "float", + "virtual": False, + } + ) + + # Causal / window -> bounds + diagonal alignment (non-deprecated path). + diagonal_alignment = "BOTTOM_RIGHT" if causal == "bottom_right" else "TOP_LEFT" + left_bound = None + right_bound = None + if causal != "none": + left_bound = -1 + right_bound = 0 + if window_left >= 0: + left_bound = window_left + if window_right >= 0: + right_bound = window_right + graph = { "nodes": [ { @@ -138,13 +300,13 @@ def build_graph_json( "compute_data_type": "float", "name": "", "inputs": { - "q_tensor_uid": 0, - "k_tensor_uid": 1, - "v_tensor_uid": 2, + "q_tensor_uid": UID_Q, + "k_tensor_uid": UID_K, + "v_tensor_uid": UID_V, "attn_mask_tensor_uid": None, "scale_tensor_uid": None, - "seq_len_q_tensor_uid": None, - "seq_len_kv_tensor_uid": None, + "seq_len_q_tensor_uid": UID_SEQ_LEN_Q if group_mode else None, + "seq_len_kv_tensor_uid": UID_SEQ_LEN_KV if group_mode else None, "seed_tensor_uid": None, "offset_tensor_uid": None, "dropout_mask_tensor_uid": None, @@ -153,16 +315,16 @@ def build_graph_json( "page_table_v_tensor_uid": None, "block_mask_tensor_uid": None, "sink_token_tensor_uid": None, - "descale_q_tensor_uid": None, - "descale_k_tensor_uid": None, - "descale_v_tensor_uid": None, + "descale_q_tensor_uid": UID_DESCALE_Q if fp8 else None, + "descale_k_tensor_uid": UID_DESCALE_K if fp8 else None, + "descale_v_tensor_uid": UID_DESCALE_V if fp8 else None, "descale_s_tensor_uid": None, "scale_s_tensor_uid": None, "scale_o_tensor_uid": None, }, "outputs": { - "o_tensor_uid": 3, - "stats_tensor_uid": 4 if stats else None, + "o_tensor_uid": UID_O, + "stats_tensor_uid": UID_LSE if stats else None, "max_tensor_uid": None, "sum_exp_tensor_uid": None, "rng_dump_tensor_uid": None, @@ -177,10 +339,10 @@ def build_graph_json( "causal_mask_bottom_right": False, "dropout_probability": None, "attn_scale_value": scale, - "left_bound": None, - "right_bound": None, + "left_bound": left_bound, + "right_bound": right_bound, "max_seq_len_kv": None, - "diagonal_alignment": "TOP_LEFT", + "diagonal_alignment": diagonal_alignment, "mma_core_mode": "float", "implementation": "AUTO", }, @@ -217,7 +379,7 @@ def build_meta_json(config, pytorch_version): "reference_backend": "pytorch_math_backend", "rocm_version": rocm_ver, "generator_version": GENERATOR_VERSION, - "generation_precision": f"like-for-like: {config['dtype'].upper()} inputs, FP32 intermediates", + "generation_precision": config["precision_note"], "direction": "forward", "seed": config["seed"], "input_range": [config["min_val"], config["max_val"]], @@ -231,7 +393,13 @@ def build_meta_json(config, pytorch_version): "head_dim_qk": config["q_dims"][3], "head_dim_v": config["v_dims"][3], "dtype": config["dtype"], - "causal": False, + "causal": config["causal"], + "window_left": config["window_left"], + "window_right": config["window_right"], + "group_mode": config["group_mode"], + "seq_lens_q": config["seq_lens_q"], + "seq_lens_kv": config["seq_lens_kv"], + "stats": config["stats"], "scale": config["scale"], "gqa_ratio": config["q_dims"][1] // config["v_dims"][1], }, @@ -366,14 +534,80 @@ def validate_against_aiter( return True +def _validate_group_args(B, seq_lens_q, seq_lens_kv, S_q, S_kv): + errors = [] + if seq_lens_q is None or seq_lens_kv is None: + errors.append( + "--variable-seq-lens requires both --seq-lens-q and --seq-lens-kv" + ) + return errors + if len(seq_lens_q) != B or len(seq_lens_kv) != B: + errors.append( + f"--seq-lens-q/--seq-lens-kv must have exactly B={B} entries " + f"(got {len(seq_lens_q)} and {len(seq_lens_kv)})" + ) + for i, s in enumerate(seq_lens_q): + if s <= 0 or s > S_q: + errors.append(f"--seq-lens-q[{i}]={s} must be in (0, S_q={S_q}]") + for i, s in enumerate(seq_lens_kv): + if s <= 0 or s > S_kv: + errors.append(f"--seq-lens-kv[{i}]={s} must be in (0, S_kv={S_kv}]") + return errors + + +def _compute_group_forward( + Q, + K, + V, + scale, + H_q, + H_kv, + causal, + window_left, + window_right, + seq_lens_q, + seq_lens_kv, + o_torch_dtype, + want_lse, +): + """GROUP-mode reference: run SDPA independently per batch sequence. + + Each batch b attends only over its valid [seq_lens_q[b], seq_lens_kv[b]] + region; padding positions are zero-filled in O (and -inf in LSE). + """ + B, _, _, _ = Q.shape + _, _, _, D_v = V.shape + O = torch.zeros((B, H_q, Q.shape[2], D_v), dtype=o_torch_dtype) + lse = ( + torch.full((B, H_q, Q.shape[2], 1), float("-inf"), dtype=torch.float32) + if want_lse + else None + ) + for b in range(B): + sq = seq_lens_q[b] + skv = seq_lens_kv[b] + q_b = Q[b : b + 1, :, :sq, :] + k_b = K[b : b + 1, :, :skv, :] + v_b = V[b : b + 1, :, :skv, :] + keep = build_mask(sq, skv, causal, window_left, window_right) + o_b = compute_forward(q_b, k_b, v_b, scale, H_q, H_kv, keep) + O[b : b + 1, :, :sq, :] = o_b.to(o_torch_dtype) + if want_lse: + lse[b : b + 1, :, :sq, :] = compute_lse(q_b, k_b, scale, H_q, H_kv, keep) + return O, lse + + def generate_forward_bundle( base_filename, q_dims, v_dims, dtype="bf16", - causal=False, + causal="none", window_left=-1, window_right=-1, + variable_seq_lens=False, + seq_lens_q=None, + seq_lens_kv=None, stats=False, seed=42, min_val=-1.0, @@ -386,14 +620,26 @@ def generate_forward_bundle( k_dims = [B, H_kv, S_kv, D_qk] o_dims = [B, H_q, S_q, D_v] - if dtype not in DTYPE_MAP: + is_fp8 = dtype == "fp8" + if not is_fp8 and dtype not in DTYPE_MAP: print( - f"ERROR: --dtype must be one of {list(DTYPE_MAP.keys())} (got '{dtype}')", + f"ERROR: --dtype must be one of {list(DTYPE_MAP.keys()) + ['fp8']} " + f"(got '{dtype}')", file=sys.stderr, ) sys.exit(1) - dtype_info = DTYPE_MAP[dtype] - torch_dtype = dtype_info["torch"] + + if is_fp8: + gen_torch_dtype = torch.float32 # quantize FP8 from FP32 + o_torch_dtype = FP8_OUTPUT_TORCH + io_json = FP8_OUTPUT_JSON # io_data_type describes the float O dtype + o_json = FP8_OUTPUT_JSON + else: + info = DTYPE_MAP[dtype] + gen_torch_dtype = info["torch"] + o_torch_dtype = info["torch"] + io_json = info["json"] + o_json = info["json"] errors = [] for name, dims in [("q", q_dims), ("v", v_dims)]: @@ -406,6 +652,8 @@ def generate_forward_bundle( errors.append(f"H_q ({H_q}) must be divisible by H_kv ({H_kv})") if min_val >= max_val: errors.append(f"--min ({min_val}) must be less than --max ({max_val})") + if variable_seq_lens: + errors += _validate_group_args(B, seq_lens_q, seq_lens_kv, S_q, S_kv) if errors: for e in errors: print(f"ERROR: {e}", file=sys.stderr) @@ -418,37 +666,90 @@ def generate_forward_bundle( print(f"Generating forward bundle: {base_filename}") print(f" Q: {q_dims}, K: {k_dims}, V: {v_dims}, O: {o_dims}") - print(f" dtype: {dtype} ({torch_dtype})") + print(f" dtype: {dtype} (output {o_torch_dtype})") print(f" H_q={H_q}, H_kv={H_kv}, GQA ratio={H_q // H_kv}") + print( + f" causal={causal}, window=({window_left},{window_right}), " + f"group={variable_seq_lens}, stats={stats}" + ) print(f" Scale: {attn_scale:.10f}, Seed: {seed}") rng = torch.Generator().manual_seed(seed) + Q = torch.empty(q_dims, dtype=gen_torch_dtype).uniform_( + min_val, max_val, generator=rng + ) + K = torch.empty(k_dims, dtype=gen_torch_dtype).uniform_( + min_val, max_val, generator=rng + ) + V = torch.empty(v_dims, dtype=gen_torch_dtype).uniform_( + min_val, max_val, generator=rng + ) + + descale_q = descale_k = descale_v = None + if is_fp8: + Q_fp8, descale_q, Q_deq = quantize_fp8_per_tensor(Q) + K_fp8, descale_k, K_deq = quantize_fp8_per_tensor(K) + V_fp8, descale_v, V_deq = quantize_fp8_per_tensor(V) + Q_compute, K_compute, V_compute = Q_deq, K_deq, V_deq + Q_store, K_store, V_store = Q_fp8, K_fp8, V_fp8 + else: + Q_compute, K_compute, V_compute = Q, K, V + Q_store, K_store, V_store = Q, K, V - Q = torch.empty(q_dims, dtype=torch_dtype).uniform_(min_val, max_val, generator=rng) - K = torch.empty(k_dims, dtype=torch_dtype).uniform_(min_val, max_val, generator=rng) - V = torch.empty(v_dims, dtype=torch_dtype).uniform_(min_val, max_val, generator=rng) + keep_mask = build_mask(S_q, S_kv, causal, window_left, window_right) try: - O = compute_forward(Q, K, V, attn_scale, H_q, H_kv) + if variable_seq_lens: + O, lse = _compute_group_forward( + Q_compute, + K_compute, + V_compute, + attn_scale, + H_q, + H_kv, + causal, + window_left, + window_right, + seq_lens_q, + seq_lens_kv, + o_torch_dtype, + want_lse=stats, + ) + else: + O = compute_forward( + Q_compute, K_compute, V_compute, attn_scale, H_q, H_kv, keep_mask + ).to(o_torch_dtype) + lse = ( + compute_lse(Q_compute, K_compute, attn_scale, H_q, H_kv, keep_mask) + if stats + else None + ) except RuntimeError as e: print(f"ERROR: PyTorch SDPA failed: {e}", file=sys.stderr) sys.exit(1) - lse = None - if stats: - lse = compute_lse(Q, K, attn_scale, H_q, H_kv) - - for name, tensor in [("Q", Q), ("K", K), ("V", V), ("O", O)]: - assert not torch.isnan(tensor).any(), f"NaN in {name}" - assert not torch.isinf(tensor).any(), f"Inf in {name}" - if lse is not None: - assert not torch.isnan(lse).any(), "NaN in LSE" - assert not torch.isinf(lse).any(), "Inf in LSE" + assert not torch.isnan(O).any(), "NaN in O" + assert not torch.isinf(O).any(), "Inf in O" # Write raw tensor data as .bin files (one per tensor UID) - tensor_list = [("Q", Q, 0), ("K", K, 1), ("V", V, 2), ("O", O, 3)] + tensor_list = [ + ("Q", Q_store, UID_Q), + ("K", K_store, UID_K), + ("V", V_store, UID_V), + ("O", O, UID_O), + ] if lse is not None: - tensor_list.append(("LSE", lse, 4)) + tensor_list.append(("LSE", lse, UID_LSE)) + if variable_seq_lens: + tensor_list.append(("SeqLenQ", cumulative_seqlens(seq_lens_q), UID_SEQ_LEN_Q)) + tensor_list.append( + ("SeqLenKv", cumulative_seqlens(seq_lens_kv), UID_SEQ_LEN_KV) + ) + if is_fp8: + tensor_list.append(("DescaleQ", descale_q, UID_DESCALE_Q)) + tensor_list.append(("DescaleK", descale_k, UID_DESCALE_K)) + tensor_list.append(("DescaleV", descale_v, UID_DESCALE_V)) + for name, tensor, uid in tensor_list: bin_path = f"{base_filename}.tensor{uid}.bin" save_tensor_bin(tensor, bin_path) @@ -464,8 +765,14 @@ def generate_forward_bundle( v_dims, o_dims, attn_scale, - dtype_str=dtype_info["json"], + dtype_str=io_json, stats=stats, + causal=causal, + window_left=window_left, + window_right=window_right, + group_mode=variable_seq_lens, + fp8=is_fp8, + o_dtype_str=o_json, ) json_path = f"{base_filename}.json" with open(json_path, "w") as f: @@ -473,16 +780,27 @@ def generate_forward_bundle( f.write("\n") print(f" Graph JSON: {json_path}") - # Write meta JSON (provenance: generator version, reference source, config) + precision_note = ( + "fp8: FP8_E4M3 inputs (per-tensor descale), FP32 intermediates, BF16 output" + if is_fp8 + else f"like-for-like: {dtype.upper()} inputs, FP32 intermediates" + ) config = { "q_dims": q_dims, "v_dims": v_dims, "dtype": dtype, + "causal": causal, + "window_left": window_left, + "window_right": window_right, + "group_mode": variable_seq_lens, + "seq_lens_q": seq_lens_q, + "seq_lens_kv": seq_lens_kv, + "stats": stats, "seed": seed, "min_val": min_val, "max_val": max_val, "scale": attn_scale, - "stats": stats, + "precision_note": precision_note, } meta_json = build_meta_json(config, torch.__version__) meta_path = f"{base_filename}.meta.json" @@ -491,23 +809,26 @@ def generate_forward_bundle( f.write("\n") print(f" Meta JSON: {meta_path}") - # Optional: cross-check golden output against AITER GPU kernel + # Optional: cross-check golden output against AITER GPU kernel (dense float only) if validate: - ok = validate_against_aiter( - Q, - K, - V, - O, - attn_scale, - causal=causal, - window_left=window_left, - window_right=window_right, - stats=stats, - ) - if ok: - print(" Validation: PASSED") + if variable_seq_lens or is_fp8: + print( + " Validation: SKIPPED (AITER cross-check only covers dense float " + "BATCH mode; GROUP/FP8 are validated by the CPU reference)" + ) else: - print(" Validation: FAILED — check warnings above") + ok = validate_against_aiter( + Q_store, + K_store, + V_store, + O, + attn_scale, + causal=(causal != "none"), + window_left=window_left, + window_right=window_right, + stats=stats, + ) + print(" Validation: PASSED" if ok else " Validation: FAILED") def main(): @@ -538,16 +859,16 @@ def main(): parser.add_argument( "--dtype", default="bf16", - choices=list(DTYPE_MAP.keys()), - help="Input/output tensor dtype (default: bf16)", + choices=list(DTYPE_MAP.keys()) + ["fp8"], + help="Input dtype. fp8 = FP8_E4M3 inputs, BF16 output, per-tensor descale " + "(default: bf16)", ) parser.add_argument( "--causal", default="none", - choices=[ - "none" - ], # TODO: add "top_left", "bottom_right" when causal kernels are imported - help="Causal mask type (default: none)", + choices=["none", "top_left", "bottom_right"], + help="Causal mask type. bottom_right maps to AITER mask=2; top_left is " + "reference-only (no AITER kernel) (default: none)", ) parser.add_argument( "--window-left", @@ -561,10 +882,30 @@ def main(): default=-1, help="Sliding window right bound, -1 = unbounded (default: -1)", ) + parser.add_argument( + "--variable-seq-lens", + action="store_true", + help="Enable GROUP mode (variable per-batch sequence lengths). Requires " + "--seq-lens-q and --seq-lens-kv.", + ) + parser.add_argument( + "--seq-lens-q", + nargs="+", + type=int, + default=None, + help="Per-batch query lengths (GROUP mode). Must have B entries.", + ) + parser.add_argument( + "--seq-lens-kv", + nargs="+", + type=int, + default=None, + help="Per-batch key/value lengths (GROUP mode). Must have B entries.", + ) parser.add_argument( "--stats", action="store_true", - help="Enable LSE output tensor (uid=4, shape [B, H_q, S_q], dtype FP32)", + help="Enable LSE output tensor (uid=4, shape [B, H_q, S_q, 1], dtype FP32)", ) parser.add_argument("--seed", type=int, default=42) parser.add_argument("--min", type=float, default=-1.0, dest="min_val") @@ -595,9 +936,12 @@ def main(): q_dims=args.q_dims, v_dims=args.v_dims, dtype=args.dtype, - causal=(args.causal != "none"), + causal=args.causal, window_left=args.window_left, window_right=args.window_right, + variable_seq_lens=args.variable_seq_lens, + seq_lens_q=args.seq_lens_q, + seq_lens_kv=args.seq_lens_kv, stats=args.stats, seed=args.seed, min_val=args.min_val, diff --git a/dnn-providers/integration-tests/src/integration_tests/sdpa/IntegrationGoldenRefSdpaFwdInference.cpp b/dnn-providers/integration-tests/src/integration_tests/sdpa/IntegrationGoldenRefSdpaFwdInference.cpp index 942444fe4081..6ea7e583d39a 100644 --- a/dnn-providers/integration-tests/src/integration_tests/sdpa/IntegrationGoldenRefSdpaFwdInference.cpp +++ b/dnn-providers/integration-tests/src/integration_tests/sdpa/IntegrationGoldenRefSdpaFwdInference.cpp @@ -141,4 +141,179 @@ INSTANTIATE_TEST_SUITE_P( TestCpuSdpaFwdGoldenRefStdFp16Hd128NomaskBatchStats, getGoldenReferenceParams("standard/SdpaFwd/bhsd/fp16/hd128_nomask_batch_stats")); +// --- causal mask (bottom-right) and hd192 head dim --- +// The CPU reference executor supports bottom-right causal (via left_bound=-1, +// right_bound=0, BOTTOM_RIGHT) and hd192 (D_qk=192, D_v=128). FP8 and GROUP-mode +// bundles are produced by the generator but are not yet runnable by the CPU +// executor, so they are intentionally not instantiated here. + +// quick tier — Small bundles + +class TestCpuSdpaFwdGoldenReferenceBf16Hd128CausalBatchBfp16 + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenReferenceBf16Hd128CausalBatchBfp16, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenReferenceBf16Hd128CausalBatchBfp16, + getGoldenReferenceParams("quick/SdpaFwd/bhsd/bf16/hd128_causal_batch")); + +class TestCpuSdpaFwdGoldenReferenceFp16Hd128CausalBatchFp16 + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenReferenceFp16Hd128CausalBatchFp16, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenReferenceFp16Hd128CausalBatchFp16, + getGoldenReferenceParams("quick/SdpaFwd/bhsd/fp16/hd128_causal_batch")); + +class TestCpuSdpaFwdGoldenReferenceBf16Hd192NomaskBatchBfp16 + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenReferenceBf16Hd192NomaskBatchBfp16, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenReferenceBf16Hd192NomaskBatchBfp16, + getGoldenReferenceParams("quick/SdpaFwd/bhsd/bf16/hd192_nomask_batch")); + +class TestCpuSdpaFwdGoldenReferenceBf16Hd192CausalBatchBfp16 + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenReferenceBf16Hd192CausalBatchBfp16, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenReferenceBf16Hd192CausalBatchBfp16, + getGoldenReferenceParams("quick/SdpaFwd/bhsd/bf16/hd192_causal_batch")); + +class TestCpuSdpaFwdGoldenReferenceFp16Hd192NomaskBatchFp16 + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenReferenceFp16Hd192NomaskBatchFp16, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenReferenceFp16Hd192NomaskBatchFp16, + getGoldenReferenceParams("quick/SdpaFwd/bhsd/fp16/hd192_nomask_batch")); + +class TestCpuSdpaFwdGoldenReferenceFp16Hd192CausalBatchFp16 + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenReferenceFp16Hd192CausalBatchFp16, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenReferenceFp16Hd192CausalBatchFp16, + getGoldenReferenceParams("quick/SdpaFwd/bhsd/fp16/hd192_causal_batch")); + +// standard tier — Medium, Gqa bundles + +class TestCpuSdpaFwdGoldenRefStdBf16Hd128CausalBatch + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenRefStdBf16Hd128CausalBatch, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenRefStdBf16Hd128CausalBatch, + getGoldenReferenceParams("standard/SdpaFwd/bhsd/bf16/hd128_causal_batch")); + +class TestCpuSdpaFwdGoldenRefStdFp16Hd128CausalBatch : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenRefStdFp16Hd128CausalBatch, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenRefStdFp16Hd128CausalBatch, + getGoldenReferenceParams("standard/SdpaFwd/bhsd/fp16/hd128_causal_batch")); + +class TestCpuSdpaFwdGoldenRefStdBf16Hd192NomaskBatch + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenRefStdBf16Hd192NomaskBatch, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenRefStdBf16Hd192NomaskBatch, + getGoldenReferenceParams("standard/SdpaFwd/bhsd/bf16/hd192_nomask_batch")); + +class TestCpuSdpaFwdGoldenRefStdBf16Hd192CausalBatch + : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenRefStdBf16Hd192CausalBatch, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenRefStdBf16Hd192CausalBatch, + getGoldenReferenceParams("standard/SdpaFwd/bhsd/bf16/hd192_causal_batch")); + +class TestCpuSdpaFwdGoldenRefStdFp16Hd192NomaskBatch : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenRefStdFp16Hd192NomaskBatch, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenRefStdFp16Hd192NomaskBatch, + getGoldenReferenceParams("standard/SdpaFwd/bhsd/fp16/hd192_nomask_batch")); + +class TestCpuSdpaFwdGoldenRefStdFp16Hd192CausalBatch : public TestCpuSdpaFwdGoldenReference +{ +}; + +TEST_P(TestCpuSdpaFwdGoldenRefStdFp16Hd192CausalBatch, Correctness) +{ + testSuite(); +} + +INSTANTIATE_TEST_SUITE_P(, + TestCpuSdpaFwdGoldenRefStdFp16Hd192CausalBatch, + getGoldenReferenceParams("standard/SdpaFwd/bhsd/fp16/hd192_causal_batch")); + #endif // HIPDNN_FLATBUFFERS_SDK_SKIP_JSON_LIB