Skip to content

Commit 3a9a185

Browse files
gtong-nvgtong-nv
authored
Merge branch 'main' into dev/gtong/fix-flashvsr
2 parents 6887866 + 4f776ab commit 3a9a185

29 files changed

Lines changed: 2814 additions & 208 deletions

‎configs/deterministic_quality_benchmarks.json‎

Lines changed: 65 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
{
22
"schema_version": 1,
3-
"description": "Local quality and performance-review scenarios for LingBot and Omnidreams. The quality scenarios use roughly 30-second seeded rollouts for baseline/candidate MP4 comparisons. The one-minute review scenarios collect performance metrics and embed longer MP4s in the HTML report, but intentionally skip baseline quality scoring.",
3+
"description": "Local quality and performance-review scenarios for LingBot and Omnidreams. The quality scenarios use roughly 10-second seeded rollouts for baseline/candidate MP4 comparisons. The one-minute review scenarios collect performance metrics, runtime stats, PAI-Bench inputs, and longer MP4s for review.",
44
"scenarios": [
55
{
66
"id": "lingbot-world-fast-taehv-quality-smoke",
77
"name": "LingBot-World Fast TAEHV seeded quality smoke",
8-
"description": "Roughly 30-second LingBot-World Fast rollout with explicit seed and strict PyTorch deterministic settings. This is a local quality signal; LingBot does not currently have the same public bitwise reproducibility CI guarantee as Omnidreams.",
8+
"description": "Roughly 10-second LingBot-World Fast rollout with explicit seed and strict PyTorch deterministic settings. This is a local quality signal; LingBot does not currently have the same public bitwise reproducibility CI guarantee as Omnidreams.",
99
"report_group": {
1010
"id": "lingbot",
1111
"name": "LingBot"
@@ -18,7 +18,7 @@
1818
"i2v",
1919
"quality",
2020
"seeded",
21-
"quality-30s"
21+
"quality-10s"
2222
],
2323
"env": {
2424
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
@@ -34,27 +34,36 @@
3434
"tools.benchmarks.strict_run",
3535
"--",
3636
"lingbot-world-fast-taehv-window15-sink3",
37-
"--example-data",
38-
"True",
39-
"--example-idx",
37+
"mp4",
38+
"--scenario.example-data",
39+
"true",
40+
"--scenario.example-idx",
4041
"0",
41-
"--pixel-height",
42+
"--scenario.pixel-height",
4243
"464",
43-
"--pixel-width",
44+
"--scenario.pixel-width",
4445
"832",
45-
"--total-blocks",
46-
"40",
46+
"--scenario.total-blocks",
47+
"14",
48+
"--output.fps",
49+
"16",
50+
"--output.path",
51+
"{output_dir}/lingbot-world-fast-taehv-quality-smoke.mp4",
52+
"--output.stats-path",
53+
"{output_dir}/stats_lingbot_world_fast_taehv_quality_smoke.json",
4754
"--pipeline.diffusion-model.seed",
4855
"1"
4956
],
50-
"warmup_steps": 1,
57+
"output_dir_arg": null,
58+
"warmup_steps": 6,
59+
"requires_runtime_stats": true,
5160
"quality_compare_region": "full",
5261
"timeout_s": 7200
5362
},
5463
{
5564
"id": "omnidreams-sv-ci-quality-smoke",
5665
"name": "Omnidreams single-view CI-style seeded quality smoke",
57-
"description": "Roughly 30-second Omnidreams single-view rollout aligned with the same-seed reproducibility CI path: non-perf runner, fixed example data, explicit seed, strict PyTorch deterministic settings, and 113 AR blocks.",
66+
"description": "Roughly 10-second Omnidreams single-view rollout aligned with the same-seed reproducibility CI path: non-perf runner, fixed example data, explicit seed, strict PyTorch deterministic settings, and 38 AR blocks.",
5867
"report_group": {
5968
"id": "omnidreams",
6069
"name": "Omnidreams"
@@ -68,7 +77,7 @@
6877
"quality",
6978
"seeded",
7079
"deterministic",
71-
"quality-30s"
80+
"quality-10s"
7281
],
7382
"env": {
7483
"CUBLAS_WORKSPACE_CONFIG": ":4096:8",
@@ -84,17 +93,24 @@
8493
"tools.benchmarks.strict_run",
8594
"--",
8695
"omnidreams",
87-
"--example-data",
88-
"True",
89-
"--example-data-uuid",
96+
"mp4",
97+
"--scenario.example-data",
98+
"true",
99+
"--scenario.example-data-uuid",
90100
"239560dc-33d1-11ef-9720-00044bcbccac",
91-
"--total-blocks",
92-
"113",
101+
"--scenario.total-blocks",
102+
"38",
103+
"--output.path",
104+
"{output_dir}/omnidreams-sv-ci-quality-smoke.mp4",
105+
"--output.stats-path",
106+
"{output_dir}/stats_omnidreams_sv_ci_quality_smoke.json",
93107
"--pipeline.diffusion-model.seed",
94108
"1"
95109
],
96-
"warmup_steps": 1,
97-
"quality_compare_region": "bottom-half",
110+
"output_dir_arg": null,
111+
"warmup_steps": 4,
112+
"requires_runtime_stats": true,
113+
"quality_compare_region": "full",
98114
"timeout_s": 7200
99115
},
100116
{
@@ -122,18 +138,27 @@
122138
"integrations/lingbot",
123139
"flashdreams-run",
124140
"lingbot-world-fast-taehv-window15-sink3",
125-
"--example-data",
126-
"True",
127-
"--example-idx",
141+
"mp4",
142+
"--scenario.example-data",
143+
"true",
144+
"--scenario.example-idx",
128145
"0",
129-
"--pixel-height",
146+
"--scenario.pixel-height",
130147
"464",
131-
"--pixel-width",
148+
"--scenario.pixel-width",
132149
"832",
133-
"--total-blocks",
134-
"81"
150+
"--scenario.total-blocks",
151+
"81",
152+
"--output.fps",
153+
"16",
154+
"--output.path",
155+
"{output_dir}/lingbot-world-fast-taehv-one-minute-review.mp4",
156+
"--output.stats-path",
157+
"{output_dir}/stats_lingbot_world_fast_taehv_one_minute_review.json"
135158
],
136-
"warmup_steps": 1,
159+
"output_dir_arg": null,
160+
"warmup_steps": 6,
161+
"requires_runtime_stats": true,
137162
"quality_baseline_compare": false,
138163
"timeout_s": 21600
139164
},
@@ -169,16 +194,23 @@
169194
"tools.benchmarks.strict_run",
170195
"--",
171196
"omnidreams",
172-
"--example-data",
173-
"True",
174-
"--example-data-uuid",
197+
"mp4",
198+
"--scenario.example-data",
199+
"true",
200+
"--scenario.example-data-uuid",
175201
"239560dc-33d1-11ef-9720-00044bcbccac",
176-
"--total-blocks",
202+
"--scenario.total-blocks",
177203
"226",
204+
"--output.path",
205+
"{output_dir}/omnidreams-sv-one-minute-review.mp4",
206+
"--output.stats-path",
207+
"{output_dir}/stats_omnidreams_sv_one_minute_review.json",
178208
"--pipeline.diffusion-model.seed",
179209
"1"
180210
],
181-
"warmup_steps": 1,
211+
"output_dir_arg": null,
212+
"warmup_steps": 4,
213+
"requires_runtime_stats": true,
182214
"quality_baseline_compare": false,
183215
"timeout_s": 21600
184216
}

‎configs/omnidreams_demo_replay_benchmarks.json‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,13 @@
7777
"--scenario.total-blocks",
7878
"226",
7979
"--output.path",
80-
"{output_dir}/omnidreams-sv-demo-replay.mp4"
80+
"{output_dir}/omnidreams-sv-demo-replay.mp4",
81+
"--output.stats-path",
82+
"{output_dir}/stats_omnidreams_sv_demo_replay.json"
8183
],
8284
"output_dir_arg": null,
83-
"warmup_steps": 1,
85+
"warmup_steps": 4,
86+
"requires_runtime_stats": true,
8487
"quality_baseline_compare": false,
8588
"timeout_s": 7200
8689
}

‎configs/one_minute_demo_benchmarks.json‎

Lines changed: 29 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,27 @@
2525
"integrations/lingbot",
2626
"flashdreams-run",
2727
"lingbot-world-fast-taehv-window15-sink3",
28-
"--example-data",
29-
"True",
30-
"--example-idx",
28+
"mp4",
29+
"--scenario.example-data",
30+
"true",
31+
"--scenario.example-idx",
3132
"0",
32-
"--pixel-height",
33+
"--scenario.pixel-height",
3334
"464",
34-
"--pixel-width",
35+
"--scenario.pixel-width",
3536
"832",
36-
"--total-blocks",
37-
"81"
37+
"--scenario.total-blocks",
38+
"81",
39+
"--output.fps",
40+
"16",
41+
"--output.path",
42+
"{output_dir}/lingbot-world-fast-taehv-one-minute.mp4",
43+
"--output.stats-path",
44+
"{output_dir}/stats_lingbot_world_fast_taehv_one_minute.json"
3845
],
39-
"warmup_steps": 1,
46+
"output_dir_arg": null,
47+
"warmup_steps": 6,
48+
"requires_runtime_stats": true,
4049
"timeout_s": 21600
4150
},
4251
{
@@ -69,16 +78,23 @@
6978
"tools.benchmarks.strict_run",
7079
"--",
7180
"omnidreams",
72-
"--example-data",
73-
"True",
74-
"--example-data-uuid",
81+
"mp4",
82+
"--scenario.example-data",
83+
"true",
84+
"--scenario.example-data-uuid",
7585
"239560dc-33d1-11ef-9720-00044bcbccac",
76-
"--total-blocks",
86+
"--scenario.total-blocks",
7787
"226",
88+
"--output.path",
89+
"{output_dir}/omnidreams-sv-one-minute.mp4",
90+
"--output.stats-path",
91+
"{output_dir}/stats_omnidreams_sv_one_minute.json",
7892
"--pipeline.diffusion-model.seed",
7993
"1"
8094
],
81-
"warmup_steps": 1,
95+
"output_dir_arg": null,
96+
"warmup_steps": 4,
97+
"requires_runtime_stats": true,
8298
"timeout_s": 21600
8399
}
84100
]

0 commit comments

Comments
 (0)