Skip to content

cmake: Add dynamic test discovery to improve parallelism #1760

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
real-or-random merged 4 commits into from
Feb 19, 2026
+96 −11
Merged

cmake: Add dynamic test discovery to improve parallelism #1760

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4ac6511
cmake, refactor: Deduplicate test-related code
hebasto Jan 13, 2026
f95b263
cmake: Add DiscoverTests module
hebasto Jan 13, 2026
29f26ec
cmake: Integrate DiscoverTests and normalize test names
hebasto Jan 20, 2026
8354618
cmake: Set `LABELS` property for tests
hebasto Jan 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 71 additions & 0 deletions cmake/DiscoverTests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# TODO: rework/remove once test discovery is implemented upstream:
# https://gitlab.kitware.com/cmake/cmake/-/issues/26920
function(discover_tests target)
set(options "")
set(oneValueArgs DISCOVERY_MATCH TEST_NAME_REPLACEMENT TEST_ARGS_REPLACEMENT)
set(multiValueArgs DISCOVERY_ARGS PROPERTIES)
cmake_parse_arguments(PARSE_ARGV 1 arg "${options}" "${oneValueArgs}" "${multiValueArgs}")

set(file_base ${CMAKE_CURRENT_BINARY_DIR}/${target})
set(include_file ${file_base}_include.cmake)

set(properties_content)
list(LENGTH arg_PROPERTIES properties_len)
if(properties_len GREATER "0")
set(properties_content " set_tests_properties(\"\${test_name}\" PROPERTIES\n")
math(EXPR num_properties "${properties_len} / 2")
foreach(i RANGE 0 ${num_properties} 2)
math(EXPR value_index "${i} + 1")
list(GET arg_PROPERTIES ${i} name)
list(GET arg_PROPERTIES ${value_index} value)
Comment on lines +16 to +20
Copy link
Copy Markdown
Member

@furszy furszy Jan 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This assumes properties come in pairs, but someone could:

discover_tests(${exe_name}
      DISCOVERY_ARGS        "--list_tests"
      DISCOVERY_MATCH       "^\\t\\\\[ *[0-9]+\\\\] ([^ ].*)$"
      TEST_NAME_REPLACEMENT "secp256k1.${exe_name}.\\\\1"
      TEST_ARGS_REPLACEMENT "--target=\\\\1 --log=1"
      PROPERTIES
        LABELS "secp256k1_${exe_name}"
        NO_TIMEOUT   // <-- property with no value
)

It would be good to check that the value exist before accessing it, and fail with a clear error if not: "Property 'name' has no associated value. Must be specified as name/value pair" or something similar.

Copy link
Copy Markdown
Member Author

@hebasto hebasto Jan 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Every set property has a value. The syntax in the snippet above is simply incorrect. Compare it with the syntax of the set_tests_properties command.

string(APPEND properties_content " \"${name}\" \"${value}\"\n")
endforeach()
string(APPEND properties_content " )\n")
endif()

string(CONCAT include_content
"set(runner [[$<TARGET_FILE:${target}>]])\n"
"set(launcher [[$<TARGET_PROPERTY:${target},TEST_LAUNCHER>]])\n"
"set(emulator [[$<$<BOOL:${CMAKE_CROSSCOMPILING}>:$<TARGET_PROPERTY:${target},CROSSCOMPILING_EMULATOR>>]])\n"
"\n"
"execute_process(\n"
" COMMAND \${launcher} \${emulator} \${runner} ${arg_DISCOVERY_ARGS}\n"
" OUTPUT_VARIABLE output OUTPUT_STRIP_TRAILING_WHITESPACE\n"
" ERROR_VARIABLE output ERROR_STRIP_TRAILING_WHITESPACE\n"
" RESULT_VARIABLE result\n"
")\n"
"\n"
"if(NOT result EQUAL 0)\n"
" add_test([[${target}_DISCOVERY_FAILURE]] \${launcher} \${emulator} \${runner} ${arg_DISCOVERY_ARGS})\n"
"else()\n"
" string(REPLACE \"\\n\" \";\" lines \"\${output}\")\n"
" foreach(line IN LISTS lines)\n"
" if(line MATCHES \"${arg_DISCOVERY_MATCH}\")\n"
" string(REGEX REPLACE \"${arg_DISCOVERY_MATCH}\" \"${arg_TEST_NAME_REPLACEMENT}\" test_name \"\${line}\")\n"
" string(REGEX REPLACE \"${arg_DISCOVERY_MATCH}\" \"${arg_TEST_ARGS_REPLACEMENT}\" test_args \"\${line}\")\n"
" separate_arguments(test_args)\n"
" add_test(\"\${test_name}\" \${launcher} \${emulator} \${runner} \${test_args})\n"
${properties_content}
" endif()\n"
" endforeach()\n"
"endif()\n"
)

get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(is_multi_config)
file(GENERATE
OUTPUT ${file_base}_include-$<CONFIG>.cmake
CONTENT "${include_content}"
)
file(WRITE ${include_file}
"include(\"${file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")"
)
else()
file(GENERATE
OUTPUT ${include_file}
CONTENT "${include_content}"
)
endif()

set_property(DIRECTORY APPEND PROPERTY TEST_INCLUDE_FILES ${include_file})
endfunction()
6 changes: 4 additions & 2 deletions examples/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ function(add_example name)
secp256k1
$<$<PLATFORM_ID:Windows>:bcrypt>
)
set(test_name ${name}_example)
add_test(NAME secp256k1_${test_name} COMMAND ${target_name})
add_test(NAME secp256k1.example.${name} COMMAND ${target_name})
set_tests_properties(secp256k1.example.${name} PROPERTIES
LABELS secp256k1_example
)
endfunction()

add_example(ecdsa)
Expand Down
30 changes: 21 additions & 9 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,15 +144,24 @@ if(SECP256K1_BUILD_TESTS)
list(APPEND TEST_DEFINITIONS SUPPORTS_CONCURRENCY=1)
endif()

add_executable(noverify_tests tests.c)
target_link_libraries(noverify_tests secp256k1_precomputed secp256k1_asm)
target_compile_definitions(noverify_tests PRIVATE ${TEST_DEFINITIONS})
add_test(NAME secp256k1_noverify_tests COMMAND noverify_tests)
function(add_executable_and_tests exe_name verify_definition)
add_executable(${exe_name} tests.c)
target_link_libraries(${exe_name} secp256k1_precomputed secp256k1_asm)
target_compile_definitions(${exe_name} PRIVATE ${verify_definition} ${TEST_DEFINITIONS})
include(DiscoverTests)
discover_tests(${exe_name}
DISCOVERY_ARGS "--list_tests"
DISCOVERY_MATCH "^\\t\\\\[ *[0-9]+\\\\] ([^ ].*)$"
TEST_NAME_REPLACEMENT "secp256k1.${exe_name}.\\\\1"
TEST_ARGS_REPLACEMENT "--target=\\\\1 --log=1"
PROPERTIES
LABELS "secp256k1_${exe_name}"
Comment on lines +157 to +158
Copy link
Copy Markdown
Member

@furszy furszy Jan 19, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would be nice to explain what LABELS are.

Copy link
Copy Markdown
Member Author

@hebasto hebasto Jan 20, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From Professional CMake: A Practical Guide 21st Edition:

Selecting tests individually by name or number can become cumbersome if a large set of related tests needs to be executed. Tests can be assigned an arbitrary list of labels using the LABELS test property, and then tests can be selected by these labels. The -L and -LE options are analogous to the -R and -E options respectively, except they operate on test labels rather than test names.
...
Labels not only enable convenient grouping for test execution, they also provide grouping for basic execution time statistics. As seen in the example output above, the ctest command prints a label summary when any test in the set of executed tests has its LABELS property set. This allows the developer to get an idea how each label group is contributing to the overall test time.

)
endfunction()

add_executable_and_tests(noverify_tests "")
if(NOT CMAKE_BUILD_TYPE STREQUAL "Coverage")
add_executable(tests tests.c)
target_compile_definitions(tests PRIVATE VERIFY ${TEST_DEFINITIONS})
target_link_libraries(tests secp256k1_precomputed secp256k1_asm)
add_test(NAME secp256k1_tests COMMAND tests)
add_executable_and_tests(tests VERIFY)
endif()
unset(TEST_DEFINITIONS)
endif()
Expand All @@ -162,7 +171,10 @@ if(SECP256K1_BUILD_EXHAUSTIVE_TESTS)
add_executable(exhaustive_tests tests_exhaustive.c)
target_link_libraries(exhaustive_tests secp256k1_asm)
target_compile_definitions(exhaustive_tests PRIVATE $<$<NOT:$<CONFIG:Coverage>>:VERIFY>)
add_test(NAME secp256k1_exhaustive_tests COMMAND exhaustive_tests)
add_test(NAME secp256k1.exhaustive_tests COMMAND exhaustive_tests)
set_tests_properties(secp256k1.exhaustive_tests PROPERTIES
LABELS secp256k1_exhaustive
)
endif()

if(SECP256K1_BUILD_CTIME_TESTS)
Expand Down