Merge #1734: Introduce (mini) unit test framework

2f4546c test: add --log option to display tests execution (furszy)
95b9953 test: Add option to display all available tests (furszy)
953f7b0 test: support running specific tests/modules targets (furszy)
0302c1a test: add --help for command-line options (furszy)
9ec3bfe test: adapt modules to the new test infrastructure (furszy)
48789da test: introduce (mini) unit test framework (furszy)
9cce703 refactor: move 'gettime_i64()' to tests_common.h (furszy)

Pull request description:

  Early Note:
  Don’t be scared by the PR’s line changes count — most of it’s just doc or part of the test framework API.

  Context:
  Currently, all tests run single-threaded sequentially and the library lacks the ability to specify which test (or group of tests) you would like to run. This is not only inconvenient as more tests are added but also time consuming during development and affects downstream projects that may want to parallelize the workload (such as Bitcoin-Core CI).

  PR Goal:
  Introduce a lightweight, extensible C89 unit test framework with no dynamic memory allocations, providing a structured way to register, execute, and report tests. The framework supports named command-line arguments in `-key=value` form, parallel test execution across multiple worker processes, granular test selection (selecting tests either by name or by module name), and time accumulation reports.

  The introduced framework supports:
  * `-help` or `-h`: display list of available commands along with their descriptions.
  * `-jobs=<num>`: distribute tests across multiple worker processes (default: sequential if 0).
  * `-target=<name>` or `-t=<name>`: run only specific tests by name; can be repeated to select multiple tests.
  *  `-target=<module name>`, `-t=<module>`  Run all tests within a specific module (can be provided multiple times)
  * `-seed=<hex>`: set a specific RNG seed (defaults to random if unspecified).
  * `-iterations=<n>`: specify the number of iterations.
  * `-list_tests`:   display list of available tests and modules you can run.
  * `-log=<0|1>`: enable or disable test execution logging (default: 0 = disabled).

  Beyond these features, the idea is to also make future developments smoother, as adding new tests require only a single entry in the central test registry, and new command-line options can be introduced easily by extending the framework’s `parse_arg()` function.

  Compatibility Note:
  The framework continues accepting the two positional arguments previously supported (iterations and seed), ensuring existing workflows remain intact.

  Testing Notes:
  Have fun. You can quickly try it through `./tests -j=<workers_num>` for parallel execution  or `./tests -t=<test_name>` to run a specific test (call `./tests -print_tests` to display all available tests and modules).

  Extra Note:
  I haven't checked the exhaustive tests file so far, but I will soon. For now, this only runs all tests declared in the `tests` binary.

  Testing Results: (Current master branch vs PR in seconds)

  * Raspberry Pi 5: master \~100 s → PR \~38 s (5 jobs)
  * MacBook Pro M1: master \~30 s → PR \~10 s (6 jobs)

ACKs for top commit:
  theStack:
    re-ACK 2f4546c
  real-or-random:
    ACK 2f4546c
  hebasto:
    ACK 2f4546c.

Tree-SHA512: 85ca2cbb620b84b35b353d5d4cf093c388fc3851ca405eeb0e458f8fa72b60534bccd357c7edabf8fc9aa93d9ad0a6fbac3dd5c4d5f9dfdf4d8701a9834755b9

Author: real-or-random

Makefile.am

@@ -45,7 +45,10 @@ noinst_HEADERS += src/precomputed_ecmult.h
 noinst_HEADERS += src/precomputed_ecmult_gen.h
 noinst_HEADERS += src/assumptions.h
 noinst_HEADERS += src/checkmem.h
+noinst_HEADERS += src/tests_common.h
 noinst_HEADERS += src/testutil.h
+noinst_HEADERS += src/unit_test.h
+noinst_HEADERS += src/unit_test.c
 noinst_HEADERS += src/util.h
 noinst_HEADERS += src/util_local_visibility.h
 noinst_HEADERS += src/int128.h
@@ -120,7 +123,7 @@ if USE_TESTS
 TESTS += noverify_tests
 noinst_PROGRAMS += noverify_tests
 noverify_tests_SOURCES = src/tests.c
-noverify_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES)
+noverify_tests_CPPFLAGS = $(SECP_CONFIG_DEFINES) $(TEST_DEFINES)
 noverify_tests_LDADD = $(COMMON_LIB) $(PRECOMPUTED_LIB)
 noverify_tests_LDFLAGS = -static
 if !ENABLE_COVERAGE

configure.ac

@@ -443,6 +443,14 @@ if test x"$enable_experimental" = x"no"; then
   fi
 fi
 
+# Check for concurrency support (tests only)
+if test "x$enable_tests" != x"no"; then
+  AC_CHECK_HEADERS([sys/types.h sys/wait.h unistd.h])
+  AS_IF([test "x$ac_cv_header_sys_types_h" = xyes && test "x$ac_cv_header_sys_wait_h" = xyes &&
+         test "x$ac_cv_header_unistd_h" = xyes], [TEST_DEFINES="-DSUPPORTS_CONCURRENCY=1"], TEST_DEFINES="")
+  AC_SUBST(TEST_DEFINES)
+fi
+
 ###
 ### Generate output
 ###

src/CMakeLists.txt

@@ -134,15 +134,27 @@ if(SECP256K1_BUILD_BENCHMARK)
 endif()
 
 if(SECP256K1_BUILD_TESTS)
+  include(CheckIncludeFile)
+  check_include_file(sys/types.h HAVE_SYS_TYPES_H)
+  check_include_file(sys/wait.h HAVE_SYS_WAIT_H)
+  check_include_file(unistd.h HAVE_UNISTD_H)
+
+  set(TEST_DEFINITIONS "")
+  if(HAVE_SYS_TYPES_H AND HAVE_SYS_WAIT_H AND HAVE_UNISTD_H)
+    list(APPEND TEST_DEFINITIONS SUPPORTS_CONCURRENCY=1)
+  endif()
+
   add_executable(noverify_tests tests.c)
   target_link_libraries(noverify_tests secp256k1_precomputed secp256k1_asm)
+  target_compile_definitions(noverify_tests PRIVATE ${TEST_DEFINITIONS})
   add_test(NAME secp256k1_noverify_tests COMMAND noverify_tests)
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Coverage")
     add_executable(tests tests.c)
-    target_compile_definitions(tests PRIVATE VERIFY)
+    target_compile_definitions(tests PRIVATE VERIFY ${TEST_DEFINITIONS})
     target_link_libraries(tests secp256k1_precomputed secp256k1_asm)
     add_test(NAME secp256k1_tests COMMAND tests)
   endif()
+  unset(TEST_DEFINITIONS)
 endif()
 
 if(SECP256K1_BUILD_EXHAUSTIVE_TESTS)

src/bench.h

@@ -12,27 +12,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#if (defined(_MSC_VER) && _MSC_VER >= 1900)
-#  include <time.h>
-#else
-#  include <sys/time.h>
-#endif
-
-static int64_t gettime_i64(void) {
-#if (defined(_MSC_VER) && _MSC_VER >= 1900)
-    /* C11 way to get wallclock time */
-    struct timespec tv;
-    if (!timespec_get(&tv, TIME_UTC)) {
-        fputs("timespec_get failed!", stderr);
-        exit(EXIT_FAILURE);
-    }
-    return (int64_t)tv.tv_nsec / 1000 + (int64_t)tv.tv_sec * 1000000LL;
-#else
-    struct timeval tv;
-    gettimeofday(&tv, NULL);
-    return (int64_t)tv.tv_usec + (int64_t)tv.tv_sec * 1000000LL;
-#endif
-}
+#include "tests_common.h"
 
 #define FP_EXP (6)
 #define FP_MULT (1000000LL)

src/modules/ecdh/tests_impl.h

@@ -7,6 +7,8 @@
 #ifndef SECP256K1_MODULE_ECDH_TESTS_H
 #define SECP256K1_MODULE_ECDH_TESTS_H
 
+#include "../../unit_test.h"
+
 static int ecdh_hash_function_test_xpassthru(unsigned char *output, const unsigned char *x, const unsigned char *y, void *data) {
     (void)y;
     (void)data;
@@ -178,12 +180,13 @@ static void test_ecdh_wycheproof(void) {
     }
 }
 
-static void run_ecdh_tests(void) {
-    test_ecdh_api();
-    test_ecdh_generator_basepoint();
-    test_bad_scalar();
-    test_result_basepoint();
-    test_ecdh_wycheproof();
-}
+/* --- Test registry --- */
+static const struct tf_test_entry tests_ecdh[] = {
+    CASE1(test_ecdh_api),
+    CASE1(test_ecdh_generator_basepoint),
+    CASE1(test_bad_scalar),
+    CASE1(test_result_basepoint),
+    CASE1(test_ecdh_wycheproof),
+};
 
 #endif /* SECP256K1_MODULE_ECDH_TESTS_H */

src/modules/ellswift/tests_impl.h

@@ -7,6 +7,7 @@
 #define SECP256K1_MODULE_ELLSWIFT_TESTS_H
 
 #include "../../../include/secp256k1_ellswift.h"
+#include "../../unit_test.h"
 
 struct ellswift_xswiftec_inv_test {
     int enc_bitmap;
@@ -433,4 +434,10 @@ void run_ellswift_tests(void) {
     }
 }
 
+/* --- Test registry --- */
+/* TODO: subdivide test in cases */
+static const struct tf_test_entry tests_ellswift[] = {
+    CASE(ellswift_tests),
+};
+
 #endif

src/modules/extrakeys/tests_impl.h

@@ -8,6 +8,7 @@
 #define SECP256K1_MODULE_EXTRAKEYS_TESTS_H
 
 #include "../../../include/secp256k1_extrakeys.h"
+#include "../../unit_test.h"
 
 static void test_xonly_pubkey(void) {
     secp256k1_pubkey pk;
@@ -467,17 +468,17 @@ static void test_keypair_add(void) {
     }
 }
 
-static void run_extrakeys_tests(void) {
+/* --- Test registry --- */
+static const struct tf_test_entry tests_extrakeys[] = {
     /* xonly key test cases */
-    test_xonly_pubkey();
-    test_xonly_pubkey_tweak();
-    test_xonly_pubkey_tweak_check();
-    test_xonly_pubkey_tweak_recursive();
-    test_xonly_pubkey_comparison();
-
+    CASE1(test_xonly_pubkey),
+    CASE1(test_xonly_pubkey_tweak),
+    CASE1(test_xonly_pubkey_tweak_check),
+    CASE1(test_xonly_pubkey_tweak_recursive),
+    CASE1(test_xonly_pubkey_comparison),
     /* keypair tests */
-    test_keypair();
-    test_keypair_add();
-}
+    CASE1(test_keypair),
+    CASE1(test_keypair_add),
+};
 
 #endif

src/modules/musig/tests_impl.h

@@ -20,6 +20,7 @@
 #include "../../group.h"
 #include "../../hash.h"
 #include "../../util.h"
+#include "../../unit_test.h"
 
 #include "vectors.h"
 
@@ -36,7 +37,7 @@ static int create_keypair_and_pk(secp256k1_keypair *keypair, secp256k1_pubkey *p
 
 /* Just a simple (non-tweaked) 2-of-2 MuSig aggregate, sign, verify
  * test. */
-static void musig_simple_test(void) {
+static void musig_simple_test_internal(void) {
     unsigned char sk[2][32];
     secp256k1_keypair keypair[2];
     secp256k1_musig_pubnonce pubnonce[2];
@@ -629,7 +630,7 @@ static void musig_tweak_test_helper(const secp256k1_xonly_pubkey* agg_pk, const
 
 /* Create aggregate public key P[0], tweak multiple times (using xonly and
  * plain tweaking) and test signing. */
-static void musig_tweak_test(void) {
+static void musig_tweak_test_internal(void) {
     unsigned char sk[2][32];
     secp256k1_pubkey pk[2];
     const secp256k1_pubkey *pk_ptr[2];
@@ -1114,28 +1115,24 @@ static void musig_test_static_nonce_gen_counter(void) {
     CHECK(secp256k1_memcmp_var(pubnonce66, expected_pubnonce, sizeof(pubnonce66)) == 0);
 }
 
-static void run_musig_tests(void) {
-    int i;
-
-    for (i = 0; i < COUNT; i++) {
-        musig_simple_test();
-    }
-    musig_api_tests();
-    musig_nonce_test();
-    for (i = 0; i < COUNT; i++) {
-        /* Run multiple times to ensure that pk and nonce have different y
-         * parities */
-        musig_tweak_test();
-    }
-    sha256_tag_test();
-    musig_test_vectors_keyagg();
-    musig_test_vectors_noncegen();
-    musig_test_vectors_nonceagg();
-    musig_test_vectors_signverify();
-    musig_test_vectors_tweak();
-    musig_test_vectors_sigagg();
-
-    musig_test_static_nonce_gen_counter();
-}
+/* --- Test registry --- */
+REPEAT_TEST(musig_simple_test)
+/* Run multiple times to ensure that pk and nonce have different y parities */
+REPEAT_TEST(musig_tweak_test)
+
+static const struct tf_test_entry tests_musig[] = {
+    CASE1(musig_simple_test),
+    CASE1(musig_api_tests),
+    CASE1(musig_nonce_test),
+    CASE1(musig_tweak_test),
+    CASE1(sha256_tag_test),
+    CASE1(musig_test_vectors_keyagg),
+    CASE1(musig_test_vectors_noncegen),
+    CASE1(musig_test_vectors_nonceagg),
+    CASE1(musig_test_vectors_signverify),
+    CASE1(musig_test_vectors_tweak),
+    CASE1(musig_test_vectors_sigagg),
+    CASE1(musig_test_static_nonce_gen_counter),
+};
 
 #endif

src/modules/recovery/tests_impl.h

@@ -7,6 +7,8 @@
 #ifndef SECP256K1_MODULE_RECOVERY_TESTS_H
 #define SECP256K1_MODULE_RECOVERY_TESTS_H
 
+#include "../../unit_test.h"
+
 static int recovery_test_nonce_function(unsigned char *nonce32, const unsigned char *msg32, const unsigned char *key32, const unsigned char *algo16, void *data, unsigned int counter) {
     (void) msg32;
     (void) key32;
@@ -28,7 +30,7 @@ static int recovery_test_nonce_function(unsigned char *nonce32, const unsigned c
     return testrand_bits(1);
 }
 
-static void test_ecdsa_recovery_api(void) {
+static void test_ecdsa_recovery_api_internal(void) {
     /* Setup contexts that just count errors */
     secp256k1_pubkey pubkey;
     secp256k1_pubkey recpubkey;
@@ -92,7 +94,7 @@ static void test_ecdsa_recovery_api(void) {
     CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, sig, recid) == 0);
 }
 
-static void test_ecdsa_recovery_end_to_end(void) {
+static void test_ecdsa_recovery_end_to_end_internal(void) {
     unsigned char extra[32] = {0x00};
     unsigned char privkey[32];
     unsigned char message[32];
@@ -324,15 +326,14 @@ static void test_ecdsa_recovery_edge_cases(void) {
     }
 }
 
-static void run_recovery_tests(void) {
-    int i;
-    for (i = 0; i < COUNT; i++) {
-        test_ecdsa_recovery_api();
-    }
-    for (i = 0; i < 64*COUNT; i++) {
-        test_ecdsa_recovery_end_to_end();
-    }
-    test_ecdsa_recovery_edge_cases();
-}
+/* --- Test registry --- */
+REPEAT_TEST(test_ecdsa_recovery_api)
+REPEAT_TEST_MULT(test_ecdsa_recovery_end_to_end, 64)
+
+static const struct tf_test_entry tests_recovery[] = {
+    CASE1(test_ecdsa_recovery_api),
+    CASE1(test_ecdsa_recovery_end_to_end),
+    CASE1(test_ecdsa_recovery_edge_cases)
+};
 
 #endif /* SECP256K1_MODULE_RECOVERY_TESTS_H */

src/modules/schnorrsig/tests_impl.h

@@ -8,6 +8,7 @@
 #define SECP256K1_MODULE_SCHNORRSIG_TESTS_H
 
 #include "../../../include/secp256k1_schnorrsig.h"
+#include "../../unit_test.h"
 
 /* Checks that a bit flip in the n_flip-th argument (that has n_bytes many
  * bytes) changes the hash function
@@ -802,7 +803,7 @@ static int nonce_function_overflowing(unsigned char *nonce32, const unsigned cha
     return 1;
 }
 
-static void test_schnorrsig_sign(void) {
+static void test_schnorrsig_sign_internal(void) {
     unsigned char sk[32];
     secp256k1_xonly_pubkey pk;
     secp256k1_keypair keypair;
@@ -852,7 +853,7 @@ static void test_schnorrsig_sign(void) {
 /* Creates N_SIGS valid signatures and verifies them with verify and
  * verify_batch (TODO). Then flips some bits and checks that verification now
  * fails. */
-static void test_schnorrsig_sign_verify(void) {
+static void test_schnorrsig_sign_verify_internal(void) {
     unsigned char sk[32];
     unsigned char msg[N_SIGS][32];
     unsigned char sig[N_SIGS][64];
@@ -965,18 +966,18 @@ static void test_schnorrsig_taproot(void) {
     CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, output_pk_bytes, pk_parity, &internal_pk, tweak) == 1);
 }
 
-static void run_schnorrsig_tests(void) {
-    int i;
-    run_nonce_function_bip340_tests();
-
-    test_schnorrsig_api();
-    test_schnorrsig_sha256_tagged();
-    test_schnorrsig_bip_vectors();
-    for (i = 0; i < COUNT; i++) {
-        test_schnorrsig_sign();
-        test_schnorrsig_sign_verify();
-    }
-    test_schnorrsig_taproot();
-}
+/* --- Test registry --- */
+REPEAT_TEST(test_schnorrsig_sign)
+REPEAT_TEST(test_schnorrsig_sign_verify)
+
+static const struct tf_test_entry tests_schnorrsig[] = {
+    CASE(nonce_function_bip340_tests),
+    CASE1(test_schnorrsig_api),
+    CASE1(test_schnorrsig_sha256_tagged),
+    CASE1(test_schnorrsig_bip_vectors),
+    CASE1(test_schnorrsig_sign),
+    CASE1(test_schnorrsig_sign_verify),
+    CASE1(test_schnorrsig_taproot),
+};
 
 #endif