Tie RSA methods to their engine lifetime

Store each allocated RSA EVP_PKEY_METHOD in its owning ENGINE rather than a
shared global pointer.  This keeps the callback valid until the ENGINE's
final structural reference is released, even if the last PKCS11_CTX is freed
while a key still retains an ENGINE reference.

OpenSSL then releases the method from its engine teardown path without
leaking it or confusing a later libp11 ENGINE instance.

Author: mtrojnar

src/p11_rsa.c

@@ -33,8 +33,8 @@ static int rsa_ex_index = 0;
 static RSA_METHOD *pkcs11_rsa_method = NULL;
 
 #if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L && !defined(OPENSSL_NO_ENGINE)
-static EVP_PKEY_METHOD *pkey_method_rsa = NULL;
 static ENGINE *pkey_engine_rsa = NULL;
+static int pkey_engine_rsa_ex_index = -1;
 #endif /* OpenSSL 3.x with ENGINE support */
 
 static RSA *pkcs11_get1_rsa(PKCS11_OBJECT_private *key)
@@ -247,14 +247,13 @@ static int pkcs11_pkey_engine_meths(ENGINE *engine,
 {
 	static int pkey_nids[] = { EVP_PKEY_RSA, 0 };
 
-	(void)engine;
 	if (!pmeth) {
 		*nids = pkey_nids;
 		return 1;
 	}
 	if (nid == EVP_PKEY_RSA) {
-		*pmeth = pkey_method_rsa;
-		return 1;
+		*pmeth = ENGINE_get_ex_data(engine, pkey_engine_rsa_ex_index);
+		return *pmeth != NULL;
 	}
 	*pmeth = NULL;
 	return 0;
@@ -263,24 +262,34 @@ static int pkcs11_pkey_engine_meths(ENGINE *engine,
 /* Initialize an ENGINE that selects the RSA method only for libp11 keys. */
 static int pkcs11_pkey_method_rsa_new(void)
 {
+	EVP_PKEY_METHOD *pkey_method_rsa;
+	ENGINE *engine;
+
 	if (pkey_engine_rsa)
 		return 1;
 
 	pkey_method_rsa = pkcs11_pkey_method_rsa();
 	if (!pkey_method_rsa)
 		return 0;
-	pkey_engine_rsa = ENGINE_new();
-	if (!pkey_engine_rsa ||
-			!ENGINE_set_id(pkey_engine_rsa, "libp11-rsa-key") ||
-			!ENGINE_set_name(pkey_engine_rsa, "libp11 RSA key method") ||
-			!ENGINE_set_pkey_meths(pkey_engine_rsa,
-				pkcs11_pkey_engine_meths)) {
-		ENGINE_free(pkey_engine_rsa);
-		pkey_engine_rsa = NULL;
+	if (pkey_engine_rsa_ex_index < 0)
+		pkey_engine_rsa_ex_index = ENGINE_get_ex_new_index(0,
+			"libp11 RSA EVP_PKEY_METHOD", NULL, NULL, NULL);
+	engine = ENGINE_new();
+	if (pkey_engine_rsa_ex_index < 0 || !engine ||
+			!ENGINE_set_id(engine, "libp11-rsa-key") ||
+			!ENGINE_set_name(engine, "libp11 RSA key method") ||
+			!ENGINE_set_ex_data(engine, pkey_engine_rsa_ex_index,
+				pkey_method_rsa)) {
+		ENGINE_free(engine);
+		EVP_PKEY_meth_free(pkey_method_rsa);
+		return 0;
+	}
+	if (!ENGINE_set_pkey_meths(engine, pkcs11_pkey_engine_meths)) {
+		ENGINE_free(engine);
 		EVP_PKEY_meth_free(pkey_method_rsa);
-		pkey_method_rsa = NULL;
 		return 0;
 	}
+	pkey_engine_rsa = engine;
 	return 1;
 }
 #endif /* OPENSSL_NO_ENGINE */
@@ -301,11 +310,12 @@ void pkcs11_rsa_key_method_free(void)
 {
 #ifndef OPENSSL_NO_ENGINE
 	if (pkey_engine_rsa) {
+		/* OpenSSL frees the method returned by the callback when the
+		 * ENGINE's final structural reference is released.  Keep the method
+		 * in ENGINE ex_data so it remains available if keys still hold refs. */
 		ENGINE_free(pkey_engine_rsa);
 		pkey_engine_rsa = NULL;
 	}
-	/* ENGINE_free() releases the methods cached by the ENGINE. */
-	pkey_method_rsa = NULL;
 #endif /* OPENSSL_NO_ENGINE */
 }