crypto: clear OpenSSL error queue after calling X509_check_private_key()

Fixes: #45485

Author: panva

src/crypto/crypto_x509.cc

@@ -453,6 +453,8 @@ void X509Certificate::CheckPrivateKey(const FunctionCallbackInfo<Value>& args) {
   ASSIGN_OR_RETURN_UNWRAP(&key, args[0]);
   CHECK_EQ(key->Data()->GetKeyType(), kKeyTypePrivate);
 
+  ClearErrorOnReturn clear_error_on_return;
+
   args.GetReturnValue().Set(
       X509_check_private_key(
           cert->get(),

test/parallel/test-crypto-x509.js

@@ -8,6 +8,7 @@ if (!common.hasCrypto)
 const {
   X509Certificate,
   createPrivateKey,
+  generateKeyPairSync,
 } = require('crypto');
 
 const {
@@ -186,10 +187,17 @@ const der = Buffer.from(
     code: 'ERR_INVALID_ARG_VALUE'
   });
 
-  // Confirm failure of X509Certificate:verify() doesn't affect other functions that use OpenSSL.
-  assert(!x509.verify(x509.publicKey));
-  // This call should not throw.
-  createPrivateKey(key);
+  {
+    // https://github.com/nodejs/node/issues/45377
+    // https://github.com/nodejs/node/issues/45485
+    // Confirm failures of X509Certificate:verify() and X509Certificate:CheckPrivateKey()
+    // do not affect other functions that use OpenSSL.
+    const keyPair = generateKeyPairSync('ed25519');
+    assert(!x509.verify(keyPair.publicKey));
+    assert(!x509.checkPrivateKey(keyPair.privateKey));
+    // This call should not throw.
+    createPrivateKey(key);
+  }
 
   // X509Certificate can be cloned via MessageChannel/MessagePort
   const mc = new MessageChannel();