PYTHON-4151 Use Centralized Scripts for CSFLE Tests (#1477)

Author: blink1073

.evergreen/config.yml

@@ -369,24 +369,21 @@ functions:
       params:
         working_dir: "src"
         shell: bash
-        background: true
+        include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
           ${PREPARE_SHELL}
           if [ -n "${test_encryption}" ]; then
-            bash ./.evergreen/start-kmip-server.sh
+            . .evergreen/tox.sh -m setup-encryption
           fi
     - command: shell.exec
       type: test
       params:
         working_dir: "src"
         shell: bash
-        include_expansions_in_env: ["test_encryption", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
           # Disable xtrace
           set +x
           ${PREPARE_SHELL}
-          # Fetch vault secrets if needed
-          bash .evergreen/fetch-aws-vault-secrets.sh
           if [ -n "${MONGODB_STARTED}" ]; then
             export PYMONGO_MUST_CONNECT=true
           fi
@@ -396,9 +393,7 @@ functions:
           if [ -n "${test_encryption}" ]; then
             # Disable xtrace (just in case it was accidentally set).
             set +x
-            export LIBMONGOCRYPT_URL="${libmongocrypt_url}"
             export TEST_ENCRYPTION=1
-            . .evergreen/setup-encryption-variables.sh
             if [ -n "${test_encryption_pyopenssl}" ]; then
               export TEST_ENCRYPTION_PYOPENSSL=1
             fi
@@ -619,7 +614,7 @@ functions:
         working_dir: "src"
         script: |
           ${PREPARE_SHELL}
-          . .evergreen/stop-kmip-server.sh
+          . .evergreen/tox.sh -m teardown-encryption
           rm -rf $DRIVERS_TOOLS || true
           rm -f ./secrets-export.sh || true
 
@@ -2164,41 +2159,32 @@ axes:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: macos-1100
         display_name: "macOS 11.00"
         run_on: macos-1100
         variables:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: macos-1100-arm64
         display_name: "macOS 11.00 Arm64"
         run_on: macos-1100-arm64
         variables:
           skip_EC2_auth_test: true
           skip_ECS_auth_test: true
           skip_web_identity_auth_test: true
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/macos/master/latest/libmongocrypt.tar.gz
       - id: rhel7
         display_name: "RHEL 7.x"
         run_on: rhel79-small
         batchtime: 10080  # 7 days
-        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-70-64-bit/master/latest/libmongocrypt.tar.gz
       - id: rhel8
         display_name: "RHEL 8.x"
         run_on: rhel87-small
         batchtime: 10080  # 7 days
-        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz
       - id: rhel80-fips
         display_name: "RHEL 8.0 FIPS"
         run_on: rhel80-fips
         batchtime: 10080  # 7 days
-        variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-80-64-bit/master/latest/libmongocrypt.tar.gz
       - id: ubuntu-22.04
         display_name: "Ubuntu 22.04"
         run_on: ubuntu2204-small
@@ -2220,7 +2206,6 @@ axes:
         run_on: rhel82-arm64-small
         batchtime: 10080  # 7 days
         variables:
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/rhel-82-arm64/master/latest/libmongocrypt.tar.gz
       - id: windows-64-vsMulti-small
         display_name: "Windows 64"
         run_on: windows-64-vsMulti-small
@@ -2230,7 +2215,6 @@ axes:
           skip_EC2_auth_test: true
           skip_web_identity_auth_test: true
           venv_bin_dir: "Scripts"
-          libmongocrypt_url: https://s3.amazonaws.com/mciuploads/libmongocrypt/windows-test/master/latest/libmongocrypt.tar.gz
 
   # Test with authentication?
   - id: auth

.evergreen/fetch-aws-vault-secrets.sh

@@ -111,32 +111,13 @@ fi
 
 if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then
 
-    # Work around for root certifi not being installed.
-    # TODO: Remove after PYTHON-3952 is deployed.
-    if [ "$(uname -s)" = "Darwin" ]; then
-        python -m pip install certifi
-        CERT_PATH=$(python -c "import certifi; print(certifi.where())")
-        export SSL_CERT_FILE=${CERT_PATH}
-        export REQUESTS_CA_BUNDLE=${CERT_PATH}
-        export AWS_CA_BUNDLE=${CERT_PATH}
-    fi
-
     python -m pip install '.[encryption]'
 
-    if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
-        # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
-        powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
+    # Install libmongocrypt if necessary.
+    if [ ! -d "libmongocrypt" ]; then
+        bash ./.evergreen/setup-libmongocrypt.sh
     fi
 
-    if [ -z "$LIBMONGOCRYPT_URL" ]; then
-        echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
-        exit 1
-    fi
-    curl -O "$LIBMONGOCRYPT_URL"
-    mkdir libmongocrypt
-    tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
-    ls -la libmongocrypt
-    ls -la libmongocrypt/nocrypto
     # Use the nocrypto build to avoid dependency issues with older windows/python versions.
     BASE=$(pwd)/libmongocrypt/nocrypto
     if [ -f "${BASE}/lib/libmongocrypt.so" ]; then
@@ -155,11 +136,6 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE
         exit 1
     fi
     export PYMONGOCRYPT_LIB
-
-    # TODO: Test with 'pip install pymongocrypt'
-    git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git
-    python -m pip install --prefer-binary -r .evergreen/test-encryption-requirements.txt
-    python -m pip install ./libmongocrypt_git/bindings/python
     python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)"
     python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())"
     # PATH is updated by PREPARE_SHELL for access to mongocryptd.
@@ -170,11 +146,6 @@ if [ -n "$TEST_ENCRYPTION" ]; then
         python -m pip install '.[ocsp]'
     fi
 
-    # Get access to the AWS temporary credentials:
-    # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
-    export AWS_SESSION_TOKEN=
-    . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
-
     if [ -n "$TEST_CRYPT_SHARED" ]; then
         CRYPT_SHARED_DIR=`dirname $CRYPT_SHARED_LIB_PATH`
         echo "using crypt_shared_dir $CRYPT_SHARED_DIR"
@@ -229,16 +200,6 @@ fi
 if [ -n "$TEST_AUTH_OIDC" ]; then
     python -m pip install ".[aws]"
 
-    # Work around for root certifi not being installed.
-    # TODO: Remove after PYTHON-3952 is deployed.
-    if [ "$(uname -s)" = "Darwin" ]; then
-        python -m pip install certifi
-        CERT_PATH=$(python -c "import certifi; print(certifi.where())")
-        export SSL_CERT_FILE=${CERT_PATH}
-        export REQUESTS_CA_BUNDLE=${CERT_PATH}
-        export AWS_CA_BUNDLE=${CERT_PATH}
-    fi
-
     TEST_ARGS="test/auth_oidc/test_auth_oidc.py"
 fi
 

.evergreen/run-tests.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set -o xtrace
+
+TARGET=""
+
+if [ "Windows_NT" = "${OS:-''}" ]; then # Magic variable in cygwin
+    # PYTHON-2808 Ensure this machine has the CA cert for google KMS.
+    powershell.exe "Invoke-WebRequest -URI https://oauth2.googleapis.com/" > /dev/null || true
+    TARGET="windows-test"
+fi
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    TARGET="macos"
+fi
+
+if [ "$(uname -s)" = "Linux" ]; then
+    rhel_ver=$(awk -F'=' '/VERSION_ID/{ gsub(/"/,""); print $2}' /etc/os-release)
+    arch=$(uname -m)
+    echo "RHEL $rhel_ver $arch"
+    if [[ $rhel_ver =~ 7 ]]; then
+        TARGET="rhel-70-64-bit"
+    elif [[ $rhel_ver =~ 8 ]]; then
+        if [ "$arch" = "x86_64" ]; then
+            TARGET="rhel-80-64-bit"
+        elif [ "$arch" = "arm" ]; then
+            TARGET="rhel-82-arm64"
+        fi
+    fi
+fi
+
+if [ -z "$LIBMONGOCRYPT_URL" ] && [ -n "$TARGET" ]; then
+    LIBMONGOCRYPT_URL="https://s3.amazonaws.com/mciuploads/libmongocrypt/$TARGET/master/latest/libmongocrypt.tar.gz"
+fi
+
+if [ -z "$LIBMONGOCRYPT_URL" ]; then
+    echo "Cannot test client side encryption without LIBMONGOCRYPT_URL!"
+    exit 1
+fi
+rm -rf libmongocrypt libmongocrypt.tar.gz
+echo "Fetching $LIBMONGOCRYPT_URL..."
+curl -O "$LIBMONGOCRYPT_URL"
+echo "Fetching $LIBMONGOCRYPT_URL...done"
+mkdir libmongocrypt
+tar xzf libmongocrypt.tar.gz -C ./libmongocrypt
+ls -la libmongocrypt
+ls -la libmongocrypt/nocrypto

.evergreen/setup-encryption-variables.sh

@@ -20,6 +20,9 @@ mongocryptd.pid
 .nova/
 venv/
 secrets-export.sh
+libmongocrypt.tar.gz
+libmongocrypt/
+libmongocrypt_git/
 
 # Lambda temp files
 test/lambda/.aws-sam

.evergreen/setup-libmongocrypt.sh

@@ -216,10 +216,9 @@ the pages will re-render and the browser will automatically refresh.
     `TEST_LOADBALANCER=1 tox -m test-eg`.
 
 ## Running Encryption Tests Locally
-- Run `AWS_PROFILE=<profile> tox -e setup-encryption` after setting up your AWS profile with `aws configure sso`
-- Run the tests with `TEST_ENCRYPTION=1 LIBMONGOCRYPT_URL=<url> tox -e test-eg`.
-  Look in `.evergreen/config.yml` for the correct `LIBMONGOCRYPT_URL` for your platform.
-- When done, run `tox -e teardown-encryption` to clean up.
+- Run `AWS_PROFILE=<profile> tox -m setup-encryption` after setting up your AWS profile with `aws configure sso`.
+- Run the tests with `TEST_ENCRYPTION=1 tox -e test-eg`.
+- When done, run `tox -m teardown-encryption` to clean up.
 
 ## Re-sync Spec Tests
 

.evergreen/start-kmip-server.sh

@@ -2007,7 +2007,7 @@ def test_invalid_kms_certificate_expired(self):
         key = {
             "region": "us-east-1",
             "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-            "endpoint": "mongodb://127.0.0.1:8000",
+            "endpoint": "mongodb://127.0.0.1:9000",
         }
         # Some examples:
         # certificate verify failed: certificate has expired (_ssl.c:1129)
@@ -2019,7 +2019,7 @@ def test_invalid_hostname_in_kms_certificate(self):
         key = {
             "region": "us-east-1",
             "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-            "endpoint": "mongodb://127.0.0.1:8001",
+            "endpoint": "mongodb://127.0.0.1:9001",
         }
         # Some examples:
         # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)"
@@ -2038,8 +2038,8 @@ def setUp(self):
         super().setUp()
         # 1, create client with only tlsCAFile.
         providers: dict = copy.deepcopy(ALL_KMS_PROVIDERS)
-        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8002"
-        providers["gcp"]["endpoint"] = "127.0.0.1:8002"
+        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9002"
+        providers["gcp"]["endpoint"] = "127.0.0.1:9002"
         kms_tls_opts_ca_only = {
             "aws": {"tlsCAFile": CA_PEM},
             "azure": {"tlsCAFile": CA_PEM},
@@ -2060,18 +2060,18 @@ def setUp(self):
         self.addCleanup(self.client_encryption_with_tls.close)
         # 3, update endpoints to expired host.
         providers: dict = copy.deepcopy(providers)
-        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8000"
-        providers["gcp"]["endpoint"] = "127.0.0.1:8000"
-        providers["kmip"]["endpoint"] = "127.0.0.1:8000"
+        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9000"
+        providers["gcp"]["endpoint"] = "127.0.0.1:9000"
+        providers["kmip"]["endpoint"] = "127.0.0.1:9000"
         self.client_encryption_expired = ClientEncryption(
             providers, "keyvault.datakeys", self.client, OPTS, kms_tls_options=kms_tls_opts_ca_only
         )
         self.addCleanup(self.client_encryption_expired.close)
         # 3, update endpoints to invalid host.
         providers: dict = copy.deepcopy(providers)
-        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:8001"
-        providers["gcp"]["endpoint"] = "127.0.0.1:8001"
-        providers["kmip"]["endpoint"] = "127.0.0.1:8001"
+        providers["azure"]["identityPlatformEndpoint"] = "127.0.0.1:9001"
+        providers["gcp"]["endpoint"] = "127.0.0.1:9001"
+        providers["kmip"]["endpoint"] = "127.0.0.1:9001"
         self.client_encryption_invalid_hostname = ClientEncryption(
             providers, "keyvault.datakeys", self.client, OPTS, kms_tls_options=kms_tls_opts_ca_only
         )
@@ -2095,7 +2095,7 @@ def test_01_aws(self):
         key = {
             "region": "us-east-1",
             "key": "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
-            "endpoint": "127.0.0.1:8002",
+            "endpoint": "127.0.0.1:9002",
         }
         with self.assertRaisesRegex(EncryptionError, self.cert_error):
             self.client_encryption_no_client_cert.create_data_key("aws", key)
@@ -2105,14 +2105,14 @@ def test_01_aws(self):
         # Some examples:
         # certificate verify failed: certificate has expired (_ssl.c:1129)
         # amazon1-2018 Python 3.6: certificate verify failed (_ssl.c:852)
-        key["endpoint"] = "127.0.0.1:8000"
+        key["endpoint"] = "127.0.0.1:9000"
         with self.assertRaisesRegex(EncryptionError, "expired|certificate verify failed"):
             self.client_encryption_expired.create_data_key("aws", key)
         # Some examples:
         # certificate verify failed: IP address mismatch, certificate is not valid for '127.0.0.1'. (_ssl.c:1129)"
         # hostname '127.0.0.1' doesn't match 'wronghost.com'
         # 127.0.0.1:8001: ('Certificate does not contain any `subjectAltName`s.',)
-        key["endpoint"] = "127.0.0.1:8001"
+        key["endpoint"] = "127.0.0.1:9001"
         with self.assertRaisesRegex(
             EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate"
         ):