DRIVERS-2585 Use AWS Secrets Manager for CSFLE

.evergreen/csfle/README.md

@@ -0,0 +1,80 @@
+# Scripts for In-Use Encryption testing
+
+This folder contains scripts for use with In-Use Encryption.
+
+See the [Azure KMS](./azurekms/README.md) and [GCP KMS](./gcpkms/README.md)
+for more information on those specific scenarios.
+
+## Prerequisities
+
+The system you are running on must have Python 3 and have access to the
+`drivers/csfle` [AWS Vault](https://wiki.corp.mongodb.com/display/DRIVERS/Using+AWS+Secrets+Manager+to+Store+Testing+Secrets).
+For legacy usage, see below.
+
+## Usage
+
+Set up In-Use Encryption by first fetching the secrets and then launching the kms servers:
+
+```bash
+$DRIVERS_TOOLS/.evergreen/csfle/setup_secrets.sh
+$DRIVERS_TOOLS/.evergreen/csfle/start_servers.sh
+```
+
+The generated `secrets-export.sh` file can be sourced from your `cwd` to get the required credentials for testing.
+
+The following servers will be started:
+
+- Mock KMIP server on port 5698
+- KMS HTTP server with an expired cert on port 8000
+- KMS HTTP server with an "wrong host" cert on port 8001
+- KMS HTTP server with a correct cert on port 8002
+- Mock Azure IMDS server on port 8080
+
+When finished, stop the servers by running:
+
+```bash
+$DRIVERS_TOOLS/.evergreen/csfle/stop_servers.sh
+```
+
+If you are starting your CSFLE servers in a separate Evergreen function, it is recommended that you setup secrets
+and start the servers in the background, and then have a separate function that uses `await_servers.sh`
+in the foreground to wait for the servers to be ready.  This will ensure the servers are not torn down
+between functions (or the function may stall and not finish because there are processes still running).  
+If you are starting the servers in a step within the same function as your tests, you
+can just start the servers directly in a foreground step.
+
+```yaml
+start-csfle-servers:
+- command: subprocess.exec
+    params:
+    working_dir: src
+    binary: bash
+    background: true
+    include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "DRIVERS_TOOLS"]
+    args:
+        - ./scripts/setup-encryption.sh
+- command: subprocess.exec
+    params:
+    working_dir: src
+    binary: bash
+    include_expansions_in_env: ["DRIVERS_TOOLS"]
+    args:
+        - ${DRIVERS_TOOLS}/.evergreen/csfle/await_servers.sh
+```
+
+Where `./scripts/setup-encryption.sh` would be:
+
+```bash
+#!/usr/bin/env bash
+# Whatever other setup needed here, like setting CSFLE_TLS_CA_FILE
+bash $DRIVERS_TOOLS/.evergreen/csfle/setup_secrets.sh
+bash $DRIVERS_TOOLS/.evergreen/csfle/start_servers.sh
+```
+
+## Legacy Usage
+
+The legacy usage involved putting the required secrets in EVG Project config, and used several steps:
+
+- Start the kmip server and http servers individually in the background.
+- Run the client in a loop until it was able to connect.
+- Use the `set-temp-creds.sh` to exchange EVG creds for CSFLE temporary credentials.

.evergreen/csfle/await_servers.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# wait for the KMS servers to start.
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+pushd $SCRIPT_DIR
+
+# Wait until the pids file has been created.
+echo "Waiting for servers to start..."
+await_pidfile() {
+    for i in $(seq 300); do
+        if [ -f ./kmip_pids.pid ]; then
+            return 0
+        fi
+        echo "PID file not detected ... sleeping"
+        sleep 2
+    done
+    echo "Could not detect PID file"
+    exit 1
+}
+await_pidfile
+echo "Waiting for servers to start...done"
+
+ # Ensure servers are running.
+await_server() {
+    echo "Waiting on $1 server on port $2"
+    for i in $(seq 10); do
+        # Exit code 7: "Failed to connect to host".
+        if curl -s "localhost:$2"; test $? -ne 7; then
+            echo "Waiting on $1 server on port $2...done"
+            return 0
+        else
+            echo "Could not connect, sleeping."
+            sleep 2
+        fi
+    done
+    echo "Could not detect '$1' server on port $2"
+    exit 1
+}
+# * List servers to await here ...
+await_server "HTTP" 9000
+await_server "HTTP" 9001
+await_server "HTTP" 9002
+await_server "Azure" 8080
+await_server "KMIP" 5698
+
+echo "Finished awaiting servers"

.evergreen/csfle/fake_azure.py

@@ -83,6 +83,9 @@ def wrapped():
 def test_params() -> 'dict[str, str]':
     return parse_qs(request.headers.get('X-MongoDB-HTTP-TestParams', ''))
 
+@imds.route('/')
+def main():
+    pass
 
 @imds.get('/metadata/identity/oauth2/token')
 @handle_asserts

.evergreen/csfle/kms_kmip_server.py

@@ -7,6 +7,7 @@
 import os
 import logging
 import argparse
+import shutil
 
 HOSTNAME = "localhost"
 PORT = 5698
@@ -19,7 +20,7 @@ def main():
         drivers_evergreen_tools, ".evergreen", "x509gen", "ca.pem")
     default_cert_file = os.path.join(
         drivers_evergreen_tools, ".evergreen", "x509gen", "server.pem")
-
+    
     parser = argparse.ArgumentParser(
         description='MongoDB Mock KMIP KMS Endpoint.')
     parser.add_argument('-p', '--port', type=int,
@@ -30,6 +31,13 @@ def main():
                         default=default_cert_file, help="TLS Server PEM file")
     args = parser.parse_args()
 
+    # Ensure we start with a fresh seed database.
+    database_path = os.path.join(
+            drivers_evergreen_tools, ".evergreen", "csfle", "pykmip.db")
+    database_seed_path = os.path.join(
+            drivers_evergreen_tools, ".evergreen", "csfle", "pykmip.db.bak")
+    shutil.copy(database_seed_path, database_path)
+
     server = KmipServer(
         hostname=HOSTNAME,
         port=args.port,
@@ -39,12 +47,11 @@ def main():
         auth_suite="TLS1.2",
         log_path=os.path.join(drivers_evergreen_tools,
                               ".evergreen", "csfle", "pykmip.log"),
-        database_path=os.path.join(
-            drivers_evergreen_tools, ".evergreen", "csfle", "pykmip.db"),
+        database_path=database_path,
         logging_level=logging.DEBUG,
     )
     with server:
-        print(f"Starting KMS KMIP server on port {args.port}")
+        print(f"\nStarting KMS KMIP server on port {args.port}")
         server.serve()
 
 

.evergreen/csfle/setup_secrets.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""
+Set up encryption secrets.
+"""
+import os
+import boto3
+
+os.environ['AWS_ACCESS_KEY_ID']=os.environ['FLE_AWS_KEY']
+os.environ['AWS_SECRET_ACCESS_KEY']=os.environ['FLE_AWS_SECRET']
+os.environ['AWS_DEFAULT_REGION']="us-east-1"
+os.environ['AWS_SESSION_TOKEN']=""
+
+print("Getting CSFLE temp creds")
+client = boto3.client('sts')
+credentials = client.get_session_token()["Credentials"]
+
+with open('secrets-export.sh', 'ab') as fid:
+    fid.write(f'\nexport CSFLE_AWS_TEMP_ACCESS_KEY_ID="{credentials["AccessKeyId"]}"'.encode('utf8'))
+    fid.write(f'\nexport CSFLE_AWS_TEMP_SECRET_ACCESS_KEY="{credentials["SecretAccessKey"]}"'.encode('utf8'))
+    fid.write(f'\nexport CSFLE_AWS_TEMP_SESSION_TOKEN="{credentials["SessionToken"]}"'.encode('utf8'))
+    for key in ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY', 'AWS_DEFAULT_REGION', 
+                'AWS_SESSION_TOKEN', 'CSFLE_TLS_CA_FILE', 'CSFLE_TLS_CERT_FILE',
+                'CSFLE_TLS_CLIENT_CERT_FILE']:
+        fid.write(f'\nexport {key}="{os.environ[key]}"'.encode('utf8'))
+    fid.write('\n'.encode('utf8'))
+
+print("Getting CSFLE temp creds...done")

.evergreen/csfle/setup_secrets.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# setup secrets for csfle testing.
+set -eu
+
+CURRENT=$(pwd)
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+PARENT_DIR=$(dirname $SCRIPT_DIR)
+pushd $SCRIPT_DIR
+
+export CSFLE_TLS_CA_FILE=${CSFLE_TLS_CA_FILE:-"$PARENT_DIR/x509gen/ca.pem"}
+export CSFLE_TLS_CERT_FILE=${CSFLE_TLS_CERT_FILE:-"$PARENT_DIR/x509gen/server.pem"}
+export CSFLE_TLS_CLIENT_CERT_FILE=${CSFLE_TLS_CLIENT_CERT_FILE:-"$PARENT_DIR/x509gen/client.pem"}
+
+if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
+    CSFLE_TLS_CA_FILE=$(cygpath -m $CSFLE_TLS_CA_FILE)
+    CSFLE_TLS_CERT_FILE=$(cygpath -m $CSFLE_TLS_CERT_FILE)
+    CSFLE_TLS_CLIENT_CERT_FILE=$(cygpath -m $CSFLE_TLS_CLIENT_CERT_FILE)
+fi
+
+bash ../auth_aws/setup_secrets.sh drivers/csfle
+source secrets-export.sh
+
+. ./activate-kmstlsvenv.sh
+python ./setup_secrets.py
+
+cp secrets-export.sh $CURRENT

.evergreen/csfle/start_servers.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# start the KMS servers in the background.
+set -eu
+
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+pushd $SCRIPT_DIR
+
+if [ ! -f ./secrets-export.sh ]; then
+    echo "Please run the setup_secrets.sh script"
+    exit 1
+fi
+
+source ./secrets-export.sh
+
+if [ -z "${CSFLE_TLS_CA_FILE-}" ]; then
+    echo "Please run the setup_secrets.sh script"
+    exit 1
+fi
+
+. ./stop_servers.sh
+. ./activate-kmstlsvenv.sh
+
+# The -u options forces the stdout and stderr streams to be unbuffered.
+# TMPDIR is required to avoid "AF_UNIX path too long" errors.
+echo "Starting KMIP Server..."
+TMPDIR="$(dirname "$DRIVERS_TOOLS")" python -u kms_kmip_server.py --ca_file $CSFLE_TLS_CA_FILE --cert_file $CSFLE_TLS_CERT_FILE --port 5698 &
+echo "$!" > kms_pids.pid
+echo "Starting KMIP Server...done."
+sleep 1
+
+echo "Starting HTTP Server 1..."
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 9000 > http1.log 2>&1 &
+echo "$!" >> kmip_pids.pid
+echo "Starting HTTP Server 1...done."
+sleep 1
+
+echo "Starting HTTP Server 2..."
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 9001 > http2.log 2>&1 &
+echo "$!" >> kmip_pids.pid
+echo "Starting HTTP Server 2...done."
+sleep 1
+
+echo "Starting HTTP Server 3..."
+python -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 9002 --require_client_cert > http3.log 2>&1 &
+echo "$!" >> kmip_pids.pid
+echo "Starting HTTP Server 3...done."
+sleep 1
+
+echo "Starting Fake Azure IMDS..."
+python bottle.py fake_azure:imds > fake_azure.log 2>&1 &
+echo "$!" >> kmip_pids.pid
+echo "Starting Fake Azure IMDS...done."
+sleep 1
+
+bash ./await_servers.sh

.evergreen/csfle/stop_servers.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# Clean up CSFLE kmip servers
+SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
+. $SCRIPT_DIR/../handle-paths.sh
+pushd $SCRIPT_DIR
+rm -f pykmip.db
+if [ -f "kmip_pids.pid" ]; then
+  < kmip_pids.pid xargs kill -9 || true
+  rm kmip_pids.pid
+fi
+popd

.gitignore

@@ -115,3 +115,6 @@ orchestration.config
 atlas-expansion.yml
 secrets-export.sh
 token_file.txt
+pykmip.log.*
+nohup.out
+pykmip.db