Tests and docs for ability to specify subdirectory to download for LocalizePath (#6308)

* Added custom localization tests for s3 and azure, added docs

Author: oandreeva-nv

docs/user_guide/model_repository.md

@@ -142,6 +142,19 @@ $ gsutil iam ch allUsers:objectViewer "${BUCKET_URL}"
 $ gsutil iam ch allUsers:legacyBucketReader "${BUCKET_URL}"
 ```
 
+By default, Triton makes a local copy of a remote model repository in
+a temporary folder, which is deleted after Triton server is shut down.
+If you would like to control where remote model repository is copied to,
+you may set the `TRITON_GCS_MOUNT_DIRECTORY` environment variable to
+a path pointing to the existing folder on your local machine.
+
+```bash
+export TRITON_GCS_MOUNT_DIRECTORY=/path/to/your/local/directory
+```
+
+**Make sure, that `TRITON_GCS_MOUNT_DIRECTORY` exists on your local machine
+and it is empty.**
+
 #### S3
 
 For a model repository residing in Amazon S3, the path must be
@@ -177,6 +190,19 @@ If the environment variables are set they will take a higher priority
 and will be used by Triton instead of the credentials set using the
 aws config command.
 
+By default, Triton makes a local copy of a remote model repository
+in a temporary folder, which is deleted after Triton server is shut down.
+If you would like to control where remote model repository is copied to,
+you may set the `TRITON_AWS_MOUNT_DIRECTORY` environment variable to
+a path pointing to the existing folder on your local machine.
+
+```bash
+export TRITON_AWS_MOUNT_DIRECTORY=/path/to/your/local/directory
+```
+
+**Make sure, that `TRITON_AWS_MOUNT_DIRECTORY` exists on your local machine
+and it is empty.**
+
 #### Azure Storage
 
 For a model repository residing in Azure Storage, the repository path
@@ -196,6 +222,19 @@ here's an example of how to find a key corresponding to your `AZURE_STORAGE_ACCO
 $ export AZURE_STORAGE_ACCOUNT="account_name"
 $ export AZURE_STORAGE_KEY=$(az storage account keys list -n $AZURE_STORAGE_ACCOUNT --query "[0].value")
 ```
+By default, Triton makes a local copy of a remote model repository in
+a temporary folder, which is deleted after Triton server is shut down.
+If you would like to control where remote model repository is copied to,
+you may set the `TRITON_AZURE_MOUNT_DIRECTORY` environment variable to a path
+pointing to the existing folder on your local machine.
+
+```bash
+export TRITON_AZURE_MOUNT_DIRECTORY=/path/to/your/local/directory
+```
+
+**Make sure, that `TRITON_AZURE_MOUNT_DIRECTORY` exists on your local machine
+and it is empty.**
+
 
 ### Cloud Storage with Credential file (Beta)
 
@@ -258,6 +297,13 @@ If the `TRITON_CLOUD_CREDENTIAL_PATH` environment variable is not set, the
 [Cloud Storage with Environment variables](#cloud-storage-with-environment-variables)
 will be used.
 
+### Caching of Cloud Storage
+
+Triton currently doesn't perform file caching for cloud storage.
+However, this functionality can be implemented through
+[repository agent API](https://github.com/triton-inference-server/server/blob/bbbcad7d87adc9596f99e3685da5d6b73380514f/docs/customization_guide/repository_agents.md) by injecting a proxy, which checks a specific local directory for caching
+given the cloud storage (original path) of the model,
+and then decides if cached files may be used.
 
 ## Model Versions
 

qa/L0_storage_S3/test.sh

@@ -292,6 +292,45 @@ set -e
 kill $SERVER_PID
 wait $SERVER_PID
 
+# Test localization to a specified location
+export TRITON_AWS_MOUNT_DIRECTORY=`pwd`/aws_localization_test
+
+if [ -d "$TRITON_AWS_MOUNT_DIRECTORY" ]; then
+  rm -rf $TRITON_AWS_MOUNT_DIRECTORY
+fi
+
+mkdir -p $TRITON_AWS_MOUNT_DIRECTORY
+
+SERVER_LOG=$SERVER_LOG_BASE.custom_localization.log
+SERVER_ARGS="--model-repository=$ROOT_REPO --exit-timeout-secs=120"
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+if [ -z "$(ls -A $TRITON_AWS_MOUNT_DIRECTORY)" ]; then
+    echo -e "\n***\n*** Test localization to a specified location failed. \n***"
+    echo -e "\n***\n*** Specified mount folder $TRITON_AWS_MOUNT_DIRECTORY is empty \n***"
+    ls -A $TRITON_AWS_MOUNT_DIRECTORY
+    exit 1
+fi
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+if [ -d "$TRITON_AWS_MOUNT_DIRECTORY" ] && [ ! -z "$(ls -A $TRITON_AWS_MOUNT_DIRECTORY)" ]; then
+    echo -e "\n***\n*** Test localization to a specified location failed. \n***"
+    echo -e "\n***\n*** Specified mount folder $TRITON_AWS_MOUNT_DIRECTORY was not cleared properly. \n***"
+    ls -A $TRITON_AWS_MOUNT_DIRECTORY
+    exit 1
+fi
+
+rm -rf $TRITON_AWS_MOUNT_DIRECTORY
+unset TRITON_AWS_MOUNT_DIRECTORY
+
 # Save models for AWS_SESSION_TOKEN test
 rm -rf tmp_cred_test_models
 mv models tmp_cred_test_models
@@ -357,6 +396,7 @@ aws configure set default.region $AWS_DEFAULT_REGION && \
 # Copy models into S3 bucket
 aws s3 cp tmp_cred_test_models/ "${BUCKET_URL_SLASH}" --recursive --include "*"
 
+SERVER_LOG=$SERVER_LOG_BASE.temporary_credentials_test.log
 SERVER_ARGS="--model-repository=$BUCKET_URL --exit-timeout-secs=120"
 
 run_server
@@ -389,6 +429,7 @@ wait $SERVER_PID
 
 # Test access decline
 export AWS_SECRET_ACCESS_KEY="[Invalid]" && export AWS_SESSION_TOKEN=""
+SERVER_LOG=$SERVER_LOG_BASE.access_decline_test.log
 SERVER_ARGS="--model-repository=$BUCKET_URL --exit-timeout-secs=120"
 run_server
 if [ "$SERVER_PID" != "0" ]; then

qa/L0_storage_azure/test.sh

@@ -169,6 +169,45 @@ for ENV_VAR in "shared_key"; do
     wait $SERVER_PID
 done
 
+# Test localization to a specified location
+export TRITON_AZURE_MOUNT_DIRECTORY=`pwd`/azure_localization_test
+
+if [ -d "$TRITON_AZURE_MOUNT_DIRECTORY" ]; then
+  rm -rf $TRITON_AZURE_MOUNT_DIRECTORY
+fi
+
+mkdir -p $TRITON_AZURE_MOUNT_DIRECTORY
+
+SERVER_LOG=$SERVER_LOG_BASE.custom_localization.log
+SERVER_ARGS="--model-repository=$MODEL_REPO --exit-timeout-secs=120"
+
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+if [ -z "$(ls -A $TRITON_AZURE_MOUNT_DIRECTORY)" ]; then
+    echo -e "\n***\n*** Test localization to a specified location failed. \n***"
+    echo -e "\n***\n*** Specified mount folder $TRITON_AZURE_MOUNT_DIRECTORY is empty \n***"
+    ls -A $TRITON_AZURE_MOUNT_DIRECTORY
+    exit 1
+fi
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+if [ -d "$TRITON_AZURE_MOUNT_DIRECTORY" ] && [ ! -z "$(ls -A $TRITON_AZURE_MOUNT_DIRECTORY)" ]; then
+    echo -e "\n***\n*** Test localization to a specified location failed. \n***"
+    echo -e "\n***\n*** Specified mount folder $TRITON_AZURE_MOUNT_DIRECTORY was not cleared properly. \n***"
+    ls -A $TRITON_AZURE_MOUNT_DIRECTORY
+    exit 1
+fi
+
+rm -rf $TRITON_AZURE_MOUNT_DIRECTORY
+unset TRITON_AZURE_MOUNT_DIRECTORY
+
 # Add test for explicit model control
 SERVER_LOG=$SERVER_LOG_BASE.explicit.log
 CLIENT_LOG=$CLIENT_LOG_BASE.explicit.log