Storageblob (#4043)

* first take

* grrrrr

* qwqwq

* ewqw

* eewew

* put back

* pipeline

* Set read-only VCPKG_BINARY_SOURCES_SECRET

* Pass InstallLanguageSteps through the perf template

* blop pipeline

* path

* ';l'l;

* add support for sync param

* --sync flag support

* typo

* sync

* sync to y

* for PR

* clang

Co-authored-by: Daniel Jurek <[email protected]>

Author: gearama

.vscode/cspell.json

@@ -74,6 +74,7 @@
     "HRESULT",
     "IMDS",
     "Intel",
+    "issecret",
     "itfactor",
     "iusg",
     "jepio",

cmake-modules/PerfTest.cmake

@@ -0,0 +1,11 @@
+macro(SetPerfDeps PACKAGE VAR_RESULT)
+    string(TOUPPER ${PACKAGE} SUFFIX)
+    string(CONCAT VAR_TRIGGER "VCPKG-" ${SUFFIX})
+    message(STATUS "trigger name ${VAR_TRIGGER}")
+    if(DEFINED ENV{${VAR_TRIGGER}})
+        find_package(${PACKAGE} $ENV{${VAR_TRIGGER}} EXACT)
+        add_compile_definitions(${VAR_RESULT}="$ENV{${VAR_TRIGGER}}")
+    else()
+        add_compile_definitions(${VAR_RESULT}="source")
+    endif()
+endmacro()

eng/pipelines/templates/jobs/perf.yml

@@ -23,6 +23,9 @@ parameters:
 - name: EnvVars
   type: object
   default: []
+- name: InstallLanguageSteps
+  type: object
+  default: []
 
 extends:
   template: /eng/common/pipelines/templates/jobs/perf.yml
@@ -38,8 +41,4 @@ extends:
     Iterations: ${{ parameters.Iterations }}
     AdditionalArguments: ${{ parameters.AdditionalArguments }}
     EnvVars: ${{ parameters.EnvVars}}
-    InstallLanguageSteps:
-      - template: /eng/pipelines/templates/steps/vcpkg.yml
-      #/eng/pipelines/templates/steps/vcpkg-clone.yml
-        parameters: 
-          RepoOwner: Microsoft
+    InstallLanguageSteps: ${{ parameters.InstallLanguageSteps }}

sdk/core/perf.yml

@@ -30,3 +30,11 @@ extends:
     Arguments: ${{ parameters.Arguments }}
     Iterations: ${{ parameters.Iterations }}
     AdditionalArguments: ${{ parameters.AdditionalArguments }}
+    InstallLanguageSteps: 
+      - pwsh: |
+          Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read"
+        displayName: Set Vcpkg Variables
+
+    EnvVars: 
+      # This is set in the InstallLanguageSteps 
+      VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET)

sdk/core/perf/src/options.cpp

@@ -85,5 +85,6 @@ std::vector<Azure::Perf::TestOption> Azure::Perf::GlobalTestOptions::GetOptionMe
       {"Rate", {"-r", "--rate"}, "Target throughput (ops/sec). Default to no throughput.", 1},
       {"Warmup", {"-w", "--warmup"}, "Duration of warmup in seconds. Default to 5 seconds.", 1},
       {"TestProxies", {"-x", "--test-proxies"}, "URIs of TestProxy Servers (separated by ';')", 1},
-      {"help", {"-h", "--help"}, "Display help information.", 0}};
+      {"help", {"-h", "--help"}, "Display help information.", 0},
+      {"Sync", {"-y", "--sync"}, "Runs sync version of test, not implemented", 0}};
 }

sdk/storage/azure-storage-blobs/perf-resources.bicep

@@ -0,0 +1,21 @@
+param baseName string = resourceGroup().name
+param location string = resourceGroup().location
+
+resource storageAccount 'Microsoft.Storage/storageAccounts@2019-06-01' = {
+  name: '${baseName}blob'
+  location: location
+  kind: 'BlockBlobStorage'
+  sku: {
+    name: 'Premium_LRS'
+  }
+}
+
+var name = storageAccount.name
+var key = storageAccount.listKeys().keys[0].value
+var connectionString = 'DefaultEndpointsProtocol=https;AccountName=${name};AccountKey=${key}'
+
+output AZURE_STORAGE_ACCOUNT_NAME string = name
+output AZURE_STORAGE_ACCOUNT_KEY string = key
+output AZURE_STORAGE_CONNECTION_STRING string = connectionString
+output STANDARD_STORAGE_CONNECTION_STRING string = connectionString
+output STORAGE_CONNECTION_STRING string = connectionString

sdk/storage/azure-storage-blobs/perf.yml

@@ -0,0 +1,40 @@
+parameters:
+- name: PackageVersions
+  displayName: PackageVersions (regex of package versions to run)
+  type: string
+  default: '12|source'
+- name: Tests
+  displayName: Tests (regex of tests to run)
+  type: string
+  default: '^(download|upload|list-blobs)$'
+- name: Arguments
+  displayName: Arguments (regex of arguments to run)
+  type: string
+  default: '(10240)|(10485760)|(1073741824)|(5 )|(500 )|(50000 )' 
+- name: Iterations
+  displayName: Iterations (times to run each test)
+  type: number
+  default: '5'
+- name: AdditionalArguments
+  displayName: AdditionalArguments (passed to PerfAutomation)
+  type: string
+  default: ' '
+
+extends:
+  template: /eng/pipelines/templates/jobs/perf.yml
+  parameters:
+    ServiceDirectory: storage/azure-storage-blobs
+    Services: "^storage-blob$"
+    PackageVersions: ${{ parameters.PackageVersions }}
+    Tests: ${{ parameters.Tests }}
+    Arguments: ${{ parameters.Arguments }}
+    Iterations: ${{ parameters.Iterations }}
+    AdditionalArguments: ${{ parameters.AdditionalArguments }}
+    InstallLanguageSteps: 
+      - pwsh: |
+          Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read"
+        displayName: Set Vcpkg Variables
+
+    EnvVars: 
+      # This is set in the InstallLanguageSteps 
+      VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET)

sdk/storage/azure-storage-blobs/test/perf/CMakeLists.txt

@@ -6,6 +6,8 @@ cmake_minimum_required (VERSION 3.13)
 project(azure-storage-blobs-perf LANGUAGES CXX)
 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
+include(AzureVcpkg)
+az_vcpkg_integrate()
 
 if(BUILD_TRANSPORT_CURL)
   set(DOWNLOAD_WITH_LIBCURL inc/azure/storage/blobs/test/download_blob_transport_only.hpp)
@@ -35,6 +37,8 @@ add_executable (
 
 create_per_service_target_build(storage azure-storage-blobs-perf)
 
+include(PerfTest)
+SETPERFDEPS(azure-storage-blobs-cpp VCPKG_STORAGE_BLOB_VERSION)
 # Include the headers from the project.
 target_include_directories(
   azure-storage-blobs-perf
@@ -47,7 +51,8 @@ if (MSVC)
     target_compile_options(azure-storage-blobs-perf PUBLIC /wd4996)
 endif()
 
+
 # link the `azure-perf` lib together with any other library which will be used for the tests. 
-target_link_libraries(azure-storage-blobs-perf PRIVATE azure-storage-blobs azure-perf)
+target_link_libraries(azure-storage-blobs-perf PRIVATE Azure::azure-storage-blobs azure-perf)
 # Make sure the project will appear in the test folder for Visual Studio CMake view
 set_target_properties(azure-storage-blobs-perf PROPERTIES FOLDER "Tests/Storage")

sdk/storage/azure-storage-blobs/test/perf/src/azure_storage_blobs_perf_test.cpp

@@ -16,6 +16,7 @@
 
 int main(int argc, char** argv)
 {
+  std::cout << "Azure-storage-blobs VERSION " << VCPKG_STORAGE_BLOB_VERSION << std::endl;
 
   // Create the test list
   std::vector<Azure::Perf::TestMetadata> tests

vcpkg.json

@@ -15,7 +15,7 @@
     },
     {
       "name": "opentelemetry-cpp",
-      "platform": "!uwp",
+      "platform": "!(windows & !static)",
       "version>=": "1.3.0"
     },
     {