Skip to content

Commit ead6ac3

Browse files
Jinming-HuJinming-Hu
authored
Merge feature/datalakestg82 (#3973)
1 parent 74d6b75 commit ead6ac3

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+4359
-432
lines changed

‎sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/blob_container_client.hpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,7 @@ namespace Azure { namespace Storage { namespace Blobs {
338338
friend class BlobServiceClient;
339339
friend class BlobLeaseClient;
340340
friend class BlobContainerBatch;
341+
friend class Files::DataLake::DataLakeFileSystemClient;
341342
};
342343

343344
}}} // namespace Azure::Storage::Blobs

‎sdk/storage/azure-storage-blobs/inc/azure/storage/blobs/rest_client.hpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1365,6 +1365,10 @@ namespace Azure { namespace Storage { namespace Blobs {
13651365
* Type of the blob.
13661366
*/
13671367
Models::BlobType BlobType;
1368+
/**
1369+
* The deletion ID associated with the deleted path.
1370+
*/
1371+
Nullable<std::string> DeletionId;
13681372
};
13691373
} // namespace _detail
13701374
/**
@@ -3447,6 +3451,7 @@ namespace Azure { namespace Storage { namespace Blobs {
34473451
Nullable<std::string> Marker;
34483452
Nullable<int32_t> MaxResults;
34493453
Nullable<Models::ListBlobsIncludeFlags> Include;
3454+
Nullable<std::string> ShowOnly;
34503455
};
34513456
static Response<Models::_detail::ListBlobsByHierarchyResult> ListBlobsByHierarchy(
34523457
Core::Http::_internal::HttpPipeline& pipeline,

‎sdk/storage/azure-storage-blobs/src/rest_client.cpp

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2310,6 +2310,7 @@ namespace Azure { namespace Storage { namespace Blobs {
23102310
kHasVersionsOnly,
23112311
kContentLength,
23122312
kBlobType,
2313+
kDeletionId,
23132314
};
23142315
const std::unordered_map<std::string, XmlTagEnum> XmlTagEnumMap{
23152316
{"EnumerationResults", XmlTagEnum::kEnumerationResults},
@@ -2370,6 +2371,7 @@ namespace Azure { namespace Storage { namespace Blobs {
23702371
{"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly},
23712372
{"Content-Length", XmlTagEnum::kContentLength},
23722373
{"BlobType", XmlTagEnum::kBlobType},
2374+
{"DeletionId", XmlTagEnum::kDeletionId},
23732375
};
23742376
std::vector<XmlTagEnum> xmlPath;
23752377
Models::_detail::BlobItem vectorElement1;
@@ -2807,6 +2809,13 @@ namespace Azure { namespace Storage { namespace Blobs {
28072809
{
28082810
vectorElement1.BlobType = Models::BlobType(node.Value);
28092811
}
2812+
else if (
2813+
xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults
2814+
&& xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob
2815+
&& xmlPath[3] == XmlTagEnum::kDeletionId)
2816+
{
2817+
vectorElement1.DeletionId = node.Value;
2818+
}
28102819
}
28112820
else if (node.Type == _internal::XmlNodeType::Attribute)
28122821
{
@@ -2908,6 +2917,11 @@ namespace Azure { namespace Storage { namespace Blobs {
29082917
ListBlobsIncludeFlagsToString(options.Include.Value())));
29092918
}
29102919
request.SetHeader("x-ms-version", "2021-04-10");
2920+
if (options.ShowOnly.HasValue() && !options.ShowOnly.Value().empty())
2921+
{
2922+
request.GetUrl().AppendQueryParameter(
2923+
"showonly", _internal::UrlEncodeQueryParameter(options.ShowOnly.Value()));
2924+
}
29112925
auto pRawResponse = pipeline.Send(request, context);
29122926
auto httpStatusCode = pRawResponse->GetStatusCode();
29132927
if (httpStatusCode != Core::Http::HttpStatusCode::Ok)
@@ -2981,6 +2995,7 @@ namespace Azure { namespace Storage { namespace Blobs {
29812995
kHasVersionsOnly,
29822996
kContentLength,
29832997
kBlobType,
2998+
kDeletionId,
29842999
kBlobPrefix,
29853000
};
29863001
const std::unordered_map<std::string, XmlTagEnum> XmlTagEnumMap{
@@ -3043,6 +3058,7 @@ namespace Azure { namespace Storage { namespace Blobs {
30433058
{"HasVersionsOnly", XmlTagEnum::kHasVersionsOnly},
30443059
{"Content-Length", XmlTagEnum::kContentLength},
30453060
{"BlobType", XmlTagEnum::kBlobType},
3061+
{"DeletionId", XmlTagEnum::kDeletionId},
30463062
{"BlobPrefix", XmlTagEnum::kBlobPrefix},
30473063
};
30483064
std::vector<XmlTagEnum> xmlPath;
@@ -3488,6 +3504,13 @@ namespace Azure { namespace Storage { namespace Blobs {
34883504
{
34893505
vectorElement1.BlobType = Models::BlobType(node.Value);
34903506
}
3507+
else if (
3508+
xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults
3509+
&& xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlob
3510+
&& xmlPath[3] == XmlTagEnum::kDeletionId)
3511+
{
3512+
vectorElement1.DeletionId = node.Value;
3513+
}
34913514
else if (
34923515
xmlPath.size() == 4 && xmlPath[0] == XmlTagEnum::kEnumerationResults
34933516
&& xmlPath[1] == XmlTagEnum::kBlobs && xmlPath[2] == XmlTagEnum::kBlobPrefix
@@ -5573,9 +5596,15 @@ namespace Azure { namespace Storage { namespace Blobs {
55735596
response.LeaseDuration
55745597
= Models::LeaseDurationType(pRawResponse->GetHeaders().at("x-ms-lease-duration"));
55755598
}
5576-
response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state"));
5577-
response.LeaseStatus
5578-
= Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status"));
5599+
if (pRawResponse->GetHeaders().count("x-ms-lease-state") != 0)
5600+
{
5601+
response.LeaseState = Models::LeaseState(pRawResponse->GetHeaders().at("x-ms-lease-state"));
5602+
}
5603+
if (pRawResponse->GetHeaders().count("x-ms-lease-status") != 0)
5604+
{
5605+
response.LeaseStatus
5606+
= Models::LeaseStatus(pRawResponse->GetHeaders().at("x-ms-lease-status"));
5607+
}
55795608
response.IsServerEncrypted
55805609
= pRawResponse->GetHeaders().at("x-ms-server-encrypted") == std::string("true");
55815610
return Response<Models::QueryBlobResult>(std::move(response), std::move(pRawResponse));

‎sdk/storage/azure-storage-blobs/swagger/README.md

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,15 @@ directive:
273273
{"value": "legalhold", "name": "LegalHold"},
274274
{"value": "deletedwithversions", "name": "DeletedWithVersions"}
275275
];
276+
$["ListBlobsShowOnly"]= {
277+
"name": "showonly",
278+
"x-ms-client-name": "ShowOnly",
279+
"in": "query",
280+
"required": false,
281+
"type": "string",
282+
"x-ms-parameter-location": "method",
283+
"description": "Include this parameter to specify one or more datasets to include in the response."
284+
};
276285
$.DeleteSnapshots["x-ms-enum"]["name"] = "DeleteSnapshotsOption";
277286
$.DeleteSnapshots["x-ms-enum"]["values"] = [{"value": "include", "name": "IncludeSnapshots"},{"value":"only", "name": "OnlySnapshots"}];
278287
$.BlobExpiryOptions["x-ms-enum"]["name"] = "ScheduleBlobExpiryOriginType";
@@ -829,6 +838,7 @@ directive:
829838
$.BlobItemInternal.properties["BlobType"] = $.BlobPropertiesInternal.properties["BlobType"];
830839
$.BlobItemInternal.properties["BlobType"]["x-ms-xml"] = {"name": "Properties/BlobType"};
831840
delete $.BlobPropertiesInternal.properties["BlobType"];
841+
$.BlobItemInternal.properties["DeletionId"] = {"type": "string"};
832842
$.BlobItemInternal.required.push("BlobType", "BlobSize");
833843
$.BlobItemInternal.properties["Name"].description = "Blob name.";
834844
$.BlobItemInternal.properties["Deleted"].description = "Indicates whether this blob was deleted.";
@@ -837,6 +847,7 @@ directive:
837847
$.BlobItemInternal.properties["IsCurrentVersion"].description = "Indicates if this is the current version of the blob.";
838848
$.BlobItemInternal.properties["BlobType"].description = "Type of the blob.";
839849
$.BlobItemInternal.properties["HasVersionsOnly"].description = "Indicates that this root blob has been deleted, but it has versions that are active.";
850+
$.BlobItemInternal.properties["DeletionId"].description = "The deletion ID associated with the deleted path.";
840851
841852
$.BlobPropertiesInternal.properties["Etag"]["x-ms-client-name"] = "ETag";
842853
$.BlobPropertiesInternal["x-ms-client-name"] = "BlobItemDetails";
@@ -916,6 +927,10 @@ directive:
916927
delete $.ListBlobsHierarchySegmentResponse.properties["Segment"];
917928
delete $.ListBlobsHierarchySegmentResponse.required;
918929
$.ListBlobsHierarchySegmentResponse.properties["NextMarker"]["x-nullable"] = true;
930+
- from: swagger-document
931+
where: $["x-ms-paths"]["/{containerName}?restype=container&comp=list&hierarchy"].get.parameters
932+
transform: >
933+
$.push({"$ref": "#/parameters/ListBlobsShowOnly"});
919934
```
920935
921936
### DownloadBlob
@@ -1312,6 +1327,10 @@ directive:
13121327
delete $[status_code].headers["x-ms-blob-content-md5"];
13131328
delete $[status_code].headers["x-ms-content-crc64"];
13141329
$[status_code].headers["x-ms-lease-duration"]["x-nullable"] = true;
1330+
$[status_code].headers["x-ms-lease-state"]["x-ms-client-default"] = "";
1331+
$[status_code].headers["x-ms-lease-state"]["x-nullable"] = true;
1332+
$[status_code].headers["x-ms-lease-status"]["x-ms-client-default"] = "";
1333+
$[status_code].headers["x-ms-lease-status"]["x-nullable"] = true;
13151334
}
13161335
```
13171336

‎sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_directory_client.hpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,13 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
234234
explicit DataLakeDirectoryClient(
235235
Azure::Core::Url directoryUrl,
236236
Blobs::BlobClient blobClient,
237-
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
238-
: DataLakePathClient(std::move(directoryUrl), std::move(blobClient), pipeline)
237+
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
238+
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
239+
: DataLakePathClient(
240+
std::move(directoryUrl),
241+
std::move(blobClient),
242+
pipeline,
243+
std::move(customerProvidedKey))
239244
{
240245
}
241246

‎sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_client.hpp

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -265,12 +265,30 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
265265
const ScheduleFileDeletionOptions& options = ScheduleFileDeletionOptions(),
266266
const Azure::Core::Context& context = Azure::Core::Context()) const;
267267

268+
/**
269+
* @brief Returns the result of a query against the file.
270+
*
271+
* @param querySqlExpression The query expression in SQL.
272+
* @param options Optional parameters to execute this function.
273+
* @param context Context for cancelling long running operations.
274+
* @return A QueryFileResult describing the query result.
275+
*/
276+
Azure::Response<Models::QueryFileResult> Query(
277+
const std::string& querySqlExpression,
278+
const QueryFileOptions& options = QueryFileOptions(),
279+
const Azure::Core::Context& context = Azure::Core::Context()) const;
280+
268281
private:
269282
explicit DataLakeFileClient(
270283
Azure::Core::Url fileUrl,
271284
Blobs::BlobClient blobClient,
272-
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
273-
: DataLakePathClient(std::move(fileUrl), std::move(blobClient), pipeline)
285+
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
286+
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
287+
: DataLakePathClient(
288+
std::move(fileUrl),
289+
std::move(blobClient),
290+
pipeline,
291+
std::move(customerProvidedKey))
274292
{
275293
}
276294

‎sdk/storage/azure-storage-files-datalake/inc/azure/storage/files/datalake/datalake_file_system_client.hpp

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -239,17 +239,46 @@ namespace Azure { namespace Storage { namespace Files { namespace DataLake {
239239
const RenameDirectoryOptions& options = RenameDirectoryOptions(),
240240
const Azure::Core::Context& context = Azure::Core::Context()) const;
241241

242+
/**
243+
* @brief Gets the paths that have recently been soft deleted in this file system.
244+
* @param options Optional parameters to list deleted paths.
245+
* @param context Context for cancelling long running operations.
246+
* @return Azure::Response<DataLakePathClient> The client targets the restored path.
247+
* @remark This request is sent to Blob endpoint.
248+
*/
249+
ListDeletedPathsPagedResponse ListDeletedPaths(
250+
const ListDeletedPathsOptions& options = ListDeletedPathsOptions(),
251+
const Azure::Core::Context& context = Azure::Core::Context()) const;
252+
253+
/**
254+
* @brief Restores a soft deleted path.
255+
* @param deletedPath The path of the deleted path.
256+
* @param deletionId The deletion ID associated with the soft deleted path. You can get soft
257+
* deleted paths and their associated deletion IDs with ListDeletedPaths.
258+
* @param context Context for cancelling long running operations.
259+
* @return Azure::Response<DataLakePathClient> The client targets the restored path.
260+
* @remark This request is sent to Blob endpoint.
261+
*/
262+
Azure::Response<DataLakePathClient> UndeletePath(
263+
const std::string& deletedPath,
264+
const std::string& deletionId,
265+
const UndeletePathOptions& options = UndeletePathOptions(),
266+
const Azure::Core::Context& context = Azure::Core::Context()) const;
267+
242268
private:
243269
Azure::Core::Url m_fileSystemUrl;
244270
Blobs::BlobContainerClient m_blobContainerClient;
245271
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> m_pipeline;
272+
Azure::Nullable<EncryptionKey> m_customerProvidedKey;
246273

247274
explicit DataLakeFileSystemClient(
248275
Azure::Core::Url fileSystemUrl,
249276
Blobs::BlobContainerClient blobContainerClient,
250-
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline)
277+
std::shared_ptr<Azure::Core::Http::_internal::HttpPipeline> pipeline,
278+
Azure::Nullable<EncryptionKey> customerProvidedKey = Azure::Nullable<EncryptionKey>())
251279
: m_fileSystemUrl(std::move(fileSystemUrl)),
252-
m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline))
280+
m_blobContainerClient(std::move(blobContainerClient)), m_pipeline(std::move(pipeline)),
281+
m_customerProvidedKey(std::move(customerProvidedKey))
253282
{
254283
}
255284
friend class DataLakeLeaseClient;

0 commit comments

Comments
 (0)