[AzDatalake] Filesystem Client Implementation + Test Framework

sdk/storage/azdatalake/assets.json

@@ -0,0 +1,6 @@
+{
+  "AssetsRepo": "Azure/azure-sdk-assets",
+  "AssetsRepoPrefixPath": "go",
+  "TagPrefix": "go/storage/azdatalake",
+  "Tag": "go/storage/azdatalake_41335bac62"
+}

sdk/storage/azdatalake/common.go

@@ -7,17 +7,30 @@
 package azdatalake
 
 import (
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/generated"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/internal/exported"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/sas"
 )
 
-// AccessConditions identifies container-specific access conditions which you optionally set.
-type AccessConditions struct {
-	ModifiedAccessConditions *ModifiedAccessConditions
-	LeaseAccessConditions    *LeaseAccessConditions
+// SharedKeyCredential contains an account's name and its primary or secondary key.
+type SharedKeyCredential = exported.SharedKeyCredential
+
+// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
+// storage account's name and either its primary or secondary key.
+func NewSharedKeyCredential(accountName, accountKey string) (*SharedKeyCredential, error) {
+	return exported.NewSharedKeyCredential(accountName, accountKey)
 }
 
-// LeaseAccessConditions contains optional parameters to access leased entity.
-type LeaseAccessConditions = generated.LeaseAccessConditions
+// URLParts object represents the components that make up an Azure Storage Container/Blob URL.
+// NOTE: Changing any SAS-related field requires computing a new SAS signature.
+type URLParts = sas.URLParts
+
+// ParseURL parses a URL initializing URLParts' fields including any SAS-related & snapshot query parameters. Any other
+// query parameters remain in the UnparsedParams field. This method overwrites all fields in the URLParts object.
+func ParseURL(u string) (URLParts, error) {
+	return sas.ParseURL(u)
+}
 
-// ModifiedAccessConditions contains a group of parameters for specifying access conditions.
-type ModifiedAccessConditions = generated.ModifiedAccessConditions
+// HTTPRange defines a range of bytes within an HTTP resource, starting at offset and
+// ending at offset+count. A zero-value HTTPRange indicates the entire resource. An HTTPRange
+// which has an offset but no zero value count indicates from the offset to the resource's end.
+type HTTPRange = exported.HTTPRange

sdk/storage/azdatalake/datalakeerror/error_codes.go

@@ -0,0 +1,181 @@
+//go:build go1.18
+// +build go1.18
+
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+package datalakeerror
+
+import (
+	"errors"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+)
+
+// HasCode returns true if the provided error is an *azcore.ResponseError
+// with its ErrorCode field equal to one of the specified Codes.
+func HasCode(err error, codes ...Code) bool {
+	var respErr *azcore.ResponseError
+	if !errors.As(err, &respErr) {
+		return false
+	}
+
+	for _, code := range codes {
+		if respErr.ErrorCode == string(code) {
+			return true
+		}
+	}
+
+	return false
+}
+
+type StorageErrorCode string
+
+// Code - Error codes returned by the service
+type Code = bloberror.Code
+
+const (
+	ContentLengthMustBeZero                StorageErrorCode = "ContentLengthMustBeZero"
+	PathAlreadyExists                      StorageErrorCode = "PathAlreadyExists"
+	InvalidFlushPosition                   StorageErrorCode = "InvalidFlushPosition"
+	InvalidPropertyName                    StorageErrorCode = "InvalidPropertyName"
+	InvalidSourceURI                       StorageErrorCode = "InvalidSourceUri"
+	UnsupportedRestVersion                 StorageErrorCode = "UnsupportedRestVersion"
+	FileSystemNotFound                     StorageErrorCode = "FilesystemNotFound"
+	PathNotFound                           StorageErrorCode = "PathNotFound"
+	RenameDestinationParentPathNotFound    StorageErrorCode = "RenameDestinationParentPathNotFound"
+	SourcePathNotFound                     StorageErrorCode = "SourcePathNotFound"
+	DestinationPathIsBeingDeleted          StorageErrorCode = "DestinationPathIsBeingDeleted"
+	FileSystemAlreadyExists                StorageErrorCode = "FilesystemAlreadyExists"
+	FileSystemBeingDeleted                 StorageErrorCode = "FilesystemBeingDeleted"
+	InvalidDestinationPath                 StorageErrorCode = "InvalidDestinationPath"
+	InvalidRenameSourcePath                StorageErrorCode = "InvalidRenameSourcePath"
+	InvalidSourceOrDestinationResourceType StorageErrorCode = "InvalidSourceOrDestinationResourceType"
+	LeaseIsAlreadyBroken                   StorageErrorCode = "LeaseIsAlreadyBroken"
+	LeaseNameMismatch                      StorageErrorCode = "LeaseNameMismatch"
+	PathConflict                           StorageErrorCode = "PathConflict"
+	SourcePathIsBeingDeleted               StorageErrorCode = "SourcePathIsBeingDeleted"
+)
+
+const (
+	AccountAlreadyExists                              Code = "AccountAlreadyExists"
+	AccountBeingCreated                               Code = "AccountBeingCreated"
+	AccountIsDisabled                                 Code = "AccountIsDisabled"
+	AppendPositionConditionNotMet                     Code = "AppendPositionConditionNotMet"
+	AuthenticationFailed                              Code = "AuthenticationFailed"
+	AuthorizationFailure                              Code = "AuthorizationFailure"
+	AuthorizationPermissionMismatch                   Code = "AuthorizationPermissionMismatch"
+	AuthorizationProtocolMismatch                     Code = "AuthorizationProtocolMismatch"
+	AuthorizationResourceTypeMismatch                 Code = "AuthorizationResourceTypeMismatch"
+	AuthorizationServiceMismatch                      Code = "AuthorizationServiceMismatch"
+	AuthorizationSourceIPMismatch                     Code = "AuthorizationSourceIPMismatch"
+	BlobAlreadyExists                                 Code = "BlobAlreadyExists"
+	PathArchived                                      Code = "BlobArchived"
+	PathBeingRehydrated                               Code = "BlobBeingRehydrated"
+	PathImmutableDueToPolicy                          Code = "BlobImmutableDueToPolicy"
+	PathNotArchived                                   Code = "BlobNotArchived"
+	BlobNotFound                                      Code = "BlobNotFound"
+	PathOverwritten                                   Code = "BlobOverwritten"
+	PathTierInadequateForContentLength                Code = "BlobTierInadequateForContentLength"
+	PathUsesCustomerSpecifiedEncryption               Code = "BlobUsesCustomerSpecifiedEncryption"
+	BlockCountExceedsLimit                            Code = "BlockCountExceedsLimit"
+	BlockListTooLong                                  Code = "BlockListTooLong"
+	CannotChangeToLowerTier                           Code = "CannotChangeToLowerTier"
+	CannotVerifyCopySource                            Code = "CannotVerifyCopySource"
+	ConditionHeadersNotSupported                      Code = "ConditionHeadersNotSupported"
+	ConditionNotMet                                   Code = "ConditionNotMet"
+	FilesystemAlreadyExists                           Code = "ContainerAlreadyExists"
+	ContainerBeingDeleted                             Code = "ContainerBeingDeleted"
+	ContainerDisabled                                 Code = "ContainerDisabled"
+	ContainerNotFound                                 Code = "ContainerNotFound"
+	ContentLengthLargerThanTierLimit                  Code = "ContentLengthLargerThanTierLimit"
+	CopyAcrossAccountsNotSupported                    Code = "CopyAcrossAccountsNotSupported"
+	CopyIDMismatch                                    Code = "CopyIdMismatch"
+	EmptyMetadataKey                                  Code = "EmptyMetadataKey"
+	FeatureVersionMismatch                            Code = "FeatureVersionMismatch"
+	IncrementalCopyPathMismatch                       Code = "IncrementalCopyBlobMismatch"
+	IncrementalCopyOfEralierVersionSnapshotNotAllowed Code = "IncrementalCopyOfEralierVersionSnapshotNotAllowed"
+	IncrementalCopySourceMustBeSnapshot               Code = "IncrementalCopySourceMustBeSnapshot"
+	InfiniteLeaseDurationRequired                     Code = "InfiniteLeaseDurationRequired"
+	InsufficientAccountPermissions                    Code = "InsufficientAccountPermissions"
+	InternalError                                     Code = "InternalError"
+	InvalidAuthenticationInfo                         Code = "InvalidAuthenticationInfo"
+	InvalidBlobOrBlock                                Code = "InvalidBlobOrBlock"
+	InvalidPathTier                                   Code = "InvalidBlobTier"
+	InvalidPathType                                   Code = "InvalidBlobType"
+	InvalidBlockID                                    Code = "InvalidBlockId"
+	InvalidBlockList                                  Code = "InvalidBlockList"
+	InvalidHTTPVerb                                   Code = "InvalidHttpVerb"
+	InvalidHeaderValue                                Code = "InvalidHeaderValue"
+	InvalidInput                                      Code = "InvalidInput"
+	InvalidMD5                                        Code = "InvalidMd5"
+	InvalidMetadata                                   Code = "InvalidMetadata"
+	InvalidOperation                                  Code = "InvalidOperation"
+	InvalidPageRange                                  Code = "InvalidPageRange"
+	InvalidQueryParameterValue                        Code = "InvalidQueryParameterValue"
+	InvalidRange                                      Code = "InvalidRange"
+	InvalidResourceName                               Code = "InvalidResourceName"
+	InvalidSourcePathType                             Code = "InvalidSourceBlobType"
+	InvalidSourcePathURL                              Code = "InvalidSourceBlobUrl"
+	InvalidURI                                        Code = "InvalidUri"
+	InvalidVersionForPageBlobOperation                Code = "InvalidVersionForPageBlobOperation"
+	InvalidXMLDocument                                Code = "InvalidXmlDocument"
+	InvalidXMLNodeValue                               Code = "InvalidXmlNodeValue"
+	LeaseAlreadyBroken                                Code = "LeaseAlreadyBroken"
+	LeaseAlreadyPresent                               Code = "LeaseAlreadyPresent"
+	LeaseIDMismatchWithBlobOperation                  Code = "LeaseIdMismatchWithBlobOperation"
+	LeaseIDMismatchWithContainerOperation             Code = "LeaseIdMismatchWithContainerOperation"
+	LeaseIDMismatchWithLeaseOperation                 Code = "LeaseIdMismatchWithLeaseOperation"
+	LeaseIDMissing                                    Code = "LeaseIdMissing"
+	LeaseIsBreakingAndCannotBeAcquired                Code = "LeaseIsBreakingAndCannotBeAcquired"
+	LeaseIsBreakingAndCannotBeChanged                 Code = "LeaseIsBreakingAndCannotBeChanged"
+	LeaseIsBrokenAndCannotBeRenewed                   Code = "LeaseIsBrokenAndCannotBeRenewed"
+	LeaseLost                                         Code = "LeaseLost"
+	LeaseNotPresentWithBlobOperation                  Code = "LeaseNotPresentWithBlobOperation"
+	LeaseNotPresentWithContainerOperation             Code = "LeaseNotPresentWithContainerOperation"
+	LeaseNotPresentWithLeaseOperation                 Code = "LeaseNotPresentWithLeaseOperation"
+	MD5Mismatch                                       Code = "Md5Mismatch"
+	CRC64Mismatch                                     Code = "Crc64Mismatch"
+	MaxBlobSizeConditionNotMet                        Code = "MaxBlobSizeConditionNotMet"
+	MetadataTooLarge                                  Code = "MetadataTooLarge"
+	MissingContentLengthHeader                        Code = "MissingContentLengthHeader"
+	MissingRequiredHeader                             Code = "MissingRequiredHeader"
+	MissingRequiredQueryParameter                     Code = "MissingRequiredQueryParameter"
+	MissingRequiredXMLNode                            Code = "MissingRequiredXmlNode"
+	MultipleConditionHeadersNotSupported              Code = "MultipleConditionHeadersNotSupported"
+	NoAuthenticationInformation                       Code = "NoAuthenticationInformation"
+	NoPendingCopyOperation                            Code = "NoPendingCopyOperation"
+	OperationNotAllowedOnIncrementalCopyBlob          Code = "OperationNotAllowedOnIncrementalCopyBlob"
+	OperationTimedOut                                 Code = "OperationTimedOut"
+	OutOfRangeInput                                   Code = "OutOfRangeInput"
+	OutOfRangeQueryParameterValue                     Code = "OutOfRangeQueryParameterValue"
+	PendingCopyOperation                              Code = "PendingCopyOperation"
+	PreviousSnapshotCannotBeNewer                     Code = "PreviousSnapshotCannotBeNewer"
+	PreviousSnapshotNotFound                          Code = "PreviousSnapshotNotFound"
+	PreviousSnapshotOperationNotSupported             Code = "PreviousSnapshotOperationNotSupported"
+	RequestBodyTooLarge                               Code = "RequestBodyTooLarge"
+	RequestURLFailedToParse                           Code = "RequestUrlFailedToParse"
+	ResourceAlreadyExists                             Code = "ResourceAlreadyExists"
+	ResourceNotFound                                  Code = "ResourceNotFound"
+	ResourceTypeMismatch                              Code = "ResourceTypeMismatch"
+	SequenceNumberConditionNotMet                     Code = "SequenceNumberConditionNotMet"
+	SequenceNumberIncrementTooLarge                   Code = "SequenceNumberIncrementTooLarge"
+	ServerBusy                                        Code = "ServerBusy"
+	SnapshotCountExceeded                             Code = "SnapshotCountExceeded"
+	SnapshotOperationRateExceeded                     Code = "SnapshotOperationRateExceeded"
+	SnapshotsPresent                                  Code = "SnapshotsPresent"
+	SourceConditionNotMet                             Code = "SourceConditionNotMet"
+	SystemInUse                                       Code = "SystemInUse"
+	TargetConditionNotMet                             Code = "TargetConditionNotMet"
+	UnauthorizedBlobOverwrite                         Code = "UnauthorizedBlobOverwrite"
+	UnsupportedHTTPVerb                               Code = "UnsupportedHttpVerb"
+	UnsupportedHeader                                 Code = "UnsupportedHeader"
+	UnsupportedQueryParameter                         Code = "UnsupportedQueryParameter"
+	UnsupportedXMLNode                                Code = "UnsupportedXmlNode"
+)
+
+var (
+	// MissingSharedKeyCredential - Error is returned when SAS URL is being created without SharedKeyCredential.
+	MissingSharedKeyCredential = bloberror.MissingSharedKeyCredential
+)

sdk/storage/azdatalake/directory/client.go

@@ -25,11 +25,41 @@ type ClientOptions base.ClientOptions
 // Client represents a URL to the Azure Datalake Storage service.
 type Client base.CompositeClient[generated.PathClient, generated.PathClient, blob.Client]
 
-//TODO: NewClient()
+// NewClient creates an instance of Client with the specified values.
+//   - directoryURL - the URL of the directory e.g. https://<account>.dfs.core.windows.net/fs/dir
+//   - cred - an Azure AD credential, typically obtained via the azidentity module
+//   - options - client options; pass nil to accept the default values
+func NewClient(directoryURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
+	blobURL := strings.Replace(directoryURL, ".dfs.", ".blob.", 1)
+	directoryURL = strings.Replace(directoryURL, ".blob.", ".dfs.", 1)
+
+	authPolicy := shared.NewStorageChallengePolicy(cred)
+	conOptions := shared.GetClientOptions(options)
+	plOpts := runtime.PipelineOptions{
+		PerRetry: []policy.Policy{authPolicy},
+	}
+	base.SetPipelineOptions((*base.ClientOptions)(conOptions), &plOpts)
+
+	azClient, err := azcore.NewClient(shared.FileClient, exported.ModuleVersion, plOpts, &conOptions.ClientOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	if options == nil {
+		options = &ClientOptions{}
+	}
+	blobClientOpts := blob.ClientOptions{
+		ClientOptions: options.ClientOptions,
+	}
+	blobClient, _ := blob.NewClient(blobURL, cred, &blobClientOpts)
+	dirClient := base.NewPathClient(directoryURL, blobURL, blobClient, azClient, nil, (*base.ClientOptions)(conOptions))
+
+	return (*Client)(dirClient), nil
+}
 
 // NewClientWithNoCredential creates an instance of Client with the specified values.
 // This is used to anonymously access a storage account or with a shared access signature (SAS) token.
-//   - serviceURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/?<sas token>
+//   - directoryURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/fs/dir?<sas token>
 //   - options - client options; pass nil to accept the default values
 func NewClientWithNoCredential(directoryURL string, options *ClientOptions) (*Client, error) {
 	blobURL := strings.Replace(directoryURL, ".dfs.", ".blob.", 1)
@@ -44,6 +74,9 @@ func NewClientWithNoCredential(directoryURL string, options *ClientOptions) (*Cl
 		return nil, err
 	}
 
+	if options == nil {
+		options = &ClientOptions{}
+	}
 	blobClientOpts := blob.ClientOptions{
 		ClientOptions: options.ClientOptions,
 	}
@@ -54,7 +87,7 @@ func NewClientWithNoCredential(directoryURL string, options *ClientOptions) (*Cl
 }
 
 // NewClientWithSharedKeyCredential creates an instance of Client with the specified values.
-//   - serviceURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/
+//   - directoryURL - the URL of the storage account e.g. https://<account>.dfs.core.windows.net/fs/dir
 //   - cred - a SharedKeyCredential created with the matching storage account and access key
 //   - options - client options; pass nil to accept the default values
 func NewClientWithSharedKeyCredential(directoryURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
@@ -73,6 +106,9 @@ func NewClientWithSharedKeyCredential(directoryURL string, cred *SharedKeyCreden
 		return nil, err
 	}
 
+	if options == nil {
+		options = &ClientOptions{}
+	}
 	blobClientOpts := blob.ClientOptions{
 		ClientOptions: options.ClientOptions,
 	}
@@ -103,13 +139,13 @@ func NewClientFromConnectionString(connectionString string, options *ClientOptio
 	return NewClientWithNoCredential(parsed.ServiceURL, options)
 }
 
-func (d *Client) generatedFSClientWithDFS() *generated.PathClient {
+func (d *Client) generatedDirClientWithDFS() *generated.PathClient {
 	//base.SharedKeyComposite((*base.CompositeClient[generated.BlobClient, generated.BlockBlobClient])(bb))
 	dirClientWithDFS, _, _ := base.InnerClients((*base.CompositeClient[generated.PathClient, generated.PathClient, blob.Client])(d))
 	return dirClientWithDFS
 }
 
-func (d *Client) generatedFSClientWithBlob() *generated.PathClient {
+func (d *Client) generatedDirClientWithBlob() *generated.PathClient {
 	_, dirClientWithBlob, _ := base.InnerClients((*base.CompositeClient[generated.PathClient, generated.PathClient, blob.Client])(d))
 	return dirClientWithBlob
 }
@@ -123,9 +159,14 @@ func (d *Client) sharedKey() *exported.SharedKeyCredential {
 	return base.SharedKeyComposite((*base.CompositeClient[generated.PathClient, generated.PathClient, blob.Client])(d))
 }
 
-// URL returns the URL endpoint used by the Client object.
-func (d *Client) URL() string {
-	return "s.generated().Endpoint()"
+// DFSURL returns the URL endpoint used by the Client object.
+func (d *Client) DFSURL() string {
+	return d.generatedDirClientWithDFS().Endpoint()
+}
+
+// BlobURL returns the URL endpoint used by the Client object.
+func (d *Client) BlobURL() string {
+	return d.generatedDirClientWithBlob().Endpoint()
 }
 
 // Create creates a new directory (dfs1).
@@ -190,3 +231,8 @@ func (d *Client) SetHTTPHeaders(ctx context.Context, httpHeaders HTTPHeaders, op
 	// TODO: call into blob
 	return SetHTTPHeadersResponse{}, nil
 }
+
+// UndeletePath restores the specified path that was previously deleted. (dfs op/blob2).
+func (d *Client) UndeletePath(ctx context.Context, path string, options *UndeletePathOptions) (UndeletePathResponse, error) {
+	return UndeletePathResponse{}, nil
+}