diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java index 0469af72f35a..b4a586785013 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java @@ -17,9 +17,11 @@ package com.google.cloud.storage.contrib.nio; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import com.google.auto.value.AutoValue; +import javax.annotation.Nullable; import java.util.Map; /** @@ -65,6 +67,25 @@ public abstract class CloudStorageConfiguration { */ public abstract int maxChannelReopens(); + /** + * Returns the project to be billed when accessing buckets. Leave empty for normal semantics, + * set to bill that project (project you own) for all accesses. This is required for accessing + * requester-pays buckets. This value cannot be null. + */ + public abstract @Nullable String userProject(); + + /** + * Returns whether userProject will be cleared for non-requester-pays buckets. That is, + * if false (the default value), setting userProject causes that project to be billed + * regardless of whether the bucket is requester-pays or not. If true, setting + * userProject will only cause that project to be billed when the project is requester-pays. + * + * Setting this will cause the bucket to be accessed when the CloudStorageFileSystem object + * is created. + */ + public abstract boolean useUserProjectOnlyForRequesterPaysBuckets(); + + /** * Creates a new builder, initialized with the following settings: * @@ -90,6 +111,9 @@ public static final class Builder { private boolean usePseudoDirectories = true; private int blockSize = CloudStorageFileSystem.BLOCK_SIZE_DEFAULT; private int maxChannelReopens = 0; + private @Nullable String userProject = null; + // This of this as "clear userProject if not RequesterPays" + private boolean useUserProjectOnlyForRequesterPaysBuckets = false; /** * Changes current working directory for new filesystem. This defaults to the root directory. @@ -99,6 +123,7 @@ public static final class Builder { * @throws IllegalArgumentException if {@code path} is not absolute. */ public Builder workingDirectory(String path) { + checkNotNull(path); checkArgument(UnixPath.getPath(false, path).isAbsolute(), "not absolute: %s", path); workingDirectory = path; return this; @@ -147,6 +172,16 @@ public Builder maxChannelReopens(int value) { return this; } + public Builder userProject(String value) { + userProject = value; + return this; + } + + public Builder autoDetectRequesterPays(boolean value) { + useUserProjectOnlyForRequesterPaysBuckets = value; + return this; + } + /** * Creates new instance without destroying builder. */ @@ -157,7 +192,9 @@ public CloudStorageConfiguration build() { stripPrefixSlash, usePseudoDirectories, blockSize, - maxChannelReopens); + maxChannelReopens, + userProject, + useUserProjectOnlyForRequesterPaysBuckets); } Builder(CloudStorageConfiguration toModify) { @@ -167,6 +204,8 @@ public CloudStorageConfiguration build() { usePseudoDirectories = toModify.usePseudoDirectories(); blockSize = toModify.blockSize(); maxChannelReopens = toModify.maxChannelReopens(); + userProject = toModify.userProject(); + useUserProjectOnlyForRequesterPaysBuckets = toModify.useUserProjectOnlyForRequesterPaysBuckets(); } Builder() {} @@ -201,6 +240,12 @@ static private CloudStorageConfiguration fromMap(Builder builder, Map case "maxChannelReopens": builder.maxChannelReopens((Integer) entry.getValue()); break; + case "userProject": + builder.userProject((String) entry.getValue()); + break; + case "useUserProjectOnlyForRequesterPaysBuckets": + builder.autoDetectRequesterPays((Boolean) entry.getValue()); + break; default: throw new IllegalArgumentException(entry.getKey()); } diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java index 04745080791f..2b613d94c5b2 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.cloud.storage.StorageOptions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; import java.io.IOException; @@ -33,6 +34,7 @@ import java.nio.file.WatchService; import java.nio.file.attribute.FileTime; import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.HashMap; import java.util.Objects; import java.util.Set; @@ -112,8 +114,9 @@ public static CloudStorageFileSystem forBucket(String bucket) { public static CloudStorageFileSystem forBucket(String bucket, CloudStorageConfiguration config) { checkArgument( !bucket.startsWith(URI_SCHEME + ":"), "Bucket name must not have schema: %s", bucket); + checkNotNull(config); return new CloudStorageFileSystem( - new CloudStorageFileSystemProvider(), bucket, checkNotNull(config)); + new CloudStorageFileSystemProvider(config.userProject()), bucket, config); } /** @@ -136,15 +139,29 @@ public static CloudStorageFileSystem forBucket(String bucket, CloudStorageConfig @Nullable StorageOptions storageOptions) { checkArgument(!bucket.startsWith(URI_SCHEME + ":"), "Bucket name must not have schema: %s", bucket); - return new CloudStorageFileSystem(new CloudStorageFileSystemProvider(storageOptions), + return new CloudStorageFileSystem(new CloudStorageFileSystemProvider(config.userProject(), storageOptions), bucket, checkNotNull(config)); } CloudStorageFileSystem( CloudStorageFileSystemProvider provider, String bucket, CloudStorageConfiguration config) { checkArgument(!bucket.isEmpty(), "bucket"); - this.provider = provider; this.bucket = bucket; + if (config.useUserProjectOnlyForRequesterPaysBuckets()) { + if (Strings.isNullOrEmpty(config.userProject())) { + throw new IllegalArgumentException("If useUserProjectOnlyForRequesterPaysBuckets is set, then userProject must be set too."); + } + // detect whether we want to pay for these accesses or not. + if (!provider.requesterPays(bucket)) { + // update config (just to ease debugging, we're not actually using config.userProject later. + HashMap disableUserProject = new HashMap<>(); + disableUserProject.put("userProject", ""); + config = CloudStorageConfiguration.fromMap(config, disableUserProject); + // update the provider (this is the most important bit) + provider = provider.withNoUserProject(); + } + } + this.provider = provider; this.config = config; } diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java index 215829cddca1..b28e833763c7 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -20,6 +20,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Strings.isNullOrEmpty; +import com.google.api.gax.paging.Page; import com.google.auto.service.AutoService; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Blob; @@ -27,11 +28,12 @@ import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.CopyWriter; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobGetOption; +import com.google.cloud.storage.Storage.BlobSourceOption; import com.google.cloud.storage.StorageException; import com.google.cloud.storage.StorageOptions; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.MoreObjects; -import com.google.common.base.Throwables; import com.google.common.collect.AbstractIterator; import com.google.common.net.UrlEscapers; import com.google.common.primitives.Ints; @@ -88,7 +90,9 @@ public final class CloudStorageFileSystemProvider extends FileSystemProvider { private Storage storage; - private StorageOptions storageOptions; + final private StorageOptions storageOptions; + // if non-null, we pay via this project. + final private @Nullable String userProject; // used only when we create a new instance of CloudStorageFileSystemProvider. private static StorageOptions futureStorageOptions; @@ -168,11 +172,23 @@ public static void setDefaultCloudStorageConfiguration(@Nullable CloudStorageCon * @see CloudStorageFileSystem#forBucket(String) */ public CloudStorageFileSystemProvider() { - this(futureStorageOptions); + this("", futureStorageOptions); } - CloudStorageFileSystemProvider(@Nullable StorageOptions gcsStorageOptions) { + /** + * Internal constructor to use the user-provided default config, and a given userProject setting. + */ + CloudStorageFileSystemProvider(@Nullable String userProject) { + this(userProject, futureStorageOptions); + } + + /** + * Internal constructor, fully configurable. Note that null options means + * to use the system defaults (NOT the user-provided ones). + */ + CloudStorageFileSystemProvider(@Nullable String userProject, @Nullable StorageOptions gcsStorageOptions) { this.storageOptions = gcsStorageOptions; + this.userProject = userProject; } // Initialize this.storage, once. This may throw an exception if default authentication @@ -252,6 +268,16 @@ public CloudStoragePath getPath(String uriInStringForm) { return getPath(URI.create(escaped)); } + /** + * Open a file for reading or writing. + * To read receiver-pays buckets, specify the BlobSourceOption.userProject option. + * + * @param path: the path to the file to open or create + * @param options: options specifying how the file is opened, e.g. StandardOpenOption.WRITE or BlobSourceOption.userProject + * @param attrs: (not supported, values will be ignored) + * @return + * @throws IOException + */ @Override public SeekableByteChannel newByteChannel( Path path, Set options, FileAttribute... attrs) throws IOException { @@ -270,6 +296,7 @@ private SeekableByteChannel newReadChannel(Path path, Set throws IOException { initStorage(); int maxChannelReopens = CloudStorageUtil.getMaxChannelReopensFromPath(path); + List blobSourceOptions = new ArrayList<>(); for (OpenOption option : options) { if (option instanceof StandardOpenOption) { switch ((StandardOpenOption) option) { @@ -293,6 +320,8 @@ private SeekableByteChannel newReadChannel(Path path, Set } } else if (option instanceof OptionMaxChannelReopens) { maxChannelReopens = ((OptionMaxChannelReopens) option).maxChannelReopens(); + } else if (option instanceof BlobSourceOption) { + blobSourceOptions.add((BlobSourceOption)option); } else { throw new UnsupportedOperationException(option.toString()); } @@ -301,7 +330,13 @@ private SeekableByteChannel newReadChannel(Path path, Set if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { throw new CloudStoragePseudoDirectoryException(cloudPath); } - return CloudStorageReadChannel.create(storage, cloudPath.getBlobId(), 0, maxChannelReopens); + return CloudStorageReadChannel.create( + storage, + cloudPath.getBlobId(), + 0, + maxChannelReopens, + userProject, + blobSourceOptions.toArray(new BlobSourceOption[blobSourceOptions.size()])); } private SeekableByteChannel newWriteChannel(Path path, Set options) @@ -361,6 +396,9 @@ private SeekableByteChannel newWriteChannel(Path path, Set throw new UnsupportedOperationException(option.toString()); } } + if (!isNullOrEmpty(userProject)) { + writeOptions.add(Storage.BlobWriteOption.userProject(userProject)); + } if (!metas.isEmpty()) { infoBuilder.setMetadata(metas); @@ -413,7 +451,11 @@ public boolean deleteIfExists(Path path) throws IOException { // Loop will terminate via an exception if all retries are exhausted while (true) { try { - return storage.delete(cloudPath.getBlobId()); + if (isNullOrEmpty(userProject)) { + return storage.delete(cloudPath.getBlobId()); + } else { + return storage.delete(cloudPath.getBlobId(), Storage.BlobSourceOption.userProject(userProject)); + } } catch (StorageException exs) { // Will rethrow a StorageException if all retries/reopens are exhausted retryHandler.handleStorageException(exs); @@ -532,7 +574,12 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep while (true) { try { if ( wantCopyAttributes ) { - BlobInfo blobInfo = storage.get(fromPath.getBlobId()); + BlobInfo blobInfo; + if (isNullOrEmpty(userProject)) { + blobInfo = storage.get(fromPath.getBlobId()); + } else { + blobInfo = storage.get(fromPath.getBlobId(), BlobGetOption.userProject(userProject)); + } if ( null == blobInfo ) { throw new NoSuchFileException(fromPath.toString()); } @@ -560,6 +607,11 @@ public void copy(Path source, Path target, CopyOption... options) throws IOExcep } else { copyReqBuilder = copyReqBuilder.setTarget(tgtInfo, Storage.BlobTargetOption.doesNotExist()); } + if (!isNullOrEmpty(fromPath.getFileSystem().config().userProject())) { + copyReqBuilder = copyReqBuilder.setSourceOptions(BlobSourceOption.userProject(fromPath.getFileSystem().config().userProject())); + } else if (!isNullOrEmpty(toPath.getFileSystem().config().userProject())) { + copyReqBuilder = copyReqBuilder.setSourceOptions(BlobSourceOption.userProject(toPath.getFileSystem().config().userProject())); + } CopyWriter copyWriter = storage.copy(copyReqBuilder.build()); copyWriter.getResult(); break; @@ -611,8 +663,20 @@ public void checkAccess(Path path, AccessMode... modes) throws IOException { if ( cloudPath.seemsLikeADirectoryAndUsePseudoDirectories() ) { return; } - if ( storage.get(cloudPath.getBlobId(), Storage.BlobGetOption.fields(Storage.BlobField.ID)) - == null ) { + boolean nullId; + if (isNullOrEmpty(userProject)) { + nullId = storage.get( + cloudPath.getBlobId(), + Storage.BlobGetOption.fields(Storage.BlobField.ID)) + == null; + } else { + nullId = storage.get( + cloudPath.getBlobId(), + Storage.BlobGetOption.fields(Storage.BlobField.ID), + Storage.BlobGetOption.userProject(userProject)) + == null; + } + if (nullId) { throw new NoSuchFileException(path.toString()); } break; @@ -644,7 +708,12 @@ public A readAttributes( A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath); return result; } - BlobInfo blobInfo = storage.get(cloudPath.getBlobId()); + BlobInfo blobInfo; + if (isNullOrEmpty(userProject)) { + blobInfo = storage.get(cloudPath.getBlobId()); + } else { + blobInfo = storage.get(cloudPath.getBlobId(), BlobGetOption.userProject(userProject)); + } // null size indicate a file that we haven't closed yet, so GCS treats it as not there yet. if ( null == blobInfo || blobInfo.getSize() == null ) { throw new NoSuchFileException( @@ -705,9 +774,17 @@ public DirectoryStream newDirectoryStream(Path dir, final Filter blobIterator = storage.list(cloudPath.bucket(), - Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(), - Storage.BlobListOption.fields()).iterateAll().iterator(); + Page dirList; + if (isNullOrEmpty(userProject)) { + dirList = storage.list(cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields()); + } else { + dirList = storage.list(cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields(), Storage.BlobListOption.userProject(userProject)); + } + final Iterator blobIterator = dirList.iterateAll().iterator(); return new DirectoryStream() { @Override public Iterator iterator() { @@ -761,6 +838,35 @@ public String toString() { return MoreObjects.toStringHelper(this).add("storage", storage).toString(); } + /** + * @param bucketName the name of the bucket to check + * @return whether requester pays is enabled for that bucket + */ + public boolean requesterPays(String bucketName) { + initStorage(); + try { + // instead of true/false, this method returns true/null. + Boolean isRP = storage.get(bucketName).requesterPays(); + return isRP != null && isRP.booleanValue(); + } catch (StorageException sex) { + if (sex.getCode() == 400 && sex.getMessage().contains("Bucket is requester pays")) { + return true; + } + throw sex; + } + } + + /** + * Returns a NEW CloudStorageFileSystemProvider identical to this one, but with + * userProject removed. + * + * Perhaps you want to call this is you realize you'll be working on a bucket that is + * not requester-pays. + */ + public CloudStorageFileSystemProvider withNoUserProject() { + return new CloudStorageFileSystemProvider("", this.storageOptions); + } + private IOException asIoException(StorageException oops) { // RPC API can only throw StorageException, but CloudStorageFileSystemProvider // can only throw IOException. Square peg, round hole. @@ -775,7 +881,9 @@ private IOException asIoException(StorageException oops) { throw new FileAlreadyExistsException(((FileAlreadyExistsException) cause).getReason()); } // fallback - Throwables.propagateIfInstanceOf(oops.getCause(), IOException.class); + if (cause != null && cause instanceof IOException) { + return (IOException)cause; + } } catch (IOException okEx) { return okEx; } diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java index eb9a8c9e0a22..fdb1ec66e451 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java @@ -20,10 +20,16 @@ import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; +import com.google.cloud.storage.Storage.BlobSourceOption; import com.google.cloud.storage.StorageException; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.Lists; + +import java.util.List; import javax.annotation.CheckReturnValue; +import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import java.io.IOException; import java.nio.ByteBuffer; @@ -32,11 +38,6 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.NoSuchFileException; -import javax.net.ssl.SSLException; -import java.io.EOFException; -import java.net.SocketException; -import java.net.SocketTimeoutException; - import static com.google.common.base.Preconditions.checkArgument; /** @@ -54,6 +55,8 @@ final class CloudStorageReadChannel implements SeekableByteChannel { final int maxChannelReopens; // max # of times we may retry a GCS operation final int maxRetries; + // open options, we keep them around for reopens. + final BlobSourceOption[] blobSourceOptions; private ReadChannel channel; private long position; private long size; @@ -62,33 +65,49 @@ final class CloudStorageReadChannel implements SeekableByteChannel { private Long generation; /** - * @param maxChannelReopens max number of times to try re-opening the channel if it closes on us unexpectedly. + * @param maxChannelReopens max number of times to try re-opening the channel if it closes on us + * unexpectedly. + * @param blobSourceOptions BlobSourceOption.userProject if you want to pay the charges (required + * for requester-pays buckets). Note: + * Buckets that have Requester Pays disabled still accept requests that include a billing + * project, and charges are applied to the billing project supplied in the request. + * Consider any billing implications prior to including a billing project in all of your + * requests. + * Source: https://cloud.google.com/storage/docs/requester-pays + * @param userProject: the project you want billed (set this for requester-pays buckets). Leave + * empty otherwise. */ @CheckReturnValue @SuppressWarnings("resource") - static CloudStorageReadChannel create(Storage gcsStorage, BlobId file, long position, int maxChannelReopens) + static CloudStorageReadChannel create(Storage gcsStorage, BlobId file, long position, int maxChannelReopens, @Nullable String userProject, BlobSourceOption... blobSourceOptions) throws IOException { - return new CloudStorageReadChannel(gcsStorage, file, position, maxChannelReopens); + return new CloudStorageReadChannel(gcsStorage, file, position, maxChannelReopens, userProject, blobSourceOptions); } - private CloudStorageReadChannel(Storage gcsStorage, BlobId file, long position, int maxChannelReopens) throws IOException { + private CloudStorageReadChannel(Storage gcsStorage, BlobId file, long position, int maxChannelReopens, @Nullable String userProject, BlobSourceOption... blobSourceOptions) throws IOException { this.gcsStorage = gcsStorage; this.file = file; this.position = position; this.maxChannelReopens = maxChannelReopens; this.maxRetries = Math.max(3, maxChannelReopens); - fetchSize(gcsStorage, file); + // get the generation, enshrine that in our options + fetchSize(gcsStorage, userProject, file); + List options = Lists.newArrayList(blobSourceOptions); + if (null != generation) { + options.add(Storage.BlobSourceOption.generationMatch(generation)); + } + if (!Strings.isNullOrEmpty(userProject)) { + options.add(BlobSourceOption.userProject(userProject)); + } + this.blobSourceOptions = (BlobSourceOption[]) options.toArray(new BlobSourceOption[options.size()]); + // innerOpen checks that it sees the same generation as fetchSize did, // which ensure the file hasn't changed. innerOpen(); } private void innerOpen() throws IOException { - if (null != generation) { - this.channel = gcsStorage.reader(file, Storage.BlobSourceOption.generationMatch(generation)); - } else { - this.channel = gcsStorage.reader(file); - } + this.channel = gcsStorage.reader(file, blobSourceOptions); if (position > 0) { channel.seek(position); } @@ -183,12 +202,17 @@ private void checkOpen() throws ClosedChannelException { } } - private long fetchSize(Storage gcsStorage, BlobId file) throws IOException { + private long fetchSize(Storage gcsStorage, @Nullable String userProject, BlobId file) throws IOException { final CloudStorageRetryHandler retryHandler = new CloudStorageRetryHandler(maxRetries, maxChannelReopens); while (true) { try { - BlobInfo blobInfo = gcsStorage.get(file, Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE)); + BlobInfo blobInfo; + if (Strings.isNullOrEmpty(userProject)) { + blobInfo = gcsStorage.get(file, Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE)); + } else { + blobInfo = gcsStorage.get(file, Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE), Storage.BlobGetOption.userProject(userProject)); + } if ( blobInfo == null ) { throw new NoSuchFileException(String.format("gs://%s/%s", file.getBucket(), file.getName())); } diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java index bccab0514092..292ddff7e727 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java @@ -67,7 +67,7 @@ public void before() throws IOException { when(gcsStorage.get(file, Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE))).thenReturn(metadata); when(gcsStorage.reader(file, Storage.BlobSourceOption.generationMatch(2L))).thenReturn(gcsChannel); when(gcsChannel.isOpen()).thenReturn(true); - chan = CloudStorageReadChannel.create(gcsStorage, file, 0, 1); + chan = CloudStorageReadChannel.create(gcsStorage, file, 0, 1, ""); verify(gcsStorage).get(eq(file), eq(Storage.BlobGetOption.fields(Storage.BlobField.GENERATION, Storage.BlobField.SIZE))); verify(gcsStorage).reader(eq(file), eq(Storage.BlobSourceOption.generationMatch(2L))); } @@ -208,7 +208,7 @@ public void testChannelPositionDoesNotGetTruncatedToInt() throws IOException { // Invoke CloudStorageReadChannel.create() to trigger a call to the private // CloudStorageReadChannel.innerOpen() method, which does a seek on our gcsChannel. - CloudStorageReadChannel.create(gcsStorage, file, startPosition, 1); + CloudStorageReadChannel.create(gcsStorage, file, startPosition, 1, ""); // Confirm that our position did not overflow during the seek in CloudStorageReadChannel.innerOpen() verify(gcsChannel).seek(captor.capture()); diff --git a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java index c7af8851b21f..3b35352154cc 100644 --- a/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java +++ b/google-cloud-clients/google-cloud-contrib/google-cloud-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java @@ -19,8 +19,12 @@ import static com.google.common.truth.Truth.assertThat; import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.api.client.http.HttpResponseException; +import com.google.cloud.storage.Storage.BlobTargetOption; +import com.google.cloud.storage.StorageException; import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration; import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem; +import com.google.cloud.storage.contrib.nio.CloudStoragePath; import com.google.common.collect.ImmutableList; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.BucketInfo; @@ -49,6 +53,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; @@ -66,7 +71,7 @@ *

This test actually talks to Google Cloud Storage (you need an account) and tests both reading * and writing. You *must* set the {@code GOOGLE_APPLICATION_CREDENTIALS} environment variable for * this test to work. It must contain the name of a local file that contains your Service Account - * JSON Key. + * JSON Key. We use the project in those credentials. * *

See * Service Accounts for instructions on how to get the Service Account JSON Key. @@ -87,11 +92,14 @@ public class ITGcsNio { private static final Logger log = Logger.getLogger(ITGcsNio.class.getName()); private static final String BUCKET = RemoteStorageHelper.generateBucketName(); + private static final String REQUESTER_PAYS_BUCKET = RemoteStorageHelper.generateBucketName()+"_rp"; private static final String SML_FILE = "tmp-test-small-file.txt"; + private static final String TMP_FILE = "tmp/tmp-test-rnd-file.txt"; private static final int SML_SIZE = 100; private static final String BIG_FILE = "tmp-test-big-file.txt"; // it's big, relatively speaking. private static final int BIG_SIZE = 2 * 1024 * 1024 - 50; // arbitrary size that's not too round. private static final String PREFIX = "tmp-test-file"; + private static String project; private static Storage storage; private static StorageOptions storageOptions; @@ -102,11 +110,15 @@ public static void beforeClass() throws IOException { // loads the credentials from local disk as par README RemoteStorageHelper gcsHelper = RemoteStorageHelper.create(); storageOptions = gcsHelper.getOptions(); + project = storageOptions.getProjectId(); storage = storageOptions.getService(); // create and populate test bucket storage.create(BucketInfo.of(BUCKET)); - fillFile(storage, SML_FILE, SML_SIZE); - fillFile(storage, BIG_FILE, BIG_SIZE); + fillFile(storage, BUCKET, SML_FILE, SML_SIZE); + fillFile(storage, BUCKET, BIG_FILE, BIG_SIZE); + BucketInfo requesterPaysBucket = BucketInfo.newBuilder(REQUESTER_PAYS_BUCKET).setRequesterPays(true).build(); + storage.create(requesterPaysBucket); + fillRequesterPaysFile(storage, SML_FILE, SML_SIZE); } @AfterClass @@ -123,10 +135,164 @@ private static byte[] randomContents(int size) { return bytes; } - private static void fillFile(Storage storage, String fname, int size) throws IOException { - storage.create(BlobInfo.newBuilder(BUCKET, fname).build(), randomContents(size)); + private static void fillFile(Storage storage, String bucket, String fname, int size) throws IOException { + storage.create(BlobInfo.newBuilder(bucket, fname).build(), randomContents(size)); + } + + private static void fillRequesterPaysFile(Storage storage, String fname, int size) throws IOException { + storage.create(BlobInfo.newBuilder(REQUESTER_PAYS_BUCKET, fname).build(), randomContents(size), + BlobTargetOption.userProject(project)); + } + + // Start of tests related to the "requester pays" feature + + @Test + public void testFileExistsRequesterPaysNoUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Path path = testBucket.getPath(SML_FILE); + try { + // fails because we must pay for every access, including metadata. + Files.exists(path); + Assert.fail("It should have thrown an exception."); + } catch (StorageException sex) { + assertIsRequesterPaysException("testFileExistsRequesterPaysNoUserProject", sex); + } + } + + @Test + public void testFileExistsRequesterPays() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, project); + Path path = testBucket.getPath(SML_FILE); + // should succeed because we specified a project + Files.exists(path); + } + + @Test + public void testFileExistsRequesterPaysWithAutodetect() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(true, project); + Path path = testBucket.getPath(SML_FILE); + // should succeed because we specified a project + Files.exists(path); + } + + @Test + public void testCantCreateWithoutUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Path path = testBucket.getPath(SML_FILE); + try { + // fails + Files.write(path, "I would like to write".getBytes()); + Assert.fail("It should have thrown an exception."); + } catch (StorageException sex) { + assertIsRequesterPaysException("testCantCreateWithoutUserProject", sex); + } + } + + @Test + public void testCanCreateWithUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, project); + Path path = testBucket.getPath(TMP_FILE); + // should succeed because we specified a project + Files.write(path, "I would like to write, please?".getBytes()); + } + + @Test + public void testCantReadWithoutUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, ""); + Path path = testBucket.getPath(SML_FILE); + try { + // fails + Files.readAllBytes(path); + Assert.fail("It should have thrown an exception."); + } catch (StorageException sex) { + assertIsRequesterPaysException("testCantReadWithoutUserProject", sex); + } + } + + @Test + public void testCanReadWithUserProject() throws IOException { + CloudStorageFileSystem testBucket = getRequesterPaysBucket(false, project); + Path path = testBucket.getPath(SML_FILE); + // should succeed because we specified a project + Files.readAllBytes(path); } + @Test + public void testCantCopyWithoutUserProject() throws IOException { + CloudStorageFileSystem testRPBucket = getRequesterPaysBucket(false, ""); + CloudStorageFileSystem testBucket = getTestBucket(); + CloudStoragePath[] sources = new CloudStoragePath[] { testBucket.getPath(SML_FILE), testRPBucket.getPath(SML_FILE) }; + CloudStoragePath[] dests = new CloudStoragePath[] {testBucket.getPath(TMP_FILE), testRPBucket.getPath(TMP_FILE) }; + for (int s=0; s<2; s++) { + for (int d=0; d<2; d++) { + // normal to normal is out of scope of RP testing. + if (s==0 && d==0) { + continue; + } + innerTestCantCopyWithoutUserProject(s==0, d==0, sources[s], dests[d]); + } + } + } + + // Try to copy the file, make sure that we were prevented. + private void innerTestCantCopyWithoutUserProject(boolean sourceNormal, boolean destNormal, Path source, Path dest) throws IOException { + String sdesc = (sourceNormal?"normal bucket":"requester-pays bucket"); + String ddesc = (destNormal?"normal bucket":"requester-pays bucket"); + String description = "Copying from " + sdesc + " to " + ddesc; + try { + Files.copy(source, dest); + Assert.fail("Shouldn't have been able to copy from " + sdesc + " to " + ddesc); + // for some reason this throws "GoogleJsonResponseException" instead of "StorageException" + // when going from requester pays bucket to requester pays bucket, but otherwise we get a + // normal StorageException. + } catch (HttpResponseException hex) { + Assert.assertEquals(description, hex.getStatusCode(), 400); + Assert.assertTrue(description, hex.getMessage().contains("Bucket is requester pays bucket but no user project provided")); + } catch (StorageException sex) { + assertIsRequesterPaysException(description, sex); + } + } + + @Test + public void testCanCopyWithUserProject() throws IOException { + CloudStorageFileSystem testRPBucket = getRequesterPaysBucket(false, project); + CloudStorageFileSystem testBucket = getTestBucket(); + CloudStoragePath[] sources = new CloudStoragePath[] {testBucket.getPath(SML_FILE), testRPBucket.getPath(SML_FILE)}; + CloudStoragePath[] dests = new CloudStoragePath[] {testBucket.getPath(TMP_FILE), testRPBucket.getPath(TMP_FILE)}; + for (int s = 0; s < 2; s++) { + for (int d = 0; d < 2; d++) { + // normal to normal is out of scope of RP testing. + if (s == 0 && d == 0) { + continue; + } + Files.copy(sources[s], dests[d], StandardCopyOption.REPLACE_EXISTING); + } + } + } + + @Test + public void testAutodetectWhenRequesterPays() throws IOException { + CloudStorageFileSystem testRPBucket = getRequesterPaysBucket(true, project); + Assert.assertEquals("Autodetect should have detected the RP bucket", testRPBucket.config().userProject(), project); + + } + + @Test + public void testAutodetectWhenNotRequesterPays() throws IOException { + CloudStorageConfiguration config = CloudStorageConfiguration.builder() + .autoDetectRequesterPays(true) + .userProject(project).build(); + CloudStorageFileSystem testBucket = CloudStorageFileSystem.forBucket(BUCKET, config, storageOptions); + Assert.assertEquals("Autodetect should have detected the bucket is not RP", testBucket.config().userProject(), ""); + } + + private void assertIsRequesterPaysException(String message, StorageException sex) { + Assert.assertEquals(message, sex.getCode(), 400); + Assert.assertTrue(message, sex.getMessage().contains("Bucket is requester pays bucket but no user project provided")); + } + + // End of tests related to the "requester pays" feature + @Test public void testFileExists() throws IOException { CloudStorageFileSystem testBucket = getTestBucket(); @@ -346,7 +512,7 @@ public void testListFiles() throws IOException { paths.addAll(goodPaths); goodPaths.add(fs.getPath("dir/dir2/")); for (Path path : paths) { - fillFile(storage, path.toString(), SML_SIZE); + fillFile(storage, BUCKET, path.toString(), SML_SIZE); } List got = new ArrayList<>(); @@ -458,4 +624,12 @@ private CloudStorageFileSystem getTestBucket() throws IOException { BUCKET, CloudStorageConfiguration.DEFAULT, storageOptions); } + // same as getTestBucket, but for the requester-pays bucket. + private CloudStorageFileSystem getRequesterPaysBucket(boolean autodetect, String userProject) throws IOException { + CloudStorageConfiguration config = CloudStorageConfiguration.builder() + .autoDetectRequesterPays(autodetect) + .userProject(userProject).build(); + return CloudStorageFileSystem.forBucket( + REQUESTER_PAYS_BUCKET, config, storageOptions); + } }