diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryError.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryError.java index 121ca578c0d3..b243cf8b1f99 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryError.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryError.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.bigquery; import com.google.api.services.bigquery.model.ErrorProto; diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java index 90ad164a7ce5..84f882fb2c53 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatistics.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.bigquery; import com.google.api.services.bigquery.model.JobStatistics2; diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatus.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatus.java index 7c948e6373f9..b2bfe99a503d 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatus.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/JobStatus.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.bigquery; import com.google.common.base.MoreObjects; diff --git a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/UserDefinedFunction.java b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/UserDefinedFunction.java index 09fa2563c59f..36ac2c27485e 100644 --- a/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/UserDefinedFunction.java +++ b/gcloud-java-bigquery/src/main/java/com/google/cloud/bigquery/UserDefinedFunction.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.bigquery; import com.google.api.services.bigquery.model.UserDefinedFunctionResource; diff --git a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryErrorTest.java b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryErrorTest.java index 1d30bb87815e..c63095b96528 100644 --- a/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryErrorTest.java +++ b/gcloud-java-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryErrorTest.java @@ -1,3 +1,19 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package com.google.cloud.bigquery; import static org.junit.Assert.assertEquals; diff --git a/gcloud-java-contrib/README.md b/gcloud-java-contrib/README.md index 4b93f6fafa9f..85808ae24a07 100644 --- a/gcloud-java-contrib/README.md +++ b/gcloud-java-contrib/README.md @@ -9,8 +9,17 @@ Packages that provide higher-level abstraction/functionality for common gcloud-j [![Codacy Badge](https://api.codacy.com/project/badge/grade/9da006ad7c3a4fe1abd142e77c003917)](https://www.codacy.com/app/mziccard/gcloud-java) [![Dependency Status](https://www.versioneye.com/user/projects/56bd8ee72a29ed002d2b0969/badge.svg?style=flat)](https://www.versioneye.com/user/projects/56bd8ee72a29ed002d2b0969) +Contents +-------- + + * [gcloud-java-nio](./gcloud-java-nio/): NIO Filesystem Provider for Google Cloud Storage. + * [gcloud-java-nio-examples](./gcloud-java-nio-examples/): How to add Google Cloud Storage NIO after the fact. + Quickstart ---------- + +### gcloud-java-nio + If you are using Maven, add this to your pom.xml file ```xml @@ -28,6 +37,10 @@ If you are using SBT, add this to your dependencies libraryDependencies += "com.google.cloud" % "gcloud-java-contrib" % "0.2.4" ``` +### gcloud-java-nio-examples + +See its [README](./gcloud-java-nio-examples/README.md) for instructions on how to run it. + Java Versions ------------- @@ -42,6 +55,11 @@ It is currently in major version zero (``0.y.z``), which means that anything may change at any time and the public API should not be considered stable. +See also +-------- + + * [gcloud-java-examples](../gcloud-java-examples) for an example of how to use NIO normally. + Contributing ------------ diff --git a/gcloud-java-contrib/gcloud-java-nio-examples/README.md b/gcloud-java-contrib/gcloud-java-nio-examples/README.md new file mode 100644 index 000000000000..9a07c31177f2 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio-examples/README.md @@ -0,0 +1,44 @@ +Example of adding the Google Cloud Storage NIO Provider to a legacy jar +======================================================================= + +This project shows how to add Google Cloud Storage capabilities to a jar file for a Java 7 +application that uses Java NIO without the need to recompile. + +Note that whenever possible, you instead want to recompile the app and use the normal +dependency mechanism to add a dependency to gcloud-java-nio. You can see examples of +this in the [gcloud-java-examples](../../gcloud-java-examples) project. + +To run this example: + +1. Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled. + +2. Log in using gcloud SDK (`gcloud auth login` in command line) + +3. Compile the JAR with: + ``` + mvn package -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true + ``` + +4. Run the sample with: + + ``` + java -cp gcloud-java-contrib/gcloud-java-nio/target/gcloud-java-nio-0.2.5-SNAPSHOT-shaded.jar:gcloud-java-contrib/gcloud-java-nio-examples/target/gcloud-java-nio-examples-0.2.5-SNAPSHOT.jar com.google.cloud.nio.examples.ListFilesystems + ``` + + Notice that it lists Google Cloud Storage, which it wouldn't if you ran it without the NIO jar: + ``` + java -cp gcloud-java-contrib/gcloud-java-nio-examples/target/gcloud-java-nio-examples-0.2.5-SNAPSHOT.jar com.google.cloud.nio.examples.ListFilesystems + ``` + +The sample doesn't have anything about Google Cloud Storage in it. It gets that ability from the NIO +jar that we're adding to the classpath. You can use the NIO "fat shaded" jar for this purpose as it +also includes the dependencies for gcloud-java-nio. +The underlying mechanism is Java's standard [ServiceLoader](https://docs.oracle.com/javase/7/docs/api/java/util/ServiceLoader.html) +facility, the [standard way](http://docs.oracle.com/javase/7/docs/technotes/guides/io/fsp/filesystemprovider.html) to plug in NIO providers like this one. + +If you have access to a project's source code you can also simply add gcloud-java-nio as +a dependency and let Maven pull in the required dependencies (this is what the NIO unit tests do). +This approach is preferable as the fat jar approach may waste memory on multiple copies of +dependencies. + +[developers-console]:https://console.developers.google.com/ diff --git a/gcloud-java-contrib/gcloud-java-nio-examples/pom.xml b/gcloud-java-contrib/gcloud-java-nio-examples/pom.xml new file mode 100644 index 000000000000..448c38a2e8e2 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio-examples/pom.xml @@ -0,0 +1,92 @@ + + + 4.0.0 + gcloud-java-nio-examples + jar + GCloud Java NIO examples + https://github.com/GoogleCloudPlatform/gcloud-java/tree/master/gcloud-java-contrib/gcloud-java-nio-examples + + Demonstrates how to use the gcloud-java-nio jar to add Google Cloud Storage functionality to legacy code. + + + com.google.cloud + gcloud-java-contrib + 0.2.5-SNAPSHOT + + + nio + + + + ${project.groupId} + gcloud-java-storage + ${project.version} + + + com.google.guava + guava + 19.0 + + + com.google.code.findbugs + jsr305 + 2.0.1 + + + javax.inject + javax.inject + 1 + + + com.google.auto.service + auto-service + 1.0-rc2 + provided + + + com.google.auto.value + auto-value + 1.1 + provided + + + junit + junit + 4.12 + test + + + com.google.guava + guava-testlib + 19.0 + test + + + com.google.truth + truth + 0.27 + test + + + org.mockito + mockito-core + 1.9.5 + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.4 + + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + + diff --git a/gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/cloud/nio/examples/ListFilesystems.java b/gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/cloud/nio/examples/ListFilesystems.java new file mode 100644 index 000000000000..b03dc13763d1 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio-examples/src/main/java/com/google/cloud/nio/examples/ListFilesystems.java @@ -0,0 +1,47 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.nio.examples; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.spi.FileSystemProvider; + +/** + * ListFilesystems is a super-simple program that lists the available NIO filesystems. + */ +public class ListFilesystems { + + /** + * See the class documentation. + */ + public static void main(String[] args) throws IOException { + listFilesystems(); + } + + private static void listFilesystems() { + System.out.println("Installed filesystem providers:"); + for (FileSystemProvider p : FileSystemProvider.installedProviders()) { + System.out.println(" " + p.getScheme()); + } + } + +} diff --git a/gcloud-java-contrib/gcloud-java-nio/README.md b/gcloud-java-contrib/gcloud-java-nio/README.md new file mode 100644 index 000000000000..ecb541971020 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/README.md @@ -0,0 +1,210 @@ +NIO Filesystem Provider for Google Cloud Storage (Alpha) +======================================================== + +Implementation of Java 7 `java.nio.file.FileSystem` for +[Google Cloud Storage](https://cloud.google.com/storage/). + +This library allows you to use the standardized Java file system API +for interacting with Google Cloud Storage. + +[![Build Status](https://travis-ci.org/GoogleCloudPlatform/gcloud-java.svg?branch=master)](https://travis-ci.org/GoogleCloudPlatform/gcloud-java) +[![Coverage Status](https://coveralls.io/repos/GoogleCloudPlatform/gcloud-java/badge.svg?branch=master)](https://coveralls.io/r/GoogleCloudPlatform/gcloud-java?branch=master) +[![Maven](https://img.shields.io/maven-central/v/com.google.cloud/gcloud-java-nio.svg)]( https://img.shields.io/maven-central/v/com.google.cloud/gcloud-java-nio.svg) +[![Codacy Badge](https://api.codacy.com/project/badge/grade/9da006ad7c3a4fe1abd142e77c003917)](https://www.codacy.com/app/mziccard/gcloud-java) +[![Dependency Status](https://www.versioneye.com/user/projects/56bd8ee72a29ed002d2b0969/badge.svg?style=flat)](https://www.versioneye.com/user/projects/56bd8ee72a29ed002d2b0969) + +- [Homepage](https://googlecloudplatform.github.io/gcloud-java/) +- [API Documentation](http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/cloud/storage/package-summary.html) + +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. + +Quickstart +---------- +If you are using Maven, add this to your pom.xml file +```xml + + com.google.cloud + gcloud-java-nio + 0.2.4 + +``` +If you are using Gradle, add this to your dependencies +```Groovy +compile 'com.google.cloud:gcloud-java-nio:0.2.4' +``` +If you are using SBT, add this to your dependencies +```Scala +libraryDependencies += "com.google.cloud" % "gcloud-java-nio" % "0.2.4" +``` + +Example Applications +------------------- + +* [`Stat`](../../gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java) +shows how to get started with NIO. + +* [`ParallelCountBytes`](../../gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java) +efficiently downloads a file from Google Cloud Storage. + +* [`ListFileSystems`](../gcloud-java-nio-examples/README.md) illustrates how +NIO can add Google Cloud Storage support to some legacy programs, without +having to modify them. + + +Authentication +-------------- + +See the [Authentication](https://github.com/GoogleCloudPlatform/gcloud-java#authentication) +section in the base directory's README. + +About Google Cloud Storage +-------------------------- + +[Google Cloud Storage][cloud-storage] is a durable and highly available +object storage service. Google Cloud Storage is almost infinitely scalable +and guarantees consistency: when a write succeeds, the latest copy of the +object will be returned to any GET, globally. + +See the [Google Cloud Storage docs][cloud-storage-activation] for more details +on how to activate Cloud Storage for your project. + +About Java NIO Providers +------------------------ + +Java NIO Providers is an extension mechanism that is part of Java and allows +third parties to extend Java's [normal File API][java-file-api] to support +additional filesystems. + +Getting Started +--------------- +#### Prerequisites + +For this tutorial, you will need a [Google Developers +Console](https://console.developers.google.com/) project with "Google Cloud +Storage" and "Google Cloud Storage JSON API" enabled via the console's API +Manager. You will need to [enable +billing](https://support.google.com/cloud/answer/6158867?hl=en) to use Google +Cloud Storage. [Follow these +instructions](https://cloud.google.com/docs/authentication#preparation) to get +your project set up. You will also need to set up the local development +environment by [installing the Google Cloud SDK](https://cloud.google.com/sdk/) +and running the following commands in command line: `gcloud auth login` and +`gcloud config set project [YOUR PROJECT ID]`. + +#### Installation and setup +You'll need to obtain the `gcloud-java-nio` library. + +There are two ways to use this library. + +The recommended way is to follow the [Quickstart](#quickstart) section to add +`gcloud-java-nio` as a dependency in your code. + +The second way is more complicated, but it allows you to add Google Cloud +Storage support to some legacy Java programs. This approach is described in the +[gcloud-java-nio-examples README](../gcloud-java-nio-examples/README.md). + +#### Accessing files + +The simplest way to get started is with `Paths` and `Files`: + + Path path = Paths.get(URI.create("gs://bucket/lolcat.csv")); + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + +If you know the paths will point to Google Cloud Storage, you can also use the +direct formulation: + + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("bucket") { + Path path = fs.getPath("lolcat.csv"); + List lines = Files.readAllLines(path, StandardCharsets.UTF_8); + } + +Once you have a `Path` you can use it as you would a normal file. For example +you can use `InputStream` and `OutputStream` for streaming: + + try (InputStream input = Files.openInputStream(path)) { + // ... + } + +You can also set various attributes using CloudStorageOptions static helpers: + + Files.write(csvPath, csvLines, StandardCharsets.UTF_8, + withMimeType(MediaType.CSV_UTF8), + withoutCaching()); + +Limitations +----------- + +This library is usable, but not yet complete. The following features are not +yet implemented: + * Listing all the buckets + * Resuming upload or download + * Generations + * File attributes + * (more - list is not exhaustive) + +Some features are not on the roadmap: this library would be a poor choice to +mirror a local filesystem onto the cloud because Google Cloud Storage has a +different set of features from your local disk. This library, by design, +does not mask those differences. Rather, it aims to expose the common +subset via a familiar interface. + +**NOTE:** Cloud Storage uses a flat namespace and therefore doesn't support real +directories. So this library supports what's known as "pseudo-directories". Any +path that includes a trailing slash, will be considered a directory. It will +always be assumed to exist, without performing any I/O. This allows you to do +path manipulation in the same manner as you would with the normal UNIX file +system implementation. You can disable this feature with +`CloudStorageConfiguration.usePseudoDirectories()`. + +#### Complete source code + +There are examples in [gcloud-java-examples](../gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/) +for your perusal. + +Java Versions +------------- + +Java 7 or above is required for using this client. + +Versioning +---------- + +This library follows [Semantic Versioning](http://semver.org/). + +It is currently in major version zero (``0.y.z``), which means that anything +may change at any time and the public API should not be considered +stable. + +Contributing +------------ + +Contributions to this library are always welcome and highly encouraged. + +See `gcloud-java`'s [CONTRIBUTING] documentation and the `gcloud-*` +[shared documentation](https://github.com/GoogleCloudPlatform/gcloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) +for more information on how to get started. + +Please note that this project is released with a Contributor Code of Conduct. +By participating in this project you agree to abide by its terms. See +[Code of Conduct][code-of-conduct] for more information. + +License +------- + +Apache 2.0 - See [LICENSE] for more information. + + +[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CONTRIBUTING.md +[code-of-conduct]:https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[LICENSE]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/LICENSE +[TESTING]: https://github.com/GoogleCloudPlatform/gcloud-java/blob/master/TESTING.md#testing-code-that-uses-storage +[cloud-platform]: https://cloud.google.com/ + +[cloud-storage]: https://cloud.google.com/storage/ +[cloud-storage-docs]: https://cloud.google.com/storage/docs/overview +[cloud-storage-create-bucket]: https://cloud.google.com/storage/docs/cloud-console#_creatingbuckets +[storage-api]: http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/cloud/storage/package-summary.html +[cloud-storage-activation]:https://cloud.google.com/storage/docs/signup?hl=en + +[java-file-api]: https://docs.oracle.com/javase/7/docs/api/java/nio/file/Files.html diff --git a/gcloud-java-contrib/gcloud-java-nio/pom.xml b/gcloud-java-contrib/gcloud-java-nio/pom.xml new file mode 100644 index 000000000000..a63b8b45856e --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/pom.xml @@ -0,0 +1,131 @@ + + + 4.0.0 + gcloud-java-nio + jar + GCloud Java NIO + https://github.com/GoogleCloudPlatform/gcloud-java/tree/master/gcloud-java-contrib/gcloud-java-nio + + FileSystemProvider for Java NIO to access Google Cloud Storage transparently. + + + com.google.cloud + gcloud-java-contrib + 0.2.5-SNAPSHOT + + + nio + + + + ${project.groupId} + gcloud-java + ${project.version} + + + com.google.guava + guava + 19.0 + + + com.google.code.findbugs + jsr305 + 2.0.1 + + + javax.inject + javax.inject + 1 + + + com.google.auto.service + auto-service + 1.0-rc2 + provided + + + com.google.auto.value + auto-value + 1.1 + provided + + + junit + junit + 4.12 + test + + + com.google.guava + guava-testlib + 19.0 + test + + + com.google.truth + truth + 0.27 + test + + + org.mockito + mockito-core + 1.9.5 + + + + + + org.codehaus.mojo + exec-maven-plugin + + false + + + + maven-compiler-plugin + 3.1 + + 1.7 + 1.7 + UTF-8 + -Xlint:unchecked + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + true + + + com + shaded.cloud-nio.com + + com.google.cloud.** + + + + org + shaded.cloud-nio.org + + + google + shaded.cloud-nio.google + + + + + + package + + shade + + + + + + + diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java new file mode 100644 index 000000000000..efa086d7db80 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageConfiguration.java @@ -0,0 +1,179 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.auto.value.AutoValue; + +import java.util.Map; + +/** + * Configuration for {@link CloudStorageFileSystem} instances. + */ +@AutoValue +public abstract class CloudStorageConfiguration { + + public static final CloudStorageConfiguration DEFAULT = builder().build(); + + /** + * Returns path of current working directory. This defaults to the root directory. + */ + public abstract String workingDirectory(); + + /** + * Returns {@code true} if we shouldn't throw an exception when encountering object names + * containing superfluous slashes, e.g. {@code a//b}. + */ + public abstract boolean permitEmptyPathComponents(); + + /** + * Returns {@code true} if '/' prefix on absolute object names should be removed before I/O. + * + *

If you disable this feature, please take into consideration that all paths created from a + * URI will have the leading slash. + */ + public abstract boolean stripPrefixSlash(); + + /** + * Returns {@code true} if paths with a trailing slash should be treated as fake directories. + */ + public abstract boolean usePseudoDirectories(); + + /** + * Returns block size (in bytes) used when talking to the Google Cloud Storage HTTP server. + */ + public abstract int blockSize(); + + /** + * Creates a new builder, initialized with the following settings: + * + *

    + *
  • Performing I/O on paths with extra slashes, e.g. {@code a//b} will throw an error. + *
  • The prefix slash on absolute paths will be removed when converting to an object name. + *
  • Pseudo-directories are enabled, so any path with a trailing slash is a fake directory. + *
+ */ + public static Builder builder() { + return new Builder(); + } + + /** + * Builder for {@link CloudStorageConfiguration}. + */ + public static final class Builder { + + private String workingDirectory = UnixPath.ROOT; + private boolean permitEmptyPathComponents; + private boolean stripPrefixSlash = true; + private boolean usePseudoDirectories = true; + private int blockSize = CloudStorageFileSystem.BLOCK_SIZE_DEFAULT; + + /** + * Changes current working directory for new filesystem. This defaults to the root directory. + * The working directory cannot be changed once it's been set. You'll need to create another + * {@link CloudStorageFileSystem} object. + * + * @throws IllegalArgumentException if {@code path} is not absolute. + */ + public Builder workingDirectory(String path) { + checkArgument(UnixPath.getPath(false, path).isAbsolute(), "not absolute: %s", path); + workingDirectory = path; + return this; + } + + /** + * Configures whether or not we should throw an exception when encountering object names + * containing superfluous slashes, e.g. {@code a//b}. + */ + public Builder permitEmptyPathComponents(boolean value) { + permitEmptyPathComponents = value; + return this; + } + + /** + * Configures if the '/' prefix on absolute object names should be removed before I/O. + * + *

If you disable this feature, please take into consideration that all paths created from a + * URI will have the leading slash. + */ + public Builder stripPrefixSlash(boolean value) { + stripPrefixSlash = value; + return this; + } + + /** + * Configures if paths with a trailing slash should be treated as fake directories. + */ + public Builder usePseudoDirectories(boolean value) { + usePseudoDirectories = value; + return this; + } + + /** + * Sets the block size in bytes that should be used for each HTTP request to the API. + * + *

The default is {@value CloudStorageFileSystem#BLOCK_SIZE_DEFAULT}. + */ + public Builder blockSize(int value) { + blockSize = value; + return this; + } + + /** + * Creates new instance without destroying builder. + */ + public CloudStorageConfiguration build() { + return new AutoValue_CloudStorageConfiguration( + workingDirectory, + permitEmptyPathComponents, + stripPrefixSlash, + usePseudoDirectories, + blockSize); + } + + Builder() {} + } + + static CloudStorageConfiguration fromMap(Map env) { + Builder builder = builder(); + for (Map.Entry entry : env.entrySet()) { + switch (entry.getKey()) { + case "workingDirectory": + builder.workingDirectory((String) entry.getValue()); + break; + case "permitEmptyPathComponents": + builder.permitEmptyPathComponents((Boolean) entry.getValue()); + break; + case "stripPrefixSlash": + builder.stripPrefixSlash((Boolean) entry.getValue()); + break; + case "usePseudoDirectories": + builder.usePseudoDirectories((Boolean) entry.getValue()); + break; + case "blockSize": + builder.blockSize((Integer) entry.getValue()); + break; + default: + throw new IllegalArgumentException(entry.getKey()); + } + } + return builder.build(); + } + + CloudStorageConfiguration() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java new file mode 100644 index 000000000000..11b92116b4c3 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java @@ -0,0 +1,93 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; +import com.google.common.base.MoreObjects; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.Objects; + +import javax.annotation.concurrent.Immutable; + +/** + * Metadata view for a Google Cloud Storage object. + */ +@Immutable +public final class CloudStorageFileAttributeView implements BasicFileAttributeView { + + private final Storage storage; + private final CloudStoragePath path; + + CloudStorageFileAttributeView(Storage storage, CloudStoragePath path) { + this.storage = checkNotNull(storage); + this.path = checkNotNull(path); + } + + /** + * Returns {@value CloudStorageFileSystem#GCS_VIEW}. + */ + @Override + public String name() { + return CloudStorageFileSystem.GCS_VIEW; + } + + @Override + public CloudStorageFileAttributes readAttributes() throws IOException { + if (path.seemsLikeADirectory() && path.getFileSystem().config().usePseudoDirectories()) { + return new CloudStoragePseudoDirectoryAttributes(path); + } + BlobInfo blobInfo = storage.get(path.getBlobId()); + if (blobInfo == null) { + throw new NoSuchFileException(path.toUri().toString()); + } + + return new CloudStorageObjectAttributes(blobInfo); + } + + /** + * This feature is not supported, since Cloud Storage objects are immutable. + */ + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) { + throw new CloudStorageObjectImmutableException(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageFileAttributeView + && Objects.equals(storage, ((CloudStorageFileAttributeView) other).storage) + && Objects.equals(path, ((CloudStorageFileAttributeView) other).path); + } + + @Override + public int hashCode() { + return Objects.hash(storage, path); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("storage", storage).add("path", path).toString(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java new file mode 100644 index 000000000000..ff77f0b96ab6 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java @@ -0,0 +1,79 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.Acl; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.attribute.BasicFileAttributes; +import java.util.List; + +/** + * Interface for attributes on a Cloud Storage file or pseudo-directory. + */ +public interface CloudStorageFileAttributes extends BasicFileAttributes { + + /** + * Returns HTTP etag hash of object contents. + * + * @see "https://developers.google.com/storage/docs/hashes-etags" + */ + Optional etag(); + + /** + * Returns mime type (e.g. text/plain), if set. + * + * @see "http://en.wikipedia.org/wiki/Internet_media_type#List_of_common_media_types" + */ + Optional mimeType(); + + /** + * Returns access control list. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + Optional> acl(); + + /** + * Returns {@code Cache-Control} HTTP header value, if set. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + Optional cacheControl(); + + /** + * Returns {@code Content-Encoding} HTTP header value, if set. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + Optional contentEncoding(); + + /** + * Returns {@code Content-Disposition} HTTP header value, if set. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + Optional contentDisposition(); + + /** + * Returns user-specified metadata. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + ImmutableMap userMetadata(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java new file mode 100644 index 000000000000..60a39fb5a817 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java @@ -0,0 +1,261 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.cloud.storage.StorageOptions; +import com.google.common.collect.ImmutableSet; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.FileTime; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.util.Objects; +import java.util.Set; + +import javax.annotation.CheckReturnValue; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Google Cloud Storage {@link FileSystem} implementation. + * + * @see + * Concepts and Terminology + * @see + * Bucket and Object Naming Guidelines + */ +@ThreadSafe +public final class CloudStorageFileSystem extends FileSystem { + + public static final String URI_SCHEME = "gs"; + public static final String GCS_VIEW = "gcs"; + public static final String BASIC_VIEW = "basic"; + public static final int BLOCK_SIZE_DEFAULT = 2 * 1024 * 1024; + public static final FileTime FILE_TIME_UNKNOWN = FileTime.fromMillis(0); + public static final ImmutableSet SUPPORTED_VIEWS = ImmutableSet.of(BASIC_VIEW, GCS_VIEW); + + private final CloudStorageFileSystemProvider provider; + private final String bucket; + private final CloudStorageConfiguration config; + + /** + * Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}. + * + *

NOTE: You may prefer to use Java's standard API instead:

   {@code
+   *
+   *   FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"));}
+ * + *

However some systems and build environments might be flaky when it comes to Java SPI. This + * is because services are generally runtime dependencies and depend on a META-INF file being + * present in your jar (generated by Google Auto at compile-time). In such cases, this method + * provides a simpler alternative. + * + * @see #forBucket(String, CloudStorageConfiguration) + * @see java.nio.file.FileSystems#getFileSystem(java.net.URI) + */ + @CheckReturnValue + public static CloudStorageFileSystem forBucket(String bucket) { + return forBucket(bucket, CloudStorageConfiguration.DEFAULT); + } + + /** + * Creates new file system instance for {@code bucket}, with customizable settings. + * + * @see #forBucket(String) + */ + @CheckReturnValue + public static CloudStorageFileSystem forBucket(String bucket, CloudStorageConfiguration config) { + checkArgument( + !bucket.startsWith(URI_SCHEME + ":"), "Bucket name must not have schema: %s", bucket); + return new CloudStorageFileSystem( + new CloudStorageFileSystemProvider(), bucket, checkNotNull(config)); + } + + /** + * Returns Google Cloud Storage {@link FileSystem} object for {@code bucket}. + * + *

Google Cloud Storage file system objects are basically free. You can create as many as you + * want, even if you have multiple instances for the same bucket. There's no actual system + * resources associated with this object. Therefore calling {@link #close()} on the returned value + * is optional. + * + *

Note: It is also possible to instantiate this class via Java's + * {@code FileSystems.getFileSystem(URI.create("gs://bucket"))}. We discourage you + * from using that if possible, for the reasons documented in + * {@link CloudStorageFileSystemProvider#newFileSystem(URI, java.util.Map)} + * + * @see java.nio.file.FileSystems#getFileSystem(URI) + */ + @CheckReturnValue + public static CloudStorageFileSystem forBucket(String bucket, CloudStorageConfiguration config, + @Nullable StorageOptions storageOptions) { + checkArgument(!bucket.startsWith(URI_SCHEME + ":"), + "Bucket name must not have schema: %s", bucket); + return new CloudStorageFileSystem(new CloudStorageFileSystemProvider(storageOptions), + bucket, checkNotNull(config)); + } + + CloudStorageFileSystem( + CloudStorageFileSystemProvider provider, String bucket, CloudStorageConfiguration config) { + checkArgument(!bucket.isEmpty(), "bucket"); + this.provider = provider; + this.bucket = bucket; + this.config = config; + } + + @Override + public CloudStorageFileSystemProvider provider() { + return provider; + } + + /** + * Returns Cloud Storage bucket name being served by this file system. + */ + public String bucket() { + return bucket; + } + + /** + * Returns configuration object for this file system instance. + */ + public CloudStorageConfiguration config() { + return config; + } + + /** + * Converts Cloud Storage object name to a {@link Path} object. + */ + @Override + public CloudStoragePath getPath(String first, String... more) { + checkArgument( + !first.startsWith(URI_SCHEME + ":"), + "Google Cloud Storage FileSystem.getPath() must not have schema and bucket name: %s", + first); + return CloudStoragePath.getPath(this, first, more); + } + + /** + * Does nothing currently. This method might be updated in the future to close all channels + * associated with this file system object. However it's unlikely that even then, calling this + * method will become mandatory. + */ + @Override + public void close() throws IOException { + // TODO(#809): Synchronously close all channels associated with this FileSystem instance. + } + + /** + * Returns {@code true}, even if you previously called the {@link #close()} method. + */ + @Override + public boolean isOpen() { + return true; + } + + /** + * Returns {@code false}. + */ + @Override + public boolean isReadOnly() { + return false; + } + + /** + * Returns {@value UnixPath#SEPARATOR}. + */ + @Override + public String getSeparator() { + return Character.toString(UnixPath.SEPARATOR); + } + + @Override + public Iterable getRootDirectories() { + return ImmutableSet.of(CloudStoragePath.getPath(this, UnixPath.ROOT)); + } + + /** + * Returns nothing because Google Cloud Storage doesn't have disk partitions of limited size, or + * anything similar. + */ + @Override + public Iterable getFileStores() { + return ImmutableSet.of(); + } + + @Override + public Set supportedFileAttributeViews() { + return SUPPORTED_VIEWS; + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public PathMatcher getPathMatcher(String syntaxAndPattern) { + // TODO(#813): Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public WatchService newWatchService() throws IOException { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageFileSystem + && Objects.equals(config, ((CloudStorageFileSystem) other).config) + && Objects.equals(bucket, ((CloudStorageFileSystem) other).bucket); + } + + @Override + public int hashCode() { + return Objects.hash(bucket); + } + + @Override + public String toString() { + try { + return new URI(URI_SCHEME, bucket, null, null).toString(); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java new file mode 100644 index 000000000000..624c410ddb5c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java @@ -0,0 +1,647 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Strings.isNullOrEmpty; + +import com.google.auto.service.AutoService; +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.CopyWriter; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.StorageOptions; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.MoreObjects; +import com.google.common.base.Throwables; +import com.google.common.collect.AbstractIterator; +import com.google.common.primitives.Ints; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessMode; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.DirectoryStream; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileStore; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; +import javax.inject.Singleton; + +/** + * Google Cloud Storage {@link FileSystemProvider} implementation. + * + *

Note: This class should never be used directly. This class is instantiated by the + * service loader and called through a standardized API, e.g. {@link java.nio.file.Files}. However + * the javadocs in this class serve as useful documentation for the behavior of the Google Cloud + * Storage NIO library. + */ +@Singleton +@ThreadSafe +@AutoService(FileSystemProvider.class) +public final class CloudStorageFileSystemProvider extends FileSystemProvider { + + private final Storage storage; + + // used only when we create a new instance of CloudStorageFileSystemProvider. + private static StorageOptions storageOptions; + + private static class LazyPathIterator extends AbstractIterator { + private final Iterator blobIterator; + private final Filter filter; + private final CloudStorageFileSystem fileSystem; + + LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator blobIterator, + Filter filter) { + this.blobIterator = blobIterator; + this.filter = filter; + this.fileSystem = fileSystem; + } + + @Override + protected Path computeNext() { + while (blobIterator.hasNext()) { + Path path = fileSystem.getPath(blobIterator.next().name()); + try { + if (filter.accept(path)) { + return path; + } + } catch (IOException ex) { + throw new DirectoryIteratorException(ex); + } + } + return endOfData(); + } + } + + /** + * Sets options that are only used by the constructor. + */ + @VisibleForTesting + public static void setGCloudOptions(StorageOptions newStorageOptions) { + storageOptions = newStorageOptions; + } + + /** + * Default constructor which should only be called by Java SPI. + * + * @see java.nio.file.FileSystems#getFileSystem(URI) + * @see CloudStorageFileSystem#forBucket(String) + */ + public CloudStorageFileSystemProvider() { + this(storageOptions); + } + + CloudStorageFileSystemProvider(@Nullable StorageOptions gcsStorageOptions) { + if (gcsStorageOptions == null) { + this.storage = StorageOptions.defaultInstance().service(); + } else { + this.storage = gcsStorageOptions.service(); + } + } + + @Override + public String getScheme() { + return CloudStorageFileSystem.URI_SCHEME; + } + + /** + * Returns Cloud Storage file system, provided a URI with no path, e.g. {@code gs://bucket}. + */ + @Override + public CloudStorageFileSystem getFileSystem(URI uri) { + return newFileSystem(uri, Collections.emptyMap()); + } + + /** + * Returns Cloud Storage file system, provided a URI with no path, e.g. {@code gs://bucket}. + * + * @param uri bucket and current working directory, e.g. {@code gs://bucket} + * @param env map of configuration options, whose keys correspond to the method names of + * {@link CloudStorageConfiguration.Builder}. However you are not allowed to set the working + * directory, as that should be provided in the {@code uri} + * @throws IllegalArgumentException if {@code uri} specifies a user, query, fragment, or scheme is + * not {@value CloudStorageFileSystem#URI_SCHEME} + */ + @Override + public CloudStorageFileSystem newFileSystem(URI uri, Map env) { + checkArgument( + uri.getScheme().equalsIgnoreCase(CloudStorageFileSystem.URI_SCHEME), + "Cloud Storage URIs must have '%s' scheme: %s", + CloudStorageFileSystem.URI_SCHEME, + uri); + checkArgument( + !isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s", + CloudStorageFileSystem.URI_SCHEME, uri); + checkArgument( + uri.getPort() == -1 + && isNullOrEmpty(uri.getPath()) + && isNullOrEmpty(uri.getQuery()) + && isNullOrEmpty(uri.getFragment()) + && isNullOrEmpty(uri.getUserInfo()), + "GCS FileSystem URIs mustn't have: port, userinfo, path, query, or fragment: %s", + uri); + CloudStorageUtil.checkBucket(uri.getHost()); + return new CloudStorageFileSystem(this, uri.getHost(), CloudStorageConfiguration.fromMap(env)); + } + + @Override + public CloudStoragePath getPath(URI uri) { + return CloudStoragePath.getPath( + getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath()); + } + + @Override + public SeekableByteChannel newByteChannel( + Path path, Set options, FileAttribute... attrs) throws IOException { + checkNotNull(path); + CloudStorageUtil.checkNotNullArray(attrs); + if (options.contains(StandardOpenOption.WRITE)) { + // TODO: Make our OpenOptions implement FileAttribute. Also remove buffer option. + return newWriteChannel(path, options); + } else { + return newReadChannel(path, options); + } + } + + private SeekableByteChannel newReadChannel(Path path, Set options) + throws IOException { + for (OpenOption option : options) { + if (option instanceof StandardOpenOption) { + switch ((StandardOpenOption) option) { + case READ: + // Default behavior. + break; + case SPARSE: + case TRUNCATE_EXISTING: + // Ignored by specification. + break; + case WRITE: + throw new IllegalArgumentException("READ+WRITE not supported yet"); + case APPEND: + case CREATE: + case CREATE_NEW: + case DELETE_ON_CLOSE: + case DSYNC: + case SYNC: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + return CloudStorageReadChannel.create(storage, cloudPath.getBlobId(), 0); + } + + private SeekableByteChannel newWriteChannel(Path path, Set options) + throws IOException { + + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + BlobId file = cloudPath.getBlobId(); + BlobInfo.Builder infoBuilder = BlobInfo.builder(file); + List writeOptions = new ArrayList<>(); + List acls = new ArrayList<>(); + + HashMap metas = new HashMap<>(); + for (OpenOption option : options) { + if (option instanceof OptionMimeType) { + infoBuilder.contentType(((OptionMimeType) option).mimeType()); + } else if (option instanceof OptionCacheControl) { + infoBuilder.cacheControl(((OptionCacheControl) option).cacheControl()); + } else if (option instanceof OptionContentDisposition) { + infoBuilder.contentDisposition(((OptionContentDisposition) option).contentDisposition()); + } else if (option instanceof OptionContentEncoding) { + infoBuilder.contentEncoding(((OptionContentEncoding) option).contentEncoding()); + } else if (option instanceof OptionUserMetadata) { + OptionUserMetadata opMeta = (OptionUserMetadata) option; + metas.put(opMeta.key(), opMeta.value()); + } else if (option instanceof OptionAcl) { + acls.add(((OptionAcl) option).acl()); + } else if (option instanceof OptionBlockSize) { + // TODO: figure out how to plumb in block size. + } else if (option instanceof StandardOpenOption) { + switch ((StandardOpenOption) option) { + case CREATE: + case TRUNCATE_EXISTING: + case WRITE: + // Default behavior. + break; + case SPARSE: + // Ignored by specification. + break; + case CREATE_NEW: + writeOptions.add(Storage.BlobWriteOption.doesNotExist()); + break; + case READ: + throw new IllegalArgumentException("READ+WRITE not supported yet"); + case APPEND: + case DELETE_ON_CLOSE: + case DSYNC: + case SYNC: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof CloudStorageOption) { + // XXX: We need to interpret these later + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + + if (!metas.isEmpty()) { + infoBuilder.metadata(metas); + } + if (!acls.isEmpty()) { + infoBuilder.acl(acls); + } + + try { + return new CloudStorageWriteChannel( + storage.writer(infoBuilder.build(), + writeOptions.toArray(new Storage.BlobWriteOption[writeOptions.size()]))); + } catch (StorageException oops) { + throw asIoException(oops); + } + } + + @Override + public InputStream newInputStream(Path path, OpenOption... options) throws IOException { + InputStream result = super.newInputStream(path, options); + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + int blockSize = cloudPath.getFileSystem().config().blockSize(); + for (OpenOption option : options) { + if (option instanceof OptionBlockSize) { + blockSize = ((OptionBlockSize) option).size(); + } + } + return new BufferedInputStream(result, blockSize); + } + + @Override + public boolean deleteIfExists(Path path) throws IOException { + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(cloudPath); + } + return storage.delete(cloudPath.getBlobId()); + } + + @Override + public void delete(Path path) throws IOException { + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (!deleteIfExists(cloudPath)) { + throw new NoSuchFileException(cloudPath.toString()); + } + } + + @Override + public void move(Path source, Path target, CopyOption... options) throws IOException { + for (CopyOption option : options) { + if (option == StandardCopyOption.ATOMIC_MOVE) { + throw new AtomicMoveNotSupportedException( + source.toString(), + target.toString(), + "Google Cloud Storage does not support atomic move operations."); + } + } + copy(source, target, options); + delete(source); + } + + @Override + public void copy(Path source, Path target, CopyOption... options) throws IOException { + boolean wantCopyAttributes = false; + boolean wantReplaceExisting = false; + boolean setContentType = false; + boolean setCacheControl = false; + boolean setContentEncoding = false; + boolean setContentDisposition = false; + + CloudStoragePath toPath = CloudStorageUtil.checkPath(target); + BlobInfo.Builder tgtInfoBuilder = BlobInfo.builder(toPath.getBlobId()).contentType(""); + + int blockSize = -1; + for (CopyOption option : options) { + if (option instanceof StandardCopyOption) { + switch ((StandardCopyOption) option) { + case COPY_ATTRIBUTES: + wantCopyAttributes = true; + break; + case REPLACE_EXISTING: + wantReplaceExisting = true; + break; + case ATOMIC_MOVE: + default: + throw new UnsupportedOperationException(option.toString()); + } + } else if (option instanceof CloudStorageOption) { + if (option instanceof OptionBlockSize) { + blockSize = ((OptionBlockSize) option).size(); + } else if (option instanceof OptionMimeType) { + tgtInfoBuilder.contentType(((OptionMimeType) option).mimeType()); + setContentType = true; + } else if (option instanceof OptionCacheControl) { + tgtInfoBuilder.cacheControl(((OptionCacheControl) option).cacheControl()); + setCacheControl = true; + } else if (option instanceof OptionContentEncoding) { + tgtInfoBuilder.contentEncoding(((OptionContentEncoding) option).contentEncoding()); + setContentEncoding = true; + } else if (option instanceof OptionContentDisposition) { + tgtInfoBuilder.contentDisposition( + ((OptionContentDisposition) option).contentDisposition()); + setContentDisposition = true; + } else { + throw new UnsupportedOperationException(option.toString()); + } + } else { + throw new UnsupportedOperationException(option.toString()); + } + } + + CloudStoragePath fromPath = CloudStorageUtil.checkPath(source); + + blockSize = + blockSize != -1 + ? blockSize + : Ints.max( + fromPath.getFileSystem().config().blockSize(), + toPath.getFileSystem().config().blockSize()); + // TODO: actually use blockSize + + if (fromPath.seemsLikeADirectory() && toPath.seemsLikeADirectory()) { + if (fromPath.getFileSystem().config().usePseudoDirectories() + && toPath.getFileSystem().config().usePseudoDirectories()) { + // NOOP: This would normally create an empty directory. + return; + } else { + checkArgument( + !fromPath.getFileSystem().config().usePseudoDirectories() + && !toPath.getFileSystem().config().usePseudoDirectories(), + "File systems associated with paths don't agree on pseudo-directories."); + } + } + if (fromPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(fromPath); + } + if (toPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + throw new CloudStoragePseudoDirectoryException(toPath); + } + + try { + if (wantCopyAttributes) { + BlobInfo blobInfo = storage.get(fromPath.getBlobId()); + if (null == blobInfo) { + throw new NoSuchFileException(fromPath.toString()); + } + if (!setCacheControl) { + tgtInfoBuilder.cacheControl(blobInfo.cacheControl()); + } + if (!setContentType) { + tgtInfoBuilder.contentType(blobInfo.contentType()); + } + if (!setContentEncoding) { + tgtInfoBuilder.contentEncoding(blobInfo.contentEncoding()); + } + if (!setContentDisposition) { + tgtInfoBuilder.contentDisposition(blobInfo.contentDisposition()); + } + tgtInfoBuilder.acl(blobInfo.acl()); + tgtInfoBuilder.metadata(blobInfo.metadata()); + } + + BlobInfo tgtInfo = tgtInfoBuilder.build(); + Storage.CopyRequest.Builder copyReqBuilder = + Storage.CopyRequest.builder().source(fromPath.getBlobId()); + if (wantReplaceExisting) { + copyReqBuilder = copyReqBuilder.target(tgtInfo); + } else { + copyReqBuilder = copyReqBuilder.target(tgtInfo, Storage.BlobTargetOption.doesNotExist()); + } + CopyWriter copyWriter = storage.copy(copyReqBuilder.build()); + copyWriter.result(); + } catch (StorageException oops) { + throw asIoException(oops); + } + } + + @Override + public boolean isSameFile(Path path, Path path2) { + return CloudStorageUtil.checkPath(path).equals(CloudStorageUtil.checkPath(path2)); + } + + /** + * Always returns {@code false}, because Google Cloud Storage doesn't support hidden files. + */ + @Override + public boolean isHidden(Path path) { + CloudStorageUtil.checkPath(path); + return false; + } + + @Override + public void checkAccess(Path path, AccessMode... modes) throws IOException { + for (AccessMode mode : modes) { + switch (mode) { + case READ: + case WRITE: + break; + case EXECUTE: + default: + throw new UnsupportedOperationException(mode.toString()); + } + } + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + return; + } + if (storage.get(cloudPath.getBlobId(), Storage.BlobGetOption.fields(Storage.BlobField.ID)) + == null) { + throw new NoSuchFileException(path.toString()); + } + } + + @Override + public A readAttributes( + Path path, Class type, LinkOption... options) throws IOException { + checkNotNull(type); + CloudStorageUtil.checkNotNullArray(options); + if (type != CloudStorageFileAttributes.class && type != BasicFileAttributes.class) { + throw new UnsupportedOperationException(type.getSimpleName()); + } + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + if (cloudPath.seemsLikeADirectoryAndUsePseudoDirectories()) { + @SuppressWarnings("unchecked") + A result = (A) new CloudStoragePseudoDirectoryAttributes(cloudPath); + return result; + } + BlobInfo blobInfo = storage.get(cloudPath.getBlobId()); + // null size indicate a file that we haven't closed yet, so GCS treats it as not there yet. + if (null == blobInfo || blobInfo.size() == null) { + throw new NoSuchFileException( + cloudPath.getBlobId().bucket() + "/" + cloudPath.getBlobId().name()); + } + CloudStorageObjectAttributes ret; + ret = new CloudStorageObjectAttributes(blobInfo); + @SuppressWarnings("unchecked") + A result = (A) ret; + return result; + } + + @Override + public Map readAttributes(Path path, String attributes, LinkOption... options) { + // TODO(#811): Java 7 NIO defines at least eleven string attributes we'd want to support + // (eg. BasicFileAttributeView and PosixFileAttributeView), so rather than a partial + // implementation we rely on the other overload for now. + throw new UnsupportedOperationException(); + } + + @Override + public V getFileAttributeView( + Path path, Class type, LinkOption... options) { + checkNotNull(type); + CloudStorageUtil.checkNotNullArray(options); + if (type != CloudStorageFileAttributeView.class && type != BasicFileAttributeView.class) { + throw new UnsupportedOperationException(type.getSimpleName()); + } + CloudStoragePath cloudPath = CloudStorageUtil.checkPath(path); + @SuppressWarnings("unchecked") + V result = (V) new CloudStorageFileAttributeView(storage, cloudPath); + return result; + } + + /** + * Does nothing since Google Cloud Storage uses fake directories. + */ + @Override + public void createDirectory(Path dir, FileAttribute... attrs) { + CloudStorageUtil.checkPath(dir); + CloudStorageUtil.checkNotNullArray(attrs); + } + + @Override + public DirectoryStream newDirectoryStream(Path dir, final Filter filter) { + final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir); + checkNotNull(filter); + String prefix = cloudPath.toString(); + final Iterator blobIterator = storage.list(cloudPath.bucket(), + Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(), + Storage.BlobListOption.fields()).iterateAll(); + return new DirectoryStream() { + @Override + public Iterator iterator() { + return new LazyPathIterator(cloudPath.getFileSystem(), blobIterator, filter); + } + + @Override + public void close() throws IOException { + // Does nothing since there's nothing to close. Commenting this method to quiet codacy. + } + }; + } + + /** + * Throws {@link UnsupportedOperationException} because Cloud Storage objects are immutable. + */ + @Override + public void setAttribute(Path path, String attribute, Object value, LinkOption... options) { + throw new CloudStorageObjectImmutableException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public FileStore getFileStore(Path path) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageFileSystemProvider + && Objects.equals(storage, ((CloudStorageFileSystemProvider) other).storage); + } + + @Override + public int hashCode() { + return Objects.hash(storage); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("storage", storage).toString(); + } + + private IOException asIoException(StorageException oops) { + // RPC API can only throw StorageException, but CloudStorageFileSystemProvider + // can only throw IOException. Square peg, round hole. + // TODO(#810): Research if other codes should be translated similarly. + if (oops.code() == 404) { + return new NoSuchFileException(oops.reason()); + } + + Throwable cause = oops.getCause(); + try { + if (cause instanceof FileAlreadyExistsException) { + throw new FileAlreadyExistsException(((FileAlreadyExistsException) cause).getReason()); + } + // fallback + Throwables.propagateIfInstanceOf(oops.getCause(), IOException.class); + } catch (IOException okEx) { + return okEx; + } + return new IOException(oops.getMessage(), oops); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java new file mode 100644 index 000000000000..56f02749c727 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java @@ -0,0 +1,148 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.cloud.storage.Acl; +import com.google.cloud.storage.BlobInfo; +import com.google.common.base.MoreObjects; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.attribute.FileTime; +import java.util.List; +import java.util.Objects; + +import javax.annotation.Nonnull; +import javax.annotation.concurrent.Immutable; + +/** + * Metadata for a Google Cloud Storage file. + */ +@Immutable +final class CloudStorageObjectAttributes implements CloudStorageFileAttributes { + + @Nonnull private final BlobInfo info; + + CloudStorageObjectAttributes(BlobInfo info) { + this.info = checkNotNull(info); + } + + @Override + public long size() { + return info.size(); + } + + @Override + public FileTime creationTime() { + if (info.updateTime() == null) { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + return FileTime.fromMillis(info.updateTime()); + } + + @Override + public FileTime lastModifiedTime() { + return creationTime(); + } + + @Override + public Optional etag() { + return Optional.fromNullable(info.etag()); + } + + @Override + public Optional mimeType() { + return Optional.fromNullable(info.contentType()); + } + + @Override + public Optional> acl() { + return Optional.fromNullable(info.acl()); + } + + @Override + public Optional cacheControl() { + return Optional.fromNullable(info.cacheControl()); + } + + @Override + public Optional contentEncoding() { + return Optional.fromNullable(info.contentEncoding()); + } + + @Override + public Optional contentDisposition() { + return Optional.fromNullable(info.contentDisposition()); + } + + @Override + public ImmutableMap userMetadata() { + if (null == info.metadata()) { + return ImmutableMap.of(); + } + return ImmutableMap.copyOf(info.metadata()); + } + + @Override + public boolean isDirectory() { + return false; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isRegularFile() { + return true; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public FileTime lastAccessTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public Object fileKey() { + return info.blobId().bucket() + info.blobId().name() + info.blobId().generation(); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStorageObjectAttributes + && Objects.equals(info, ((CloudStorageObjectAttributes) other).info); + } + + @Override + public int hashCode() { + return info.hashCode(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this).add("info", info).toString(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectImmutableException.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectImmutableException.java new file mode 100644 index 000000000000..523e5dae7ed4 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectImmutableException.java @@ -0,0 +1,27 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +/** + * Exception reminding user that Cloud Storage objects can't be mutated. + */ +public final class CloudStorageObjectImmutableException extends UnsupportedOperationException { + + CloudStorageObjectImmutableException() { + super("Cloud Storage objects are immutable."); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOption.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOption.java new file mode 100644 index 000000000000..d4921e10a0ec --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOption.java @@ -0,0 +1,41 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import java.nio.file.CopyOption; +import java.nio.file.OpenOption; + +/** + * Master interface for file operation option classes related to Google Cloud Storage. + */ +public interface CloudStorageOption { + + /** + * Interface for Google Cloud Storage options that can be specified when opening files. + */ + interface Open extends CloudStorageOption, OpenOption {} + + /** + * Interface for Google Cloud Storage options that can be specified when copying files. + */ + interface Copy extends CloudStorageOption, CopyOption {} + + /** + * Interface for Google Cloud Storage options that can be specified when opening or copying files. + */ + interface OpenCopy extends Open, Copy {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java new file mode 100644 index 000000000000..74293b7a79f2 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageOptions.java @@ -0,0 +1,95 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.Acl; + +/** + * Helper class for specifying options when opening and copying Cloud Storage files. + */ +public final class CloudStorageOptions { + + /** + * Sets the mime type header on an object, e.g. {@code "text/plain"}. + */ + public static CloudStorageOption.OpenCopy withMimeType(String mimeType) { + return OptionMimeType.create(mimeType); + } + + /** + * Disables caching on an object. Same as: {@code withCacheControl("no-cache")}. + */ + public static CloudStorageOption.OpenCopy withoutCaching() { + return withCacheControl("no-cache"); + } + + /** + * Sets the {@code Cache-Control} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#cachecontrol" + */ + public static CloudStorageOption.OpenCopy withCacheControl(String cacheControl) { + return OptionCacheControl.create(cacheControl); + } + + /** + * Sets the {@code Content-Disposition} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentdisposition" + */ + public static CloudStorageOption.OpenCopy withContentDisposition(String contentDisposition) { + return OptionContentDisposition.create(contentDisposition); + } + + /** + * Sets the {@code Content-Encoding} HTTP header on an object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#contentencoding" + */ + public static CloudStorageOption.OpenCopy withContentEncoding(String contentEncoding) { + return OptionContentEncoding.create(contentEncoding); + } + + /** + * Sets the ACL value on a Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#acl" + */ + public static CloudStorageOption.OpenCopy withAcl(Acl acl) { + return OptionAcl.create(acl); + } + + /** + * Sets an unmodifiable piece of user metadata on a Cloud Storage object. + * + * @see "https://developers.google.com/storage/docs/reference-headers#xgoogmeta" + */ + public static CloudStorageOption.OpenCopy withUserMetadata(String key, String value) { + return OptionUserMetadata.create(key, value); + } + + /** + * Sets the block size (in bytes) when talking to the Google Cloud Storage server. + * + *

The default is {@value CloudStorageFileSystem#BLOCK_SIZE_DEFAULT}. + */ + public static CloudStorageOption.OpenCopy withBlockSize(int size) { + return OptionBlockSize.create(size); + } + + private CloudStorageOptions() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java new file mode 100644 index 000000000000..7c9154b20d93 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java @@ -0,0 +1,354 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.cloud.storage.BlobId; +import com.google.common.collect.UnmodifiableIterator; + +import java.io.File; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.WatchEvent.Kind; +import java.nio.file.WatchEvent.Modifier; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.Collections; +import java.util.Iterator; +import java.util.Objects; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Google Cloud Storage {@link Path}. + * + * @see UnixPath + */ +@Immutable +public final class CloudStoragePath implements Path { + + private static final Pattern EXTRA_SLASHES_OR_DOT_DIRS_PATTERN = + Pattern.compile("^\\.\\.?/|//|/\\.\\.?/|/\\.\\.?$"); + + private final CloudStorageFileSystem fileSystem; + private final UnixPath path; + + private CloudStoragePath(CloudStorageFileSystem fileSystem, UnixPath path) { + this.fileSystem = fileSystem; + this.path = path; + } + + static CloudStoragePath getPath(CloudStorageFileSystem fileSystem, String path, String... more) { + return new CloudStoragePath( + fileSystem, UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), path, more)); + } + + /** + * Returns the Cloud Storage bucket name being served by this file system. + */ + public String bucket() { + return fileSystem.bucket(); + } + + /** + * Returns path converted to a {@link BlobId} so I/O can be performed. + */ + BlobId getBlobId() { + return BlobId.of(bucket(), toRealPath().path.toString()); + } + + boolean seemsLikeADirectory() { + return path.seemsLikeADirectory(); + } + + boolean seemsLikeADirectoryAndUsePseudoDirectories() { + return path.seemsLikeADirectory() && fileSystem.config().usePseudoDirectories(); + } + + @Override + public CloudStorageFileSystem getFileSystem() { + return fileSystem; + } + + @Nullable + @Override + public CloudStoragePath getRoot() { + return newPath(path.getRoot()); + } + + @Override + public boolean isAbsolute() { + return path.isAbsolute(); + } + + /** + * Changes relative path to be absolute, using + * {@link CloudStorageConfiguration#workingDirectory() workingDirectory} as current dir. + */ + @Override + public CloudStoragePath toAbsolutePath() { + return newPath(path.toAbsolutePath(getWorkingDirectory())); + } + + /** + * Returns this path rewritten to the Cloud Storage object name that'd be used to perform i/o. + * + *

This method makes path {@link #toAbsolutePath() absolute} and removes the prefix slash from + * the absolute path when {@link CloudStorageConfiguration#stripPrefixSlash() stripPrefixSlash} + * is {@code true}. + * + * @throws IllegalArgumentException if path contains extra slashes or dot-dirs when + * {@link CloudStorageConfiguration#permitEmptyPathComponents() permitEmptyPathComponents} + * is {@code false}, or if the resulting path is empty. + */ + @Override + public CloudStoragePath toRealPath(LinkOption... options) { + CloudStorageUtil.checkNotNullArray(options); + return newPath(toRealPathInternal(true)); + } + + private UnixPath toRealPathInternal(boolean errorCheck) { + UnixPath objectName = path.toAbsolutePath(getWorkingDirectory()); + if (errorCheck && !fileSystem.config().permitEmptyPathComponents()) { + checkArgument( + !EXTRA_SLASHES_OR_DOT_DIRS_PATTERN.matcher(objectName).find(), + "I/O not allowed on dot-dirs or extra slashes when !permitEmptyPathComponents: %s", + objectName); + } + if (fileSystem.config().stripPrefixSlash()) { + objectName = objectName.removeBeginningSeparator(); + } + checkArgument( + !errorCheck || !objectName.isEmpty(), + "I/O not allowed on empty Google Cloud Storage object names."); + return objectName; + } + + /** + * Returns path without extra slashes or {@code .} and {@code ..} and preserves trailing slash. + */ + @Override + public CloudStoragePath normalize() { + return newPath(path.normalize()); + } + + @Override + public CloudStoragePath resolve(Path object) { + return newPath(path.resolve(CloudStorageUtil.checkPath(object).path)); + } + + @Override + public CloudStoragePath resolve(String other) { + return newPath(path.resolve(getUnixPath(other))); + } + + @Override + public CloudStoragePath resolveSibling(Path other) { + return newPath(path.resolveSibling(CloudStorageUtil.checkPath(other).path)); + } + + @Override + public CloudStoragePath resolveSibling(String other) { + return newPath(path.resolveSibling(getUnixPath(other))); + } + + @Override + public CloudStoragePath relativize(Path object) { + return newPath(path.relativize(CloudStorageUtil.checkPath(object).path)); + } + + @Nullable + @Override + public CloudStoragePath getParent() { + return newPath(path.getParent()); + } + + @Nullable + @Override + public CloudStoragePath getFileName() { + return newPath(path.getFileName()); + } + + @Override + public CloudStoragePath subpath(int beginIndex, int endIndex) { + return newPath(path.subpath(beginIndex, endIndex)); + } + + @Override + public int getNameCount() { + return path.getNameCount(); + } + + @Override + public CloudStoragePath getName(int index) { + return newPath(path.getName(index)); + } + + @Override + public boolean startsWith(Path other) { + if (!(checkNotNull(other) instanceof CloudStoragePath)) { + return false; + } + CloudStoragePath that = (CloudStoragePath) other; + if (!bucket().equals(that.bucket())) { + return false; + } + return path.startsWith(that.path); + } + + @Override + public boolean startsWith(String other) { + return path.startsWith(getUnixPath(other)); + } + + @Override + public boolean endsWith(Path other) { + if (!(checkNotNull(other) instanceof CloudStoragePath)) { + return false; + } + CloudStoragePath that = (CloudStoragePath) other; + if (!bucket().equals(that.bucket())) { + return false; + } + return path.endsWith(that.path); + } + + @Override + public boolean endsWith(String other) { + return path.endsWith(getUnixPath(other)); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public WatchKey register(WatchService watcher, Kind[] events, Modifier... modifiers) { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because this feature hasn't been implemented yet. + */ + @Override + public WatchKey register(WatchService watcher, Kind... events) { + // TODO: Implement me. + throw new UnsupportedOperationException(); + } + + /** + * Throws {@link UnsupportedOperationException} because Google Cloud Storage files are not backed + * by the local file system. + */ + @Override + public File toFile() { + throw new UnsupportedOperationException("GCS objects aren't available locally"); + } + + @Override + public Iterator iterator() { + if (path.isEmpty()) { + return Collections.singleton(this).iterator(); + } else if (path.isRoot()) { + return Collections.emptyIterator(); + } else { + return new PathIterator(); + } + } + + @Override + public int compareTo(Path other) { + // Documented to throw CCE if other is associated with a different FileSystemProvider. + CloudStoragePath that = (CloudStoragePath) other; + int res = bucket().compareTo(that.bucket()); + if (res != 0) { + return res; + } + return toRealPathInternal(false).compareTo(that.toRealPathInternal(false)); + } + + @Override + public boolean equals(Object other) { + return this == other + || other instanceof CloudStoragePath + && Objects.equals(bucket(), ((CloudStoragePath) other).bucket()) + && Objects.equals( + toRealPathInternal(false), ((CloudStoragePath) other).toRealPathInternal(false)); + } + + @Override + public int hashCode() { + return Objects.hash(bucket(), toRealPathInternal(false)); + } + + @Override + public String toString() { + return path.toString(); + } + + @Override + public URI toUri() { + try { + return new URI( + CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + } + + @Nullable + private CloudStoragePath newPath(@Nullable UnixPath newPath) { + if (newPath == path) { // Nonuse of equals is intentional. + return this; + } else if (newPath != null) { + return new CloudStoragePath(fileSystem, newPath); + } else { + return null; + } + } + + private UnixPath getUnixPath(String newPath) { + return UnixPath.getPath(fileSystem.config().permitEmptyPathComponents(), newPath); + } + + private UnixPath getWorkingDirectory() { + return getUnixPath(fileSystem.config().workingDirectory()); + } + + /** + * Transform iterator providing a slight performance boost over {@code FluentIterable}. + */ + private final class PathIterator extends UnmodifiableIterator { + private final Iterator delegate = path.split(); + + @Override + public Path next() { + return newPath(getUnixPath(delegate.next())); + } + + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java new file mode 100644 index 000000000000..a7a8c7293688 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java @@ -0,0 +1,116 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.storage.Acl; +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableMap; + +import java.nio.file.attribute.FileTime; +import java.util.List; + +/** + * Metadata for a Cloud Storage pseudo-directory. + */ +final class CloudStoragePseudoDirectoryAttributes implements CloudStorageFileAttributes { + + private final String id; + + CloudStoragePseudoDirectoryAttributes(CloudStoragePath path) { + this.id = path.toUri().toString(); + } + + @Override + public boolean isDirectory() { + return true; + } + + @Override + public boolean isOther() { + return false; + } + + @Override + public boolean isRegularFile() { + return false; + } + + @Override + public boolean isSymbolicLink() { + return false; + } + + @Override + public Object fileKey() { + return id; + } + + @Override + public long size() { + return 1; // Allow I/O to happen before we fail. + } + + @Override + public FileTime lastModifiedTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public FileTime creationTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public FileTime lastAccessTime() { + return CloudStorageFileSystem.FILE_TIME_UNKNOWN; + } + + @Override + public Optional etag() { + return Optional.absent(); + } + + @Override + public Optional mimeType() { + return Optional.absent(); + } + + @Override + public Optional> acl() { + return Optional.absent(); + } + + @Override + public Optional cacheControl() { + return Optional.absent(); + } + + @Override + public Optional contentEncoding() { + return Optional.absent(); + } + + @Override + public Optional contentDisposition() { + return Optional.absent(); + } + + @Override + public ImmutableMap userMetadata() { + return ImmutableMap.of(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java new file mode 100644 index 000000000000..5a63bc790c6c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryException.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import java.nio.file.InvalidPathException; + +/** + * Exception thrown when erroneously trying to operate on a path with a trailing slash. + */ +public final class CloudStoragePseudoDirectoryException extends InvalidPathException { + + CloudStoragePseudoDirectoryException(CloudStoragePath path) { + super(path.toString(), "Can't perform I/O on pseudo-directories (trailing slash)"); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java new file mode 100644 index 000000000000..bd9cb9def6c5 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannel.java @@ -0,0 +1,150 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.Storage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.NoSuchFileException; + +import javax.annotation.CheckReturnValue; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Cloud Storage read channel. + * + * @see CloudStorageWriteChannel + */ +@ThreadSafe +final class CloudStorageReadChannel implements SeekableByteChannel { + + private final ReadChannel channel; + private long position; + private long size; + + @CheckReturnValue + @SuppressWarnings("resource") + static CloudStorageReadChannel create(Storage gcsStorage, BlobId file, long position) + throws IOException { + // XXX: Reading size and opening file should be atomic. + long size = fetchSize(gcsStorage, file); + ReadChannel channel = gcsStorage.reader(file); + if (position > 0) { + channel.seek((int) position); + } + return new CloudStorageReadChannel(position, size, channel); + } + + private CloudStorageReadChannel(long position, long size, ReadChannel channel) { + this.position = position; + this.size = size; + this.channel = channel; + } + + @Override + public boolean isOpen() { + synchronized (this) { + return channel.isOpen(); + } + } + + @Override + public void close() throws IOException { + synchronized (this) { + channel.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + synchronized (this) { + checkOpen(); + int amt = channel.read(dst); + if (amt > 0) { + position += amt; + // XXX: This would only ever happen if the fetchSize() race-condition occurred. + if (position > size) { + size = position; + } + } + return amt; + } + } + + @Override + public long size() throws IOException { + synchronized (this) { + checkOpen(); + return size; + } + } + + @Override + public long position() throws IOException { + synchronized (this) { + checkOpen(); + return position; + } + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + checkArgument(newPosition >= 0); + synchronized (this) { + checkOpen(); + if (newPosition == position) { + return this; + } + channel.seek(newPosition); + position = newPosition; + return this; + } + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new NonWritableChannelException(); + } + + @Override + public SeekableByteChannel truncate(long size) throws IOException { + throw new NonWritableChannelException(); + } + + private void checkOpen() throws ClosedChannelException { + if (!channel.isOpen()) { + throw new ClosedChannelException(); + } + } + + private static long fetchSize(Storage gcsStorage, BlobId file) throws IOException { + BlobInfo blobInfo = gcsStorage.get(file); + if (blobInfo == null) { + throw new NoSuchFileException(String.format("gs://%s/%s", file.bucket(), file.name())); + } + return blobInfo.size(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageUtil.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageUtil.java new file mode 100644 index 000000000000..14b1921a7b9e --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageUtil.java @@ -0,0 +1,80 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; +import java.util.regex.Pattern; + +final class CloudStorageUtil { + + private static final Pattern BUCKET_PATTERN = Pattern.compile("[a-z0-9][-._a-z0-9]+[a-z0-9]"); + + static void checkBucket(String bucket) { + // TODO: The true check is actually more complicated. Consider implementing it. + checkArgument( + BUCKET_PATTERN.matcher(bucket).matches(), + "Invalid bucket name: '" + + bucket + + "'. " + + "Google Cloud Storage bucket names must contain only lowercase letters, numbers, " + + "dashes (-), underscores (_), and dots (.). Bucket names must start and end with a " + + "number or a letter. See the following page for more details: " + + "https://developers.google.com/storage/docs/bucketnaming"); + } + + static CloudStoragePath checkPath(Path path) { + if (!(checkNotNull(path) instanceof CloudStoragePath)) { + throw new ProviderMismatchException( + String.format( + "Not a Cloud Storage path: %s (%s)", path, path.getClass().getSimpleName())); + } + return (CloudStoragePath) path; + } + + static URI stripPathFromUri(URI uri) { + try { + return new URI( + uri.getScheme(), + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + null, + uri.getQuery(), + uri.getFragment()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException(e.getMessage()); + } + } + + /** + * Makes {@code NullPointerTester} happy. + */ + @SafeVarargs + static void checkNotNullArray(T... values) { + for (T value : values) { + checkNotNull(value); + } + } + + private CloudStorageUtil() {} +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannel.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannel.java new file mode 100644 index 000000000000..e3aad30b3579 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannel.java @@ -0,0 +1,112 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.WriteChannel; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; +import java.nio.channels.SeekableByteChannel; + +import javax.annotation.concurrent.ThreadSafe; + +/** + * Cloud Storage write channel. + * + *

This class does not support seeking, reading, or append. + * + * @see CloudStorageReadChannel + */ +@ThreadSafe +final class CloudStorageWriteChannel implements SeekableByteChannel { + + private final WriteChannel channel; + private long position; + private long size; + + CloudStorageWriteChannel(WriteChannel channel) { + this.channel = channel; + } + + @Override + public boolean isOpen() { + synchronized (this) { + return channel.isOpen(); + } + } + + @Override + public void close() throws IOException { + synchronized (this) { + channel.close(); + } + } + + @Override + public int read(ByteBuffer dst) throws IOException { + throw new NonReadableChannelException(); + } + + @Override + public int write(ByteBuffer src) throws IOException { + synchronized (this) { + checkOpen(); + int amt = channel.write(src); + if (amt > 0) { + position += amt; + size += amt; + } + return amt; + } + } + + @Override + public long position() throws IOException { + synchronized (this) { + checkOpen(); + return position; + } + } + + @Override + public SeekableByteChannel position(long newPosition) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long size() throws IOException { + synchronized (this) { + checkOpen(); + return size; + } + } + + @Override + public SeekableByteChannel truncate(long newSize) throws IOException { + // TODO: Emulate this functionality by closing and rewriting old file up to newSize. + // Or maybe just swap out GcsStorage for the API client. + throw new UnsupportedOperationException(); + } + + private void checkOpen() throws ClosedChannelException { + if (!channel.isOpen()) { + throw new ClosedChannelException(); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAcl.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAcl.java new file mode 100644 index 000000000000..ea9b4b0a62a6 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionAcl.java @@ -0,0 +1,30 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; +import com.google.cloud.storage.Acl; + +@AutoValue +abstract class OptionAcl implements CloudStorageOption.OpenCopy { + + static OptionAcl create(Acl acl) { + return new AutoValue_OptionAcl(acl); + } + + abstract Acl acl(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionBlockSize.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionBlockSize.java new file mode 100644 index 000000000000..61d3ba6e7d8e --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionBlockSize.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionBlockSize implements CloudStorageOption.OpenCopy { + + static OptionBlockSize create(int size) { + return new AutoValue_OptionBlockSize(size); + } + + abstract int size(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionCacheControl.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionCacheControl.java new file mode 100644 index 000000000000..25b18ac96162 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionCacheControl.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionCacheControl implements CloudStorageOption.OpenCopy { + + static OptionCacheControl create(String cacheControl) { + return new AutoValue_OptionCacheControl(cacheControl); + } + + abstract String cacheControl(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentDisposition.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentDisposition.java new file mode 100644 index 000000000000..8df320d687d7 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentDisposition.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionContentDisposition implements CloudStorageOption.OpenCopy { + + static OptionContentDisposition create(String contentDisposition) { + return new AutoValue_OptionContentDisposition(contentDisposition); + } + + abstract String contentDisposition(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentEncoding.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentEncoding.java new file mode 100644 index 000000000000..8d599fb11d52 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionContentEncoding.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionContentEncoding implements CloudStorageOption.OpenCopy { + + static OptionContentEncoding create(String contentEncoding) { + return new AutoValue_OptionContentEncoding(contentEncoding); + } + + abstract String contentEncoding(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMimeType.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMimeType.java new file mode 100644 index 000000000000..759cb00bee5b --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionMimeType.java @@ -0,0 +1,29 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionMimeType implements CloudStorageOption.OpenCopy { + + static OptionMimeType create(String mimeType) { + return new AutoValue_OptionMimeType(mimeType); + } + + abstract String mimeType(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionUserMetadata.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionUserMetadata.java new file mode 100644 index 000000000000..3c45e384c964 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/OptionUserMetadata.java @@ -0,0 +1,31 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.auto.value.AutoValue; + +@AutoValue +abstract class OptionUserMetadata implements CloudStorageOption.OpenCopy { + + static OptionUserMetadata create(String key, String value) { + return new AutoValue_OptionUserMetadata(key, value); + } + + abstract String key(); + + abstract String value(); +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/UnixPath.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/UnixPath.java new file mode 100644 index 000000000000..3d8c6ae77299 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/UnixPath.java @@ -0,0 +1,541 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; +import com.google.common.collect.Ordering; +import com.google.common.collect.PeekingIterator; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** + * Unix file system path. + * + *

This class is helpful for writing {@link java.nio.file.Path Path} implementations. + * + *

This implementation behaves almost identically to {@code sun.nio.fs.UnixPath}. The only + * difference is that some methods (like {@link #relativize(UnixPath)} go to greater lengths to + * preserve trailing backslashes, in order to ensure the path will continue to be recognized as a + * directory. + * + *

Note: This code might not play nice with + * Supplementary + * Characters as Surrogates. + */ +@Immutable +final class UnixPath implements CharSequence { + + public static final char DOT = '.'; + public static final char SEPARATOR = '/'; + public static final String ROOT = "" + SEPARATOR; + public static final String CURRENT_DIR = "" + DOT; + public static final String PARENT_DIR = "" + DOT + DOT; + public static final UnixPath EMPTY_PATH = new UnixPath(false, ""); + public static final UnixPath ROOT_PATH = new UnixPath(false, ROOT); + + private static final Splitter SPLITTER = Splitter.on(SEPARATOR).omitEmptyStrings(); + private static final Splitter SPLITTER_PERMIT_EMPTY_COMPONENTS = Splitter.on(SEPARATOR); + private static final Joiner JOINER = Joiner.on(SEPARATOR); + private static final Ordering> ORDERING = Ordering.natural().lexicographical(); + + private final String path; + private List lazyStringParts; + private final boolean permitEmptyComponents; + + private UnixPath(boolean permitEmptyComponents, String path) { + this.path = checkNotNull(path); + this.permitEmptyComponents = permitEmptyComponents; + } + + /** + * Returns new path of {@code first}. + */ + public static UnixPath getPath(boolean permitEmptyComponents, String path) { + if (path.isEmpty()) { + return EMPTY_PATH; + } else if (isRootInternal(path)) { + return ROOT_PATH; + } else { + return new UnixPath(permitEmptyComponents, path); + } + } + + /** + * Returns new path of {@code first} with {@code more} components resolved against it. + * + * @see #resolve(UnixPath) + * @see java.nio.file.FileSystem#getPath(String, String...) + */ + public static UnixPath getPath(boolean permitEmptyComponents, String first, String... more) { + if (more.length == 0) { + return getPath(permitEmptyComponents, first); + } + StringBuilder builder = new StringBuilder(first); + for (int i = 0; i < more.length; i++) { + String part = more[i]; + if (part.isEmpty()) { + continue; + } else if (isAbsoluteInternal(part)) { + if (i == more.length - 1) { + return new UnixPath(permitEmptyComponents, part); + } else { + builder.replace(0, builder.length(), part); + } + } else if (hasTrailingSeparatorInternal(builder)) { + builder.append(part); + } else { + builder.append(SEPARATOR); + builder.append(part); + } + } + return new UnixPath(permitEmptyComponents, builder.toString()); + } + + /** + * Returns {@code true} consists only of {@code separator}. + */ + public boolean isRoot() { + return isRootInternal(path); + } + + private static boolean isRootInternal(String path) { + return path.length() == 1 && path.charAt(0) == SEPARATOR; + } + + /** + * Returns {@code true} if path starts with {@code separator}. + */ + public boolean isAbsolute() { + return isAbsoluteInternal(path); + } + + private static boolean isAbsoluteInternal(String path) { + return !path.isEmpty() && path.charAt(0) == SEPARATOR; + } + + /** + * Returns {@code true} if path ends with {@code separator}. + */ + public boolean hasTrailingSeparator() { + return hasTrailingSeparatorInternal(path); + } + + private static boolean hasTrailingSeparatorInternal(CharSequence path) { + return path.length() != 0 && path.charAt(path.length() - 1) == SEPARATOR; + } + + /** + * Returns {@code true} if path ends with a trailing slash, or would after normalization. + */ + public boolean seemsLikeADirectory() { + int length = path.length(); + return path.isEmpty() + || path.charAt(length - 1) == SEPARATOR + || path.endsWith(".") && (length == 1 || path.charAt(length - 2) == SEPARATOR) + || path.endsWith("..") && (length == 2 || path.charAt(length - 3) == SEPARATOR); + } + + /** + * Returns last component in {@code path}. + * + * @see java.nio.file.Path#getFileName() + */ + @Nullable + public UnixPath getFileName() { + if (path.isEmpty()) { + return EMPTY_PATH; + } else if (isRoot()) { + return null; + } else { + List parts = getParts(); + String last = parts.get(parts.size() - 1); + return parts.size() == 1 && path.equals(last) + ? this + : new UnixPath(permitEmptyComponents, last); + } + } + + /** + * Returns parent directory (including trailing separator) or {@code null} if no parent remains. + * + * @see java.nio.file.Path#getParent() + */ + @Nullable + public UnixPath getParent() { + if (path.isEmpty() || isRoot()) { + return null; + } + int index = + hasTrailingSeparator() + ? path.lastIndexOf(SEPARATOR, path.length() - 2) + : path.lastIndexOf(SEPARATOR); + if (index == -1) { + return isAbsolute() ? ROOT_PATH : null; + } else { + return new UnixPath(permitEmptyComponents, path.substring(0, index + 1)); + } + } + + /** + * Returns root component if an absolute path, otherwise {@code null}. + * + * @see java.nio.file.Path#getRoot() + */ + @Nullable + public UnixPath getRoot() { + return isAbsolute() ? ROOT_PATH : null; + } + + /** + * Returns specified range of sub-components in path joined together. + * + * @see java.nio.file.Path#subpath(int, int) + */ + public UnixPath subpath(int beginIndex, int endIndex) { + if (path.isEmpty() && beginIndex == 0 && endIndex == 1) { + return this; + } + checkArgument(beginIndex >= 0 && endIndex > beginIndex); + List subList; + try { + subList = getParts().subList(beginIndex, endIndex); + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(); + } + return new UnixPath(permitEmptyComponents, JOINER.join(subList)); + } + + /** + * Returns number of components in {@code path}. + * + * @see java.nio.file.Path#getNameCount() + */ + public int getNameCount() { + if (path.isEmpty()) { + return 1; + } else if (isRoot()) { + return 0; + } else { + return getParts().size(); + } + } + + /** + * Returns component in {@code path} at {@code index}. + * + * @see java.nio.file.Path#getName(int) + */ + public UnixPath getName(int index) { + if (path.isEmpty()) { + return this; + } + try { + return new UnixPath(permitEmptyComponents, getParts().get(index)); + } catch (IndexOutOfBoundsException e) { + throw new IllegalArgumentException(); + } + } + + /** + * Returns path without extra separators or {@code .} and {@code ..}, preserving trailing slash. + * + * @see java.nio.file.Path#normalize() + */ + public UnixPath normalize() { + List parts = new ArrayList<>(); + boolean mutated = false; + int resultLength = 0; + int mark = 0; + int index; + do { + index = path.indexOf(SEPARATOR, mark); + String part = path.substring(mark, index == -1 ? path.length() : index + 1); + switch (part) { + case CURRENT_DIR: + case CURRENT_DIR + SEPARATOR: + mutated = true; + break; + case PARENT_DIR: + case PARENT_DIR + SEPARATOR: + mutated = true; + if (!parts.isEmpty()) { + resultLength -= parts.remove(parts.size() - 1).length(); + } + break; + default: + if (index != mark || index == 0) { + parts.add(part); + resultLength = part.length(); + } else { + mutated = true; + } + } + mark = index + 1; + } while (index != -1); + if (!mutated) { + return this; + } + StringBuilder result = new StringBuilder(resultLength); + for (String part : parts) { + result.append(part); + } + return new UnixPath(permitEmptyComponents, result.toString()); + } + + /** + * Returns {@code other} appended to {@code path}. + * + * @see java.nio.file.Path#resolve(java.nio.file.Path) + */ + public UnixPath resolve(UnixPath other) { + if (other.path.isEmpty()) { + return this; + } else if (other.isAbsolute()) { + return other; + } else if (hasTrailingSeparator()) { + return new UnixPath(permitEmptyComponents, path + other.path); + } else { + return new UnixPath(permitEmptyComponents, path + SEPARATOR + other.path); + } + } + + /** + * Returns {@code other} resolved against parent of {@code path}. + * + * @see java.nio.file.Path#resolveSibling(java.nio.file.Path) + */ + public UnixPath resolveSibling(UnixPath other) { + checkNotNull(other); + UnixPath parent = getParent(); + return parent == null ? other : parent.resolve(other); + } + + /** + * Returns {@code other} made relative to {@code path}. + * + * @see java.nio.file.Path#relativize(java.nio.file.Path) + */ + public UnixPath relativize(UnixPath other) { + checkArgument(isAbsolute() == other.isAbsolute(), "'other' is different type of Path"); + if (path.isEmpty()) { + return other; + } + PeekingIterator left = Iterators.peekingIterator(split()); + PeekingIterator right = Iterators.peekingIterator(other.split()); + while (left.hasNext() && right.hasNext()) { + if (!left.peek().equals(right.peek())) { + break; + } + left.next(); + right.next(); + } + StringBuilder result = new StringBuilder(path.length() + other.path.length()); + while (left.hasNext()) { + result.append(PARENT_DIR); + result.append(SEPARATOR); + left.next(); + } + while (right.hasNext()) { + result.append(right.next()); + result.append(SEPARATOR); + } + if (result.length() > 0 && !other.hasTrailingSeparator()) { + result.deleteCharAt(result.length() - 1); + } + return new UnixPath(permitEmptyComponents, result.toString()); + } + + /** + * Returns {@code true} if {@code path} starts with {@code other}. + * + * @see java.nio.file.Path#startsWith(java.nio.file.Path) + */ + public boolean startsWith(UnixPath other) { + UnixPath me = removeTrailingSeparator(); + other = other.removeTrailingSeparator(); + if (other.path.length() > me.path.length()) { + return false; + } else if (me.isAbsolute() != other.isAbsolute()) { + return false; + } else if (!me.path.isEmpty() && other.path.isEmpty()) { + return false; + } + return startsWith(split(), other.split()); + } + + private static boolean startsWith(Iterator lefts, Iterator rights) { + while (rights.hasNext()) { + if (!lefts.hasNext() || !rights.next().equals(lefts.next())) { + return false; + } + } + return true; + } + + /** + * Returns {@code true} if {@code path} ends with {@code other}. + * + * @see java.nio.file.Path#endsWith(java.nio.file.Path) + */ + public boolean endsWith(UnixPath other) { + UnixPath me = removeTrailingSeparator(); + other = other.removeTrailingSeparator(); + if (other.path.length() > me.path.length()) { + return false; + } else if (!me.path.isEmpty() && other.path.isEmpty()) { + return false; + } else if (other.isAbsolute()) { + return me.isAbsolute() && me.path.equals(other.path); + } + return startsWith(me.splitReverse(), other.splitReverse()); + } + + /** + * Compares two paths lexicographically for ordering. + * + * @see java.nio.file.Path#compareTo(java.nio.file.Path) + */ + public int compareTo(UnixPath other) { + return ORDERING.compare(getParts(), other.getParts()); + } + + /** + * Converts relative path to an absolute path. + */ + public UnixPath toAbsolutePath(UnixPath currentWorkingDirectory) { + checkArgument(currentWorkingDirectory.isAbsolute()); + return isAbsolute() ? this : currentWorkingDirectory.resolve(this); + } + + /** + * Returns {@code toAbsolutePath(ROOT_PATH)}. + */ + public UnixPath toAbsolutePath() { + return toAbsolutePath(ROOT_PATH); + } + + /** + * Removes beginning separator from path, if an absolute path. + */ + public UnixPath removeBeginningSeparator() { + return isAbsolute() ? new UnixPath(permitEmptyComponents, path.substring(1)) : this; + } + + /** + * Adds trailing separator to path, if it isn't present. + */ + public UnixPath addTrailingSeparator() { + return hasTrailingSeparator() ? this : new UnixPath(permitEmptyComponents, path + SEPARATOR); + } + + /** + * Removes trailing separator from path, unless it's root. + */ + public UnixPath removeTrailingSeparator() { + if (!isRoot() && hasTrailingSeparator()) { + return new UnixPath(permitEmptyComponents, path.substring(0, path.length() - 1)); + } else { + return this; + } + } + + /** + * Splits path into components, excluding separators and empty strings. + */ + public Iterator split() { + return getParts().iterator(); + } + + /** + * Splits path into components in reverse, excluding separators and empty strings. + */ + public Iterator splitReverse() { + return Lists.reverse(getParts()).iterator(); + } + + @Override + public boolean equals(Object other) { + return this == other || other instanceof UnixPath && path.equals(((UnixPath) other).path); + } + + @Override + public int hashCode() { + return path.hashCode(); + } + + /** + * Returns path as a string. + */ + @Override + public String toString() { + return path; + } + + @Override + public int length() { + return path.length(); + } + + @Override + public char charAt(int index) { + return path.charAt(index); + } + + @Override + public CharSequence subSequence(int start, int end) { + return path.subSequence(start, end); + } + + /** + * Returns {@code true} if this path is an empty string. + */ + public boolean isEmpty() { + return path.isEmpty(); + } + + /** + * Returns list of path components, excluding slashes. + */ + private List getParts() { + List result = lazyStringParts; + return result != null + ? result + : (lazyStringParts = + path.isEmpty() || isRoot() ? Collections.emptyList() : createParts()); + } + + private List createParts() { + if (permitEmptyComponents) { + return SPLITTER_PERMIT_EMPTY_COMPONENTS.splitToList( + path.charAt(0) == SEPARATOR ? path.substring(1) : path); + } else { + return SPLITTER.splitToList(path); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/package-info.java b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/package-info.java new file mode 100644 index 000000000000..1376ca25d02e --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/package-info.java @@ -0,0 +1,100 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Java 7 nio FileSystem client library for Google Cloud Storage. + * + *

This client library allows you to easily interact with Google Cloud Storage, using Java's + * standard file system API, introduced in Java 7. + * + *

How It Works

+ * + * The simplest way to get started is with {@code Paths} and {@code Files}: + * 
{@code
+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ * }
+ * + *

For the complete source code see + * + * ReadAllLines.java. + * + *

If you want to configure the bucket per-environment, it might make more sense to use the + * {@code FileSystem} API: + * 

{@code
+ *   FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"));
+ *   byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
+ *   Path path = fs.getPath("/object");
+ *   Files.write(path, data);
+ *   List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+ * }
+ * + *

For the complete source code see + * + * GetFileSystem.java. + * + *

You can also use {@code InputStream} and {@code OutputStream} for streaming: + * 

+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   try (InputStream input = Files.newInputStream(path)) {
+ *     // use input stream
+ *   }
+ *
+ * + *

For the complete source code see + * + * CreateInputStream.java. + * + *

You can set various attributes using + * {@link com.google.cloud.storage.contrib.nio.CloudStorageOptions CloudStorageOptions} static + * helpers: + * 

+ *   Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+ *   Files.write(path, csvLines, StandardCharsets.UTF_8,
+ *       withMimeType("text/csv; charset=UTF-8"),
+ *       withoutCaching());
+ *
+ * + *

For the complete source code see + * + * WriteFileWithAttributes.java. + * + *

NOTE: Cloud Storage uses a flat namespace and therefore doesn't support real + * directories. So this library supports what's known as "pseudo-directories". Any path that + * includes a trailing slash, will be considered a directory. It will always be assumed to exist, + * without performing any I/O. This allows you to do path manipulation in the same manner as you + * would with the normal UNIX file system implementation. You can disable this feature with + * {@link com.google.cloud.storage.contrib.nio.CloudStorageConfiguration#usePseudoDirectories()}. + * + *

Non-SPI Interface

+ * + *

If you don't want to rely on Java SPI, which requires a META-INF file in your jar generated by + * Google Auto, you can instantiate this file system directly as follows: + * + * 

+ *   CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("bucket");
+ *   byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
+ *   Path path = fs.getPath("/object");
+ *   Files.write(path, data);
+ *   data = Files.readAllBytes(path);
+ *
+ * + *

For the complete source code see + * + * CreateCloudStorageFileSystem.java. + */ +@javax.annotation.ParametersAreNonnullByDefault +package com.google.cloud.storage.contrib.nio; diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageConfigurationTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageConfigurationTest.java new file mode 100644 index 000000000000..a2f155d2fd2c --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageConfigurationTest.java @@ -0,0 +1,77 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.ImmutableMap; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link CloudStorageConfiguration}. + */ +@RunWith(JUnit4.class) +public class CloudStorageConfigurationTest { + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void testBuilder() { + CloudStorageConfiguration config = + CloudStorageConfiguration.builder() + .workingDirectory("/omg") + .permitEmptyPathComponents(true) + .stripPrefixSlash(false) + .usePseudoDirectories(false) + .blockSize(666) + .build(); + assertThat(config.workingDirectory()).isEqualTo("/omg"); + assertThat(config.permitEmptyPathComponents()).isTrue(); + assertThat(config.stripPrefixSlash()).isFalse(); + assertThat(config.usePseudoDirectories()).isFalse(); + assertThat(config.blockSize()).isEqualTo(666); + } + + @Test + public void testFromMap() { + CloudStorageConfiguration config = + CloudStorageConfiguration.fromMap( + new ImmutableMap.Builder() + .put("workingDirectory", "/omg") + .put("permitEmptyPathComponents", true) + .put("stripPrefixSlash", false) + .put("usePseudoDirectories", false) + .put("blockSize", 666) + .build()); + assertThat(config.workingDirectory()).isEqualTo("/omg"); + assertThat(config.permitEmptyPathComponents()).isTrue(); + assertThat(config.stripPrefixSlash()).isFalse(); + assertThat(config.usePseudoDirectories()).isFalse(); + assertThat(config.blockSize()).isEqualTo(666); + } + + @Test + public void testFromMap_badKey_throwsIae() { + thrown.expect(IllegalArgumentException.class); + CloudStorageConfiguration.fromMap(ImmutableMap.of("lol", "/omg")); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java new file mode 100644 index 000000000000..4a437a288f5f --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeViewTest.java @@ -0,0 +1,113 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileTime; + +/** + * Unit tests for {@link CloudStorageFileAttributeView}. + */ +@RunWith(JUnit4.class) +public class CloudStorageFileAttributeViewTest { + + private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + private Path path; + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options()); + path = Paths.get(URI.create("gs://red/water")); + } + + @Test + public void testReadAttributes() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withCacheControl("potato")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.readAttributes().cacheControl().get()).isEqualTo("potato"); + } + + @Test + public void testReadAttributes_notFound_throwsNoSuchFileException() throws IOException { + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + thrown.expect(NoSuchFileException.class); + lazyAttributes.readAttributes(); + } + + @Test + public void testReadAttributes_pseudoDirectory() throws IOException { + Path dir = Paths.get(URI.create("gs://red/rum/")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(dir, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.readAttributes()) + .isInstanceOf(CloudStoragePseudoDirectoryAttributes.class); + } + + @Test + public void testName() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withCacheControl("potato")); + CloudStorageFileAttributeView lazyAttributes = + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class); + assertThat(lazyAttributes.name()).isEqualTo("gcs"); + } + + @Test + public void testEquals_equalsTester() { + new EqualsTester() + .addEqualityGroup( + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/rum")), CloudStorageFileAttributeView.class), + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/rum")), CloudStorageFileAttributeView.class)) + .addEqualityGroup( + Files.getFileAttributeView( + Paths.get(URI.create("gs://red/lol/dog")), CloudStorageFileAttributeView.class)) + .testEquals(); + } + + @Test + public void testNullness() throws NoSuchMethodException, SecurityException { + new NullPointerTester() + .ignore(CloudStorageFileAttributeView.class.getMethod("equals", Object.class)) + .setDefault(FileTime.class, FileTime.fromMillis(0)) + .testAllPublicInstanceMethods( + Files.getFileAttributeView(path, CloudStorageFileAttributeView.class)); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java new file mode 100644 index 000000000000..c5001114c14e --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributesTest.java @@ -0,0 +1,178 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.Acl; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Unit tests for {@link CloudStorageFileAttributes}. + */ +@RunWith(JUnit4.class) +public class CloudStorageFileAttributesTest { + + private static final byte[] HAPPY = "(✿◕ ‿◕ )ノ".getBytes(UTF_8); + + private Path path; + private Path dir; + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options()); + path = Paths.get(URI.create("gs://bucket/randompath")); + dir = Paths.get(URI.create("gs://bucket/randompath/")); + } + + @Test + public void testCacheControl() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withCacheControl("potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("potato"); + } + + @Test + public void testMimeType() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).mimeType().get()) + .isEqualTo("text/potato"); + } + + @Test + public void testAcl() throws IOException { + Acl acl = Acl.of(new Acl.User("serf@example.com"), Acl.Role.READER); + Files.write(path, HAPPY, CloudStorageOptions.withAcl(acl)); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) + .contains(acl); + } + + @Test + public void testContentDisposition() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withContentDisposition("crash call")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).contentDisposition().get()) + .isEqualTo("crash call"); + } + + @Test + public void testContentEncoding() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withContentEncoding("my content encoding")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).contentEncoding().get()) + .isEqualTo("my content encoding"); + } + + @Test + public void testUserMetadata() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withUserMetadata("green", "bean")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class) + .userMetadata() + .get("green")) + .isEqualTo("bean"); + } + + @Test + public void testIsDirectory() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isDirectory()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isDirectory()).isTrue(); + } + + @Test + public void testIsRegularFile() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isRegularFile()) + .isTrue(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isRegularFile()) + .isFalse(); + } + + @Test + public void testIsOther() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isOther()).isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isOther()).isFalse(); + } + + @Test + public void testIsSymbolicLink() throws IOException { + Files.write(path, HAPPY); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).isSymbolicLink()) + .isFalse(); + assertThat(Files.readAttributes(dir, CloudStorageFileAttributes.class).isSymbolicLink()) + .isFalse(); + } + + @Test + public void testEquals_equalsTester() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/plain")); + CloudStorageFileAttributes a1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes a2 = Files.readAttributes(path, CloudStorageFileAttributes.class); + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/potato")); + CloudStorageFileAttributes b1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes b2 = Files.readAttributes(path, CloudStorageFileAttributes.class); + new EqualsTester().addEqualityGroup(a1, a2).addEqualityGroup(b1, b2).testEquals(); + } + + @Test + public void testFilekey() throws IOException { + Files.write(path, HAPPY, CloudStorageOptions.withMimeType("text/plain")); + Path path2 = Paths.get(URI.create("gs://bucket/anotherrandompath")); + Files.write(path2, HAPPY, CloudStorageOptions.withMimeType("text/plain")); + + // diff files cannot have same filekey + CloudStorageFileAttributes a1 = Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes a2 = Files.readAttributes(path2, CloudStorageFileAttributes.class); + assertThat(a1.fileKey()).isNotEqualTo(a2.fileKey()); + + // same for directories + CloudStorageFileAttributes b1 = Files.readAttributes(dir, CloudStorageFileAttributes.class); + CloudStorageFileAttributes b2 = + Files.readAttributes( + Paths.get(URI.create("gs://bucket/jacket/")), CloudStorageFileAttributes.class); + assertThat(a1.fileKey()).isNotEqualTo(b1.fileKey()); + assertThat(b1.fileKey()).isNotEqualTo(b2.fileKey()); + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + Files.write(path, HAPPY); + CloudStorageFileAttributes pathAttributes = + Files.readAttributes(path, CloudStorageFileAttributes.class); + CloudStorageFileAttributes dirAttributes = + Files.readAttributes(dir, CloudStorageFileAttributes.class); + NullPointerTester tester = new NullPointerTester(); + tester.ignore(CloudStorageObjectAttributes.class.getMethod("equals", Object.class)); + tester.testAllPublicInstanceMethods(pathAttributes); + tester.testAllPublicInstanceMethods(dirAttributes); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java new file mode 100644 index 000000000000..5475459d9376 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProviderTest.java @@ -0,0 +1,654 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.cloud.storage.contrib.nio.CloudStorageFileSystem.forBucket; +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.COPY_ATTRIBUTES; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.CREATE_NEW; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; + +import com.google.common.collect.ImmutableList; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.CopyOption; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +/** + * Unit tests for {@link CloudStorageFileSystemProvider}. + */ +@RunWith(JUnit4.class) +public class CloudStorageFileSystemProviderTest { + + private static final List FILE_CONTENTS = + ImmutableList.of( + "Fanatics have their dreams, wherewith they weave", + "A paradise for a sect; the savage too", + "From forth the loftiest fashion of his sleep", + "Guesses at Heaven; pity these have not", + "Trac'd upon vellum or wild Indian leaf", + "The shadows of melodious utterance.", + "But bare of laurel they live, dream, and die;", + "For Poesy alone can tell her dreams,", + "With the fine spell of words alone can save", + "Imagination from the sable charm", + "And dumb enchantment. Who alive can say,", + "'Thou art no Poet may'st not tell thy dreams?'", + "Since every man whose soul is not a clod", + "Hath visions, and would speak, if he had loved", + "And been well nurtured in his mother tongue.", + "Whether the dream now purpos'd to rehearse", + "Be poet's or fanatic's will be known", + "When this warm scribe my hand is in the grave."); + + private static final String SINGULARITY = "A string"; + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options()); + } + + @Test + public void testSize() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + assertThat(Files.size(path)).isEqualTo(SINGULARITY.getBytes(UTF_8).length); + } + + @Test + public void testSize_trailingSlash_returnsFakePseudoDirectorySize() throws IOException { + assertThat(Files.size(Paths.get(URI.create("gs://bucket/wat/")))).isEqualTo(1); + } + + @Test + public void testSize_trailingSlash_disablePseudoDirectories() throws IOException { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + Path path = fs.getPath("wat/"); + byte[] rapture = SINGULARITY.getBytes(UTF_8); + Files.write(path, rapture); + assertThat(Files.size(path)).isEqualTo(rapture.length); + } + } + + @Test + public void testReadAllBytes() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testReadAllBytes_trailingSlash() throws IOException { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.readAllBytes(Paths.get(URI.create("gs://bucket/wat/"))); + } + + @Test + public void testNewByteChannelRead() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + byte[] data = SINGULARITY.getBytes(UTF_8); + Files.write(path, data); + try (ReadableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(data.length); + assertThat(input.read(buffer)).isEqualTo(data.length); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo(SINGULARITY); + buffer.rewind(); + assertThat(input.read(buffer)).isEqualTo(-1); + } + } + + @Test + public void testNewByteChannelRead_seeking() throws IOException { + Path path = Paths.get(URI.create("gs://lol/cat")); + Files.write(path, "helloworld".getBytes(UTF_8)); + try (SeekableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(5); + input.position(5); + assertThat(input.position()).isEqualTo(5); + assertThat(input.read(buffer)).isEqualTo(5); + assertThat(input.position()).isEqualTo(10); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo("world"); + buffer.rewind(); + assertThat(input.read(buffer)).isEqualTo(-1); + input.position(0); + assertThat(input.position()).isEqualTo(0); + assertThat(input.read(buffer)).isEqualTo(5); + assertThat(input.position()).isEqualTo(5); + assertThat(new String(buffer.array(), UTF_8)).isEqualTo("hello"); + } + } + + @Test + public void testNewByteChannelRead_seekBeyondSize_reportsEofOnNextRead() throws IOException { + Path path = Paths.get(URI.create("gs://lol/cat")); + Files.write(path, "hellocat".getBytes(UTF_8)); + try (SeekableByteChannel input = Files.newByteChannel(path)) { + ByteBuffer buffer = ByteBuffer.allocate(5); + input.position(10); + assertThat(input.read(buffer)).isEqualTo(-1); + input.position(11); + assertThat(input.read(buffer)).isEqualTo(-1); + assertThat(input.size()).isEqualTo(8); + } + } + + @Test + public void testNewByteChannelRead_trailingSlash() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.newByteChannel(path); + } + + @Test + public void testNewByteChannelRead_notFound() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wednesday")); + thrown.expect(NoSuchFileException.class); + Files.newByteChannel(path); + } + + @Test + public void testNewByteChannelWrite() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/tests")); + try (SeekableByteChannel output = Files.newByteChannel(path, WRITE)) { + assertThat(output.position()).isEqualTo(0); + assertThat(output.size()).isEqualTo(0); + ByteBuffer buffer = ByteBuffer.wrap("filec".getBytes(UTF_8)); + assertThat(output.write(buffer)).isEqualTo(5); + assertThat(output.position()).isEqualTo(5); + assertThat(output.size()).isEqualTo(5); + buffer = ByteBuffer.wrap("onten".getBytes(UTF_8)); + assertThat(output.write(buffer)).isEqualTo(5); + assertThat(output.position()).isEqualTo(10); + assertThat(output.size()).isEqualTo(10); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo("fileconten"); + } + + @Test + public void testNewInputStream() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + try (InputStream input = Files.newInputStream(path)) { + byte[] data = new byte[SINGULARITY.getBytes(UTF_8).length]; + input.read(data); + assertThat(new String(data, UTF_8)).isEqualTo(SINGULARITY); + } + } + + @Test + public void testNewInputStream_trailingSlash() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + thrown.expect(CloudStoragePseudoDirectoryException.class); + try (InputStream input = Files.newInputStream(path)) { + input.read(); + } + } + + @Test + public void testNewInputStream_notFound() throws IOException { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + thrown.expect(NoSuchFileException.class); + try (InputStream input = Files.newInputStream(path)) { + input.read(); + } + } + + @Test + public void testNewOutputStream() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_truncateByDefault() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.write(path, "hello".getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_truncateExplicitly() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + Files.write(path, "hello".getBytes(UTF_8)); + try (OutputStream output = Files.newOutputStream(path, TRUNCATE_EXISTING)) { + output.write(SINGULARITY.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(path), UTF_8)).isEqualTo(SINGULARITY); + } + + @Test + public void testNewOutputStream_trailingSlash() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/wat/")); + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.newOutputStream(path); + } + + @Test + public void testNewOutputStream_createNew() throws IOException { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + Files.newOutputStream(path, CREATE_NEW); + } + + @Test + public void testNewOutputStream_createNew_alreadyExists() throws IOException { + Path path = Paths.get(URI.create("gs://cry/wednesday")); + Files.write(path, SINGULARITY.getBytes(UTF_8)); + thrown.expect(FileAlreadyExistsException.class); + Files.newOutputStream(path, CREATE_NEW); + } + + @Test + public void testWrite_objectNameWithExtraSlashes_throwsIae() throws IOException { + Path path = Paths.get(URI.create("gs://double/slash//yep")); + thrown.expect(IllegalArgumentException.class); + Files.write(path, FILE_CONTENTS, UTF_8); + } + + @Test + public void testWrite_objectNameWithExtraSlashes_canBeNormalized() throws IOException { + try (CloudStorageFileSystem fs = forBucket("greenbean", permitEmptyPathComponents(false))) { + Path path = fs.getPath("adipose//yep").normalize(); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(fs.getPath("adipose", "yep"))).isTrue(); + } + } + + @Test + public void testWrite_objectNameWithExtraSlashes_permitEmptyPathComponents() throws IOException { + try (CloudStorageFileSystem fs = forBucket("greenbean", permitEmptyPathComponents(true))) { + Path path = fs.getPath("adipose//yep"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite_absoluteObjectName_prefixSlashGetsRemoved() throws IOException { + Path path = Paths.get(URI.create("gs://greenbean/adipose/yep")); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testWrite_absoluteObjectName_disableStrip_slashGetsPreserved() throws IOException { + try (CloudStorageFileSystem fs = + forBucket( + "greenbean", CloudStorageConfiguration.builder().stripPrefixSlash(false).build())) { + Path path = fs.getPath("/adipose/yep"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + assertThat(Files.exists(path)).isTrue(); + } + } + + @Test + public void testWrite() throws IOException { + Path path = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.readAllLines(path, UTF_8)).isEqualTo(FILE_CONTENTS); + } + + @Test + public void testWriteOnClose() throws IOException { + Path path = Paths.get(URI.create("gs://greenbean/adipose")); + try (SeekableByteChannel chan = Files.newByteChannel(path, WRITE)) { + // writing lots of contents to defeat channel-internal buffering. + for (int i = 0; i < 9999; i++) { + for (String s : FILE_CONTENTS) { + chan.write(ByteBuffer.wrap(s.getBytes(UTF_8))); + } + } + try { + Files.size(path); + // we shouldn't make it to this line. Not using thrown.expect because + // I still want to run a few lines after the exception. + assertThat(false).isTrue(); + } catch (NoSuchFileException nsf) { + // that's what we wanted, we're good. + } + } + // channel now closed, the file should be there and with the new contents. + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.size(path)).isGreaterThan(100L); + } + + @Test + public void testWrite_trailingSlash() throws IOException { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.write(Paths.get(URI.create("gs://greenbean/adipose/")), FILE_CONTENTS, UTF_8); + } + + @Test + public void testExists() throws IOException { + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion")))).isFalse(); + Files.write(Paths.get(URI.create("gs://military/fashion")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion")))).isTrue(); + } + + @Test + public void testExists_trailingSlash() { + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/")))).isTrue(); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/.")))).isTrue(); + assertThat(Files.exists(Paths.get(URI.create("gs://military/fashion/..")))).isTrue(); + } + + @Test + public void testExists_trailingSlash_disablePseudoDirectories() throws IOException { + try (CloudStorageFileSystem fs = forBucket("military", usePseudoDirectories(false))) { + assertThat(Files.exists(fs.getPath("fashion/"))).isFalse(); + } + } + + @Test + public void testDelete() throws IOException { + Files.write(Paths.get(URI.create("gs://love/fashion")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.exists(Paths.get(URI.create("gs://love/fashion")))).isTrue(); + Files.delete(Paths.get(URI.create("gs://love/fashion"))); + assertThat(Files.exists(Paths.get(URI.create("gs://love/fashion")))).isFalse(); + } + + @Test + public void testDelete_dotDirNotNormalized_throwsIae() throws IOException { + thrown.expect(IllegalArgumentException.class); + Files.delete(Paths.get(URI.create("gs://love/fly/../passion"))); + } + + @Test + public void testDelete_trailingSlash() throws IOException { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.delete(Paths.get(URI.create("gs://love/passion/"))); + } + + @Test + public void testDelete_trailingSlash_disablePseudoDirectories() throws IOException { + try (CloudStorageFileSystem fs = forBucket("pumpkin", usePseudoDirectories(false))) { + Path path = fs.getPath("wat/"); + Files.write(path, FILE_CONTENTS, UTF_8); + assertThat(Files.exists(path)); + Files.delete(path); + assertThat(!Files.exists(path)); + } + } + + @Test + public void testDelete_notFound() throws IOException { + thrown.expect(NoSuchFileException.class); + Files.delete(Paths.get(URI.create("gs://loveh/passionehu"))); + } + + @Test + public void testDeleteIfExists() throws IOException { + Files.write(Paths.get(URI.create("gs://love/passionz")), "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + assertThat(Files.deleteIfExists(Paths.get(URI.create("gs://love/passionz")))).isTrue(); + } + + @Test + public void testDeleteIfExists_trailingSlash() throws IOException { + thrown.expect(CloudStoragePseudoDirectoryException.class); + Files.deleteIfExists(Paths.get(URI.create("gs://love/passion/"))); + } + + @Test + public void testCopy() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target); + assertThat(new String(Files.readAllBytes(target), UTF_8)).isEqualTo("(✿◕ ‿◕ )ノ"); + assertThat(Files.exists(source)).isTrue(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testCopy_sourceMissing_throwsNoSuchFileException() throws IOException { + thrown.expect(NoSuchFileException.class); + Files.copy( + Paths.get(URI.create("gs://military/fashion.show")), + Paths.get(URI.create("gs://greenbean/adipose"))); + } + + @Test + public void testCopy_targetExists_throwsFileAlreadyExistsException() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.write(target, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + thrown.expect(FileAlreadyExistsException.class); + Files.copy(source, target); + } + + @Test + public void testCopyReplace_targetExists_works() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.write(target, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.copy(source, target, REPLACE_EXISTING); + } + + @Test + public void testCopy_directory_doesNothing() throws IOException { + Path source = Paths.get(URI.create("gs://military/fundir/")); + Path target = Paths.get(URI.create("gs://greenbean/loldir/")); + Files.copy(source, target); + } + + @Test + public void testCopy_atomic_throwsUnsupported() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + thrown.expect(UnsupportedOperationException.class); + Files.copy(source, target, ATOMIC_MOVE); + } + + @Test + public void testMove() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + Files.move(source, target); + assertThat(new String(Files.readAllBytes(target), UTF_8)).isEqualTo("(✿◕ ‿◕ )ノ"); + assertThat(Files.exists(source)).isFalse(); + assertThat(Files.exists(target)).isTrue(); + } + + @Test + public void testCreateDirectory() throws IOException { + Path path = Paths.get(URI.create("gs://greenbean/dir/")); + Files.createDirectory(path); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testMove_atomicMove_notSupported() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write(source, "(✿◕ ‿◕ )ノ".getBytes(UTF_8)); + thrown.expect(AtomicMoveNotSupportedException.class); + Files.move(source, target, ATOMIC_MOVE); + } + + @Test + public void testIsDirectory() throws IOException { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://doodle"))) { + assertThat(Files.isDirectory(fs.getPath(""))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("/"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("./"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("cat/.."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("hello/cat/.."))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("cat/../"))).isTrue(); + assertThat(Files.isDirectory(fs.getPath("hello/cat/../"))).isTrue(); + } + } + + @Test + public void testIsDirectory_trailingSlash_alwaysTrue() { + assertThat(Files.isDirectory(Paths.get(URI.create("gs://military/fundir/")))).isTrue(); + } + + @Test + public void testIsDirectory_trailingSlash_pseudoDirectoriesDisabled_false() throws IOException { + try (CloudStorageFileSystem fs = forBucket("doodle", usePseudoDirectories(false))) { + assertThat(Files.isDirectory(fs.getPath("fundir/"))).isFalse(); + } + } + + @Test + public void testCopy_withCopyAttributes_preservesAttributes() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write( + source, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withMimeType("text/lolcat"), + CloudStorageOptions.withCacheControl("public; max-age=666"), + CloudStorageOptions.withContentEncoding("foobar"), + CloudStorageOptions.withContentDisposition("my-content-disposition"), + CloudStorageOptions.withUserMetadata("answer", "42")); + Files.copy(source, target, COPY_ATTRIBUTES); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/lolcat"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + assertThat(attributes.contentEncoding()).hasValue("foobar"); + assertThat(attributes.contentDisposition()).hasValue("my-content-disposition"); + assertThat(attributes.userMetadata().containsKey("answer")).isTrue(); + assertThat(attributes.userMetadata().get("answer")).isEqualTo("42"); + } + + @Test + public void testCopy_withoutOptions_doesntPreservesAttributes() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target = Paths.get(URI.create("gs://greenbean/adipose")); + Files.write( + source, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withMimeType("text/lolcat"), + CloudStorageOptions.withCacheControl("public; max-age=666"), + CloudStorageOptions.withUserMetadata("answer", "42")); + Files.copy(source, target); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target, CloudStorageFileAttributes.class); + String mimeType = attributes.mimeType().orNull(); + String cacheControl = attributes.cacheControl().orNull(); + assertThat(mimeType).isNotEqualTo("text/lolcat"); + assertThat(cacheControl).isNull(); + assertThat(attributes.userMetadata().containsKey("answer")).isFalse(); + } + + @Test + public void testCopy_overwriteAttributes() throws IOException { + Path source = Paths.get(URI.create("gs://military/fashion.show")); + Path target1 = Paths.get(URI.create("gs://greenbean/adipose")); + Path target2 = Paths.get(URI.create("gs://greenbean/round")); + Files.write( + source, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withMimeType("text/lolcat"), + CloudStorageOptions.withCacheControl("public; max-age=666")); + Files.copy(source, target1, COPY_ATTRIBUTES); + Files.copy(source, target2, COPY_ATTRIBUTES, CloudStorageOptions.withMimeType("text/palfun")); + + CloudStorageFileAttributes attributes = + Files.readAttributes(target1, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/lolcat"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + + attributes = Files.readAttributes(target2, CloudStorageFileAttributes.class); + assertThat(attributes.mimeType()).hasValue("text/palfun"); + assertThat(attributes.cacheControl()).hasValue("public; max-age=666"); + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://blood"))) { + NullPointerTester tester = new NullPointerTester(); + tester.ignore(CloudStorageFileSystemProvider.class.getMethod("equals", Object.class)); + tester.setDefault(URI.class, URI.create("gs://blood")); + tester.setDefault(Path.class, fs.getPath("and/one")); + tester.setDefault(OpenOption.class, CREATE); + tester.setDefault(CopyOption.class, COPY_ATTRIBUTES); + // can't do that, setGCloudOptions accepts a null argument. + // TODO(jart): Figure out how to re-enable this. + // tester.testAllPublicStaticMethods(CloudStorageFileSystemProvider.class); + tester.testAllPublicInstanceMethods(new CloudStorageFileSystemProvider()); + } + } + + @Test + public void testProviderEquals() { + Path path1 = Paths.get(URI.create("gs://bucket/tuesday")); + Path path2 = Paths.get(URI.create("gs://blood/wednesday")); + Path path3 = Paths.get("tmp"); + assertThat(path1.getFileSystem().provider()).isEqualTo(path2.getFileSystem().provider()); + assertThat(path1.getFileSystem().provider()).isNotEqualTo(path3.getFileSystem().provider()); + } + + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { + return CloudStorageConfiguration.builder().permitEmptyPathComponents(value).build(); + } + + private static CloudStorageConfiguration usePseudoDirectories(boolean value) { + return CloudStorageConfiguration.builder().usePseudoDirectories(value).build(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java new file mode 100644 index 000000000000..90e61ded721a --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.StorageOptions; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +/** + * Unit tests for {@link CloudStorageFileSystem}. + */ +@RunWith(JUnit4.class) +public class CloudStorageFileSystemTest { + + private static final String ALONE = + "To be, or not to be, that is the question—\n" + + "Whether 'tis Nobler in the mind to suffer\n" + + "The Slings and Arrows of outrageous Fortune,\n" + + "Or to take Arms against a Sea of troubles,\n" + + "And by opposing, end them? To die, to sleep—\n" + + "No more; and by a sleep, to say we end\n" + + "The Heart-ache, and the thousand Natural shocks\n" + + "That Flesh is heir to? 'Tis a consummation\n"; + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options()); + } + + @Test + public void testGetPath() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(fs.getPath("/angel").toString()).isEqualTo("/angel"); + assertThat(fs.getPath("/angel").toUri().toString()).isEqualTo("gs://bucket/angel"); + } + } + + @Test + public void testWrite() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + Files.write(fs.getPath("/angel"), ALONE.getBytes(UTF_8)); + } + assertThat(new String(Files.readAllBytes(Paths.get(URI.create("gs://bucket/angel"))), UTF_8)) + .isEqualTo(ALONE); + } + + @Test + public void testRead() throws IOException { + Files.write(Paths.get(URI.create("gs://bucket/angel")), ALONE.getBytes(UTF_8)); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(new String(Files.readAllBytes(fs.getPath("/angel")), UTF_8)).isEqualTo(ALONE); + } + } + + @Test + public void testExists_false() throws IOException { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"))) { + assertThat(Files.exists(fs.getPath("/angel"))).isFalse(); + } + } + + @Test + public void testExists_true() throws IOException { + Files.write(Paths.get(URI.create("gs://bucket/angel")), ALONE.getBytes(UTF_8)); + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(Files.exists(fs.getPath("/angel"))).isTrue(); + } + } + + @Test + public void testGetters() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + assertThat(fs.isOpen()).isTrue(); + assertThat(fs.isReadOnly()).isFalse(); + assertThat(fs.getRootDirectories()).containsExactly(fs.getPath("/")); + assertThat(fs.getFileStores()).isEmpty(); + assertThat(fs.getSeparator()).isEqualTo("/"); + assertThat(fs.supportedFileAttributeViews()).containsExactly("basic", "gcs"); + } + } + + @Test + public void testEquals() throws IOException { + try (FileSystem bucket1 = CloudStorageFileSystem.forBucket("bucket"); + FileSystem bucket2 = FileSystems.getFileSystem(URI.create("gs://bucket")); + FileSystem doge1 = CloudStorageFileSystem.forBucket("doge"); + FileSystem doge2 = FileSystems.getFileSystem(URI.create("gs://doge"))) { + new EqualsTester() + .addEqualityGroup(bucket1, bucket2) + .addEqualityGroup(doge1, doge2) + .testEquals(); + } + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + try (FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"))) { + NullPointerTester tester = + new NullPointerTester() + .ignore(CloudStorageFileSystem.class.getMethod("equals", Object.class)) + .setDefault(CloudStorageConfiguration.class, CloudStorageConfiguration.DEFAULT) + .setDefault(StorageOptions.class, LocalStorageHelper.options()); + tester.testAllPublicStaticMethods(CloudStorageFileSystem.class); + tester.testAllPublicInstanceMethods(fs); + } + } + + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = CloudStorageFileSystem.forBucket("bucket")) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + Files.write(path, ALONE.getBytes(UTF_8)); + } + + List got = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(fs.getPath("/dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + } + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java new file mode 100644 index 000000000000..e23ba5055511 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageOptionsTest.java @@ -0,0 +1,120 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.Acl; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Unit tests for {@link CloudStorageOptions}. + */ +@RunWith(JUnit4.class) +public class CloudStorageOptionsTest { + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options()); + } + + @Test + public void testWithoutCaching() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withoutCaching()); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("no-cache"); + } + + @Test + public void testCacheControl() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withCacheControl("potato")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).cacheControl().get()) + .isEqualTo("potato"); + } + + @Test + public void testWithAcl() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Acl acl = Acl.of(new Acl.User("king@example.com"), Acl.Role.OWNER); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withAcl(acl)); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).acl().get()) + .contains(acl); + } + + @Test + public void testWithContentDisposition() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withContentDisposition("bubbly fun")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).contentDisposition().get()) + .isEqualTo("bubbly fun"); + } + + @Test + public void testWithContentEncoding() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withContentEncoding("gzip")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).contentEncoding().get()) + .isEqualTo("gzip"); + } + + @Test + public void testWithUserMetadata() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write( + path, + "(✿◕ ‿◕ )ノ".getBytes(UTF_8), + CloudStorageOptions.withUserMetadata("nolo", "contendere"), + CloudStorageOptions.withUserMetadata("eternal", "sadness")); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class).userMetadata().get("nolo")) + .isEqualTo("contendere"); + assertThat( + Files.readAttributes(path, CloudStorageFileAttributes.class) + .userMetadata() + .get("eternal")) + .isEqualTo("sadness"); + } + + @Test + public void testWithMimeType_string() throws IOException { + Path path = Paths.get(URI.create("gs://bucket/path")); + Files.write(path, "(✿◕ ‿◕ )ノ".getBytes(UTF_8), CloudStorageOptions.withMimeType("text/plain")); + assertThat(Files.readAttributes(path, CloudStorageFileAttributes.class).mimeType().get()) + .isEqualTo("text/plain"); + } + + @Test + public void testNullness() { + NullPointerTester tester = new NullPointerTester(); + tester.testAllPublicStaticMethods(CloudStorageOptions.class); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java new file mode 100644 index 000000000000..7ba37211cabb --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStoragePathTest.java @@ -0,0 +1,498 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.common.collect.Iterables; +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.ProviderMismatchException; + +/** + * Unit tests for {@link CloudStoragePath}. + */ +@RunWith(JUnit4.class) +public class CloudStoragePathTest { + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + @Before + public void before() { + CloudStorageFileSystemProvider.setGCloudOptions(LocalStorageHelper.options()); + } + + @Test + public void testCreate_neverRemoveExtraSlashes() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("lol//cat").toString()).isEqualTo("lol//cat"); + assertThat((Object) fs.getPath("lol//cat")).isEqualTo(fs.getPath("lol//cat")); + } + } + + @Test + public void testCreate_preservesTrailingSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("lol/cat/").toString()).isEqualTo("lol/cat/"); + assertThat((Object) fs.getPath("lol/cat/")).isEqualTo(fs.getPath("lol/cat/")); + } + } + + @Test + public void testGetGcsFilename_empty_notAllowed() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("").getBlobId(); + } + } + + @Test + public void testGetGcsFilename_stripsPrefixSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi").getBlobId().name()).isEqualTo("hi"); + } + } + + @Test + public void testGetGcsFilename_overrideStripPrefixSlash_doesntStripPrefixSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", stripPrefixSlash(false))) { + assertThat(fs.getPath("/hi").getBlobId().name()).isEqualTo("/hi"); + } + } + + @Test + public void testGetGcsFilename_extraSlashes_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("a//b").getBlobId().name(); + } + } + + @Test + public void testGetGcsFilename_overridepermitEmptyPathComponents() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("a//b").getBlobId().name()).isEqualTo("a//b"); + } + } + + @Test + public void testGetGcsFilename_freaksOutOnExtraSlashesAndDotDirs() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("a//b/..").getBlobId().name(); + } + } + + @Test + public void testNameCount() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").getNameCount()).isEqualTo(1); + assertThat(fs.getPath("/").getNameCount()).isEqualTo(0); + assertThat(fs.getPath("/hi/").getNameCount()).isEqualTo(1); + assertThat(fs.getPath("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(fs.getPath("hi/yo").getNameCount()).isEqualTo(2); + } + } + + @Test + public void testGetName() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").getName(0).toString()).isEqualTo(""); + assertThat(fs.getPath("/hi").getName(0).toString()).isEqualTo("hi"); + assertThat(fs.getPath("hi/there").getName(1).toString()).isEqualTo("there"); + } + } + + @Test + public void testGetName_negative_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("angel").getName(-1); + } + } + + @Test + public void testGetName_overflow_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + fs.getPath("angel").getName(1); + } + } + + @Test + public void testIterator() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(Iterables.get(fs.getPath("/dog/mog"), 0).toString()).isEqualTo("dog"); + assertThat(Iterables.get(fs.getPath("/dog/mog"), 1).toString()).isEqualTo("mog"); + assertThat(Iterables.size(fs.getPath("/"))).isEqualTo(0); + assertThat(Iterables.size(fs.getPath(""))).isEqualTo(1); + assertThat(Iterables.get(fs.getPath(""), 0).toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/").normalize().toString()).isEqualTo("/"); + assertThat(fs.getPath("a/x/../b/x/..").normalize().toString()).isEqualTo("a/b/"); + assertThat(fs.getPath("/x/x/../../♡").normalize().toString()).isEqualTo("/♡"); + assertThat(fs.getPath("/x/x/./.././.././♡").normalize().toString()).isEqualTo("/♡"); + } + } + + @Test + public void testNormalize_dot_becomesBlank() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").normalize().toString()).isEqualTo(""); + assertThat(fs.getPath(".").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_trailingSlash_isPreserved() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("o/").normalize().toString()).isEqualTo("o/"); + } + } + + @Test + public void testNormalize_doubleDot_becomesBlank() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("..").normalize().toString()).isEqualTo(""); + assertThat(fs.getPath("../..").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_extraSlashes_getRemoved() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("//life///b/good//").normalize().toString()).isEqualTo("/life/b/good/"); + } + } + + @Test + public void testToRealPath_hasDotDir_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a/hi./b").toRealPath(); + fs.getPath("a/.hi/b").toRealPath(); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("dot-dir"); + fs.getPath("a/./b").toRealPath(); + } + } + + @Test + public void testToRealPath_hasDotDotDir_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + fs.getPath("a/hi../b").toRealPath(); + fs.getPath("a/..hi/b").toRealPath(); + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("dot-dir"); + fs.getPath("a/../b").toRealPath(); + } + } + + @Test + public void testToRealPath_extraSlashes_throwsIae() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(IllegalArgumentException.class); + thrown.expectMessage("extra slashes"); + fs.getPath("a//b").toRealPath(); + } + } + + @Test + public void testToRealPath_overridePermitEmptyPathComponents_extraSlashes_slashesRemain() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("/life///b/./good/").toRealPath().toString()) + .isEqualTo("life///b/./good/"); + } + } + + @Test + public void testToRealPath_permitEmptyPathComponents_doesNotNormalize() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", permitEmptyPathComponents(true))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("a"); + assertThat(fs.getPath("a//b").toRealPath().toString()).isEqualTo("a//b"); + assertThat(fs.getPath("a//./b//..").toRealPath().toString()).isEqualTo("a//./b//.."); + } + } + + @Test + public void testToRealPath_withWorkingDirectory_makesAbsolute() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", workingDirectory("/lol"))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("lol/a"); + } + } + + @Test + public void testToRealPath_disableStripPrefixSlash_makesPathAbsolute() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", stripPrefixSlash(false))) { + assertThat(fs.getPath("a").toRealPath().toString()).isEqualTo("/a"); + assertThat(fs.getPath("/a").toRealPath().toString()).isEqualTo("/a"); + } + } + + @Test + public void testToRealPath_trailingSlash_getsPreserved() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("a/b/").toRealPath().toString()).isEqualTo("a/b/"); + } + } + + @Test + public void testNormalize_empty_returnsEmpty() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("").normalize().toString()).isEqualTo(""); + } + } + + @Test + public void testNormalize_preserveTrailingSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("a/b/../c/").normalize().toString()).isEqualTo("a/c/"); + assertThat(fs.getPath("a/b/./c/").normalize().toString()).isEqualTo("a/b/c/"); + } + } + + @Test + public void testGetParent_preserveTrailingSlash() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("a/b/c").getParent().toString()).isEqualTo("a/b/"); + assertThat(fs.getPath("a/b/c/").getParent().toString()).isEqualTo("a/b/"); + assertThat((Object) fs.getPath("").getParent()).isNull(); + assertThat((Object) fs.getPath("/").getParent()).isNull(); + assertThat((Object) fs.getPath("aaa").getParent()).isNull(); + assertThat((Object) (fs.getPath("aaa/").getParent())).isNull(); + } + } + + @Test + public void testGetRoot() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hello").getRoot().toString()).isEqualTo("/"); + assertThat((Object) fs.getPath("hello").getRoot()).isNull(); + } + } + + @Test + public void testRelativize() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat( + fs.getPath("/foo/bar/lol/cat").relativize(fs.getPath("/foo/a/b/../../c")).toString()) + .isEqualTo("../../../a/b/../../c"); + } + } + + @Test + public void testRelativize_providerMismatch() throws IOException { + try (CloudStorageFileSystem gcs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(ProviderMismatchException.class); + gcs.getPath("/etc").relativize(FileSystems.getDefault().getPath("/dog")); + } + } + + @Test + @SuppressWarnings("ReturnValueIgnored") // testing that an Exception is thrown + public void testRelativize_providerMismatch2() throws IOException { + try (CloudStorageFileSystem gcs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(ProviderMismatchException.class); + gcs.getPath("/dog").relativize(FileSystems.getDefault().getPath("/etc")); + } + } + + @Test + public void testResolve() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi").resolve("there").toString()).isEqualTo("/hi/there"); + assertThat(fs.getPath("hi").resolve("there").toString()).isEqualTo("hi/there"); + } + } + + @Test + public void testResolve_providerMismatch() throws IOException { + try (CloudStorageFileSystem gcs = CloudStorageFileSystem.forBucket("doodle")) { + thrown.expect(ProviderMismatchException.class); + gcs.getPath("etc").resolve(FileSystems.getDefault().getPath("/dog")); + } + } + + @Test + public void testIsAbsolute() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi").isAbsolute()).isTrue(); + assertThat(fs.getPath("hi").isAbsolute()).isFalse(); + } + } + + @Test + public void testToAbsolutePath() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat((Object) fs.getPath("/hi").toAbsolutePath()).isEqualTo(fs.getPath("/hi")); + assertThat((Object) fs.getPath("hi").toAbsolutePath()).isEqualTo(fs.getPath("/hi")); + } + } + + @Test + public void testToAbsolutePath_withWorkingDirectory() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", workingDirectory("/lol"))) { + assertThat(fs.getPath("a").toAbsolutePath().toString()).isEqualTo("/lol/a"); + } + } + + @Test + public void testGetFileName() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").getFileName().toString()).isEqualTo("there"); + assertThat(fs.getPath("military/fashion/show").getFileName().toString()).isEqualTo("show"); + } + } + + @Test + public void testCompareTo() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/there"))).isEqualTo(0); + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/therf"))).isEqualTo(-1); + assertThat(fs.getPath("/hi/there").compareTo(fs.getPath("/hi/therd"))).isEqualTo(1); + } + } + + @Test + public void testStartsWith() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/there"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/hi/"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("hi"))).isFalse(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath("/"))).isTrue(); + assertThat(fs.getPath("/hi/there").startsWith(fs.getPath(""))).isFalse(); + } + } + + @Test + public void testEndsWith() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("there"))).isTrue(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/blag/therf"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/hi/there"))).isTrue(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath("/there"))).isFalse(); + assertThat(fs.getPath("/human/that/you/cry").endsWith(fs.getPath("that/you/cry"))).isTrue(); + assertThat(fs.getPath("/human/that/you/cry").endsWith(fs.getPath("that/you/cry/"))).isTrue(); + assertThat(fs.getPath("/hi/there/").endsWith(fs.getPath("/"))).isFalse(); + assertThat(fs.getPath("/hi/there").endsWith(fs.getPath(""))).isFalse(); + assertThat(fs.getPath("").endsWith(fs.getPath(""))).isTrue(); + } + } + + @Test + public void testResolve_willWorkWithRecursiveCopy() throws IOException { + // See: http://stackoverflow.com/a/10068306 + try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); + FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { + Path targetPath = fsTarget.getPath("/some/folder/"); + Path relSrcPath = fsSource.getPath("file.txt"); + assertThat((Object) targetPath.resolve(relSrcPath)) + .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); + } + } + + @Test + public void testRelativize_willWorkWithRecursiveCopy() throws IOException { + // See: http://stackoverflow.com/a/10068306 + try (FileSystem fsSource = FileSystems.getFileSystem(URI.create("gs://hello")); + FileSystem fsTarget = FileSystems.getFileSystem(URI.create("gs://cat"))) { + Path targetPath = fsTarget.getPath("/some/folder/"); + Path sourcePath = fsSource.getPath("/sloth/"); + Path file = fsSource.getPath("/sloth/file.txt"); + assertThat((Object) targetPath.resolve(sourcePath.relativize(file))) + .isEqualTo(fsTarget.getPath("/some/folder/file.txt")); + } + } + + @Test + public void testToFile_unsupported() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + Path path = fs.getPath("/lol"); + thrown.expect(UnsupportedOperationException.class); + path.toFile(); + } + } + + @Test + public void testEquals() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + new EqualsTester() + // These are obviously equal. + .addEqualityGroup(fs.getPath("/hello/cat"), fs.getPath("/hello/cat")) + // These are equal because equals() runs things through toRealPath() + .addEqualityGroup(fs.getPath("great/commandment"), fs.getPath("/great/commandment")) + .addEqualityGroup(fs.getPath("great/commandment/"), fs.getPath("/great/commandment/")) + // Equals shouldn't do error checking or normalization. + .addEqualityGroup(fs.getPath("foo/../bar"), fs.getPath("foo/../bar")) + .addEqualityGroup(fs.getPath("bar")) + .testEquals(); + } + } + + @Test + public void testEquals_currentDirectoryIsTakenIntoConsideration() throws IOException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle", workingDirectory("/hello"))) { + new EqualsTester() + .addEqualityGroup(fs.getPath("cat"), fs.getPath("/hello/cat")) + .addEqualityGroup(fs.getPath(""), fs.getPath("/hello")) + .testEquals(); + } + } + + @Test + public void testNullness() throws IOException, NoSuchMethodException, SecurityException { + try (CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("doodle")) { + NullPointerTester tester = new NullPointerTester(); + tester.ignore(CloudStoragePath.class.getMethod("equals", Object.class)); + tester.setDefault(Path.class, fs.getPath("sup")); + tester.testAllPublicStaticMethods(CloudStoragePath.class); + tester.testAllPublicInstanceMethods(fs.getPath("sup")); + } + } + + private static CloudStorageConfiguration stripPrefixSlash(boolean value) { + return CloudStorageConfiguration.builder().stripPrefixSlash(value).build(); + } + + private static CloudStorageConfiguration permitEmptyPathComponents(boolean value) { + return CloudStorageConfiguration.builder().permitEmptyPathComponents(value).build(); + } + + private static CloudStorageConfiguration workingDirectory(String value) { + return CloudStorageConfiguration.builder().workingDirectory(value).build(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java new file mode 100644 index 000000000000..ac42be8cd155 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageReadChannelTest.java @@ -0,0 +1,156 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.cloud.ReadChannel; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.BlobId; +import com.google.cloud.storage.Storage; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonWritableChannelException; + +/** + * Unit tests for {@link CloudStorageReadChannel}. + */ +@RunWith(JUnit4.class) +public class CloudStorageReadChannelTest { + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + private CloudStorageReadChannel chan; + + private final Storage gcsStorage = mock(Storage.class); + private final BlobId file = BlobId.of("blob", "attack"); + private final Blob metadata = mock(Blob.class); + private final ReadChannel gcsChannel = mock(ReadChannel.class); + + @Before + public void before() throws IOException { + when(metadata.size()).thenReturn(42L); + when(gcsStorage.get(file)).thenReturn(metadata); + when(gcsStorage.reader(eq(file))).thenReturn(gcsChannel); + when(gcsChannel.isOpen()).thenReturn(true); + chan = CloudStorageReadChannel.create(gcsStorage, file, 0); + verify(gcsStorage).get(eq(file)); + verify(gcsStorage).reader(eq(file)); + } + + @Test + public void testRead() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + when(gcsChannel.read(eq(buffer))).thenReturn(1); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.read(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + verify(gcsChannel).read(any(ByteBuffer.class)); + verify(gcsChannel, times(3)).isOpen(); + } + + @Test + public void testRead_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.read(ByteBuffer.allocate(1)); + } + + @Test + public void testWrite_throwsNonWritableChannelException() throws IOException { + thrown.expect(NonWritableChannelException.class); + chan.write(ByteBuffer.allocate(1)); + } + + @Test + public void testTruncate_throwsNonWritableChannelException() throws IOException { + thrown.expect(NonWritableChannelException.class); + chan.truncate(0); + } + + @Test + public void testIsOpen() throws IOException { + when(gcsChannel.isOpen()).thenReturn(true).thenReturn(false); + assertThat(chan.isOpen()).isTrue(); + chan.close(); + assertThat(chan.isOpen()).isFalse(); + verify(gcsChannel, times(2)).isOpen(); + verify(gcsChannel).close(); + } + + @Test + public void testSize() throws IOException { + assertThat(chan.size()).isEqualTo(42L); + verify(gcsChannel).isOpen(); + verifyZeroInteractions(gcsChannel); + } + + @Test + public void testSize_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.size(); + } + + @Test + public void testPosition_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.position(); + } + + @Test + public void testSetPosition_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.position(0); + } + + @Test + public void testClose_calledMultipleTimes_doesntThrowAnError() throws IOException { + chan.close(); + chan.close(); + chan.close(); + } + + @Test + public void testSetPosition() throws IOException { + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.size()).isEqualTo(42L); + chan.position(1L); + assertThat(chan.position()).isEqualTo(1L); + assertThat(chan.size()).isEqualTo(42L); + verify(gcsChannel).seek(1); + verify(gcsChannel, times(5)).isOpen(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannelTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannelTest.java new file mode 100644 index 000000000000..8e95e04b0616 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/CloudStorageWriteChannelTest.java @@ -0,0 +1,125 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.Mockito.when; + +import com.google.cloud.WriteChannel; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; + +/** + * Unit tests for {@link CloudStorageWriteChannel}. + */ +@RunWith(JUnit4.class) +public class CloudStorageWriteChannelTest { + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + private final WriteChannel gcsChannel = mock(WriteChannel.class); + private final CloudStorageWriteChannel chan = new CloudStorageWriteChannel(gcsChannel); + + @Before + public void before() { + when(gcsChannel.isOpen()).thenReturn(true); + } + + @Test + public void testRead_throwsNonReadableChannelException() throws IOException { + thrown.expect(NonReadableChannelException.class); + chan.read(ByteBuffer.allocate(1)); + } + + @Test + public void testWrite() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(1); + buffer.put((byte) 'B'); + assertThat(chan.position()).isEqualTo(0L); + assertThat(chan.size()).isEqualTo(0L); + when(gcsChannel.write(eq(buffer))).thenReturn(1); + assertThat(chan.write(buffer)).isEqualTo(1); + assertThat(chan.position()).isEqualTo(1L); + assertThat(chan.size()).isEqualTo(1L); + verify(gcsChannel).write(any(ByteBuffer.class)); + verify(gcsChannel, times(5)).isOpen(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testWrite_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.write(ByteBuffer.allocate(1)); + } + + @Test + public void testIsOpen() throws IOException { + when(gcsChannel.isOpen()).thenReturn(true).thenReturn(false); + assertThat(chan.isOpen()).isTrue(); + chan.close(); + assertThat(chan.isOpen()).isFalse(); + verify(gcsChannel, times(2)).isOpen(); + verify(gcsChannel).close(); + verifyNoMoreInteractions(gcsChannel); + } + + @Test + public void testSize() throws IOException { + assertThat(chan.size()).isEqualTo(0L); + verify(gcsChannel).isOpen(); + verifyZeroInteractions(gcsChannel); + } + + @Test + public void testSize_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.size(); + } + + @Test + public void testPosition_whenClosed_throwsCce() throws IOException { + when(gcsChannel.isOpen()).thenReturn(false); + thrown.expect(ClosedChannelException.class); + chan.position(); + } + + @Test + public void testClose_calledMultipleTimes_doesntThrowAnError() throws IOException { + chan.close(); + chan.close(); + chan.close(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/FakeStorageRpc.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/FakeStorageRpc.java new file mode 100644 index 000000000000..e7b10677cfcf --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/FakeStorageRpc.java @@ -0,0 +1,369 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file exctain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unept in compliance with the License. + * You may obless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.StorageObject; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageException; +import com.google.cloud.storage.spi.RpcBatch; +import com.google.cloud.storage.spi.StorageRpc; + +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.nio.file.FileAlreadyExistsException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.annotation.concurrent.NotThreadSafe; + +/** + * A bare-bones in-memory implementation of Storage, meant for testing. + * + *

This class is not thread-safe. It's also (currently) limited in the following ways: + *

    + *
  • Supported + *
      + *
    • object create + *
    • object get + *
    • object delete + *
    • list the contents of a bucket + *
    + *
  • Unsupported + *
      + *
    • bucket create + *
    • bucket get + *
    • bucket delete + *
    • list all buckets + *
    • generations + *
    • file attributes + *
    • patch + *
    • continueRewrite + *
    • createBatch + *
    • checksums, etags + *
    + *
+ */ +@NotThreadSafe +class FakeStorageRpc implements StorageRpc { + + // fullname -> metadata + Map metadata = new HashMap<>(); + // fullname -> contents + Map contents = new HashMap<>(); + // fullname -> future contents that will be visible on close. + Map futureContents = new HashMap<>(); + + private final boolean throwIfOption; + + /** + * @param throwIfOption if true, we throw when given any option + */ + public FakeStorageRpc(boolean throwIfOption) { + this.throwIfOption = throwIfOption; + } + + // remove all files + void reset() { + metadata = new HashMap<>(); + contents = new HashMap<>(); + } + + @Override + public Bucket create(Bucket bucket, Map options) throws StorageException { + throw new UnsupportedOperationException(); + } + + @Override + public StorageObject create(StorageObject object, InputStream content, Map options) + throws StorageException { + potentiallyThrow(options); + String key = fullname(object); + metadata.put(key, object); + try { + contents.put(key, com.google.common.io.ByteStreams.toByteArray(content)); + } catch (IOException e) { + throw new StorageException(e); + } + // TODO: crc, etc + return object; + } + + @Override + public Tuple> list(Map options) throws StorageException { + throw new UnsupportedOperationException(); + } + + @Override + public Tuple> list(String bucket, Map options) + throws StorageException { + String delimiter = null; + String preprefix = ""; + for (Map.Entry e : options.entrySet()) { + switch (e.getKey()) { + case PREFIX: + preprefix = (String) e.getValue(); + if (preprefix.startsWith("/")) { + preprefix = preprefix.substring(1); + } + break; + case DELIMITER: + delimiter = (String) e.getValue(); + break; + case FIELDS: + // ignore and return all the fields + break; + default: + throw new UnsupportedOperationException("Unknown option: " + e.getKey()); + } + } + final String prefix = preprefix; + + List values = new ArrayList<>(); + Map folders = new HashMap<>(); + for (StorageObject so : metadata.values()) { + if (!so.getName().startsWith(prefix)) { + continue; + } + if (processedAsFolder(so, delimiter, prefix, folders)) { + continue; + } + values.add(so); + } + values.addAll(folders.values()); + // null cursor to indicate there is no more data (empty string would cause us to be called again). + // The type cast seems to be necessary to help Java's typesystem remember that collections are iterable. + return Tuple.of(null, (Iterable) values); + } + + /** + * Returns the requested bucket or {@code null} if not found. + */ + @Override + public Bucket get(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + /** + * Returns the requested storage object or {@code null} if not found. + */ + @Override + public StorageObject get(StorageObject object, Map options) throws StorageException { + // we allow the "ID" option because we need to, but then we give a whole answer anyways + // because the caller won't mind the extra fields. + if (throwIfOption && !options.isEmpty() && options.size() > 1 + && options.keySet().toArray()[0] != Storage.BlobGetOption.fields(Storage.BlobField.ID)) { + throw new UnsupportedOperationException(); + } + + String key = fullname(object); + if (metadata.containsKey(key)) { + StorageObject ret = metadata.get(key); + if (contents.containsKey(key)) { + ret.setSize(BigInteger.valueOf(contents.get(key).length)); + } + ret.setId(key); + return ret; + } + return null; + } + + @Override + public Bucket patch(Bucket bucket, Map options) throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public StorageObject patch(StorageObject storageObject, Map options) + throws StorageException { + potentiallyThrow(options); + return null; + } + + @Override + public boolean delete(Bucket bucket, Map options) throws StorageException { + return false; + } + + @Override + public boolean delete(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + contents.remove(key); + return null != metadata.remove(key); + } + + @Override + public RpcBatch createBatch() { + //return new DefaultRpcBatch(storage); + throw new UnsupportedOperationException(); + } + + @Override + public StorageObject compose(Iterable sources, StorageObject target, + Map targetOptions) throws StorageException { + return null; + } + + @Override + public byte[] load(StorageObject storageObject, Map options) throws StorageException { + String key = fullname(storageObject); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + return contents.get(key); + } + + @Override + public Tuple read( + StorageObject from, Map options, long zposition, int zbytes) + throws StorageException { + potentiallyThrow(options); + String key = fullname(from); + if (!contents.containsKey(key)) { + throw new StorageException(404, "File not found: " + key); + } + long position = zposition; + int bytes = zbytes; + if (position < 0) { + position = 0; + } + byte[] full = contents.get(key); + if ((int) position + bytes > full.length) { + bytes = full.length - (int) position; + } + if (bytes <= 0) { + // special case: you're trying to read past the end + return Tuple.of("etag-goes-here", new byte[0]); + } + byte[] ret = new byte[bytes]; + System.arraycopy(full, (int) position, ret, 0, bytes); + return Tuple.of("etag-goes-here", ret); + } + + @Override + public String open(StorageObject object, Map options) throws StorageException { + String key = fullname(object); + boolean mustNotExist = false; + for (Option option : options.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH && ((Long) options.get(option)) == 0L) { + mustNotExist = true; + } + } + if (mustNotExist && metadata.containsKey(key)) { + throw new StorageException(new FileAlreadyExistsException(key)); + } + metadata.put(key, object); + + return fullname(object); + } + + @Override + public void write(String uploadId, byte[] toWrite, int toWriteOffset, long destOffset, + int length, boolean last) throws StorageException { + // this may have a lot more allocations than ideal, but it'll work. + byte[] bytes; + if (futureContents.containsKey(uploadId)) { + bytes = futureContents.get(uploadId); + if (bytes.length < length + destOffset) { + bytes = new byte[(int) (length + destOffset)]; + } + } else { + bytes = new byte[(int) (length + destOffset)]; + } + System.arraycopy(toWrite, toWriteOffset, bytes, (int) destOffset, length); + // we want to mimic the GCS behavior that file contents are only visible on close. + if (last) { + contents.put(uploadId, bytes); + futureContents.remove(uploadId); + } else { + futureContents.put(uploadId, bytes); + } + } + + @Override + public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException { + String sourceKey = fullname(rewriteRequest.source); + // a little hackish, just good enough for the tests to work. + if (!contents.containsKey(sourceKey)) { + throw new StorageException(404, "File not found: " + sourceKey); + } + + boolean mustNotExist = false; + for (Option option : rewriteRequest.targetOptions.keySet()) { + // this is a bit of a hack, since we don't implement generations. + if (option == Option.IF_GENERATION_MATCH + && (Long) rewriteRequest.targetOptions.get(option) == 0L) { + mustNotExist = true; + } + } + + String destKey = fullname(rewriteRequest.target); + if (mustNotExist && contents.containsKey(destKey)) { + throw new StorageException(new FileAlreadyExistsException(destKey)); + } + + metadata.put(destKey, rewriteRequest.target); + + byte[] data = contents.get(sourceKey); + contents.put(destKey, Arrays.copyOf(data, data.length)); + return new RewriteResponse(rewriteRequest, rewriteRequest.target, data.length, true, + "rewriteToken goes here", data.length); + } + + @Override + public RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException { + throw new UnsupportedOperationException(); + } + + private String fullname(StorageObject so) { + return (so.getBucket() + "/" + so.getName()); + } + + private void potentiallyThrow(Map options) throws UnsupportedOperationException { + if (throwIfOption && !options.isEmpty()) { + throw new UnsupportedOperationException(); + } + } + + // Returns true if this is a folder. Adds it to folders if it isn't already there. + private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map folders) { + if (delimiter == null) { + return false; + } + int nextSlash = so.getName().indexOf(delimiter, prefix.length()); + if (nextSlash < 0) { + return false; + } + String folderName = so.getName().substring(0, nextSlash + 1); + if (folders.containsKey(folderName)) { + return true; + } + StorageObject fakeFolder = new StorageObject(); + fakeFolder.setName(folderName); + fakeFolder.setBucket(so.getBucket()); + fakeFolder.setGeneration(so.getGeneration()); + folders.put(folderName, fakeFolder); + return true; + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/LocalStorageHelper.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/LocalStorageHelper.java new file mode 100644 index 000000000000..a27a0df3a43a --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/LocalStorageHelper.java @@ -0,0 +1,66 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import com.google.cloud.spi.ServiceRpcFactory; +import com.google.cloud.storage.spi.StorageRpc; +import com.google.cloud.storage.StorageOptions; + +/** + * Utility to create an in-memory storage configuration for testing. Storage options can be + * obtained via the {@link #options()} method. Returned options will point to FakeStorageRpc. + */ +class LocalStorageHelper { + + // used for testing. Will throw if you pass it an option. + private static final FakeStorageRpc instance = new FakeStorageRpc(true); + + /** + * Returns a {@link StorageOptions} that use the static FakeStorageRpc instance, and resets it + * first so you start from a clean slate. That instance will throw if you pass it any option. + */ + public static StorageOptions options() { + instance.reset(); + return StorageOptions.builder() + .projectId("dummy-project-for-testing") + .serviceRpcFactory( + new ServiceRpcFactory() { + @Override + public StorageRpc create(StorageOptions options) { + return instance; + } + }) + .build(); + } + + /** + * Returns a {@link StorageOptions} that creates a new FakeStorageRpc instance with the given + * option. + */ + public static StorageOptions customOptions(final boolean throwIfOptions) { + return StorageOptions.builder() + .projectId("dummy-project-for-testing") + .serviceRpcFactory( + new ServiceRpcFactory() { + @Override + public StorageRpc create(StorageOptions options) { + return new FakeStorageRpc(throwIfOptions); + } + }) + .build(); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/UnixPathTest.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/UnixPathTest.java new file mode 100644 index 000000000000..7bdf5b9cc248 --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/UnixPathTest.java @@ -0,0 +1,402 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import com.google.common.testing.EqualsTester; +import com.google.common.testing.NullPointerTester; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Unit tests for {@link UnixPath}. + */ +@RunWith(JUnit4.class) +public class UnixPathTest { + + @Rule public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void testNormalize() { + assertThat(p(".").normalize()).isEqualTo(p("")); + assertThat(p("/").normalize()).isEqualTo(p("/")); + assertThat(p("/.").normalize()).isEqualTo(p("/")); + assertThat(p("/a/b/../c").normalize()).isEqualTo(p("/a/c")); + assertThat(p("/a/b/./c").normalize()).isEqualTo(p("/a/b/c")); + assertThat(p("a/b/../c").normalize()).isEqualTo(p("a/c")); + assertThat(p("a/b/./c").normalize()).isEqualTo(p("a/b/c")); + assertThat(p("/a/b/../../c").normalize()).isEqualTo(p("/c")); + assertThat(p("/a/b/./.././.././c").normalize()).isEqualTo(p("/c")); + } + + @Test + public void testNormalize_empty_returnsEmpty() { + assertThat(p("").normalize()).isEqualTo(p("")); + } + + @Test + public void testNormalize_underflow_isAllowed() { + assertThat(p("../").normalize()).isEqualTo(p("")); + } + + @Test + public void testNormalize_extraSlashes_getRemoved() { + assertThat(p("///").normalize()).isEqualTo(p("/")); + assertThat(p("/hi//there").normalize()).isEqualTo(p("/hi/there")); + assertThat(p("/hi////.///there").normalize()).isEqualTo(p("/hi/there")); + } + + @Test + public void testNormalize_trailingSlash() { + assertThat(p("/hi/there/").normalize()).isEqualTo(p("/hi/there/")); + assertThat(p("/hi/there/../").normalize()).isEqualTo(p("/hi/")); + assertThat(p("/hi/there/..").normalize()).isEqualTo(p("/hi/")); + assertThat(p("hi/../").normalize()).isEqualTo(p("")); + assertThat(p("/hi/../").normalize()).isEqualTo(p("/")); + assertThat(p("hi/..").normalize()).isEqualTo(p("")); + assertThat(p("/hi/..").normalize()).isEqualTo(p("/")); + } + + @Test + public void testNormalize_sameObjectOptimization() { + UnixPath path = p("/hi/there"); + assertThat(path.normalize()).isSameAs(path); + path = p("/hi/there/"); + assertThat(path.normalize()).isSameAs(path); + } + + @Test + public void testResolve() { + assertThat(p("/hello").resolve(p("cat"))).isEqualTo(p("/hello/cat")); + assertThat(p("/hello/").resolve(p("cat"))).isEqualTo(p("/hello/cat")); + assertThat(p("hello/").resolve(p("cat"))).isEqualTo(p("hello/cat")); + assertThat(p("hello/").resolve(p("cat/"))).isEqualTo(p("hello/cat/")); + assertThat(p("hello/").resolve(p(""))).isEqualTo(p("hello/")); + assertThat(p("hello/").resolve(p("/hi/there"))).isEqualTo(p("/hi/there")); + } + + @Test + public void testResolve_sameObjectOptimization() { + UnixPath path = p("/hi/there"); + assertThat(path.resolve(p(""))).isSameAs(path); + assertThat(p("hello").resolve(path)).isSameAs(path); + } + + @Test + public void testGetPath() { + assertThat(UnixPath.getPath(false, "hello")).isEqualTo(p("hello")); + assertThat(UnixPath.getPath(false, "hello", "cat")).isEqualTo(p("hello/cat")); + assertThat(UnixPath.getPath(false, "/hello", "cat")).isEqualTo(p("/hello/cat")); + assertThat(UnixPath.getPath(false, "/hello", "cat", "inc.")).isEqualTo(p("/hello/cat/inc.")); + assertThat(UnixPath.getPath(false, "hello/", "/hi/there")).isEqualTo(p("/hi/there")); + } + + @Test + public void testResolveSibling() { + assertThat(p("/hello/cat").resolveSibling(p("dog"))).isEqualTo(p("/hello/dog")); + assertThat(p("/").resolveSibling(p("dog"))).isEqualTo(p("dog")); + } + + @Test + public void testResolveSibling_preservesTrailingSlash() { + assertThat(p("/hello/cat").resolveSibling(p("dog/"))).isEqualTo(p("/hello/dog/")); + assertThat(p("/").resolveSibling(p("dog/"))).isEqualTo(p("dog/")); + } + + @Test + public void testRelativize() { + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/mop/top"))) + .isEqualTo(p("../../../mop/top")); + assertThat(p("/foo/bar/dog").relativize(p("/foo/mop/top"))).isEqualTo(p("../../mop/top")); + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/mop/top/../../mog"))) + .isEqualTo(p("../../../mop/top/../../mog")); + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/../mog"))).isEqualTo(p("../../../../mog")); + assertThat(p("").relativize(p("foo/mop/top/"))).isEqualTo(p("foo/mop/top/")); + } + + @Test + public void testRelativize_absoluteMismatch_notAllowed() { + thrown.expect(IllegalArgumentException.class); + p("/a/b/").relativize(p("")); + } + + @Test + public void testRelativize_preservesTrailingSlash() { + // This behavior actually diverges from sun.nio.fs.UnixPath: + // bsh % print(Paths.get("/a/b/").relativize(Paths.get("/etc/"))); + // ../../etc + assertThat(p("/foo/bar/hop/dog").relativize(p("/foo/../mog/"))) + .isEqualTo(p("../../../../mog/")); + assertThat(p("/a/b/").relativize(p("/etc/"))).isEqualTo(p("../../etc/")); + } + + @Test + public void testStartsWith() { + assertThat(p("/hi/there").startsWith(p("/hi/there"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("/hi/therf"))).isFalse(); + assertThat(p("/hi/there").startsWith(p("/hi"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("/hi/"))).isTrue(); + assertThat(p("/hi/there").startsWith(p("hi"))).isFalse(); + assertThat(p("/hi/there").startsWith(p("/"))).isTrue(); + assertThat(p("/hi/there").startsWith(p(""))).isFalse(); + assertThat(p("/a/b").startsWith(p("a/b/"))).isFalse(); + assertThat(p("/a/b/").startsWith(p("a/b/"))).isFalse(); + assertThat(p("/hi/there").startsWith(p(""))).isFalse(); + assertThat(p("").startsWith(p(""))).isTrue(); + } + + @Test + public void testStartsWith_comparesComponentsIndividually() { + assertThat(p("/hello").startsWith(p("/hell"))).isFalse(); + assertThat(p("/hello").startsWith(p("/hello"))).isTrue(); + } + + @Test + public void testEndsWith() { + assertThat(p("/hi/there").endsWith(p("there"))).isTrue(); + assertThat(p("/hi/there").endsWith(p("therf"))).isFalse(); + assertThat(p("/hi/there").endsWith(p("/blag/therf"))).isFalse(); + assertThat(p("/hi/there").endsWith(p("/hi/there"))).isTrue(); + assertThat(p("/hi/there").endsWith(p("/there"))).isFalse(); + assertThat(p("/human/that/you/cry").endsWith(p("that/you/cry"))).isTrue(); + assertThat(p("/human/that/you/cry").endsWith(p("that/you/cry/"))).isTrue(); + assertThat(p("/hi/there/").endsWith(p("/"))).isFalse(); + assertThat(p("/hi/there").endsWith(p(""))).isFalse(); + assertThat(p("").endsWith(p(""))).isTrue(); + } + + @Test + public void testEndsWith_comparesComponentsIndividually() { + assertThat(p("/hello").endsWith(p("lo"))).isFalse(); + assertThat(p("/hello").endsWith(p("hello"))).isTrue(); + } + + @Test + public void testGetParent() { + assertThat(p("").getParent()).isNull(); + assertThat(p("/").getParent()).isNull(); + assertThat(p("aaa/").getParent()).isNull(); + assertThat(p("aaa").getParent()).isNull(); + assertThat(p("/aaa/").getParent()).isEqualTo(p("/")); + assertThat(p("a/b/c").getParent()).isEqualTo(p("a/b/")); + assertThat(p("a/b/c/").getParent()).isEqualTo(p("a/b/")); + assertThat(p("a/b/").getParent()).isEqualTo(p("a/")); + } + + @Test + public void testGetRoot() { + assertThat(p("/hello").getRoot()).isEqualTo(p("/")); + assertThat(p("hello").getRoot()).isNull(); + } + + @Test + public void testGetFileName() { + assertThat(p("").getFileName()).isEqualTo(p("")); + assertThat(p("/").getFileName()).isNull(); + assertThat(p("/dark").getFileName()).isEqualTo(p("dark")); + assertThat(p("/angels/").getFileName()).isEqualTo(p("angels")); + } + + @Test + public void testEquals() { + assertThat(p("/a/").equals(p("/a/"))).isTrue(); + assertThat(p("/a/").equals(p("/b/"))).isFalse(); + assertThat(p("/b/").equals(p("/b"))).isFalse(); + assertThat(p("/b").equals(p("/b/"))).isFalse(); + assertThat(p("b").equals(p("/b"))).isFalse(); + assertThat(p("b").equals(p("b"))).isTrue(); + } + + @Test + public void testSplit() { + assertThat(p("").split().hasNext()).isFalse(); + assertThat(p("hi/there").split().hasNext()).isTrue(); + assertThat(p(p("hi/there").split().next())).isEqualTo(p("hi")); + } + + @Test + public void testToAbsolute() { + assertThat(p("lol").toAbsolutePath(UnixPath.ROOT_PATH)).isEqualTo(p("/lol")); + assertThat(p("lol/cat").toAbsolutePath(UnixPath.ROOT_PATH)).isEqualTo(p("/lol/cat")); + } + + @Test + public void testToAbsolute_withCurrentDirectory() { + assertThat(p("cat").toAbsolutePath(p("/lol"))).isEqualTo(p("/lol/cat")); + assertThat(p("cat").toAbsolutePath(p("/lol/"))).isEqualTo(p("/lol/cat")); + assertThat(p("/hi/there").toAbsolutePath(p("/lol"))).isEqualTo(p("/hi/there")); + } + + @Test + public void testToAbsolute_preservesTrailingSlash() { + assertThat(p("cat/").toAbsolutePath(p("/lol"))).isEqualTo(p("/lol/cat/")); + } + + @Test + public void testSubpath() { + assertThat(p("/eins/zwei/drei/vier").subpath(0, 1)).isEqualTo(p("eins")); + assertThat(p("/eins/zwei/drei/vier").subpath(0, 2)).isEqualTo(p("eins/zwei")); + assertThat(p("eins/zwei/drei/vier/").subpath(1, 4)).isEqualTo(p("zwei/drei/vier")); + assertThat(p("eins/zwei/drei/vier/").subpath(2, 4)).isEqualTo(p("drei/vier")); + } + + @Test + public void testSubpath_empty_returnsEmpty() { + assertThat(p("").subpath(0, 1)).isEqualTo(p("")); + } + + @Test + public void testSubpath_root_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/").subpath(0, 1); + } + + @Test + public void testSubpath_negativeIndex_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(-1, 1); + } + + @Test + public void testSubpath_notEnoughElements_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(0, 5); + } + + @Test + public void testSubpath_beginAboveEnd_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(1, 0); + } + + @Test + public void testSubpath_beginAndEndEqual_throwsIae() { + thrown.expect(IllegalArgumentException.class); + p("/eins/zwei/drei/vier").subpath(0, 0); + } + + @Test + public void testNameCount() { + assertThat(p("").getNameCount()).isEqualTo(1); + assertThat(p("/").getNameCount()).isEqualTo(0); + assertThat(p("/hi/").getNameCount()).isEqualTo(1); + assertThat(p("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(p("hi/yo").getNameCount()).isEqualTo(2); + } + + @Test + public void testNameCount_dontPermitEmptyComponents_emptiesGetIgnored() { + assertThat(p("hi//yo").getNameCount()).isEqualTo(2); + assertThat(p("//hi//yo//").getNameCount()).isEqualTo(2); + } + + @Test + public void testNameCount_permitEmptyComponents_emptiesGetCounted() { + assertThat(pp("hi//yo").getNameCount()).isEqualTo(3); + assertThat(pp("hi//yo/").getNameCount()).isEqualTo(4); + assertThat(pp("hi//yo//").getNameCount()).isEqualTo(5); + } + + @Test + public void testNameCount_permitEmptyComponents_rootComponentDoesntCount() { + assertThat(pp("hi/yo").getNameCount()).isEqualTo(2); + assertThat(pp("/hi/yo").getNameCount()).isEqualTo(2); + assertThat(pp("//hi/yo").getNameCount()).isEqualTo(3); + } + + @Test + public void testGetName() { + assertThat(p("").getName(0)).isEqualTo(p("")); + assertThat(p("/hi").getName(0)).isEqualTo(p("hi")); + assertThat(p("hi/there").getName(1)).isEqualTo(p("there")); + } + + @Test + public void testCompareTo() { + assertThat(p("/hi/there").compareTo(p("/hi/there"))).isEqualTo(0); + assertThat(p("/hi/there").compareTo(p("/hi/therf"))).isEqualTo(-1); + assertThat(p("/hi/there").compareTo(p("/hi/therd"))).isEqualTo(1); + } + + @Test + public void testCompareTo_dontPermitEmptyComponents_emptiesGetIgnored() { + assertThat(p("a/b").compareTo(p("a//b"))).isEqualTo(0); + } + + @Test + public void testCompareTo_permitEmptyComponents_behaviorChanges() { + assertThat(p("a/b").compareTo(pp("a//b"))).isEqualTo(1); + assertThat(pp("a/b").compareTo(pp("a//b"))).isEqualTo(1); + } + + @Test + public void testCompareTo_comparesComponentsIndividually() { + assumeTrue('.' < '/'); + assertThat("hi./there".compareTo("hi/there")).isEqualTo(-1); + assertThat("hi.".compareTo("hi")).isEqualTo(1); + assertThat(p("hi./there").compareTo(p("hi/there"))).isEqualTo(1); + assertThat(p("hi./there").compareTo(p("hi/there"))).isEqualTo(1); + assumeTrue('0' > '/'); + assertThat("hi0/there".compareTo("hi/there")).isEqualTo(1); + assertThat("hi0".compareTo("hi")).isEqualTo(1); + assertThat(p("hi0/there").compareTo(p("hi/there"))).isEqualTo(1); + } + + @Test + public void testSeemsLikeADirectory() { + assertThat(p("a").seemsLikeADirectory()).isFalse(); + assertThat(p("a.").seemsLikeADirectory()).isFalse(); + assertThat(p("a..").seemsLikeADirectory()).isFalse(); + assertThat(p("").seemsLikeADirectory()).isTrue(); + assertThat(p("/").seemsLikeADirectory()).isTrue(); + assertThat(p(".").seemsLikeADirectory()).isTrue(); + assertThat(p("/.").seemsLikeADirectory()).isTrue(); + assertThat(p("..").seemsLikeADirectory()).isTrue(); + assertThat(p("/..").seemsLikeADirectory()).isTrue(); + } + + @Test + public void testEquals_equalsTester() { + new EqualsTester() + .addEqualityGroup(p("/lol"), p("/lol")) + .addEqualityGroup(p("/lol//"), p("/lol//")) + .addEqualityGroup(p("dust")) + .testEquals(); + } + + @Test + public void testNullness() throws Exception { + NullPointerTester tester = new NullPointerTester(); + tester.ignore(UnixPath.class.getMethod("equals", Object.class)); + tester.testAllPublicStaticMethods(UnixPath.class); + tester.testAllPublicInstanceMethods(p("solo")); + } + + private static UnixPath p(String path) { + return UnixPath.getPath(false, path); + } + + private static UnixPath pp(String path) { + return UnixPath.getPath(true, path); + } +} diff --git a/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java new file mode 100644 index 000000000000..dad6fc35ba2d --- /dev/null +++ b/gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java @@ -0,0 +1,389 @@ +/* + * Copyright 2016 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.storage.contrib.nio.it; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration; +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem; +import com.google.common.collect.ImmutableList; +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.BucketInfo; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.cloud.storage.testing.RemoteStorageHelper; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Integration test for gcloud-java-nio. + * + *

This test actually talks to Google Cloud Storage (you need an account) and tests both reading + * and writing. You *must* set the {@code GOOGLE_APPLICATION_CREDENTIALS} environment variable for + * this test to work. It must contain the name of a local file that contains your Service Account + * JSON Key. + * + *

See + * Service Accounts for instructions on how to get the Service Account JSON Key. + * + *

The short version is this: go to cloud.google.com/console, select your project, search for + * "API manager", click "Credentials", click "create credentials/service account key", new service + * account, JSON. The contents of the file that's sent to your browsers is your + * "Service Account JSON Key". + */ +@RunWith(JUnit4.class) +public class ITGcsNio { + + private static final List FILE_CONTENTS = + ImmutableList.of( + "Tous les êtres humains naissent libres et égaux en dignité et en droits.", + "Ils sont doués de raison et de conscience et doivent agir ", + "les uns envers les autres dans un esprit de fraternité."); + + private static final Logger log = Logger.getLogger(ITGcsNio.class.getName()); + private static final String BUCKET = RemoteStorageHelper.generateBucketName(); + private static final String SML_FILE = "tmp-test-small-file.txt"; + private static final int SML_SIZE = 100; + private static final String BIG_FILE = "tmp-test-big-file.txt"; // it's big, relatively speaking. + private static final int BIG_SIZE = 2 * 1024 * 1024 - 50; // arbitrary size that's not too round. + private static final String PREFIX = "tmp-test-file"; + private static Storage storage; + private static StorageOptions storageOptions; + + private final Random rnd = new Random(); + + @BeforeClass + public static void beforeClass() throws IOException { + // loads the credentials from local disk as par README + RemoteStorageHelper gcsHelper = RemoteStorageHelper.create(); + storageOptions = gcsHelper.options(); + storage = storageOptions.service(); + // create and populate test bucket + storage.create(BucketInfo.of(BUCKET)); + fillFile(storage, SML_FILE, SML_SIZE); + fillFile(storage, BIG_FILE, BIG_SIZE); + } + + @AfterClass + public static void afterClass() throws ExecutionException, InterruptedException { + if (storage != null && !RemoteStorageHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS) && + log.isLoggable(Level.WARNING)) { + log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET); + } + } + + private static byte[] randomContents(int size) { + byte[] bytes = new byte[size]; + new Random(size).nextBytes(bytes); + return bytes; + } + + private static void fillFile(Storage storage, String fname, int size) throws IOException { + storage.create(BlobInfo.builder(BUCKET, fname).build(), randomContents(size)); + } + + @Test + public void testFileExists() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + assertThat(Files.exists(path)).isTrue(); + } + + @Test + public void testFileSize() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + assertThat(Files.size(path)).isEqualTo(SML_SIZE); + } + + @Test(timeout = 60_000) + public void testReadByteChannel() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(SML_FILE); + long size = Files.size(path); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + assertThat(chan.size()).isEqualTo(size); + ByteBuffer buf = ByteBuffer.allocate(SML_SIZE); + int read = 0; + while (chan.isOpen()) { + int rc = chan.read(buf); + assertThat(chan.size()).isEqualTo(size); + if (rc < 0) { + // EOF + break; + } + assertThat(rc).isGreaterThan(0); + read += rc; + assertThat(chan.position()).isEqualTo(read); + } + assertThat(read).isEqualTo(size); + byte[] expected = new byte[SML_SIZE]; + new Random(SML_SIZE).nextBytes(expected); + assertThat(Arrays.equals(buf.array(), expected)).isTrue(); + } + + @Test + public void testSeek() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(BIG_FILE); + int size = BIG_SIZE; + byte[] contents = randomContents(size); + byte[] sample = new byte[100]; + byte[] wanted; + byte[] wanted2; + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + assertThat(chan.size()).isEqualTo(size); + + // check seek + int dest = size / 2; + chan.position(dest); + readFully(chan, sample); + wanted = Arrays.copyOfRange(contents, dest, dest + 100); + assertThat(wanted).isEqualTo(sample); + // now go back and check the beginning + // (we do 2 locations because 0 is sometimes a special case). + chan.position(0); + readFully(chan, sample); + wanted2 = Arrays.copyOf(contents, 100); + assertThat(wanted2).isEqualTo(sample); + // if the two spots in the file have the same contents, then this isn't a good file for this + // test. + assertThat(wanted).isNotEqualTo(wanted2); + } + + @Test + public void testCreate() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially. If it does it's either because it's a leftover + // from a previous run (so we should delete the file) + // or because we're misconfigured and pointing to an actually important file + // (so we should absolutely not delete it). + // So if the file's here, don't try to fix it automatically, let the user deal with it. + assertThat(Files.exists(path)).isFalse(); + try { + Files.createFile(path); + // now it does, and it has size 0. + assertThat(Files.exists(path)).isTrue(); + long size = Files.size(path); + assertThat(size).isEqualTo(0); + } finally { + // let's not leave files around + Files.deleteIfExists(path); + } + } + + @Test + public void testWrite() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially. If it does it's either because it's a leftover + // from a previous run (so we should delete the file) + // or because we're misconfigured and pointing to an actually important file + // (so we should absolutely not delete it). + // So if the file's here, don't try to fix it automatically, let the user deal with it. + assertThat(Files.exists(path)).isFalse(); + try { + Files.write(path, FILE_CONTENTS, UTF_8); + // now it does. + assertThat(Files.exists(path)).isTrue(); + + // let's check that the contents is OK. + ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); + for (String content : FILE_CONTENTS) { + writer.println(content); + } + writer.close(); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + byte[] gotBytes = new byte[(int) chan.size()]; + readFully(chan, gotBytes); + assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); + } finally { + // let's not leave files around + Files.deleteIfExists(path); + } + } + + @Test + public void testCreateAndWrite() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially (see above). + assertThat(Files.exists(path)).isFalse(); + try { + Files.createFile(path); + Files.write(path, FILE_CONTENTS, UTF_8); + // now it does. + assertThat(Files.exists(path)).isTrue(); + + // let's check that the contents is OK. + ByteArrayOutputStream wantBytes = new ByteArrayOutputStream(); + PrintWriter writer = new PrintWriter(new OutputStreamWriter(wantBytes, UTF_8)); + for (String content : FILE_CONTENTS) { + writer.println(content); + } + writer.close(); + SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.READ); + byte[] gotBytes = new byte[(int) chan.size()]; + readFully(chan, gotBytes); + assertThat(gotBytes).isEqualTo(wantBytes.toByteArray()); + } finally { + // let's not leave files around + Files.deleteIfExists(path); + } + } + + @Test + public void testWriteOnClose() throws Exception { + CloudStorageFileSystem testBucket = getTestBucket(); + Path path = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially (see above) + assertThat(Files.exists(path)).isFalse(); + try { + long expectedSize = 0; + try (SeekableByteChannel chan = Files.newByteChannel(path, StandardOpenOption.WRITE)) { + // writing lots of contents to defeat channel-internal buffering. + for (String s : FILE_CONTENTS) { + byte[] sBytes = s.getBytes(UTF_8); + expectedSize += sBytes.length * 9999; + for (int i = 0; i < 9999; i++) { + chan.write(ByteBuffer.wrap(sBytes)); + } + } + try { + Files.size(path); + // we shouldn't make it to this line. Not using thrown.expect because + // I still want to run a few lines after the exception. + Assert.fail("Files.size should have thrown an exception"); + } catch (NoSuchFileException nsf) { + // that's what we wanted, we're good. + } + } + // channel now closed, the file should be there and with the new contents. + assertThat(Files.exists(path)).isTrue(); + assertThat(Files.size(path)).isEqualTo(expectedSize); + } finally { + Files.deleteIfExists(path); + } + } + + @Test + public void testCopy() throws IOException { + CloudStorageFileSystem testBucket = getTestBucket(); + Path src = testBucket.getPath(SML_FILE); + Path dst = testBucket.getPath(PREFIX + randomSuffix()); + // file shouldn't exist initially (see above). + assertThat(Files.exists(dst)).isFalse(); + try { + Files.copy(src, dst); + + assertThat(Files.exists(dst)).isTrue(); + assertThat(Files.size(dst)).isEqualTo(SML_SIZE); + byte[] got = new byte[SML_SIZE]; + readFully(Files.newByteChannel(dst), got); + assertThat(got).isEqualTo(randomContents(SML_SIZE)); + } finally { + // let's not leave files around + Files.deleteIfExists(dst); + } + } + + @Test + public void testListFiles() throws IOException { + try (FileSystem fs = getTestBucket()) { + List goodPaths = new ArrayList<>(); + List paths = new ArrayList<>(); + goodPaths.add(fs.getPath("dir/angel")); + goodPaths.add(fs.getPath("dir/alone")); + paths.add(fs.getPath("dir/dir2/another_angel")); + paths.add(fs.getPath("atroot")); + paths.addAll(goodPaths); + goodPaths.add(fs.getPath("dir/dir2/")); + for (Path path : paths) { + fillFile(storage, path.toString(), SML_SIZE); + } + + List got = new ArrayList<>(); + for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) { + got.add(path); + } + assertThat(got).containsExactlyElementsIn(goodPaths); + } + } + + private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException { + ByteBuffer buf = ByteBuffer.wrap(outputBuf); + int sofar = 0; + int bytes = buf.remaining(); + while (sofar < bytes) { + int read = chan.read(buf); + if (read < 0) { + throw new EOFException("channel EOF"); + } + sofar += read; + } + return sofar; + } + + private String randomSuffix() { + return "-" + rnd.nextInt(99999); + } + + + private CloudStorageFileSystem getTestBucket() throws IOException { + // in typical usage we use the single-argument version of forBucket + // and rely on the user being logged into their project with the + // gcloud tool, and then everything authenticates automagically + // (or we just use paths that start with "gs://" and rely on NIO's magic). + // + // However for the tests we want to be able to run in automated environments + // where we can set environment variables but not necessarily install gcloud + // or run it. That's why we're setting the credentials programmatically. + return CloudStorageFileSystem.forBucket( + BUCKET, CloudStorageConfiguration.DEFAULT, storageOptions); + } + +} diff --git a/gcloud-java-contrib/pom.xml b/gcloud-java-contrib/pom.xml index 61a43bd070c3..37ddb20e708d 100644 --- a/gcloud-java-contrib/pom.xml +++ b/gcloud-java-contrib/pom.xml @@ -2,7 +2,7 @@ 4.0.0 gcloud-java-contrib - jar + pom GCloud Java contributions https://github.com/GoogleCloudPlatform/gcloud-java/tree/master/gcloud-java-contrib @@ -16,6 +16,10 @@ gcloud-java-contrib + + gcloud-java-nio + gcloud-java-nio-examples + ${project.groupId} diff --git a/gcloud-java-core/src/main/java/com/google/cloud/ReadChannel.java b/gcloud-java-core/src/main/java/com/google/cloud/ReadChannel.java index 2afb8b2d5b32..1ac45902b27d 100644 --- a/gcloud-java-core/src/main/java/com/google/cloud/ReadChannel.java +++ b/gcloud-java-core/src/main/java/com/google/cloud/ReadChannel.java @@ -38,7 +38,7 @@ public interface ReadChannel extends ReadableByteChannel, Closeable, Restorable< @Override void close(); - void seek(int position) throws IOException; + void seek(long position) throws IOException; /** * Sets the minimum size that will be read by a single RPC. diff --git a/gcloud-java-examples/README.md b/gcloud-java-examples/README.md index 75947a6dcca3..0a09b14f31e5 100644 --- a/gcloud-java-examples/README.md +++ b/gcloud-java-examples/README.md @@ -9,8 +9,8 @@ Examples for gcloud-java (Java idiomatic client for [Google Cloud Platform][clou [![Codacy Badge](https://api.codacy.com/project/badge/grade/9da006ad7c3a4fe1abd142e77c003917)](https://www.codacy.com/app/mziccard/gcloud-java) [![Dependency Status](https://www.versioneye.com/user/projects/56bd8ee72a29ed002d2b0969/badge.svg?style=flat)](https://www.versioneye.com/user/projects/56bd8ee72a29ed002d2b0969) -- [Homepage] (https://googlecloudplatform.github.io/gcloud-java/) -- [Examples] (http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/cloud/examples/package-summary.html) +- [Homepage](https://googlecloudplatform.github.io/gcloud-java/) +- [Examples](http://googlecloudplatform.github.io/gcloud-java/apidocs/index.html?com/google/cloud/examples/package-summary.html) Quickstart ---------- @@ -33,21 +33,22 @@ libraryDependencies += "com.google.cloud" % "gcloud-java-examples" % "0.2.4" To run examples from your command line: -1. Login using gcloud SDK (`gcloud auth login` in command line) +1. Log in using gcloud SDK (`gcloud auth login` in command line) 2. Set your current project using `gcloud config set project PROJECT_ID`. This step is not necessary for `ResourceManagerExample`. -3. Compile using Maven (`mvn compile` in command line from your base project directory) +3. Compile using Maven: `mvn install -DskipTests` in command line from your base project directory + then `cd gcloud-java-examples` and finally `mvn package appassembler:assemble -DskipTests`. -4. Run an example using Maven from command line. +4. Run an example from the command line using the Maven-generated scripts. * Here's an example run of `BigQueryExample`. Before running the example, go to the [Google Developers Console][developers-console] to ensure that BigQuery API is enabled. You can upload a CSV file `my_csv_file` to the `my_bucket` bucket - (replace `my_csv_file` and `my_bucket` with actual file and bucket names) using the GCS - [web browser](https://console.developers.google.com/storage/browser). The CSV file will be used to - load data into a BigQuery table and should look something like: + (replace `my_csv_file` and `my_bucket` with actual file and bucket names) using the Google Cloud + Storage [web browser](https://console.developers.google.com/storage/browser). The CSV file will + be used to load data into a BigQuery table and should look something like: ```csv value1 value2 @@ -55,11 +56,11 @@ To run examples from your command line: ``` Then you are ready to run the following example: ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.bigquery.BigQueryExample" -Dexec.args="create dataset new_dataset_id" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.bigquery.BigQueryExample" -Dexec.args="create table new_dataset_id new_table_id field_name:string" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.bigquery.BigQueryExample" -Dexec.args="list tables new_dataset_id" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.bigquery.BigQueryExample" -Dexec.args="load new_dataset_id new_table_id CSV gs://my_bucket/my_csv_file" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.bigquery.BigQueryExample" -Dexec.args="query 'select * from new_dataset_id.new_table_id'" + target/appassembler/bin/BigQueryExample create dataset new_dataset_id + target/appassembler/bin/BigQueryExample create table new_dataset_id new_table_id field_name:string + target/appassembler/bin/BigQueryExample list tables new_dataset_id + target/appassembler/bin/BigQueryExample load new_dataset_id new_table_id CSV gs://my_bucket/my_csv_file + target/appassembler/bin/BigQueryExample query 'select * from new_dataset_id.new_table_id' ``` * Here's an example run of `ComputeExample`. @@ -67,21 +68,21 @@ To run examples from your command line: Before running the example, go to the [Google Developers Console][developers-console] to ensure that Compute API is enabled. ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.compute.ComputeExample" -Dexec.args="create image-disk us-central1-a test-disk debian-cloud debian-8-jessie-v20160329" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.compute.ComputeExample" -Dexec.args="create instance us-central1-a test-instance n1-standard-1 test-disk default" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.compute.ComputeExample" -Dexec.args="add-access-config us-central1-a test-instance nic0 NAT" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.compute.ComputeExample" -Dexec.args="delete instance us-central1-a test-instance" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.compute.ComputeExample" -Dexec.args="delete disk us-central1-a test-disk" + target/appassembler/bin/ComputeExample create image-disk us-central1-a test-disk debian-cloud debian-8-jessie-v20160329 + target/appassembler/bin/ComputeExample create instance us-central1-a test-instance n1-standard-1 test-disk default + target/appassembler/bin/ComputeExample add-access-config us-central1-a test-instance nic0 NAT + target/appassembler/bin/ComputeExample delete instance us-central1-a test-instance + target/appassembler/bin/ComputeExample delete disk us-central1-a test-disk ``` * Here's an example run of `DatastoreExample`. Be sure to change the placeholder project ID "your-project-id" with your own project ID. Also note that you have to enable the Google Cloud Datastore API on the [Google Developers Console][developers-console] before running the following commands. ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name add my\ comment" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name display" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name delete" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.datastore.DatastoreExample" -Dexec.args="your-project-id my_name set myname@mydomain.com 1234" + target/appassembler/bin/DatastoreExample your-project-id my_name add my\ comment + target/appassembler/bin/DatastoreExample your-project-id my_name display + target/appassembler/bin/DatastoreExample your-project-id my_name delete + target/appassembler/bin/DatastoreExample your-project-id my_name set myname@mydomain.com 1234 ``` * Here's an example run of `DnsExample`. @@ -90,45 +91,63 @@ To run examples from your command line: You will need to replace the domain name `elaborateexample.com` with your own domain name with [verified ownership] (https://www.google.com/webmasters/verification/home). Also, note that the example creates and deletes record sets of type A only. Operations with other record types are not implemented in the example. ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="create some-sample-zone elaborateexample.com. description" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="list" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="list some-sample-zone records" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="add-record some-sample-zone www.elaborateexample.com. 12.13.14.15 69" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="get some-sample-zone" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="delete-record some-sample-zone www.elaborateexample.com. 12.13.14.15 69" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="list some-sample-zone changes ascending" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample" -Dexec.args="delete some-sample-zone" + target/appassembler/bin/DnsExample create some-sample-zone elaborateexample.com. description + target/appassembler/bin/DnsExample list + target/appassembler/bin/DnsExample list some-sample-zone records + target/appassembler/bin/DnsExample add-record some-sample-zone www.elaborateexample.com. 12.13.14.15 69 + target/appassembler/bin/DnsExample get some-sample-zone + target/appassembler/bin/DnsExample delete-record some-sample-zone www.elaborateexample.com. 12.13.14.15 69 + target/appassembler/bin/DnsExample list some-sample-zone changes ascending + target/appassembler/bin/DnsExample delete some-sample-zone ``` * Here's an example run of `PubSubExample`. Before running the example, go to the [Google Developers Console][developers-console] to ensure that "Google Cloud Pub/Sub" is enabled. ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.pubsub.PubSubExample" -Dexec.args="create topic test-topic" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.pubsub.PubSubExample" -Dexec.args="create subscription test-topic test-subscription" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.pubsub.PubSubExample" -Dexec.args="publish test-topic message1 message2" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.pubsub.PubSubExample" -Dexec.args="pull sync test-subscription 2" + target/appassembler/bin/PubSubExample create topic test-topic + target/appassembler/bin/PubSubExample create subscription test-topic test-subscription + target/appassembler/bin/PubSubExample publish test-topic message1 message2 + target/appassembler/bin/PubSubExample pull sync test-subscription 2 ``` * Here's an example run of `ResourceManagerExample`. Be sure to change the placeholder project ID "your-project-id" with your own globally unique project ID. ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.resourcemanager.ResourceManagerExample" -Dexec.args="create your-project-id" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.resourcemanager.ResourceManagerExample" -Dexec.args="list" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.resourcemanager.ResourceManagerExample" -Dexec.args="get your-project-id" + target/appassembler/bin/ResourceManagerExample create your-project-id + target/appassembler/bin/ResourceManagerExample list + target/appassembler/bin/ResourceManagerExample get your-project-id ``` * Here's an example run of `StorageExample`. Before running the example, go to the [Google Developers Console][developers-console] to ensure that "Google Cloud Storage" and "Google Cloud Storage JSON API" are enabled and that you have a bucket. Also ensure that you have a test file (`test.txt` is chosen here) to upload to Cloud Storage stored locally on your machine. ``` - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.storage.StorageExample" -Dexec.args="upload /path/to/test.txt " - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.storage.StorageExample" -Dexec.args="list " - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.storage.StorageExample" -Dexec.args="download test.txt" - mvn exec:java -Dexec.mainClass="com.google.cloud.examples.storage.StorageExample" -Dexec.args="delete test.txt" + target/appassembler/bin/StorageExample upload /path/to/test.txt + target/appassembler/bin/StorageExample list + target/appassembler/bin/StorageExample download test.txt + target/appassembler/bin/StorageExample delete test.txt ``` + * Here's an example run of `Stat`, illustrating the use of gcloud-java-nio. + + Before running the example, go to the [Google Developers Console][developers-console] to ensure that Google Cloud Storage API is enabled and that you have a bucket with a file in it. + + Run the sample with (from the gcloud-java-examples folder): + ``` + target/appassembler/bin/Stat --check + + ``` + Or, if you have a file in `gs://mybucket/myfile.txt`, you can run: + ``` + target/appassembler/bin/Stat gs://mybucket/myfile.txt + ``` + + The sample doesn't have anything special about Google Cloud Storage in it, it just opens files + via the NIO API. It lists gcloud-java-nio as a dependency, and that enables it to interpret + `gs://` paths. + Troubleshooting --------------- @@ -142,7 +161,7 @@ Java 7 or above is required for using this client. Versioning ---------- -This library follows [Semantic Versioning] (http://semver.org/). +This library follows [Semantic Versioning](http://semver.org/). It is currently in major version zero (``0.y.z``), which means that anything may change at any time and the public API should not be considered diff --git a/gcloud-java-examples/pom.xml b/gcloud-java-examples/pom.xml index 4b6885d5d057..66920628bfbe 100644 --- a/gcloud-java-examples/pom.xml +++ b/gcloud-java-examples/pom.xml @@ -22,6 +22,16 @@ gcloud-java ${project.version} + + ${project.groupId} + gcloud-java-nio + ${project.version} + + + org.apache.maven.plugins + maven-assembly-plugin + 2.5.4 + @@ -33,6 +43,62 @@ false + + + org.codehaus.mojo + appassembler-maven-plugin + 1.10 + + + + com.google.cloud.examples.bigquery.BigQueryExample + BigQueryExample + + + com.google.cloud.examples.compute.ComputeExample + ComputeExample + + + com.google.cloud.examples.datastore.DatastoreExample + DatastoreExample + + + com.google.cloud.examples.nio.Stat + Stat + + + com.google.cloud.examples.dns.DnsExample + DnsExample + + + com.google.cloud.examples.nio.CountBytes + CountBytes + + + com.google.cloud.examples.nio.ParallelCountBytes + ParallelCountBytes + + + + com.google.cloud.examples.pubsub.PubSubExample + + PubSubExample + + + + com.google.cloud.examples.resourcemanager.ResourceManagerExample + + ResourceManagerExample + + + com.google.cloud.examples.storage.StorageExample + StorageExample + + + + diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java index 3ac1335cca7a..d7c1c1ef0b1e 100644 --- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java +++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/bigquery/BigQueryExample.java @@ -58,34 +58,29 @@ * *

This example demonstrates a simple/typical BigQuery usage. * - *

Steps needed for running the example: - *

    - *
  1. login using gcloud SDK - {@code gcloud auth login}.
    2. - *
  2. compile using maven - {@code mvn compile}
    3. - *
  3. run using maven - - * 
    {@code mvn exec:java -Dexec.mainClass="com.google.cloud.examples.bigquery.BigQueryExample"
- *  -Dexec.args="[]
- *  list datasets |
- *  list tables  |
- *  list jobs |
- *  list data   |
- *  info dataset  |
- *  info table  
     |
- *  info job  |
- *  create dataset  |
- *  create table  
     (:)+ |
- *  create view  
     |
- *  create external-table  
     
     (:)+  |
- *  delete dataset  |
- *  delete table  
     
     |
- *  cancel  |
- *  copy     |
- *  load  
     + |
- *  extract  
     
     + |
- *  query  |
- *  load-file  
     
     "}
- * 
- * 
+ * 
    See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
    {@code target/appassembler/bin/BigQueryExample []
+ * list datasets |
+ * list tables  |
+ * list jobs |
+ * list data  
     
     |
+ * info dataset  |
+ * info table  
     |
+ * info job  |
+ * create dataset  |
+ * create table  
     (:)+ |
+ * create view  
     |
+ * create external-table  
     
     (:)+  |
+ * delete dataset  |
+ * delete table  
     
     |
+ * cancel  |
+ * copy     |
+ * load  
     + |
+ * extract  
     
     + |
+ * query  |
+ * load-file  
     
     }
  *
  * 
    The first parameter is an optional {@code project_id} (logged-in project will be used if not
  * supplied). Second parameter is a BigQuery operation and can be used to demonstrate its usage. For
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/compute/ComputeExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/compute/ComputeExample.java
index 23421e197f58..a66fe20e52c8 100644
--- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/compute/ComputeExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/compute/ComputeExample.java
@@ -83,13 +83,10 @@
  *
  * 
    This example demonstrates a simple/typical Compute usage.
  *
- * 
    Steps needed for running the example:
- * 
      
- * 
    1. login using gcloud SDK - {@code gcloud auth login}.
      2. 
- * 
    2. compile using maven - {@code mvn compile}
      3. 
- * 
    3. run using maven -
- * 
      {@code mvn exec:java -Dexec.mainClass="com.google.cloud.examples.compute.ComputeExample"
- *  -Dexec.args="[]
+ * 
      See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
      {@code target/appassembler/bin/ComputeExample []
  * list networks |
  * list region-operations  |
  * list instances ? |
@@ -151,9 +148,7 @@
  * stop   |
  * reset   |
  * set-tags   * |
- * set-metadata   *"}
      
- * 
      4. 
- * 
    
+ * set-metadata   *}
  *
  * 
    The first parameter is an optional {@code project_id} (logged-in project will be used if not
  * supplied). Second parameter is a Compute operation and can be used to demonstrate its usage. For
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/datastore/DatastoreExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/datastore/DatastoreExample.java
index 4b83f04e7165..0e3e505dd308 100644
--- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/datastore/DatastoreExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/datastore/DatastoreExample.java
@@ -41,18 +41,11 @@
  * 
    This example adds, displays or clears comments for a given user. This example also sets
  * contact information for a user.
  *
- * 
    Steps needed for running the example:
      
- * 
    1. login using gcloud SDK - {@code gcloud auth login}.
      2. 
- * 
    2. compile using maven - {@code mvn compile}
      3. 
- * 
    3. run using maven -
- * 
      {@code mvn exec:java -Dexec.mainClass="com.google.cloud.examples.datastore.DatastoreExample"
- *  -Dexec.args=" 
- *  delete |
- *  display |
- *  add  |
- *  set  }
      
- * 
      4. 
- * 
    
+ * 
    See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
    {@code target/appassembler/bin/DatastoreExample
+ * [projectId] [user] [delete|display|add comment|set  ]}
    
  *
  * 
    If no action is provided {@code display} is executed.
  */
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/dns/DnsExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/dns/DnsExample.java
index 8b3c91cbd664..f49659c3158f 100644
--- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/dns/DnsExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/dns/DnsExample.java
@@ -42,21 +42,17 @@
  * 
    This example creates, deletes, gets, and lists zones. It also creates and deletes
  * record sets of type A, and lists record sets.
  *
- * 
    Steps needed for running the example:
- * 
      
- * 
    1. login using gcloud SDK - {@code gcloud auth login}.
      2. 
- * 
    2. compile using maven - {@code mvn compile}
      3. 
- * 
    3. run using maven - {@code mvn exec:java
- * -Dexec.mainClass="com.google.cloud.examples.dns.DnsExample"
- * -Dexec.args="[]
+ * 
      See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
      {@code target/appassembler/bin/DnsExample []
  * create    |
  * get  |
  * delete  |
  * list [ [changes [descending | ascending] | records]] |
  * add-record     |
  * delete-record    [] |
- * quota}
      4. 
- * 
    
+ * quota}
  *
  * 
    The first parameter is an optional {@code project_id}. The project specified in the Google
  * Cloud SDK configuration (see {@code gcloud config list}) will be used if the project ID is not
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java
new file mode 100644
index 000000000000..90d6f43e5f3f
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/CountBytes.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.io.BaseEncoding;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * CountBytes will read through the whole file given as input.
+ *
+ * 
    This example shows how to read a file size using NIO.
+ * File.size returns the size of the file as saved in Storage metadata.
+ * This class also shows how to read all of the file's contents using NIO,
+ * computes a MD5 hash, and reports how long it took.
+ *
+ * 
    See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
    {@code target/appassembler/bin/CountBytes }
    
+ */
+public class CountBytes {
+
+  /**
+   * See the class documentation.
+   */
+  public static void main(String[] args) throws IOException {
+    if (args.length == 0 || args[0].equals("--help")) {
+      help();
+      return;
+    }
+    for (String a : args) {
+      countFile(a);
+    }
+  }
+
+  /**
+   * Print the length of the indicated file.
+   *
+   * 
    This uses the normal Java NIO Api, so it can take advantage of any installed
+   * NIO Filesystem provider without any extra effort.
+   */
+  private static void countFile(String fname) {
+    // large buffers pay off
+    final int bufSize = 50 * 1024 * 1024;
+    try {
+      Path path = Paths.get(new URI(fname));
+      long size = Files.size(path);
+      System.out.println(fname + ": " + size + " bytes.");
+      ByteBuffer buf = ByteBuffer.allocate(bufSize);
+      System.out.println("Reading the whole file...");
+      Stopwatch sw = Stopwatch.createStarted();
+      try (SeekableByteChannel chan = Files.newByteChannel(path)) {
+        long total = 0;
+        int readCalls = 0;
+        MessageDigest md = MessageDigest.getInstance("MD5");
+        while (chan.read(buf) > 0) {
+          readCalls++;
+          md.update(buf.array(), 0, buf.position());
+          total += buf.position();
+          buf.flip();
+        }
+        readCalls++; // We must count the last call
+        long elapsed = sw.elapsed(TimeUnit.SECONDS);
+        System.out.println("Read all " + total + " bytes in " + elapsed + "s. " +
+            "(" + readCalls +" calls to chan.read)");
+        String hex = String.valueOf(BaseEncoding.base16().encode(md.digest()));
+        System.out.println("The MD5 is: 0x" + hex);
+        if (total != size) {
+          System.out.println("Wait, this doesn't match! We saw " + total + " bytes, " +
+              "yet the file size is listed at " + size + " bytes.");
+        }
+      }
+    } catch (Exception ex) {
+      System.out.println(fname + ": " + ex.toString());
+    }
+  }
+
+  private static void help() {
+    String[] help =
+        {"The argument is a ",
+         "and we show the length of that file."
+        };
+    for (String s : help) {
+      System.out.println(s);
+    }
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java
new file mode 100644
index 000000000000..2ccd68b6587d
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/ParallelCountBytes.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.io.BaseEncoding;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.ByteBuffer;
+import java.nio.channels.SeekableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.MessageDigest;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * ParallelCountBytes will read through the whole file given as input.
+ *
+ * 
    This example shows how to go through all the contents of a file,
+ * in order, using multithreaded NIO reads.
+ * It prints a MD5 hash and reports how long it took.
+ *
+ * 
    See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
    {@code target/appassembler/bin/ParallelCountBytes }
    
+ */
+public class ParallelCountBytes {
+
+  /**
+   * WorkUnit holds a buffer and the instructions for what to put in it.
+   *
+   * 
    Use it like this:
+   * 
      
+   *   
    1.  call()
+   *   
    2.  the data is now in buf, you can access it directly
+   *   
    3.  if need more, call resetForIndex(...) and go back to the top.
+   *   
    4.  else, call close()
+   * 
    
+   */
+  private static class WorkUnit implements Callable, Closeable {
+    public final ByteBuffer buf;
+    final SeekableByteChannel chan;
+    final int blockSize;
+    int blockIndex;
+
+    public WorkUnit(SeekableByteChannel chan, int blockSize, int blockIndex) {
+      this.chan = chan;
+      this.buf = ByteBuffer.allocate(blockSize);
+      this.blockSize = blockSize;
+      this.blockIndex = blockIndex;
+    }
+
+    @Override
+    public WorkUnit call() throws IOException {
+      long pos = ((long)blockSize) * blockIndex;
+      if (pos > chan.size()) {
+        return this;
+      }
+      chan.position(pos);
+      // read until buffer is full, or EOF
+      while (chan.read(buf) > 0) {};
+      return this;
+    }
+
+    public WorkUnit resetForIndex(int blockIndex) {
+      this.blockIndex = blockIndex;
+      buf.flip();
+      return this;
+    }
+
+
+    public void close() throws IOException {
+      chan.close();
+    }
+  }
+
+  /**
+   * See the class documentation.
+   */
+  public static void main(String[] args) throws Exception {
+    if (args.length == 0 || args[0].equals("--help")) {
+      help();
+      return;
+    }
+    for (String a : args) {
+      countFile(a);
+    }
+  }
+
+  /**
+   * Print the length and MD5 of the indicated file.
+   *
+   * 
    This uses the normal Java NIO Api, so it can take advantage of any installed
+   * NIO Filesystem provider without any extra effort.
+   */
+  private static void countFile(String fname) throws Exception {
+    // large buffers pay off
+    final int bufSize = 50 * 1024 * 1024;
+    Queue> work = new ArrayDeque<>();
+      Path path = Paths.get(new URI(fname));
+      long size = Files.size(path);
+      System.out.println(fname + ": " + size + " bytes.");
+      int nThreads = (int) Math.ceil(size / (double) bufSize);
+      if (nThreads > 4) nThreads = 4;
+      System.out.println("Reading the whole file using " + nThreads + " threads...");
+      Stopwatch sw = Stopwatch.createStarted();
+      long total = 0;
+      MessageDigest md = MessageDigest.getInstance("MD5");
+
+      ExecutorService exec = Executors.newFixedThreadPool(nThreads);
+      int blockIndex;
+      for (blockIndex = 0; blockIndex < nThreads; blockIndex++) {
+        work.add(exec.submit(new WorkUnit(Files.newByteChannel(path), bufSize, blockIndex)));
+      }
+      while (!work.isEmpty()) {
+        WorkUnit full = work.remove().get();
+        md.update(full.buf.array(), 0, full.buf.position());
+        total += full.buf.position();
+        if (full.buf.hasRemaining()) {
+          full.close();
+        } else {
+          work.add(exec.submit(full.resetForIndex(blockIndex++)));
+        }
+      }
+      exec.shutdown();
+
+      long elapsed = sw.elapsed(TimeUnit.SECONDS);
+      System.out.println("Read all " + total + " bytes in " + elapsed + "s. ");
+      String hex = String.valueOf(BaseEncoding.base16().encode(md.digest()));
+      System.out.println("The MD5 is: 0x" + hex);
+      if (total != size) {
+        System.out.println("Wait, this doesn't match! We saw " + total + " bytes, "
+            + "yet the file size is listed at " + size + " bytes.");
+      }
+  }
+
+  private static void help() {
+    String[] help =
+        {"The argument is a ",
+         "and we show the length of that file."
+        };
+    for (String s : help) {
+      System.out.println(s);
+    }
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java
new file mode 100644
index 000000000000..35a1a2ae7c7f
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/Stat.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.spi.FileSystemProvider;
+
+/**
+ * Stat is a super-simple program that just displays the size of the file
+ * passed as argument.
+ *
+ * 
    It's meant to be used to test GCloud's integration with Java NIO.
+ *
+ * 
    You can either use the '--check' argument to see whether Google Cloud Storage is enabled, or
+ * you can directly pass in a Google Cloud Storage file name to use. In that case you have to be
+ * logged in (using e.g. the gcloud auth command).
+ *
+ * 
    See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
    {@code target/appassembler/bin/Stat --help | --check | --list | }
    
+ *
+ * 
    In short, this version (in gcloud-java-examples) is in a package that lists gcloud-java-nio
+ * as a dependency, so it will work directly without having to do any special work.
+ */
+public class Stat {
+
+  /**
+   * See the class documentation.
+   */
+  public static void main(String[] args) throws IOException {
+    if (args.length == 0 || args[0].equals("--help")) {
+      help();
+      return;
+    }
+    if (args[0].equals("--list")) {
+      listFilesystems();
+      return;
+    }
+    if (args[0].equals("--check")) {
+      checkGcs();
+      return;
+    }
+    for (String a : args) {
+      statFile(a);
+    }
+  }
+
+  /**
+   * Print the length of the indicated file.
+   *
+   * 
    This uses the normal Java NIO Api, so it can take advantage of any installed
+   * NIO Filesystem provider without any extra effort.
+   */
+  private static void statFile(String fname) {
+    try {
+      Path path = Paths.get(new URI(fname));
+      long size = Files.size(path);
+      System.out.println(fname + ": " + size + " bytes.");
+    } catch (Exception ex) {
+      System.out.println(fname + ": " + ex.toString());
+    }
+  }
+
+  private static void help() {
+    String[] help =
+        {"The arguments can be one of:",
+         " * ",
+         "   to display the length of that file.",
+         "",
+         " * --list",
+         "   to list the filesystem providers.",
+         "",
+         " * --check",
+         "   to double-check the Google Cloud Storage provider is installed.",
+         "",
+         "The purpose of this tool is to demonstrate that the gcloud NIO filesystem provider",
+         "can add Google Cloud Storage support to programs not explicitly designed for it.",
+         "",
+         "This tool normally knows nothing of Google Cloud Storage. If you pass it --check",
+         "or a Google Cloud Storage file name (e.g. gs://mybucket/myfile), it will show an error.",
+         "However, by just adding the gcloud-nio jar as a dependency and recompiling, this tool is",
+         "made aware of gs:// paths and can access files on the cloud.",
+         "",
+         "The gcloud NIO filesystem provider can similarly enable existing Java 7 programs",
+         "to read and write cloud files, even if they have no special built-in cloud support."
+        };
+    for (String s : help) {
+      System.out.println(s);
+    }
+  }
+
+  private static void listFilesystems() {
+    System.out.println("Installed filesystem providers:");
+    for (FileSystemProvider p : FileSystemProvider.installedProviders()) {
+      System.out.println("  " + p.getScheme());
+    }
+  }
+
+  private static void checkGcs() {
+    FileSystem fs = FileSystems.getFileSystem(URI.create("gs://domain-registry-alpha"));
+    System.out.println("Success! We can instantiate a gs:// filesystem.");
+    System.out.println("isOpen: " + fs.isOpen());
+    System.out.println("isReadOnly: " + fs.isReadOnly());
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/CreateCloudStorageFileSystem.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/CreateCloudStorageFileSystem.java
new file mode 100644
index 000000000000..b646fa4f1f90
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/CreateCloudStorageFileSystem.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio.snippets;
+
+import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * A snippet for Google Cloud Storage NIO that shows how to create a {@link CloudStorageFileSystem}
+ * for a bucket. The snippet also shows how to create a file, given the file system.
+ */
+public class CreateCloudStorageFileSystem {
+
+  public static void main(String... args) throws IOException {
+    // Create a file system for the bucket
+    CloudStorageFileSystem fs = CloudStorageFileSystem.forBucket("bucket");
+    byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
+    Path path = fs.getPath("/object");
+    // Write a file in the bucket
+    Files.write(path, data);
+    // Read a file from the bucket
+    data = Files.readAllBytes(path);
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/CreateInputStream.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/CreateInputStream.java
new file mode 100644
index 000000000000..243102d31e01
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/CreateInputStream.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio.snippets;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * A snippet showing how to create an input stream for a Google Cloud Storage file using NIO.
+ */
+public class CreateInputStream {
+
+  public static void main(String... args) throws IOException {
+    Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+    try (InputStream input = Files.newInputStream(path)) {
+      // use input stream
+    }
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/GetFileSystem.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/GetFileSystem.java
new file mode 100644
index 000000000000..2819340fa14a
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/GetFileSystem.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio.snippets;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+/**
+ * A snippet showing how to get a {@link FileSystem} instance for a Google Cloud Storage bucket.
+ * This snippet also shows how to create a file and read its lines.
+ */
+public class GetFileSystem {
+
+  public static void main(String... args) throws IOException {
+    FileSystem fs = FileSystems.getFileSystem(URI.create("gs://bucket"));
+    byte[] data = "hello world".getBytes(StandardCharsets.UTF_8);
+    Path path = fs.getPath("/object");
+    Files.write(path, data);
+    List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/ReadAllLines.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/ReadAllLines.java
new file mode 100644
index 000000000000..cbc7baacf48c
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/ReadAllLines.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio.snippets;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+/**
+ * A snippet showing how to read all lines of a Google Cloud Storage file using NIO.
+ */
+public class ReadAllLines {
+
+  public static void main(String... args) throws IOException {
+    Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+    List lines = Files.readAllLines(path, StandardCharsets.UTF_8);
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/WriteFileWithAttributes.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/WriteFileWithAttributes.java
new file mode 100644
index 000000000000..d6486b3cfdc2
--- /dev/null
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/nio/snippets/WriteFileWithAttributes.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.examples.nio.snippets;
+
+import static com.google.cloud.storage.contrib.nio.CloudStorageOptions.withMimeType;
+import static com.google.cloud.storage.contrib.nio.CloudStorageOptions.withoutCaching;
+
+import com.google.cloud.storage.contrib.nio.CloudStorageOptions;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A snippet showing how to write a file to Google Cloud Storage using NIO. This example also shows
+ * how to set file attributes, using {@link CloudStorageOptions} static helpers.
+ */
+public class WriteFileWithAttributes {
+
+  private static final String[] LINES = {"value1,", "value"};
+
+  public static void main(String... args) throws IOException {
+    List csvLines =  Arrays.asList(LINES);
+    Path path = Paths.get(URI.create("gs://bucket/lolcat.csv"));
+    Files.write(path, csvLines, StandardCharsets.UTF_8,
+        withMimeType("text/csv; charset=UTF-8"),
+        withoutCaching());
+  }
+}
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/pubsub/PubSubExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/pubsub/PubSubExample.java
index c488405345be..167ba610cc9d 100644
--- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/pubsub/PubSubExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/pubsub/PubSubExample.java
@@ -43,12 +43,10 @@
  *
  * 
    This example demonstrates a simple/typical Pub/Sub usage.
  *
- * 
    Steps needed for running the example:
- * 
      
- * 
    1. login using gcloud SDK - {@code gcloud auth login}.
      2. 
- * 
    2. compile using maven - {@code mvn compile}
      3. 
- * 
    3. run using maven -
- * 
      {@code mvn exec:java -Dexec.mainClass="com.google.cloud.examples.pubsub.PubSubExample"
+ * 
      See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
      {@code target/appassembler/bin/PubSubExample
  *  -Dexec.args="[]
  *  pull async  ?
  *  pull sync  
@@ -64,8 +62,6 @@
  *  delete subscription 
  *  info topic 
  *  info subscription "}
      
- * 
      4. 
- * 
    
  *
  * 
    The first parameter is an optional {@code project_id} (logged-in project will be used if not
  * supplied). Second parameter is a Pub/Sub operation and can be used to demonstrate its usage. For
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/resourcemanager/ResourceManagerExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/resourcemanager/ResourceManagerExample.java
index fd2eb3ed6d91..53bcd5357594 100644
--- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/resourcemanager/ResourceManagerExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/resourcemanager/ResourceManagerExample.java
@@ -32,13 +32,11 @@
  *
  * 
    This example creates, deletes, gets, and lists projects.
  *
- * 
     Steps needed for running the example:
      
- * 
    1. login using gcloud SDK - {@code gcloud auth login}.
      2. 
- * 
    2. compile using maven - {@code mvn compile}
      3. 
- * 
    3. run using maven - {@code mvn exec:java
- * -Dexec.mainClass="com.google.cloud.examples.resourcemanager.ResourceManagerExample"
- * -Dexec.args="[list | [create | delete | get] projectId]"}
      4. 
- * 
    
+ * 
    See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
    {@code target/appassembler/bin/ResourceManagerExample
+ * [list | [create | delete | get] projectId]}
    
  */
 public class ResourceManagerExample {
 
diff --git a/gcloud-java-examples/src/main/java/com/google/cloud/examples/storage/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/cloud/examples/storage/StorageExample.java
index d8273e65797d..f803377c1255 100644
--- a/gcloud-java-examples/src/main/java/com/google/cloud/examples/storage/StorageExample.java
+++ b/gcloud-java-examples/src/main/java/com/google/cloud/examples/storage/StorageExample.java
@@ -64,13 +64,10 @@
  *
  * 
    This example demonstrates a simple/typical storage usage.
  *
- * 
    Steps needed for running the example:
- * 
      
- * 
    1. login using gcloud SDK - {@code gcloud auth login}.
      2. 
- * 
    2. compile using maven - {@code mvn compile}
      3. 
- * 
    3. run using maven -
- * 
      {@code mvn exec:java -Dexec.mainClass="com.google.cloud.examples.storage.StorageExample"
- *  -Dexec.args="[]
+ * 
      See the
+ * 
+ * README for compilation instructions. Run this code with
+ * 
      {@code target/appassembler/bin/StorageExample []
  *  list [] |
  *  info [ []] |
  *  download   [local_file] |
@@ -83,9 +80,7 @@
  *  add-acl domain  ?  OWNER|READER|WRITER |
  *  add-acl project  ? :(OWNERS|EDITORS|VIEWERS) OWNER|READER|WRITER |
  *  add-acl user  ? |allUsers|allAuthenticatedUsers OWNER|READER|WRITER |
- *  add-acl group  ?  OWNER|READER|WRITER"}
      
- * 
      4. 
- * 
    
+ *  add-acl group  ?  OWNER|READER|WRITER}
  *
  * 
    The first parameter is an optional {@code project_id} (logged-in project will be used if not
  * supplied). Second parameter is a Storage operation (list, delete, compose,...) and can be used to
diff --git a/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/spi/DefaultResourceManagerRpc.java b/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/spi/DefaultResourceManagerRpc.java
index 427478baef5f..43cb40bcd306 100644
--- a/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/spi/DefaultResourceManagerRpc.java
+++ b/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/spi/DefaultResourceManagerRpc.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.cloud.resourcemanager.spi;
 
 import static com.google.cloud.resourcemanager.spi.ResourceManagerRpc.Option.FIELDS;
diff --git a/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelper.java b/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelper.java
index 6c16372c0424..48934d602bfe 100644
--- a/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelper.java
+++ b/gcloud-java-resourcemanager/src/main/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelper.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.cloud.resourcemanager.testing;
 
 import static com.google.common.base.Preconditions.checkArgument;
diff --git a/gcloud-java-resourcemanager/src/test/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelperTest.java b/gcloud-java-resourcemanager/src/test/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelperTest.java
index cf65b4981bfc..cfe475cb038b 100644
--- a/gcloud-java-resourcemanager/src/test/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelperTest.java
+++ b/gcloud-java-resourcemanager/src/test/java/com/google/cloud/resourcemanager/testing/LocalResourceManagerHelperTest.java
@@ -1,3 +1,19 @@
+/*
+ * Copyright 2016 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package com.google.cloud.resourcemanager.testing;
 
 import static org.junit.Assert.assertEquals;
diff --git a/gcloud-java-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java b/gcloud-java-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java
index 0352e8a9d550..04b085574844 100644
--- a/gcloud-java-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java
+++ b/gcloud-java-storage/src/main/java/com/google/cloud/storage/BlobReadChannel.java
@@ -45,7 +45,7 @@ class BlobReadChannel implements ReadChannel {
   private final BlobId blob;
   private final Map requestOptions;
   private String lastEtag;
-  private int position;
+  private long position;
   private boolean isOpen;
   private boolean endOfStream;
   private int chunkSize = DEFAULT_CHUNK_SIZE;
@@ -99,7 +99,7 @@ private void validateOpen() throws ClosedChannelException {
   }
 
   @Override
-  public void seek(int position) throws IOException {
+  public void seek(long position) throws IOException {
     validateOpen();
     this.position = position;
     buffer = null;
@@ -164,7 +164,7 @@ static class StateImpl implements RestorableState, Serializable {
     private final BlobId blob;
     private final Map requestOptions;
     private final String lastEtag;
-    private final int position;
+    private final long position;
     private final boolean isOpen;
     private final boolean endOfStream;
     private final int chunkSize;
@@ -185,7 +185,7 @@ static class Builder {
       private final BlobId blob;
       private final Map requestOptions;
       private String lastEtag;
-      private int position;
+      private long position;
       private boolean isOpen;
       private boolean endOfStream;
       private int chunkSize;
@@ -201,7 +201,7 @@ Builder lastEtag(String lastEtag) {
         return this;
       }
 
-      Builder position(int position) {
+      Builder position(long position) {
         this.position = position;
         return this;
       }
diff --git a/gcloud-java-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java
index 65d540458c7c..883ce80003bb 100644
--- a/gcloud-java-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java
+++ b/gcloud-java-storage/src/main/java/com/google/cloud/storage/spi/DefaultStorageRpc.java
@@ -31,6 +31,7 @@
 import static com.google.cloud.storage.spi.StorageRpc.Option.PREFIX;
 import static com.google.cloud.storage.spi.StorageRpc.Option.VERSIONS;
 import static com.google.common.base.MoreObjects.firstNonNull;
+import static com.google.common.base.Preconditions.checkArgument;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 import static javax.servlet.http.HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE;
 
@@ -467,10 +468,11 @@ public Tuple read(StorageObject from, Map options, lo
           .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(options))
           .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(options))
           .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(options));
+      checkArgument(position >= 0, "Position should be non-negative, is %d", position);
       StringBuilder range = new StringBuilder();
       range.append("bytes=").append(position).append("-").append(position + bytes - 1);
       req.getRequestHeaders().setRange(range.toString());
-      ByteArrayOutputStream output = new ByteArrayOutputStream();
+      ByteArrayOutputStream output = new ByteArrayOutputStream(bytes);
       req.executeMedia().download(output);
       String etag = req.getLastResponseHeaders().getETag();
       return Tuple.of(etag, output.toByteArray());
diff --git a/utilities/update_docs_version.sh b/utilities/update_docs_version.sh
index c9305a157bab..bec6813d1610 100755
--- a/utilities/update_docs_version.sh
+++ b/utilities/update_docs_version.sh
@@ -21,6 +21,9 @@ if [ "${RELEASED_VERSION##*-}" != "SNAPSHOT" ]; then
             readmes="$readmes ${item}/README.md"
         fi
     done
+    NEW_VERSION=${RELEASED_VERSION%.*}.$((${RELEASED_VERSION##*.}+1))-SNAPSHOT
+    echo "Changing version to $NEW_VERSION in gcloud-java-nio-example README"
+    sed -ri "s/gcloud-java-nio-examples-[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT/gcloud-java-nio-examples-$NEW_VERSION/g" gcloud-java-contrib/gcloud-java-nio-examples/README.md
     git add $readmes
     git config --global user.name "travis-ci"
     git config --global user.email "travis@travis-ci.org"