Skip to content

Commit 0b77a08

Browse files
mziccardmziccard
committed
Add support for blob rewrite
- Add rewrite method to StorageRpc and DefaultStorageRpc - Add rewriter method to Storage and StorageImpl - Add unit and integration tests
1 parent 6e25709 commit 0b77a08

File tree

7 files changed

+604
-0
lines changed

7 files changed

+604
-0
lines changed

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java

+28
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import com.google.api.services.storage.model.ComposeRequest;
5656
import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions;
5757
import com.google.api.services.storage.model.Objects;
58+
import com.google.api.services.storage.model.RewriteResponse;
5859
import com.google.api.services.storage.model.StorageObject;
5960
import com.google.common.base.MoreObjects;
6061
import com.google.common.collect.ImmutableSet;
@@ -521,4 +522,31 @@ public String open(StorageObject object, Map<Option, ?> options)
521522
throw translate(ex);
522523
}
523524
}
525+
526+
@Override
527+
public RewriteResponse rewrite(StorageObject source, Map<Option, ?> sourceOptions,
528+
StorageObject target, Map<Option, ?> targetOptions, String token, Long maxByteRewrittenPerCall)
529+
throws StorageException {
530+
try {
531+
return storage
532+
.objects()
533+
.rewrite(source.getBucket(), source.getName(), target.getBucket(), target.getName(),
534+
target)
535+
.setRewriteToken(token)
536+
.setMaxBytesRewrittenPerCall(maxByteRewrittenPerCall)
537+
.setProjection(DEFAULT_PROJECTION)
538+
.setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions))
539+
.setIfSourceMetagenerationNotMatch(
540+
IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions))
541+
.setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions))
542+
.setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions))
543+
.setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions))
544+
.setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions))
545+
.setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions))
546+
.setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions))
547+
.execute();
548+
} catch (IOException ex) {
549+
throw translate(ex);
550+
}
551+
}
524552
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java

+5
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static com.google.common.base.MoreObjects.firstNonNull;
2020

2121
import com.google.api.services.storage.model.Bucket;
22+
import com.google.api.services.storage.model.RewriteResponse;
2223
import com.google.api.services.storage.model.StorageObject;
2324
import com.google.common.collect.ImmutableList;
2425
import com.google.common.collect.ImmutableMap;
@@ -174,4 +175,8 @@ byte[] read(StorageObject from, Map<Option, ?> options, long position, int bytes
174175

175176
void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest,
176177
long destOffset, int length, boolean last) throws StorageException;
178+
179+
RewriteResponse rewrite(StorageObject source, Map<Option, ?> sourceOptions,
180+
StorageObject target, Map<Option, ?> targetOptions, String token, Long maxByteRewrittenPerCall)
181+
throws StorageException;
177182
}

‎gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java

+203
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/*
2+
* Copyright 2015 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.gcloud.storage;
18+
19+
import static com.google.common.base.MoreObjects.firstNonNull;
20+
import static com.google.gcloud.RetryHelper.runWithRetries;
21+
22+
import com.google.api.services.storage.model.RewriteResponse;
23+
import com.google.gcloud.RetryHelper;
24+
import com.google.gcloud.spi.StorageRpc;
25+
26+
import java.math.BigInteger;
27+
import java.util.Map;
28+
import java.util.concurrent.Callable;
29+
30+
/**
31+
* Google Storage blob rewriter.
32+
*/
33+
public final class BlobRewriter {
34+
35+
private final StorageOptions serviceOptions;
36+
private final BlobId source;
37+
private final Map<StorageRpc.Option, ?> sourceOptions;
38+
private final Map<StorageRpc.Option, ?> targetOptions;
39+
private final Long maxBytesRewrittenPerCall;
40+
private BigInteger blobSize;
41+
private BlobInfo target;
42+
private Boolean isDone;
43+
private String rewriteToken;
44+
private BigInteger totalBytesRewritten;
45+
46+
private final StorageRpc storageRpc;
47+
48+
private BlobRewriter(Builder builder) {
49+
this.serviceOptions = builder.serviceOptions;
50+
this.source = builder.source;
51+
this.sourceOptions = builder.sourceOptions;
52+
this.target = builder.target;
53+
this.targetOptions = builder.targetOptions;
54+
this.blobSize = builder.blobSize;
55+
this.isDone = builder.isDone;
56+
this.rewriteToken = builder.rewriteToken;
57+
this.totalBytesRewritten = firstNonNull(builder.totalBytesRewritten, BigInteger.ZERO);
58+
this.maxBytesRewrittenPerCall = builder.maxBytesRewrittenPerCall;
59+
this.storageRpc = serviceOptions.storageRpc();
60+
}
61+
62+
static class Builder {
63+
64+
private final StorageOptions serviceOptions;
65+
private final BlobId source;
66+
private final Map<StorageRpc.Option, ?> sourceOptions;
67+
private final BlobInfo target;
68+
private final Map<StorageRpc.Option, ?> targetOptions;
69+
private BigInteger blobSize;
70+
private Boolean isDone;
71+
private String rewriteToken;
72+
private BigInteger totalBytesRewritten;
73+
private Long maxBytesRewrittenPerCall;
74+
75+
Builder(StorageOptions serviceOptions, BlobId source, Map<StorageRpc.Option, ?> sourceOptions,
76+
BlobInfo target, Map<StorageRpc.Option, ?> targetOptions) {
77+
this.serviceOptions = serviceOptions;
78+
this.source = source;
79+
this.sourceOptions = sourceOptions;
80+
this.target = target;
81+
this.targetOptions = targetOptions;
82+
}
83+
84+
Builder blobSize(BigInteger blobSize) {
85+
this.blobSize = blobSize;
86+
return this;
87+
}
88+
89+
Builder isDone(Boolean isDone) {
90+
this.isDone = isDone;
91+
return this;
92+
}
93+
94+
Builder rewriteToken(String rewriteToken) {
95+
this.rewriteToken = rewriteToken;
96+
return this;
97+
}
98+
99+
Builder totalBytesRewritten(BigInteger totalBytesRewritten) {
100+
this.totalBytesRewritten = totalBytesRewritten;
101+
return this;
102+
}
103+
104+
Builder maxBytesRewrittenPerCall(Long maxBytesRewrittenPerCall) {
105+
this.maxBytesRewrittenPerCall = maxBytesRewrittenPerCall;
106+
return this;
107+
}
108+
109+
BlobRewriter build() {
110+
return new BlobRewriter(this);
111+
}
112+
}
113+
114+
static Builder builder(StorageOptions options, BlobId source,
115+
Map<StorageRpc.Option, ?> sourceOpt,
116+
BlobInfo target, Map<StorageRpc.Option, ?> targetOpt) {
117+
return new Builder(options, source, sourceOpt, target, targetOpt);
118+
}
119+
120+
/**
121+
* Returns the id of the source blob.
122+
*/
123+
public BlobId source() {
124+
return source;
125+
}
126+
127+
/**
128+
* Returns the info for the target blob. When {@link #isDone} is {@code true} this method returns
129+
* the updated information for the just written blob.
130+
*/
131+
public BlobInfo target() {
132+
return target;
133+
}
134+
135+
/**
136+
* Size of the blob being copied. Returns {@code null} until the first copy request returns.
137+
*/
138+
public BigInteger blobSize() {
139+
return blobSize;
140+
}
141+
142+
/**
143+
* Returns {@code true} of blob rewrite finished, {@code false} otherwise.
144+
*/
145+
public Boolean isDone() {
146+
return isDone;
147+
}
148+
149+
/**
150+
* Returns the token to be used to rewrite the next chunk of the blob.
151+
*/
152+
public String rewriteToken() {
153+
return rewriteToken;
154+
}
155+
156+
/**
157+
* Returns the number of bytes written.
158+
*/
159+
public BigInteger totalBytesRewritten() {
160+
return totalBytesRewritten;
161+
}
162+
163+
/**
164+
* Returns the maximum number of bytes to be copied with each {@link #copyChunk()} call. This
165+
* parameter is ignored if source and target blob share the same location and storage class as
166+
* rewrite is made with one single RPC.
167+
*/
168+
public Long maxBytesRewrittenPerCall() {
169+
return maxBytesRewrittenPerCall;
170+
}
171+
172+
/**
173+
* Rewrite the next chunk of the blob. An RPC is issued only if rewrite has not finished yet
174+
* ({@link #isDone} returns {@code false}).
175+
*
176+
* @throws StorageException upon failure
177+
*/
178+
public void copyChunk() {
179+
if (!isDone) {
180+
try {
181+
RewriteResponse response = runWithRetries(new Callable<RewriteResponse>() {
182+
@Override
183+
public RewriteResponse call() {
184+
return storageRpc.rewrite(
185+
source.toPb(),
186+
sourceOptions,
187+
target.toPb(),
188+
targetOptions,
189+
rewriteToken,
190+
maxBytesRewrittenPerCall);
191+
}
192+
}, serviceOptions.retryParams(), StorageImpl.EXCEPTION_HANDLER);
193+
rewriteToken = response.getRewriteToken();
194+
isDone = response.getDone();
195+
blobSize = response.getObjectSize();
196+
totalBytesRewritten = response.getTotalBytesRewritten();
197+
target = response.getResource() != null ? BlobInfo.fromPb(response.getResource()) : target;
198+
} catch (RetryHelper.RetryHelperException e) {
199+
throw StorageException.translateAndThrow(e);
200+
}
201+
}
202+
}
203+
}

0 commit comments

Comments
 (0)