Skip to content

Commit 5137b75

Browse files
soosinhasoosinha
committed
Serialize using codec version
Signed-off-by: Sooraj Sinha <[email protected]>
1 parent ad7f9e7 commit 5137b75

File tree

5 files changed

+110
-21
lines changed

5 files changed

+110
-21
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
8989
- Fix inefficient Stream API call chains ending with count() ([#15386](https://github.com/opensearch-project/OpenSearch/pull/15386))
9090
- Fix array hashCode calculation in ResyncReplicationRequest ([#16378](https://github.com/opensearch-project/OpenSearch/pull/16378))
9191
- Fix typo super->sb in method toString() of RemoteStoreNodeAttribute ([#15362](https://github.com/opensearch-project/OpenSearch/pull/15362))
92+
- Fix version upgrade for remote state enabled cluster ([#16403](https://github.com/opensearch-project/OpenSearch/pull/16403))
9293

9394
### Security
9495

‎server/src/main/java/org/opensearch/gateway/remote/ClusterMetadataManifest.java

+12-2
Original file line numberDiff line numberDiff line change
@@ -1216,12 +1216,18 @@ private static String componentPrefix(Object[] fields) {
12161216

12171217
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V0 = new ConstructingObjectParser<>(
12181218
"uploaded_index_metadata",
1219-
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields))
1219+
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), CODEC_V0)
12201220
);
12211221

12221222
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> PARSER_V2 = new ConstructingObjectParser<>(
12231223
"uploaded_index_metadata",
1224-
fields -> new UploadedIndexMetadata(indexName(fields), indexUUID(fields), uploadedFilename(fields), componentPrefix(fields))
1224+
fields -> new UploadedIndexMetadata(
1225+
indexName(fields),
1226+
indexUUID(fields),
1227+
uploadedFilename(fields),
1228+
componentPrefix(fields),
1229+
CODEC_V2
1230+
)
12251231
);
12261232

12271233
private static final ConstructingObjectParser<UploadedIndexMetadata, Void> CURRENT_PARSER = PARSER_V2;
@@ -1306,6 +1312,10 @@ public String getComponentPrefix() {
13061312
return componentPrefix;
13071313
}
13081314

1315+
protected void setCodecVersion(long codecVersion) {
1316+
this.codecVersion = codecVersion;
1317+
}
1318+
13091319
@Override
13101320
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
13111321
builder.field(INDEX_NAME_FIELD.getPreferredName(), getIndexName())

‎server/src/main/java/org/opensearch/gateway/remote/RemoteManifestManager.java

+1
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ RemoteClusterStateManifestInfo uploadManifest(
105105
) {
106106
synchronized (this) {
107107
ClusterMetadataManifest.Builder manifestBuilder = ClusterMetadataManifest.builder();
108+
uploadedMetadataResult.uploadedIndexMetadata.forEach(md -> md.setCodecVersion(codecVersion));
108109
manifestBuilder.clusterTerm(clusterState.term())
109110
.stateVersion(clusterState.getVersion())
110111
.clusterUUID(clusterState.metadata().clusterUUID())

‎server/src/main/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifest.java

+13-3
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,8 @@ public UploadedMetadata getUploadedMetadata() {
123123

124124
@Override
125125
public InputStream serialize() throws IOException {
126-
return CLUSTER_METADATA_MANIFEST_FORMAT.serialize(
126+
ChecksumBlobStoreFormat<ClusterMetadataManifest> blobStoreFormat = getClusterMetadataManifestBlobStoreFormatForUpload();
127+
return blobStoreFormat.serialize(
127128
clusterMetadataManifest,
128129
generateBlobFileName(),
129130
getCompressor(),
@@ -133,7 +134,7 @@ public InputStream serialize() throws IOException {
133134

134135
@Override
135136
public ClusterMetadataManifest deserialize(final InputStream inputStream) throws IOException {
136-
ChecksumBlobStoreFormat<ClusterMetadataManifest> blobStoreFormat = getClusterMetadataManifestBlobStoreFormat();
137+
ChecksumBlobStoreFormat<ClusterMetadataManifest> blobStoreFormat = getClusterMetadataManifestBlobStoreFormatForDownload();
137138
return blobStoreFormat.deserialize(blobName, getNamedXContentRegistry(), Streams.readFully(inputStream));
138139
}
139140

@@ -151,8 +152,17 @@ int getManifestCodecVersion() {
151152
}
152153
}
153154

154-
private ChecksumBlobStoreFormat<ClusterMetadataManifest> getClusterMetadataManifestBlobStoreFormat() {
155+
private ChecksumBlobStoreFormat<ClusterMetadataManifest> getClusterMetadataManifestBlobStoreFormatForDownload() {
155156
long codecVersion = getManifestCodecVersion();
157+
return getClusterMetadataManifestBlobStoreFormat(codecVersion);
158+
}
159+
160+
private ChecksumBlobStoreFormat<ClusterMetadataManifest> getClusterMetadataManifestBlobStoreFormatForUpload() {
161+
long codecVersion = clusterMetadataManifest.getCodecVersion();
162+
return getClusterMetadataManifestBlobStoreFormat(codecVersion);
163+
}
164+
165+
private ChecksumBlobStoreFormat<ClusterMetadataManifest> getClusterMetadataManifestBlobStoreFormat(long codecVersion) {
156166
if (codecVersion == ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION) {
157167
return CLUSTER_METADATA_MANIFEST_FORMAT;
158168
} else if (codecVersion == ClusterMetadataManifest.CODEC_V3) {

‎server/src/test/java/org/opensearch/gateway/remote/model/RemoteClusterMetadataManifestTests.java

+83-16
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.opensearch.core.compress.NoneCompressor;
2121
import org.opensearch.core.xcontent.NamedXContentRegistry;
2222
import org.opensearch.gateway.remote.ClusterMetadataManifest;
23+
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
2324
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata;
2425
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadataAttribute;
2526
import org.opensearch.gateway.remote.RemoteClusterStateUtils;
@@ -37,11 +38,13 @@
3738
import java.io.IOException;
3839
import java.io.InputStream;
3940
import java.util.List;
41+
import java.util.Map;
4042
import java.util.function.Function;
4143
import java.util.stream.Stream;
4244

4345
import static java.util.stream.Collectors.toList;
4446
import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V0;
47+
import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V1;
4548
import static org.opensearch.gateway.remote.ClusterMetadataManifest.CODEC_V2;
4649
import static org.opensearch.gateway.remote.ClusterMetadataManifest.MANIFEST_CURRENT_CODEC_VERSION;
4750
import static org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest.MANIFEST;
@@ -214,18 +217,20 @@ public void testGetUploadedMetadata() throws IOException {
214217
}
215218

216219
public void testSerDe() throws IOException {
217-
ClusterMetadataManifest manifest = getClusterMetadataManifest();
218-
RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(
219-
manifest,
220-
clusterUUID,
221-
compressor,
222-
namedXContentRegistry
223-
);
224-
try (InputStream inputStream = remoteObjectForUpload.serialize()) {
225-
remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath());
226-
assertThat(inputStream.available(), greaterThan(0));
227-
ClusterMetadataManifest readManifest = remoteObjectForUpload.deserialize(inputStream);
228-
assertThat(readManifest, is(manifest));
220+
for (int codecVersion : ClusterMetadataManifest.CODEC_VERSIONS) {
221+
ClusterMetadataManifest manifest = getClusterMetadataManifestForCodecVersion(codecVersion);
222+
RemoteClusterMetadataManifest remoteObjectForUpload = new RemoteClusterMetadataManifest(
223+
manifest,
224+
clusterUUID,
225+
compressor,
226+
namedXContentRegistry
227+
);
228+
try (InputStream inputStream = remoteObjectForUpload.serialize()) {
229+
remoteObjectForUpload.setFullBlobName(BlobPath.cleanPath());
230+
assertThat(inputStream.available(), greaterThan(0));
231+
ClusterMetadataManifest readManifest = remoteObjectForUpload.deserialize(inputStream);
232+
validateManifest(manifest, readManifest);
233+
}
229234
}
230235

231236
String blobName = "/usr/local/manifest__1__2__3__4__5__6";
@@ -261,9 +266,13 @@ public void testGetManifestCodecVersion() {
261266
}
262267

263268
private ClusterMetadataManifest getClusterMetadataManifest() {
264-
return ClusterMetadataManifest.builder()
265-
.opensearchVersion(Version.CURRENT)
266-
.codecVersion(MANIFEST_CURRENT_CODEC_VERSION)
269+
return getClusterMetadataManifestForCodecVersion(MANIFEST_CURRENT_CODEC_VERSION);
270+
}
271+
272+
private ClusterMetadataManifest getClusterMetadataManifestForCodecVersion(int codecVersion) {
273+
ClusterMetadataManifest.Builder builder = ClusterMetadataManifest.builder();
274+
builder.opensearchVersion(Version.CURRENT)
275+
.codecVersion(codecVersion)
267276
.nodeId("test-node")
268277
.clusterUUID("test-uuid")
269278
.previousClusterUUID("_NA_")
@@ -272,6 +281,64 @@ private ClusterMetadataManifest getClusterMetadataManifest() {
272281
.stateVersion(VERSION)
273282
.committed(true)
274283
.coordinationMetadata(new UploadedMetadataAttribute("test-attr", "uploaded-file"))
275-
.build();
284+
.indices(List.of(new UploadedIndexMetadata("test-index", "tst-idx", "uploaded-index-file", codecVersion)))
285+
.clusterUUIDCommitted(true);
286+
287+
if (codecVersion == CODEC_V1) {
288+
builder.globalMetadataFileName("global-metadata-file-name");
289+
}
290+
if (codecVersion >= CODEC_V2) {
291+
builder.coordinationMetadata(new UploadedMetadataAttribute("uploaded_coordination", "coordination-metadata-file"));
292+
builder.settingMetadata(new UploadedMetadataAttribute("uploaded_settings", "settings-metadata-file"));
293+
builder.templatesMetadata(new UploadedMetadataAttribute("uploaded_templates", "templates-metadata-file"));
294+
builder.customMetadataMap(Map.of("uploaded_custom", new UploadedMetadataAttribute("uploaded_custom", "custom-metadata-file")));
295+
builder.routingTableVersion(1L);
296+
builder.indicesRouting(List.of(new UploadedIndexMetadata("test-index", "tst-idx", "uploaded_routing", "routing--")));
297+
builder.discoveryNodesMetadata(new UploadedMetadataAttribute("uploaded_discovery", "discovery-metadata-file"));
298+
builder.clusterBlocksMetadata(new UploadedMetadataAttribute("uploaded_blocks", "blocks-metadata-file"));
299+
builder.metadataVersion(1L);
300+
builder.transientSettingsMetadata(
301+
new UploadedMetadataAttribute("uploaded_transient_settings", "transient-settings-metadata-file")
302+
);
303+
builder.hashesOfConsistentSettings(new UploadedMetadataAttribute("uploaded_hashes_settings", "hashes-settings-metadata-file"));
304+
builder.clusterStateCustomMetadataMap(
305+
Map.of("uploaded_custom", new UploadedMetadataAttribute("uploaded_custom", "custom-metadata-file"))
306+
);
307+
}
308+
309+
return builder.build();
310+
}
311+
312+
private void validateManifest(ClusterMetadataManifest writeManifest, ClusterMetadataManifest readManifest) {
313+
assertThat(readManifest.getOpensearchVersion(), is(writeManifest.getOpensearchVersion()));
314+
assertThat(readManifest.getCodecVersion(), is(writeManifest.getCodecVersion()));
315+
assertThat(readManifest.getNodeId(), is(writeManifest.getNodeId()));
316+
assertThat(readManifest.getClusterUUID(), is(writeManifest.getClusterUUID()));
317+
assertThat(readManifest.getPreviousClusterUUID(), is(writeManifest.getPreviousClusterUUID()));
318+
assertThat(readManifest.getStateUUID(), is(writeManifest.getStateUUID()));
319+
assertThat(readManifest.getClusterTerm(), is(writeManifest.getClusterTerm()));
320+
assertThat(readManifest.getStateVersion(), is(writeManifest.getStateVersion()));
321+
assertThat(readManifest.isCommitted(), is(writeManifest.isCommitted()));
322+
assertThat(readManifest.getPreviousClusterUUID(), is(writeManifest.getPreviousClusterUUID()));
323+
assertThat(readManifest.isClusterUUIDCommitted(), is(writeManifest.isClusterUUIDCommitted()));
324+
assertThat(readManifest.getIndices(), is(writeManifest.getIndices()));
325+
if (writeManifest.getCodecVersion() == CODEC_V1) {
326+
assertThat(readManifest.getGlobalMetadataFileName(), is(writeManifest.getGlobalMetadataFileName()));
327+
}
328+
if (writeManifest.getCodecVersion() >= CODEC_V2) {
329+
assertThat(readManifest.getCoordinationMetadata(), is(writeManifest.getCoordinationMetadata()));
330+
assertThat(readManifest.getSettingsMetadata(), is(writeManifest.getSettingsMetadata()));
331+
assertThat(readManifest.getTemplatesMetadata(), is(writeManifest.getTemplatesMetadata()));
332+
assertThat(readManifest.getCustomMetadataMap(), is(writeManifest.getCustomMetadataMap()));
333+
assertThat(readManifest.getRoutingTableVersion(), is(writeManifest.getRoutingTableVersion()));
334+
assertThat(readManifest.getIndicesRouting(), is(writeManifest.getIndicesRouting()));
335+
assertThat(readManifest.getDiscoveryNodesMetadata(), is(writeManifest.getDiscoveryNodesMetadata()));
336+
assertThat(readManifest.getClusterBlocksMetadata(), is(writeManifest.getClusterBlocksMetadata()));
337+
assertThat(readManifest.getMetadataVersion(), is(writeManifest.getMetadataVersion()));
338+
assertThat(readManifest.getDiffManifest(), is(writeManifest.getDiffManifest()));
339+
assertThat(readManifest.getTransientSettingsMetadata(), is(writeManifest.getTransientSettingsMetadata()));
340+
assertThat(readManifest.getHashesOfConsistentSettings(), is(writeManifest.getHashesOfConsistentSettings()));
341+
assertThat(readManifest.getClusterStateCustomMap(), is(writeManifest.getClusterStateCustomMap()));
342+
}
276343
}
277344
}

0 commit comments

Comments
 (0)