Skip to content

Commit 7712bea

Browse files
soosinhasoosinha
authored
Add dedicated string prefix for remote index metadata and remote routing table (#15575) (#15739)
* Hashed prefix for index metadata Signed-off-by: Sooraj Sinha <[email protected]>
1 parent 3b8a741 commit 7712bea

File tree

11 files changed

+298
-17
lines changed

11 files changed

+298
-17
lines changed

‎server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,11 @@
1717
import org.opensearch.common.settings.Settings;
1818
import org.opensearch.common.util.FeatureFlags;
1919
import org.opensearch.discovery.DiscoveryStats;
20+
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
2021
import org.opensearch.gateway.remote.model.RemoteClusterMetadataManifest;
22+
import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore;
23+
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
24+
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
2125
import org.opensearch.indices.recovery.RecoverySettings;
2226
import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
2327
import org.opensearch.repositories.RepositoriesService;
@@ -48,19 +52,27 @@
4852
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
4953
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
5054
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
55+
import static org.hamcrest.Matchers.is;
56+
import static org.hamcrest.Matchers.startsWith;
5157

5258
@ClusterScope(scope = Scope.TEST, numDataNodes = 0)
5359
public class RemoteStatePublicationIT extends RemoteStoreBaseIntegTestCase {
5460

55-
private static String INDEX_NAME = "test-index";
61+
private static final String INDEX_NAME = "test-index";
62+
private static final String REMOTE_STATE_PREFIX = "!";
63+
private static final String REMOTE_ROUTING_PREFIX = "_";
5664
private boolean isRemoteStateEnabled = true;
5765
private String isRemotePublicationEnabled = "true";
66+
private boolean hasRemoteStateCharPrefix;
67+
private boolean hasRemoteRoutingCharPrefix;
5868

5969
@Before
6070
public void setup() {
6171
asyncUploadMockFsRepo = false;
6272
isRemoteStateEnabled = true;
6373
isRemotePublicationEnabled = "true";
74+
hasRemoteStateCharPrefix = randomBoolean();
75+
hasRemoteRoutingCharPrefix = randomBoolean();
6476
}
6577

6678
@Override
@@ -84,6 +96,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
8496
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
8597
routingTableRepoName
8698
);
99+
87100
return Settings.builder()
88101
.put(super.nodeSettings(nodeOrdinal))
89102
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), isRemoteStateEnabled)
@@ -94,6 +107,19 @@ protected Settings nodeSettings(int nodeOrdinal) {
94107
RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(),
95108
RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE
96109
)
110+
.put(
111+
RemoteClusterStateService.CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.getKey(),
112+
hasRemoteStateCharPrefix ? REMOTE_STATE_PREFIX : ""
113+
)
114+
.put(
115+
RemoteRoutingTableBlobStore.CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX.getKey(),
116+
hasRemoteRoutingCharPrefix ? REMOTE_ROUTING_PREFIX : ""
117+
)
118+
.put(RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX.toString())
119+
.put(
120+
RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING.getKey(),
121+
PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString()
122+
)
97123
.build();
98124
}
99125

@@ -137,6 +163,27 @@ public void testPublication() throws Exception {
137163
Map<String, Integer> manifestFiles = getMetadataFiles(repository, RemoteClusterMetadataManifest.MANIFEST);
138164
assertTrue(manifestFiles.containsKey(RemoteClusterMetadataManifest.MANIFEST));
139165

166+
RemoteClusterStateService remoteClusterStateService = internalCluster().getInstance(
167+
RemoteClusterStateService.class,
168+
internalCluster().getClusterManagerName()
169+
);
170+
ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
171+
getClusterState().getClusterName().value(),
172+
getClusterState().metadata().clusterUUID()
173+
).get();
174+
assertThat(manifest.getIndices().size(), is(1));
175+
if (hasRemoteStateCharPrefix) {
176+
for (UploadedIndexMetadata md : manifest.getIndices()) {
177+
assertThat(md.getUploadedFilename(), startsWith(REMOTE_STATE_PREFIX));
178+
}
179+
}
180+
assertThat(manifest.getIndicesRouting().size(), is(1));
181+
if (hasRemoteRoutingCharPrefix) {
182+
for (UploadedIndexMetadata md : manifest.getIndicesRouting()) {
183+
assertThat(md.getUploadedFilename(), startsWith(REMOTE_ROUTING_PREFIX));
184+
}
185+
}
186+
140187
// get settings from each node and verify that it is updated
141188
Settings settings = clusterService().getSettings();
142189
logger.info("settings : {}", settings);

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -328,10 +328,17 @@ public void testFullClusterRestoreManifestFilePointsToInvalidIndexMetadataPathTh
328328
internalCluster().stopAllNodes();
329329
// Step - 3 Delete index metadata file in remote
330330
try {
331-
Files.move(
332-
segmentRepoPath.resolve(encodeString(clusterName) + "/cluster-state/" + prevClusterUUID + "/index"),
333-
segmentRepoPath.resolve("cluster-state/")
331+
RemoteClusterStateService remoteClusterStateService = internalCluster().getInstance(
332+
RemoteClusterStateService.class,
333+
internalCluster().getClusterManagerName()
334334
);
335+
ClusterMetadataManifest manifest = remoteClusterStateService.getLatestClusterMetadataManifest(
336+
getClusterState().getClusterName().value(),
337+
getClusterState().metadata().clusterUUID()
338+
).get();
339+
for (UploadedIndexMetadata md : manifest.getIndices()) {
340+
Files.move(segmentRepoPath.resolve(md.getUploadedFilename()), segmentRepoPath.resolve("cluster-state/"));
341+
}
335342
} catch (IOException e) {
336343
throw new RuntimeException(e);
337344
}

‎server/src/main/java/org/opensearch/common/remote/RemoteWriteableBlobEntity.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,17 @@ public String[] getBlobPathTokens() {
6363

6464
public abstract String generateBlobFileName();
6565

66+
/**
67+
* Generate the blob path for the remote entity by adding a custom prefix.
68+
* This custom prefix may be generated by any of the strategies defined in {@link org.opensearch.index.remote.RemoteStoreEnums}
69+
* The default implementation returns the same path as passed in the argument.
70+
* @param blobPath The remote path on which the remote entity is to be uploaded
71+
* @return The modified remote path after adding a custom prefix at which the remote entity will be uploaded.
72+
*/
73+
public BlobPath getPrefixedPath(BlobPath blobPath) {
74+
return blobPath;
75+
}
76+
6677
public String clusterUUID() {
6778
return clusterUUID;
6879
}

‎server/src/main/java/org/opensearch/common/remote/RemoteWriteableEntityBlobStore.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public BlobPath getBlobPathForUpload(final RemoteWriteableBlobEntity<T> obj) {
102102
for (String token : obj.getBlobPathParameters().getPathTokens()) {
103103
blobPath = blobPath.add(token);
104104
}
105-
return blobPath;
105+
return obj.getPrefixedPath(blobPath);
106106
}
107107

108108
public BlobPath getBlobPathForDownload(final RemoteWriteableBlobEntity<T> obj) {

‎server/src/main/java/org/opensearch/common/settings/ClusterSettings.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
import org.opensearch.gateway.ShardsBatchGatewayAllocator;
110110
import org.opensearch.gateway.remote.RemoteClusterStateCleanupManager;
111111
import org.opensearch.gateway.remote.RemoteClusterStateService;
112+
import org.opensearch.gateway.remote.RemoteIndexMetadataManager;
112113
import org.opensearch.gateway.remote.model.RemoteRoutingTableBlobStore;
113114
import org.opensearch.http.HttpTransportSettings;
114115
import org.opensearch.index.IndexModule;
@@ -738,6 +739,9 @@ public void apply(Settings value, Settings current, Settings previous) {
738739
GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING,
739740
METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING,
740741
RemoteClusterStateService.REMOTE_STATE_READ_TIMEOUT_SETTING,
742+
RemoteClusterStateService.CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX,
743+
RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_TYPE_SETTING,
744+
RemoteIndexMetadataManager.REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING,
741745
RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
742746
RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING,
743747

@@ -746,6 +750,7 @@ public void apply(Settings value, Settings current, Settings previous) {
746750
RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_TYPE_SETTING,
747751
RemoteRoutingTableBlobStore.REMOTE_ROUTING_TABLE_PATH_HASH_ALGO_SETTING,
748752
RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING,
753+
RemoteRoutingTableBlobStore.CLUSTER_REMOTE_STORE_ROUTING_TABLE_PATH_PREFIX,
749754

750755
AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
751756
CpuBasedAdmissionControllerSettings.CPU_BASED_ADMISSION_CONTROLLER_TRANSPORT_LAYER_MODE,

‎server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,16 @@ public class RemoteClusterStateService implements Closeable {
150150
Setting.Property.NodeScope
151151
);
152152

153+
/**
154+
* Controls the fixed prefix for the cluster state path on remote store.
155+
*/
156+
public static final Setting<String> CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX = Setting.simpleString(
157+
"cluster.remote_store.state.path.prefix",
158+
"",
159+
Property.NodeScope,
160+
Property.Final
161+
);
162+
153163
/**
154164
* Validation mode for cluster state checksum.
155165
* None: Validation will be disabled.
@@ -211,6 +221,7 @@ public static RemoteClusterStateValidationMode parseString(String mode) {
211221
+ "indices, coordination metadata updated : [{}], settings metadata updated : [{}], templates metadata "
212222
+ "updated : [{}], custom metadata updated : [{}], indices routing updated : [{}]";
213223
private final boolean isPublicationEnabled;
224+
private final String remotePathPrefix;
214225

215226
// ToXContent Params with gateway mode.
216227
// We are using gateway context mode to persist all custom metadata.
@@ -252,6 +263,7 @@ public RemoteClusterStateService(
252263
this.isPublicationEnabled = FeatureFlags.isEnabled(REMOTE_PUBLICATION_EXPERIMENTAL)
253264
&& RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled(settings)
254265
&& RemoteStoreNodeAttribute.isRemoteRoutingTableEnabled(settings);
266+
this.remotePathPrefix = CLUSTER_REMOTE_STORE_STATE_PATH_PREFIX.get(settings);
255267
this.remoteRoutingTableService = RemoteRoutingTableServiceFactory.getService(
256268
repositoriesService,
257269
settings,
@@ -728,7 +740,10 @@ UploadedMetadataResults writeMetadataInParallel(
728740
indexMetadata,
729741
clusterState.metadata().clusterUUID(),
730742
blobStoreRepository.getCompressor(),
731-
blobStoreRepository.getNamedXContentRegistry()
743+
blobStoreRepository.getNamedXContentRegistry(),
744+
remoteIndexMetadataManager.getPathTypeSetting(),
745+
remoteIndexMetadataManager.getPathHashAlgoSetting(),
746+
remotePathPrefix
732747
),
733748
listener
734749
);

‎server/src/main/java/org/opensearch/gateway/remote/RemoteIndexMetadataManager.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.opensearch.core.xcontent.NamedXContentRegistry;
2121
import org.opensearch.gateway.remote.model.RemoteIndexMetadata;
2222
import org.opensearch.gateway.remote.model.RemoteReadResult;
23+
import org.opensearch.index.remote.RemoteStoreEnums;
2324
import org.opensearch.index.translog.transfer.BlobStoreTransferService;
2425
import org.opensearch.repositories.blobstore.BlobStoreRepository;
2526
import org.opensearch.threadpool.ThreadPool;
@@ -44,11 +45,38 @@ public class RemoteIndexMetadataManager extends AbstractRemoteWritableEntityMana
4445
Setting.Property.Deprecated
4546
);
4647

48+
/**
49+
* This setting is used to set the remote index metadata blob store path type strategy.
50+
*/
51+
public static final Setting<RemoteStoreEnums.PathType> REMOTE_INDEX_METADATA_PATH_TYPE_SETTING = new Setting<>(
52+
"cluster.remote_store.index_metadata.path_type",
53+
RemoteStoreEnums.PathType.HASHED_PREFIX.toString(),
54+
RemoteStoreEnums.PathType::parseString,
55+
Setting.Property.NodeScope,
56+
Setting.Property.Dynamic
57+
);
58+
59+
/**
60+
* This setting is used to set the remote index metadata blob store path hash algorithm strategy.
61+
* This setting will come to effect if the {@link #REMOTE_INDEX_METADATA_PATH_TYPE_SETTING}
62+
* is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
63+
*/
64+
public static final Setting<RemoteStoreEnums.PathHashAlgorithm> REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING = new Setting<>(
65+
"cluster.remote_store.index_metadata.path_hash_algo",
66+
RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64.toString(),
67+
RemoteStoreEnums.PathHashAlgorithm::parseString,
68+
Setting.Property.NodeScope,
69+
Setting.Property.Dynamic
70+
);
71+
4772
private final Compressor compressor;
4873
private final NamedXContentRegistry namedXContentRegistry;
4974

5075
private volatile TimeValue indexMetadataUploadTimeout;
5176

77+
private RemoteStoreEnums.PathType pathType;
78+
private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo;
79+
5280
public RemoteIndexMetadataManager(
5381
ClusterSettings clusterSettings,
5482
String clusterName,
@@ -70,7 +98,11 @@ public RemoteIndexMetadataManager(
7098
this.namedXContentRegistry = blobStoreRepository.getNamedXContentRegistry();
7199
this.compressor = blobStoreRepository.getCompressor();
72100
this.indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING);
101+
this.pathType = clusterSettings.get(REMOTE_INDEX_METADATA_PATH_TYPE_SETTING);
102+
this.pathHashAlgo = clusterSettings.get(REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING);
73103
clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout);
104+
clusterSettings.addSettingsUpdateConsumer(REMOTE_INDEX_METADATA_PATH_TYPE_SETTING, this::setPathTypeSetting);
105+
clusterSettings.addSettingsUpdateConsumer(REMOTE_INDEX_METADATA_PATH_HASH_ALGO_SETTING, this::setPathHashAlgoSetting);
74106
}
75107

76108
/**
@@ -127,4 +159,20 @@ protected ActionListener<Object> getWrappedReadListener(
127159
ex -> listener.onFailure(new RemoteStateTransferException("Download failed for " + component, remoteEntity, ex))
128160
);
129161
}
162+
163+
private void setPathTypeSetting(RemoteStoreEnums.PathType pathType) {
164+
this.pathType = pathType;
165+
}
166+
167+
private void setPathHashAlgoSetting(RemoteStoreEnums.PathHashAlgorithm pathHashAlgo) {
168+
this.pathHashAlgo = pathHashAlgo;
169+
}
170+
171+
protected RemoteStoreEnums.PathType getPathTypeSetting() {
172+
return pathType;
173+
}
174+
175+
protected RemoteStoreEnums.PathHashAlgorithm getPathHashAlgoSetting() {
176+
return pathHashAlgo;
177+
}
130178
}

‎server/src/main/java/org/opensearch/gateway/remote/model/RemoteIndexMetadata.java

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.opensearch.gateway.remote.model;
1010

1111
import org.opensearch.cluster.metadata.IndexMetadata;
12+
import org.opensearch.common.blobstore.BlobPath;
1213
import org.opensearch.common.io.Streams;
1314
import org.opensearch.common.remote.AbstractClusterMetadataWriteableBlobEntity;
1415
import org.opensearch.common.remote.BlobPathParameters;
@@ -17,6 +18,8 @@
1718
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
1819
import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedMetadata;
1920
import org.opensearch.gateway.remote.RemoteClusterStateUtils;
21+
import org.opensearch.index.remote.RemoteStoreEnums;
22+
import org.opensearch.index.remote.RemoteStorePathStrategy;
2023
import org.opensearch.index.remote.RemoteStoreUtils;
2124
import org.opensearch.repositories.blobstore.ChecksumBlobStoreFormat;
2225

@@ -41,15 +44,24 @@ public class RemoteIndexMetadata extends AbstractClusterMetadataWriteableBlobEnt
4144
public static final String INDEX = "index";
4245

4346
private IndexMetadata indexMetadata;
47+
private RemoteStoreEnums.PathType pathType;
48+
private RemoteStoreEnums.PathHashAlgorithm pathHashAlgo;
49+
private String fixedPrefix;
4450

4551
public RemoteIndexMetadata(
4652
final IndexMetadata indexMetadata,
4753
final String clusterUUID,
4854
final Compressor compressor,
49-
final NamedXContentRegistry namedXContentRegistry
55+
final NamedXContentRegistry namedXContentRegistry,
56+
final RemoteStoreEnums.PathType pathType,
57+
final RemoteStoreEnums.PathHashAlgorithm pathHashAlgo,
58+
final String fixedPrefix
5059
) {
5160
super(clusterUUID, compressor, namedXContentRegistry);
5261
this.indexMetadata = indexMetadata;
62+
this.pathType = pathType;
63+
this.pathHashAlgo = pathHashAlgo;
64+
this.fixedPrefix = fixedPrefix;
5365
}
5466

5567
public RemoteIndexMetadata(
@@ -86,6 +98,22 @@ public String generateBlobFileName() {
8698
return blobFileName;
8799
}
88100

101+
@Override
102+
public BlobPath getPrefixedPath(BlobPath blobPath) {
103+
if (pathType == null) {
104+
return blobPath;
105+
}
106+
assert pathHashAlgo != null;
107+
return pathType.path(
108+
RemoteStorePathStrategy.BasePathInput.builder()
109+
.fixedPrefix(fixedPrefix)
110+
.basePath(blobPath)
111+
.indexUUID(indexMetadata.getIndexUUID())
112+
.build(),
113+
pathHashAlgo
114+
);
115+
}
116+
89117
@Override
90118
public UploadedMetadata getUploadedMetadata() {
91119
assert blobName != null;

0 commit comments

Comments
 (0)