Skip to content

Commit 915ea10

Browse files
rajiv-kvrajiv-kv
committed
support list of prefixes for repository attribute keys
1 parent acf209f commit 915ea10

File tree

13 files changed

+368
-118
lines changed

13 files changed

+368
-118
lines changed

‎server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import org.opensearch.persistent.PersistentTasksCustomMetadata;
5555

5656
import java.util.ArrayList;
57+
import java.util.Arrays;
5758
import java.util.Collection;
5859
import java.util.Collections;
5960
import java.util.HashMap;
@@ -67,7 +68,8 @@
6768

6869
import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned;
6970
import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
70-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES;
71+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getClusterStateRepoName;
72+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.getRoutingTableRepoName;
7173
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
7274
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
7375
import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
@@ -517,7 +519,12 @@ private static void ensureRemoteClusterStateNodesCompatibility(DiscoveryNode joi
517519
.findFirst();
518520

519521
if (remotePublicationNode.isPresent() && joiningNode.isRemoteStatePublicationConfigured()) {
520-
ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), REMOTE_CLUSTER_PUBLICATION_REPO_NAME_ATTRIBUTES);
522+
List<String> repos = Arrays.asList(
523+
getClusterStateRepoName(remotePublicationNode.get().getAttributes()),
524+
getRoutingTableRepoName(remotePublicationNode.get().getAttributes())
525+
);
526+
527+
ensureRepositoryCompatibility(joiningNode, remotePublicationNode.get(), repos);
521528
}
522529
}
523530

@@ -546,16 +553,12 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod
546553
List<String> reposToSkip = new ArrayList<>(1);
547554
// find a remote node which has routing table configured
548555
Optional<DiscoveryNode> remoteRoutingTableNode = existingNodes.stream()
549-
.filter(
550-
node -> node.isRemoteStoreNode()
551-
&& node.getAttributes().get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY) != null
552-
)
556+
.filter(node -> node.isRemoteStoreNode() && RemoteStoreNodeAttribute.getRoutingTableRepoName(node.getAttributes()) != null)
553557
.findFirst();
554558
// If none of the existing nodes have routing table repo, then we skip this repo check if present in joining node.
555559
// This ensures a new node with remote routing table repo is able to join the cluster.
556560
if (remoteRoutingTableNode.isEmpty()) {
557-
String joiningNodeRepoName = joiningNode.getAttributes()
558-
.get(RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY);
561+
String joiningNodeRepoName = getRoutingTableRepoName(joiningNode.getAttributes());
559562
if (joiningNodeRepoName != null) {
560563
reposToSkip.add(joiningNodeRepoName);
561564
}

‎server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1166,12 +1166,9 @@ public static void updateRemoteStoreSettings(
11661166
.findFirst();
11671167

11681168
if (remoteNode.isPresent()) {
1169-
translogRepo = remoteNode.get()
1170-
.getAttributes()
1171-
.get(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY);
1172-
segmentRepo = remoteNode.get()
1173-
.getAttributes()
1174-
.get(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY);
1169+
translogRepo = RemoteStoreNodeAttribute.getTranslogRepoName(remoteNode.get().getAttributes());
1170+
1171+
segmentRepo = RemoteStoreNodeAttribute.getSegmentRepoName(remoteNode.get().getAttributes());
11751172
if (segmentRepo != null && translogRepo != null) {
11761173
settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true)
11771174
.put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo)

‎server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.opensearch.core.xcontent.ToXContentFragment;
4646
import org.opensearch.core.xcontent.XContentBuilder;
4747
import org.opensearch.node.Node;
48+
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
4849

4950
import java.io.IOException;
5051
import java.util.Collections;
@@ -62,10 +63,9 @@
6263
import java.util.stream.Stream;
6364

6465
import static org.opensearch.node.NodeRoleSettings.NODE_ROLES_SETTING;
65-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
6666
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX;
67-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
68-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
67+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isClusterStateRepoConfigured;
68+
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRoutingTableRepoConfigured;
6969

7070
/**
7171
* A discovery node represents a node that is part of the cluster.
@@ -510,20 +510,15 @@ public boolean isSearchNode() {
510510
* @return true if the node contains remote store node attributes, false otherwise
511511
*/
512512
public boolean isRemoteStoreNode() {
513-
return this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY))
514-
&& this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY));
513+
return isClusterStateRepoConfigured(this.getAttributes()) && RemoteStoreNodeAttribute.isSegmentRepoConfigured(this.getAttributes());
515514
}
516515

517516
/**
518517
* Returns whether settings required for remote cluster state publication is configured
519518
* @return true if the node contains remote cluster state node attribute and remote routing table node attribute
520519
*/
521520
public boolean isRemoteStatePublicationConfigured() {
522-
return this.getAttributes()
523-
.keySet()
524-
.stream()
525-
.anyMatch(key -> (key.equals(REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY)))
526-
&& this.getAttributes().keySet().stream().anyMatch(key -> key.equals(REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY));
521+
return isClusterStateRepoConfigured(this.getAttributes()) && isRoutingTableRepoConfigured(this.getAttributes());
527522
}
528523

529524
/**
@@ -587,13 +582,16 @@ public String toString() {
587582
sb.append('}');
588583
}
589584
if (!attributes.isEmpty()) {
590-
sb.append(
591-
attributes.entrySet()
592-
.stream()
593-
.filter(entry -> !entry.getKey().startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX)) // filter remote_store attributes
594-
// from logging to reduce noise.
595-
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
596-
);
585+
sb.append(attributes.entrySet().stream().filter(entry -> {
586+
for (String prefix : REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) {
587+
if (entry.getKey().startsWith(prefix)) {
588+
return false;
589+
}
590+
}
591+
return true;
592+
}) // filter remote_store attributes
593+
// from logging to reduce noise.
594+
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)));
597595
}
598596
return sb.toString();
599597
}

‎server/src/main/java/org/opensearch/cluster/routing/remote/InternalRemoteRoutingTableService.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333
import org.opensearch.gateway.remote.routingtable.RemoteIndexRoutingTable;
3434
import org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff;
3535
import org.opensearch.index.translog.transfer.BlobStoreTransferService;
36-
import org.opensearch.node.Node;
3736
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
3837
import org.opensearch.repositories.RepositoriesService;
3938
import org.opensearch.repositories.Repository;
@@ -235,9 +234,7 @@ protected void doClose() throws IOException {
235234
@Override
236235
protected void doStart() {
237236
assert isRemoteRoutingTableEnabled(settings) == true : "Remote routing table is not enabled";
238-
final String remoteStoreRepo = settings.get(
239-
Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY
240-
);
237+
final String remoteStoreRepo = RemoteStoreNodeAttribute.getRoutingTableRepoName(settings);
241238
assert remoteStoreRepo != null : "Remote routing table repository is not configured";
242239
final Repository repository = repositoriesService.get().repository(remoteStoreRepo);
243240
assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";

‎server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,6 @@
6161
import org.opensearch.gateway.remote.model.RemoteTransientSettingsMetadata;
6262
import org.opensearch.gateway.remote.routingtable.RemoteRoutingTableDiff;
6363
import org.opensearch.index.translog.transfer.BlobStoreTransferService;
64-
import org.opensearch.node.Node;
6564
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
6665
import org.opensearch.repositories.RepositoriesService;
6766
import org.opensearch.repositories.Repository;
@@ -1062,9 +1061,8 @@ public void close() throws IOException {
10621061

10631062
public void start() {
10641063
assert isRemoteClusterStateConfigured(settings) == true : "Remote cluster state is not enabled";
1065-
final String remoteStoreRepo = settings.get(
1066-
Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY
1067-
);
1064+
final String remoteStoreRepo = RemoteStoreNodeAttribute.getClusterStateRepoName(settings);
1065+
10681066
assert remoteStoreRepo != null : "Remote Cluster State repository is not configured";
10691067
final Repository repository = repositoriesService.get().repository(remoteStoreRepo);
10701068
assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";

‎server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import org.opensearch.gateway.remote.RemoteStateTransferException;
2727
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
2828
import org.opensearch.indices.RemoteStoreSettings;
29-
import org.opensearch.node.Node;
3029
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
3130
import org.opensearch.repositories.RepositoriesService;
3231
import org.opensearch.repositories.Repository;
@@ -70,11 +69,6 @@ public class RemoteIndexPathUploader extends IndexMetadataUploadListener {
7069

7170
private static final String TIMEOUT_EXCEPTION_MSG = "Timed out waiting while uploading remote index path file for indexes=%s";
7271
private static final String UPLOAD_EXCEPTION_MSG = "Exception occurred while uploading remote index paths for indexes=%s";
73-
static final String TRANSLOG_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey()
74-
+ RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
75-
static final String SEGMENT_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey()
76-
+ RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
77-
7872
private static final Logger logger = LogManager.getLogger(RemoteIndexPathUploader.class);
7973

8074
private final Settings settings;
@@ -226,9 +220,8 @@ private void writePathToRemoteStore(
226220
}
227221
}
228222

229-
private Repository validateAndGetRepository(String repoSetting) {
230-
final String repo = settings.get(repoSetting);
231-
assert repo != null : "Remote " + repoSetting + " repository is not configured";
223+
private Repository validateAndGetRepository(String repo) {
224+
assert repo != null : "Remote repository is not configured";
232225
final Repository repository = repositoriesService.get().repository(repo);
233226
assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";
234227
return repository;
@@ -240,15 +233,16 @@ public void start() {
240233
// If remote store data attributes are not present than we skip this.
241234
return;
242235
}
243-
translogRepository = (BlobStoreRepository) validateAndGetRepository(TRANSLOG_REPO_NAME_KEY);
244-
segmentRepository = (BlobStoreRepository) validateAndGetRepository(SEGMENT_REPO_NAME_KEY);
236+
237+
translogRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings));
238+
segmentRepository = (BlobStoreRepository) validateAndGetRepository(RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings));
245239
}
246240

247241
private boolean isTranslogSegmentRepoSame() {
248242
// TODO - The current comparison checks the repository name. But it is also possible that the repository are same
249243
// by attributes, but different by name. We need to handle this.
250-
String translogRepoName = settings.get(TRANSLOG_REPO_NAME_KEY);
251-
String segmentRepoName = settings.get(SEGMENT_REPO_NAME_KEY);
244+
String translogRepoName = RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(settings);
245+
String segmentRepoName = RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(settings);
252246
return Objects.equals(translogRepoName, segmentRepoName);
253247
}
254248

‎server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import org.opensearch.common.settings.Settings;
1919
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
2020
import org.opensearch.indices.replication.common.ReplicationType;
21+
import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
2122

2223
import java.util.List;
2324
import java.util.Map;
@@ -30,8 +31,6 @@
3031
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
3132
import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStoreCustomMetadataDuringMigration;
3233
import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName;
33-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
34-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
3534

3635
/**
3736
* Utils for checking and mutating cluster state during remote migration
@@ -74,8 +73,9 @@ public void maybeAddRemoteIndexSettings(IndexMetadata.Builder indexMetadataBuild
7473
index
7574
);
7675
Map<String, String> remoteRepoNames = getRemoteStoreRepoName(discoveryNodes);
77-
String segmentRepoName = remoteRepoNames.get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY);
78-
String tlogRepoName = remoteRepoNames.get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY);
76+
String segmentRepoName = RemoteStoreNodeAttribute.getSegmentRepoName(remoteRepoNames);
77+
String tlogRepoName = RemoteStoreNodeAttribute.getTranslogRepoName(remoteRepoNames);
78+
7979
assert Objects.nonNull(segmentRepoName) && Objects.nonNull(tlogRepoName) : "Remote repo names cannot be null";
8080
Settings.Builder indexSettingsBuilder = Settings.builder().put(currentIndexSettings);
8181
updateRemoteStoreSettings(indexSettingsBuilder, segmentRepoName, tlogRepoName);

0 commit comments

Comments
 (0)