Skip to content

Commit bf2a70a

Browse files
rajiv-kvrajiv-kv
committed
Support prefix list for remote repository attributes (opensearch-project#16271)
* Support prefix list for remote repository attributes Signed-off-by: Rajiv Kumar Vaidyanathan <[email protected]>
1 parent 19b2e9a commit bf2a70a

File tree

17 files changed

+635
-392
lines changed

17 files changed

+635
-392
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
1414
- Make IndexStoreListener a pluggable interface ([#16583](https://github.com/opensearch-project/OpenSearch/pull/16583))
1515
- Add a flag in QueryShardContext to differentiate inner hit query ([#16600](https://github.com/opensearch-project/OpenSearch/pull/16600))
1616
- Add vertical scaling and SoftReference for snapshot repository data cache ([#16489](https://github.com/opensearch-project/OpenSearch/pull/16489))
17+
- Support prefix list for remote repository attributes([#16271](https://github.com/opensearch-project/OpenSearch/pull/16271))
1718
- Add new configuration setting `synonym_analyzer`, to the `synonym` and `synonym_graph` filters, enabling the specification of a custom analyzer for reading the synonym file ([#16488](https://github.com/opensearch-project/OpenSearch/pull/16488)).
1819
- Add stats for remote publication failure and move download failure stats to remote methods([#16682](https://github.com/opensearch-project/OpenSearch/pull/16682/))
1920

‎server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemotePublicationConfigurationIT.java

Lines changed: 2 additions & 272 deletions
Original file line numberDiff line numberDiff line change
@@ -8,34 +8,9 @@
88

99
package org.opensearch.gateway.remote;
1010

11-
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
12-
import org.opensearch.common.settings.Settings;
13-
import org.opensearch.plugins.Plugin;
14-
import org.opensearch.remotemigration.MigrationBaseTestCase;
15-
import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin;
16-
import org.opensearch.repositories.blobstore.BlobStoreRepository;
17-
import org.opensearch.repositories.fs.FsRepository;
18-
import org.opensearch.repositories.fs.ReloadableFsRepository;
19-
import org.opensearch.test.InternalSettingsPlugin;
2011
import org.opensearch.test.OpenSearchIntegTestCase;
21-
import org.opensearch.test.transport.MockTransportService;
22-
import org.junit.Assert;
2312
import org.junit.Before;
2413

25-
import java.util.Collection;
26-
import java.util.Locale;
27-
import java.util.stream.Collectors;
28-
import java.util.stream.Stream;
29-
30-
import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
31-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
32-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
33-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
34-
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY;
35-
import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
36-
import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
37-
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
38-
3914
/**
4015
* Tests the compatibility between types of nodes based on the configured repositories
4116
* Non Remote node [No Repositories configured]
@@ -44,260 +19,15 @@
4419
* Remote Node With Routing Table [Cluster State + Segment + Translog + Routing Table]
4520
*/
4621
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
47-
public class RemotePublicationConfigurationIT extends MigrationBaseTestCase {
48-
private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica";
49-
50-
@Override
51-
protected Collection<Class<? extends Plugin>> nodePlugins() {
52-
/* Adding the following mock plugins:
53-
- InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync
54-
- MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated
55-
*/
56-
return Stream.concat(
57-
super.nodePlugins().stream(),
58-
Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class)
59-
).collect(Collectors.toList());
60-
}
61-
22+
public class RemotePublicationConfigurationIT extends RemoteRepositoryConfigurationIT {
6223
@Before
6324
public void setUp() throws Exception {
6425
if (segmentRepoPath == null || translogRepoPath == null) {
6526
segmentRepoPath = randomRepoPath().toAbsolutePath();
6627
translogRepoPath = randomRepoPath().toAbsolutePath();
6728
}
29+
super.remoteRepoPrefix = "remote_publication";
6830
super.setUp();
6931
}
7032

71-
public Settings.Builder remotePublishConfiguredNodeSetting() {
72-
String stateRepoSettingsAttributeKeyPrefix = String.format(
73-
Locale.getDefault(),
74-
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
75-
REPOSITORY_NAME
76-
);
77-
String prefixModeVerificationSuffix = BlobStoreRepository.PREFIX_MODE_VERIFICATION_SETTING.getKey();
78-
String stateRepoTypeAttributeKey = String.format(
79-
Locale.getDefault(),
80-
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
81-
REPOSITORY_NAME
82-
);
83-
String routingTableRepoTypeAttributeKey = String.format(
84-
Locale.getDefault(),
85-
"node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
86-
ROUTING_TABLE_REPO_NAME
87-
);
88-
String routingTableRepoSettingsAttributeKeyPrefix = String.format(
89-
Locale.getDefault(),
90-
"node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
91-
ROUTING_TABLE_REPO_NAME
92-
);
93-
94-
Settings.Builder builder = Settings.builder()
95-
.put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME)
96-
.put(stateRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
97-
.put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
98-
.put(stateRepoSettingsAttributeKeyPrefix + prefixModeVerificationSuffix, true)
99-
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
100-
.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, ROUTING_TABLE_REPO_NAME)
101-
.put(routingTableRepoTypeAttributeKey, FsRepository.TYPE)
102-
.put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath);
103-
return builder;
104-
}
105-
106-
public Settings.Builder remoteWithRoutingTableNodeSetting() {
107-
// Remote Cluster with Routing table
108-
109-
return Settings.builder()
110-
.put(
111-
remoteStoreClusterSettings(
112-
REPOSITORY_NAME,
113-
segmentRepoPath,
114-
ReloadableFsRepository.TYPE,
115-
REPOSITORY_2_NAME,
116-
translogRepoPath,
117-
ReloadableFsRepository.TYPE,
118-
REPOSITORY_NAME,
119-
segmentRepoPath,
120-
ReloadableFsRepository.TYPE
121-
)
122-
)
123-
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true);
124-
}
125-
126-
public void testRemoteClusterStateServiceNotInitialized_WhenNodeAttributesNotPresent() {
127-
internalCluster().startClusterManagerOnlyNode();
128-
internalCluster().startDataOnlyNodes(2);
129-
130-
ensureStableCluster(3);
131-
ensureGreen();
132-
133-
internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNull);
134-
}
135-
136-
public void testServiceInitialized_WhenNodeAttributesPresent() {
137-
internalCluster().startClusterManagerOnlyNode(
138-
buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE)
139-
);
140-
internalCluster().startDataOnlyNodes(
141-
2,
142-
buildRemoteStateNodeAttributes(REPOSITORY_NAME, segmentRepoPath, ReloadableFsRepository.TYPE)
143-
);
144-
145-
ensureStableCluster(3);
146-
ensureGreen();
147-
148-
internalCluster().getDataOrClusterManagerNodeInstances(RemoteClusterStateService.class).forEach(Assert::assertNotNull);
149-
}
150-
151-
public void testRemotePublishConfigNodeJoinNonRemoteCluster() throws Exception {
152-
internalCluster().startClusterManagerOnlyNode();
153-
internalCluster().startDataOnlyNodes(2);
154-
155-
Settings.Builder build = remotePublishConfiguredNodeSetting();
156-
internalCluster().startClusterManagerOnlyNode(build.build());
157-
internalCluster().startDataOnlyNodes(2, build.build());
158-
159-
ensureStableCluster(6);
160-
ensureGreen();
161-
}
162-
163-
public void testRemotePublishConfigNodeJoinRemoteCluster() throws Exception {
164-
// Remote Cluster without Routing table
165-
setAddRemote(true);
166-
internalCluster().startClusterManagerOnlyNode();
167-
internalCluster().startDataOnlyNodes(2);
168-
setAddRemote(false);
169-
170-
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
171-
updateSettingsRequest.persistentSettings(
172-
Settings.builder()
173-
.put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
174-
.put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
175-
);
176-
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
177-
Settings.Builder build = remotePublishConfiguredNodeSetting();
178-
internalCluster().startClusterManagerOnlyNode(build.build());
179-
ensureStableCluster(4);
180-
ensureGreen();
181-
}
182-
183-
public void testRemoteNodeWithRoutingTableJoinRemoteCluster() throws Exception {
184-
setAddRemote(true);
185-
internalCluster().startClusterManagerOnlyNode();
186-
internalCluster().startDataOnlyNodes(2);
187-
setAddRemote(false);
188-
189-
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
190-
updateSettingsRequest.persistentSettings(
191-
Settings.builder()
192-
.put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
193-
.put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
194-
);
195-
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
196-
197-
// Remote Repo with Routing table
198-
Settings settings = remoteWithRoutingTableNodeSetting().build();
199-
internalCluster().startClusterManagerOnlyNode(settings);
200-
ensureStableCluster(4);
201-
ensureGreen();
202-
}
203-
204-
public void testNonRemoteNodeJoinRemoteWithRoutingCluster() throws Exception {
205-
Settings settings = remoteWithRoutingTableNodeSetting().build();
206-
internalCluster().startClusterManagerOnlyNode(settings);
207-
internalCluster().startDataOnlyNodes(2, settings);
208-
209-
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
210-
updateSettingsRequest.persistentSettings(
211-
Settings.builder()
212-
.put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
213-
.put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
214-
);
215-
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
216-
217-
internalCluster().startClusterManagerOnlyNode();
218-
ensureStableCluster(4);
219-
ensureGreen();
220-
}
221-
222-
public void testRemotePublishConfigNodeJoinRemoteWithRoutingCluster() throws Exception {
223-
Settings settings = remoteWithRoutingTableNodeSetting().build();
224-
internalCluster().startClusterManagerOnlyNode(settings);
225-
internalCluster().startDataOnlyNodes(2, settings);
226-
227-
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
228-
updateSettingsRequest.persistentSettings(
229-
Settings.builder()
230-
.put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
231-
.put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
232-
);
233-
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
234-
235-
internalCluster().startClusterManagerOnlyNode(remotePublishConfiguredNodeSetting().build());
236-
237-
ensureStableCluster(4);
238-
ensureGreen();
239-
}
240-
241-
public void testNonRemoteNodeJoiningPublishConfigCluster() throws Exception {
242-
Settings.Builder build = remotePublishConfiguredNodeSetting();
243-
internalCluster().startClusterManagerOnlyNode(build.build());
244-
internalCluster().startDataOnlyNodes(2, build.build());
245-
246-
internalCluster().startClusterManagerOnlyNode();
247-
248-
ensureStableCluster(4);
249-
ensureGreen();
250-
}
251-
252-
public void testRemoteNodeJoiningPublishConfigCluster() throws Exception {
253-
Settings.Builder build = remotePublishConfiguredNodeSetting();
254-
internalCluster().startClusterManagerOnlyNode(build.build());
255-
internalCluster().startDataOnlyNodes(2, build.build());
256-
257-
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
258-
updateSettingsRequest.persistentSettings(
259-
Settings.builder()
260-
.put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
261-
.put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
262-
);
263-
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
264-
265-
setAddRemote(true);
266-
internalCluster().startClusterManagerOnlyNode();
267-
ensureStableCluster(4);
268-
ensureGreen();
269-
}
270-
271-
public void testRemoteNodeWithRoutingTableJoiningPublishConfigCluster() throws Exception {
272-
Settings.Builder build = remotePublishConfiguredNodeSetting();
273-
internalCluster().startClusterManagerOnlyNode(build.build());
274-
internalCluster().startDataOnlyNodes(2, build.build());
275-
276-
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
277-
updateSettingsRequest.persistentSettings(
278-
Settings.builder()
279-
.put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
280-
.put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
281-
);
282-
283-
Settings settings = Settings.builder()
284-
.put(
285-
buildRemoteStoreNodeAttributes(
286-
REPOSITORY_NAME,
287-
segmentRepoPath,
288-
REPOSITORY_2_NAME,
289-
translogRepoPath,
290-
ROUTING_TABLE_REPO_NAME,
291-
segmentRepoPath,
292-
false
293-
)
294-
)
295-
.put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
296-
.build();
297-
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
298-
internalCluster().startClusterManagerOnlyNode(settings);
299-
300-
ensureStableCluster(4);
301-
ensureGreen();
302-
}
30333
}

0 commit comments

Comments
 (0)