Skip to content

Commit b0c0c7c

Browse files
ltaragiltaragi
committed
Snapshot Status API changes for V2 snapshots
Signed-off-by: Lakshya Taragi <[email protected]>
1 parent 80bf6cc commit b0c0c7c

29 files changed

+700
-85
lines changed

‎server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,11 @@
3232

3333
package org.opensearch.snapshots;
3434

35+
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
3536
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
3637
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage;
3738
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus;
39+
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus;
3840
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus;
3941
import org.opensearch.cluster.SnapshotsInProgress;
4042
import org.opensearch.common.action.ActionFuture;
@@ -44,8 +46,13 @@
4446
import org.junit.Before;
4547

4648
import java.nio.file.Path;
49+
import java.util.Map;
50+
import java.util.concurrent.TimeUnit;
4751

4852
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
53+
import static org.opensearch.repositories.blobstore.BlobStoreRepository.SNAPSHOT_V2;
54+
import static org.opensearch.snapshots.SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API;
55+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
4956
import static org.hamcrest.Matchers.equalTo;
5057
import static org.hamcrest.Matchers.greaterThan;
5158
import static org.hamcrest.Matchers.is;
@@ -192,6 +199,102 @@ public void testStatusAPICallInProgressShallowSnapshot() throws Exception {
192199
createSnapshotResponseActionFuture.actionGet();
193200
}
194201

202+
public void testStatusAPICallForShallowV2Snapshot() throws Exception {
203+
disableRepoConsistencyCheck("Remote store repository is being used for the test");
204+
internalCluster().startClusterManagerOnlyNode();
205+
internalCluster().startDataOnlyNodes(2);
206+
207+
logger.info("Create repository for shallow V2 snapshots");
208+
final String snapshotRepoName = "snapshot-repo-name";
209+
Settings.Builder snapshotV2RepoSettings = snapshotRepoSettingsForShallowCopy().put(SNAPSHOT_V2.getKey(), Boolean.TRUE);
210+
createRepository(snapshotRepoName, "fs", snapshotV2RepoSettings);
211+
212+
final String index1 = "remote-index-1";
213+
final String index2 = "remote-index-2";
214+
final String index3 = "remote-index-3";
215+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
216+
createIndex(index1, remoteStoreEnabledIndexSettings);
217+
createIndex(index2, remoteStoreEnabledIndexSettings);
218+
createIndex(index3, remoteStoreEnabledIndexSettings);
219+
ensureGreen();
220+
221+
logger.info("Indexing some data");
222+
for (int i = 0; i < 50; i++) {
223+
index(index1, "_doc", Integer.toString(i), "foo", "bar" + i);
224+
index(index2, "_doc", Integer.toString(i), "foo", "bar" + i);
225+
index(index3, "_doc", Integer.toString(i), "foo", "bar" + i);
226+
}
227+
refresh();
228+
229+
final String snapshot = "snapshot";
230+
SnapshotInfo snapshotInfo = createFullSnapshot(snapshotRepoName, snapshot);
231+
assertTrue(snapshotInfo.getPinnedTimestamp() > 0); // to assert creation of a shallow v2 snapshot
232+
233+
logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value");
234+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
235+
updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2));
236+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
237+
238+
// without index filter
239+
assertBusy(() -> {
240+
// although no. of shards in snapshot (3) is greater than the max value allowed in a status api call, the request does not fail
241+
SnapshotStatus snapshotStatusWithoutIndexFilter = client().admin()
242+
.cluster()
243+
.prepareSnapshotStatus(snapshotRepoName)
244+
.setSnapshots(snapshot)
245+
.execute()
246+
.actionGet()
247+
.getSnapshots()
248+
.get(0);
249+
250+
assertShallowV2SnapshotStatus(snapshotStatusWithoutIndexFilter, false);
251+
252+
SnapshotStatus snapshotStatusWithIndexFilter = client().admin()
253+
.cluster()
254+
.prepareSnapshotStatus(snapshotRepoName)
255+
.setSnapshots(snapshot)
256+
.setIndices(index1, index2)
257+
.execute()
258+
.actionGet()
259+
.getSnapshots()
260+
.get(0);
261+
262+
assertShallowV2SnapshotStatus(snapshotStatusWithIndexFilter, true);
263+
264+
}, 1, TimeUnit.MINUTES);
265+
266+
}
267+
268+
private void assertShallowV2SnapshotStatus(SnapshotStatus snapshotStatus, boolean hasIndexFilter) {
269+
if (hasIndexFilter) {
270+
assertEquals(0, snapshotStatus.getStats().getTotalSize());
271+
} else {
272+
assertTrue(snapshotStatus.getStats().getTotalSize() > 0);
273+
}
274+
// assert that total and incremental values of file count and size_in_bytes are 0 at index and shard levels
275+
assertEquals(0, snapshotStatus.getStats().getTotalFileCount());
276+
assertEquals(0, snapshotStatus.getStats().getIncrementalSize());
277+
assertEquals(0, snapshotStatus.getStats().getIncrementalFileCount());
278+
279+
for (Map.Entry<String, SnapshotIndexStatus> entry : snapshotStatus.getIndices().entrySet()) {
280+
// index level
281+
SnapshotIndexStatus snapshotIndexStatus = entry.getValue();
282+
assertEquals(0, snapshotIndexStatus.getStats().getTotalSize());
283+
assertEquals(0, snapshotIndexStatus.getStats().getTotalFileCount());
284+
assertEquals(0, snapshotIndexStatus.getStats().getIncrementalSize());
285+
assertEquals(0, snapshotIndexStatus.getStats().getIncrementalFileCount());
286+
287+
for (SnapshotIndexShardStatus snapshotIndexShardStatus : snapshotStatus.getShards()) {
288+
// shard level
289+
assertEquals(0, snapshotIndexShardStatus.getStats().getTotalSize());
290+
assertEquals(0, snapshotIndexShardStatus.getStats().getTotalFileCount());
291+
assertEquals(0, snapshotIndexShardStatus.getStats().getIncrementalSize());
292+
assertEquals(0, snapshotIndexShardStatus.getStats().getIncrementalFileCount());
293+
assertEquals(SnapshotIndexShardStage.DONE, snapshotIndexShardStatus.getStage());
294+
}
295+
}
296+
}
297+
195298
private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) {
196299
return snapshotStatus.getIndices().get(indexName).getShards().get(0);
197300
}

‎server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,14 @@
3333
package org.opensearch.snapshots;
3434

3535
import org.opensearch.Version;
36+
import org.opensearch.action.ActionRequestValidationException;
37+
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
3638
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
3739
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
3840
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
3941
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage;
4042
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus;
43+
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus;
4144
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats;
4245
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus;
4346
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
@@ -49,6 +52,7 @@
4952
import org.opensearch.common.util.io.IOUtils;
5053
import org.opensearch.core.common.Strings;
5154
import org.opensearch.core.common.unit.ByteSizeUnit;
55+
import org.opensearch.core.rest.RestStatus;
5256
import org.opensearch.repositories.blobstore.BlobStoreRepository;
5357
import org.opensearch.threadpool.ThreadPool;
5458

@@ -59,9 +63,12 @@
5963
import java.util.Collections;
6064
import java.util.List;
6165
import java.util.Locale;
66+
import java.util.Map;
67+
import java.util.Set;
6268
import java.util.concurrent.TimeUnit;
6369
import java.util.stream.Collectors;
6470

71+
import static org.opensearch.snapshots.SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API;
6572
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
6673
import static org.hamcrest.Matchers.equalTo;
6774
import static org.hamcrest.Matchers.greaterThan;
@@ -564,6 +571,192 @@ public void testGetSnapshotsRequest() throws Exception {
564571
waitForCompletion(repositoryName, inProgressSnapshot, TimeValue.timeValueSeconds(60));
565572
}
566573

574+
public void testSnapshotStatusApiFailureForTooManyShardsAcrossSnapshots() throws Exception {
575+
String repositoryName = "test-repo";
576+
String index1 = "test-idx-1";
577+
String index2 = "test-idx-2";
578+
String index3 = "test-idx-3";
579+
createRepository(repositoryName, "fs");
580+
581+
logger.info("Create indices");
582+
createIndex(index1, index2, index3);
583+
ensureGreen();
584+
585+
logger.info("Indexing some data");
586+
for (int i = 0; i < 10; i++) {
587+
index(index1, "_doc", Integer.toString(i), "foo", "bar" + i);
588+
index(index2, "_doc", Integer.toString(i), "foo", "baz" + i);
589+
index(index3, "_doc", Integer.toString(i), "foo", "baz" + i);
590+
}
591+
refresh();
592+
String snapshot1 = "test-snap-1";
593+
String snapshot2 = "test-snap-2";
594+
createSnapshot(repositoryName, snapshot1, List.of(index1, index2, index3));
595+
createSnapshot(repositoryName, snapshot2, List.of(index1, index2));
596+
597+
logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value");
598+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
599+
updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2));
600+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
601+
602+
// across a single snapshot
603+
assertBusy(() -> {
604+
SnapshotException snapshotException = expectThrows(
605+
SnapshotException.class,
606+
() -> client().admin().cluster().prepareSnapshotStatus(repositoryName).setSnapshots(snapshot1).execute().actionGet()
607+
);
608+
assertEquals(snapshotException.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE);
609+
assertTrue(
610+
snapshotException.getMessage()
611+
.endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request")
612+
);
613+
}, 1, TimeUnit.MINUTES);
614+
615+
// across multiple snapshots
616+
assertBusy(() -> {
617+
SnapshotException snapshotException = expectThrows(
618+
SnapshotException.class,
619+
() -> client().admin()
620+
.cluster()
621+
.prepareSnapshotStatus(repositoryName)
622+
.setSnapshots(snapshot1, snapshot2)
623+
.execute()
624+
.actionGet()
625+
);
626+
assertEquals(snapshotException.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE);
627+
assertTrue(
628+
snapshotException.getMessage()
629+
.endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request")
630+
);
631+
}, 1, TimeUnit.MINUTES);
632+
633+
logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value");
634+
updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey()));
635+
assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
636+
}
637+
638+
public void testSnapshotStatusForIndexFilter() throws Exception {
639+
String repositoryName = "test-repo";
640+
String index1 = "test-idx-1";
641+
String index2 = "test-idx-2";
642+
String index3 = "test-idx-3";
643+
createRepository(repositoryName, "fs");
644+
645+
logger.info("Create indices");
646+
createIndex(index1, index2, index3);
647+
ensureGreen();
648+
649+
logger.info("Indexing some data");
650+
for (int i = 0; i < 10; i++) {
651+
index(index1, "_doc", Integer.toString(i), "foo", "bar" + i);
652+
index(index2, "_doc", Integer.toString(i), "foo", "baz" + i);
653+
index(index3, "_doc", Integer.toString(i), "foo", "baz" + i);
654+
}
655+
refresh();
656+
String snapshot = "test-snap-1";
657+
createSnapshot(repositoryName, snapshot, List.of(index1, index2, index3));
658+
659+
assertBusy(() -> {
660+
SnapshotStatus snapshotsStatus = client().admin()
661+
.cluster()
662+
.prepareSnapshotStatus(repositoryName)
663+
.setSnapshots(snapshot)
664+
.setIndices(index1, index2)
665+
.get()
666+
.getSnapshots()
667+
.get(0);
668+
Map<String, SnapshotIndexStatus> snapshotIndexStatusMap = snapshotsStatus.getIndices();
669+
// Although the snapshot contains 3 indices, the response of status api call only contains results for 2
670+
assertEquals(snapshotIndexStatusMap.size(), 2);
671+
assertEquals(snapshotIndexStatusMap.keySet(), Set.of(index1, index2));
672+
}, 1, TimeUnit.MINUTES);
673+
}
674+
675+
public void testSnapshotStatusFailuresWithIndexFilter() throws Exception {
676+
String repositoryName = "test-repo";
677+
String index1 = "test-idx-1";
678+
String index2 = "test-idx-2";
679+
String index3 = "test-idx-3";
680+
createRepository(repositoryName, "fs");
681+
682+
logger.info("Create indices");
683+
createIndex(index1, index2, index3);
684+
ensureGreen();
685+
686+
logger.info("Indexing some data");
687+
for (int i = 0; i < 10; i++) {
688+
index(index1, "_doc", Integer.toString(i), "foo", "bar" + i);
689+
index(index2, "_doc", Integer.toString(i), "foo", "baz" + i);
690+
index(index3, "_doc", Integer.toString(i), "foo", "baz" + i);
691+
}
692+
refresh();
693+
String snapshot1 = "test-snap-1";
694+
String snapshot2 = "test-snap-2";
695+
createSnapshot(repositoryName, snapshot1, List.of(index1, index2, index3));
696+
createSnapshot(repositoryName, snapshot2, List.of(index1));
697+
698+
assertBusy(() -> {
699+
// failure due to passing index filter for multiple snapshots
700+
ActionRequestValidationException ex1 = expectThrows(
701+
ActionRequestValidationException.class,
702+
() -> client().admin()
703+
.cluster()
704+
.prepareSnapshotStatus(repositoryName)
705+
.setSnapshots(snapshot1, snapshot2)
706+
.setIndices(index1, index2, index3)
707+
.execute()
708+
.actionGet()
709+
);
710+
String cause = "index list filter is supported only for a single snapshot";
711+
assertTrue(ex1.getMessage().contains(cause));
712+
713+
// failure due to index not found in snapshot
714+
SnapshotException ex2 = expectThrows(
715+
SnapshotException.class,
716+
() -> client().admin()
717+
.cluster()
718+
.prepareSnapshotStatus(repositoryName)
719+
.setSnapshots(snapshot2)
720+
.setIndices(index1, index2, index3)
721+
.execute()
722+
.actionGet()
723+
);
724+
assertEquals(ex2.status(), RestStatus.NOT_FOUND);
725+
cause = String.format(
726+
"[%s:%s] indices [%s] missing in snapshot [%s]",
727+
repositoryName,
728+
snapshot2,
729+
String.join(", ", List.of(index2, index3)),
730+
snapshot2
731+
);
732+
assertEquals(cause, ex2.getMessage());
733+
734+
// failure due to too many shards requested
735+
logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value");
736+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
737+
updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2));
738+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
739+
740+
SnapshotException ex3 = expectThrows(
741+
SnapshotException.class,
742+
() -> client().admin()
743+
.cluster()
744+
.prepareSnapshotStatus(repositoryName)
745+
.setSnapshots(snapshot1)
746+
.setIndices(index1, index2, index3)
747+
.execute()
748+
.actionGet()
749+
);
750+
assertEquals(ex3.status(), RestStatus.REQUEST_ENTITY_TOO_LARGE);
751+
assertTrue(ex3.getMessage().endsWith(" is more than the maximum allowed value of shard count [2] for snapshot status request"));
752+
753+
logger.info("Reset MAX_SHARDS_ALLOWED_IN_STATUS_API to default value");
754+
updateSettingsRequest.persistentSettings(Settings.builder().putNull(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey()));
755+
assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
756+
757+
}, 1, TimeUnit.MINUTES);
758+
}
759+
567760
private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) {
568761
return snapshotStatus.getIndices().get(indexName).getShards().get(0);
569762
}

0 commit comments

Comments
 (0)