use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.
the class IndexRecoveryIT method testSnapshotRecovery.
public void testSnapshotRecovery() throws Exception {
logger.info("--> start node A");
String nodeA = internalCluster().startNode();
logger.info("--> create repository");
assertAcked(client().admin().cluster().preparePutRepository(REPO_NAME).setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", false)).get());
ensureGreen();
logger.info("--> create index on node: {}", nodeA);
createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
logger.info("--> snapshot");
CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).setIndices(INDEX_NAME).get();
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
assertThat(client().admin().cluster().prepareGetSnapshots(REPO_NAME).setSnapshots(SNAP_NAME).get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
client().admin().indices().prepareClose(INDEX_NAME).execute().actionGet();
logger.info("--> restore");
RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot(REPO_NAME, SNAP_NAME).setWaitForCompletion(true).execute().actionGet();
int totalShards = restoreSnapshotResponse.getRestoreInfo().totalShards();
assertThat(totalShards, greaterThan(0));
ensureGreen();
logger.info("--> request recoveries");
RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
Repository repository = internalCluster().getMasterNodeInstance(RepositoriesService.class).repository(REPO_NAME);
final RepositoryData repositoryData = PlainActionFuture.get(repository::getRepositoryData);
for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
assertThat(indexRecoveryStates.getKey(), equalTo(INDEX_NAME));
List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
assertThat(recoveryStates.size(), equalTo(totalShards));
for (RecoveryState recoveryState : recoveryStates) {
SnapshotRecoverySource recoverySource = new SnapshotRecoverySource(((SnapshotRecoverySource) recoveryState.getRecoverySource()).restoreUUID(), new Snapshot(REPO_NAME, createSnapshotResponse.getSnapshotInfo().snapshotId()), Version.CURRENT, repositoryData.resolveIndexId(INDEX_NAME));
assertRecoveryState(recoveryState, 0, recoverySource, true, Stage.DONE, null, nodeA);
validateIndexRecoveryState(recoveryState.getIndex());
}
}
}
use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.
the class ExceptionSerializationTests method testSnapshotException.
public void testSnapshotException() throws IOException {
final Snapshot snapshot = new Snapshot("repo", new SnapshotId("snap", UUIDs.randomBase64UUID()));
SnapshotException ex = serialize(new SnapshotException(snapshot, "no such snapshot", new NullPointerException()));
assertEquals(ex.getRepositoryName(), snapshot.getRepository());
assertEquals(ex.getSnapshotName(), snapshot.getSnapshotId().getName());
assertEquals(ex.getMessage(), "[" + snapshot + "] no such snapshot");
assertTrue(ex.getCause() instanceof NullPointerException);
ex = serialize(new SnapshotException(null, "no such snapshot", new NullPointerException()));
assertNull(ex.getRepositoryName());
assertNull(ex.getSnapshotName());
assertEquals(ex.getMessage(), "[_na] no such snapshot");
assertTrue(ex.getCause() instanceof NullPointerException);
}
use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.
the class SnapshotStatusTests method createTestInstance.
@Override
protected SnapshotStatus createTestInstance() {
SnapshotsInProgress.State state = randomFrom(SnapshotsInProgress.State.values());
String uuid = UUIDs.randomBase64UUID();
SnapshotId id = new SnapshotId("test-snap", uuid);
Snapshot snapshot = new Snapshot("test-repo", id);
SnapshotIndexShardStatusTests builder = new SnapshotIndexShardStatusTests();
builder.createTestInstance();
List<SnapshotIndexShardStatus> snapshotIndexShardStatuses = new ArrayList<>();
for (int idx = 0; idx < randomIntBetween(0, 10); idx++) {
SnapshotIndexShardStatus snapshotIndexShardStatus = builder.createTestInstance();
snapshotIndexShardStatuses.add(snapshotIndexShardStatus);
}
boolean includeGlobalState = randomBoolean();
return new SnapshotStatus(snapshot, state, snapshotIndexShardStatuses, includeGlobalState, 0L, 0L);
}
use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.
the class MetadataIndexStateServiceTests method addRestoredIndex.
private static ClusterState addRestoredIndex(final String index, final int numShards, final int numReplicas, final ClusterState state) {
ClusterState newState = addOpenedIndex(index, numShards, numReplicas, state);
final ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shardsBuilder = ImmutableOpenMap.builder();
for (ShardRouting shardRouting : newState.routingTable().index(index).randomAllActiveShardsIt()) {
shardsBuilder.put(shardRouting.shardId(), new RestoreInProgress.ShardRestoreStatus(shardRouting.currentNodeId()));
}
final Snapshot snapshot = new Snapshot(randomAlphaOfLength(10), new SnapshotId(randomAlphaOfLength(5), randomAlphaOfLength(5)));
final RestoreInProgress.Entry entry = new RestoreInProgress.Entry("_uuid", snapshot, RestoreInProgress.State.INIT, Collections.singletonList(index), shardsBuilder.build());
return ClusterState.builder(newState).putCustom(RestoreInProgress.TYPE, new RestoreInProgress.Builder().add(entry).build()).build();
}
use of org.opensearch.snapshots.Snapshot in project OpenSearch by opensearch-project.
the class DiskThresholdDeciderTests method testDiskThresholdWithSnapshotShardSizes.
public void testDiskThresholdWithSnapshotShardSizes() {
final long shardSizeInBytes = randomBoolean() ? 10L : 50L;
logger.info("--> using shard size [{}]", shardSizeInBytes);
final Settings diskSettings = Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_THRESHOLD_ENABLED_SETTING.getKey(), true).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), "90%").put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), "95%").build();
final ImmutableOpenMap.Builder<String, DiskUsage> usagesBuilder = ImmutableOpenMap.builder();
// 79% used
usagesBuilder.put("node1", new DiskUsage("node1", "n1", "/dev/null", 100, 21));
// 99% used
usagesBuilder.put("node2", new DiskUsage("node2", "n2", "/dev/null", 100, 1));
final ImmutableOpenMap<String, DiskUsage> usages = usagesBuilder.build();
final ClusterInfoService clusterInfoService = () -> new DevNullClusterInfo(usages, usages, ImmutableOpenMap.of());
final AllocationDeciders deciders = new AllocationDeciders(new HashSet<>(Arrays.asList(new RestoreInProgressAllocationDecider(), new SameShardAllocationDecider(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)), makeDecider(diskSettings))));
final Snapshot snapshot = new Snapshot("_repository", new SnapshotId("_snapshot_name", UUIDs.randomBase64UUID(random())));
final IndexId indexId = new IndexId("_indexid_name", UUIDs.randomBase64UUID(random()));
final ShardId shardId = new ShardId(new Index("test", IndexMetadata.INDEX_UUID_NA_VALUE), 0);
final Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0).putInSyncAllocationIds(0, singleton(AllocationId.newInitializing().getId()))).build();
final RoutingTable routingTable = RoutingTable.builder().addAsNewRestore(metadata.index("test"), new RecoverySource.SnapshotRecoverySource("_restore_uuid", snapshot, Version.CURRENT, indexId), new IntHashSet()).build();
final ImmutableOpenMap.Builder<ShardId, RestoreInProgress.ShardRestoreStatus> shards = ImmutableOpenMap.builder();
shards.put(shardId, new RestoreInProgress.ShardRestoreStatus("node1"));
final RestoreInProgress.Builder restores = new RestoreInProgress.Builder().add(new RestoreInProgress.Entry("_restore_uuid", snapshot, RestoreInProgress.State.INIT, singletonList("test"), shards.build()));
ClusterState clusterState = ClusterState.builder(new ClusterName(getTestName())).metadata(metadata).routingTable(routingTable).putCustom(RestoreInProgress.TYPE, restores.build()).nodes(// node2 is added because DiskThresholdDecider
DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo).allMatch(unassignedInfo -> Reason.NEW_INDEX_RESTORED.equals(unassignedInfo.getReason())), is(true));
assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo).allMatch(unassignedInfo -> AllocationStatus.NO_ATTEMPT.equals(unassignedInfo.getLastAllocationStatus())), is(true));
assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1));
final AtomicReference<SnapshotShardSizeInfo> snapshotShardSizeInfoRef = new AtomicReference<>(SnapshotShardSizeInfo.EMPTY);
final AllocationService strategy = new AllocationService(deciders, new TestGatewayAllocator(), new BalancedShardsAllocator(Settings.EMPTY), clusterInfoService, snapshotShardSizeInfoRef::get);
// reroute triggers snapshot shard size fetching
clusterState = strategy.reroute(clusterState, "reroute");
logShardStates(clusterState);
// shard cannot be assigned yet as the snapshot shard size is unknown
assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().map(ShardRouting::unassignedInfo).allMatch(unassignedInfo -> AllocationStatus.FETCHING_SHARD_DATA.equals(unassignedInfo.getLastAllocationStatus())), is(true));
assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(1));
final SnapshotShard snapshotShard = new SnapshotShard(snapshot, indexId, shardId);
final ImmutableOpenMap.Builder<SnapshotShard, Long> snapshotShardSizes = ImmutableOpenMap.builder();
final boolean shouldAllocate;
if (randomBoolean()) {
logger.info("--> simulating snapshot shards size retrieval success");
snapshotShardSizes.put(snapshotShard, shardSizeInBytes);
logger.info("--> shard allocation depends on its size");
shouldAllocate = shardSizeInBytes < usages.get("node1").getFreeBytes();
} else {
logger.info("--> simulating snapshot shards size retrieval failure");
snapshotShardSizes.put(snapshotShard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
logger.info("--> shard is always allocated when its size could not be retrieved");
shouldAllocate = true;
}
snapshotShardSizeInfoRef.set(new SnapshotShardSizeInfo(snapshotShardSizes.build()));
// reroute uses the previous snapshot shard size
clusterState = startInitializingShardsAndReroute(strategy, clusterState);
logShardStates(clusterState);
assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(shouldAllocate ? 0 : 1));
assertThat(clusterState.getRoutingNodes().shardsWithState("test", INITIALIZING, STARTED).size(), equalTo(shouldAllocate ? 1 : 0));
}
Aggregations