use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.
the class RemoveCorruptedShardDataCommandIT method testCorruptTranslogTruncationOfReplica.
public void testCorruptTranslogTruncationOfReplica() throws Exception {
internalCluster().startMasterOnlyNode();
final String node1 = internalCluster().startDataOnlyNode();
final String node2 = internalCluster().startDataOnlyNode();
logger.info("--> nodes name: {}, {}", node1, node2);
final String indexName = "test";
assertAcked(prepareCreate(indexName).setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1").put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), // never flush - always recover from translog
true).put("index.routing.allocation.exclude._name", node2)));
ensureYellow();
assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder().put("index.routing.allocation.exclude._name", (String) null)));
ensureGreen();
// Index some documents
int numDocsToKeep = randomIntBetween(0, 100);
logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName).setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
flush(indexName);
disableTranslogFlush(indexName);
// having no extra docs is an interesting case for seq no based recoveries - test it more often
int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
logger.info("--> indexing [{}] more docs to be truncated", numDocsToTruncate);
builders = new IndexRequestBuilder[numDocsToTruncate];
for (int i = 0; i < builders.length; i++) {
builders[i] = client().prepareIndex(indexName).setSource("foo", "bar");
}
indexRandom(false, false, false, Arrays.asList(builders));
final int totalDocs = numDocsToKeep + numDocsToTruncate;
// sample the replica node translog dirs
final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
final Path translogDir = getPathToShardData(node2, shardId, ShardPath.TRANSLOG_FOLDER_NAME);
final Settings node1PathSettings = internalCluster().dataPathSettings(node1);
final Settings node2PathSettings = internalCluster().dataPathSettings(node2);
assertBusy(() -> internalCluster().getInstances(GatewayMetaState.class).forEach(gw -> assertTrue(gw.allPendingAsyncStatesWritten())));
// stop data nodes
internalCluster().stopRandomDataNode();
internalCluster().stopRandomDataNode();
// Corrupt the translog file(s) on the replica
logger.info("--> corrupting translog");
TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir);
// Start the node with the non-corrupted data path
logger.info("--> starting node");
internalCluster().startNode(node1PathSettings);
ensureYellow();
// Run a search and make sure it succeeds
assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), totalDocs);
// check replica corruption
final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
final MockTerminal terminal = new MockTerminal();
final OptionParser parser = command.getParser();
final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put(internalCluster().getDefaultSettings()).put(node2PathSettings).build());
terminal.addTextInput("y");
OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString());
logger.info("--> running command for [{}]", translogDir.toAbsolutePath());
command.execute(terminal, options, environment);
logger.info("--> output:\n{}", terminal.getOutput());
logger.info("--> starting the replica node to test recovery");
internalCluster().startNode(node2PathSettings);
ensureGreen(indexName);
for (String node : internalCluster().nodesInclude(indexName)) {
assertHitCount(client().prepareSearch(indexName).setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs);
}
final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).setActiveOnly(false).get();
final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get(indexName).stream().filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
// the replica translog was disabled so it doesn't know what hte global checkpoint is and thus can't do ops based recovery
assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
// Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
final SeqNoStats seqNoStats = getSeqNoStats(indexName, 0);
assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
}
use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.
the class ReplicaShardAllocatorIT method assertNoOpRecoveries.
private void assertNoOpRecoveries(String indexName) {
for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
if (recovery.getPrimary() == false) {
assertThat(recovery.getIndex().fileDetails(), empty());
assertThat(recovery.getTranslog().totalLocal(), equalTo(recovery.getTranslog().totalOperations()));
}
}
}
use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.
the class ReplicaShardAllocatorIT method testRecentPrimaryInformation.
/**
* Ensure that we fetch the latest shard store from the primary when a new node joins so we won't cancel the current recovery
* for the copy on the newly joined node unless we can perform a noop recovery with that node.
*/
public void testRecentPrimaryInformation() throws Exception {
String indexName = "test";
String nodeWithPrimary = internalCluster().startNode();
assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 0.1f).put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms").put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1ms")));
String nodeWithReplica = internalCluster().startDataOnlyNode();
DiscoveryNode discoNodeWithReplica = internalCluster().getInstance(ClusterService.class, nodeWithReplica).localNode();
Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica);
ensureGreen(indexName);
indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithReplica));
if (randomBoolean()) {
indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
}
CountDownLatch blockRecovery = new CountDownLatch(1);
CountDownLatch recoveryStarted = new CountDownLatch(1);
MockTransportService transportServiceOnPrimary = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeWithPrimary);
transportServiceOnPrimary.addSendBehavior((connection, requestId, action, request, options) -> {
if (PeerRecoveryTargetService.Actions.FILES_INFO.equals(action)) {
recoveryStarted.countDown();
try {
blockRecovery.await();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
connection.sendRequest(requestId, action, request, options);
});
String newNode = internalCluster().startDataOnlyNode();
recoveryStarted.await();
// Index more documents and flush to destroy sync_id and remove the retention lease (as file_based_recovery_threshold reached).
indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, between(50, 200)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
client().admin().indices().prepareFlush(indexName).get();
assertBusy(() -> {
for (ShardStats shardStats : client().admin().indices().prepareStats(indexName).get().getShards()) {
for (RetentionLease lease : shardStats.getRetentionLeaseStats().retentionLeases().leases()) {
assertThat(lease.id(), not(equalTo(ReplicationTracker.getPeerRecoveryRetentionLeaseId(discoNodeWithReplica.getId()))));
}
}
});
// AllocationService only calls GatewayAllocator if there are unassigned shards
assertAcked(client().admin().indices().prepareCreate("dummy-index").setWaitForActiveShards(0).setSettings(Settings.builder().put("index.routing.allocation.require.attr", "not-found")));
internalCluster().startDataOnlyNode(nodeWithReplicaSettings);
// need to wait for events to ensure the reroute has happened since we perform it async when a new node joins.
client().admin().cluster().prepareHealth(indexName).setWaitForYellowStatus().setWaitForEvents(Priority.LANGUID).get();
blockRecovery.countDown();
ensureGreen(indexName);
assertThat(internalCluster().nodesInclude(indexName), hasItem(newNode));
for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
if (recovery.getPrimary() == false) {
assertThat(recovery.getIndex().fileDetails(), not(empty()));
}
}
transportServiceOnPrimary.clearAllRules();
}
use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.
the class AbortedRestoreIT method testAbortedRestoreAlsoAbortFileRestores.
public void testAbortedRestoreAlsoAbortFileRestores() throws Exception {
internalCluster().startMasterOnlyNode();
final String dataNode = internalCluster().startDataOnlyNode();
final String indexName = "test-abort-restore";
createIndex(indexName, indexSettingsNoReplicas(1).build());
indexRandomDocs(indexName, scaledRandomIntBetween(10, 1_000));
ensureGreen();
forceMerge();
final String repositoryName = "repository";
createRepository(repositoryName, "mock");
final String snapshotName = "snapshot";
createFullSnapshot(repositoryName, snapshotName);
assertAcked(client().admin().indices().prepareDelete(indexName));
logger.info("--> blocking all data nodes for repository [{}]", repositoryName);
blockAllDataNodes(repositoryName);
failReadsAllDataNodes(repositoryName);
logger.info("--> starting restore");
final ActionFuture<RestoreSnapshotResponse> future = client().admin().cluster().prepareRestoreSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(indexName).execute();
assertBusy(() -> {
final RecoveryResponse recoveries = client().admin().indices().prepareRecoveries(indexName).setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN).setActiveOnly(true).get();
assertThat(recoveries.hasRecoveries(), is(true));
final List<RecoveryState> shardRecoveries = recoveries.shardRecoveryStates().get(indexName);
assertThat(shardRecoveries, hasSize(1));
assertThat(future.isDone(), is(false));
for (RecoveryState shardRecovery : shardRecoveries) {
assertThat(shardRecovery.getRecoverySource().getType(), equalTo(RecoverySource.Type.SNAPSHOT));
assertThat(shardRecovery.getStage(), equalTo(RecoveryState.Stage.INDEX));
}
});
final ThreadPool.Info snapshotThreadPoolInfo = threadPool(dataNode).info(ThreadPool.Names.SNAPSHOT);
assertThat(snapshotThreadPoolInfo.getMax(), greaterThan(0));
logger.info("--> waiting for snapshot thread [max={}] pool to be full", snapshotThreadPoolInfo.getMax());
waitForMaxActiveSnapshotThreads(dataNode, equalTo(snapshotThreadPoolInfo.getMax()));
logger.info("--> aborting restore by deleting the index");
assertAcked(client().admin().indices().prepareDelete(indexName));
logger.info("--> unblocking repository [{}]", repositoryName);
unblockAllDataNodes(repositoryName);
logger.info("--> restore should have failed");
final RestoreSnapshotResponse restoreSnapshotResponse = future.get();
assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(1));
assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(0));
logger.info("--> waiting for snapshot thread pool to be empty");
waitForMaxActiveSnapshotThreads(dataNode, equalTo(0));
}
use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.
the class CloseIndexIT method testRecoverExistingReplica.
/**
* Ensures that if a replica of a closed index does not have the same content as the primary, then a file-based recovery will occur.
*/
public void testRecoverExistingReplica() throws Exception {
final String indexName = "test-recover-existing-replica";
internalCluster().ensureAtLeastNumDataNodes(2);
List<String> dataNodes = randomSubsetOf(2, Sets.newHashSet(clusterService().state().nodes().getDataNodes().valuesIt()).stream().map(DiscoveryNode::getName).collect(Collectors.toSet()));
createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put("index.routing.allocation.include._name", String.join(",", dataNodes)).build());
indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, randomIntBetween(0, 50)).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
ensureGreen(indexName);
client().admin().indices().prepareFlush(indexName).get();
// index more documents while one shard copy is offline
internalCluster().restartNode(dataNodes.get(1), new InternalTestCluster.RestartCallback() {
@Override
public Settings onNodeStopped(String nodeName) throws Exception {
Client client = client(dataNodes.get(0));
int moreDocs = randomIntBetween(1, 50);
for (int i = 0; i < moreDocs; i++) {
client.prepareIndex(indexName).setSource("num", i).get();
}
assertAcked(client.admin().indices().prepareClose(indexName));
return super.onNodeStopped(nodeName);
}
});
assertIndexIsClosed(indexName);
ensureGreen(indexName);
internalCluster().assertSameDocIdsOnShards();
for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
if (recovery.getPrimary() == false) {
assertThat(recovery.getIndex().fileDetails(), not(empty()));
}
}
}
Aggregations