Examples with RecoveryRequest org.elasticsearch.action.admin.indices.recovery.RecoveryRequest used on opensource projects

Example 1 with RecoveryRequest

use of org.elasticsearch.action.admin.indices.recovery.RecoveryRequest in project crate by crate.

the class IndexRecoveryIT method testSnapshotRecovery.

@Test
public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create repository");
    execute("CREATE REPOSITORY " + REPO_NAME + " TYPE FS WITH (location = '" + randomRepoPath() + "', compress=false)");
    ensureGreen();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> snapshot");
    var snapshotName = REPO_NAME + "." + SNAP_NAME;
    execute("CREATE SNAPSHOT " + snapshotName + " ALL WITH (wait_for_completion=true)");
    execute("SELECT state FROM sys.snapshots WHERE name = '" + SNAP_NAME + "'");
    assertThat(response.rows()[0][0], is("SUCCESS"));
    execute("ALTER TABLE " + INDEX_NAME + " CLOSE");
    logger.info("--> restore");
    execute("RESTORE SNAPSHOT " + snapshotName + " ALL WITH (wait_for_completion=true)");
    ensureGreen();
    var snapshotInfo = client().execute(GetSnapshotsAction.INSTANCE, new GetSnapshotsRequest(REPO_NAME, new String[] { SNAP_NAME })).get().getSnapshots().get(0);
    logger.info("--> request recoveries");
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
        assertThat(indexRecoveryStates.getKey(), equalTo(indexName));
        List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
        assertThat(recoveryStates.size(), equalTo(SHARD_COUNT));
        for (RecoveryState recoveryState : recoveryStates) {
            RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource(((RecoverySource.SnapshotRecoverySource) recoveryState.getRecoverySource()).restoreUUID(), new Snapshot(REPO_NAME, snapshotInfo.snapshotId()), Version.CURRENT, indexName);
            assertRecoveryState(recoveryState, 0, recoverySource, true, RecoveryState.Stage.DONE, null, nodeA);
            validateIndexRecoveryState(recoveryState.getIndex());
        }
    }
}

Example 2 with RecoveryRequest

use of org.elasticsearch.action.admin.indices.recovery.RecoveryRequest in project crate by crate.

the class IndexRecoveryIT method testUsesFileBasedRecoveryIfRetentionLeaseMissing.

@Test
public void testUsesFileBasedRecoveryIfRetentionLeaseMissing() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    String indexName = "test";
    execute("CREATE TABLE doc.test (num INT)" + " CLUSTERED INTO 1 SHARDS" + " WITH (" + "  number_of_replicas = 1," + "  \"unassigned.node_left.delayed_timeout\"='12h'," + "  \"soft_deletes.enabled\"=true" + " )");
    int numDocs = randomIntBetween(1, 100);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO doc.test (num) VALUES (?)", args);
    ensureGreen(indexName);
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final DiscoveryNodes discoveryNodes = clusterService().state().nodes();
    final IndexShardRoutingTable indexShardRoutingTable = clusterService().state().routingTable().shardRoutingTable(shardId);
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, discoveryNodes.get(indexShardRoutingTable.primaryShard().currentNodeId()).getName()).getShardOrNull(shardId);
    final ShardRouting replicaShardRouting = indexShardRoutingTable.replicaShards().get(0);
    internalCluster().restartNode(discoveryNodes.get(replicaShardRouting.currentNodeId()).getName(), new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(discoveryNodes.getSize() - 1)).setWaitForEvents(Priority.LANGUID).get().isTimedOut());
            final PlainActionFuture<ReplicationResponse> future = new PlainActionFuture<>();
            primary.removeRetentionLease(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replicaShardRouting), future);
            future.get();
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen(indexName);
    // noinspection OptionalGetWithoutIsPresent because it fails the test if absent
    final var recoveryState = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).get().shardRecoveryStates().get(indexName).stream().filter(rs -> rs.getPrimary() == false).findFirst().get();
    assertThat(recoveryState.getIndex().totalFileCount(), greaterThan(0));
}

Example 3 with RecoveryRequest

use of org.elasticsearch.action.admin.indices.recovery.RecoveryRequest in project crate by crate.

the class IndexRecoveryIT method testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint.

@Test
public void testUsesFileBasedRecoveryIfRetentionLeaseAheadOfGlobalCheckpoint() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    String indexName = "test";
    execute("CREATE TABLE doc.test (num INT)" + " CLUSTERED INTO 1 SHARDS" + " WITH (" + "  number_of_replicas = 1," + "  \"unassigned.node_left.delayed_timeout\"='12h'," + "  \"soft_deletes.enabled\"=true" + " )");
    int numDocs = randomIntBetween(1, 100);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO doc.test (num) VALUES (?)", args);
    ensureGreen(indexName);
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final DiscoveryNodes discoveryNodes = clusterService().state().nodes();
    final IndexShardRoutingTable indexShardRoutingTable = clusterService().state().routingTable().shardRoutingTable(shardId);
    final IndexShard primary = internalCluster().getInstance(IndicesService.class, discoveryNodes.get(indexShardRoutingTable.primaryShard().currentNodeId()).getName()).getShardOrNull(shardId);
    final ShardRouting replicaShardRouting = indexShardRoutingTable.replicaShards().get(0);
    internalCluster().restartNode(discoveryNodes.get(replicaShardRouting.currentNodeId()).getName(), new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes(Integer.toString(discoveryNodes.getSize() - 1)).setWaitForEvents(Priority.LANGUID).get().isTimedOut());
            execute("INSERT INTO doc.test (num) VALUES (?)", args);
            // We do not guarantee that the replica can recover locally all the way to its own global checkpoint before starting
            // to recover from the primary, so we must be careful not to perform an operations-based recovery if this would require
            // some operations that are not being retained. Emulate this by advancing the lease ahead of the replica's GCP:
            primary.renewRetentionLease(ReplicationTracker.getPeerRecoveryRetentionLeaseId(replicaShardRouting), primary.seqNoStats().getMaxSeqNo() + 1, ReplicationTracker.PEER_RECOVERY_RETENTION_LEASE_SOURCE);
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen(indexName);
    // noinspection OptionalGetWithoutIsPresent because it fails the test if absent
    final var recoveryState = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).get().shardRecoveryStates().get(indexName).stream().filter(rs -> rs.getPrimary() == false).findFirst().get();
    assertThat(recoveryState.getIndex().totalFileCount(), greaterThan(0));
}

Example 4 with RecoveryRequest

use of org.elasticsearch.action.admin.indices.recovery.RecoveryRequest in project crate by crate.

the class IndexRecoveryIT method testGatewayRecovery.

@Test
public void testGatewayRecovery() throws Exception {
    logger.info("--> start nodes");
    String node = internalCluster().startNode();
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> restarting cluster");
    internalCluster().fullRestart();
    ensureGreen();
    logger.info("--> request recoveries");
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    assertThat(response.shardRecoveryStates().size(), equalTo(SHARD_COUNT));
    assertThat(response.shardRecoveryStates().get(indexName).size(), equalTo(1));
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(indexName);
    assertThat(recoveryStates.size(), equalTo(1));
    RecoveryState recoveryState = recoveryStates.get(0);
    assertRecoveryState(recoveryState, 0, RecoverySource.ExistingStoreRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, null, node);
    validateIndexRecoveryState(recoveryState.getIndex());
}

Example 5 with RecoveryRequest

use of org.elasticsearch.action.admin.indices.recovery.RecoveryRequest in project crate by crate.

the class IndexRecoveryIT method testReplicaRecovery.

@Test
public void testReplicaRecovery() throws Exception {
    final String nodeA = internalCluster().startNode();
    execute("CREATE TABLE " + INDEX_NAME + " (id BIGINT, data TEXT) " + " CLUSTERED INTO " + SHARD_COUNT + " SHARDS WITH (number_of_replicas=" + REPLICA_COUNT + ")");
    ensureGreen();
    final int numOfDocs = scaledRandomIntBetween(1, 200);
    try (BackgroundIndexer indexer = new BackgroundIndexer(IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null), "data", sqlExecutor.jdbcUrl(), numOfDocs, scaledRandomIntBetween(2, 5), true, null)) {
        waitForDocs(numOfDocs, indexer, sqlExecutor);
    }
    refresh();
    execute("SELECT COUNT(*) FROM " + INDEX_NAME);
    assertThat(response.rows()[0][0], is((long) numOfDocs));
    // We do not support ALTER on a closed table
    // final boolean closedIndex = randomBoolean();
    final boolean closedIndex = false;
    if (closedIndex) {
        execute("ALTER TABLE " + INDEX_NAME + " CLOSE");
        ensureGreen();
    }
    // force a shard recovery from nodeA to nodeB
    final String nodeB = internalCluster().startNode();
    execute("ALTER TABLE " + INDEX_NAME + " SET (number_of_replicas=1)");
    ensureGreen();
    // we should now have two total shards, one primary and one replica
    execute("SELECT * FROM sys.shards WHERE table_name = '" + INDEX_NAME + "'");
    assertThat(response.rowCount(), is(2L));
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    final RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    // we should now have two total shards, one primary and one replica
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(indexName);
    assertThat(recoveryStates.size(), equalTo(2));
    List<RecoveryState> nodeAResponses = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeAResponses.size(), equalTo(1));
    List<RecoveryState> nodeBResponses = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBResponses.size(), equalTo(1));
    // validate node A recovery
    final RecoveryState nodeARecoveryState = nodeAResponses.get(0);
    final RecoverySource expectedRecoverySource;
    if (closedIndex == false) {
        expectedRecoverySource = RecoverySource.EmptyStoreRecoverySource.INSTANCE;
    } else {
        expectedRecoverySource = RecoverySource.ExistingStoreRecoverySource.INSTANCE;
    }
    assertRecoveryState(nodeARecoveryState, 0, expectedRecoverySource, true, RecoveryState.Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryState.getIndex());
    // validate node B recovery
    final RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
    assertRecoveryState(nodeBRecoveryState, 0, RecoverySource.PeerRecoverySource.INSTANCE, false, RecoveryState.Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryState.getIndex());
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
    if (closedIndex) {
        execute("ALTER TABLE " + INDEX_NAME + " OPEN");
    }
    var res = execute("SELECT COUNT(*) FROM " + INDEX_NAME);
    assertThat(res.rows()[0][0], is((long) numOfDocs));
}