Examples with RecoveryResponse org.elasticsearch.action.admin.indices.recovery.RecoveryResponse used on opensource projects

Example 16 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class IndexRecoveryIT method testReplicaRecovery.

public void testReplicaRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> start node B");
    String nodeB = internalCluster().startNode();
    ensureGreen();
    // force a shard recovery from nodeA to nodeB
    logger.info("--> bump replica count");
    client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet();
    ensureGreen();
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    // we should now have two total shards, one primary and one replica
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(2));
    List<RecoveryState> nodeAResponses = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeAResponses.size(), equalTo(1));
    List<RecoveryState> nodeBResponses = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBResponses.size(), equalTo(1));
    // validate node A recovery
    RecoveryState nodeARecoveryState = nodeAResponses.get(0);
    assertRecoveryState(nodeARecoveryState, 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryState.getIndex());
    // validate node B recovery
    RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
    assertRecoveryState(nodeBRecoveryState, 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryState.getIndex());
}

Example 17 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class IndexRecoveryIT method testGatewayRecovery.

public void testGatewayRecovery() throws Exception {
    logger.info("--> start nodes");
    String node = internalCluster().startNode();
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> restarting cluster");
    internalCluster().fullRestart();
    ensureGreen();
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    assertThat(response.shardRecoveryStates().size(), equalTo(SHARD_COUNT));
    assertThat(response.shardRecoveryStates().get(INDEX_NAME).size(), equalTo(1));
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));
    RecoveryState recoveryState = recoveryStates.get(0);
    assertRecoveryState(recoveryState, 0, StoreRecoverySource.EXISTING_STORE_INSTANCE, true, Stage.DONE, null, node);
    validateIndexRecoveryState(recoveryState.getIndex());
}

Example 18 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class IndexRecoveryIT method testRerouteRecovery.

public void testRerouteRecovery() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    ByteSizeValue shardSize = createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT).getShards()[0].getStats().getStore().size();
    logger.info("--> start node B");
    final String nodeB = internalCluster().startNode();
    ensureGreen();
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeB)).execute().actionGet().getState();
    logger.info("--> waiting for recovery to start both on source and target");
    final Index index = resolveIndex(INDEX_NAME);
    assertBusy(new Runnable() {

        @Override
        public void run() {
            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
            indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
        }
    });
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    logger.info("--> request node recovery stats");
    NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    long nodeAThrottling = Long.MAX_VALUE;
    long nodeBThrottling = Long.MAX_VALUE;
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A should have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(1));
            assertThat("node A should not have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(0));
            nodeAThrottling = recoveryStats.throttleTime().millis();
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B should not have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(0));
            assertThat("node B should have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(1));
            nodeBThrottling = recoveryStats.throttleTime().millis();
        }
    }
    logger.info("--> checking throttling increases");
    final long finalNodeAThrottling = nodeAThrottling;
    final long finalNodeBThrottling = nodeBThrottling;
    assertBusy(new Runnable() {

        @Override
        public void run() {
            NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
            assertThat(statsResponse.getNodes(), hasSize(2));
            for (NodeStats nodeStats : statsResponse.getNodes()) {
                final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
                if (nodeStats.getNode().getName().equals(nodeA)) {
                    assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
                }
                if (nodeStats.getNode().getName().equals(nodeB)) {
                    assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
                }
            }
        }
    });
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    // wait for it to be finished
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));
    assertRecoveryState(recoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(recoveryStates.get(0).getIndex());
    statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B throttling should be >0 ", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
    }
    logger.info("--> bump replica count");
    client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet();
    ensureGreen();
    statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B throttling should be >0 ", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
    }
    logger.info("--> start node C");
    String nodeC = internalCluster().startNode();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut());
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move replica shard from: {} to: {}", nodeA, nodeC);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeC)).execute().actionGet().getState();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    if (randomBoolean()) {
        // shutdown node with relocation source of replica shard and check if recovery continues
        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
        ensureStableCluster(2);
        response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
        recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
        nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
        assertThat(nodeARecoveryStates.size(), equalTo(0));
        nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
        assertThat(nodeBRecoveryStates.size(), equalTo(1));
        nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
        assertThat(nodeCRecoveryStates.size(), equalTo(1));
        assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
        validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
        validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    }
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(0));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
}

Example 19 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project crate by crate.

the class IndexRecoveryIT method testCancelNewShardRecoveryAndUsesExistingShardCopy.

@Test
public void testCancelNewShardRecoveryAndUsesExistingShardCopy() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> start node B");
    // force a shard recovery from nodeA to nodeB
    final String nodeB = internalCluster().startNode();
    logger.info("--> add replica for {} on node: {}", INDEX_NAME, nodeB);
    execute("ALTER TABLE " + INDEX_NAME + " SET (number_of_replicas=1, \"unassigned.node_left.delayed_timeout\"=0)");
    ensureGreen();
    logger.info("--> start node C");
    final String nodeC = internalCluster().startNode();
    // do sync flush to gen sync id
    execute("OPTIMIZE TABLE " + INDEX_NAME);
    // assertThat(client().admin().indices().prepareSyncedFlush(INDEX_NAME).get().failedShards(), equalTo(0));
    // hold peer recovery on phase 2 after nodeB down
    CountDownLatch phase1ReadyBlocked = new CountDownLatch(1);
    CountDownLatch allowToCompletePhase1Latch = new CountDownLatch(1);
    MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeA);
    transportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (PeerRecoveryTargetService.Actions.CLEAN_FILES.equals(action)) {
            phase1ReadyBlocked.countDown();
            try {
                allowToCompletePhase1Latch.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    logger.info("--> restart node B");
    internalCluster().restartNode(nodeB, new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            phase1ReadyBlocked.await();
            // nodeB stopped, peer recovery from nodeA to nodeC, it will be cancelled after nodeB get started.
            var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
            RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
            List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(indexName);
            List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
            assertThat(nodeCRecoveryStates.size(), equalTo(1));
            assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, false, nodeA, nodeC);
            validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
            return super.onNodeStopped(nodeName);
        }
    });
    // wait for peer recovery from nodeA to nodeB which is a no-op recovery so it skips the CLEAN_FILES stage and hence is not blocked
    ensureGreen();
    allowToCompletePhase1Latch.countDown();
    transportService.clearAllRules();
    // make sure nodeA has primary and nodeB has replica
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    List<ShardRouting> startedShards = state.routingTable().shardsWithState(ShardRoutingState.STARTED);
    assertThat(startedShards.size(), equalTo(2));
    for (ShardRouting shardRouting : startedShards) {
        if (shardRouting.primary()) {
            assertThat(state.nodes().get(shardRouting.currentNodeId()).getName(), equalTo(nodeA));
        } else {
            assertThat(state.nodes().get(shardRouting.currentNodeId()).getName(), equalTo(nodeB));
        }
    }
}

Example 20 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project crate by crate.

the class IndexRecoveryIT method testHistoryRetention.

@Test
public void testHistoryRetention() throws Exception {
    internalCluster().startNodes(3);
    final String indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), "test", null);
    execute("CREATE TABLE test (id int) CLUSTERED INTO 1 SHARDS " + "WITH (" + " number_of_replicas=2," + " \"recovery.file_based_threshold\" = 1.0" + ")");
    // Perform some replicated operations so the replica isn't simply empty, because ops-based recovery isn't better in that case
    int numDocs = scaledRandomIntBetween(25, 250);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO test (id) VALUES (?)", args);
    if (randomBoolean()) {
        execute("OPTIMIZE TABLE test");
    }
    ensureGreen();
    String firstNodeToStop = randomFrom(internalCluster().getNodeNames());
    Settings firstNodeToStopDataPathSettings = internalCluster().dataPathSettings(firstNodeToStop);
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(firstNodeToStop));
    String secondNodeToStop = randomFrom(internalCluster().getNodeNames());
    Settings secondNodeToStopDataPathSettings = internalCluster().dataPathSettings(secondNodeToStop);
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(secondNodeToStop));
    final long desyncNanoTime = System.nanoTime();
    // noinspection StatementWithEmptyBody
    while (System.nanoTime() <= desyncNanoTime) {
    // time passes
    }
    final int numNewDocs = scaledRandomIntBetween(25, 250);
    args = new Object[numNewDocs][];
    for (int i = 0; i < numNewDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO test (id) VALUES (?)", args);
    refresh();
    // Flush twice to update the safe commit's local checkpoint
    execute("OPTIMIZE TABLE test");
    execute("OPTIMIZE TABLE test");
    execute("ALTER TABLE test SET (number_of_replicas = 1)");
    internalCluster().startNode(randomFrom(firstNodeToStopDataPathSettings, secondNodeToStopDataPathSettings));
    ensureGreen(indexName);
    final RecoveryResponse recoveryResponse = client().admin().indices().recoveries(new RecoveryRequest(indexName)).get();
    final List<RecoveryState> recoveryStates = recoveryResponse.shardRecoveryStates().get(indexName);
    recoveryStates.removeIf(r -> r.getTimer().getStartNanoTime() <= desyncNanoTime);
    assertThat(recoveryStates, hasSize(1));
    assertThat(recoveryStates.get(0).getIndex().totalFileCount(), is(0));
    assertThat(recoveryStates.get(0).getTranslog().recoveredOperations(), greaterThan(0));
}