Examples with RecoveryResponse org.elasticsearch.action.admin.indices.recovery.RecoveryResponse used on opensource projects

Example 11 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project crate by crate.

the class IndexRecoveryIT method testRerouteRecovery.

@Test
public void testRerouteRecovery() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    execute("SELECT size FROM sys.shards WHERE table_name = '" + INDEX_NAME + "' AND primary=true");
    long shardSize = (long) response.rows()[0][0];
    logger.info("--> start node B");
    final String nodeB = internalCluster().startNode();
    ensureGreen();
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    execute("ALTER TABLE " + INDEX_NAME + " REROUTE MOVE SHARD 0 FROM '" + nodeA + "' TO '" + nodeB + "'");
    logger.info("--> waiting for recovery to start both on source and target");
    final Index index = resolveIndex(IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null));
    assertBusy(() -> {
        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
        assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
        indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
        assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
    });
    logger.info("--> request recoveries");
    RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index.getName())).actionGet();
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(index.getName());
    List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, RecoverySource.EmptyStoreRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    logger.info("--> request node recovery stats");
    IndicesService indicesServiceNodeA = internalCluster().getInstance(IndicesService.class, nodeA);
    var recoveryStatsNodeA = indicesServiceNodeA.indexServiceSafe(index).getShard(0).recoveryStats();
    assertThat("node A should have ongoing recovery as source", recoveryStatsNodeA.currentAsSource(), equalTo(1));
    assertThat("node A should not have ongoing recovery as target", recoveryStatsNodeA.currentAsTarget(), equalTo(0));
    long nodeAThrottling = recoveryStatsNodeA.throttleTime().millis();
    IndicesService indicesServiceNodeB = internalCluster().getInstance(IndicesService.class, nodeB);
    var recoveryStatsNodeB = indicesServiceNodeB.indexServiceSafe(index).getShard(0).recoveryStats();
    assertThat("node B should not have ongoing recovery as source", recoveryStatsNodeB.currentAsSource(), equalTo(0));
    assertThat("node B should have ongoing recovery as target", recoveryStatsNodeB.currentAsTarget(), equalTo(1));
    long nodeBThrottling = recoveryStatsNodeB.throttleTime().millis();
    logger.info("--> checking throttling increases");
    final long finalNodeAThrottling = nodeAThrottling;
    final long finalNodeBThrottling = nodeBThrottling;
    assertBusy(() -> {
        var recoveryStats = indicesServiceNodeA.indexServiceSafe(index).getShard(0).recoveryStats();
        assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
        recoveryStats = indicesServiceNodeB.indexServiceSafe(index).getShard(0).recoveryStats();
        assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
    });
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    // wait for it to be finished
    ensureGreen(index.getName());
    response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index.getName())).actionGet();
    recoveryStates = response.shardRecoveryStates().get(index.getName());
    assertThat(recoveryStates.size(), equalTo(1));
    assertRecoveryState(recoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(recoveryStates.get(0).getIndex());
    Consumer<String> assertNodeHasThrottleTimeAndNoRecoveries = nodeName -> {
        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName);
        var recoveryStats = indicesService.indexServiceSafe(index).getShard(0).recoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        assertThat(nodeName + " throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
    };
    // we have to use assertBusy as recovery counters are decremented only when the last reference to the RecoveryTarget
    // is decremented, which may happen after the recovery was done.
    // CrateDB does not expose the RecoveryStats via an API, it can only be retrieved by by the IndicesService.
    // NodeA does not hold the index anymore, such resolving RecoveryStats via NodeA is not possible.
    // assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeA));
    assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeB));
    logger.info("--> bump replica count");
    execute("ALTER TABLE " + INDEX_NAME + " SET (number_of_replicas=1)");
    ensureGreen();
    // TODO: NodeA should now contain the replica shards, thus resolving RecoveryStats via the IndicesService
    // is possible again. BUT checked throttle time never increased on target NodeA, too fast?
    // assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeA));
    assertBusy(() -> assertNodeHasThrottleTimeAndNoRecoveries.accept(nodeB));
    logger.info("--> start node C");
    String nodeC = internalCluster().startNode();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut());
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move replica shard from: {} to: {}", nodeA, nodeC);
    execute("ALTER TABLE " + INDEX_NAME + " REROUTE MOVE SHARD 0 FROM '" + nodeA + "' TO '" + nodeC + "'");
    response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index.getName())).actionGet();
    recoveryStates = response.shardRecoveryStates().get(index.getName());
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, false, RecoveryState.Stage.DONE, nodeB, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    if (randomBoolean()) {
        // shutdown node with relocation source of replica shard and check if recovery continues
        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
        ensureStableCluster(2);
        response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index.getName())).actionGet();
        recoveryStates = response.shardRecoveryStates().get(index.getName());
        nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
        assertThat(nodeARecoveryStates.size(), equalTo(0));
        nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
        assertThat(nodeBRecoveryStates.size(), equalTo(1));
        nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
        assertThat(nodeCRecoveryStates.size(), equalTo(1));
        assertRecoveryState(nodeBRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, nodeA, nodeB);
        validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
        validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    }
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    ensureGreen();
    response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(index.getName())).actionGet();
    recoveryStates = response.shardRecoveryStates().get(index.getName());
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(0));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertRecoveryState(nodeCRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, false, RecoveryState.Stage.DONE, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
}

Example 12 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class ReusePeerRecoverySharedTest method testCase.

/**
     * Test peer reuse on recovery. This is shared between RecoverFromGatewayIT
     * and RecoveryBackwardsCompatibilityIT.
     *
     * @param indexSettings
     *            settings for the index to test
     * @param restartCluster
     *            runnable that will restart the cluster under test
     * @param logger
     *            logger for logging
     * @param useSyncIds
     *            should this use synced flush? can't use synced from in the bwc
     *            tests
     */
public static void testCase(Settings indexSettings, Runnable restartCluster, Logger logger, boolean useSyncIds) {
    /*
         * prevent any rebalance actions during the peer recovery if we run into
         * a relocation the reuse count will be 0 and this fails the test. We
         * are testing here if we reuse the files on disk after full restarts
         * for replicas.
         */
    assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put(indexSettings).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("30s").get();
    logger.info("--> indexing docs");
    for (int i = 0; i < 1000; i++) {
        client().prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
        if ((i % 200) == 0) {
            client().admin().indices().prepareFlush().execute().actionGet();
        }
    }
    if (randomBoolean()) {
        client().admin().indices().prepareFlush().execute().actionGet();
    }
    logger.info("--> running cluster health");
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("30s").get();
    // just wait for merges
    client().admin().indices().prepareForceMerge("test").setMaxNumSegments(100).get();
    client().admin().indices().prepareFlush().setForce(true).get();
    if (useSyncIds == false) {
        logger.info("--> disabling allocation while the cluster is shut down");
        // Disable allocations while we are closing nodes
        client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE)).get();
        logger.info("--> full cluster restart");
        restartCluster.run();
        logger.info("--> waiting for cluster to return to green after first shutdown");
        client().admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("30s").get();
    } else {
        logger.info("--> trying to sync flush");
        assertEquals(client().admin().indices().prepareSyncedFlush("test").get().failedShards(), 0);
        assertSyncIdsNotNull();
    }
    logger.info("--> disabling allocation while the cluster is shut down{}", useSyncIds ? "" : " a second time");
    // Disable allocations while we are closing nodes
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE)).get();
    logger.info("--> full cluster restart");
    restartCluster.run();
    logger.info("--> waiting for cluster to return to green after {}shutdown", useSyncIds ? "" : "second ");
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().setTimeout("30s").get();
    if (useSyncIds) {
        assertSyncIdsNotNull();
    }
    RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
    for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
        long recovered = 0;
        for (RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
            if (file.name().startsWith("segments")) {
                recovered += file.length();
            }
        }
        if (!recoveryState.getPrimary() && (useSyncIds == false)) {
            logger.info("--> replica shard {} recovered from {} to {}, recovered {}, reuse {}", recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(), recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
            assertThat("no bytes should be recovered", recoveryState.getIndex().recoveredBytes(), equalTo(recovered));
            assertThat("data should have been reused", recoveryState.getIndex().reusedBytes(), greaterThan(0L));
            // we have to recover the segments file since we commit the translog ID on engine startup
            assertThat("all bytes should be reused except of the segments file", recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes() - recovered));
            assertThat("no files should be recovered except of the segments file", recoveryState.getIndex().recoveredFileCount(), equalTo(1));
            assertThat("all files should be reused except of the segments file", recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount() - 1));
            assertThat("> 0 files should be reused", recoveryState.getIndex().reusedFileCount(), greaterThan(0));
        } else {
            if (useSyncIds && !recoveryState.getPrimary()) {
                logger.info("--> replica shard {} recovered from {} to {} using sync id, recovered {}, reuse {}", recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(), recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
            }
            assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0L));
            assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes()));
            assertThat(recoveryState.getIndex().recoveredFileCount(), equalTo(0));
            assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount()));
        }
    }
}

Example 13 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class RestRecoveryAction method doCatRequest.

@Override
public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
    final RecoveryRequest recoveryRequest = new RecoveryRequest(Strings.splitStringByCommaToArray(request.param("index")));
    recoveryRequest.detailed(request.paramAsBoolean("detailed", false));
    recoveryRequest.activeOnly(request.paramAsBoolean("active_only", false));
    recoveryRequest.indicesOptions(IndicesOptions.fromRequest(request, recoveryRequest.indicesOptions()));
    return channel -> client.admin().indices().recoveries(recoveryRequest, new RestResponseListener<RecoveryResponse>(channel) {

        @Override
        public RestResponse buildResponse(final RecoveryResponse response) throws Exception {
            return RestTable.buildResponse(buildRecoveryTable(request, response), channel);
        }
    });
}

Example 14 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class RestRecoveryAction method prepareRequest.

@Override
public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
    final RecoveryRequest recoveryRequest = new RecoveryRequest(Strings.splitStringByCommaToArray(request.param("index")));
    recoveryRequest.detailed(request.paramAsBoolean("detailed", false));
    recoveryRequest.activeOnly(request.paramAsBoolean("active_only", false));
    recoveryRequest.indicesOptions(IndicesOptions.fromRequest(request, recoveryRequest.indicesOptions()));
    return channel -> client.admin().indices().recoveries(recoveryRequest, new RestBuilderListener<RecoveryResponse>(channel) {

        @Override
        public RestResponse buildResponse(RecoveryResponse response, XContentBuilder builder) throws Exception {
            response.detailed(recoveryRequest.detailed());
            builder.startObject();
            response.toXContent(builder, request);
            builder.endObject();
            return new BytesRestResponse(OK, builder);
        }
    });
}

Example 15 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class IndexRecoveryIT method testGatewayRecoveryTestActiveOnly.

public void testGatewayRecoveryTestActiveOnly() throws Exception {
    logger.info("--> start nodes");
    internalCluster().startNode();
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> restarting cluster");
    internalCluster().fullRestart();
    ensureGreen();
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).setActiveOnly(true).execute().actionGet();
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    // Should not expect any responses back
    assertThat(recoveryStates.size(), equalTo(0));
}