Examples with RecoveryResponse org.elasticsearch.action.admin.indices.recovery.RecoveryResponse used on opensource projects

Example 6 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project elasticsearch by elastic.

the class RecoveryFromGatewayIT method testReuseInFileBasedPeerRecovery.

public void testReuseInFileBasedPeerRecovery() throws Exception {
    internalCluster().startMasterOnlyNode();
    final String primaryNode = internalCluster().startDataOnlyNode(nodeSettings(0));
    // create the index with our mapping
    client(primaryNode).admin().indices().prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 1)).get();
    logger.info("--> indexing docs");
    for (int i = 0; i < randomIntBetween(1, 1024); i++) {
        client(primaryNode).prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
    }
    client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get();
    // start the replica node; we do this after indexing so a file-based recovery is triggered to ensure the files are identical
    final String replicaNode = internalCluster().startDataOnlyNode(nodeSettings(1));
    ensureGreen();
    final RecoveryResponse initialRecoveryReponse = client().admin().indices().prepareRecoveries("test").get();
    final Set<String> files = new HashSet<>();
    for (final RecoveryState recoveryState : initialRecoveryReponse.shardRecoveryStates().get("test")) {
        if (recoveryState.getTargetNode().getName().equals(replicaNode)) {
            for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
                files.add(file.name());
            }
            break;
        }
    }
    logger.info("--> restart replica node");
    internalCluster().restartNode(replicaNode, new RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            // index some more documents; we expect to reuse the files that already exist on the replica
            for (int i = 0; i < randomIntBetween(1, 1024); i++) {
                client(primaryNode).prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
            }
            // prevent a sequence-number-based recovery from being possible
            client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get();
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen();
    final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
    for (final RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
        long recovered = 0;
        long reused = 0;
        int filesRecovered = 0;
        int filesReused = 0;
        for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
            if (files.contains(file.name()) == false) {
                recovered += file.length();
                filesRecovered++;
            } else {
                reused += file.length();
                filesReused++;
            }
        }
        if (recoveryState.getPrimary()) {
            assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0L));
            assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes()));
            assertThat(recoveryState.getIndex().recoveredFileCount(), equalTo(0));
            assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount()));
        } else {
            logger.info("--> replica shard {} recovered from {} to {}, recovered {}, reuse {}", recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(), recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
            assertThat("bytes should have been recovered", recoveryState.getIndex().recoveredBytes(), equalTo(recovered));
            assertThat("data should have been reused", recoveryState.getIndex().reusedBytes(), greaterThan(0L));
            // we have to recover the segments file since we commit the translog ID on engine startup
            assertThat("all existing files should be reused, byte count mismatch", recoveryState.getIndex().reusedBytes(), equalTo(reused));
            assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes() - recovered));
            assertThat("the segment from the last round of indexing should be recovered", recoveryState.getIndex().recoveredFileCount(), equalTo(filesRecovered));
            assertThat("all existing files should be reused, file count mismatch", recoveryState.getIndex().reusedFileCount(), equalTo(filesReused));
            assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount() - filesRecovered));
            assertThat("> 0 files should be reused", recoveryState.getIndex().reusedFileCount(), greaterThan(0));
            assertThat("no translog ops should be recovered", recoveryState.getTranslog().recoveredOperations(), equalTo(0));
        }
    }
}

Example 7 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project pentaho-kettle by pentaho.

the class ElasticSearchBulkDialog method test.

private void test(TestType testType) {
    try {
        ElasticSearchBulkMeta tempMeta = new ElasticSearchBulkMeta();
        toModel(tempMeta);
        // if ( !tempMeta.getServers().isEmpty() ) {
        Settings.Builder settingsBuilder = Settings.builder();
        settingsBuilder.put(Settings.Builder.EMPTY_SETTINGS);
        tempMeta.getSettingsMap().entrySet().stream().forEach((s) -> settingsBuilder.put(s.getKey(), transMeta.environmentSubstitute(s.getValue())));
        try (PreBuiltTransportClient client = new PreBuiltTransportClient(settingsBuilder.build())) {
            for (Server server : tempMeta.getServers()) {
                client.addTransportAddress(new TransportAddress(InetAddress.getByName(transMeta.environmentSubstitute(server.getAddress())), server.getPort()));
            }
            AdminClient admin = client.admin();
            switch(testType) {
                case INDEX:
                    if (StringUtils.isBlank(tempMeta.getIndex())) {
                        showError(BaseMessages.getString(PKG, "ElasticSearchBulk.Error.NoIndex"));
                        break;
                    }
                    // First check to see if the index exists
                    IndicesExistsRequestBuilder indicesExistBld = admin.indices().prepareExists(tempMeta.getIndex());
                    IndicesExistsResponse indicesExistResponse = indicesExistBld.execute().get();
                    if (!indicesExistResponse.isExists()) {
                        showError(BaseMessages.getString(PKG, "ElasticSearchBulkDialog.Error.NoIndex"));
                        return;
                    }
                    RecoveryRequestBuilder indicesBld = admin.indices().prepareRecoveries(tempMeta.getIndex());
                    ActionFuture<RecoveryResponse> lafInd = indicesBld.execute();
                    String shards = "" + lafInd.get().getSuccessfulShards() + "/" + lafInd.get().getTotalShards();
                    showMessage(BaseMessages.getString(PKG, "ElasticSearchBulkDialog.TestIndex.TestOK", shards));
                    break;
                case CLUSTER:
                    ClusterStateRequestBuilder clusterBld = admin.cluster().prepareState();
                    ActionFuture<ClusterStateResponse> lafClu = clusterBld.execute();
                    ClusterStateResponse cluResp = lafClu.actionGet();
                    String name = cluResp.getClusterName().value();
                    ClusterState cluState = cluResp.getState();
                    int numNodes = cluState.getNodes().getSize();
                    showMessage(BaseMessages.getString(PKG, "ElasticSearchBulkDialog.TestCluster.TestOK", name, numNodes));
                    break;
                default:
                    break;
            }
        }
    } catch (NoNodeAvailableException | MasterNotDiscoveredException e) {
        showError(BaseMessages.getString(PKG, "ElasticSearchBulkDialog.Error.NoNodesFound"));
    } catch (Exception e) {
        showError(e.getLocalizedMessage());
    }
}

Example 8 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project crate by crate.

the class IndexRecoveryIT method testSnapshotRecovery.

@Test
public void testSnapshotRecovery() throws Exception {
    logger.info("--> start node A");
    String nodeA = internalCluster().startNode();
    logger.info("--> create repository");
    execute("CREATE REPOSITORY " + REPO_NAME + " TYPE FS WITH (location = '" + randomRepoPath() + "', compress=false)");
    ensureGreen();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> snapshot");
    var snapshotName = REPO_NAME + "." + SNAP_NAME;
    execute("CREATE SNAPSHOT " + snapshotName + " ALL WITH (wait_for_completion=true)");
    execute("SELECT state FROM sys.snapshots WHERE name = '" + SNAP_NAME + "'");
    assertThat(response.rows()[0][0], is("SUCCESS"));
    execute("ALTER TABLE " + INDEX_NAME + " CLOSE");
    logger.info("--> restore");
    execute("RESTORE SNAPSHOT " + snapshotName + " ALL WITH (wait_for_completion=true)");
    ensureGreen();
    var snapshotInfo = client().execute(GetSnapshotsAction.INSTANCE, new GetSnapshotsRequest(REPO_NAME, new String[] { SNAP_NAME })).get().getSnapshots().get(0);
    logger.info("--> request recoveries");
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    for (Map.Entry<String, List<RecoveryState>> indexRecoveryStates : response.shardRecoveryStates().entrySet()) {
        assertThat(indexRecoveryStates.getKey(), equalTo(indexName));
        List<RecoveryState> recoveryStates = indexRecoveryStates.getValue();
        assertThat(recoveryStates.size(), equalTo(SHARD_COUNT));
        for (RecoveryState recoveryState : recoveryStates) {
            RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource(((RecoverySource.SnapshotRecoverySource) recoveryState.getRecoverySource()).restoreUUID(), new Snapshot(REPO_NAME, snapshotInfo.snapshotId()), Version.CURRENT, indexName);
            assertRecoveryState(recoveryState, 0, recoverySource, true, RecoveryState.Stage.DONE, null, nodeA);
            validateIndexRecoveryState(recoveryState.getIndex());
        }
    }
}

Example 9 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project crate by crate.

the class IndexRecoveryIT method testGatewayRecovery.

@Test
public void testGatewayRecovery() throws Exception {
    logger.info("--> start nodes");
    String node = internalCluster().startNode();
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> restarting cluster");
    internalCluster().fullRestart();
    ensureGreen();
    logger.info("--> request recoveries");
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    assertThat(response.shardRecoveryStates().size(), equalTo(SHARD_COUNT));
    assertThat(response.shardRecoveryStates().get(indexName).size(), equalTo(1));
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(indexName);
    assertThat(recoveryStates.size(), equalTo(1));
    RecoveryState recoveryState = recoveryStates.get(0);
    assertRecoveryState(recoveryState, 0, RecoverySource.ExistingStoreRecoverySource.INSTANCE, true, RecoveryState.Stage.DONE, null, node);
    validateIndexRecoveryState(recoveryState.getIndex());
}

Example 10 with RecoveryResponse

use of org.elasticsearch.action.admin.indices.recovery.RecoveryResponse in project crate by crate.

the class IndexRecoveryIT method testReplicaRecovery.

@Test
public void testReplicaRecovery() throws Exception {
    final String nodeA = internalCluster().startNode();
    execute("CREATE TABLE " + INDEX_NAME + " (id BIGINT, data TEXT) " + " CLUSTERED INTO " + SHARD_COUNT + " SHARDS WITH (number_of_replicas=" + REPLICA_COUNT + ")");
    ensureGreen();
    final int numOfDocs = scaledRandomIntBetween(1, 200);
    try (BackgroundIndexer indexer = new BackgroundIndexer(IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null), "data", sqlExecutor.jdbcUrl(), numOfDocs, scaledRandomIntBetween(2, 5), true, null)) {
        waitForDocs(numOfDocs, indexer, sqlExecutor);
    }
    refresh();
    execute("SELECT COUNT(*) FROM " + INDEX_NAME);
    assertThat(response.rows()[0][0], is((long) numOfDocs));
    // We do not support ALTER on a closed table
    // final boolean closedIndex = randomBoolean();
    final boolean closedIndex = false;
    if (closedIndex) {
        execute("ALTER TABLE " + INDEX_NAME + " CLOSE");
        ensureGreen();
    }
    // force a shard recovery from nodeA to nodeB
    final String nodeB = internalCluster().startNode();
    execute("ALTER TABLE " + INDEX_NAME + " SET (number_of_replicas=1)");
    ensureGreen();
    // we should now have two total shards, one primary and one replica
    execute("SELECT * FROM sys.shards WHERE table_name = '" + INDEX_NAME + "'");
    assertThat(response.rowCount(), is(2L));
    var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
    final RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
    // we should now have two total shards, one primary and one replica
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(indexName);
    assertThat(recoveryStates.size(), equalTo(2));
    List<RecoveryState> nodeAResponses = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeAResponses.size(), equalTo(1));
    List<RecoveryState> nodeBResponses = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBResponses.size(), equalTo(1));
    // validate node A recovery
    final RecoveryState nodeARecoveryState = nodeAResponses.get(0);
    final RecoverySource expectedRecoverySource;
    if (closedIndex == false) {
        expectedRecoverySource = RecoverySource.EmptyStoreRecoverySource.INSTANCE;
    } else {
        expectedRecoverySource = RecoverySource.ExistingStoreRecoverySource.INSTANCE;
    }
    assertRecoveryState(nodeARecoveryState, 0, expectedRecoverySource, true, RecoveryState.Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryState.getIndex());
    // validate node B recovery
    final RecoveryState nodeBRecoveryState = nodeBResponses.get(0);
    assertRecoveryState(nodeBRecoveryState, 0, RecoverySource.PeerRecoverySource.INSTANCE, false, RecoveryState.Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryState.getIndex());
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
    if (closedIndex) {
        execute("ALTER TABLE " + INDEX_NAME + " OPEN");
    }
    var res = execute("SELECT COUNT(*) FROM " + INDEX_NAME);
    assertThat(res.rows()[0][0], is((long) numOfDocs));
}