Examples with RecoveryState org.elasticsearch.indices.recovery.RecoveryState used on opensource projects

Example 1 with RecoveryState

use of org.elasticsearch.indices.recovery.RecoveryState in project crate by crate.

the class SysShardsExpressionsTest method mockIndexShard.

private IndexShard mockIndexShard() {
    IndexService indexService = mock(IndexService.class);
    Index index = new Index(indexName);
    ShardId shardId = new ShardId(indexName, 1);
    IndexShard indexShard = mock(IndexShard.class);
    when(indexService.index()).thenReturn(index);
    when(indexShard.indexService()).thenReturn(indexService);
    when(indexShard.shardId()).thenReturn(shardId);
    when(indexShard.state()).thenReturn(IndexShardState.STARTED);
    StoreStats storeStats = mock(StoreStats.class);
    when(storeStats.getSizeInBytes()).thenReturn(123456L);
    when(indexShard.storeStats()).thenReturn(storeStats);
    Path dataPath = Paths.get("/dummy/" + indexName + "/1");
    when(indexShard.shardPath()).thenReturn(new ShardPath(false, dataPath, dataPath, "123", shardId));
    DocsStats docsStats = new DocsStats(654321L, 0L);
    when(indexShard.docStats()).thenReturn(docsStats).thenThrow(IllegalIndexShardStateException.class);
    ShardRouting shardRouting = ShardRouting.newUnassigned(index.name(), shardId.id(), null, true, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    ShardRoutingHelper.initialize(shardRouting, "node1");
    ShardRoutingHelper.moveToStarted(shardRouting);
    ShardRoutingHelper.relocate(shardRouting, "node_X");
    when(indexShard.routingEntry()).thenReturn(shardRouting);
    when(indexShard.minimumCompatibleVersion()).thenReturn(Version.LATEST);
    RecoveryState recoveryState = mock(RecoveryState.class);
    when(indexShard.recoveryState()).thenReturn(recoveryState);
    RecoveryState.Index recoveryStateIndex = mock(RecoveryState.Index.class);
    RecoveryState.Timer recoveryStateTimer = mock(RecoveryState.Timer.class);
    when(recoveryState.getIndex()).thenReturn(recoveryStateIndex);
    when(recoveryState.getStage()).thenReturn(RecoveryState.Stage.DONE);
    when(recoveryState.getTimer()).thenReturn(recoveryStateTimer);
    when(recoveryState.getType()).thenReturn(RecoveryState.Type.REPLICA);
    when(recoveryStateIndex.totalBytes()).thenReturn(2048L);
    when(recoveryStateIndex.reusedBytes()).thenReturn(1024L);
    when(recoveryStateIndex.recoveredBytes()).thenReturn(1024L);
    when(recoveryStateIndex.totalFileCount()).thenReturn(2);
    when(recoveryStateIndex.reusedFileCount()).thenReturn(1);
    when(recoveryStateIndex.recoveredFileCount()).thenReturn(1);
    when(recoveryStateTimer.time()).thenReturn(10000L);
    return indexShard;
}

Example 2 with RecoveryState

use of org.elasticsearch.indices.recovery.RecoveryState in project elasticsearch by elastic.

the class IndexShardTestCase method recoverReplica.

/**
     * Recovers a replica from the give primary, allow the user to supply a custom recovery target. A typical usage of a custom recovery
     * target is to assert things in the various stages of recovery.
     * @param replica                the recovery target shard
     * @param primary                the recovery source shard
     * @param targetSupplier         supplies an instance of {@link RecoveryTarget}
     * @param markAsRecovering       set to {@code false} if the replica is marked as recovering
     */
protected final void recoverReplica(final IndexShard replica, final IndexShard primary, final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier, final boolean markAsRecovering) throws IOException {
    final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
    final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
    if (markAsRecovering) {
        replica.markAsRecovering("remote", new RecoveryState(replica.routingEntry(), pNode, rNode));
    } else {
        assertEquals(replica.state(), IndexShardState.RECOVERING);
    }
    replica.prepareForIndexRecovery();
    final RecoveryTarget recoveryTarget = targetSupplier.apply(replica, pNode);
    final Store.MetadataSnapshot snapshot = getMetadataSnapshotOrEmpty(replica);
    final long startingSeqNo;
    if (snapshot.size() > 0) {
        startingSeqNo = PeerRecoveryTargetService.getStartingSeqNo(recoveryTarget);
    } else {
        startingSeqNo = SequenceNumbersService.UNASSIGNED_SEQ_NO;
    }
    final StartRecoveryRequest request = new StartRecoveryRequest(replica.shardId(), pNode, rNode, snapshot, false, 0, startingSeqNo);
    final RecoverySourceHandler recovery = new RecoverySourceHandler(primary, recoveryTarget, request, () -> 0L, e -> () -> {
    }, (int) ByteSizeUnit.MB.toBytes(1), Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), pNode.getName()).build());
    recovery.recoverToTarget();
    recoveryTarget.markAsDone();
    replica.updateRoutingEntry(ShardRoutingHelper.moveToStarted(replica.routingEntry()));
}

Example 3 with RecoveryState

use of org.elasticsearch.indices.recovery.RecoveryState in project elasticsearch by elastic.

the class RecoveryFromGatewayIT method testReuseInFileBasedPeerRecovery.

public void testReuseInFileBasedPeerRecovery() throws Exception {
    internalCluster().startMasterOnlyNode();
    final String primaryNode = internalCluster().startDataOnlyNode(nodeSettings(0));
    // create the index with our mapping
    client(primaryNode).admin().indices().prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 1)).get();
    logger.info("--> indexing docs");
    for (int i = 0; i < randomIntBetween(1, 1024); i++) {
        client(primaryNode).prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
    }
    client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get();
    // start the replica node; we do this after indexing so a file-based recovery is triggered to ensure the files are identical
    final String replicaNode = internalCluster().startDataOnlyNode(nodeSettings(1));
    ensureGreen();
    final RecoveryResponse initialRecoveryReponse = client().admin().indices().prepareRecoveries("test").get();
    final Set<String> files = new HashSet<>();
    for (final RecoveryState recoveryState : initialRecoveryReponse.shardRecoveryStates().get("test")) {
        if (recoveryState.getTargetNode().getName().equals(replicaNode)) {
            for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
                files.add(file.name());
            }
            break;
        }
    }
    logger.info("--> restart replica node");
    internalCluster().restartNode(replicaNode, new RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            // index some more documents; we expect to reuse the files that already exist on the replica
            for (int i = 0; i < randomIntBetween(1, 1024); i++) {
                client(primaryNode).prepareIndex("test", "type").setSource("field", "value").execute().actionGet();
            }
            // prevent a sequence-number-based recovery from being possible
            client(primaryNode).admin().indices().prepareFlush("test").setForce(true).get();
            return super.onNodeStopped(nodeName);
        }
    });
    ensureGreen();
    final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
    for (final RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
        long recovered = 0;
        long reused = 0;
        int filesRecovered = 0;
        int filesReused = 0;
        for (final RecoveryState.File file : recoveryState.getIndex().fileDetails()) {
            if (files.contains(file.name()) == false) {
                recovered += file.length();
                filesRecovered++;
            } else {
                reused += file.length();
                filesReused++;
            }
        }
        if (recoveryState.getPrimary()) {
            assertThat(recoveryState.getIndex().recoveredBytes(), equalTo(0L));
            assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes()));
            assertThat(recoveryState.getIndex().recoveredFileCount(), equalTo(0));
            assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount()));
        } else {
            logger.info("--> replica shard {} recovered from {} to {}, recovered {}, reuse {}", recoveryState.getShardId().getId(), recoveryState.getSourceNode().getName(), recoveryState.getTargetNode().getName(), recoveryState.getIndex().recoveredBytes(), recoveryState.getIndex().reusedBytes());
            assertThat("bytes should have been recovered", recoveryState.getIndex().recoveredBytes(), equalTo(recovered));
            assertThat("data should have been reused", recoveryState.getIndex().reusedBytes(), greaterThan(0L));
            // we have to recover the segments file since we commit the translog ID on engine startup
            assertThat("all existing files should be reused, byte count mismatch", recoveryState.getIndex().reusedBytes(), equalTo(reused));
            assertThat(recoveryState.getIndex().reusedBytes(), equalTo(recoveryState.getIndex().totalBytes() - recovered));
            assertThat("the segment from the last round of indexing should be recovered", recoveryState.getIndex().recoveredFileCount(), equalTo(filesRecovered));
            assertThat("all existing files should be reused, file count mismatch", recoveryState.getIndex().reusedFileCount(), equalTo(filesReused));
            assertThat(recoveryState.getIndex().reusedFileCount(), equalTo(recoveryState.getIndex().totalFileCount() - filesRecovered));
            assertThat("> 0 files should be reused", recoveryState.getIndex().reusedFileCount(), greaterThan(0));
            assertThat("no translog ops should be recovered", recoveryState.getTranslog().recoveredOperations(), equalTo(0));
        }
    }
}

Example 4 with RecoveryState

use of org.elasticsearch.indices.recovery.RecoveryState in project elasticsearch by elastic.

the class IndexShardTests method testShardActiveDuringInternalRecovery.

public void testShardActiveDuringInternalRecovery() throws IOException {
    IndexShard shard = newStartedShard(true);
    indexDoc(shard, "type", "0");
    shard = reinitShard(shard);
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    shard.markAsRecovering("for testing", new RecoveryState(shard.routingEntry(), localNode, null));
    // Shard is still inactive since we haven't started recovering yet
    assertFalse(shard.isActive());
    shard.prepareForIndexRecovery();
    // Shard is still inactive since we haven't started recovering yet
    assertFalse(shard.isActive());
    shard.performTranslogRecovery(true);
    // Shard should now be active since we did recover:
    assertTrue(shard.isActive());
    closeShards(shard);
}

Example 5 with RecoveryState

use of org.elasticsearch.indices.recovery.RecoveryState in project elasticsearch by elastic.

the class IndexShardTests method testRecoverFromStore.

public void testRecoverFromStore() throws IOException {
    final IndexShard shard = newStartedShard(true);
    int translogOps = 1;
    indexDoc(shard, "test", "0");
    if (randomBoolean()) {
        flushShard(shard);
        translogOps = 0;
    }
    IndexShard newShard = reinitShard(shard);
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
    assertTrue(newShard.recoverFromStore());
    assertEquals(translogOps, newShard.recoveryState().getTranslog().recoveredOperations());
    assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperations());
    assertEquals(translogOps, newShard.recoveryState().getTranslog().totalOperationsOnStart());
    assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
    newShard.updateRoutingEntry(newShard.routingEntry().moveToStarted());
    assertDocCount(newShard, 1);
    closeShards(newShard);
}