Examples with RecoveryState org.opensearch.indices.recovery.RecoveryState used on opensource projects

Example 11 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class StoreRecovery method recoveryListener.

private ActionListener<Boolean> recoveryListener(IndexShard indexShard, ActionListener<Boolean> listener) {
    return ActionListener.wrap(res -> {
        if (res) {
            // Check that the gateway didn't leave the shard in init or recovering stage. it is up to the gateway
            // to call post recovery.
            final IndexShardState shardState = indexShard.state();
            final RecoveryState recoveryState = indexShard.recoveryState();
            assert shardState != IndexShardState.CREATED && shardState != IndexShardState.RECOVERING : "recovery process of " + shardId + " didn't get to post_recovery. shardState [" + shardState + "]";
            if (logger.isTraceEnabled()) {
                ReplicationLuceneIndex index = recoveryState.getIndex();
                StringBuilder sb = new StringBuilder();
                sb.append("    index    : files           [").append(index.totalFileCount()).append("] with total_size [").append(new ByteSizeValue(index.totalBytes())).append("], took[").append(TimeValue.timeValueMillis(index.time())).append("]\n");
                sb.append("             : recovered_files [").append(index.recoveredFileCount()).append("] with total_size [").append(new ByteSizeValue(index.recoveredBytes())).append("]\n");
                sb.append("             : reusing_files   [").append(index.reusedFileCount()).append("] with total_size [").append(new ByteSizeValue(index.reusedBytes())).append("]\n");
                sb.append("    verify_index    : took [").append(TimeValue.timeValueMillis(recoveryState.getVerifyIndex().time())).append("], check_index [").append(timeValueMillis(recoveryState.getVerifyIndex().checkIndexTime())).append("]\n");
                sb.append("    translog : number_of_operations [").append(recoveryState.getTranslog().recoveredOperations()).append("], took [").append(TimeValue.timeValueMillis(recoveryState.getTranslog().time())).append("]");
                logger.trace("recovery completed from [shard_store], took [{}]\n{}", timeValueMillis(recoveryState.getTimer().time()), sb);
            } else if (logger.isDebugEnabled()) {
                logger.debug("recovery completed from [shard_store], took [{}]", timeValueMillis(recoveryState.getTimer().time()));
            }
        }
        listener.onResponse(res);
    }, ex -> {
        if (ex instanceof IndexShardRecoveryException) {
            if (indexShard.state() == IndexShardState.CLOSED) {
                // got closed on us, just ignore this recovery
                listener.onResponse(false);
                return;
            }
            if ((ex.getCause() instanceof IndexShardClosedException) || (ex.getCause() instanceof IndexShardNotStartedException)) {
                // got closed on us, just ignore this recovery
                listener.onResponse(false);
                return;
            }
            listener.onFailure(ex);
        } else if (ex instanceof IndexShardClosedException || ex instanceof IndexShardNotStartedException) {
            listener.onResponse(false);
        } else {
            if (indexShard.state() == IndexShardState.CLOSED) {
                // got closed on us, just ignore this recovery
                listener.onResponse(false);
            } else {
                listener.onFailure(new IndexShardRecoveryException(shardId, "failed recovery", ex));
            }
        }
    });
}

Example 12 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class ReplicaShardAllocatorIT method assertNoOpRecoveries.

private void assertNoOpRecoveries(String indexName) {
    for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
        if (recovery.getPrimary() == false) {
            assertThat(recovery.getIndex().fileDetails(), empty());
            assertThat(recovery.getTranslog().totalLocal(), equalTo(recovery.getTranslog().totalOperations()));
        }
    }
}

Example 13 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class ReplicaShardAllocatorIT method testRecentPrimaryInformation.

/**
 * Ensure that we fetch the latest shard store from the primary when a new node joins so we won't cancel the current recovery
 * for the copy on the newly joined node unless we can perform a noop recovery with that node.
 */
public void testRecentPrimaryInformation() throws Exception {
    String indexName = "test";
    String nodeWithPrimary = internalCluster().startNode();
    assertAcked(client().admin().indices().prepareCreate(indexName).setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.FILE_BASED_RECOVERY_THRESHOLD_SETTING.getKey(), 0.1f).put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms").put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "1ms")));
    String nodeWithReplica = internalCluster().startDataOnlyNode();
    DiscoveryNode discoNodeWithReplica = internalCluster().getInstance(ClusterService.class, nodeWithReplica).localNode();
    Settings nodeWithReplicaSettings = internalCluster().dataPathSettings(nodeWithReplica);
    ensureGreen(indexName);
    indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
    internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithReplica));
    if (randomBoolean()) {
        indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, between(10, 100)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
    }
    CountDownLatch blockRecovery = new CountDownLatch(1);
    CountDownLatch recoveryStarted = new CountDownLatch(1);
    MockTransportService transportServiceOnPrimary = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeWithPrimary);
    transportServiceOnPrimary.addSendBehavior((connection, requestId, action, request, options) -> {
        if (PeerRecoveryTargetService.Actions.FILES_INFO.equals(action)) {
            recoveryStarted.countDown();
            try {
                blockRecovery.await();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    String newNode = internalCluster().startDataOnlyNode();
    recoveryStarted.await();
    // Index more documents and flush to destroy sync_id and remove the retention lease (as file_based_recovery_threshold reached).
    indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, between(50, 200)).mapToObj(n -> client().prepareIndex(indexName).setSource("f", "v")).collect(Collectors.toList()));
    client().admin().indices().prepareFlush(indexName).get();
    assertBusy(() -> {
        for (ShardStats shardStats : client().admin().indices().prepareStats(indexName).get().getShards()) {
            for (RetentionLease lease : shardStats.getRetentionLeaseStats().retentionLeases().leases()) {
                assertThat(lease.id(), not(equalTo(ReplicationTracker.getPeerRecoveryRetentionLeaseId(discoNodeWithReplica.getId()))));
            }
        }
    });
    // AllocationService only calls GatewayAllocator if there are unassigned shards
    assertAcked(client().admin().indices().prepareCreate("dummy-index").setWaitForActiveShards(0).setSettings(Settings.builder().put("index.routing.allocation.require.attr", "not-found")));
    internalCluster().startDataOnlyNode(nodeWithReplicaSettings);
    // need to wait for events to ensure the reroute has happened since we perform it async when a new node joins.
    client().admin().cluster().prepareHealth(indexName).setWaitForYellowStatus().setWaitForEvents(Priority.LANGUID).get();
    blockRecovery.countDown();
    ensureGreen(indexName);
    assertThat(internalCluster().nodesInclude(indexName), hasItem(newNode));
    for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
        if (recovery.getPrimary() == false) {
            assertThat(recovery.getIndex().fileDetails(), not(empty()));
        }
    }
    transportServiceOnPrimary.clearAllRules();
}

Example 14 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class FullRollingRestartIT method testNoRebalanceOnRollingRestart.

public void testNoRebalanceOnRollingRestart() throws Exception {
    // see https://github.com/elastic/elasticsearch/issues/14387
    internalCluster().startClusterManagerOnlyNode(Settings.EMPTY);
    internalCluster().startDataOnlyNodes(3);
    /**
     * We start 3 nodes and a dedicated cluster-manager. Restart on of the data-nodes and ensure that we got no relocations.
     * Yet we have 6 shards 0 replica so that means if the restarting node comes back both other nodes are subject
     * to relocating to the restarting node since all had 2 shards and now one node has nothing allocated.
     * We have a fix for this to wait until we have allocated unallocated shards now so this shouldn't happen.
     */
    prepareCreate("test").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, "6").put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "0").put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), TimeValue.timeValueMinutes(1))).get();
    for (int i = 0; i < 100; i++) {
        client().prepareIndex("test").setId(Long.toString(i)).setSource(MapBuilder.<String, Object>newMapBuilder().put("test", "value" + i).map()).execute().actionGet();
    }
    ensureGreen();
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
    for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
        assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "\n" + state, recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
    }
    internalCluster().restartRandomDataNode();
    ensureGreen();
    client().admin().cluster().prepareState().get().getState();
    recoveryResponse = client().admin().indices().prepareRecoveries("test").get();
    for (RecoveryState recoveryState : recoveryResponse.shardRecoveryStates().get("test")) {
        assertTrue("relocated from: " + recoveryState.getSourceNode() + " to: " + recoveryState.getTargetNode() + "-- \nbefore: \n" + state, recoveryState.getRecoverySource().getType() != RecoverySource.Type.PEER || recoveryState.getPrimary() == false);
    }
}

Example 15 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndexingMemoryControllerTests method testTranslogRecoveryWorksWithIMC.

public void testTranslogRecoveryWorksWithIMC() throws IOException {
    IndexShard shard = newStartedShard(true);
    for (int i = 0; i < 100; i++) {
        indexDoc(shard, Integer.toString(i), "{\"foo\" : \"bar\"}", XContentType.JSON, null);
    }
    shard.close("simon says", false);
    AtomicReference<IndexShard> shardRef = new AtomicReference<>();
    Settings settings = Settings.builder().put("indices.memory.index_buffer_size", "50kb").build();
    Iterable<IndexShard> iterable = () -> (shardRef.get() == null) ? Collections.emptyIterator() : Collections.singleton(shardRef.get()).iterator();
    AtomicInteger flushes = new AtomicInteger();
    IndexingMemoryController imc = new IndexingMemoryController(settings, threadPool, iterable) {

        @Override
        protected void writeIndexingBufferAsync(IndexShard shard) {
            assertEquals(shard, shardRef.get());
            flushes.incrementAndGet();
            shard.writeIndexingBuffer();
        }
    };
    shard = reinitShard(shard, imc);
    shardRef.set(shard);
    assertEquals(0, imc.availableShards().size());
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    shard.markAsRecovering("store", new RecoveryState(shard.routingEntry(), localNode, null));
    assertEquals(1, imc.availableShards().size());
    assertTrue(recoverFromStore(shard));
    assertThat("we should have flushed in IMC at least once", flushes.get(), greaterThanOrEqualTo(1));
    closeShards(shard);
}