Examples with RecoveryState org.opensearch.indices.recovery.RecoveryState used on opensource projects

Example 16 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndexShardTests method testRecoverFromCleanStore.

public void testRecoverFromCleanStore() throws IOException {
    final IndexShard shard = newStartedShard(true);
    indexDoc(shard, "_doc", "0");
    if (randomBoolean()) {
        flushShard(shard);
    }
    final ShardRouting shardRouting = shard.routingEntry();
    IndexShard newShard = reinitShard(shard, ShardRoutingHelper.initWithSameId(shardRouting, RecoverySource.EmptyStoreRecoverySource.INSTANCE));
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    newShard.markAsRecovering("store", new RecoveryState(newShard.routingEntry(), localNode, null));
    assertTrue(recoverFromStore(newShard));
    assertEquals(0, newShard.recoveryState().getTranslog().recoveredOperations());
    assertEquals(0, newShard.recoveryState().getTranslog().totalOperations());
    assertEquals(0, newShard.recoveryState().getTranslog().totalOperationsOnStart());
    assertEquals(100.0f, newShard.recoveryState().getTranslog().recoveredPercent(), 0.01f);
    IndexShardTestCase.updateRoutingEntry(newShard, newShard.routingEntry().moveToStarted());
    assertDocCount(newShard, 0);
    closeShards(newShard);
}

Example 17 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndicesLifecycleListenerSingleNodeTests method testStartDeleteIndexEventCallback.

public void testStartDeleteIndexEventCallback() throws Throwable {
    IndicesService indicesService = getInstanceFromNode(IndicesService.class);
    assertAcked(client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)));
    ensureGreen();
    Index idx = resolveIndex("test");
    IndexMetadata metadata = indicesService.indexService(idx).getMetadata();
    ShardRouting shardRouting = indicesService.indexService(idx).getShard(0).routingEntry();
    final AtomicInteger counter = new AtomicInteger(1);
    IndexEventListener countingListener = new IndexEventListener() {

        @Override
        public void beforeIndexCreated(Index index, Settings indexSettings) {
            assertEquals("test", index.getName());
            assertEquals(1, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void afterIndexCreated(IndexService indexService) {
            assertEquals("test", indexService.index().getName());
            assertEquals(2, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void beforeIndexShardCreated(ShardId shardId, Settings indexSettings) {
            assertEquals(3, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void afterIndexShardCreated(IndexShard indexShard) {
            assertEquals(4, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void afterIndexShardStarted(IndexShard indexShard) {
            assertEquals(5, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
            assertEquals(DELETED, reason);
            assertEquals(6, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void beforeIndexShardDeleted(ShardId shardId, Settings indexSettings) {
            assertEquals(7, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void afterIndexShardDeleted(ShardId shardId, Settings indexSettings) {
            assertEquals(8, counter.get());
            counter.incrementAndGet();
        }

        @Override
        public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRemovalReason reason) {
            assertEquals(DELETED, reason);
            assertEquals(9, counter.get());
            counter.incrementAndGet();
        }
    };
    indicesService.removeIndex(idx, DELETED, "simon says");
    try {
        IndexService index = indicesService.createIndex(metadata, Arrays.asList(countingListener), false);
        assertEquals(3, counter.get());
        idx = index.index();
        ShardRouting newRouting = shardRouting;
        String nodeId = newRouting.currentNodeId();
        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom");
        newRouting = newRouting.moveToUnassigned(unassignedInfo).updateUnassigned(unassignedInfo, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
        newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
        IndexShard shard = index.createShard(newRouting, s -> {
        }, RetentionLeaseSyncer.EMPTY);
        IndexShardTestCase.updateRoutingEntry(shard, newRouting);
        assertEquals(5, counter.get());
        final DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
        shard.markAsRecovering("store", new RecoveryState(newRouting, localNode, null));
        IndexShardTestCase.recoverFromStore(shard);
        newRouting = ShardRoutingHelper.moveToStarted(newRouting);
        IndexShardTestCase.updateRoutingEntry(shard, newRouting);
        assertEquals(6, counter.get());
    } finally {
        indicesService.removeIndex(idx, DELETED, "simon says");
    }
    assertEquals(10, counter.get());
}

Example 18 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndexingMemoryControllerTests method testTranslogRecoveryWorksWithIMC.

public void testTranslogRecoveryWorksWithIMC() throws IOException {
    IndexShard shard = newStartedShard(true);
    for (int i = 0; i < 100; i++) {
        indexDoc(shard, "_doc", Integer.toString(i), "{\"foo\" : \"bar\"}", XContentType.JSON, null);
    }
    shard.close("simon says", false);
    AtomicReference<IndexShard> shardRef = new AtomicReference<>();
    Settings settings = Settings.builder().put("indices.memory.index_buffer_size", "50kb").build();
    Iterable<IndexShard> iterable = () -> (shardRef.get() == null) ? Collections.emptyIterator() : Collections.singleton(shardRef.get()).iterator();
    AtomicInteger flushes = new AtomicInteger();
    IndexingMemoryController imc = new IndexingMemoryController(settings, threadPool, iterable) {

        @Override
        protected void writeIndexingBufferAsync(IndexShard shard) {
            assertEquals(shard, shardRef.get());
            flushes.incrementAndGet();
            shard.writeIndexingBuffer();
        }
    };
    shard = reinitShard(shard, imc);
    shardRef.set(shard);
    assertEquals(0, imc.availableShards().size());
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
    shard.markAsRecovering("store", new RecoveryState(shard.routingEntry(), localNode, null));
    assertEquals(1, imc.availableShards().size());
    assertTrue(recoverFromStore(shard));
    assertThat("we should have flushed in IMC at least once", flushes.get(), greaterThanOrEqualTo(1));
    closeShards(shard);
}

Example 19 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class StoreRecovery method recoveryListener.

private ActionListener<Boolean> recoveryListener(IndexShard indexShard, ActionListener<Boolean> listener) {
    return ActionListener.wrap(res -> {
        if (res) {
            // Check that the gateway didn't leave the shard in init or recovering stage. it is up to the gateway
            // to call post recovery.
            final IndexShardState shardState = indexShard.state();
            final RecoveryState recoveryState = indexShard.recoveryState();
            assert shardState != IndexShardState.CREATED && shardState != IndexShardState.RECOVERING : "recovery process of " + shardId + " didn't get to post_recovery. shardState [" + shardState + "]";
            if (logger.isTraceEnabled()) {
                RecoveryState.Index index = recoveryState.getIndex();
                StringBuilder sb = new StringBuilder();
                sb.append("    index    : files           [").append(index.totalFileCount()).append("] with total_size [").append(new ByteSizeValue(index.totalBytes())).append("], took[").append(TimeValue.timeValueMillis(index.time())).append("]\n");
                sb.append("             : recovered_files [").append(index.recoveredFileCount()).append("] with total_size [").append(new ByteSizeValue(index.recoveredBytes())).append("]\n");
                sb.append("             : reusing_files   [").append(index.reusedFileCount()).append("] with total_size [").append(new ByteSizeValue(index.reusedBytes())).append("]\n");
                sb.append("    verify_index    : took [").append(TimeValue.timeValueMillis(recoveryState.getVerifyIndex().time())).append("], check_index [").append(timeValueMillis(recoveryState.getVerifyIndex().checkIndexTime())).append("]\n");
                sb.append("    translog : number_of_operations [").append(recoveryState.getTranslog().recoveredOperations()).append("], took [").append(TimeValue.timeValueMillis(recoveryState.getTranslog().time())).append("]");
                logger.trace("recovery completed from [shard_store], took [{}]\n{}", timeValueMillis(recoveryState.getTimer().time()), sb);
            } else if (logger.isDebugEnabled()) {
                logger.debug("recovery completed from [shard_store], took [{}]", timeValueMillis(recoveryState.getTimer().time()));
            }
        }
        listener.onResponse(res);
    }, ex -> {
        if (ex instanceof IndexShardRecoveryException) {
            if (indexShard.state() == IndexShardState.CLOSED) {
                // got closed on us, just ignore this recovery
                listener.onResponse(false);
                return;
            }
            if ((ex.getCause() instanceof IndexShardClosedException) || (ex.getCause() instanceof IndexShardNotStartedException)) {
                // got closed on us, just ignore this recovery
                listener.onResponse(false);
                return;
            }
            listener.onFailure(ex);
        } else if (ex instanceof IndexShardClosedException || ex instanceof IndexShardNotStartedException) {
            listener.onResponse(false);
        } else {
            if (indexShard.state() == IndexShardState.CLOSED) {
                // got closed on us, just ignore this recovery
                listener.onResponse(false);
            } else {
                listener.onFailure(new IndexShardRecoveryException(shardId, "failed recovery", ex));
            }
        }
    });
}

Example 20 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndexShard method storeStats.

public StoreStats storeStats() {
    try {
        final RecoveryState recoveryState = this.recoveryState;
        final long bytesStillToRecover = recoveryState == null ? -1L : recoveryState.getIndex().bytesStillToRecover();
        return store.stats(bytesStillToRecover == -1 ? StoreStats.UNKNOWN_RESERVED_BYTES : bytesStillToRecover);
    } catch (IOException e) {
        failShard("Failing shard because of exception during storeStats", e);
        throw new OpenSearchException("io exception while building 'store stats'", e);
    }
}