Search in sources :

Example 6 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class RemoveCorruptedShardDataCommandIT method testCorruptTranslogTruncationOfReplica.

public void testCorruptTranslogTruncationOfReplica() throws Exception {
    internalCluster().startClusterManagerOnlyNode();
    final String node1 = internalCluster().startDataOnlyNode();
    final String node2 = internalCluster().startDataOnlyNode();
    logger.info("--> nodes name: {}, {}", node1, node2);
    final String indexName = "test";
    assertAcked(prepareCreate(indexName).setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "-1").put(MockEngineSupport.DISABLE_FLUSH_ON_CLOSE.getKey(), // never flush - always recover from translog
    true).put("index.routing.allocation.exclude._name", node2)));
    ensureYellow();
    assertAcked(client().admin().indices().prepareUpdateSettings(indexName).setSettings(Settings.builder().put("index.routing.allocation.exclude._name", (String) null)));
    ensureGreen();
    // Index some documents
    int numDocsToKeep = randomIntBetween(0, 100);
    logger.info("--> indexing [{}] docs to be kept", numDocsToKeep);
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocsToKeep];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex(indexName).setSource("foo", "bar");
    }
    indexRandom(false, false, false, Arrays.asList(builders));
    flush(indexName);
    disableTranslogFlush(indexName);
    // having no extra docs is an interesting case for seq no based recoveries - test it more often
    int numDocsToTruncate = randomBoolean() ? 0 : randomIntBetween(0, 100);
    logger.info("--> indexing [{}] more docs to be truncated", numDocsToTruncate);
    builders = new IndexRequestBuilder[numDocsToTruncate];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex(indexName).setSource("foo", "bar");
    }
    indexRandom(false, false, false, Arrays.asList(builders));
    final int totalDocs = numDocsToKeep + numDocsToTruncate;
    // sample the replica node translog dirs
    final ShardId shardId = new ShardId(resolveIndex(indexName), 0);
    final Path translogDir = getPathToShardData(node2, shardId, ShardPath.TRANSLOG_FOLDER_NAME);
    final Settings node1PathSettings = internalCluster().dataPathSettings(node1);
    final Settings node2PathSettings = internalCluster().dataPathSettings(node2);
    assertBusy(() -> internalCluster().getInstances(GatewayMetaState.class).forEach(gw -> assertTrue(gw.allPendingAsyncStatesWritten())));
    // stop data nodes
    internalCluster().stopRandomDataNode();
    internalCluster().stopRandomDataNode();
    // Corrupt the translog file(s) on the replica
    logger.info("--> corrupting translog");
    TestTranslog.corruptRandomTranslogFile(logger, random(), translogDir);
    // Start the node with the non-corrupted data path
    logger.info("--> starting node");
    internalCluster().startNode(node1PathSettings);
    ensureYellow();
    // Run a search and make sure it succeeds
    assertHitCount(client().prepareSearch(indexName).setQuery(matchAllQuery()).get(), totalDocs);
    // check replica corruption
    final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand();
    final MockTerminal terminal = new MockTerminal();
    final OptionParser parser = command.getParser();
    final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put(internalCluster().getDefaultSettings()).put(node2PathSettings).build());
    terminal.addTextInput("y");
    OptionSet options = parser.parse("-d", translogDir.toAbsolutePath().toString());
    logger.info("--> running command for [{}]", translogDir.toAbsolutePath());
    command.execute(terminal, options, environment);
    logger.info("--> output:\n{}", terminal.getOutput());
    logger.info("--> starting the replica node to test recovery");
    internalCluster().startNode(node2PathSettings);
    ensureGreen(indexName);
    for (String node : internalCluster().nodesInclude(indexName)) {
        assertHitCount(client().prepareSearch(indexName).setPreference("_only_nodes:" + node).setQuery(matchAllQuery()).get(), totalDocs);
    }
    final RecoveryResponse recoveryResponse = client().admin().indices().prepareRecoveries(indexName).setActiveOnly(false).get();
    final RecoveryState replicaRecoveryState = recoveryResponse.shardRecoveryStates().get(indexName).stream().filter(recoveryState -> recoveryState.getPrimary() == false).findFirst().get();
    // the replica translog was disabled so it doesn't know what hte global checkpoint is and thus can't do ops based recovery
    assertThat(replicaRecoveryState.getIndex().toString(), replicaRecoveryState.getIndex().recoveredFileCount(), greaterThan(0));
    // Ensure that the global checkpoint and local checkpoint are restored from the max seqno of the last commit.
    final SeqNoStats seqNoStats = getSeqNoStats(indexName, 0);
    assertThat(seqNoStats.getGlobalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
    assertThat(seqNoStats.getLocalCheckpoint(), equalTo(seqNoStats.getMaxSeqNo()));
}
Also used : Path(java.nio.file.Path) IndexRequestBuilder(org.opensearch.action.index.IndexRequestBuilder) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) Arrays(java.util.Arrays) TranslogCorruptedException(org.opensearch.index.translog.TranslogCorruptedException) MockTerminal(org.opensearch.cli.MockTerminal) Matchers.not(org.hamcrest.Matchers.not) ByteSizeUnit(org.opensearch.common.unit.ByteSizeUnit) AllocateStalePrimaryAllocationCommand(org.opensearch.cluster.routing.allocation.command.AllocateStalePrimaryAllocationCommand) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) ClusterAllocationExplanation(org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplanation) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Matcher(java.util.regex.Matcher) OpenSearchAssertions.assertHitCount(org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount) QueryBuilders.matchAllQuery(org.opensearch.index.query.QueryBuilders.matchAllQuery) RecoveryState(org.opensearch.indices.recovery.RecoveryState) Directory(org.apache.lucene.store.Directory) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) OptionParser(joptsimple.OptionParser) SearchRequestBuilder(org.opensearch.action.search.SearchRequestBuilder) UnassignedInfo(org.opensearch.cluster.routing.UnassignedInfo) Path(java.nio.file.Path) OptionSet(joptsimple.OptionSet) NodeEnvironment(org.opensearch.env.NodeEnvironment) CollectionUtils.iterableAsArrayList(org.opensearch.common.util.CollectionUtils.iterableAsArrayList) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Index(org.opensearch.index.Index) Matchers.allOf(org.hamcrest.Matchers.allOf) Collection(java.util.Collection) ShardAllocationDecision(org.opensearch.cluster.routing.allocation.ShardAllocationDecision) IndicesService(org.opensearch.indices.IndicesService) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) Collectors(java.util.stream.Collectors) Matchers.startsWith(org.hamcrest.Matchers.startsWith) IndexWriter(org.apache.lucene.index.IndexWriter) GatewayMetaState(org.opensearch.gateway.GatewayMetaState) List(java.util.List) Stream(java.util.stream.Stream) Matchers.equalTo(org.hamcrest.Matchers.equalTo) IndexSettings(org.opensearch.index.IndexSettings) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Pattern(java.util.regex.Pattern) OpenSearchIntegTestCase(org.opensearch.test.OpenSearchIntegTestCase) Matchers.containsString(org.hamcrest.Matchers.containsString) PathUtils(org.opensearch.common.io.PathUtils) FS(org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest.Metric.FS) TestEnvironment(org.opensearch.env.TestEnvironment) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) RandomPicks(com.carrotsearch.randomizedtesting.generators.RandomPicks) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) ShardIterator(org.opensearch.cluster.routing.ShardIterator) LockObtainFailedException(org.apache.lucene.store.LockObtainFailedException) NativeFSLockFactory(org.apache.lucene.store.NativeFSLockFactory) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) HashMap(java.util.HashMap) MockTransportService(org.opensearch.test.transport.MockTransportService) InternalTestCluster(org.opensearch.test.InternalTestCluster) ClusterState(org.opensearch.cluster.ClusterState) Lock(org.apache.lucene.store.Lock) InternalSettingsPlugin(org.opensearch.test.InternalSettingsPlugin) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) Matchers.hasSize(org.hamcrest.Matchers.hasSize) StreamSupport(java.util.stream.StreamSupport) CorruptionUtils(org.opensearch.test.CorruptionUtils) FSDirectory(org.apache.lucene.store.FSDirectory) AllocationDecision(org.opensearch.cluster.routing.allocation.AllocationDecision) Environment(org.opensearch.env.Environment) OpenSearchAssertions.assertAcked(org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked) MockEngineFactoryPlugin(org.opensearch.index.MockEngineFactoryPlugin) Files(java.nio.file.Files) TestTranslog(org.opensearch.index.translog.TestTranslog) IOException(java.io.IOException) MockEngineSupport(org.opensearch.test.engine.MockEngineSupport) Plugin(org.opensearch.plugins.Plugin) ShardRouting(org.opensearch.cluster.routing.ShardRouting) GroupShardsIterator(org.opensearch.cluster.routing.GroupShardsIterator) RecoveryResponse(org.opensearch.action.admin.indices.recovery.RecoveryResponse) NodesStatsResponse(org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) MergePolicyConfig(org.opensearch.index.MergePolicyConfig) Matchers.containsString(org.hamcrest.Matchers.containsString) MockTerminal(org.opensearch.cli.MockTerminal) OptionParser(joptsimple.OptionParser) RecoveryResponse(org.opensearch.action.admin.indices.recovery.RecoveryResponse) IndexRequestBuilder(org.opensearch.action.index.IndexRequestBuilder) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) NodeEnvironment(org.opensearch.env.NodeEnvironment) TestEnvironment(org.opensearch.env.TestEnvironment) Environment(org.opensearch.env.Environment) OptionSet(joptsimple.OptionSet) RecoveryState(org.opensearch.indices.recovery.RecoveryState) Settings(org.opensearch.common.settings.Settings) IndexSettings(org.opensearch.index.IndexSettings)

Example 7 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class CloseIndexIT method testRecoverExistingReplica.

/**
 * Ensures that if a replica of a closed index does not have the same content as the primary, then a file-based recovery will occur.
 */
public void testRecoverExistingReplica() throws Exception {
    final String indexName = "test-recover-existing-replica";
    internalCluster().ensureAtLeastNumDataNodes(2);
    List<String> dataNodes = randomSubsetOf(2, Sets.newHashSet(clusterService().state().nodes().getDataNodes().valuesIt()).stream().map(DiscoveryNode::getName).collect(Collectors.toSet()));
    createIndex(indexName, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put("index.routing.allocation.include._name", String.join(",", dataNodes)).build());
    indexRandom(randomBoolean(), randomBoolean(), randomBoolean(), IntStream.range(0, randomIntBetween(0, 50)).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
    ensureGreen(indexName);
    client().admin().indices().prepareFlush(indexName).get();
    // index more documents while one shard copy is offline
    internalCluster().restartNode(dataNodes.get(1), new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            Client client = client(dataNodes.get(0));
            int moreDocs = randomIntBetween(1, 50);
            for (int i = 0; i < moreDocs; i++) {
                client.prepareIndex(indexName).setSource("num", i).get();
            }
            assertAcked(client.admin().indices().prepareClose(indexName));
            return super.onNodeStopped(nodeName);
        }
    });
    assertIndexIsClosed(indexName);
    ensureGreen(indexName);
    internalCluster().assertSameDocIdsOnShards();
    for (RecoveryState recovery : client().admin().indices().prepareRecoveries(indexName).get().shardRecoveryStates().get(indexName)) {
        if (recovery.getPrimary() == false) {
            assertThat(recovery.getIndex().fileDetails(), not(empty()));
        }
    }
}
Also used : DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) InternalTestCluster(org.opensearch.test.InternalTestCluster) Matchers.containsString(org.hamcrest.Matchers.containsString) Client(org.opensearch.client.Client) RecoveryState(org.opensearch.indices.recovery.RecoveryState) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) IndexClosedException(org.opensearch.indices.IndexClosedException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Settings(org.opensearch.common.settings.Settings) IndexSettings(org.opensearch.index.IndexSettings)

Example 8 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class AbortedRestoreIT method testAbortedRestoreAlsoAbortFileRestores.

public void testAbortedRestoreAlsoAbortFileRestores() throws Exception {
    internalCluster().startClusterManagerOnlyNode();
    final String dataNode = internalCluster().startDataOnlyNode();
    final String indexName = "test-abort-restore";
    createIndex(indexName, indexSettingsNoReplicas(1).build());
    indexRandomDocs(indexName, scaledRandomIntBetween(10, 1_000));
    ensureGreen();
    forceMerge();
    final String repositoryName = "repository";
    createRepository(repositoryName, "mock");
    final String snapshotName = "snapshot";
    createFullSnapshot(repositoryName, snapshotName);
    assertAcked(client().admin().indices().prepareDelete(indexName));
    logger.info("--> blocking all data nodes for repository [{}]", repositoryName);
    blockAllDataNodes(repositoryName);
    failReadsAllDataNodes(repositoryName);
    logger.info("--> starting restore");
    final ActionFuture<RestoreSnapshotResponse> future = client().admin().cluster().prepareRestoreSnapshot(repositoryName, snapshotName).setWaitForCompletion(true).setIndices(indexName).execute();
    assertBusy(() -> {
        final RecoveryResponse recoveries = client().admin().indices().prepareRecoveries(indexName).setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN).setActiveOnly(true).get();
        assertThat(recoveries.hasRecoveries(), is(true));
        final List<RecoveryState> shardRecoveries = recoveries.shardRecoveryStates().get(indexName);
        assertThat(shardRecoveries, hasSize(1));
        assertThat(future.isDone(), is(false));
        for (RecoveryState shardRecovery : shardRecoveries) {
            assertThat(shardRecovery.getRecoverySource().getType(), equalTo(RecoverySource.Type.SNAPSHOT));
            assertThat(shardRecovery.getStage(), equalTo(RecoveryState.Stage.INDEX));
        }
    });
    final ThreadPool.Info snapshotThreadPoolInfo = threadPool(dataNode).info(ThreadPool.Names.SNAPSHOT);
    assertThat(snapshotThreadPoolInfo.getMax(), greaterThan(0));
    logger.info("--> waiting for snapshot thread [max={}] pool to be full", snapshotThreadPoolInfo.getMax());
    waitForMaxActiveSnapshotThreads(dataNode, equalTo(snapshotThreadPoolInfo.getMax()));
    logger.info("--> aborting restore by deleting the index");
    assertAcked(client().admin().indices().prepareDelete(indexName));
    logger.info("--> unblocking repository [{}]", repositoryName);
    unblockAllDataNodes(repositoryName);
    logger.info("--> restore should have failed");
    final RestoreSnapshotResponse restoreSnapshotResponse = future.get();
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(1));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(0));
    logger.info("--> waiting for snapshot thread pool to be empty");
    waitForMaxActiveSnapshotThreads(dataNode, equalTo(0));
}
Also used : ThreadPool(org.opensearch.threadpool.ThreadPool) RecoveryState(org.opensearch.indices.recovery.RecoveryState) RestoreSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse) RecoveryResponse(org.opensearch.action.admin.indices.recovery.RecoveryResponse)

Example 9 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndexShard method startRecovery.

public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService, RecoveryListener recoveryListener, RepositoriesService repositoriesService, Consumer<MappingMetadata> mappingUpdateConsumer, IndicesService indicesService) {
    // }
    assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
    switch(recoveryState.getRecoverySource().getType()) {
        case EMPTY_STORE:
        case EXISTING_STORE:
            executeRecovery("from store", recoveryState, recoveryListener, this::recoverFromStore);
            break;
        case PEER:
            try {
                markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
            } catch (Exception e) {
                failShard("corrupted preexisting index", e);
                recoveryListener.onFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
            }
            break;
        case SNAPSHOT:
            final String repo = ((SnapshotRecoverySource) recoveryState.getRecoverySource()).snapshot().getRepository();
            executeRecovery("from snapshot", recoveryState, recoveryListener, l -> restoreFromRepository(repositoriesService.repository(repo), l));
            break;
        case LOCAL_SHARDS:
            final IndexMetadata indexMetadata = indexSettings().getIndexMetadata();
            final Index resizeSourceIndex = indexMetadata.getResizeSourceIndex();
            final List<IndexShard> startedShards = new ArrayList<>();
            final IndexService sourceIndexService = indicesService.indexService(resizeSourceIndex);
            final Set<ShardId> requiredShards;
            final int numShards;
            if (sourceIndexService != null) {
                requiredShards = IndexMetadata.selectRecoverFromShards(shardId().id(), sourceIndexService.getMetadata(), indexMetadata.getNumberOfShards());
                for (IndexShard shard : sourceIndexService) {
                    if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) {
                        startedShards.add(shard);
                    }
                }
                numShards = requiredShards.size();
            } else {
                numShards = -1;
                requiredShards = Collections.emptySet();
            }
            if (numShards == startedShards.size()) {
                assert requiredShards.isEmpty() == false;
                executeRecovery("from local shards", recoveryState, recoveryListener, l -> recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream().filter((s) -> requiredShards.contains(s.shardId())).collect(Collectors.toList()), l));
            } else {
                final RuntimeException e;
                if (numShards == -1) {
                    e = new IndexNotFoundException(resizeSourceIndex);
                } else {
                    e = new IllegalStateException("not all required shards of index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId());
                }
                throw e;
            }
            break;
        default:
            throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
    }
}
Also used : Query(org.apache.lucene.search.Query) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) LongSupplier(java.util.function.LongSupplier) CheckedFunction(org.opensearch.common.CheckedFunction) SnapshotRecoverySource(org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource) SearchStats(org.opensearch.index.search.stats.SearchStats) MetadataSnapshot(org.opensearch.index.store.Store.MetadataSnapshot) Term(org.apache.lucene.index.Term) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ReferenceManager(org.apache.lucene.search.ReferenceManager) DocumentMapperForType(org.opensearch.index.mapper.DocumentMapperForType) MergeStats(org.opensearch.index.merge.MergeStats) UsageTrackingQueryCachingPolicy(org.apache.lucene.search.UsageTrackingQueryCachingPolicy) WriteStateException(org.opensearch.gateway.WriteStateException) RecoveryState(org.opensearch.indices.recovery.RecoveryState) RefreshFailedEngineException(org.opensearch.index.engine.RefreshFailedEngineException) Map(java.util.Map) Lucene(org.opensearch.common.lucene.Lucene) ObjectLongMap(com.carrotsearch.hppc.ObjectLongMap) QueryCachingPolicy(org.apache.lucene.search.QueryCachingPolicy) ActionListener(org.opensearch.action.ActionListener) GetResult(org.opensearch.index.engine.Engine.GetResult) IndexModule(org.opensearch.index.IndexModule) Segment(org.opensearch.index.engine.Segment) EnumSet(java.util.EnumSet) Repository(org.opensearch.repositories.Repository) TimeValue(org.opensearch.common.unit.TimeValue) ShardRequestCache(org.opensearch.index.cache.request.ShardRequestCache) Index(org.opensearch.index.Index) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) ShardBitsetFilterCache(org.opensearch.index.cache.bitset.ShardBitsetFilterCache) GetStats(org.opensearch.index.get.GetStats) StandardCharsets(java.nio.charset.StandardCharsets) Engine(org.opensearch.index.engine.Engine) ClosedByInterruptException(java.nio.channels.ClosedByInterruptException) CountDownLatch(java.util.concurrent.CountDownLatch) VersionType(org.opensearch.index.VersionType) Logger(org.apache.logging.log4j.Logger) EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) CheckedRunnable(org.opensearch.common.CheckedRunnable) ThreadInterruptedException(org.apache.lucene.util.ThreadInterruptedException) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) IndexCommit(org.apache.lucene.index.IndexCommit) ShardFieldData(org.opensearch.index.fielddata.ShardFieldData) RepositoriesService(org.opensearch.repositories.RepositoriesService) CodecService(org.opensearch.index.codec.CodecService) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) Releasable(org.opensearch.common.lease.Releasable) MeanMetric(org.opensearch.common.metrics.MeanMetric) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) RecoverySource(org.opensearch.cluster.routing.RecoverySource) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) FilterDirectoryReader(org.apache.lucene.index.FilterDirectoryReader) LegacyESVersion(org.opensearch.LegacyESVersion) Mapping(org.opensearch.index.mapper.Mapping) Booleans(org.opensearch.common.Booleans) ReplicationCheckpoint(org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint) BiConsumer(java.util.function.BiConsumer) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) RetentionLeaseStats(org.opensearch.index.seqno.RetentionLeaseStats) StreamSupport(java.util.stream.StreamSupport) CommitStats(org.opensearch.index.engine.CommitStats) EngineFactory(org.opensearch.index.engine.EngineFactory) PublishCheckpointRequest(org.opensearch.indices.replication.checkpoint.PublishCheckpointRequest) CompletionStats(org.opensearch.search.suggest.completion.CompletionStats) SetOnce(org.apache.lucene.util.SetOnce) RecoveryFailedException(org.opensearch.indices.recovery.RecoveryFailedException) PendingReplicationActions(org.opensearch.action.support.replication.PendingReplicationActions) RETAIN_ALL(org.opensearch.index.seqno.RetentionLeaseActions.RETAIN_ALL) FlushStats(org.opensearch.index.flush.FlushStats) IOException(java.io.IOException) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) IndexService(org.opensearch.index.IndexService) XContentHelper(org.opensearch.common.xcontent.XContentHelper) AtomicLong(java.util.concurrent.atomic.AtomicLong) RecoveryStats(org.opensearch.index.recovery.RecoveryStats) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) CounterMetric(org.opensearch.common.metrics.CounterMetric) ShardIndexWarmerService(org.opensearch.index.warmer.ShardIndexWarmerService) RetentionLeaseSyncer(org.opensearch.index.seqno.RetentionLeaseSyncer) SafeCommitInfo(org.opensearch.index.engine.SafeCommitInfo) EngineException(org.opensearch.index.engine.EngineException) ReadOnlyEngine(org.opensearch.index.engine.ReadOnlyEngine) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) FieldDataStats(org.opensearch.index.fielddata.FieldDataStats) TimeoutException(java.util.concurrent.TimeoutException) OpenSearchException(org.opensearch.OpenSearchException) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) Releasables(org.opensearch.common.lease.Releasables) UpgradeRequest(org.opensearch.action.admin.indices.upgrade.post.UpgradeRequest) RunOnce(org.opensearch.common.util.concurrent.RunOnce) ForceMergeRequest(org.opensearch.action.admin.indices.forcemerge.ForceMergeRequest) MapperService(org.opensearch.index.mapper.MapperService) RefreshStats(org.opensearch.index.refresh.RefreshStats) RecoveryListener(org.opensearch.indices.recovery.RecoveryListener) Locale(java.util.Locale) Assertions(org.opensearch.Assertions) SegmentsStats(org.opensearch.index.engine.SegmentsStats) SegmentReplicationCheckpointPublisher(org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) IndicesClusterStateService(org.opensearch.indices.cluster.IndicesClusterStateService) CheckIndex(org.apache.lucene.index.CheckIndex) Sort(org.apache.lucene.search.Sort) DirectoryReader(org.apache.lucene.index.DirectoryReader) IndicesService(org.opensearch.indices.IndicesService) TranslogConfig(org.opensearch.index.translog.TranslogConfig) IndexingMemoryController(org.opensearch.indices.IndexingMemoryController) RestStatus(org.opensearch.rest.RestStatus) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) SegmentInfos(org.apache.lucene.index.SegmentInfos) Nullable(org.opensearch.common.Nullable) Tuple(org.opensearch.common.collect.Tuple) WarmerStats(org.opensearch.index.warmer.WarmerStats) Objects(java.util.Objects) GatedCloseable(org.opensearch.common.concurrent.GatedCloseable) List(java.util.List) ResyncTask(org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask) LeafReader(org.apache.lucene.index.LeafReader) IndexSettings(org.opensearch.index.IndexSettings) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) Optional(java.util.Optional) BigArrays(org.opensearch.common.util.BigArrays) Uid(org.opensearch.index.mapper.Uid) TranslogStats(org.opensearch.index.translog.TranslogStats) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardSearchStats(org.opensearch.index.search.stats.ShardSearchStats) ActionRunnable(org.opensearch.action.ActionRunnable) SimilarityService(org.opensearch.index.similarity.SimilarityService) CheckedConsumer(org.opensearch.common.CheckedConsumer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) EngineConfig(org.opensearch.index.engine.EngineConfig) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) HashSet(java.util.HashSet) SourceToParse(org.opensearch.index.mapper.SourceToParse) AsyncIOProcessor(org.opensearch.common.util.concurrent.AsyncIOProcessor) PeerRecoveryTargetService(org.opensearch.indices.recovery.PeerRecoveryTargetService) Translog(org.opensearch.index.translog.Translog) StoreFileMetadata(org.opensearch.index.store.StoreFileMetadata) StoreStats(org.opensearch.index.store.StoreStats) RetentionLease(org.opensearch.index.seqno.RetentionLease) PrintStream(java.io.PrintStream) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) RecoveryTarget(org.opensearch.indices.recovery.RecoveryTarget) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) IndexCache(org.opensearch.index.cache.IndexCache) DocumentMapper(org.opensearch.index.mapper.DocumentMapper) RootObjectMapper(org.opensearch.index.mapper.RootObjectMapper) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) ShardGetService(org.opensearch.index.get.ShardGetService) CircuitBreakerService(org.opensearch.indices.breaker.CircuitBreakerService) Closeable(java.io.Closeable) Collections(java.util.Collections) IndexService(org.opensearch.index.IndexService) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) Index(org.opensearch.index.Index) CheckIndex(org.apache.lucene.index.CheckIndex) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) WriteStateException(org.opensearch.gateway.WriteStateException) RefreshFailedEngineException(org.opensearch.index.engine.RefreshFailedEngineException) ClosedByInterruptException(java.nio.channels.ClosedByInterruptException) ThreadInterruptedException(org.apache.lucene.util.ThreadInterruptedException) RecoveryFailedException(org.opensearch.indices.recovery.RecoveryFailedException) IOException(java.io.IOException) EngineException(org.opensearch.index.engine.EngineException) TimeoutException(java.util.concurrent.TimeoutException) OpenSearchException(org.opensearch.OpenSearchException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ReplicationCheckpoint(org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint) RecoveryFailedException(org.opensearch.indices.recovery.RecoveryFailedException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata)

Example 10 with RecoveryState

use of org.opensearch.indices.recovery.RecoveryState in project OpenSearch by opensearch-project.

the class IndexShard method storeStats.

public StoreStats storeStats() {
    try {
        final RecoveryState recoveryState = this.recoveryState;
        final long bytesStillToRecover = recoveryState == null ? -1L : recoveryState.getIndex().bytesStillToRecover();
        return store.stats(bytesStillToRecover == -1 ? StoreStats.UNKNOWN_RESERVED_BYTES : bytesStillToRecover);
    } catch (IOException e) {
        failShard("Failing shard because of exception during storeStats", e);
        throw new OpenSearchException("io exception while building 'store stats'", e);
    }
}
Also used : OpenSearchException(org.opensearch.OpenSearchException) IOException(java.io.IOException) RecoveryState(org.opensearch.indices.recovery.RecoveryState)

Aggregations

RecoveryState (org.opensearch.indices.recovery.RecoveryState)48 DiscoveryNode (org.opensearch.cluster.node.DiscoveryNode)28 ShardRouting (org.opensearch.cluster.routing.ShardRouting)17 Settings (org.opensearch.common.settings.Settings)16 IndexSettings (org.opensearch.index.IndexSettings)13 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)12 IOException (java.io.IOException)11 Matchers.containsString (org.hamcrest.Matchers.containsString)11 TestShardRouting (org.opensearch.cluster.routing.TestShardRouting)11 TestShardRouting.newShardRouting (org.opensearch.cluster.routing.TestShardRouting.newShardRouting)11 Store (org.opensearch.index.store.Store)10 FlushRequest (org.opensearch.action.admin.indices.flush.FlushRequest)8 List (java.util.List)7 CountDownLatch (java.util.concurrent.CountDownLatch)7 SourceToParse (org.opensearch.index.mapper.SourceToParse)7 ArrayList (java.util.ArrayList)6 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6 Collectors (java.util.stream.Collectors)6 Matchers.hasToString (org.hamcrest.Matchers.hasToString)6 UnassignedInfo (org.opensearch.cluster.routing.UnassignedInfo)6