Search in sources :

Example 1 with NoOpEngine

use of org.opensearch.index.engine.NoOpEngine in project OpenSearch by opensearch-project.

the class PeerRecoveryTargetServiceTests method testClosedIndexSkipsLocalRecovery.

public void testClosedIndexSkipsLocalRecovery() throws Exception {
    DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Collections.emptyMap(), Collections.emptySet(), Version.CURRENT);
    IndexShard shard = newStartedShard(false);
    long globalCheckpoint = populateRandomData(shard).getGlobalCheckpoint();
    Optional<SequenceNumbers.CommitInfo> safeCommit = shard.store().findSafeIndexCommit(globalCheckpoint);
    assertTrue(safeCommit.isPresent());
    final IndexMetadata indexMetadata;
    if (randomBoolean()) {
        indexMetadata = IndexMetadata.builder(shard.indexSettings().getIndexMetadata()).settings(shard.indexSettings().getSettings()).state(IndexMetadata.State.CLOSE).build();
    } else {
        indexMetadata = IndexMetadata.builder(shard.indexSettings().getIndexMetadata()).settings(Settings.builder().put(shard.indexSettings().getSettings()).put(IndexMetadata.SETTING_BLOCKS_WRITE, true)).build();
    }
    IndexShard replica = reinitShard(shard, ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.PeerRecoverySource.INSTANCE), indexMetadata, NoOpEngine::new, new EngineConfigFactory(shard.indexSettings()));
    replica.markAsRecovering("for testing", new RecoveryState(replica.routingEntry(), localNode, localNode));
    replica.prepareForIndexRecovery();
    assertThat(replica.recoverLocallyUpToGlobalCheckpoint(), equalTo(safeCommit.get().localCheckpoint + 1));
    assertThat(replica.recoveryState().getTranslog().totalLocal(), equalTo(0));
    assertThat(replica.recoveryState().getTranslog().recoveredOperations(), equalTo(0));
    assertThat(replica.getLastKnownGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
    closeShards(replica);
}
Also used : EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) NoOpEngine(org.opensearch.index.engine.NoOpEngine) IndexShard(org.opensearch.index.shard.IndexShard) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata)

Example 2 with NoOpEngine

use of org.opensearch.index.engine.NoOpEngine in project OpenSearch by opensearch-project.

the class IndicesService method getEngineFactory.

private EngineFactory getEngineFactory(final IndexSettings idxSettings) {
    final IndexMetadata indexMetadata = idxSettings.getIndexMetadata();
    if (indexMetadata != null && indexMetadata.getState() == IndexMetadata.State.CLOSE) {
        // NoOpEngine takes precedence as long as the index is closed
        return NoOpEngine::new;
    }
    final List<Optional<EngineFactory>> engineFactories = engineFactoryProviders.stream().map(engineFactoryProvider -> engineFactoryProvider.apply(idxSettings)).filter(maybe -> Objects.requireNonNull(maybe).isPresent()).collect(Collectors.toList());
    if (engineFactories.isEmpty()) {
        if (idxSettings.isSegRepEnabled()) {
            return new NRTReplicationEngineFactory();
        }
        return new InternalEngineFactory();
    } else if (engineFactories.size() == 1) {
        assert engineFactories.get(0).isPresent();
        return engineFactories.get(0).get();
    } else {
        final String message = String.format(Locale.ROOT, "multiple engine factories provided for %s: %s", idxSettings.getIndex(), engineFactories.stream().map(t -> {
            assert t.isPresent();
            return "[" + t.get().getClass().getName() + "]";
        }).collect(Collectors.joining(",")));
        throw new IllegalStateException(message);
    }
}
Also used : OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) NRTReplicationEngineFactory(org.opensearch.index.engine.NRTReplicationEngineFactory) LongSupplier(java.util.function.LongSupplier) CheckedFunction(org.opensearch.common.CheckedFunction) IndexScopedSettings(org.opensearch.common.settings.IndexScopedSettings) SearchStats(org.opensearch.index.search.stats.SearchStats) Iterables(org.opensearch.common.util.iterable.Iterables) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) MapBuilder.newMapBuilder(org.opensearch.common.collect.MapBuilder.newMapBuilder) IndexShardStats(org.opensearch.action.admin.indices.stats.IndexShardStats) FileSystemUtils(org.opensearch.common.io.FileSystemUtils) MergeStats(org.opensearch.index.merge.MergeStats) AbstractRefCounted(org.opensearch.common.util.concurrent.AbstractRefCounted) IndexFieldDataCache(org.opensearch.index.fielddata.IndexFieldDataCache) RecoveryState(org.opensearch.indices.recovery.RecoveryState) AliasFilter(org.opensearch.search.internal.AliasFilter) Map(java.util.Map) IndexModule(org.opensearch.index.IndexModule) NodeEnvironment(org.opensearch.env.NodeEnvironment) ScriptService(org.opensearch.script.ScriptService) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) ShardRequestCache(org.opensearch.index.cache.request.ShardRequestCache) Index(org.opensearch.index.Index) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) IndexingStats(org.opensearch.index.shard.IndexingStats) GetStats(org.opensearch.index.get.GetStats) Executors(java.util.concurrent.Executors) QueryPhase(org.opensearch.search.query.QueryPhase) OpenSearchThreadPoolExecutor(org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor) UncheckedIOException(java.io.UncheckedIOException) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) CountDownLatch(java.util.concurrent.CountDownLatch) Logger(org.apache.logging.log4j.Logger) EngineConfigFactory(org.opensearch.index.engine.EngineConfigFactory) XContentType(org.opensearch.common.xcontent.XContentType) ShardLock(org.opensearch.env.ShardLock) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) RepositoriesService(org.opensearch.repositories.RepositoriesService) IndexShardState(org.opensearch.index.shard.IndexShardState) ThreadPool(org.opensearch.threadpool.ThreadPool) ValuesSourceRegistry(org.opensearch.search.aggregations.support.ValuesSourceRegistry) Releasable(org.opensearch.common.lease.Releasable) Node(org.opensearch.node.Node) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) ArrayList(java.util.ArrayList) RecoverySource(org.opensearch.cluster.routing.RecoverySource) ClusterState(org.opensearch.cluster.ClusterState) LegacyESVersion(org.opensearch.LegacyESVersion) IndexingOperationListener(org.opensearch.index.shard.IndexingOperationListener) MapperRegistry(org.opensearch.indices.mapper.MapperRegistry) RetentionLeaseStats(org.opensearch.index.seqno.RetentionLeaseStats) CommitStats(org.opensearch.index.engine.CommitStats) NamedWriteableAwareStreamInput(org.opensearch.common.io.stream.NamedWriteableAwareStreamInput) EngineFactory(org.opensearch.index.engine.EngineFactory) Files(java.nio.file.Files) SearchType(org.opensearch.action.search.SearchType) FlushStats(org.opensearch.index.flush.FlushStats) IOException(java.io.IOException) BytesStreamOutput(org.opensearch.common.io.stream.BytesStreamOutput) IndexService(org.opensearch.index.IndexService) CollectionUtil(org.apache.lucene.util.CollectionUtil) PluginsService(org.opensearch.plugins.PluginsService) Flag(org.opensearch.action.admin.indices.stats.CommonStatsFlags.Flag) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) RecoveryStats(org.opensearch.index.recovery.RecoveryStats) ClusterService(org.opensearch.cluster.service.ClusterService) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) RetentionLeaseSyncer(org.opensearch.index.seqno.RetentionLeaseSyncer) CommonStats(org.opensearch.action.admin.indices.stats.CommonStats) CREATE_INDEX(org.opensearch.index.IndexService.IndexCreationContext.CREATE_INDEX) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) SearchContext(org.opensearch.search.internal.SearchContext) OpenSearchException(org.opensearch.OpenSearchException) CircuitBreaker(org.opensearch.common.breaker.CircuitBreaker) CommonStatsFlags(org.opensearch.action.admin.indices.stats.CommonStatsFlags) XContentParser(org.opensearch.common.xcontent.XContentParser) IndexStorePlugin(org.opensearch.plugins.IndexStorePlugin) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) AnalysisRegistry(org.opensearch.index.analysis.AnalysisRegistry) MapperService(org.opensearch.index.mapper.MapperService) RefreshStats(org.opensearch.index.refresh.RefreshStats) CheckedSupplier(org.opensearch.common.CheckedSupplier) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecoveryListener(org.opensearch.indices.recovery.RecoveryListener) Locale(java.util.Locale) OpenSearchExecutors.daemonThreadFactory(org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory) Property(org.opensearch.common.settings.Setting.Property) XContentFactory(org.opensearch.common.xcontent.XContentFactory) SegmentReplicationCheckpointPublisher(org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher) IndicesClusterStateService(org.opensearch.indices.cluster.IndicesClusterStateService) CacheHelper(org.apache.lucene.index.IndexReader.CacheHelper) IndexEventListener(org.opensearch.index.shard.IndexEventListener) Predicate(java.util.function.Predicate) Collections.emptyList(java.util.Collections.emptyList) DirectoryReader(org.apache.lucene.index.DirectoryReader) Collection(java.util.Collection) LoggingDeprecationHandler(org.opensearch.common.xcontent.LoggingDeprecationHandler) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) Objects(java.util.Objects) RamUsageEstimator(org.apache.lucene.util.RamUsageEstimator) List(java.util.List) QueryBuilder(org.opensearch.index.query.QueryBuilder) IndexSettings(org.opensearch.index.IndexSettings) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) Optional(java.util.Optional) BigArrays(org.opensearch.common.util.BigArrays) MetaStateService(org.opensearch.gateway.MetaStateService) AbstractQueryBuilder.parseInnerQueryBuilder(org.opensearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder) BytesReference(org.opensearch.common.bytes.BytesReference) NoOpEngine(org.opensearch.index.engine.NoOpEngine) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardSearchRequest(org.opensearch.search.internal.ShardSearchRequest) CheckedConsumer(org.opensearch.common.CheckedConsumer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StreamOutput(org.opensearch.common.io.stream.StreamOutput) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) HashMap(java.util.HashMap) EngineConfig(org.opensearch.index.engine.EngineConfig) IndicesFieldDataCache(org.opensearch.indices.fielddata.cache.IndicesFieldDataCache) QuerySearchResult(org.opensearch.search.query.QuerySearchResult) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) NamedWriteableRegistry(org.opensearch.common.io.stream.NamedWriteableRegistry) IndexShard(org.opensearch.index.shard.IndexShard) PeerRecoveryTargetService(org.opensearch.indices.recovery.PeerRecoveryTargetService) MetadataStateFormat(org.opensearch.gateway.MetadataStateFormat) IllegalIndexShardStateException(org.opensearch.index.shard.IllegalIndexShardStateException) ExecutorService(java.util.concurrent.ExecutorService) StreamInput(org.opensearch.common.io.stream.StreamInput) InternalEngineFactory(org.opensearch.index.engine.InternalEngineFactory) Collections.emptyMap(java.util.Collections.emptyMap) Setting(org.opensearch.common.settings.Setting) Iterator(java.util.Iterator) ShardLockObtainFailedException(org.opensearch.env.ShardLockObtainFailedException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) CollectionUtils.arrayAsArrayList(org.opensearch.common.util.CollectionUtils.arrayAsArrayList) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Sets(org.opensearch.common.util.set.Sets) METADATA_VERIFICATION(org.opensearch.index.IndexService.IndexCreationContext.METADATA_VERIFICATION) QueryRewriteContext(org.opensearch.index.query.QueryRewriteContext) CircuitBreakerService(org.opensearch.indices.breaker.CircuitBreakerService) Closeable(java.io.Closeable) ALLOW_EXPENSIVE_QUERIES(org.opensearch.search.SearchService.ALLOW_EXPENSIVE_QUERIES) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) InputStream(java.io.InputStream) Optional(java.util.Optional) InternalEngineFactory(org.opensearch.index.engine.InternalEngineFactory) NRTReplicationEngineFactory(org.opensearch.index.engine.NRTReplicationEngineFactory) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata)

Aggregations

IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)2 DiscoveryNode (org.opensearch.cluster.node.DiscoveryNode)2 EngineConfigFactory (org.opensearch.index.engine.EngineConfigFactory)2 NoOpEngine (org.opensearch.index.engine.NoOpEngine)2 IndexShard (org.opensearch.index.shard.IndexShard)2 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 UncheckedIOException (java.io.UncheckedIOException)1 Files (java.nio.file.Files)1 ArrayList (java.util.ArrayList)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 Collections.emptyList (java.util.Collections.emptyList)1 Collections.emptyMap (java.util.Collections.emptyMap)1 Collections.unmodifiableMap (java.util.Collections.unmodifiableMap)1 HashMap (java.util.HashMap)1 Iterator (java.util.Iterator)1 List (java.util.List)1 Locale (java.util.Locale)1