Search in sources :

Example 1 with OpenSearchDirectoryReader

use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.

the class Engine method acquireSearcherSupplier.

/**
 * Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
 */
public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper, SearcherScope scope) throws EngineException {
    /* Acquire order here is store -> manager since we need
         * to make sure that the store is not closed before
         * the searcher is acquired. */
    if (store.tryIncRef() == false) {
        throw new AlreadyClosedException(shardId + " store is closed", failedEngine.get());
    }
    Releasable releasable = store::decRef;
    try {
        ReferenceManager<OpenSearchDirectoryReader> referenceManager = getReferenceManager(scope);
        OpenSearchDirectoryReader acquire = referenceManager.acquire();
        SearcherSupplier reader = new SearcherSupplier(wrapper) {

            @Override
            public Searcher acquireSearcherInternal(String source) {
                assert assertSearcherIsWarmedUp(source, scope);
                return new Searcher(source, acquire, engineConfig.getSimilarity(), engineConfig.getQueryCache(), engineConfig.getQueryCachingPolicy(), () -> {
                });
            }

            @Override
            protected void doClose() {
                try {
                    referenceManager.release(acquire);
                } catch (IOException e) {
                    throw new UncheckedIOException("failed to close", e);
                } catch (AlreadyClosedException e) {
                    // This means there's a bug somewhere: don't suppress it
                    throw new AssertionError(e);
                } finally {
                    store.decRef();
                }
            }
        };
        // success - hand over the reference to the engine reader
        releasable = null;
        return reader;
    } catch (AlreadyClosedException ex) {
        throw ex;
    } catch (Exception ex) {
        maybeFailEngine("acquire_reader", ex);
        // throw EngineCloseException here if we are already closed
        ensureOpen(ex);
        logger.error(() -> new ParameterizedMessage("failed to acquire reader"), ex);
        throw new EngineException(shardId, "failed to acquire reader", ex);
    } finally {
        Releasables.close(releasable);
    }
}
Also used : OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) IndexSearcher(org.apache.lucene.search.IndexSearcher) UncheckedIOException(java.io.UncheckedIOException) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) NoSuchFileException(java.nio.file.NoSuchFileException) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) FileNotFoundException(java.io.FileNotFoundException) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) Releasable(org.opensearch.common.lease.Releasable) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage)

Example 2 with OpenSearchDirectoryReader

use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.

the class InternalEngineTests method testNotWarmUpSearcherInEngineCtor.

public void testNotWarmUpSearcherInEngineCtor() throws Exception {
    try (Store store = createStore()) {
        List<OpenSearchDirectoryReader> warmedUpReaders = new ArrayList<>();
        Engine.Warmer warmer = reader -> {
            assertNotNull(reader);
            assertThat(reader, not(in(warmedUpReaders)));
            warmedUpReaders.add(reader);
        };
        EngineConfig config = engine.config();
        final TranslogConfig translogConfig = new TranslogConfig(config.getTranslogConfig().getShardId(), createTempDir(), config.getTranslogConfig().getIndexSettings(), config.getTranslogConfig().getBigArrays());
        EngineConfig configWithWarmer = new EngineConfig(config.getShardId(), config.getThreadPool(), config.getIndexSettings(), warmer, store, config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), translogConfig, config.getFlushMergesAfter(), config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), config.getTombstoneDocSupplier());
        try (InternalEngine engine = createEngine(configWithWarmer)) {
            assertThat(warmedUpReaders, empty());
            assertThat(expectThrows(Throwable.class, () -> engine.acquireSearcher("test")).getMessage(), equalTo("searcher was not warmed up yet for source[test]"));
            int times = randomIntBetween(1, 10);
            for (int i = 0; i < times; i++) {
                engine.refresh("test");
            }
            assertThat(warmedUpReaders, hasSize(1));
            try (Engine.Searcher internalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                try (Engine.Searcher externalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)) {
                    assertSame(internalSearcher.getDirectoryReader(), externalSearcher.getDirectoryReader());
                    assertSame(warmedUpReaders.get(0), externalSearcher.getDirectoryReader());
                }
            }
            index(engine, randomInt());
            if (randomBoolean()) {
                engine.refresh("test", Engine.SearcherScope.INTERNAL, true);
                assertThat(warmedUpReaders, hasSize(1));
                try (Engine.Searcher internalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                    try (Engine.Searcher externalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)) {
                        assertNotSame(internalSearcher.getDirectoryReader(), externalSearcher.getDirectoryReader());
                    }
                }
            }
            engine.refresh("test");
            assertThat(warmedUpReaders, hasSize(2));
            try (Engine.Searcher internalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
                try (Engine.Searcher externalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)) {
                    assertSame(internalSearcher.getDirectoryReader(), externalSearcher.getDirectoryReader());
                    assertSame(warmedUpReaders.get(1), externalSearcher.getDirectoryReader());
                }
            }
        }
    }
}
Also used : Term(org.apache.lucene.index.Term) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) PEER_RECOVERY(org.opensearch.index.engine.Engine.Operation.Origin.PEER_RECOVERY) MergePolicy(org.apache.lucene.index.MergePolicy) Map(java.util.Map) Mockito.doAnswer(org.mockito.Mockito.doAnswer) DefaultTranslogDeletionPolicy(org.opensearch.index.translog.DefaultTranslogDeletionPolicy) Path(java.nio.file.Path) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) SoftDeletesRetentionMergePolicy(org.apache.lucene.index.SoftDeletesRetentionMergePolicy) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) PointValues(org.apache.lucene.index.PointValues) CountDownLatch(java.util.concurrent.CountDownLatch) Logger(org.apache.logging.log4j.Logger) Randomness(org.opensearch.common.Randomness) BytesArray(org.opensearch.common.bytes.BytesArray) DocIdAndSeqNo(org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver.DocIdAndSeqNo) XContentType(org.opensearch.common.xcontent.XContentType) CodecService(org.opensearch.index.codec.CodecService) ThreadPool(org.opensearch.threadpool.ThreadPool) LogDocMergePolicy(org.apache.lucene.index.LogDocMergePolicy) FixedBitSet(org.apache.lucene.util.FixedBitSet) RegexFilter(org.apache.logging.log4j.core.filter.RegexFilter) Mockito.spy(org.mockito.Mockito.spy) Supplier(java.util.function.Supplier) LinkedHashMap(java.util.LinkedHashMap) ToLongBiFunction(java.util.function.ToLongBiFunction) IndexWriterMaxDocsChanger(org.apache.lucene.index.IndexWriterMaxDocsChanger) Lock(org.apache.lucene.store.Lock) Mapping(org.opensearch.index.mapper.Mapping) VersionFieldMapper(org.opensearch.index.mapper.VersionFieldMapper) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Bits(org.apache.lucene.util.Bits) TieredMergePolicy(org.apache.lucene.index.TieredMergePolicy) Versions(org.opensearch.common.lucene.uid.Versions) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) TestTranslog(org.opensearch.index.translog.TestTranslog) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) UNASSIGNED_PRIMARY_TERM(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM) SourceFieldMapper(org.opensearch.index.mapper.SourceFieldMapper) AtomicLong(java.util.concurrent.atomic.AtomicLong) Phaser(java.util.concurrent.Phaser) TextField(org.apache.lucene.document.TextField) SeqNoFieldMapper(org.opensearch.index.mapper.SeqNoFieldMapper) Matchers.emptyArray(org.hamcrest.Matchers.emptyArray) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) BiFunction(java.util.function.BiFunction) StoredField(org.apache.lucene.document.StoredField) Matchers.hasKey(org.hamcrest.Matchers.hasKey) Matchers.everyItem(org.hamcrest.Matchers.everyItem) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Terms(org.apache.lucene.index.Terms) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Store(org.opensearch.index.store.Store) Collectors(java.util.stream.Collectors) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) LeafReader(org.apache.lucene.index.LeafReader) IndexSettings(org.opensearch.index.IndexSettings) MetadataFieldMapper(org.opensearch.index.mapper.MetadataFieldMapper) ShardUtils(org.opensearch.index.shard.ShardUtils) Queue(java.util.Queue) BigArrays(org.opensearch.common.util.BigArrays) IndexReader(org.apache.lucene.index.IndexReader) IndexSearcher(org.apache.lucene.search.IndexSearcher) IndexSettingsModule(org.opensearch.test.IndexSettingsModule) BytesReference(org.opensearch.common.bytes.BytesReference) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) CheckedBiConsumer(org.opensearch.common.CheckedBiConsumer) LOCAL_RESET(org.opensearch.index.engine.Engine.Operation.Origin.LOCAL_RESET) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Function(java.util.function.Function) HashSet(java.util.HashSet) Charset(java.nio.charset.Charset) Translog(org.opensearch.index.translog.Translog) PRIMARY(org.opensearch.index.engine.Engine.Operation.Origin.PRIMARY) IntSupplier(java.util.function.IntSupplier) RetentionLease(org.opensearch.index.seqno.RetentionLease) CoreMatchers.sameInstance(org.hamcrest.CoreMatchers.sameInstance) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.empty(org.hamcrest.Matchers.empty) Semaphore(java.util.concurrent.Semaphore) Mockito.when(org.mockito.Mockito.when) ShardRouting(org.opensearch.cluster.routing.ShardRouting) IOUtils(org.opensearch.core.internal.io.IOUtils) ShardId(org.opensearch.index.shard.ShardId) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) VersionsAndSeqNoResolver(org.opensearch.common.lucene.uid.VersionsAndSeqNoResolver) Field(org.apache.lucene.document.Field) TransportActions(org.opensearch.action.support.TransportActions) Comparator(java.util.Comparator) LogManager(org.apache.logging.log4j.LogManager) NoMergePolicy(org.apache.lucene.index.NoMergePolicy) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) Arrays(java.util.Arrays) LongSupplier(java.util.function.LongSupplier) Matchers.not(org.hamcrest.Matchers.not) Level(org.apache.logging.log4j.Level) ContentPath(org.opensearch.index.mapper.ContentPath) LogEvent(org.apache.logging.log4j.core.LogEvent) ReferenceManager(org.apache.lucene.search.ReferenceManager) Document(org.opensearch.index.mapper.ParseContext.Document) Strings(org.opensearch.common.Strings) CoreMatchers.instanceOf(org.hamcrest.CoreMatchers.instanceOf) RandomNumbers(com.carrotsearch.randomizedtesting.generators.RandomNumbers) REPLICA(org.opensearch.index.engine.Engine.Operation.Origin.REPLICA) TermsEnum(org.apache.lucene.index.TermsEnum) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Lucene(org.opensearch.common.lucene.Lucene) ActionListener(org.opensearch.action.ActionListener) SequentialStoredFieldsLeafReader(org.opensearch.common.lucene.index.SequentialStoredFieldsLeafReader) FieldsVisitor(org.opensearch.index.fieldvisitor.FieldsVisitor) NumericDocValuesField(org.apache.lucene.document.NumericDocValuesField) NO_OPS_PERFORMED(org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED) Mockito.atLeastOnce(org.mockito.Mockito.atLeastOnce) Set(java.util.Set) SortedSetSortField(org.apache.lucene.search.SortedSetSortField) Settings(org.opensearch.common.settings.Settings) ReplicationTracker(org.opensearch.index.seqno.ReplicationTracker) BuilderContext(org.opensearch.index.mapper.Mapper.BuilderContext) UncheckedIOException(java.io.UncheckedIOException) VersionType(org.opensearch.index.VersionType) Matchers.contains(org.hamcrest.Matchers.contains) CheckedRunnable(org.opensearch.common.CheckedRunnable) LocalCheckpointTracker(org.opensearch.index.seqno.LocalCheckpointTracker) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) Matchers.in(org.hamcrest.Matchers.in) TriFunction(org.opensearch.common.TriFunction) IndexCommit(org.apache.lucene.index.IndexCommit) LiveIndexWriterConfig(org.apache.lucene.index.LiveIndexWriterConfig) TranslogDeletionPolicies.createTranslogDeletionPolicy(org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy) ArrayList(java.util.ArrayList) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) VersionUtils(org.opensearch.test.VersionUtils) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) TopDocs(org.apache.lucene.search.TopDocs) ParseContext(org.opensearch.index.mapper.ParseContext) LongStream(java.util.stream.LongStream) SetOnce(org.apache.lucene.util.SetOnce) Files(java.nio.file.Files) AbstractAppender(org.apache.logging.log4j.core.appender.AbstractAppender) Matchers.hasItem(org.hamcrest.Matchers.hasItem) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) SnapshotMatchers(org.opensearch.index.translog.SnapshotMatchers) LOCAL_TRANSLOG_RECOVERY(org.opensearch.index.engine.Engine.Operation.Origin.LOCAL_TRANSLOG_RECOVERY) Collections.shuffle(java.util.Collections.shuffle) IdFieldMapper(org.opensearch.index.mapper.IdFieldMapper) IndexableField(org.apache.lucene.index.IndexableField) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) OpenSearchException(org.opensearch.OpenSearchException) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) ObjectObjectCursor(com.carrotsearch.hppc.cursors.ObjectObjectCursor) MapperService(org.opensearch.index.mapper.MapperService) Directory(org.apache.lucene.store.Directory) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) TotalHitCountCollector(org.apache.lucene.search.TotalHitCountCollector) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) CyclicBarrier(java.util.concurrent.CyclicBarrier) Sort(org.apache.lucene.search.Sort) BytesRef(org.apache.lucene.util.BytesRef) DirectoryReader(org.apache.lucene.index.DirectoryReader) TranslogConfig(org.opensearch.index.translog.TranslogConfig) TranslogDeletionPolicyFactory(org.opensearch.index.translog.TranslogDeletionPolicyFactory) SegmentInfos(org.apache.lucene.index.SegmentInfos) Tuple(org.opensearch.common.collect.Tuple) IndexWriter(org.apache.lucene.index.IndexWriter) List(java.util.List) MatcherAssert(org.hamcrest.MatcherAssert) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig) MockDirectoryWrapper(org.apache.lucene.store.MockDirectoryWrapper) Uid(org.opensearch.index.mapper.Uid) LongPoint(org.apache.lucene.document.LongPoint) NumericDocValues(org.apache.lucene.index.NumericDocValues) ByteSizeValue(org.opensearch.common.unit.ByteSizeValue) HashMap(java.util.HashMap) ReleasableLock(org.opensearch.common.util.concurrent.ReleasableLock) AtomicReference(java.util.concurrent.atomic.AtomicReference) Loggers(org.opensearch.common.logging.Loggers) UUIDs(org.opensearch.common.UUIDs) Iterator(java.util.Iterator) Matchers(org.hamcrest.Matchers) RootObjectMapper(org.opensearch.index.mapper.RootObjectMapper) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) TermQuery(org.apache.lucene.search.TermQuery) Closeable(java.io.Closeable) IndexRequest(org.opensearch.action.index.IndexRequest) Collections(java.util.Collections) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) TranslogConfig(org.opensearch.index.translog.TranslogConfig) ArrayList(java.util.ArrayList) Store(org.opensearch.index.store.Store) LongPoint(org.apache.lucene.document.LongPoint) CodecService(org.opensearch.index.codec.CodecService)

Example 3 with OpenSearchDirectoryReader

use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.

the class ReadOnlyEngineTests method testReadOnlyEngine.

public void testReadOnlyEngine() throws Exception {
    IOUtils.close(engine, store);
    Engine readOnlyEngine = null;
    final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
    try (Store store = createStore()) {
        EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
        int numDocs = scaledRandomIntBetween(10, 1000);
        final SeqNoStats lastSeqNoStats;
        final List<DocIdSeqNoAndSource> lastDocIds;
        try (InternalEngine engine = createEngine(config)) {
            Engine.Get get = null;
            for (int i = 0; i < numDocs; i++) {
                ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
                engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
                if (get == null || rarely()) {
                    get = newGet(randomBoolean(), doc);
                }
                if (rarely()) {
                    engine.flush();
                }
                globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
            }
            engine.syncTranslog();
            globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
            engine.flush();
            readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, engine.getSeqNoStats(globalCheckpoint.get()), engine.getTranslogStats(), false, Function.identity(), true);
            lastSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
            lastDocIds = getDocIds(engine, true);
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
            for (int i = 0; i < numDocs; i++) {
                if (randomBoolean()) {
                    String delId = Integer.toString(i);
                    engine.delete(new Engine.Delete("test", delId, newUid(delId), primaryTerm.get()));
                }
                if (rarely()) {
                    engine.flush();
                }
            }
            Engine.Searcher external = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL);
            Engine.Searcher internal = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
            assertSame(external.getIndexReader(), internal.getIndexReader());
            assertThat(external.getIndexReader(), instanceOf(DirectoryReader.class));
            DirectoryReader dirReader = external.getDirectoryReader();
            OpenSearchDirectoryReader esReader = getOpenSearchDirectoryReader(dirReader);
            IndexReader.CacheHelper helper = esReader.getReaderCacheHelper();
            assertNotNull(helper);
            assertEquals(helper.getKey(), dirReader.getReaderCacheHelper().getKey());
            IOUtils.close(external, internal);
            // the locked down engine should still point to the previous commit
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
            try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) {
                assertTrue(getResult.exists());
            }
        }
        // Close and reopen the main engine
        try (InternalEngine recoveringEngine = new InternalEngine(config)) {
            recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
            // the locked down engine should still point to the previous commit
            assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
            assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
            assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
        }
    } finally {
        IOUtils.close(readOnlyEngine);
    }
}
Also used : BytesArray(org.opensearch.common.bytes.BytesArray) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) OpenSearchDirectoryReader.getOpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader.getOpenSearchDirectoryReader) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) DirectoryReader(org.apache.lucene.index.DirectoryReader) OpenSearchDirectoryReader.getOpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader.getOpenSearchDirectoryReader) Store(org.opensearch.index.store.Store) AtomicLong(java.util.concurrent.atomic.AtomicLong) SeqNoStats(org.opensearch.index.seqno.SeqNoStats) ParsedDocument(org.opensearch.index.mapper.ParsedDocument) IndexReader(org.apache.lucene.index.IndexReader)

Example 4 with OpenSearchDirectoryReader

use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.

the class ShardUtilsTests method testExtractShardId.

public void testExtractShardId() throws IOException {
    BaseDirectoryWrapper dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.commit();
    ShardId id = new ShardId("foo", "_na_", random().nextInt());
    try (DirectoryReader reader = DirectoryReader.open(writer)) {
        OpenSearchDirectoryReader wrap = OpenSearchDirectoryReader.wrap(reader, id);
        assertEquals(id, ShardUtils.extractShardId(wrap));
    }
    final int numDocs = 1 + random().nextInt(5);
    for (int i = 0; i < numDocs; i++) {
        Document d = new Document();
        d.add(newField("name", "foobar", StringField.TYPE_STORED));
        writer.addDocument(d);
        if (random().nextBoolean()) {
            writer.commit();
        }
    }
    try (DirectoryReader reader = DirectoryReader.open(writer)) {
        OpenSearchDirectoryReader wrap = OpenSearchDirectoryReader.wrap(reader, id);
        assertEquals(id, ShardUtils.extractShardId(wrap));
        CompositeReaderContext context = wrap.getContext();
        for (LeafReaderContext leaf : context.leaves()) {
            assertEquals(id, ShardUtils.extractShardId(leaf.reader()));
        }
    }
    IOUtils.close(writer, dir);
}
Also used : OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) CompositeReaderContext(org.apache.lucene.index.CompositeReaderContext) IndexWriter(org.apache.lucene.index.IndexWriter) OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) DirectoryReader(org.apache.lucene.index.DirectoryReader) BaseDirectoryWrapper(org.apache.lucene.store.BaseDirectoryWrapper) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) Document(org.apache.lucene.document.Document)

Example 5 with OpenSearchDirectoryReader

use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.

the class InternalEngine method createReaderManager.

private ExternalReaderManager createReaderManager(RefreshWarmerListener externalRefreshListener) throws EngineException {
    boolean success = false;
    OpenSearchReaderManager internalReaderManager = null;
    try {
        try {
            final OpenSearchDirectoryReader directoryReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
            internalReaderManager = new OpenSearchReaderManager(directoryReader);
            lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
            ExternalReaderManager externalReaderManager = new ExternalReaderManager(internalReaderManager, externalRefreshListener);
            success = true;
            return externalReaderManager;
        } catch (IOException e) {
            maybeFailEngine("start", e);
            try {
                indexWriter.rollback();
            } catch (IOException inner) {
                // iw is closed below
                e.addSuppressed(inner);
            }
            throw new EngineCreationFailureException(shardId, "failed to open reader on writer", e);
        }
    } finally {
        if (success == false) {
            // release everything we created on a failure
            IOUtils.closeWhileHandlingException(internalReaderManager, indexWriter);
        }
    }
}
Also used : OpenSearchDirectoryReader(org.opensearch.common.lucene.index.OpenSearchDirectoryReader) IOException(java.io.IOException)

Aggregations

OpenSearchDirectoryReader (org.opensearch.common.lucene.index.OpenSearchDirectoryReader)7 DirectoryReader (org.apache.lucene.index.DirectoryReader)5 IOException (java.io.IOException)3 LeafReaderContext (org.apache.lucene.index.LeafReaderContext)3 UncheckedIOException (java.io.UncheckedIOException)2 HashSet (java.util.HashSet)2 AtomicLong (java.util.concurrent.atomic.AtomicLong)2 IndexReader (org.apache.lucene.index.IndexReader)2 IndexWriter (org.apache.lucene.index.IndexWriter)2 IndexSearcher (org.apache.lucene.search.IndexSearcher)2 AlreadyClosedException (org.apache.lucene.store.AlreadyClosedException)2 BytesArray (org.opensearch.common.bytes.BytesArray)2 ParsedDocument (org.opensearch.index.mapper.ParsedDocument)2 SeqNoStats (org.opensearch.index.seqno.SeqNoStats)2 Store (org.opensearch.index.store.Store)2 ObjectObjectCursor (com.carrotsearch.hppc.cursors.ObjectObjectCursor)1 RandomNumbers (com.carrotsearch.randomizedtesting.generators.RandomNumbers)1 Closeable (java.io.Closeable)1 FileNotFoundException (java.io.FileNotFoundException)1 Charset (java.nio.charset.Charset)1