use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.
the class Engine method acquireSearcherSupplier.
/**
* Acquires a point-in-time reader that can be used to create {@link Engine.Searcher}s on demand.
*/
public SearcherSupplier acquireSearcherSupplier(Function<Searcher, Searcher> wrapper, SearcherScope scope) throws EngineException {
/* Acquire order here is store -> manager since we need
* to make sure that the store is not closed before
* the searcher is acquired. */
if (store.tryIncRef() == false) {
throw new AlreadyClosedException(shardId + " store is closed", failedEngine.get());
}
Releasable releasable = store::decRef;
try {
ReferenceManager<OpenSearchDirectoryReader> referenceManager = getReferenceManager(scope);
OpenSearchDirectoryReader acquire = referenceManager.acquire();
SearcherSupplier reader = new SearcherSupplier(wrapper) {
@Override
public Searcher acquireSearcherInternal(String source) {
assert assertSearcherIsWarmedUp(source, scope);
return new Searcher(source, acquire, engineConfig.getSimilarity(), engineConfig.getQueryCache(), engineConfig.getQueryCachingPolicy(), () -> {
});
}
@Override
protected void doClose() {
try {
referenceManager.release(acquire);
} catch (IOException e) {
throw new UncheckedIOException("failed to close", e);
} catch (AlreadyClosedException e) {
// This means there's a bug somewhere: don't suppress it
throw new AssertionError(e);
} finally {
store.decRef();
}
}
};
// success - hand over the reference to the engine reader
releasable = null;
return reader;
} catch (AlreadyClosedException ex) {
throw ex;
} catch (Exception ex) {
maybeFailEngine("acquire_reader", ex);
// throw EngineCloseException here if we are already closed
ensureOpen(ex);
logger.error(() -> new ParameterizedMessage("failed to acquire reader"), ex);
throw new EngineException(shardId, "failed to acquire reader", ex);
} finally {
Releasables.close(releasable);
}
}
use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.
the class InternalEngineTests method testNotWarmUpSearcherInEngineCtor.
public void testNotWarmUpSearcherInEngineCtor() throws Exception {
try (Store store = createStore()) {
List<OpenSearchDirectoryReader> warmedUpReaders = new ArrayList<>();
Engine.Warmer warmer = reader -> {
assertNotNull(reader);
assertThat(reader, not(in(warmedUpReaders)));
warmedUpReaders.add(reader);
};
EngineConfig config = engine.config();
final TranslogConfig translogConfig = new TranslogConfig(config.getTranslogConfig().getShardId(), createTempDir(), config.getTranslogConfig().getIndexSettings(), config.getTranslogConfig().getBigArrays());
EngineConfig configWithWarmer = new EngineConfig(config.getShardId(), config.getThreadPool(), config.getIndexSettings(), warmer, store, config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), translogConfig, config.getFlushMergesAfter(), config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), config.getTombstoneDocSupplier());
try (InternalEngine engine = createEngine(configWithWarmer)) {
assertThat(warmedUpReaders, empty());
assertThat(expectThrows(Throwable.class, () -> engine.acquireSearcher("test")).getMessage(), equalTo("searcher was not warmed up yet for source[test]"));
int times = randomIntBetween(1, 10);
for (int i = 0; i < times; i++) {
engine.refresh("test");
}
assertThat(warmedUpReaders, hasSize(1));
try (Engine.Searcher internalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
try (Engine.Searcher externalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)) {
assertSame(internalSearcher.getDirectoryReader(), externalSearcher.getDirectoryReader());
assertSame(warmedUpReaders.get(0), externalSearcher.getDirectoryReader());
}
}
index(engine, randomInt());
if (randomBoolean()) {
engine.refresh("test", Engine.SearcherScope.INTERNAL, true);
assertThat(warmedUpReaders, hasSize(1));
try (Engine.Searcher internalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
try (Engine.Searcher externalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)) {
assertNotSame(internalSearcher.getDirectoryReader(), externalSearcher.getDirectoryReader());
}
}
}
engine.refresh("test");
assertThat(warmedUpReaders, hasSize(2));
try (Engine.Searcher internalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.INTERNAL)) {
try (Engine.Searcher externalSearcher = engine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL)) {
assertSame(internalSearcher.getDirectoryReader(), externalSearcher.getDirectoryReader());
assertSame(warmedUpReaders.get(1), externalSearcher.getDirectoryReader());
}
}
}
}
}
use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.
the class ReadOnlyEngineTests method testReadOnlyEngine.
public void testReadOnlyEngine() throws Exception {
IOUtils.close(engine, store);
Engine readOnlyEngine = null;
final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);
try (Store store = createStore()) {
EngineConfig config = config(defaultSettings, store, createTempDir(), newMergePolicy(), null, null, globalCheckpoint::get);
int numDocs = scaledRandomIntBetween(10, 1000);
final SeqNoStats lastSeqNoStats;
final List<DocIdSeqNoAndSource> lastDocIds;
try (InternalEngine engine = createEngine(config)) {
Engine.Get get = null;
for (int i = 0; i < numDocs; i++) {
ParsedDocument doc = testParsedDocument(Integer.toString(i), null, testDocument(), new BytesArray("{}"), null);
engine.index(new Engine.Index(newUid(doc), doc, i, primaryTerm.get(), 1, null, Engine.Operation.Origin.REPLICA, System.nanoTime(), -1, false, SequenceNumbers.UNASSIGNED_SEQ_NO, 0));
if (get == null || rarely()) {
get = newGet(randomBoolean(), doc);
}
if (rarely()) {
engine.flush();
}
globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
}
engine.syncTranslog();
globalCheckpoint.set(randomLongBetween(globalCheckpoint.get(), engine.getPersistedLocalCheckpoint()));
engine.flush();
readOnlyEngine = new ReadOnlyEngine(engine.engineConfig, engine.getSeqNoStats(globalCheckpoint.get()), engine.getTranslogStats(), false, Function.identity(), true);
lastSeqNoStats = engine.getSeqNoStats(globalCheckpoint.get());
lastDocIds = getDocIds(engine, true);
assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
for (int i = 0; i < numDocs; i++) {
if (randomBoolean()) {
String delId = Integer.toString(i);
engine.delete(new Engine.Delete("test", delId, newUid(delId), primaryTerm.get()));
}
if (rarely()) {
engine.flush();
}
}
Engine.Searcher external = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.EXTERNAL);
Engine.Searcher internal = readOnlyEngine.acquireSearcher("test", Engine.SearcherScope.INTERNAL);
assertSame(external.getIndexReader(), internal.getIndexReader());
assertThat(external.getIndexReader(), instanceOf(DirectoryReader.class));
DirectoryReader dirReader = external.getDirectoryReader();
OpenSearchDirectoryReader esReader = getOpenSearchDirectoryReader(dirReader);
IndexReader.CacheHelper helper = esReader.getReaderCacheHelper();
assertNotNull(helper);
assertEquals(helper.getKey(), dirReader.getReaderCacheHelper().getKey());
IOUtils.close(external, internal);
// the locked down engine should still point to the previous commit
assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
try (Engine.GetResult getResult = readOnlyEngine.get(get, readOnlyEngine::acquireSearcher)) {
assertTrue(getResult.exists());
}
}
// Close and reopen the main engine
try (InternalEngine recoveringEngine = new InternalEngine(config)) {
recoveringEngine.recoverFromTranslog(translogHandler, Long.MAX_VALUE);
// the locked down engine should still point to the previous commit
assertThat(readOnlyEngine.getPersistedLocalCheckpoint(), equalTo(lastSeqNoStats.getLocalCheckpoint()));
assertThat(readOnlyEngine.getSeqNoStats(globalCheckpoint.get()).getMaxSeqNo(), equalTo(lastSeqNoStats.getMaxSeqNo()));
assertThat(getDocIds(readOnlyEngine, false), equalTo(lastDocIds));
}
} finally {
IOUtils.close(readOnlyEngine);
}
}
use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.
the class ShardUtilsTests method testExtractShardId.
public void testExtractShardId() throws IOException {
BaseDirectoryWrapper dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
writer.commit();
ShardId id = new ShardId("foo", "_na_", random().nextInt());
try (DirectoryReader reader = DirectoryReader.open(writer)) {
OpenSearchDirectoryReader wrap = OpenSearchDirectoryReader.wrap(reader, id);
assertEquals(id, ShardUtils.extractShardId(wrap));
}
final int numDocs = 1 + random().nextInt(5);
for (int i = 0; i < numDocs; i++) {
Document d = new Document();
d.add(newField("name", "foobar", StringField.TYPE_STORED));
writer.addDocument(d);
if (random().nextBoolean()) {
writer.commit();
}
}
try (DirectoryReader reader = DirectoryReader.open(writer)) {
OpenSearchDirectoryReader wrap = OpenSearchDirectoryReader.wrap(reader, id);
assertEquals(id, ShardUtils.extractShardId(wrap));
CompositeReaderContext context = wrap.getContext();
for (LeafReaderContext leaf : context.leaves()) {
assertEquals(id, ShardUtils.extractShardId(leaf.reader()));
}
}
IOUtils.close(writer, dir);
}
use of org.opensearch.common.lucene.index.OpenSearchDirectoryReader in project OpenSearch by opensearch-project.
the class InternalEngine method createReaderManager.
private ExternalReaderManager createReaderManager(RefreshWarmerListener externalRefreshListener) throws EngineException {
boolean success = false;
OpenSearchReaderManager internalReaderManager = null;
try {
try {
final OpenSearchDirectoryReader directoryReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(indexWriter), shardId);
internalReaderManager = new OpenSearchReaderManager(directoryReader);
lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
ExternalReaderManager externalReaderManager = new ExternalReaderManager(internalReaderManager, externalRefreshListener);
success = true;
return externalReaderManager;
} catch (IOException e) {
maybeFailEngine("start", e);
try {
indexWriter.rollback();
} catch (IOException inner) {
// iw is closed below
e.addSuppressed(inner);
}
throw new EngineCreationFailureException(shardId, "failed to open reader on writer", e);
}
} finally {
if (success == false) {
// release everything we created on a failure
IOUtils.closeWhileHandlingException(internalReaderManager, indexWriter);
}
}
}
Aggregations