Examples with EngineConfig org.opensearch.index.engine.EngineConfig used on opensource projects

Example 1 with EngineConfig

use of org.opensearch.index.engine.EngineConfig in project OpenSearch by opensearch-project.

the class IndexShardTests method testCloseShardWhileEngineIsWarming.

public void testCloseShardWhileEngineIsWarming() throws Exception {
    CountDownLatch warmerStarted = new CountDownLatch(1);
    CountDownLatch warmerBlocking = new CountDownLatch(1);
    IndexShard shard = newShard(true, Settings.EMPTY, config -> {
        Engine.Warmer warmer = reader -> {
            try {
                warmerStarted.countDown();
                warmerBlocking.await();
                config.getWarmer().warm(reader);
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
        };
        EngineConfig configWithWarmer = new EngineConfig(config.getShardId(), config.getThreadPool(), config.getIndexSettings(), warmer, config.getStore(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), config.getTombstoneDocSupplier());
        return new InternalEngine(configWithWarmer);
    });
    Thread recoveryThread = new Thread(() -> expectThrows(AlreadyClosedException.class, () -> recoverShardFromStore(shard)));
    recoveryThread.start();
    try {
        warmerStarted.await();
        shard.close("testing", false);
        assertThat(shard.state, equalTo(IndexShardState.CLOSED));
    } finally {
        warmerBlocking.countDown();
    }
    recoveryThread.join();
    shard.store().close();
}

Example 2 with EngineConfig

use of org.opensearch.index.engine.EngineConfig in project OpenSearch by opensearch-project.

the class RefreshListenersTests method setupListeners.

@Before
public void setupListeners() throws Exception {
    // Setup dependencies of the listeners
    maxListeners = randomIntBetween(1, 1000);
    // Now setup the InternalEngine which is much more complicated because we aren't mocking anything
    threadPool = new TestThreadPool(getTestName());
    refreshMetric = new MeanMetric();
    listeners = new RefreshListeners(() -> maxListeners, () -> engine.refresh("too-many-listeners"), logger, threadPool.getThreadContext(), refreshMetric);
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
    ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
    String allocationId = UUIDs.randomBase64UUID(random());
    Directory directory = newDirectory();
    store = new Store(shardId, indexSettings, directory, new DummyShardLock(shardId));
    IndexWriterConfig iwc = newIndexWriterConfig();
    TranslogConfig translogConfig = new TranslogConfig(shardId, createTempDir("translog"), indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    Engine.EventListener eventListener = new Engine.EventListener() {

        @Override
        public void onFailedEngine(String reason, @Nullable Exception e) {
        // we don't need to notify anybody in this test
        }
    };
    store.createEmpty(Version.CURRENT.luceneVersion);
    final long primaryTerm = randomNonNegativeLong();
    final String translogUUID = Translog.createEmptyTranslog(translogConfig.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm);
    store.associateIndexWithNewTranslog(translogUUID);
    EngineConfig config = new EngineConfig(shardId, threadPool, indexSettings, null, store, newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), Collections.emptyList(), null, new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED, () -> RetentionLeases.EMPTY, () -> primaryTerm, EngineTestCase.tombstoneDocSupplier());
    engine = new InternalEngine(config);
    engine.recoverFromTranslog((e, s) -> 0, Long.MAX_VALUE);
    listeners.setCurrentRefreshLocationSupplier(engine::getTranslogLastWriteLocation);
}

Example 3 with EngineConfig

use of org.opensearch.index.engine.EngineConfig in project OpenSearch by opensearch-project.

the class IndexingMemoryControllerTests method configWithRefreshListener.

EngineConfig configWithRefreshListener(EngineConfig config, ReferenceManager.RefreshListener listener) {
    final List<ReferenceManager.RefreshListener> internalRefreshListener = new ArrayList<>(config.getInternalRefreshListener());
    ;
    internalRefreshListener.add(listener);
    return new EngineConfig(config.getShardId(), config.getThreadPool(), config.getIndexSettings(), config.getWarmer(), config.getStore(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getExternalRefreshListener(), internalRefreshListener, config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), config.getTombstoneDocSupplier());
}

Example 4 with EngineConfig

use of org.opensearch.index.engine.EngineConfig in project OpenSearch by opensearch-project.

the class IndexShard method newEngineConfig.

private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) {
    final Sort indexSort = indexSortSupplier.get();
    final Engine.Warmer warmer = reader -> {
        assert Thread.holdsLock(mutex) == false : "warming engine under mutex";
        assert reader != null;
        if (this.warmer != null) {
            this.warmer.warm(reader);
        }
    };
    return this.engineConfigFactory.newEngineConfig(shardId, threadPool, indexSettings, warmer, store, indexSettings.getMergePolicy(), mapperService != null ? mapperService.indexAnalyzer() : null, similarityService.similarity(mapperService), engineConfigFactory.newCodecServiceOrDefault(indexSettings, mapperService, logger, codecService), shardEventListener, indexCache != null ? indexCache.query() : null, cachingPolicy, translogConfig, IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.get(indexSettings.getSettings()), Arrays.asList(refreshListeners, refreshPendingLocationListener), Collections.singletonList(new RefreshMetricUpdater(refreshMetric)), indexSort, circuitBreakerService, globalCheckpointSupplier, replicationTracker::getRetentionLeases, () -> getOperationPrimaryTerm(), tombstoneDocSupplier());
}

Example 5 with EngineConfig

use of org.opensearch.index.engine.EngineConfig in project OpenSearch by opensearch-project.

the class IndexShard method innerOpenEngineAndTranslog.

private void innerOpenEngineAndTranslog(LongSupplier globalCheckpointSupplier) throws IOException {
    assert Thread.holdsLock(mutex) == false : "opening engine under mutex";
    if (state != IndexShardState.RECOVERING) {
        throw new IndexShardNotRecoveringException(shardId, state);
    }
    final EngineConfig config = newEngineConfig(globalCheckpointSupplier);
    // we disable deletes since we allow for operations to be executed against the shard while recovering
    // but we need to make sure we don't loose deletes until we are done recovering
    config.setEnableGcDeletes(false);
    updateRetentionLeasesOnReplica(loadRetentionLeases());
    assert recoveryState.getRecoverySource().expectEmptyRetentionLeases() == false || getRetentionLeases().leases().isEmpty() : "expected empty set of retention leases with recovery source [" + recoveryState.getRecoverySource() + "] but got " + getRetentionLeases();
    synchronized (engineMutex) {
        assert currentEngineReference.get() == null : "engine is running";
        verifyNotClosed();
        // we must create a new engine under mutex (see IndexShard#snapshotStoreMetadata).
        final Engine newEngine = engineFactory.newReadWriteEngine(config);
        onNewEngine(newEngine);
        currentEngineReference.set(newEngine);
        // We set active because we are now writing operations to the engine; this way,
        // we can flush if we go idle after some time and become inactive.
        active.set(true);
    }
    // time elapses after the engine is created above (pulling the config settings) until we set the engine reference, during
    // which settings changes could possibly have happened, so here we forcefully push any config changes to the new engine.
    onSettingsChanged();
    assert assertSequenceNumbersInCommit();
    recoveryState.validateCurrentStage(RecoveryState.Stage.TRANSLOG);
}