Examples with InternalEngine org.opensearch.index.engine.InternalEngine used on opensource projects

Example 6 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexShardTests method testCloseShardWhileResettingEngine.

/**
 * This test simulates a scenario seen rarely in ConcurrentSeqNoVersioningIT. Closing a shard while engine is inside
 * resetEngineToGlobalCheckpoint can lead to check index failure in integration tests.
 */
public void testCloseShardWhileResettingEngine() throws Exception {
    CountDownLatch readyToCloseLatch = new CountDownLatch(1);
    CountDownLatch closeDoneLatch = new CountDownLatch(1);
    IndexShard shard = newStartedShard(false, Settings.EMPTY, config -> new InternalEngine(config) {

        @Override
        public InternalEngine recoverFromTranslog(TranslogRecoveryRunner translogRecoveryRunner, long recoverUpToSeqNo) throws IOException {
            readyToCloseLatch.countDown();
            try {
                closeDoneLatch.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
            return super.recoverFromTranslog(translogRecoveryRunner, recoverUpToSeqNo);
        }
    });
    Thread closeShardThread = new Thread(() -> {
        try {
            readyToCloseLatch.await();
            shard.close("testing", false);
            // in integration tests, this is done as a listener on IndexService.
            MockFSDirectoryFactory.checkIndex(logger, shard.store(), shard.shardId);
        } catch (InterruptedException | IOException e) {
            throw new AssertionError(e);
        } finally {
            closeDoneLatch.countDown();
        }
    });
    closeShardThread.start();
    final CountDownLatch engineResetLatch = new CountDownLatch(1);
    shard.acquireAllReplicaOperationsPermits(shard.getOperationPrimaryTerm(), shard.getLastKnownGlobalCheckpoint(), 0L, ActionListener.wrap(r -> {
        try (Releasable dummy = r) {
            shard.resetEngineToGlobalCheckpoint();
        } finally {
            engineResetLatch.countDown();
        }
    }, Assert::assertNotNull), TimeValue.timeValueMinutes(1L));
    engineResetLatch.await();
    closeShardThread.join();
    // close store.
    closeShard(shard, false);
}

Example 7 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexShardTests method testRecordsForceMerges.

public void testRecordsForceMerges() throws IOException {
    IndexShard shard = newStartedShard(true);
    final String initialForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
    assertThat(initialForceMergeUUID, nullValue());
    final ForceMergeRequest firstForceMergeRequest = new ForceMergeRequest().maxNumSegments(1);
    shard.forceMerge(firstForceMergeRequest);
    final String secondForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
    assertThat(secondForceMergeUUID, notNullValue());
    assertThat(secondForceMergeUUID, equalTo(firstForceMergeRequest.forceMergeUUID()));
    final ForceMergeRequest secondForceMergeRequest = new ForceMergeRequest().maxNumSegments(1);
    shard.forceMerge(secondForceMergeRequest);
    final String thirdForceMergeUUID = ((InternalEngine) shard.getEngine()).getForceMergeUUID();
    assertThat(thirdForceMergeUUID, notNullValue());
    assertThat(thirdForceMergeUUID, not(equalTo(secondForceMergeUUID)));
    assertThat(thirdForceMergeUUID, equalTo(secondForceMergeRequest.forceMergeUUID()));
    closeShards(shard);
}

Example 8 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexLevelReplicationTests method testAppendOnlyRecoveryThenReplication.

public void testAppendOnlyRecoveryThenReplication() throws Exception {
    CountDownLatch indexedOnPrimary = new CountDownLatch(1);
    CountDownLatch recoveryDone = new CountDownLatch(1);
    try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetadata(1)) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            return config -> new InternalEngine(config) {

                @Override
                public IndexResult index(Index op) throws IOException {
                    IndexResult result = super.index(op);
                    if (op.origin() == Operation.Origin.PRIMARY) {
                        indexedOnPrimary.countDown();
                        // to make sure that this operation is replicated to the replica via recovery, then via replication.
                        try {
                            recoveryDone.await();
                        } catch (InterruptedException e) {
                            throw new AssertionError(e);
                        }
                    }
                    return result;
                }
            };
        }
    }) {
        shards.startAll();
        Thread thread = new Thread(() -> {
            IndexRequest indexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON);
            try {
                shards.index(indexRequest);
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        });
        thread.start();
        IndexShard replica = shards.addReplica();
        Future<Void> fut = shards.asyncRecoverReplica(replica, (shard, node) -> new RecoveryTarget(shard, node, recoveryListener) {

            @Override
            public void prepareForTranslogOperations(int totalTranslogOps, ActionListener<Void> listener) {
                try {
                    indexedOnPrimary.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
                super.prepareForTranslogOperations(totalTranslogOps, listener);
            }
        });
        fut.get();
        recoveryDone.countDown();
        thread.join();
        shards.assertAllEqual(1);
    }
}

Example 9 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project k-NN by opensearch-project.

the class KNNEngineFactory method newReadWriteEngine.

@Override
public Engine newReadWriteEngine(EngineConfig config) {
    codecService.setPostingsFormat(config.getCodec().postingsFormat());
    EngineConfig engineConfig = new EngineConfig(config.getShardId(), config.getThreadPool(), config.getIndexSettings(), config.getWarmer(), config.getStore(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), codecService, config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), config.getTombstoneDocSupplier());
    return new InternalEngine(engineConfig);
}