Examples with InternalEngine org.opensearch.index.engine.InternalEngine used on opensource projects

Example 1 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexLevelReplicationTests method testAppendWhileRecovering.

public void testAppendWhileRecovering() throws Exception {
    try (ReplicationGroup shards = createGroup(0)) {
        shards.startAll();
        CountDownLatch latch = new CountDownLatch(2);
        int numDocs = randomIntBetween(100, 200);
        // just append one to the translog so we can assert below
        shards.appendDocs(1);
        Thread thread = new Thread() {

            @Override
            public void run() {
                try {
                    latch.countDown();
                    latch.await();
                    shards.appendDocs(numDocs - 1);
                } catch (Exception e) {
                    throw new AssertionError(e);
                }
            }
        };
        thread.start();
        IndexShard replica = shards.addReplica();
        Future<Void> future = shards.asyncRecoverReplica(replica, (indexShard, node) -> new RecoveryTarget(indexShard, node, recoveryListener) {

            @Override
            public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetadata, ActionListener<Void> listener) {
                super.cleanFiles(totalTranslogOps, globalCheckpoint, sourceMetadata, ActionListener.runAfter(listener, () -> {
                    latch.countDown();
                    try {
                        latch.await();
                    } catch (InterruptedException e) {
                        throw new AssertionError(e);
                    }
                }));
            }
        });
        future.get();
        thread.join();
        shards.assertAllEqual(numDocs);
        Engine engine = IndexShardTests.getEngineFromShard(shards.getPrimary());
        assertEquals(0, InternalEngineTests.getNumIndexVersionsLookups((InternalEngine) engine));
        assertEquals(0, InternalEngineTests.getNumVersionLookups((InternalEngine) engine));
    }
}

Example 2 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexShardTests method testCloseShardWhileEngineIsWarming.

public void testCloseShardWhileEngineIsWarming() throws Exception {
    CountDownLatch warmerStarted = new CountDownLatch(1);
    CountDownLatch warmerBlocking = new CountDownLatch(1);
    IndexShard shard = newShard(true, Settings.EMPTY, config -> {
        Engine.Warmer warmer = reader -> {
            try {
                warmerStarted.countDown();
                warmerBlocking.await();
                config.getWarmer().warm(reader);
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
        };
        EngineConfig configWithWarmer = new EngineConfig(config.getShardId(), config.getThreadPool(), config.getIndexSettings(), warmer, config.getStore(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(), new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getExternalRefreshListener(), config.getInternalRefreshListener(), config.getIndexSort(), config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.retentionLeasesSupplier(), config.getPrimaryTermSupplier(), config.getTombstoneDocSupplier());
        return new InternalEngine(configWithWarmer);
    });
    Thread recoveryThread = new Thread(() -> expectThrows(AlreadyClosedException.class, () -> recoverShardFromStore(shard)));
    recoveryThread.start();
    try {
        warmerStarted.await();
        shard.close("testing", false);
        assertThat(shard.state, equalTo(IndexShardState.CLOSED));
    } finally {
        warmerBlocking.countDown();
    }
    recoveryThread.join();
    shard.store().close();
}

Example 3 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class RefreshListenersTests method setupListeners.

@Before
public void setupListeners() throws Exception {
    // Setup dependencies of the listeners
    maxListeners = randomIntBetween(1, 1000);
    // Now setup the InternalEngine which is much more complicated because we aren't mocking anything
    threadPool = new TestThreadPool(getTestName());
    refreshMetric = new MeanMetric();
    listeners = new RefreshListeners(() -> maxListeners, () -> engine.refresh("too-many-listeners"), logger, threadPool.getThreadContext(), refreshMetric);
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
    ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
    String allocationId = UUIDs.randomBase64UUID(random());
    Directory directory = newDirectory();
    store = new Store(shardId, indexSettings, directory, new DummyShardLock(shardId));
    IndexWriterConfig iwc = newIndexWriterConfig();
    TranslogConfig translogConfig = new TranslogConfig(shardId, createTempDir("translog"), indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    Engine.EventListener eventListener = new Engine.EventListener() {

        @Override
        public void onFailedEngine(String reason, @Nullable Exception e) {
        // we don't need to notify anybody in this test
        }
    };
    store.createEmpty(Version.CURRENT.luceneVersion);
    final long primaryTerm = randomNonNegativeLong();
    final String translogUUID = Translog.createEmptyTranslog(translogConfig.getTranslogPath(), SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm);
    store.associateIndexWithNewTranslog(translogUUID);
    EngineConfig config = new EngineConfig(shardId, threadPool, indexSettings, null, store, newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), Collections.singletonList(listeners), Collections.emptyList(), null, new NoneCircuitBreakerService(), () -> SequenceNumbers.NO_OPS_PERFORMED, () -> RetentionLeases.EMPTY, () -> primaryTerm, EngineTestCase.tombstoneDocSupplier());
    engine = new InternalEngine(config);
    engine.recoverFromTranslog((e, s) -> 0, Long.MAX_VALUE);
    listeners.setCurrentRefreshLocationSupplier(engine::getTranslogLastWriteLocation);
}

Example 4 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexingMemoryControllerTests method testSkipRefreshIfShardIsRefreshingAlready.

public void testSkipRefreshIfShardIsRefreshingAlready() throws Exception {
    SetOnce<CountDownLatch> refreshLatch = new SetOnce<>();
    ReferenceManager.RefreshListener refreshListener = new ReferenceManager.RefreshListener() {

        @Override
        public void beforeRefresh() {
            if (refreshLatch.get() != null) {
                try {
                    refreshLatch.get().await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
            }
        }

        @Override
        public void afterRefresh(boolean didRefresh) {
        }
    };
    IndexShard shard = newStartedShard(randomBoolean(), Settings.EMPTY, config -> new InternalEngine(configWithRefreshListener(config, refreshListener)));
    // block refresh
    refreshLatch.set(new CountDownLatch(1));
    final RefreshStats refreshStats = shard.refreshStats();
    final IndexingMemoryController controller = new IndexingMemoryController(Settings.builder().put("indices.memory.interval", // disable it
    "200h").put("indices.memory.index_buffer_size", "1024b").build(), threadPool, Collections.singleton(shard)) {

        @Override
        protected long getIndexBufferRAMBytesUsed(IndexShard shard) {
            return randomLongBetween(1025, 10 * 1024 * 1024);
        }

        @Override
        protected long getShardWritingBytes(IndexShard shard) {
            return 0L;
        }
    };
    int iterations = randomIntBetween(10, 100);
    ThreadPoolStats.Stats beforeStats = getRefreshThreadPoolStats();
    for (int i = 0; i < iterations; i++) {
        controller.forceCheck();
    }
    assertBusy(() -> {
        ThreadPoolStats.Stats stats = getRefreshThreadPoolStats();
        assertThat(stats.getCompleted(), equalTo(beforeStats.getCompleted() + iterations - 1));
    });
    // allow refresh
    refreshLatch.get().countDown();
    assertBusy(() -> {
        ThreadPoolStats.Stats stats = getRefreshThreadPoolStats();
        assertThat(stats.getCompleted(), equalTo(beforeStats.getCompleted() + iterations));
    });
    assertThat(shard.refreshStats().getTotal(), equalTo(refreshStats.getTotal() + 1));
    closeShards(shard);
}

Example 5 with InternalEngine

use of org.opensearch.index.engine.InternalEngine in project OpenSearch by opensearch-project.

the class IndexLevelReplicationTests method testAppendOnlyRecoveryThenReplication.

public void testAppendOnlyRecoveryThenReplication() throws Exception {
    CountDownLatch indexedOnPrimary = new CountDownLatch(1);
    CountDownLatch recoveryDone = new CountDownLatch(1);
    try (ReplicationGroup shards = new ReplicationGroup(buildIndexMetadata(1)) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            return config -> new InternalEngine(config) {

                @Override
                public IndexResult index(Index op) throws IOException {
                    IndexResult result = super.index(op);
                    if (op.origin() == Operation.Origin.PRIMARY) {
                        indexedOnPrimary.countDown();
                        // to make sure that this operation is replicated to the replica via recovery, then via replication.
                        try {
                            recoveryDone.await();
                        } catch (InterruptedException e) {
                            throw new AssertionError(e);
                        }
                    }
                    return result;
                }
            };
        }
    }) {
        shards.startAll();
        Thread thread = new Thread(() -> {
            IndexRequest indexRequest = new IndexRequest(index.getName()).source("{}", XContentType.JSON);
            try {
                shards.index(indexRequest);
            } catch (Exception e) {
                throw new AssertionError(e);
            }
        });
        thread.start();
        IndexShard replica = shards.addReplica();
        Future<Void> fut = shards.asyncRecoverReplica(replica, (shard, node) -> new RecoveryTarget(shard, node, recoveryListener) {

            @Override
            public void prepareForTranslogOperations(int totalTranslogOps, ActionListener<Void> listener) {
                try {
                    indexedOnPrimary.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
                super.prepareForTranslogOperations(totalTranslogOps, listener);
            }
        });
        fut.get();
        recoveryDone.countDown();
        thread.join();
        shards.assertAllEqual(1);
    }
}