Examples with InternalEngine org.elasticsearch.index.engine.InternalEngine used on opensource projects

Example 1 with InternalEngine

use of org.elasticsearch.index.engine.InternalEngine in project elasticsearch by elastic.

the class IndexLevelReplicationTests method testAppendWhileRecovering.

public void testAppendWhileRecovering() throws Exception {
    try (ReplicationGroup shards = createGroup(0)) {
        shards.startAll();
        IndexShard replica = shards.addReplica();
        CountDownLatch latch = new CountDownLatch(2);
        int numDocs = randomIntBetween(100, 200);
        // just append one to the translog so we can assert below
        shards.appendDocs(1);
        Thread thread = new Thread() {

            @Override
            public void run() {
                try {
                    latch.countDown();
                    latch.await();
                    shards.appendDocs(numDocs - 1);
                } catch (Exception e) {
                    throw new AssertionError(e);
                }
            }
        };
        thread.start();
        Future<Void> future = shards.asyncRecoverReplica(replica, (indexShard, node) -> new RecoveryTarget(indexShard, node, recoveryListener, version -> {
        }) {

            @Override
            public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaData) throws IOException {
                super.cleanFiles(totalTranslogOps, sourceMetaData);
                latch.countDown();
                try {
                    latch.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
            }
        });
        future.get();
        thread.join();
        shards.assertAllEqual(numDocs);
        Engine engine = IndexShardTests.getEngineFromShard(replica);
        assertEquals("expected at no version lookups ", InternalEngineTests.getNumVersionLookups((InternalEngine) engine), 0);
        for (IndexShard shard : shards) {
            engine = IndexShardTests.getEngineFromShard(shard);
            assertEquals(0, InternalEngineTests.getNumIndexVersionsLookups((InternalEngine) engine));
            assertEquals(0, InternalEngineTests.getNumVersionLookups((InternalEngine) engine));
        }
    }
}

Example 2 with InternalEngine

use of org.elasticsearch.index.engine.InternalEngine in project elasticsearch by elastic.

the class RefreshListenersTests method setupListeners.

@Before
public void setupListeners() throws Exception {
    // Setup dependencies of the listeners
    maxListeners = randomIntBetween(1, 1000);
    listeners = new RefreshListeners(() -> maxListeners, () -> engine.refresh("too-many-listeners"), // Immediately run listeners rather than adding them to the listener thread pool like IndexShard does to simplify the test.
    Runnable::run, logger);
    // Now setup the InternalEngine which is much more complicated because we aren't mocking anything
    threadPool = new TestThreadPool(getTestName());
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("index", Settings.EMPTY);
    ShardId shardId = new ShardId(new Index("index", "_na_"), 1);
    Directory directory = newDirectory();
    DirectoryService directoryService = new DirectoryService(shardId, indexSettings) {

        @Override
        public Directory newDirectory() throws IOException {
            return directory;
        }
    };
    store = new Store(shardId, indexSettings, directoryService, new DummyShardLock(shardId));
    IndexWriterConfig iwc = newIndexWriterConfig();
    TranslogConfig translogConfig = new TranslogConfig(shardId, createTempDir("translog"), indexSettings, BigArrays.NON_RECYCLING_INSTANCE);
    Engine.EventListener eventListener = new Engine.EventListener() {

        @Override
        public void onFailedEngine(String reason, @Nullable Exception e) {
        // we don't need to notify anybody in this test
        }
    };
    TranslogHandler translogHandler = new TranslogHandler(xContentRegistry(), shardId.getIndexName(), logger);
    EngineConfig config = new EngineConfig(EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG, shardId, threadPool, indexSettings, null, store, new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()), newMergePolicy(), iwc.getAnalyzer(), iwc.getSimilarity(), new CodecService(null, logger), eventListener, translogHandler, IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), translogConfig, TimeValue.timeValueMinutes(5), listeners, IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP);
    engine = new InternalEngine(config);
    listeners.setTranslog(engine.getTranslog());
}

Example 3 with InternalEngine

use of org.elasticsearch.index.engine.InternalEngine in project crate by crate.

the class IndexingMemoryControllerTests method testSkipRefreshIfShardIsRefreshingAlready.

public void testSkipRefreshIfShardIsRefreshingAlready() throws Exception {
    SetOnce<CountDownLatch> refreshLatch = new SetOnce<>();
    ReferenceManager.RefreshListener refreshListener = new ReferenceManager.RefreshListener() {

        @Override
        public void beforeRefresh() {
            if (refreshLatch.get() != null) {
                try {
                    refreshLatch.get().await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                }
            }
        }

        @Override
        public void afterRefresh(boolean didRefresh) {
        }
    };
    IndexShard shard = newStartedShard(randomBoolean(), Settings.EMPTY, config -> new InternalEngine(configWithRefreshListener(config, refreshListener)));
    // block refresh
    refreshLatch.set(new CountDownLatch(1));
    final IndexingMemoryController controller = new IndexingMemoryController(// disable it
    Settings.builder().put("indices.memory.interval", "200h").put("indices.memory.index_buffer_size", "1024b").build(), threadPool, Collections.singleton(shard)) {

        @Override
        protected long getIndexBufferRAMBytesUsed(IndexShard shard) {
            return randomLongBetween(1025, 10 * 1024 * 1024);
        }

        @Override
        protected long getShardWritingBytes(IndexShard shard) {
            return 0L;
        }
    };
    int iterations = randomIntBetween(10, 100);
    ThreadPoolStats.Stats beforeStats = getRefreshThreadPoolStats();
    for (int i = 0; i < iterations; i++) {
        controller.forceCheck();
    }
    assertBusy(() -> {
        ThreadPoolStats.Stats stats = getRefreshThreadPoolStats();
        assertThat(stats.getCompleted(), equalTo(beforeStats.getCompleted() + iterations - 1));
    });
    // allow refresh
    refreshLatch.get().countDown();
    assertBusy(() -> {
        ThreadPoolStats.Stats stats = getRefreshThreadPoolStats();
        assertThat(stats.getCompleted(), equalTo(beforeStats.getCompleted() + iterations));
    });
    closeShards(shard);
}