Examples with EngineFactory org.elasticsearch.index.engine.EngineFactory used on opensource projects

Example 1 with EngineFactory

use of org.elasticsearch.index.engine.EngineFactory in project elasticsearch by elastic.

the class IndexShardTestCase method newShard.

/**
     * creates a new initializing shard.
     *
     * @param routing              shard routing to use
     * @param shardPath            path to use for shard data
     * @param indexMetaData        indexMetaData for the shard, including any mapping
     * @param indexSearcherWrapper an optional wrapper to be used during searchers
     * @param listeners            an optional set of listeners to add to the shard
     */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData, @Nullable IndexSearcherWrapper indexSearcherWrapper, Runnable globalCheckpointSyncer, @Nullable EngineFactory engineFactory, IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
    final IndexShard indexShard;
    final Store store = createStore(indexSettings, shardPath);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), indexSettings.getSettings());
        mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
        final IndexEventListener indexEventListener = new IndexEventListener() {
        };
        final Engine.Warmer warmer = searcher -> {
        };
        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
        });
        IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), mapperService);
        indexShard = new IndexShard(routing, indexSettings, shardPath, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, indexSearcherWrapper, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, globalCheckpointSyncer, Collections.emptyList(), Arrays.asList(listeners));
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}

Example 2 with EngineFactory

use of org.elasticsearch.index.engine.EngineFactory in project elasticsearch by elastic.

the class IndexService method createShard.

public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
    final boolean primary = routing.primary();
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false || (primary && this.indexSettings.isOnSharedFilesystem());
        final Engine.Warmer engineWarmer = (searcher) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(searcher, shard, IndexService.this.indexSettings);
            }
        };
        store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
        if (useShadowEngine(primary, this.indexSettings)) {
            indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners);
        // no indexing listeners - shadow  engines don't index
        } else {
            indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, () -> globalCheckpointSyncer.accept(shardId), searchOperationListeners, indexingOperationListeners);
        }
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}

Example 3 with EngineFactory

use of org.elasticsearch.index.engine.EngineFactory in project elasticsearch by elastic.

the class RecoveryDuringReplicationTests method testWaitForPendingSeqNo.

@TestLogging("_root:DEBUG,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.action.get:TRACE," + "org.elasticsearch.discovery:TRACE," + "org.elasticsearch.cluster.service:TRACE,org.elasticsearch.indices.recovery:TRACE," + "org.elasticsearch.indices.cluster:TRACE,org.elasticsearch.index.shard:TRACE," + "org.elasticsearch.index.seqno:TRACE")
public void testWaitForPendingSeqNo() throws Exception {
    IndexMetaData metaData = buildIndexMetaData(1);
    final int pendingDocs = randomIntBetween(1, 5);
    final AtomicReference<Semaphore> blockIndexingOnPrimary = new AtomicReference<>();
    final CountDownLatch blockedIndexers = new CountDownLatch(pendingDocs);
    try (ReplicationGroup shards = new ReplicationGroup(metaData) {

        @Override
        protected EngineFactory getEngineFactory(ShardRouting routing) {
            if (routing.primary()) {
                return new EngineFactory() {

                    @Override
                    public Engine newReadWriteEngine(EngineConfig config) {
                        return InternalEngineTests.createInternalEngine((directory, writerConfig) -> new IndexWriter(directory, writerConfig) {

                            @Override
                            public long addDocument(Iterable<? extends IndexableField> doc) throws IOException {
                                Semaphore block = blockIndexingOnPrimary.get();
                                if (block != null) {
                                    blockedIndexers.countDown();
                                    try {
                                        block.acquire();
                                    } catch (InterruptedException e) {
                                        throw new AssertionError("unexpectedly interrupted", e);
                                    }
                                }
                                return super.addDocument(doc);
                            }
                        }, null, config);
                    }

                    @Override
                    public Engine newReadOnlyEngine(EngineConfig config) {
                        throw new UnsupportedOperationException();
                    }
                };
            } else {
                return null;
            }
        }
    }) {
        shards.startAll();
        int docs = shards.indexDocs(randomIntBetween(1, 10));
        IndexShard replica = shards.getReplicas().get(0);
        shards.removeReplica(replica);
        closeShards(replica);
        docs += pendingDocs;
        final Semaphore pendingDocsSemaphore = new Semaphore(pendingDocs);
        blockIndexingOnPrimary.set(pendingDocsSemaphore);
        blockIndexingOnPrimary.get().acquire(pendingDocs);
        CountDownLatch pendingDocsDone = new CountDownLatch(pendingDocs);
        for (int i = 0; i < pendingDocs; i++) {
            final String id = "pending_" + i;
            threadPool.generic().submit(() -> {
                try {
                    shards.index(new IndexRequest(index.getName(), "type", id).source("{}", XContentType.JSON));
                } catch (Exception e) {
                    throw new AssertionError(e);
                } finally {
                    pendingDocsDone.countDown();
                }
            });
        }
        // wait for the pending ops to "hang"
        blockedIndexers.await();
        blockIndexingOnPrimary.set(null);
        // index some more
        docs += shards.indexDocs(randomInt(5));
        IndexShard newReplica = shards.addReplicaWithExistingPath(replica.shardPath(), replica.routingEntry().currentNodeId());
        CountDownLatch recoveryStart = new CountDownLatch(1);
        AtomicBoolean preparedForTranslog = new AtomicBoolean(false);
        final Future<Void> recoveryFuture = shards.asyncRecoverReplica(newReplica, (indexShard, node) -> {
            recoveryStart.countDown();
            return new RecoveryTarget(indexShard, node, recoveryListener, l -> {
            }) {

                @Override
                public void prepareForTranslogOperations(int totalTranslogOps, long maxUnsafeAutoIdTimestamp) throws IOException {
                    preparedForTranslog.set(true);
                    super.prepareForTranslogOperations(totalTranslogOps, maxUnsafeAutoIdTimestamp);
                }
            };
        });
        recoveryStart.await();
        for (int i = 0; i < pendingDocs; i++) {
            assertFalse((pendingDocs - i) + " pending operations, recovery should wait", preparedForTranslog.get());
            pendingDocsSemaphore.release();
        }
        pendingDocsDone.await();
        // now recovery can finish
        recoveryFuture.get();
        assertThat(newReplica.recoveryState().getIndex().fileDetails(), empty());
        assertThat(newReplica.recoveryState().getTranslog().recoveredOperations(), equalTo(docs));
        shards.assertAllEqual(docs);
    }
}

Example 4 with EngineFactory

use of org.elasticsearch.index.engine.EngineFactory in project crate by crate.

the class IndicesService method getEngineFactory.

private EngineFactory getEngineFactory(final IndexSettings idxSettings) {
    final IndexMetadata indexMetadata = idxSettings.getIndexMetadata();
    if (indexMetadata != null && indexMetadata.getState() == IndexMetadata.State.CLOSE) {
        // NoOpEngine takes precedence as long as the index is closed
        return NoOpEngine::new;
    }
    final List<Optional<EngineFactory>> engineFactories = engineFactoryProviders.stream().map(engineFactoryProvider -> engineFactoryProvider.apply(idxSettings)).filter(maybe -> Objects.requireNonNull(maybe).isPresent()).collect(Collectors.toList());
    if (engineFactories.isEmpty()) {
        return new InternalEngineFactory();
    } else if (engineFactories.size() == 1) {
        assert engineFactories.get(0).isPresent();
        return engineFactories.get(0).get();
    } else {
        final String message = String.format(Locale.ROOT, "multiple engine factories provided for %s: %s", idxSettings.getIndex(), engineFactories.stream().map(t -> {
            assert t.isPresent();
            return "[" + t.get().getClass().getName() + "]";
        }).collect(Collectors.joining(",")));
        throw new IllegalStateException(message);
    }
}