Examples with ThreadPool org.elasticsearch.threadpool.ThreadPool used on opensource projects

Example 61 with ThreadPool

use of org.elasticsearch.threadpool.ThreadPool in project elasticsearch by elastic.

the class ResourceWatcherServiceTests method testSettings.

public void testSettings() throws Exception {
    ThreadPool threadPool = new TestThreadPool("test");
    // checking the defaults
    Settings settings = Settings.builder().build();
    ResourceWatcherService service = new ResourceWatcherService(settings, threadPool);
    assertThat(service.highMonitor.interval, is(ResourceWatcherService.Frequency.HIGH.interval));
    assertThat(service.mediumMonitor.interval, is(ResourceWatcherService.Frequency.MEDIUM.interval));
    assertThat(service.lowMonitor.interval, is(ResourceWatcherService.Frequency.LOW.interval));
    // checking bwc
    settings = Settings.builder().put("resource.reload.interval", // only applies to medium
    "40s").build();
    service = new ResourceWatcherService(settings, threadPool);
    assertThat(service.highMonitor.interval.millis(), is(timeValueSeconds(5).millis()));
    assertThat(service.mediumMonitor.interval.millis(), is(timeValueSeconds(40).millis()));
    assertThat(service.lowMonitor.interval.millis(), is(timeValueSeconds(60).millis()));
    // checking custom
    settings = Settings.builder().put("resource.reload.interval.high", "10s").put("resource.reload.interval.medium", "20s").put("resource.reload.interval.low", "30s").build();
    service = new ResourceWatcherService(settings, threadPool);
    assertThat(service.highMonitor.interval.millis(), is(timeValueSeconds(10).millis()));
    assertThat(service.mediumMonitor.interval.millis(), is(timeValueSeconds(20).millis()));
    assertThat(service.lowMonitor.interval.millis(), is(timeValueSeconds(30).millis()));
    terminate(threadPool);
}

Example 62 with ThreadPool

use of org.elasticsearch.threadpool.ThreadPool in project elasticsearch by elastic.

the class IndexShardTestCase method newShard.

/**
     * creates a new initializing shard.
     *
     * @param routing              shard routing to use
     * @param shardPath            path to use for shard data
     * @param indexMetaData        indexMetaData for the shard, including any mapping
     * @param indexSearcherWrapper an optional wrapper to be used during searchers
     * @param listeners            an optional set of listeners to add to the shard
     */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData, @Nullable IndexSearcherWrapper indexSearcherWrapper, Runnable globalCheckpointSyncer, @Nullable EngineFactory engineFactory, IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
    final IndexShard indexShard;
    final Store store = createStore(indexSettings, shardPath);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), indexSettings.getSettings());
        mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
        final IndexEventListener indexEventListener = new IndexEventListener() {
        };
        final Engine.Warmer warmer = searcher -> {
        };
        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
        });
        IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), mapperService);
        indexShard = new IndexShard(routing, indexSettings, shardPath, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, indexSearcherWrapper, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, globalCheckpointSyncer, Collections.emptyList(), Arrays.asList(listeners));
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}

Example 63 with ThreadPool

use of org.elasticsearch.threadpool.ThreadPool in project elasticsearch by elastic.

the class ClusterServiceTests method testDisconnectFromNewlyAddedNodesIfClusterStatePublishingFails.

public void testDisconnectFromNewlyAddedNodesIfClusterStatePublishingFails() throws InterruptedException {
    TimedClusterService timedClusterService = new TimedClusterService(Settings.builder().put("cluster.name", "ClusterServiceTests").build(), new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), threadPool, () -> new DiscoveryNode("node1", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT));
    Set<DiscoveryNode> currentNodes = new HashSet<>();
    timedClusterService.setNodeConnectionsService(new NodeConnectionsService(Settings.EMPTY, null, null) {

        @Override
        public void connectToNodes(DiscoveryNodes discoveryNodes) {
            discoveryNodes.forEach(currentNodes::add);
        }

        @Override
        public void disconnectFromNodesExcept(DiscoveryNodes nodesToKeep) {
            Set<DiscoveryNode> nodeSet = new HashSet<>();
            nodesToKeep.iterator().forEachRemaining(nodeSet::add);
            currentNodes.removeIf(node -> nodeSet.contains(node) == false);
        }
    });
    AtomicBoolean failToCommit = new AtomicBoolean();
    timedClusterService.setClusterStatePublisher((event, ackListener) -> {
        if (failToCommit.get()) {
            throw new Discovery.FailedToCommitClusterStateException("just to test this");
        }
    });
    timedClusterService.setDiscoverySettings(new DiscoverySettings(Settings.EMPTY, new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
    timedClusterService.start();
    ClusterState state = timedClusterService.state();
    final DiscoveryNodes nodes = state.nodes();
    final DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder(nodes).masterNodeId(nodes.getLocalNodeId());
    state = ClusterState.builder(state).blocks(ClusterBlocks.EMPTY_CLUSTER_BLOCK).nodes(nodesBuilder).build();
    setState(timedClusterService, state);
    assertThat(currentNodes, equalTo(Sets.newHashSet(timedClusterService.state().getNodes())));
    final CountDownLatch latch = new CountDownLatch(1);
    // try to add node when cluster state publishing fails
    failToCommit.set(true);
    timedClusterService.submitStateUpdateTask("test", new ClusterStateUpdateTask() {

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            DiscoveryNode newNode = new DiscoveryNode("node2", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
            return ClusterState.builder(currentState).nodes(DiscoveryNodes.builder(currentState.nodes()).add(newNode)).build();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            latch.countDown();
        }

        @Override
        public void onFailure(String source, Exception e) {
            latch.countDown();
        }
    });
    latch.await();
    assertThat(currentNodes, equalTo(Sets.newHashSet(timedClusterService.state().getNodes())));
    timedClusterService.close();
}

Example 64 with ThreadPool

use of org.elasticsearch.threadpool.ThreadPool in project elasticsearch by elastic.

the class IndexService method createShard.

public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
    final boolean primary = routing.primary();
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false || (primary && this.indexSettings.isOnSharedFilesystem());
        final Engine.Warmer engineWarmer = (searcher) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(searcher, shard, IndexService.this.indexSettings);
            }
        };
        store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
        if (useShadowEngine(primary, this.indexSettings)) {
            indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners);
        // no indexing listeners - shadow  engines don't index
        } else {
            indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, () -> globalCheckpointSyncer.accept(shardId), searchOperationListeners, indexingOperationListeners);
        }
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}

Example 65 with ThreadPool

use of org.elasticsearch.threadpool.ThreadPool in project elasticsearch by elastic.

the class ESSingleNodeTestCase method createSearchContext.

/**
     * Create a new search context.
     */
protected SearchContext createSearchContext(IndexService indexService) {
    BigArrays bigArrays = indexService.getBigArrays();
    ThreadPool threadPool = indexService.getThreadPool();
    return new TestSearchContext(threadPool, bigArrays, indexService);
}