Search in sources :

Example 86 with Supplier

use of org.apache.logging.log4j.util.Supplier in project elasticsearch by elastic.

the class ClusterService method executeTasks.

private ClusterTasksResult<Object> executeTasks(TaskInputs taskInputs, long startTimeNS, ClusterState previousClusterState) {
    ClusterTasksResult<Object> clusterTasksResult;
    try {
        List<Object> inputs = taskInputs.updateTasks.stream().map(tUpdateTask -> tUpdateTask.task).collect(Collectors.toList());
        clusterTasksResult = taskInputs.executor.execute(previousClusterState, inputs);
    } catch (Exception e) {
        TimeValue executionTime = TimeValue.timeValueMillis(Math.max(0, TimeValue.nsecToMSec(currentTimeInNanos() - startTimeNS)));
        if (logger.isTraceEnabled()) {
            logger.trace((Supplier<?>) () -> new ParameterizedMessage("failed to execute cluster state update in [{}], state:\nversion [{}], source [{}]\n{}{}{}", executionTime, previousClusterState.version(), taskInputs.summary, previousClusterState.nodes(), previousClusterState.routingTable(), previousClusterState.getRoutingNodes()), e);
        }
        warnAboutSlowTaskIfNeeded(executionTime, taskInputs.summary);
        clusterTasksResult = ClusterTasksResult.builder().failures(taskInputs.updateTasks.stream().map(updateTask -> updateTask.task)::iterator, e).build(previousClusterState);
    }
    assert clusterTasksResult.executionResults != null;
    assert clusterTasksResult.executionResults.size() == taskInputs.updateTasks.size() : String.format(Locale.ROOT, "expected [%d] task result%s but was [%d]", taskInputs.updateTasks.size(), taskInputs.updateTasks.size() == 1 ? "" : "s", clusterTasksResult.executionResults.size());
    boolean assertsEnabled = false;
    assert (assertsEnabled = true);
    if (assertsEnabled) {
        for (UpdateTask updateTask : taskInputs.updateTasks) {
            assert clusterTasksResult.executionResults.containsKey(updateTask.task) : "missing task result for " + updateTask;
        }
    }
    return clusterTasksResult;
}
Also used : MetaData(org.elasticsearch.cluster.metadata.MetaData) ScheduledFuture(java.util.concurrent.ScheduledFuture) Nullable(org.elasticsearch.common.Nullable) Property(org.elasticsearch.common.settings.Setting.Property) ConcurrentCollections(org.elasticsearch.common.util.concurrent.ConcurrentCollections) UnaryOperator(java.util.function.UnaryOperator) ClusterBlocks(org.elasticsearch.cluster.block.ClusterBlocks) ClusterStateListener(org.elasticsearch.cluster.ClusterStateListener) ClusterTasksResult(org.elasticsearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult) ClusterState(org.elasticsearch.cluster.ClusterState) Future(java.util.concurrent.Future) Settings(org.elasticsearch.common.settings.Settings) ClusterBlock(org.elasticsearch.cluster.block.ClusterBlock) Locale(java.util.Locale) Map(java.util.Map) CountDown(org.elasticsearch.common.util.concurrent.CountDown) ThreadPool(org.elasticsearch.threadpool.ThreadPool) ClusterName(org.elasticsearch.cluster.ClusterName) ClusterStateObserver(org.elasticsearch.cluster.ClusterStateObserver) Priority(org.elasticsearch.common.Priority) IdentityHashMap(java.util.IdentityHashMap) Setting(org.elasticsearch.common.settings.Setting) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterChangedEvent(org.elasticsearch.cluster.ClusterChangedEvent) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) PrioritizedEsThreadPoolExecutor(org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor) Supplier(org.apache.logging.log4j.util.Supplier) Queue(java.util.Queue) EsExecutors.daemonThreadFactory(org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) PrioritizedRunnable(org.elasticsearch.common.util.concurrent.PrioritizedRunnable) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) ProcessClusterEventTimeoutException(org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException) ArrayList(java.util.ArrayList) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) ClusterStateTaskListener(org.elasticsearch.cluster.ClusterStateTaskListener) Text(org.elasticsearch.common.text.Text) TimeValue(org.elasticsearch.common.unit.TimeValue) Iterables(org.elasticsearch.common.util.iterable.Iterables) BiConsumer(java.util.function.BiConsumer) ClusterStateApplier(org.elasticsearch.cluster.ClusterStateApplier) LinkedHashSet(java.util.LinkedHashSet) TimeoutClusterStateListener(org.elasticsearch.cluster.TimeoutClusterStateListener) Loggers(org.elasticsearch.common.logging.Loggers) Builder(org.elasticsearch.cluster.ClusterState.Builder) DiscoveryNodes(org.elasticsearch.cluster.node.DiscoveryNodes) OperationRouting(org.elasticsearch.cluster.routing.OperationRouting) EsExecutors(org.elasticsearch.common.util.concurrent.EsExecutors) FutureUtils(org.elasticsearch.common.util.concurrent.FutureUtils) Iterator(java.util.Iterator) Executor(java.util.concurrent.Executor) Discovery(org.elasticsearch.discovery.Discovery) DiscoverySettings(org.elasticsearch.discovery.DiscoverySettings) ClusterStateTaskConfig(org.elasticsearch.cluster.ClusterStateTaskConfig) ClusterStateTaskExecutor(org.elasticsearch.cluster.ClusterStateTaskExecutor) AbstractLifecycleComponent(org.elasticsearch.common.component.AbstractLifecycleComponent) TimeUnit(java.util.concurrent.TimeUnit) LocalNodeMasterListener(org.elasticsearch.cluster.LocalNodeMasterListener) NodeConnectionsService(org.elasticsearch.cluster.NodeConnectionsService) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) EsRejectedExecutionException(org.elasticsearch.common.util.concurrent.EsRejectedExecutionException) RoutingTable(org.elasticsearch.cluster.routing.RoutingTable) AckedClusterStateTaskListener(org.elasticsearch.cluster.AckedClusterStateTaskListener) Collections(java.util.Collections) Supplier(org.apache.logging.log4j.util.Supplier) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) ProcessClusterEventTimeoutException(org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException) EsRejectedExecutionException(org.elasticsearch.common.util.concurrent.EsRejectedExecutionException) TimeValue(org.elasticsearch.common.unit.TimeValue)

Example 87 with Supplier

use of org.apache.logging.log4j.util.Supplier in project elasticsearch by elastic.

the class ClusterService method publishAndApplyChanges.

private void publishAndApplyChanges(TaskInputs taskInputs, TaskOutputs taskOutputs) {
    ClusterState previousClusterState = taskOutputs.previousClusterState;
    ClusterState newClusterState = taskOutputs.newClusterState;
    ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent(taskInputs.summary, newClusterState, previousClusterState);
    // new cluster state, notify all listeners
    final DiscoveryNodes.Delta nodesDelta = clusterChangedEvent.nodesDelta();
    if (nodesDelta.hasChanges() && logger.isInfoEnabled()) {
        String summary = nodesDelta.shortSummary();
        if (summary.length() > 0) {
            logger.info("{}, reason: {}", summary, taskInputs.summary);
        }
    }
    final Discovery.AckListener ackListener = newClusterState.nodes().isLocalNodeElectedMaster() ? taskOutputs.createAckListener(threadPool, newClusterState) : null;
    nodeConnectionsService.connectToNodes(newClusterState.nodes());
    // we don't want to notify
    if (newClusterState.nodes().isLocalNodeElectedMaster()) {
        logger.debug("publishing cluster state version [{}]", newClusterState.version());
        try {
            clusterStatePublisher.accept(clusterChangedEvent, ackListener);
        } catch (Discovery.FailedToCommitClusterStateException t) {
            final long version = newClusterState.version();
            logger.warn((Supplier<?>) () -> new ParameterizedMessage("failing [{}]: failed to commit cluster state version [{}]", taskInputs.summary, version), t);
            // ensure that list of connected nodes in NodeConnectionsService is in-sync with the nodes of the current cluster state
            nodeConnectionsService.connectToNodes(previousClusterState.nodes());
            nodeConnectionsService.disconnectFromNodesExcept(previousClusterState.nodes());
            taskOutputs.publishingFailed(t);
            return;
        }
    }
    logger.debug("applying cluster state version {}", newClusterState.version());
    try {
        // nothing to do until we actually recover from the gateway or any other block indicates we need to disable persistency
        if (clusterChangedEvent.state().blocks().disableStatePersistence() == false && clusterChangedEvent.metaDataChanged()) {
            final Settings incomingSettings = clusterChangedEvent.state().metaData().settings();
            clusterSettings.applySettings(incomingSettings);
        }
    } catch (Exception ex) {
        logger.warn("failed to apply cluster settings", ex);
    }
    logger.debug("set local cluster state to version {}", newClusterState.version());
    callClusterStateAppliers(newClusterState, clusterChangedEvent);
    nodeConnectionsService.disconnectFromNodesExcept(newClusterState.nodes());
    updateState(css -> newClusterState);
    Stream.concat(clusterStateListeners.stream(), timeoutClusterStateListeners.stream()).forEach(listener -> {
        try {
            logger.trace("calling [{}] with change to version [{}]", listener, newClusterState.version());
            listener.clusterChanged(clusterChangedEvent);
        } catch (Exception ex) {
            logger.warn("failed to notify ClusterStateListener", ex);
        }
    });
    //manual ack only from the master at the end of the publish
    if (newClusterState.nodes().isLocalNodeElectedMaster()) {
        try {
            ackListener.onNodeAck(newClusterState.nodes().getLocalNode(), null);
        } catch (Exception e) {
            final DiscoveryNode localNode = newClusterState.nodes().getLocalNode();
            logger.debug((Supplier<?>) () -> new ParameterizedMessage("error while processing ack for master node [{}]", localNode), e);
        }
    }
    taskOutputs.processedDifferentClusterState(previousClusterState, newClusterState);
    if (newClusterState.nodes().isLocalNodeElectedMaster()) {
        try {
            taskOutputs.clusterStatePublished(clusterChangedEvent);
        } catch (Exception e) {
            logger.error((Supplier<?>) () -> new ParameterizedMessage("exception thrown while notifying executor of new cluster state publication [{}]", taskInputs.summary), e);
        }
    }
}
Also used : ClusterState(org.elasticsearch.cluster.ClusterState) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) Discovery(org.elasticsearch.discovery.Discovery) ClusterChangedEvent(org.elasticsearch.cluster.ClusterChangedEvent) ProcessClusterEventTimeoutException(org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException) EsRejectedExecutionException(org.elasticsearch.common.util.concurrent.EsRejectedExecutionException) Supplier(org.apache.logging.log4j.util.Supplier) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) DiscoveryNodes(org.elasticsearch.cluster.node.DiscoveryNodes) Settings(org.elasticsearch.common.settings.Settings) DiscoverySettings(org.elasticsearch.discovery.DiscoverySettings) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings)

Example 88 with Supplier

use of org.apache.logging.log4j.util.Supplier in project elasticsearch by elastic.

the class NodeConnectionsService method connectToNodes.

public void connectToNodes(DiscoveryNodes discoveryNodes) {
    CountDownLatch latch = new CountDownLatch(discoveryNodes.getSize());
    for (final DiscoveryNode node : discoveryNodes) {
        final boolean connected;
        try (Releasable ignored = nodeLocks.acquire(node)) {
            nodes.putIfAbsent(node, 0);
            connected = transportService.nodeConnected(node);
        }
        if (connected) {
            latch.countDown();
        } else {
            // spawn to another thread to do in parallel
            threadPool.executor(ThreadPool.Names.MANAGEMENT).execute(new AbstractRunnable() {

                @Override
                public void onFailure(Exception e) {
                    // both errors and rejections are logged here. the service
                    // will try again after `cluster.nodes.reconnect_interval` on all nodes but the current master.
                    // On the master, node fault detection will remove these nodes from the cluster as their are not
                    // connected. Note that it is very rare that we end up here on the master.
                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to connect to {}", node), e);
                }

                @Override
                protected void doRun() throws Exception {
                    try (Releasable ignored = nodeLocks.acquire(node)) {
                        validateAndConnectIfNeeded(node);
                    }
                }

                @Override
                public void onAfter() {
                    latch.countDown();
                }
            });
        }
    }
    try {
        latch.await();
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}
Also used : AbstractRunnable(org.elasticsearch.common.util.concurrent.AbstractRunnable) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) Releasable(org.elasticsearch.common.lease.Releasable) Supplier(org.apache.logging.log4j.util.Supplier) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) CountDownLatch(java.util.concurrent.CountDownLatch)

Example 89 with Supplier

use of org.apache.logging.log4j.util.Supplier in project elasticsearch by elastic.

the class ShardStateAction method sendShardAction.

private void sendShardAction(final String actionName, final ClusterState currentState, final ShardEntry shardEntry, final Listener listener) {
    ClusterStateObserver observer = new ClusterStateObserver(currentState, clusterService, null, logger, threadPool.getThreadContext());
    DiscoveryNode masterNode = currentState.nodes().getMasterNode();
    Predicate<ClusterState> changePredicate = MasterNodeChangePredicate.build(currentState);
    if (masterNode == null) {
        logger.warn("{} no master known for action [{}] for shard entry [{}]", shardEntry.shardId, actionName, shardEntry);
        waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
    } else {
        logger.debug("{} sending [{}] to [{}] for shard entry [{}]", shardEntry.shardId, actionName, masterNode.getId(), shardEntry);
        transportService.sendRequest(masterNode, actionName, shardEntry, new EmptyTransportResponseHandler(ThreadPool.Names.SAME) {

            @Override
            public void handleResponse(TransportResponse.Empty response) {
                listener.onSuccess();
            }

            @Override
            public void handleException(TransportException exp) {
                if (isMasterChannelException(exp)) {
                    waitForNewMasterAndRetry(actionName, observer, shardEntry, listener, changePredicate);
                } else {
                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("{} unexpected failure while sending request [{}] to [{}] for shard entry [{}]", shardEntry.shardId, actionName, masterNode, shardEntry), exp);
                    listener.onFailure(exp instanceof RemoteTransportException ? (Exception) (exp.getCause() instanceof Exception ? exp.getCause() : new ElasticsearchException(exp.getCause())) : exp);
                }
            }
        });
    }
}
Also used : ClusterState(org.elasticsearch.cluster.ClusterState) RemoteTransportException(org.elasticsearch.transport.RemoteTransportException) ClusterStateObserver(org.elasticsearch.cluster.ClusterStateObserver) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) ElasticsearchException(org.elasticsearch.ElasticsearchException) TransportResponse(org.elasticsearch.transport.TransportResponse) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) RemoteTransportException(org.elasticsearch.transport.RemoteTransportException) TransportException(org.elasticsearch.transport.TransportException) ElasticsearchException(org.elasticsearch.ElasticsearchException) NodeClosedException(org.elasticsearch.node.NodeClosedException) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) NodeDisconnectedException(org.elasticsearch.transport.NodeDisconnectedException) RemoteTransportException(org.elasticsearch.transport.RemoteTransportException) TransportException(org.elasticsearch.transport.TransportException) NotMasterException(org.elasticsearch.cluster.NotMasterException) IOException(java.io.IOException) Supplier(org.apache.logging.log4j.util.Supplier) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) EmptyTransportResponseHandler(org.elasticsearch.transport.EmptyTransportResponseHandler)

Example 90 with Supplier

use of org.apache.logging.log4j.util.Supplier in project elasticsearch by elastic.

the class MetaDataIndexUpgradeService method archiveBrokenIndexSettings.

IndexMetaData archiveBrokenIndexSettings(IndexMetaData indexMetaData) {
    final Settings settings = indexMetaData.getSettings();
    final Settings upgrade = indexScopedSettings.archiveUnknownOrInvalidSettings(settings, e -> logger.warn("{} ignoring unknown index setting: [{}] with value [{}]; archiving", indexMetaData.getIndex(), e.getKey(), e.getValue()), (e, ex) -> logger.warn((Supplier<?>) () -> new ParameterizedMessage("{} ignoring invalid index setting: [{}] with value [{}]; archiving", indexMetaData.getIndex(), e.getKey(), e.getValue()), ex));
    if (upgrade != settings) {
        return IndexMetaData.builder(indexMetaData).settings(upgrade).build();
    } else {
        return indexMetaData;
    }
}
Also used : Supplier(org.apache.logging.log4j.util.Supplier) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) IndexScopedSettings(org.elasticsearch.common.settings.IndexScopedSettings) Settings(org.elasticsearch.common.settings.Settings) IndexSettings(org.elasticsearch.index.IndexSettings)

Aggregations

Supplier (org.apache.logging.log4j.util.Supplier)94 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)91 IOException (java.io.IOException)55 ElasticsearchException (org.elasticsearch.ElasticsearchException)27 ArrayList (java.util.ArrayList)25 ClusterState (org.elasticsearch.cluster.ClusterState)21 DiscoveryNode (org.elasticsearch.cluster.node.DiscoveryNode)21 TimeValue (org.elasticsearch.common.unit.TimeValue)14 HashMap (java.util.HashMap)12 Map (java.util.Map)11 Settings (org.elasticsearch.common.settings.Settings)11 TransportException (org.elasticsearch.transport.TransportException)11 List (java.util.List)10 ExecutionException (java.util.concurrent.ExecutionException)10 Index (org.elasticsearch.index.Index)10 CountDownLatch (java.util.concurrent.CountDownLatch)9 NotMasterException (org.elasticsearch.cluster.NotMasterException)8 ClusterStateUpdateResponse (org.elasticsearch.cluster.ack.ClusterStateUpdateResponse)8 ClusterBlockException (org.elasticsearch.cluster.block.ClusterBlockException)8 NoSuchFileException (java.nio.file.NoSuchFileException)7