Search in sources :

Example 1 with ConnectionRequestor

use of org.apache.ignite.spi.communication.tcp.internal.ConnectionRequestor in project gridgain by gridgain.

the class TcpCommunicationSpi method spiStart.

/**
 * {@inheritDoc}
 */
@Override
public void spiStart(String igniteInstanceName) throws IgniteSpiException {
    final Function<UUID, ClusterNode> nodeGetter = (nodeId) -> getSpiContext().node(nodeId);
    final Supplier<ClusterNode> locNodeSupplier = () -> getSpiContext().localNode();
    final Supplier<Ignite> igniteExSupplier = this::ignite;
    final Function<UUID, Boolean> pingNode = (nodeId) -> getSpiContext().pingNode(nodeId);
    final Supplier<FailureProcessor> failureProcessorSupplier = () -> ignite instanceof IgniteEx ? ((IgniteEx) ignite).context().failure() : null;
    final Supplier<TcpCommunicationMetricsListener> metricLsnrSupplier = () -> metricsLsnr;
    final Supplier<Boolean> isStopped = () -> getSpiContext().isStopping();
    cfg.failureDetectionTimeout(ignite.configuration().getFailureDetectionTimeout());
    attributeNames = new AttributeNames(createSpiAttributeName(ATTR_PAIRED_CONN), createSpiAttributeName(ATTR_SHMEM_PORT), createSpiAttributeName(ATTR_ADDRS), createSpiAttributeName(ATTR_HOST_NAMES), createSpiAttributeName(ATTR_EXT_ADDRS), createSpiAttributeName(ATTR_PORT), createSpiAttributeName(ATTR_FORCE_CLIENT_SERVER_CONNECTIONS));
    boolean client = Boolean.TRUE.equals(ignite().configuration().isClientMode());
    this.stateProvider = new ClusterStateProvider(ignite, locNodeSupplier, this, isStopped, () -> super.getSpiContext(), log, igniteExSupplier);
    try {
        cfg.localHost(U.resolveLocalHost(cfg.localAddress()));
    } catch (IOException e) {
        throw new IgniteSpiException("Failed to initialize local address: " + cfg.localAddress(), e);
    }
    if (cfg.connectionsPerNode() > 1)
        connPlc = new RoundRobinConnectionPolicy(cfg);
    else
        connPlc = new FirstConnectionPolicy();
    this.srvLsnr = resolve(ignite, new InboundConnectionHandler(log, cfg, nodeGetter, locNodeSupplier, stateProvider, clientPool, commWorker, connectGate, failureProcessorSupplier, attributeNames, metricLsnrSupplier, nioSrvWrapper, ctxInitLatch, client, igniteExSupplier, new CommunicationListener<Message>() {

        @Override
        public void onMessage(UUID nodeId, Message msg, IgniteRunnable msgC) {
            notifyListener(nodeId, msg, msgC);
        }

        @Override
        public void onDisconnected(UUID nodeId) {
            if (lsnr != null)
                lsnr.onDisconnected(nodeId);
        }
    }));
    TcpHandshakeExecutor tcpHandshakeExecutor = resolve(ignite, new TcpHandshakeExecutor(log, stateProvider, cfg.directBuffer()));
    this.nioSrvWrapper = resolve(ignite, new GridNioServerWrapper(log, cfg, attributeNames, tracing, nodeGetter, locNodeSupplier, connectGate, stateProvider, this::getExceptionRegistry, commWorker, ignite.configuration(), this.srvLsnr, ignite.configuration().getIgniteInstanceName(), getWorkersRegistry(ignite), ignite instanceof IgniteEx ? ((IgniteEx) ignite).context().metric() : null, this::createTcpClient, tcpHandshakeExecutor));
    this.srvLsnr.setNioSrvWrapper(nioSrvWrapper);
    this.clientPool = resolve(ignite, new ConnectionClientPool(cfg, attributeNames, log, metricLsnrSupplier, locNodeSupplier, nodeGetter, null, getWorkersRegistry(ignite), this, stateProvider, nioSrvWrapper, connectionRequestor, getName()));
    this.srvLsnr.setClientPool(clientPool);
    nioSrvWrapper.clientPool(clientPool);
    discoLsnr = new CommunicationDiscoveryEventListener(clientPool, metricLsnrSupplier);
    try {
        shmemSrv = resetShmemServer();
    } catch (IgniteCheckedException e) {
        U.warn(log, "Failed to start shared memory communication server.", e);
    }
    try {
        // This method potentially resets local port to the value
        // local node was bound to.
        nioSrvWrapper.nio(nioSrvWrapper.resetNioServer());
    } catch (IgniteCheckedException e) {
        throw new IgniteSpiException("Failed to initialize TCP server: " + cfg.localHost(), e);
    }
    boolean forceClientToSrvConnections = forceClientToServerConnections() || cfg.localPort() == -1;
    if (cfg.usePairedConnections() && forceClientToSrvConnections) {
        throw new IgniteSpiException("Node using paired connections " + "is not allowed to start in forced client to server connections mode.");
    }
    assert cfg.localHost() != null;
    // Start SPI start stopwatch.
    startStopwatch();
    if (log.isDebugEnabled()) {
        log.debug(configInfo("locAddr", cfg.localAddress()));
        log.debug(configInfo("locPort", cfg.localPort()));
        log.debug(configInfo("locPortRange", cfg.localPortRange()));
        log.debug(configInfo("idleConnTimeout", cfg.idleConnectionTimeout()));
        log.debug(configInfo("directBuf", cfg.directBuffer()));
        log.debug(configInfo("directSendBuf", cfg.directSendBuffer()));
        log.debug(configInfo("selectorsCnt", cfg.selectorsCount()));
        log.debug(configInfo("tcpNoDelay", cfg.tcpNoDelay()));
        log.debug(configInfo("sockSndBuf", cfg.socketSendBuffer()));
        log.debug(configInfo("sockRcvBuf", cfg.socketReceiveBuffer()));
        log.debug(configInfo("shmemPort", cfg.shmemPort()));
        log.debug(configInfo("msgQueueLimit", cfg.messageQueueLimit()));
        log.debug(configInfo("connectionsPerNode", cfg.connectionsPerNode()));
        if (failureDetectionTimeoutEnabled()) {
            log.debug(configInfo("connTimeout", cfg.connectionTimeout()));
            log.debug(configInfo("maxConnTimeout", cfg.maxConnectionTimeout()));
            log.debug(configInfo("reconCnt", cfg.reconCount()));
        } else
            log.debug(configInfo("failureDetectionTimeout", failureDetectionTimeout()));
        log.debug(configInfo("sockWriteTimeout", cfg.socketWriteTimeout()));
        log.debug(configInfo("ackSndThreshold", cfg.ackSendThreshold()));
        log.debug(configInfo("unackedMsgsBufSize", cfg.unackedMsgsBufferSize()));
    }
    if (!cfg.tcpNoDelay())
        U.quietAndWarn(log, "'TCP_NO_DELAY' for communication is off, which should be used with caution " + "since may produce significant delays with some scenarios.");
    if (cfg.slowClientQueueLimit() > 0 && cfg.messageQueueLimit() > 0 && cfg.slowClientQueueLimit() >= cfg.messageQueueLimit()) {
        U.quietAndWarn(log, "Slow client queue limit is set to a value greater than or equal to message " + "queue limit (slow client queue limit will have no effect) [msgQueueLimit=" + cfg.messageQueueLimit() + ", slowClientQueueLimit=" + cfg.slowClientQueueLimit() + ']');
    }
    if (cfg.messageQueueLimit() == 0)
        U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " + "potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " + "due to message queues growth on sender and receiver sides.");
    registerMBean(igniteInstanceName, new TcpCommunicationSpiMBeanImpl(this, metricLsnrSupplier, cfg, stateProvider), TcpCommunicationSpiMBean.class);
    if (shmemSrv != null) {
        MessageFactory msgFactory = new MessageFactory() {

            private MessageFactory impl;

            @Nullable
            @Override
            public Message create(short type) {
                if (impl == null)
                    impl = getSpiContext().messageFactory();
                assert impl != null;
                return impl.create(type);
            }
        };
        GridNioMessageWriterFactory writerFactory = new GridNioMessageWriterFactory() {

            private MessageFormatter formatter;

            @Override
            public MessageWriter writer(GridNioSession ses) throws IgniteCheckedException {
                if (formatter == null)
                    formatter = getSpiContext().messageFormatter();
                assert formatter != null;
                ConnectionKey connKey = ses.meta(CONN_IDX_META);
                return connKey != null ? formatter.writer(connKey.nodeId()) : null;
            }
        };
        GridNioMessageReaderFactory readerFactory = new GridNioMessageReaderFactory() {

            private MessageFormatter formatter;

            @Override
            public MessageReader reader(GridNioSession ses, MessageFactory msgFactory) throws IgniteCheckedException {
                if (formatter == null)
                    formatter = getSpiContext().messageFormatter();
                assert formatter != null;
                ConnectionKey connKey = ses.meta(CONN_IDX_META);
                return connKey != null ? formatter.reader(connKey.nodeId(), msgFactory) : null;
            }
        };
        shmemAcceptWorker = new ShmemAcceptWorker(igniteInstanceName, srvLsnr, shmemSrv, metricsLsnr, log, msgFactory, writerFactory, readerFactory, tracing);
        new IgniteThread(shmemAcceptWorker).start();
    }
    nioSrvWrapper.start();
    this.commWorker = new CommunicationWorker(igniteInstanceName, log, cfg, attributeNames, clientPool, failureProcessorSupplier, nodeGetter, pingNode, this::getExceptionRegistry, nioSrvWrapper, getWorkersRegistry(ignite), getName());
    this.srvLsnr.communicationWorker(commWorker);
    this.nioSrvWrapper.communicationWorker(commWorker);
    new IgniteSpiThread(igniteInstanceName, commWorker.name(), log) {

        @Override
        protected void body() {
            commWorker.run();
        }
    }.start();
    // Ack start.
    if (log.isDebugEnabled())
        log.debug(startInfo());
}
Also used : IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) CommunicationWorker(org.apache.ignite.spi.communication.tcp.internal.CommunicationWorker) TcpHandshakeExecutor(org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor) GridNioSession(org.apache.ignite.internal.util.nio.GridNioSession) TcpCommunicationSpiMBeanImpl(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationSpiMBeanImpl) EVT_NODE_LEFT(org.apache.ignite.events.EventType.EVT_NODE_LEFT) AddressResolver(org.apache.ignite.configuration.AddressResolver) GridFinishedFuture(org.apache.ignite.internal.util.future.GridFinishedFuture) ConnectionRequestor(org.apache.ignite.spi.communication.tcp.internal.ConnectionRequestor) IgniteEx(org.apache.ignite.internal.IgniteEx) TcpCommunicationConfigInitializer(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConfigInitializer) ClusterStateProvider(org.apache.ignite.spi.communication.tcp.internal.ClusterStateProvider) ByteBuffer(java.nio.ByteBuffer) GridNioSessionMetaKey(org.apache.ignite.internal.util.nio.GridNioSessionMetaKey) Map(java.util.Map) FirstConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.FirstConnectionPolicy) ConnectionClientPool(org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool) FailureProcessor(org.apache.ignite.internal.processors.failure.FailureProcessor) WorkersRegistry(org.apache.ignite.internal.worker.WorkersRegistry) LoggerResource(org.apache.ignite.resources.LoggerResource) RoundRobinConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.RoundRobinConnectionPolicy) IgniteKernal(org.apache.ignite.internal.IgniteKernal) NOOP(org.apache.ignite.spi.communication.tcp.internal.CommunicationTcpUtils.NOOP) IgniteFuture(org.apache.ignite.lang.IgniteFuture) IgniteClientDisconnectedCheckedException(org.apache.ignite.internal.IgniteClientDisconnectedCheckedException) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) MessageFormatter(org.apache.ignite.plugin.extensions.communication.MessageFormatter) GridNioServer(org.apache.ignite.internal.util.nio.GridNioServer) ShmemAcceptWorker(org.apache.ignite.spi.communication.tcp.internal.shmem.ShmemAcceptWorker) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) CommunicationTcpUtils(org.apache.ignite.spi.communication.tcp.internal.CommunicationTcpUtils) TcpConnectionIndexAwareMessage(org.apache.ignite.spi.communication.tcp.internal.TcpConnectionIndexAwareMessage) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UNDEFINED_CONNECTION_INDEX(org.apache.ignite.spi.communication.tcp.internal.TcpConnectionIndexAwareMessage.UNDEFINED_CONNECTION_INDEX) UUID(java.util.UUID) InetSocketAddress(java.net.InetSocketAddress) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) CountDownLatch(java.util.concurrent.CountDownLatch) GridResourceProcessor(org.apache.ignite.internal.processors.resource.GridResourceProcessor) Nullable(org.jetbrains.annotations.Nullable) List(java.util.List) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) EVT_NODE_FAILED(org.apache.ignite.events.EventType.EVT_NODE_FAILED) MTC(org.apache.ignite.internal.processors.tracing.MTC) Message(org.apache.ignite.plugin.extensions.communication.Message) GridCommunicationClient(org.apache.ignite.internal.util.nio.GridCommunicationClient) IgniteFutureImpl(org.apache.ignite.internal.util.future.IgniteFutureImpl) ConnectGateway(org.apache.ignite.spi.communication.tcp.internal.ConnectGateway) CommunicationListener(org.apache.ignite.spi.communication.CommunicationListener) IgniteSpiContext(org.apache.ignite.spi.IgniteSpiContext) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) IgniteThread(org.apache.ignite.thread.IgniteThread) IgniteSpiThread(org.apache.ignite.spi.IgniteSpiThread) U(org.apache.ignite.internal.util.typedef.internal.U) IgnitePortProtocol(org.apache.ignite.spi.IgnitePortProtocol) InboundConnectionHandler(org.apache.ignite.spi.communication.tcp.internal.InboundConnectionHandler) IgniteLogger(org.apache.ignite.IgniteLogger) GridIoManager(org.apache.ignite.internal.managers.communication.GridIoManager) Function(java.util.function.Function) Supplier(java.util.function.Supplier) GridNioMessageWriterFactory(org.apache.ignite.internal.util.nio.GridNioMessageWriterFactory) CommunicationDiscoveryEventListener(org.apache.ignite.spi.communication.tcp.internal.CommunicationDiscoveryEventListener) GridNioMessageReaderFactory(org.apache.ignite.internal.util.nio.GridNioMessageReaderFactory) ClusterNode(org.apache.ignite.cluster.ClusterNode) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) IgniteSpiConsistencyChecked(org.apache.ignite.spi.IgniteSpiConsistencyChecked) IgniteSpiMultipleInstancesSupport(org.apache.ignite.spi.IgniteSpiMultipleInstancesSupport) S(org.apache.ignite.internal.util.typedef.internal.S) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) MessageWriter(org.apache.ignite.plugin.extensions.communication.MessageWriter) GridNioServerWrapper(org.apache.ignite.spi.communication.tcp.internal.GridNioServerWrapper) MessageFactory(org.apache.ignite.plugin.extensions.communication.MessageFactory) IOException(java.io.IOException) Ignite(org.apache.ignite.Ignite) IgniteExperimental(org.apache.ignite.lang.IgniteExperimental) ConnectionKey(org.apache.ignite.spi.communication.tcp.internal.ConnectionKey) MessageReader(org.apache.ignite.plugin.extensions.communication.MessageReader) TestOnly(org.jetbrains.annotations.TestOnly) CommunicationSpi(org.apache.ignite.spi.communication.CommunicationSpi) TcpCommunicationConnectionCheckFuture(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture) MetricUtils(org.apache.ignite.internal.processors.metric.impl.MetricUtils) BitSet(java.util.BitSet) GridNioRecoveryDescriptor(org.apache.ignite.internal.util.nio.GridNioRecoveryDescriptor) GridLocalEventListener(org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener) TcpConnectionIndexAwareMessage(org.apache.ignite.spi.communication.tcp.internal.TcpConnectionIndexAwareMessage) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) Message(org.apache.ignite.plugin.extensions.communication.Message) GridNioSession(org.apache.ignite.internal.util.nio.GridNioSession) ConnectionKey(org.apache.ignite.spi.communication.tcp.internal.ConnectionKey) GridNioServerWrapper(org.apache.ignite.spi.communication.tcp.internal.GridNioServerWrapper) MessageFormatter(org.apache.ignite.plugin.extensions.communication.MessageFormatter) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) CommunicationDiscoveryEventListener(org.apache.ignite.spi.communication.tcp.internal.CommunicationDiscoveryEventListener) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) RoundRobinConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.RoundRobinConnectionPolicy) Ignite(org.apache.ignite.Ignite) ShmemAcceptWorker(org.apache.ignite.spi.communication.tcp.internal.shmem.ShmemAcceptWorker) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) UUID(java.util.UUID) GridNioMessageWriterFactory(org.apache.ignite.internal.util.nio.GridNioMessageWriterFactory) ClusterNode(org.apache.ignite.cluster.ClusterNode) FirstConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.FirstConnectionPolicy) TcpHandshakeExecutor(org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor) ClusterStateProvider(org.apache.ignite.spi.communication.tcp.internal.ClusterStateProvider) MessageFactory(org.apache.ignite.plugin.extensions.communication.MessageFactory) TcpCommunicationSpiMBeanImpl(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationSpiMBeanImpl) IOException(java.io.IOException) CommunicationWorker(org.apache.ignite.spi.communication.tcp.internal.CommunicationWorker) IgniteSpiThread(org.apache.ignite.spi.IgniteSpiThread) IgniteEx(org.apache.ignite.internal.IgniteEx) ConnectionClientPool(org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool) IgniteThread(org.apache.ignite.thread.IgniteThread) InboundConnectionHandler(org.apache.ignite.spi.communication.tcp.internal.InboundConnectionHandler) FailureProcessor(org.apache.ignite.internal.processors.failure.FailureProcessor) GridNioMessageReaderFactory(org.apache.ignite.internal.util.nio.GridNioMessageReaderFactory)

Aggregations

IOException (java.io.IOException)1 InetSocketAddress (java.net.InetSocketAddress)1 ByteBuffer (java.nio.ByteBuffer)1 BitSet (java.util.BitSet)1 Collection (java.util.Collection)1 List (java.util.List)1 Map (java.util.Map)1 UUID (java.util.UUID)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 Function (java.util.function.Function)1 Supplier (java.util.function.Supplier)1 Ignite (org.apache.ignite.Ignite)1 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)1 IgniteException (org.apache.ignite.IgniteException)1 IgniteLogger (org.apache.ignite.IgniteLogger)1 ClusterNode (org.apache.ignite.cluster.ClusterNode)1 AddressResolver (org.apache.ignite.configuration.AddressResolver)1 IgniteConfiguration (org.apache.ignite.configuration.IgniteConfiguration)1 EVT_NODE_FAILED (org.apache.ignite.events.EventType.EVT_NODE_FAILED)1 EVT_NODE_LEFT (org.apache.ignite.events.EventType.EVT_NODE_LEFT)1