Search in sources :

Example 1 with TcpHandshakeExecutor

use of org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor in project ignite by apache.

the class TcpCommunicationSpi method spiStart.

/**
 * {@inheritDoc}
 */
@Override
public void spiStart(String igniteInstanceName) throws IgniteSpiException {
    final Function<UUID, ClusterNode> nodeGetter = (nodeId) -> getSpiContext().node(nodeId);
    final Supplier<ClusterNode> locNodeSupplier = () -> getSpiContext().localNode();
    final Supplier<Ignite> igniteExSupplier = this::ignite;
    final Function<UUID, Boolean> pingNode = (nodeId) -> getSpiContext().pingNode(nodeId);
    final Supplier<FailureProcessor> failureProcessorSupplier = () -> ignite instanceof IgniteEx ? ((IgniteEx) ignite).context().failure() : null;
    final Supplier<Boolean> isStopped = () -> getSpiContext().isStopping();
    this.igniteInstanceName = igniteInstanceName;
    cfg.failureDetectionTimeout(ignite.configuration().getFailureDetectionTimeout());
    attributeNames = new AttributeNames(createSpiAttributeName(ATTR_PAIRED_CONN), createSpiAttributeName(ATTR_ADDRS), createSpiAttributeName(ATTR_HOST_NAMES), createSpiAttributeName(ATTR_EXT_ADDRS), createSpiAttributeName(ATTR_PORT), createSpiAttributeName(ATTR_FORCE_CLIENT_SERVER_CONNECTIONS));
    boolean client = Boolean.TRUE.equals(ignite().configuration().isClientMode());
    this.stateProvider = new ClusterStateProvider(ignite, locNodeSupplier, this, isStopped, () -> super.getSpiContext(), log, igniteExSupplier);
    try {
        cfg.localHost(U.resolveLocalHost(cfg.localAddress()));
    } catch (IOException e) {
        throw new IgniteSpiException("Failed to initialize local address: " + cfg.localAddress(), e);
    }
    if (cfg.connectionsPerNode() > 1)
        connPlc = new RoundRobinConnectionPolicy(cfg);
    else
        connPlc = new FirstConnectionPolicy();
    this.srvLsnr = resolve(ignite, new InboundConnectionHandler(log, cfg, nodeGetter, locNodeSupplier, stateProvider, clientPool, commWorker, connectGate, failureProcessorSupplier, attributeNames, metricsLsnr, nioSrvWrapper, ctxInitLatch, client, igniteExSupplier, new CommunicationListener<Message>() {

        @Override
        public void onMessage(UUID nodeId, Message msg, IgniteRunnable msgC) {
            notifyListener(nodeId, msg, msgC);
        }

        @Override
        public void onDisconnected(UUID nodeId) {
            if (lsnr != null)
                lsnr.onDisconnected(nodeId);
        }
    }));
    TcpHandshakeExecutor tcpHandshakeExecutor = resolve(ignite, new TcpHandshakeExecutor(log, stateProvider, cfg.directBuffer()));
    this.nioSrvWrapper = resolve(ignite, new GridNioServerWrapper(log, cfg, attributeNames, tracing, nodeGetter, locNodeSupplier, connectGate, stateProvider, this::getExceptionRegistry, commWorker, ignite.configuration(), this.srvLsnr, getName(), getWorkersRegistry(ignite), ignite instanceof IgniteEx ? ((IgniteEx) ignite).context().metric() : null, this::createTcpClient, new CommunicationListenerEx<Message>() {

        @Override
        public void onMessage(UUID nodeId, Message msg, IgniteRunnable msgC) {
            notifyListener(nodeId, msg, msgC);
        }

        @Override
        public void onDisconnected(UUID nodeId) {
            if (lsnr != null)
                lsnr.onDisconnected(nodeId);
        }

        @Override
        public void onChannelOpened(UUID rmtNodeId, Message initMsg, Channel channel) {
            if (lsnr instanceof CommunicationListenerEx)
                ((CommunicationListenerEx<Message>) lsnr).onChannelOpened(rmtNodeId, initMsg, channel);
        }
    }, tcpHandshakeExecutor));
    this.srvLsnr.setNioSrvWrapper(nioSrvWrapper);
    this.clientPool = resolve(ignite, new ConnectionClientPool(cfg, attributeNames, log, metricsLsnr, locNodeSupplier, nodeGetter, null, getWorkersRegistry(ignite), this, stateProvider, nioSrvWrapper, getName()));
    this.srvLsnr.setClientPool(clientPool);
    nioSrvWrapper.clientPool(clientPool);
    discoLsnr = new CommunicationDiscoveryEventListener(clientPool, metricsLsnr);
    try {
        // This method potentially resets local port to the value
        // local node was bound to.
        nioSrvWrapper.nio(nioSrvWrapper.resetNioServer());
    } catch (IgniteCheckedException e) {
        throw new IgniteSpiException("Failed to initialize TCP server: " + cfg.localHost(), e);
    }
    boolean forceClientToSrvConnections = forceClientToServerConnections() || cfg.localPort() == -1;
    if (cfg.usePairedConnections() && forceClientToSrvConnections) {
        throw new IgniteSpiException("Node using paired connections " + "is not allowed to start in forced client to server connections mode.");
    }
    assert cfg.localHost() != null;
    // Start SPI start stopwatch.
    startStopwatch();
    if (log.isDebugEnabled()) {
        log.debug(configInfo("locAddr", cfg.localAddress()));
        log.debug(configInfo("locPort", cfg.localPort()));
        log.debug(configInfo("locPortRange", cfg.localPortRange()));
        log.debug(configInfo("idleConnTimeout", cfg.idleConnectionTimeout()));
        log.debug(configInfo("directBuf", cfg.directBuffer()));
        log.debug(configInfo("directSendBuf", cfg.directSendBuffer()));
        log.debug(configInfo("selectorsCnt", cfg.selectorsCount()));
        log.debug(configInfo("tcpNoDelay", cfg.tcpNoDelay()));
        log.debug(configInfo("sockSndBuf", cfg.socketSendBuffer()));
        log.debug(configInfo("sockRcvBuf", cfg.socketReceiveBuffer()));
        log.debug(configInfo("msgQueueLimit", cfg.messageQueueLimit()));
        log.debug(configInfo("connectionsPerNode", cfg.connectionsPerNode()));
        if (failureDetectionTimeoutEnabled()) {
            log.debug(configInfo("connTimeout", cfg.connectionTimeout()));
            log.debug(configInfo("maxConnTimeout", cfg.maxConnectionTimeout()));
            log.debug(configInfo("reconCnt", cfg.reconCount()));
        } else
            log.debug(configInfo("failureDetectionTimeout", failureDetectionTimeout()));
        log.debug(configInfo("sockWriteTimeout", cfg.socketWriteTimeout()));
        log.debug(configInfo("ackSndThreshold", cfg.ackSendThreshold()));
        log.debug(configInfo("unackedMsgsBufSize", cfg.unackedMsgsBufferSize()));
    }
    if (!cfg.tcpNoDelay())
        U.quietAndWarn(log, "'TCP_NO_DELAY' for communication is off, which should be used with caution " + "since may produce significant delays with some scenarios.");
    if (cfg.slowClientQueueLimit() > 0 && cfg.messageQueueLimit() > 0 && cfg.slowClientQueueLimit() >= cfg.messageQueueLimit()) {
        U.quietAndWarn(log, "Slow client queue limit is set to a value greater than or equal to message " + "queue limit (slow client queue limit will have no effect) [msgQueueLimit=" + cfg.messageQueueLimit() + ", slowClientQueueLimit=" + cfg.slowClientQueueLimit() + ']');
    }
    if (cfg.messageQueueLimit() == 0)
        U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " + "potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " + "due to message queues growth on sender and receiver sides.");
    nioSrvWrapper.start();
    this.commWorker = new CommunicationWorker(igniteInstanceName, log, cfg, attributeNames, clientPool, failureProcessorSupplier, nodeGetter, pingNode, this::getExceptionRegistry, nioSrvWrapper, getWorkersRegistry(ignite), getName());
    this.srvLsnr.communicationWorker(commWorker);
    this.nioSrvWrapper.communicationWorker(commWorker);
    new IgniteSpiThread(igniteInstanceName, commWorker.name(), log) {

        @Override
        protected void body() {
            commWorker.run();
        }
    }.start();
    // Ack start.
    if (log.isDebugEnabled())
        log.debug(startInfo());
}
Also used : IgniteInternalFuture(org.apache.ignite.internal.IgniteInternalFuture) CommunicationWorker(org.apache.ignite.spi.communication.tcp.internal.CommunicationWorker) TcpHandshakeExecutor(org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor) GridNioSession(org.apache.ignite.internal.util.nio.GridNioSession) TcpCommunicationSpiMBeanImpl(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationSpiMBeanImpl) EVT_NODE_LEFT(org.apache.ignite.events.EventType.EVT_NODE_LEFT) AddressResolver(org.apache.ignite.configuration.AddressResolver) GridFinishedFuture(org.apache.ignite.internal.util.future.GridFinishedFuture) CommunicationListenerEx(org.apache.ignite.spi.communication.tcp.internal.CommunicationListenerEx) IgniteEx(org.apache.ignite.internal.IgniteEx) TcpCommunicationConfigInitializer(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConfigInitializer) ClusterStateProvider(org.apache.ignite.spi.communication.tcp.internal.ClusterStateProvider) ByteBuffer(java.nio.ByteBuffer) GridNioSessionMetaKey(org.apache.ignite.internal.util.nio.GridNioSessionMetaKey) Map(java.util.Map) FirstConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.FirstConnectionPolicy) ConnectionClientPool(org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool) FailureProcessor(org.apache.ignite.internal.processors.failure.FailureProcessor) WorkersRegistry(org.apache.ignite.internal.worker.WorkersRegistry) LoggerResource(org.apache.ignite.resources.LoggerResource) RoundRobinConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.RoundRobinConnectionPolicy) Channel(java.nio.channels.Channel) IgniteKernal(org.apache.ignite.internal.IgniteKernal) NOOP(org.apache.ignite.spi.communication.tcp.internal.CommunicationTcpUtils.NOOP) IgniteFuture(org.apache.ignite.lang.IgniteFuture) IgniteClientDisconnectedCheckedException(org.apache.ignite.internal.IgniteClientDisconnectedCheckedException) IgniteInClosure(org.apache.ignite.lang.IgniteInClosure) GridNioServer(org.apache.ignite.internal.util.nio.GridNioServer) Collection(java.util.Collection) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) IgniteException(org.apache.ignite.IgniteException) CommunicationTcpUtils(org.apache.ignite.spi.communication.tcp.internal.CommunicationTcpUtils) TcpConnectionIndexAwareMessage(org.apache.ignite.spi.communication.tcp.internal.TcpConnectionIndexAwareMessage) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) UNDEFINED_CONNECTION_INDEX(org.apache.ignite.spi.communication.tcp.internal.TcpConnectionIndexAwareMessage.UNDEFINED_CONNECTION_INDEX) UUID(java.util.UUID) InetSocketAddress(java.net.InetSocketAddress) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) CountDownLatch(java.util.concurrent.CountDownLatch) GridResourceProcessor(org.apache.ignite.internal.processors.resource.GridResourceProcessor) List(java.util.List) IgniteConfiguration(org.apache.ignite.configuration.IgniteConfiguration) EVT_NODE_FAILED(org.apache.ignite.events.EventType.EVT_NODE_FAILED) MTC(org.apache.ignite.internal.processors.tracing.MTC) Message(org.apache.ignite.plugin.extensions.communication.Message) GridCommunicationClient(org.apache.ignite.internal.util.nio.GridCommunicationClient) IgniteFutureImpl(org.apache.ignite.internal.util.future.IgniteFutureImpl) ConnectGateway(org.apache.ignite.spi.communication.tcp.internal.ConnectGateway) CommunicationListener(org.apache.ignite.spi.communication.CommunicationListener) IgniteSpiContext(org.apache.ignite.spi.IgniteSpiContext) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) IgniteSpiThread(org.apache.ignite.spi.IgniteSpiThread) U(org.apache.ignite.internal.util.typedef.internal.U) IgnitePortProtocol(org.apache.ignite.spi.IgnitePortProtocol) InboundConnectionHandler(org.apache.ignite.spi.communication.tcp.internal.InboundConnectionHandler) IgniteLogger(org.apache.ignite.IgniteLogger) GridIoManager(org.apache.ignite.internal.managers.communication.GridIoManager) Function(java.util.function.Function) Supplier(java.util.function.Supplier) CommunicationDiscoveryEventListener(org.apache.ignite.spi.communication.tcp.internal.CommunicationDiscoveryEventListener) ClusterNode(org.apache.ignite.cluster.ClusterNode) IgnitePredicate(org.apache.ignite.lang.IgnitePredicate) IgniteSpiConsistencyChecked(org.apache.ignite.spi.IgniteSpiConsistencyChecked) IgniteSpiMultipleInstancesSupport(org.apache.ignite.spi.IgniteSpiMultipleInstancesSupport) S(org.apache.ignite.internal.util.typedef.internal.S) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) GridNioServerWrapper(org.apache.ignite.spi.communication.tcp.internal.GridNioServerWrapper) IOException(java.io.IOException) Ignite(org.apache.ignite.Ignite) ConnectionKey(org.apache.ignite.spi.communication.tcp.internal.ConnectionKey) TestOnly(org.jetbrains.annotations.TestOnly) CommunicationSpi(org.apache.ignite.spi.communication.CommunicationSpi) TcpCommunicationConnectionCheckFuture(org.apache.ignite.spi.communication.tcp.internal.TcpCommunicationConnectionCheckFuture) NodeUnreachableException(org.apache.ignite.spi.communication.tcp.internal.NodeUnreachableException) MetricUtils(org.apache.ignite.internal.processors.metric.impl.MetricUtils) BitSet(java.util.BitSet) GridNioRecoveryDescriptor(org.apache.ignite.internal.util.nio.GridNioRecoveryDescriptor) GridLocalEventListener(org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener) TcpConnectionIndexAwareMessage(org.apache.ignite.spi.communication.tcp.internal.TcpConnectionIndexAwareMessage) GridIoMessage(org.apache.ignite.internal.managers.communication.GridIoMessage) Message(org.apache.ignite.plugin.extensions.communication.Message) GridNioServerWrapper(org.apache.ignite.spi.communication.tcp.internal.GridNioServerWrapper) IgniteRunnable(org.apache.ignite.lang.IgniteRunnable) CommunicationDiscoveryEventListener(org.apache.ignite.spi.communication.tcp.internal.CommunicationDiscoveryEventListener) IgniteCheckedException(org.apache.ignite.IgniteCheckedException) RoundRobinConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.RoundRobinConnectionPolicy) Ignite(org.apache.ignite.Ignite) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) UUID(java.util.UUID) ClusterNode(org.apache.ignite.cluster.ClusterNode) FirstConnectionPolicy(org.apache.ignite.spi.communication.tcp.internal.FirstConnectionPolicy) TcpHandshakeExecutor(org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor) ClusterStateProvider(org.apache.ignite.spi.communication.tcp.internal.ClusterStateProvider) CommunicationListenerEx(org.apache.ignite.spi.communication.tcp.internal.CommunicationListenerEx) Channel(java.nio.channels.Channel) IOException(java.io.IOException) CommunicationWorker(org.apache.ignite.spi.communication.tcp.internal.CommunicationWorker) IgniteSpiThread(org.apache.ignite.spi.IgniteSpiThread) IgniteEx(org.apache.ignite.internal.IgniteEx) ConnectionClientPool(org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool) InboundConnectionHandler(org.apache.ignite.spi.communication.tcp.internal.InboundConnectionHandler) FailureProcessor(org.apache.ignite.internal.processors.failure.FailureProcessor)

Example 2 with TcpHandshakeExecutor

use of org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor in project ignite by apache.

the class TcpCommunicationHandshakeTimeoutTest method testSocketForcedClosedBecauseSlowReadFromSocket.

/**
 * 1. Cluster from three nodes.
 * 2. Waiting when communication connection goes idle.
 * 4. Configure the delay during the communication connection handshake.
 * 5. Force establishing of new connection from node2 to node1 from timeout object processor thread
 * (it use compute in this test).
 * 6. Expected:  The frozen attempt of handshake would be successfully handled and a new connection
 * would be established.
 *
 * @throws Exception If fail.
 */
@Test
public void testSocketForcedClosedBecauseSlowReadFromSocket() throws Exception {
    // given: Two ordinary nodes.
    startGrid(0);
    IgniteEx g1 = startGrid(1);
    // and: One more node which communication connection can be delayed by demand.
    AtomicBoolean delayHandshakeUntilSocketClosed = new AtomicBoolean();
    IgniteEx g2 = startGrid(2, new DependencyResolver() {

        @Override
        public <T> T resolve(T instance) {
            if (instance instanceof TcpHandshakeExecutor) {
                TcpHandshakeExecutor gridNioServer = (TcpHandshakeExecutor) instance;
                return (T) new DelaydTcpHandshakeExecutor(gridNioServer, delayHandshakeUntilSocketClosed);
            }
            return instance;
        }
    });
    awaitPartitionMapExchange();
    AtomicBoolean result = new AtomicBoolean(false);
    // Wait for connections go idle.
    doSleep(1000);
    // when: Initiate communication connection from timeout object processor thread.
    g2.context().timeout().addTimeoutObject(new GridTimeoutObjectAdapter(0) {

        @Override
        public void onTimeout() {
            delayHandshakeUntilSocketClosed.set(true);
            g2.compute(g2.cluster().forNodes(Arrays.asList(g1.localNode()))).withNoFailover().call(() -> true);
            result.set(true);
        }
    });
    // then: Despite the first attempt of handshake would be frozen the compute should be handled well eventually.
    assertTrue("Compute should be successfully handled.", waitForCondition(result::get, 20_000));
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TcpHandshakeExecutor(org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor) GridTimeoutObjectAdapter(org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter) IgniteEx(org.apache.ignite.internal.IgniteEx) DependencyResolver(org.apache.ignite.internal.processors.resource.DependencyResolver) GridCommonAbstractTest(org.apache.ignite.testframework.junits.common.GridCommonAbstractTest) Test(org.junit.Test)

Aggregations

IgniteEx (org.apache.ignite.internal.IgniteEx)2 TcpHandshakeExecutor (org.apache.ignite.spi.communication.tcp.internal.TcpHandshakeExecutor)2 IOException (java.io.IOException)1 InetSocketAddress (java.net.InetSocketAddress)1 ByteBuffer (java.nio.ByteBuffer)1 Channel (java.nio.channels.Channel)1 BitSet (java.util.BitSet)1 Collection (java.util.Collection)1 List (java.util.List)1 Map (java.util.Map)1 UUID (java.util.UUID)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Function (java.util.function.Function)1 Supplier (java.util.function.Supplier)1 Ignite (org.apache.ignite.Ignite)1 IgniteCheckedException (org.apache.ignite.IgniteCheckedException)1 IgniteException (org.apache.ignite.IgniteException)1 IgniteLogger (org.apache.ignite.IgniteLogger)1 ClusterNode (org.apache.ignite.cluster.ClusterNode)1