Examples with ConnectionClientPool org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool used on opensource projects

Example 1 with ConnectionClientPool

use of org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool in project ignite by apache.

the class TcpCommunicationSpi method spiStart.

/**
 * {@inheritDoc}
 */
@Override
public void spiStart(String igniteInstanceName) throws IgniteSpiException {
    final Function<UUID, ClusterNode> nodeGetter = (nodeId) -> getSpiContext().node(nodeId);
    final Supplier<ClusterNode> locNodeSupplier = () -> getSpiContext().localNode();
    final Supplier<Ignite> igniteExSupplier = this::ignite;
    final Function<UUID, Boolean> pingNode = (nodeId) -> getSpiContext().pingNode(nodeId);
    final Supplier<FailureProcessor> failureProcessorSupplier = () -> ignite instanceof IgniteEx ? ((IgniteEx) ignite).context().failure() : null;
    final Supplier<Boolean> isStopped = () -> getSpiContext().isStopping();
    this.igniteInstanceName = igniteInstanceName;
    cfg.failureDetectionTimeout(ignite.configuration().getFailureDetectionTimeout());
    attributeNames = new AttributeNames(createSpiAttributeName(ATTR_PAIRED_CONN), createSpiAttributeName(ATTR_ADDRS), createSpiAttributeName(ATTR_HOST_NAMES), createSpiAttributeName(ATTR_EXT_ADDRS), createSpiAttributeName(ATTR_PORT), createSpiAttributeName(ATTR_FORCE_CLIENT_SERVER_CONNECTIONS));
    boolean client = Boolean.TRUE.equals(ignite().configuration().isClientMode());
    this.stateProvider = new ClusterStateProvider(ignite, locNodeSupplier, this, isStopped, () -> super.getSpiContext(), log, igniteExSupplier);
    try {
        cfg.localHost(U.resolveLocalHost(cfg.localAddress()));
    } catch (IOException e) {
        throw new IgniteSpiException("Failed to initialize local address: " + cfg.localAddress(), e);
    }
    if (cfg.connectionsPerNode() > 1)
        connPlc = new RoundRobinConnectionPolicy(cfg);
    else
        connPlc = new FirstConnectionPolicy();
    this.srvLsnr = resolve(ignite, new InboundConnectionHandler(log, cfg, nodeGetter, locNodeSupplier, stateProvider, clientPool, commWorker, connectGate, failureProcessorSupplier, attributeNames, metricsLsnr, nioSrvWrapper, ctxInitLatch, client, igniteExSupplier, new CommunicationListener<Message>() {

        @Override
        public void onMessage(UUID nodeId, Message msg, IgniteRunnable msgC) {
            notifyListener(nodeId, msg, msgC);
        }

        @Override
        public void onDisconnected(UUID nodeId) {
            if (lsnr != null)
                lsnr.onDisconnected(nodeId);
        }
    }));
    TcpHandshakeExecutor tcpHandshakeExecutor = resolve(ignite, new TcpHandshakeExecutor(log, stateProvider, cfg.directBuffer()));
    this.nioSrvWrapper = resolve(ignite, new GridNioServerWrapper(log, cfg, attributeNames, tracing, nodeGetter, locNodeSupplier, connectGate, stateProvider, this::getExceptionRegistry, commWorker, ignite.configuration(), this.srvLsnr, getName(), getWorkersRegistry(ignite), ignite instanceof IgniteEx ? ((IgniteEx) ignite).context().metric() : null, this::createTcpClient, new CommunicationListenerEx<Message>() {

        @Override
        public void onMessage(UUID nodeId, Message msg, IgniteRunnable msgC) {
            notifyListener(nodeId, msg, msgC);
        }

        @Override
        public void onDisconnected(UUID nodeId) {
            if (lsnr != null)
                lsnr.onDisconnected(nodeId);
        }

        @Override
        public void onChannelOpened(UUID rmtNodeId, Message initMsg, Channel channel) {
            if (lsnr instanceof CommunicationListenerEx)
                ((CommunicationListenerEx<Message>) lsnr).onChannelOpened(rmtNodeId, initMsg, channel);
        }
    }, tcpHandshakeExecutor));
    this.srvLsnr.setNioSrvWrapper(nioSrvWrapper);
    this.clientPool = resolve(ignite, new ConnectionClientPool(cfg, attributeNames, log, metricsLsnr, locNodeSupplier, nodeGetter, null, getWorkersRegistry(ignite), this, stateProvider, nioSrvWrapper, getName()));
    this.srvLsnr.setClientPool(clientPool);
    nioSrvWrapper.clientPool(clientPool);
    discoLsnr = new CommunicationDiscoveryEventListener(clientPool, metricsLsnr);
    try {
        // This method potentially resets local port to the value
        // local node was bound to.
        nioSrvWrapper.nio(nioSrvWrapper.resetNioServer());
    } catch (IgniteCheckedException e) {
        throw new IgniteSpiException("Failed to initialize TCP server: " + cfg.localHost(), e);
    }
    boolean forceClientToSrvConnections = forceClientToServerConnections() || cfg.localPort() == -1;
    if (cfg.usePairedConnections() && forceClientToSrvConnections) {
        throw new IgniteSpiException("Node using paired connections " + "is not allowed to start in forced client to server connections mode.");
    }
    assert cfg.localHost() != null;
    // Start SPI start stopwatch.
    startStopwatch();
    if (log.isDebugEnabled()) {
        log.debug(configInfo("locAddr", cfg.localAddress()));
        log.debug(configInfo("locPort", cfg.localPort()));
        log.debug(configInfo("locPortRange", cfg.localPortRange()));
        log.debug(configInfo("idleConnTimeout", cfg.idleConnectionTimeout()));
        log.debug(configInfo("directBuf", cfg.directBuffer()));
        log.debug(configInfo("directSendBuf", cfg.directSendBuffer()));
        log.debug(configInfo("selectorsCnt", cfg.selectorsCount()));
        log.debug(configInfo("tcpNoDelay", cfg.tcpNoDelay()));
        log.debug(configInfo("sockSndBuf", cfg.socketSendBuffer()));
        log.debug(configInfo("sockRcvBuf", cfg.socketReceiveBuffer()));
        log.debug(configInfo("msgQueueLimit", cfg.messageQueueLimit()));
        log.debug(configInfo("connectionsPerNode", cfg.connectionsPerNode()));
        if (failureDetectionTimeoutEnabled()) {
            log.debug(configInfo("connTimeout", cfg.connectionTimeout()));
            log.debug(configInfo("maxConnTimeout", cfg.maxConnectionTimeout()));
            log.debug(configInfo("reconCnt", cfg.reconCount()));
        } else
            log.debug(configInfo("failureDetectionTimeout", failureDetectionTimeout()));
        log.debug(configInfo("sockWriteTimeout", cfg.socketWriteTimeout()));
        log.debug(configInfo("ackSndThreshold", cfg.ackSendThreshold()));
        log.debug(configInfo("unackedMsgsBufSize", cfg.unackedMsgsBufferSize()));
    }
    if (!cfg.tcpNoDelay())
        U.quietAndWarn(log, "'TCP_NO_DELAY' for communication is off, which should be used with caution " + "since may produce significant delays with some scenarios.");
    if (cfg.slowClientQueueLimit() > 0 && cfg.messageQueueLimit() > 0 && cfg.slowClientQueueLimit() >= cfg.messageQueueLimit()) {
        U.quietAndWarn(log, "Slow client queue limit is set to a value greater than or equal to message " + "queue limit (slow client queue limit will have no effect) [msgQueueLimit=" + cfg.messageQueueLimit() + ", slowClientQueueLimit=" + cfg.slowClientQueueLimit() + ']');
    }
    if (cfg.messageQueueLimit() == 0)
        U.quietAndWarn(log, "Message queue limit is set to 0 which may lead to " + "potential OOMEs when running cache operations in FULL_ASYNC or PRIMARY_SYNC modes " + "due to message queues growth on sender and receiver sides.");
    nioSrvWrapper.start();
    this.commWorker = new CommunicationWorker(igniteInstanceName, log, cfg, attributeNames, clientPool, failureProcessorSupplier, nodeGetter, pingNode, this::getExceptionRegistry, nioSrvWrapper, getWorkersRegistry(ignite), getName());
    this.srvLsnr.communicationWorker(commWorker);
    this.nioSrvWrapper.communicationWorker(commWorker);
    new IgniteSpiThread(igniteInstanceName, commWorker.name(), log) {

        @Override
        protected void body() {
            commWorker.run();
        }
    }.start();
    // Ack start.
    if (log.isDebugEnabled())
        log.debug(startInfo());
}

Example 2 with ConnectionClientPool

use of org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool in project ignite by apache.

the class TxDeadlockOnEntryToStringTest method resolve.

/**
 * The method reference implementation of {@link DependencyResolver}. It adds an additional behavior to {@link
 * InboundConnectionHandler}.
 *
 * @param instance Delegated instance.
 */
private <T> T resolve(T instance) {
    if (instance instanceof InboundConnectionHandler) {
        InboundConnectionHandler hnd = (InboundConnectionHandler) instance;
        return (T) (new InboundConnectionHandler(null, null, null, null, null, null, null, null, null, null, null, null, null, false, null, null) {

            @Override
            public void setNioSrvWrapper(GridNioServerWrapper nioSrvWrapper) {
                hnd.setNioSrvWrapper(nioSrvWrapper);
            }

            @Override
            public void setClientPool(ConnectionClientPool pool) {
                hnd.setClientPool(pool);
            }

            @Override
            public void onSessionWriteTimeout(GridNioSession ses) {
                hnd.onSessionWriteTimeout(ses);
            }

            @Override
            public void onConnected(GridNioSession ses) {
                hnd.onConnected(ses);
            }

            @Override
            public void onMessageSent(GridNioSession ses, Message msg) {
                hnd.onMessageSent(ses, msg);
            }

            @Override
            public void onMessage(GridNioSession ses, Message msg) {
                if (rejectHandshake.get() && msg instanceof HandshakeMessage2) {
                    rejectHandshake.set(false);
                    ses.close();
                    return;
                }
                hnd.onMessage(ses, msg);
            }

            @Override
            public void onFailure(FailureType failureType, Throwable failure) {
                hnd.onFailure(failureType, failure);
            }

            @Override
            public void onDisconnected(GridNioSession ses, @Nullable Exception e) {
                hnd.onDisconnected(ses, e);
            }

            @Override
            public void stop() {
                hnd.stop();
            }

            @Override
            public void communicationWorker(CommunicationWorker commWorker) {
                hnd.communicationWorker(commWorker);
            }

            @Override
            public void onSessionIdleTimeout(GridNioSession ses) {
                hnd.onSessionIdleTimeout(ses);
            }

            @Override
            public void metricsListener(@Nullable TcpCommunicationMetricsListener metricsLsnr) {
                hnd.metricsListener(metricsLsnr);
            }
        });
    }
    return instance;
}

Example 3 with ConnectionClientPool

use of org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool in project ignite by apache.

the class TooManyOpenFilesTcpCommunicationSpiTest method testTooManyOpenFilesErr.

/**
 * Test checks that node will fail in case of error "Too many open files" in {@link TcpCommunicationSpi}.
 *
 * @throws Exception If failed.
 */
@Test
public void testTooManyOpenFilesErr() throws Exception {
    IgniteEx crd = startGrids(3);
    crd.cluster().state(ClusterState.ACTIVE);
    IgniteEx stopNode = grid(2);
    CommunicationSpi stopNodeSpi = stopNode.context().config().getCommunicationSpi();
    ConnectionClientPool connPool = U.field(stopNodeSpi, "clientPool");
    GridNioServerWrapper nioSrvWrapper = U.field(stopNodeSpi, "nioSrvWrapper");
    IgniteEx txNode = grid(1);
    try (Transaction tx = txNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 60_000, 4)) {
        IgniteCache<Object, Object> cache = txNode.cache(DEFAULT_CACHE_NAME);
        cache.put(0, 1);
        nioSrvWrapper.socketChannelFactory(() -> {
            throw new SocketException("Too many open files");
        });
        connPool.forceCloseConnection(txNode.localNode().id());
        cache.put(1, 2);
        cache.put(2, 3);
        cache.put(3, 4);
        // hungs here.
        tx.commit();
    } catch (ClusterTopologyException e) {
        log.error("Error wait commit", e);
    }
    assertTrue(waitForCondition(((IgniteKernal) stopNode)::isStopping, 60_000));
}

Example 4 with ConnectionClientPool

use of org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool in project ignite by apache.

the class TcpCommunicationSpiFaultyClientTest method testFailClient.

/**
 * @param srv Server.
 * @param expDelay Expected delay until client is gone while trying to establish connection.
 * @throws Exception If failed.
 */
private void testFailClient(FakeServer srv, long expDelay) throws Exception {
    IgniteInternalFuture<Long> fut = null;
    try {
        if (srv != null)
            fut = GridTestUtils.runMultiThreadedAsync(srv, 1, "fake-server");
        startGrids(2);
        startClientGrid(2);
        startClientGrid(3);
        // Need to wait for PME to avoid opening new connections during closing idle connections.
        awaitPartitionMapExchange();
        CommunicationSpi commSpi = grid(0).configuration().getCommunicationSpi();
        ConnectionClientPool clientPool = U.field(commSpi, "clientPool");
        ConcurrentMap<UUID, GridCommunicationClient[]> clients = U.field(clientPool, "clients");
        // Wait for write timeout and closing idle connections.
        assertTrue("Failed to wait for closing idle connections.", GridTestUtils.waitForCondition(() -> {
            for (GridCommunicationClient[] clients0 : clients.values()) {
                for (GridCommunicationClient client : clients0) {
                    if (client != null)
                        return false;
                }
            }
            return true;
        }, 1000));
        final CountDownLatch latch = new CountDownLatch(1);
        grid(0).events().localListen(evt -> {
            latch.countDown();
            return true;
        }, EVT_NODE_FAILED);
        block = true;
        long t1 = U.currentTimeMillis();
        try {
            grid(0).compute(grid(0).cluster().forClients()).withNoFailover().broadcast(() -> {
            // No-op.
            });
        } catch (IgniteException ignored) {
        // No-op.
        }
        final long time = U.currentTimeMillis() - t1;
        assertTrue("Must try longer than expected delay", time >= expDelay);
        assertTrue(latch.await(expDelay + 1000, TimeUnit.MILLISECONDS));
        assertTrue(GridTestUtils.waitForCondition(() -> grid(0).cluster().forClients().nodes().size() == 1, 5000));
        for (int i = 0; i < 5; i++) {
            U.sleep(1000);
            log.info("Check topology (" + (i + 1) + "): " + grid(0).cluster().nodes());
            assertEquals(1, grid(0).cluster().forClients().nodes().size());
        }
    } finally {
        if (srv != null) {
            srv.stop();
            assert fut != null;
            fut.get();
        }
        stopAllGrids();
    }
}

Example 5 with ConnectionClientPool

use of org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool in project ignite by apache.

the class TxDeadlockOnEntryToStringTest method testDeadlockOnTimeoutWorkerAndToString.

/**
 * We removed locks from toString on Entry. The case, a thread X lock entry, after that trying to do a handshake
 * with connecting node. But, handshake fails on the first attempt. Between these events, timeout worked trying to
 * print this entry, but I locked. The thread X can't reconnect, because timeout worker hangs.
 */
@Test
public void testDeadlockOnTimeoutWorkerAndToString() throws Exception {
    // Setup
    TestDependencyResolver nearDepRslvr = new TestDependencyResolver();
    IgniteEx nearNode = startGrid(0, nearDepRslvr);
    TestDependencyResolver incomingDepRslvr = new TestDependencyResolver(this::resolve);
    IgniteEx incomingNode = startGrid(1, incomingDepRslvr);
    GridTimeoutProcessor tp = nearNode.context().timeout();
    ConnectionClientPool pool = nearDepRslvr.getDependency(ConnectionClientPool.class);
    GridCacheEntryEx ex = getEntry(nearNode, DEFAULT_CACHE_NAME, TEST_KEY);
    // Act
    try {
        // Lock entry in current thread
        ex.lockEntry();
        // Print the entry from another thread via timeObject.
        CountDownLatch entryPrinted = new CountDownLatch(1);
        CountDownLatch entryReadyToPrint = new CountDownLatch(1);
        tp.addTimeoutObject(new EntryPrinterTimeoutObject(ex, entryPrinted, entryReadyToPrint));
        entryReadyToPrint.await();
        // Try to do first handshake with hangs, after reconnect handshake should be passed.
        rejectHandshake.set(true);
        pool.forceCloseConnection(incomingNode.localNode().id());
        nearNode.configuration().getCommunicationSpi().sendMessage(incomingNode.localNode(), UUIDCollectionMessage.of(UUID.randomUUID()));
        // Check
        assertTrue(GridTestUtils.waitForCondition(() -> entryPrinted.getCount() == 0, 5_000));
    } finally {
        ex.unlockEntry();
    }
}