Examples with IgniteTooManyOpenFilesException org.apache.ignite.internal.IgniteTooManyOpenFilesException used on opensource projects

Example 1 with IgniteTooManyOpenFilesException

use of org.apache.ignite.internal.IgniteTooManyOpenFilesException in project ignite by apache.

the class CommunicationWorker method processDisconnect.

/**
 * @param sesInfo Disconnected session information.
 */
private void processDisconnect(DisconnectedSessionInfo sesInfo) {
    GridNioRecoveryDescriptor recoveryDesc = sesInfo.recoveryDescription();
    ClusterNode node = recoveryDesc.node();
    if (!recoveryDesc.nodeAlive(nodeGetter.apply(node.id())))
        return;
    try {
        if (log.isDebugEnabled())
            log.debug("Recovery reconnect [rmtNode=" + recoveryDesc.node().id() + ']');
        GridCommunicationClient client = clientPool.reserveClient(node, sesInfo.connectionIndex());
        client.release();
    } catch (ClusterTopologyCheckedException e) {
        if (log.isDebugEnabled())
            log.debug("Recovery reconnect failed, node stopping [rmtNode=" + recoveryDesc.node().id() + ']');
    } catch (IgniteTooManyOpenFilesException e) {
        eRegistrySupplier.get().onException(e.getMessage(), e);
        throw e;
    } catch (IgniteCheckedException | IgniteException e) {
        try {
            if (recoveryDesc.nodeAlive(nodeGetter.apply(node.id())) && pingNode.apply(node.id())) {
                if (log.isDebugEnabled()) {
                    log.debug("Recovery reconnect failed, will retry " + "[rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
                }
                addProcessDisconnectRequest(sesInfo);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("Recovery reconnect failed, " + "node left [rmtNode=" + recoveryDesc.node().id() + ", err=" + e + ']');
                }
                eRegistrySupplier.get().onException("Recovery reconnect failed, node left [rmtNode=" + recoveryDesc.node().id() + "]", e);
            }
        } catch (IgniteClientDisconnectedException ignored) {
            if (log.isDebugEnabled())
                log.debug("Failed to ping node, client disconnected.");
        }
    }
}

Example 2 with IgniteTooManyOpenFilesException

use of org.apache.ignite.internal.IgniteTooManyOpenFilesException in project ignite by apache.

the class ConnectionClientPool method reserveClient.

/**
 * Returns existing or just created client to node.
 *
 * @param node Node to which client should be open.
 * @param connIdx Connection index.
 * @return The existing or just created client.
 * @throws IgniteCheckedException Thrown if any exception occurs.
 */
public GridCommunicationClient reserveClient(ClusterNode node, int connIdx) throws IgniteCheckedException {
    assert node != null;
    assert (connIdx >= 0 && connIdx < cfg.connectionsPerNode()) || !(cfg.usePairedConnections() && usePairedConnections(node, attrs.pairedConnection())) : connIdx;
    if (locNodeSupplier.get().isClient()) {
        if (node.isClient()) {
            if (DISABLED_CLIENT_PORT.equals(node.attribute(attrs.port())))
                throw new IgniteSpiException("Cannot send message to the client node with no server socket opened.");
        }
    }
    UUID nodeId = node.id();
    if (log.isDebugEnabled())
        log.debug("The node client is going to reserve a connection [nodeId=" + node.id() + ", connIdx=" + connIdx + "]");
    while (true) {
        GridCommunicationClient[] curClients = clients.get(nodeId);
        GridCommunicationClient client = curClients != null && connIdx < curClients.length ? curClients[connIdx] : null;
        if (client == null) {
            if (stopping)
                throw new IgniteSpiException("Node is stopping.");
            // Do not allow concurrent connects.
            GridFutureAdapter<GridCommunicationClient> fut = new ConnectFuture();
            ConnectionKey connKey = new ConnectionKey(nodeId, connIdx, -1);
            GridFutureAdapter<GridCommunicationClient> oldFut = clientFuts.putIfAbsent(connKey, fut);
            if (oldFut == null) {
                try {
                    GridCommunicationClient[] curClients0 = clients.get(nodeId);
                    GridCommunicationClient client0 = curClients0 != null && connIdx < curClients0.length ? curClients0[connIdx] : null;
                    if (client0 == null) {
                        client0 = createCommunicationClient(node, connIdx);
                        if (client0 != null) {
                            addNodeClient(node, connIdx, client0);
                            if (client0 instanceof GridTcpNioCommunicationClient) {
                                GridTcpNioCommunicationClient tcpClient = ((GridTcpNioCommunicationClient) client0);
                                if (tcpClient.session().closeTime() > 0 && removeNodeClient(nodeId, client0)) {
                                    if (log.isDebugEnabled()) {
                                        log.debug("Session was closed after client creation, will retry " + "[node=" + node + ", client=" + client0 + ']');
                                    }
                                    client0 = null;
                                }
                            }
                        } else {
                            U.sleep(200);
                            if (nodeGetter.apply(node.id()) == null)
                                throw new ClusterTopologyCheckedException("Failed to send message " + "(node left topology): " + node);
                        }
                    }
                    fut.onDone(client0);
                } catch (NodeUnreachableException e) {
                    log.warning(e.getMessage());
                    fut = handleUnreachableNodeException(node, connIdx, fut, e);
                } catch (Throwable e) {
                    if (e instanceof NodeUnreachableException)
                        throw e;
                    fut.onDone(e);
                    if (e instanceof IgniteTooManyOpenFilesException)
                        throw e;
                    if (e instanceof Error)
                        throw (Error) e;
                } finally {
                    clientFuts.remove(connKey, fut);
                }
            } else
                fut = oldFut;
            long clientReserveWaitTimeout = registry != null ? registry.getSystemWorkerBlockedTimeout() / 3 : cfg.connectionTimeout() / 3;
            long currTimeout = System.currentTimeMillis();
            // This cycle will eventually quit when future is completed by concurrent thread reserving client.
            while (true) {
                try {
                    client = fut.get(clientReserveWaitTimeout, TimeUnit.MILLISECONDS);
                    break;
                } catch (IgniteFutureTimeoutCheckedException ignored) {
                    currTimeout += clientReserveWaitTimeout;
                    if (log.isDebugEnabled()) {
                        log.debug("Still waiting for reestablishing connection to node " + "[nodeId=" + node.id() + ", waitingTime=" + currTimeout + "ms]");
                    }
                    if (registry != null) {
                        GridWorker wrkr = registry.worker(Thread.currentThread().getName());
                        if (wrkr != null)
                            wrkr.updateHeartbeat();
                    }
                }
            }
            if (client == null) {
                if (clusterStateProvider.isLocalNodeDisconnected())
                    throw new IgniteCheckedException("Unable to create TCP client due to local node disconnecting.");
                else
                    continue;
            }
            if (nodeGetter.apply(nodeId) == null) {
                if (removeNodeClient(nodeId, client))
                    client.forceClose();
                throw new IgniteSpiException("Destination node is not in topology: " + node.id());
            }
        }
        assert connIdx == client.connectionIndex() : client;
        if (client.reserve())
            return client;
        else
            // Client has just been closed by idle worker. Help it and try again.
            removeNodeClient(nodeId, client);
    }
}

Example 3 with IgniteTooManyOpenFilesException

use of org.apache.ignite.internal.IgniteTooManyOpenFilesException in project ignite by apache.

the class GridNioServerWrapper method createNioSession.

/**
 * Returns the established TCP/IP connection between the current node and remote server. A handshake process of
 * negotiation between two communicating nodes will be performed before the {@link GridNioSession} created.
 * <p>
 * The handshaking process contains of these steps:
 *
 * <ol>
 * <li>The local node opens a new {@link SocketChannel} in the <em>blocking</em> mode.</li>
 * <li>The local node calls {@link SocketChannel#connect(SocketAddress)} to remote node.</li>
 * <li>The remote GridNioAcceptWorker thread accepts new connection.</li>
 * <li>The remote node sends back the {@link NodeIdMessage}.</li>
 * <li>The local node reads NodeIdMessage from created channel.</li>
 * <li>The local node sends the {@link HandshakeMessage2} to remote.</li>
 * <li>The remote node processes {@link HandshakeMessage2} in {@link GridNioServerListener#onMessage(GridNioSession,
 * Object)}.</li>
 * <li>The remote node sends back the {@link RecoveryLastReceivedMessage}.</li>
 * </ol>
 *
 * The handshaking process ends.
 * </p>
 * <p>
 * <em>Note.</em> The {@link HandshakeTimeoutObject} is created to control execution timeout during the
 * whole handshaking process.
 * </p>
 *
 * @param node Remote node identifier to connect with.
 * @param connIdx Connection index based on configured {@link ConnectionPolicy}.
 * @return A {@link GridNioSession} connection representation.
 * @throws IgniteCheckedException If establish connection fails.
 */
public GridNioSession createNioSession(ClusterNode node, int connIdx) throws IgniteCheckedException {
    boolean locNodeIsSrv = !locNodeSupplier.get().isClient() && !locNodeSupplier.get().isDaemon();
    if (!(Thread.currentThread() instanceof IgniteDiscoveryThread) && locNodeIsSrv) {
        if (node.isClient() && forceClientToServerConnections(node)) {
            String msg = "Failed to connect to node " + node.id() + " because it is started" + " in 'forceClientToServerConnections' mode; inverse connection will be requested.";
            throw new NodeUnreachableException(msg);
        }
    }
    Collection<InetSocketAddress> addrs = nodeAddresses(node, cfg.filterReachableAddresses(), attrs, locNodeSupplier);
    GridNioSession ses = null;
    IgniteCheckedException errs = null;
    long totalTimeout;
    if (cfg.failureDetectionTimeoutEnabled())
        totalTimeout = node.isClient() ? stateProvider.clientFailureDetectionTimeout() : cfg.failureDetectionTimeout();
    else {
        totalTimeout = ExponentialBackoffTimeoutStrategy.totalBackoffTimeout(cfg.connectionTimeout(), cfg.maxConnectionTimeout(), cfg.reconCount());
    }
    Set<InetSocketAddress> failedAddrsSet = new HashSet<>();
    int skippedAddrs = 0;
    for (InetSocketAddress addr : addrs) {
        if (addr.isUnresolved()) {
            failedAddrsSet.add(addr);
            continue;
        }
        TimeoutStrategy connTimeoutStgy = new ExponentialBackoffTimeoutStrategy(totalTimeout, cfg.failureDetectionTimeoutEnabled() ? DFLT_INITIAL_TIMEOUT : cfg.connectionTimeout(), cfg.maxConnectionTimeout());
        while (ses == null) {
            // Reconnection on handshake timeout.
            if (stopping)
                throw new IgniteSpiException("Node is stopping.");
            if (isLocalNodeAddress(addr)) {
                if (log.isDebugEnabled())
                    log.debug("Skipping local address [addr=" + addr + ", locAddrs=" + node.attribute(attrs.addresses()) + ", node=" + node + ']');
                skippedAddrs++;
                break;
            }
            long timeout = 0;
            connectGate.enter();
            try {
                if (nodeGetter.apply(node.id()) == null)
                    throw new ClusterTopologyCheckedException("Failed to send message (node left topology): " + node);
                SocketChannel ch = socketChannelFactory.get();
                ch.configureBlocking(true);
                ch.socket().setTcpNoDelay(cfg.tcpNoDelay());
                ch.socket().setKeepAlive(true);
                if (cfg.socketReceiveBuffer() > 0)
                    ch.socket().setReceiveBufferSize(cfg.socketReceiveBuffer());
                if (cfg.socketSendBuffer() > 0)
                    ch.socket().setSendBufferSize(cfg.socketSendBuffer());
                ConnectionKey connKey = new ConnectionKey(node.id(), connIdx, -1);
                GridNioRecoveryDescriptor recoveryDesc = outRecoveryDescriptor(node, connKey);
                assert recoveryDesc != null : "Recovery descriptor not found [connKey=" + connKey + ", rmtNode=" + node.id() + ']';
                if (!recoveryDesc.reserve()) {
                    U.closeQuiet(ch);
                    // Ensure the session is closed.
                    GridNioSession sesFromRecovery = recoveryDesc.session();
                    if (sesFromRecovery != null) {
                        while (sesFromRecovery.closeTime() == 0) sesFromRecovery.close();
                    }
                    return null;
                }
                long rcvCnt;
                Map<Integer, Object> meta = new HashMap<>();
                GridSslMeta sslMeta = null;
                try {
                    timeout = connTimeoutStgy.nextTimeout();
                    ch.socket().connect(addr, (int) timeout);
                    if (nodeGetter.apply(node.id()) == null)
                        throw new ClusterTopologyCheckedException("Failed to send message (node left topology): " + node);
                    if (stateProvider.isSslEnabled()) {
                        meta.put(SSL_META.ordinal(), sslMeta = new GridSslMeta());
                        SSLEngine sslEngine = stateProvider.createSSLEngine();
                        sslEngine.setUseClientMode(true);
                        sslMeta.sslEngine(sslEngine);
                    }
                    ClusterNode locNode = locNodeSupplier.get();
                    if (locNode == null)
                        throw new IgniteCheckedException("Local node has not been started or " + "fully initialized [isStopping=" + stateProvider.isStopping() + ']');
                    timeout = connTimeoutStgy.nextTimeout(timeout);
                    rcvCnt = safeTcpHandshake(ch, node.id(), timeout, sslMeta, new HandshakeMessage2(locNode.id(), recoveryDesc.incrementConnectCount(), recoveryDesc.received(), connIdx));
                    if (rcvCnt == ALREADY_CONNECTED)
                        return null;
                    else if (rcvCnt == NODE_STOPPING) {
                        // Safe to remap on remote node stopping.
                        throw new ClusterTopologyCheckedException("Remote node started stop procedure: " + node.id());
                    } else if (rcvCnt == UNKNOWN_NODE)
                        throw new IgniteCheckedException("Remote node does not observe current node " + "in topology : " + node.id());
                    else if (rcvCnt == NEED_WAIT) {
                        // scenarios with delayed client node join.
                        if (log.isDebugEnabled())
                            log.debug("NEED_WAIT received, handshake after delay [node = " + node + ", outOfTopologyDelay = " + DFLT_NEED_WAIT_DELAY + "ms]");
                        U.sleep(DFLT_NEED_WAIT_DELAY);
                        continue;
                    } else if (rcvCnt < 0)
                        throw new IgniteCheckedException("Unsupported negative receivedCount [rcvCnt=" + rcvCnt + ", senderNode=" + node + ']');
                    recoveryDesc.onHandshake(rcvCnt);
                    meta.put(CONSISTENT_ID_META, node.consistentId());
                    meta.put(CONN_IDX_META, connKey);
                    meta.put(GridNioServer.RECOVERY_DESC_META_KEY, recoveryDesc);
                    ses = nioSrv.createSession(ch, meta, false, null).get();
                } finally {
                    if (ses == null) {
                        U.closeQuiet(ch);
                        if (recoveryDesc != null)
                            recoveryDesc.release();
                    }
                }
            } catch (IgniteSpiOperationTimeoutException e) {
                // Handshake is timed out.
                if (ses != null) {
                    ses.close();
                    ses = null;
                }
                eRegistrySupplier.get().onException("Handshake timed out (will retry with increased timeout) [connTimeoutStrategy=" + connTimeoutStgy + ", addr=" + addr + ']', e);
                if (log.isDebugEnabled())
                    log.debug("Handshake timed out (will retry with increased timeout) [connTimeoutStrategy=" + connTimeoutStgy + ", addr=" + addr + ", err=" + e + ']');
                if (connTimeoutStgy.checkTimeout()) {
                    U.warn(log, "Handshake timed out (will stop attempts to perform the handshake) " + "[node=" + node.id() + ", connTimeoutStrategy=" + connTimeoutStgy + ", err=" + e.getMessage() + ", addr=" + addr + ", failureDetectionTimeoutEnabled=" + cfg.failureDetectionTimeoutEnabled() + ", timeout=" + timeout + ']');
                    String msg = "Failed to connect to node (is node still alive?). " + "Make sure that each ComputeTask and cache Transaction has a timeout set " + "in order to prevent parties from waiting forever in case of network issues " + "[nodeId=" + node.id() + ", addrs=" + addrs + ']';
                    if (errs == null)
                        errs = new IgniteCheckedException(msg, e);
                    else
                        errs.addSuppressed(new IgniteCheckedException(msg, e));
                    break;
                }
            } catch (ClusterTopologyCheckedException e) {
                throw e;
            } catch (Exception e) {
                // Most probably IO error on socket connect or handshake.
                if (ses != null) {
                    ses.close();
                    ses = null;
                }
                eRegistrySupplier.get().onException("Client creation failed [addr=" + addr + ", err=" + e + ']', e);
                if (log.isDebugEnabled())
                    log.debug("Client creation failed [addr=" + addr + ", err=" + e + ']');
                if (X.hasCause(e, "Too many open files", SocketException.class))
                    throw new IgniteTooManyOpenFilesException(e);
                // check if timeout occured in case of unrecoverable exception
                if (connTimeoutStgy.checkTimeout()) {
                    U.warn(log, "Connection timed out (will stop attempts to perform the connect) " + "[node=" + node.id() + ", connTimeoutStgy=" + connTimeoutStgy + ", failureDetectionTimeoutEnabled=" + cfg.failureDetectionTimeoutEnabled() + ", timeout=" + timeout + ", err=" + e.getMessage() + ", addr=" + addr + ']');
                    String msg = "Failed to connect to node (is node still alive?). " + "Make sure that each ComputeTask and cache Transaction has a timeout set " + "in order to prevent parties from waiting forever in case of network issues " + "[nodeId=" + node.id() + ", addrs=" + addrs + ']';
                    if (errs == null)
                        errs = new IgniteCheckedException(msg, e);
                    else
                        errs.addSuppressed(new IgniteCheckedException(msg, e));
                    break;
                }
                // Inverse communication protocol works only for client nodes.
                if (node.isClient() && isNodeUnreachableException(e))
                    failedAddrsSet.add(addr);
                if (isRecoverableException(e))
                    U.sleep(DFLT_RECONNECT_DELAY);
                else {
                    String msg = "Failed to connect to node due to unrecoverable exception (is node still alive?). " + "Make sure that each ComputeTask and cache Transaction has a timeout set " + "in order to prevent parties from waiting forever in case of network issues " + "[nodeId=" + node.id() + ", addrs=" + addrs + ", err= " + e + ']';
                    if (errs == null)
                        errs = new IgniteCheckedException(msg, e);
                    else
                        errs.addSuppressed(new IgniteCheckedException(msg, e));
                    break;
                }
            } finally {
                connectGate.leave();
            }
            CommunicationWorker commWorker0 = commWorker;
            if (commWorker0 != null && commWorker0.runner() == Thread.currentThread())
                commWorker0.updateHeartbeat();
        }
        if (ses != null)
            break;
    }
    if (ses == null) {
        // inverse connection so no point in throwing NodeUnreachableException
        if (!cfg.usePairedConnections() || !Boolean.TRUE.equals(node.attribute(attrs.pairedConnection()))) {
            if (!(Thread.currentThread() instanceof IgniteDiscoveryThread) && locNodeIsSrv) {
                if (node.isClient() && (addrs.size() - skippedAddrs == failedAddrsSet.size())) {
                    String msg = "Failed to connect to all addresses of node " + node.id() + ": " + failedAddrsSet + "; inverse connection will be requested.";
                    throw new NodeUnreachableException(msg);
                }
            }
        }
        processSessionCreationError(node, addrs, errs == null ? new IgniteCheckedException("No session found") : errs);
    }
    return ses;
}