Examples with RecoveryLastReceivedMessage org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage used on opensource projects

Example 1 with RecoveryLastReceivedMessage

use of org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage in project ignite by apache.

the class CommunicationWorker method sendAckOnTimeout.

/**
 * @param recovery Recovery descriptor.
 * @param ses Session.
 */
private void sendAckOnTimeout(GridNioRecoveryDescriptor recovery, GridNioSession ses) {
    if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
        RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
        if (log.isDebugEnabled()) {
            log.debug("Send recovery acknowledgement on timeout [rmtNode=" + recovery.node().id() + ", rcvCnt=" + msg.received() + ", lastAcked=" + recovery.lastAcknowledged() + ']');
        }
        try {
            nioSrvWrapper.nio().sendSystem(ses, msg);
            recovery.lastAcknowledged(msg.received());
        } catch (IgniteCheckedException e) {
            U.error(log, "Failed to send message: " + e, e);
        }
    }
}

Example 2 with RecoveryLastReceivedMessage

use of org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage in project ignite by apache.

the class TcpCommunicationSpi method resetNioServer.

/**
 * Recreates tpcSrvr socket instance.
 *
 * @return Server instance.
 * @throws IgniteCheckedException Thrown if it's not possible to create server.
 */
private GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
    if (boundTcpPort >= 0)
        throw new IgniteCheckedException("Tcp NIO server was already created on port " + boundTcpPort);
    IgniteCheckedException lastEx = null;
    // If configured TCP port is busy, find first available in range.
    int lastPort = locPortRange == 0 ? locPort : locPort + locPortRange - 1;
    for (int port = locPort; port <= lastPort; port++) {
        try {
            MessageFactory msgFactory = new MessageFactory() {

                private MessageFactory impl;

                @Nullable
                @Override
                public Message create(short type) {
                    if (impl == null)
                        impl = getSpiContext().messageFactory();
                    assert impl != null;
                    return impl.create(type);
                }
            };
            GridNioMessageReaderFactory readerFactory = new GridNioMessageReaderFactory() {

                private MessageFormatter formatter;

                @Override
                public MessageReader reader(GridNioSession ses, MessageFactory msgFactory) throws IgniteCheckedException {
                    if (formatter == null)
                        formatter = getSpiContext().messageFormatter();
                    assert formatter != null;
                    ConnectionKey key = ses.meta(CONN_IDX_META);
                    return key != null ? formatter.reader(key.nodeId(), msgFactory) : null;
                }
            };
            GridNioMessageWriterFactory writerFactory = new GridNioMessageWriterFactory() {

                private MessageFormatter formatter;

                @Override
                public MessageWriter writer(GridNioSession ses) throws IgniteCheckedException {
                    if (formatter == null)
                        formatter = getSpiContext().messageFormatter();
                    assert formatter != null;
                    ConnectionKey key = ses.meta(CONN_IDX_META);
                    return key != null ? formatter.writer(key.nodeId()) : null;
                }
            };
            GridDirectParser parser = new GridDirectParser(log.getLogger(GridDirectParser.class), msgFactory, readerFactory);
            IgnitePredicate<Message> skipRecoveryPred = new IgnitePredicate<Message>() {

                @Override
                public boolean apply(Message msg) {
                    return msg instanceof RecoveryLastReceivedMessage;
                }
            };
            boolean clientMode = Boolean.TRUE.equals(ignite.configuration().isClientMode());
            IgniteBiInClosure<GridNioSession, Integer> queueSizeMonitor = !clientMode && slowClientQueueLimit > 0 ? new CI2<GridNioSession, Integer>() {

                @Override
                public void apply(GridNioSession ses, Integer qSize) {
                    checkClientQueueSize(ses, qSize);
                }
            } : null;
            GridNioFilter[] filters;
            if (isSslEnabled()) {
                GridNioSslFilter sslFilter = new GridNioSslFilter(ignite.configuration().getSslContextFactory().create(), true, ByteOrder.nativeOrder(), log);
                sslFilter.directMode(true);
                sslFilter.wantClientAuth(true);
                sslFilter.needClientAuth(true);
                filters = new GridNioFilter[] { new GridNioCodecFilter(parser, log, true), new GridConnectionBytesVerifyFilter(log), sslFilter };
            } else
                filters = new GridNioFilter[] { new GridNioCodecFilter(parser, log, true), new GridConnectionBytesVerifyFilter(log) };
            GridNioServer<Message> srvr = GridNioServer.<Message>builder().address(locHost).port(port).listener(srvLsnr).logger(log).selectorCount(selectorsCnt).igniteInstanceName(igniteInstanceName).serverName("tcp-comm").tcpNoDelay(tcpNoDelay).directBuffer(directBuf).byteOrder(ByteOrder.nativeOrder()).socketSendBufferSize(sockSndBuf).socketReceiveBufferSize(sockRcvBuf).sendQueueLimit(msgQueueLimit).directMode(true).metricsListener(metricsLsnr).writeTimeout(sockWriteTimeout).selectorSpins(selectorSpins).filters(filters).writerFactory(writerFactory).skipRecoveryPredicate(skipRecoveryPred).messageQueueSizeListener(queueSizeMonitor).readWriteSelectorsAssign(usePairedConnections).build();
            boundTcpPort = port;
            // Ack Port the TCP server was bound to.
            if (log.isInfoEnabled()) {
                log.info("Successfully bound communication NIO server to TCP port " + "[port=" + boundTcpPort + ", locHost=" + locHost + ", selectorsCnt=" + selectorsCnt + ", selectorSpins=" + srvr.selectorSpins() + ", pairedConn=" + usePairedConnections + ']');
            }
            srvr.idleTimeout(idleConnTimeout);
            return srvr;
        } catch (IgniteCheckedException e) {
            if (X.hasCause(e, SSLException.class))
                throw new IgniteSpiException("Failed to create SSL context. SSL factory: " + ignite.configuration().getSslContextFactory() + '.', e);
            lastEx = e;
            if (log.isDebugEnabled())
                log.debug("Failed to bind to local port (will try next port within range) [port=" + port + ", locHost=" + locHost + ']');
            onException("Failed to bind to local port (will try next port within range) [port=" + port + ", locHost=" + locHost + ']', e);
        }
    }
    // If free port wasn't found.
    throw new IgniteCheckedException("Failed to bind to any port within range [startPort=" + locPort + ", portRange=" + locPortRange + ", locHost=" + locHost + ']', lastEx);
}

Example 3 with RecoveryLastReceivedMessage

use of org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage in project ignite by apache.

the class GridNioServerWrapper method resetNioServer.

/**
 * Recreates tpcSrvr socket instance.
 *
 * @return Server instance.
 * @throws IgniteCheckedException Thrown if it's not possible to create server.
 */
public GridNioServer<Message> resetNioServer() throws IgniteCheckedException {
    if (cfg.boundTcpPort() >= 0)
        throw new IgniteCheckedException("Tcp NIO server was already created on port " + cfg.boundTcpPort());
    IgniteCheckedException lastEx = null;
    // If configured TCP port is busy, find first available in range.
    int lastPort = cfg.localPort() == -1 ? -1 : cfg.localPortRange() == 0 ? cfg.localPort() : cfg.localPort() + cfg.localPortRange() - 1;
    for (int port = cfg.localPort(); port <= lastPort; port++) {
        try {
            MessageFactory msgFactory = new MessageFactory() {

                private MessageFactory impl;

                @Nullable
                @Override
                public Message create(short type) {
                    if (impl == null)
                        impl = stateProvider.getSpiContext().messageFactory();
                    assert impl != null;
                    return impl.create(type);
                }
            };
            GridNioMessageReaderFactory readerFactory = new GridNioMessageReaderFactory() {

                private IgniteSpiContext context;

                private MessageFormatter formatter;

                @Override
                public MessageReader reader(GridNioSession ses, MessageFactory msgFactory) throws IgniteCheckedException {
                    final IgniteSpiContext ctx = stateProvider.getSpiContextWithoutInitialLatch();
                    if (formatter == null || context != ctx) {
                        context = ctx;
                        formatter = context.messageFormatter();
                    }
                    assert formatter != null;
                    ConnectionKey key = ses.meta(CONN_IDX_META);
                    return key != null ? formatter.reader(key.nodeId(), msgFactory) : null;
                }
            };
            GridNioMessageWriterFactory writerFactory = new GridNioMessageWriterFactory() {

                private IgniteSpiContext context;

                private MessageFormatter formatter;

                @Override
                public MessageWriter writer(GridNioSession ses) throws IgniteCheckedException {
                    final IgniteSpiContext ctx = stateProvider.getSpiContextWithoutInitialLatch();
                    if (formatter == null || context != ctx) {
                        context = ctx;
                        formatter = context.messageFormatter();
                    }
                    assert formatter != null;
                    ConnectionKey key = ses.meta(CONN_IDX_META);
                    return key != null ? formatter.writer(key.nodeId()) : null;
                }
            };
            GridDirectParser parser = new GridDirectParser(log.getLogger(GridDirectParser.class), msgFactory, readerFactory);
            IgnitePredicate<Message> skipRecoveryPred = msg -> msg instanceof RecoveryLastReceivedMessage;
            boolean clientMode = Boolean.TRUE.equals(igniteCfg.isClientMode());
            IgniteBiInClosure<GridNioSession, Integer> queueSizeMonitor = !clientMode && cfg.slowClientQueueLimit() > 0 ? this::checkClientQueueSize : null;
            List<GridNioFilter> filters = new ArrayList<>();
            if (tracing instanceof GridTracingManager && ((GridManager) tracing).enabled())
                filters.add(new GridNioTracerFilter(log, tracing));
            filters.add(new GridNioCodecFilter(parser, log, true));
            filters.add(new GridConnectionBytesVerifyFilter(log));
            if (stateProvider.isSslEnabled()) {
                GridNioSslFilter sslFilter = new GridNioSslFilter(igniteCfg.getSslContextFactory().create(), true, ByteOrder.LITTLE_ENDIAN, log, metricMgr == null ? null : metricMgr.registry(COMMUNICATION_METRICS_GROUP_NAME));
                sslFilter.directMode(true);
                sslFilter.wantClientAuth(true);
                sslFilter.needClientAuth(true);
                filters.add(sslFilter);
            }
            GridNioServer.Builder<Message> builder = GridNioServer.<Message>builder().address(cfg.localHost()).port(port).listener(srvLsnr).logger(log).selectorCount(cfg.selectorsCount()).igniteInstanceName(igniteInstanceName).serverName("tcp-comm").tcpNoDelay(cfg.tcpNoDelay()).directBuffer(cfg.directBuffer()).byteOrder(ByteOrder.LITTLE_ENDIAN).socketSendBufferSize(cfg.socketSendBuffer()).socketReceiveBufferSize(cfg.socketReceiveBuffer()).sendQueueLimit(cfg.messageQueueLimit()).directMode(true).writeTimeout(cfg.socketWriteTimeout()).selectorSpins(cfg.selectorSpins()).filters(filters.toArray(new GridNioFilter[filters.size()])).writerFactory(writerFactory).skipRecoveryPredicate(skipRecoveryPred).messageQueueSizeListener(queueSizeMonitor).tracing(tracing).readWriteSelectorsAssign(cfg.usePairedConnections());
            if (metricMgr != null) {
                builder.workerListener(workersRegistry).metricRegistry(metricMgr.registry(COMMUNICATION_METRICS_GROUP_NAME));
            }
            GridNioServer<Message> srvr = builder.build();
            cfg.boundTcpPort(port);
            // Ack Port the TCP server was bound to.
            if (log.isInfoEnabled()) {
                log.info("Successfully bound communication NIO server to TCP port " + "[port=" + cfg.boundTcpPort() + ", locHost=" + cfg.localHost() + ", selectorsCnt=" + cfg.selectorsCount() + ", selectorSpins=" + srvr.selectorSpins() + ", pairedConn=" + cfg.usePairedConnections() + ']');
            }
            srvr.idleTimeout(cfg.idleConnectionTimeout());
            return srvr;
        } catch (IgniteCheckedException e) {
            if (X.hasCause(e, SSLException.class))
                throw new IgniteSpiException("Failed to create SSL context. SSL factory: " + igniteCfg.getSslContextFactory() + '.', e);
            lastEx = e;
            if (log.isDebugEnabled())
                log.debug("Failed to bind to local port (will try next port within range) [port=" + port + ", locHost=" + cfg.localHost() + ']');
            eRegistrySupplier.get().onException("Failed to bind to local port (will try next port within range) [port=" + port + ", locHost=" + cfg.localHost() + ']', e);
        }
    }
    // If free port wasn't found.
    throw new IgniteCheckedException("Failed to bind to any port within range [startPort=" + cfg.localPort() + ", portRange=" + cfg.localPortRange() + ", locHost=" + cfg.localHost() + ']', lastEx);
}

Example 4 with RecoveryLastReceivedMessage

use of org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage in project ignite by apache.

the class CommunicationWorker method processIdle.

/**
 * Process idle.
 */
private void processIdle() {
    cleanupRecovery();
    for (Map.Entry<UUID, GridCommunicationClient[]> e : clientPool.entrySet()) {
        UUID nodeId = e.getKey();
        for (GridCommunicationClient client : e.getValue()) {
            if (client == null)
                continue;
            ClusterNode node = nodeGetter.apply(nodeId);
            if (node == null) {
                if (log.isDebugEnabled())
                    log.debug("Forcing close of non-existent node connection: " + nodeId);
                client.forceClose();
                clientPool.removeNodeClient(nodeId, client);
                continue;
            }
            GridNioRecoveryDescriptor recovery = null;
            if (!(cfg.usePairedConnections() && usePairedConnections(node, attrs.pairedConnection())) && client instanceof GridTcpNioCommunicationClient) {
                recovery = nioSrvWrapper.recoveryDescs().get(new ConnectionKey(node.id(), client.connectionIndex(), -1));
                if (recovery != null && recovery.lastAcknowledged() != recovery.received()) {
                    RecoveryLastReceivedMessage msg = new RecoveryLastReceivedMessage(recovery.received());
                    if (log.isDebugEnabled())
                        log.debug("Send recovery acknowledgement on timeout [rmtNode=" + nodeId + ", rcvCnt=" + msg.received() + ']');
                    try {
                        nioSrvWrapper.nio().sendSystem(((GridTcpNioCommunicationClient) client).session(), msg);
                        recovery.lastAcknowledged(msg.received());
                    } catch (IgniteCheckedException err) {
                        U.error(log, "Failed to send message: " + err, err);
                    }
                    continue;
                }
            }
            long idleTime = client.getIdleTime();
            if (idleTime >= cfg.idleConnectionTimeout()) {
                if (recovery == null && cfg.usePairedConnections() && usePairedConnections(node, attrs.pairedConnection()))
                    recovery = nioSrvWrapper.outRecDescs().get(new ConnectionKey(node.id(), client.connectionIndex(), -1));
                if (recovery != null && recovery.nodeAlive(nodeGetter.apply(nodeId)) && !recovery.messagesRequests().isEmpty()) {
                    if (log.isDebugEnabled())
                        log.debug("Node connection is idle, but there are unacknowledged messages, " + "will wait: " + nodeId);
                    continue;
                }
                if (log.isDebugEnabled())
                    log.debug("Closing idle node connection: " + nodeId);
                if (client.close() || client.closed())
                    clientPool.removeNodeClient(nodeId, client);
            }
        }
    }
    for (GridNioSession ses : nioSrvWrapper.nio().sessions()) {
        GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
        if (recovery != null && cfg.usePairedConnections() && usePairedConnections(recovery.node(), attrs.pairedConnection())) {
            assert ses.accepted() : ses;
            sendAckOnTimeout(recovery, ses);
        }
    }
}

Example 5 with RecoveryLastReceivedMessage

use of org.apache.ignite.spi.communication.tcp.messages.RecoveryLastReceivedMessage in project ignite by apache.

the class InboundConnectionHandler method onMessage.

/**
 * {@inheritDoc}
 */
@Override
public void onMessage(final GridNioSession ses, Message msg) {
    Span span = MTC.span();
    span.addLog(() -> "Communication received");
    span.addTag(SpanTags.MESSAGE, () -> traceName(msg));
    ConnectionKey connKey = ses.meta(CONN_IDX_META);
    if (connKey == null) {
        assert ses.accepted() : ses;
        if (!connectGate.tryEnter()) {
            if (log.isDebugEnabled())
                log.debug("Close incoming connection, failed to enter gateway.");
            ses.send(new RecoveryLastReceivedMessage(NODE_STOPPING)).listen(fut -> ses.close());
            return;
        }
        try {
            onFirstMessage(ses, msg);
        } finally {
            connectGate.leave();
        }
    } else {
        if (isChannelConnIdx(connKey.connectionIndex())) {
            if (ses.meta(CHANNEL_FUT_META) == null)
                nioSrvWrapper.onChannelCreate((GridSelectorNioSessionImpl) ses, connKey, msg);
            else {
                GridFutureAdapter<Channel> fut = ses.meta(CHANNEL_FUT_META);
                GridSelectorNioSessionImpl ses0 = (GridSelectorNioSessionImpl) ses;
                ses0.closeSocketOnSessionClose(false);
                ses0.close().listen(f -> {
                    if (f.error() != null) {
                        fut.onDone(f.error());
                        return;
                    }
                    fut.onDone(ses0.key().channel());
                });
            }
            return;
        }
        Object consistentId = ses.meta(CONSISTENT_ID_META);
        assert consistentId != null;
        if (msg instanceof RecoveryLastReceivedMessage) {
            metricsLsnr.onMessageReceived(msg, consistentId);
            GridNioRecoveryDescriptor recovery = ses.outRecoveryDescriptor();
            if (recovery != null) {
                RecoveryLastReceivedMessage msg0 = (RecoveryLastReceivedMessage) msg;
                if (log.isDebugEnabled()) {
                    log.debug("Received recovery acknowledgement [rmtNode=" + connKey.nodeId() + ", connIdx=" + connKey.connectionIndex() + ", rcvCnt=" + msg0.received() + ']');
                }
                recovery.ackReceived(msg0.received());
            }
            return;
        } else {
            GridNioRecoveryDescriptor recovery = ses.inRecoveryDescriptor();
            if (recovery != null) {
                long rcvCnt = recovery.onReceived();
                if (rcvCnt % cfg.ackSendThreshold() == 0) {
                    if (log.isDebugEnabled()) {
                        log.debug("Send recovery acknowledgement [rmtNode=" + connKey.nodeId() + ", connIdx=" + connKey.connectionIndex() + ", rcvCnt=" + rcvCnt + ']');
                    }
                    ses.systemMessage(new RecoveryLastReceivedMessage(rcvCnt));
                    recovery.lastAcknowledged(rcvCnt);
                }
            } else if (connKey.dummy()) {
                assert msg instanceof NodeIdMessage : msg;
                TcpCommunicationNodeConnectionCheckFuture fut = ses.meta(SES_FUT_META);
                assert fut != null : msg;
                fut.onConnected(U.bytesToUuid(((NodeIdMessage) msg).nodeIdBytes(), 0));
                nioSrvWrapper.nio().closeFromWorkerThread(ses);
                return;
            }
        }
        metricsLsnr.onMessageReceived(msg, consistentId);
        IgniteRunnable c;
        if (cfg.messageQueueLimit() > 0) {
            GridNioMessageTracker tracker = ses.meta(TRACKER_META);
            if (tracker == null) {
                GridNioMessageTracker old = ses.addMeta(TRACKER_META, tracker = new GridNioMessageTracker(ses, cfg.messageQueueLimit()));
                assert old == null;
            }
            tracker.onMessageReceived();
            c = tracker;
        } else
            c = NOOP;
        lsnr.onMessage(connKey.nodeId(), msg, c);
    }
}