Examples with CommunicationsSession org.apache.nifi.remote.protocol.CommunicationsSession used on opensource projects

Example 11 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class SocketFlowFileServerProtocol method negotiateCodec.

@Override
public FlowFileCodec negotiateCodec(final Peer peer) throws IOException, ProtocolException {
    if (!handshakeCompleted) {
        throw new IllegalStateException("Handshake has not been completed");
    }
    if (shutdown) {
        throw new IllegalStateException("Protocol is shutdown");
    }
    logger.debug("{} Negotiating Codec with {} using {}", new Object[] { this, peer, peer.getCommunicationsSession() });
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    if (port == null) {
        RemoteResourceFactory.rejectCodecNegotiation(dis, dos, "Cannot transfer FlowFiles because no port was specified");
    }
    // Negotiate the FlowFileCodec to use.
    try {
        negotiatedFlowFileCodec = RemoteResourceFactory.receiveCodecNegotiation(dis, dos);
        logger.debug("{} Negotiated Codec {} with {}", new Object[] { this, negotiatedFlowFileCodec, peer });
        return negotiatedFlowFileCodec;
    } catch (final HandshakeException e) {
        throw new ProtocolException(e.toString());
    }
}

Example 12 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class EndpointConnectionPool method establishSiteToSiteConnection.

private CommunicationsSession establishSiteToSiteConnection(final String hostname, final int port) throws IOException {
    final boolean siteToSiteSecure = siteInfoProvider.isSecure();
    CommunicationsSession commsSession = null;
    try {
        if (siteToSiteSecure) {
            if (sslContext == null) {
                throw new IOException("Unable to communicate with " + hostname + ":" + port + " because it requires Secure Site-to-Site communications, but this instance is not configured for secure communications");
            }
            final SSLSocketChannel socketChannel = new SSLSocketChannel(sslContext, hostname, port, localAddress, true);
            socketChannel.connect();
            commsSession = new SSLSocketChannelCommunicationsSession(socketChannel);
            try {
                commsSession.setUserDn(socketChannel.getDn());
            } catch (final CertificateException ex) {
                throw new IOException(ex);
            }
        } else {
            final SocketChannel socketChannel = SocketChannel.open();
            if (localAddress != null) {
                final SocketAddress localSocketAddress = new InetSocketAddress(localAddress, 0);
                socketChannel.socket().bind(localSocketAddress);
            }
            socketChannel.socket().connect(new InetSocketAddress(hostname, port), commsTimeout);
            socketChannel.socket().setSoTimeout(commsTimeout);
            commsSession = new SocketChannelCommunicationsSession(socketChannel);
        }
        commsSession.getOutput().getOutputStream().write(CommunicationsSession.MAGIC_BYTES);
    } catch (final IOException ioe) {
        if (commsSession != null) {
            commsSession.close();
        }
        throw ioe;
    }
    return commsSession;
}

Example 13 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class EndpointConnectionPool method getEndpointConnection.

public EndpointConnection getEndpointConnection(final TransferDirection direction, final SiteToSiteClientConfig config) throws IOException {
    // 
    // Attempt to get a connection state that already exists for this URL.
    // 
    FlowFileCodec codec = null;
    CommunicationsSession commsSession = null;
    SocketClientProtocol protocol = null;
    EndpointConnection connection;
    Peer peer = null;
    final URI clusterUrl;
    try {
        clusterUrl = siteInfoProvider.getActiveClusterUrl();
    } catch (final IOException ioe) {
        throw new UnreachableClusterException("Unable to refresh details from any of the configured remote instances.", ioe);
    }
    do {
        final List<EndpointConnection> addBack = new ArrayList<>();
        logger.debug("{} getting next peer status", this);
        final PeerStatus peerStatus = peerSelector.getNextPeerStatus(direction);
        logger.debug("{} next peer status = {}", this, peerStatus);
        if (peerStatus == null) {
            return null;
        }
        final PeerDescription peerDescription = peerStatus.getPeerDescription();
        BlockingQueue<EndpointConnection> connectionQueue = connectionQueueMap.get(peerDescription);
        if (connectionQueue == null) {
            connectionQueue = new LinkedBlockingQueue<>();
            BlockingQueue<EndpointConnection> existing = connectionQueueMap.putIfAbsent(peerDescription, connectionQueue);
            if (existing != null) {
                connectionQueue = existing;
            }
        }
        try {
            connection = connectionQueue.poll();
            logger.debug("{} Connection State for {} = {}", this, clusterUrl, connection);
            final String portId = getPortIdentifier(direction);
            if (connection == null && !addBack.isEmpty()) {
                // all available connections have been penalized.
                logger.debug("{} all Connections for {} are penalized; returning no Connection", this, portId);
                return null;
            }
            if (connection != null && connection.getPeer().isPenalized(portId)) {
                // we have a connection, but it's penalized. We want to add it back to the queue
                // when we've found one to use.
                addBack.add(connection);
                continue;
            }
            // if we can't get an existing Connection, create one
            if (connection == null) {
                logger.debug("{} No Connection available for Port {}; creating new Connection", this, portId);
                protocol = new SocketClientProtocol();
                protocol.setDestination(new IdEnrichedRemoteDestination(remoteDestination, portId));
                protocol.setEventReporter(eventReporter);
                final long penalizationMillis = remoteDestination.getYieldPeriod(TimeUnit.MILLISECONDS);
                try {
                    logger.debug("{} Establishing site-to-site connection with {}", this, peerStatus);
                    commsSession = establishSiteToSiteConnection(peerStatus);
                } catch (final IOException ioe) {
                    peerSelector.penalize(peerStatus.getPeerDescription(), penalizationMillis);
                    throw ioe;
                }
                final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
                final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
                try {
                    logger.debug("{} Negotiating protocol", this);
                    RemoteResourceInitiator.initiateResourceNegotiation(protocol, dis, dos);
                } catch (final HandshakeException e) {
                    try {
                        commsSession.close();
                    } catch (final IOException ioe) {
                        throw e;
                    }
                }
                final String peerUrl = "nifi://" + peerDescription.getHostname() + ":" + peerDescription.getPort();
                peer = new Peer(peerDescription, commsSession, peerUrl, clusterUrl.toString());
                // set properties based on config
                if (config != null) {
                    protocol.setTimeout((int) config.getTimeout(TimeUnit.MILLISECONDS));
                    protocol.setPreferredBatchCount(config.getPreferredBatchCount());
                    protocol.setPreferredBatchSize(config.getPreferredBatchSize());
                    protocol.setPreferredBatchDuration(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
                }
                // perform handshake
                try {
                    logger.debug("{} performing handshake", this);
                    protocol.handshake(peer);
                    // handle error cases
                    if (protocol.isDestinationFull()) {
                        logger.warn("{} {} indicates that port {}'s destination is full; penalizing peer", this, peer, config.getPortName() == null ? config.getPortIdentifier() : config.getPortName());
                        peerSelector.penalize(peer, penalizationMillis);
                        try {
                            peer.close();
                        } catch (final IOException ioe) {
                        }
                        continue;
                    } else if (protocol.isPortInvalid()) {
                        peerSelector.penalize(peer, penalizationMillis);
                        cleanup(protocol, peer);
                        throw new PortNotRunningException(peer.toString() + " indicates that port " + portId + " is not running");
                    } else if (protocol.isPortUnknown()) {
                        peerSelector.penalize(peer, penalizationMillis);
                        cleanup(protocol, peer);
                        throw new UnknownPortException(peer.toString() + " indicates that port " + portId + " is not known");
                    }
                    // negotiate the FlowFileCodec to use
                    logger.debug("{} negotiating codec", this);
                    codec = protocol.negotiateCodec(peer);
                    logger.debug("{} negotiated codec is {}", this, codec);
                } catch (final PortNotRunningException | UnknownPortException e) {
                    throw e;
                } catch (final Exception e) {
                    peerSelector.penalize(peer, penalizationMillis);
                    cleanup(protocol, peer);
                    final String message = String.format("%s failed to communicate with %s due to %s", this, peer == null ? clusterUrl : peer, e.toString());
                    error(logger, eventReporter, message);
                    if (logger.isDebugEnabled()) {
                        logger.error("", e);
                    }
                    throw e;
                }
                connection = new EndpointConnection(peer, protocol, codec);
            } else {
                final long lastTimeUsed = connection.getLastTimeUsed();
                final long millisSinceLastUse = System.currentTimeMillis() - lastTimeUsed;
                if (commsTimeout > 0L && millisSinceLastUse >= commsTimeout) {
                    cleanup(connection.getSocketClientProtocol(), connection.getPeer());
                    connection = null;
                } else {
                    codec = connection.getCodec();
                    peer = connection.getPeer();
                    commsSession = peer.getCommunicationsSession();
                    protocol = connection.getSocketClientProtocol();
                }
            }
        } catch (final Throwable t) {
            if (commsSession != null) {
                try {
                    commsSession.close();
                } catch (final IOException ioe) {
                }
            }
            throw t;
        } finally {
            if (!addBack.isEmpty()) {
                connectionQueue.addAll(addBack);
                addBack.clear();
            }
        }
    } while (connection == null || codec == null || commsSession == null || protocol == null);
    activeConnections.add(connection);
    return connection;
}

Example 14 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class EndpointConnectionPool method fetchRemotePeerStatuses.

@Override
public Set<PeerStatus> fetchRemotePeerStatuses(final PeerDescription peerDescription) throws IOException {
    final String hostname = peerDescription.getHostname();
    final int port = peerDescription.getPort();
    final URI clusterUrl = siteInfoProvider.getActiveClusterUrl();
    final PeerDescription clusterPeerDescription = new PeerDescription(hostname, port, clusterUrl.toString().startsWith("https://"));
    final CommunicationsSession commsSession = establishSiteToSiteConnection(hostname, port);
    final Peer peer = new Peer(clusterPeerDescription, commsSession, "nifi://" + hostname + ":" + port, clusterUrl.toString());
    final SocketClientProtocol clientProtocol = new SocketClientProtocol();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    RemoteResourceInitiator.initiateResourceNegotiation(clientProtocol, dis, dos);
    clientProtocol.setTimeout(commsTimeout);
    if (clientProtocol.getVersionNegotiator().getVersion() < 5) {
        String portId = getPortIdentifier(TransferDirection.RECEIVE);
        if (portId == null) {
            portId = getPortIdentifier(TransferDirection.SEND);
        }
        if (portId == null) {
            peer.close();
            throw new IOException("Failed to determine the identifier of port " + remoteDestination.getName());
        }
        clientProtocol.handshake(peer, portId);
    } else {
        clientProtocol.handshake(peer, null);
    }
    final Set<PeerStatus> peerStatuses = clientProtocol.getPeerStatuses(peer);
    try {
        clientProtocol.shutdown(peer);
    } catch (final IOException e) {
        final String message = String.format("%s Failed to shutdown protocol when updating list of peers due to %s", this, e.toString());
        warn(logger, eventReporter, message);
        if (logger.isDebugEnabled()) {
            logger.warn("", e);
        }
    }
    try {
        peer.close();
    } catch (final IOException e) {
        final String message = String.format("%s Failed to close resources when updating list of peers due to %s", this, e.toString());
        warn(logger, eventReporter, message);
        if (logger.isDebugEnabled()) {
            logger.warn("", e);
        }
    }
    return peerStatuses;
}

Example 15 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class SocketClientProtocol method handshake.

public void handshake(final Peer peer, final String destinationId) throws IOException, HandshakeException {
    if (handshakeComplete) {
        throw new IllegalStateException("Handshake has already been completed");
    }
    commsIdentifier = UUID.randomUUID().toString();
    logger.debug("{} handshaking with {}", this, peer);
    final Map<HandshakeProperty, String> properties = new HashMap<>();
    properties.put(HandshakeProperty.GZIP, String.valueOf(useCompression));
    if (destinationId != null) {
        properties.put(HandshakeProperty.PORT_IDENTIFIER, destinationId);
    }
    properties.put(HandshakeProperty.REQUEST_EXPIRATION_MILLIS, String.valueOf(timeoutMillis));
    if (versionNegotiator.getVersion() >= 5) {
        if (batchCount > 0) {
            properties.put(HandshakeProperty.BATCH_COUNT, String.valueOf(batchCount));
        }
        if (batchSize > 0L) {
            properties.put(HandshakeProperty.BATCH_SIZE, String.valueOf(batchSize));
        }
        if (batchMillis > 0L) {
            properties.put(HandshakeProperty.BATCH_DURATION, String.valueOf(batchMillis));
        }
    }
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    commsSession.setTimeout(timeoutMillis);
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    dos.writeUTF(commsIdentifier);
    if (versionNegotiator.getVersion() >= 3) {
        dos.writeUTF(peer.getUrl());
        transitUriPrefix = peer.getUrl();
        if (!transitUriPrefix.endsWith("/")) {
            transitUriPrefix = transitUriPrefix + "/";
        }
    }
    logger.debug("Handshaking with properties {}", properties);
    dos.writeInt(properties.size());
    for (final Map.Entry<HandshakeProperty, String> entry : properties.entrySet()) {
        dos.writeUTF(entry.getKey().name());
        dos.writeUTF(entry.getValue());
    }
    dos.flush();
    try {
        handshakeResponse = Response.read(dis);
    } catch (final ProtocolException e) {
        throw new HandshakeException(e);
    }
    switch(handshakeResponse.getCode()) {
        case PORT_NOT_IN_VALID_STATE:
        case UNKNOWN_PORT:
        case PORTS_DESTINATION_FULL:
            break;
        case PROPERTIES_OK:
            readyForFileTransfer = true;
            break;
        default:
            logger.error("{} received unexpected response {} from {} when negotiating Codec", new Object[] { this, handshakeResponse, peer });
            peer.close();
            throw new HandshakeException("Received unexpected response " + handshakeResponse);
    }
    logger.debug("{} Finished handshake with {}", this, peer);
    handshakeComplete = true;
}