Examples with CommunicationsSession org.apache.nifi.remote.protocol.CommunicationsSession used on opensource projects

Example 6 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class SocketClientProtocol method getPeerStatuses.

@Override
public Set<PeerStatus> getPeerStatuses(final Peer peer) throws IOException {
    if (!handshakeComplete) {
        throw new IllegalStateException("Handshake has not been performed");
    }
    logger.debug("{} Get Peer Statuses from {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    final boolean queryPeersForOtherPeers = getVersionNegotiator().getVersion() >= 6;
    RequestType.REQUEST_PEER_LIST.writeRequestType(dos);
    dos.flush();
    final int numPeers = dis.readInt();
    final Set<PeerStatus> peers = new HashSet<>(numPeers);
    for (int i = 0; i < numPeers; i++) {
        final String hostname = dis.readUTF();
        final int port = dis.readInt();
        final boolean secure = dis.readBoolean();
        final int flowFileCount = dis.readInt();
        peers.add(new PeerStatus(new PeerDescription(hostname, port, secure), flowFileCount, queryPeersForOtherPeers));
    }
    logger.debug("{} Received {} Peer Statuses from {}", this, peers.size(), peer);
    return peers;
}

Example 7 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class SocketRemoteSiteListener method start.

@Override
public void start() throws IOException {
    final boolean secure = (sslContext != null);
    final List<Thread> threads = new ArrayList<Thread>();
    final ServerSocketChannel serverSocketChannel = ServerSocketChannel.open();
    serverSocketChannel.configureBlocking(true);
    serverSocketChannel.bind(new InetSocketAddress(socketPort));
    stopped.set(false);
    final Thread listenerThread = new Thread(new Runnable() {

        private int threadCount = 0;

        @Override
        public void run() {
            while (!stopped.get()) {
                final ProcessGroup processGroup = rootGroup.get();
                // the processGroup.
                if ((nodeInformant == null) && (processGroup == null || (processGroup.getInputPorts().isEmpty() && processGroup.getOutputPorts().isEmpty()))) {
                    try {
                        Thread.sleep(2000L);
                    } catch (final Exception e) {
                    }
                    continue;
                }
                LOG.trace("Accepting Connection...");
                Socket acceptedSocket = null;
                try {
                    serverSocketChannel.configureBlocking(false);
                    final ServerSocket serverSocket = serverSocketChannel.socket();
                    serverSocket.setSoTimeout(2000);
                    while (!stopped.get() && acceptedSocket == null) {
                        try {
                            acceptedSocket = serverSocket.accept();
                        } catch (final SocketTimeoutException ste) {
                            continue;
                        }
                    }
                } catch (final IOException e) {
                    LOG.error("RemoteSiteListener Unable to accept connection due to {}", e.toString());
                    if (LOG.isDebugEnabled()) {
                        LOG.error("", e);
                    }
                    continue;
                }
                LOG.trace("Got connection");
                if (stopped.get()) {
                    break;
                }
                final Socket socket = acceptedSocket;
                final SocketChannel socketChannel = socket.getChannel();
                final Thread thread = new Thread(new Runnable() {

                    @Override
                    public void run() {
                        LOG.debug("{} Determining URL of connection", this);
                        final InetAddress inetAddress = socket.getInetAddress();
                        String clientHostName = inetAddress.getHostName();
                        final int slashIndex = clientHostName.indexOf("/");
                        if (slashIndex == 0) {
                            clientHostName = clientHostName.substring(1);
                        } else if (slashIndex > 0) {
                            clientHostName = clientHostName.substring(0, slashIndex);
                        }
                        final int clientPort = socket.getPort();
                        final String peerUri = "nifi://" + clientHostName + ":" + clientPort;
                        LOG.debug("{} Connection URL is {}", this, peerUri);
                        final CommunicationsSession commsSession;
                        final String dn;
                        try {
                            if (secure) {
                                final SSLSocketChannel sslSocketChannel = new SSLSocketChannel(sslContext, socketChannel, false);
                                LOG.trace("Channel is secure; connecting...");
                                sslSocketChannel.connect();
                                LOG.trace("Channel connected");
                                commsSession = new SSLSocketChannelCommunicationsSession(sslSocketChannel);
                                dn = sslSocketChannel.getDn();
                                commsSession.setUserDn(dn);
                            } else {
                                LOG.trace("{} Channel is not secure", this);
                                commsSession = new SocketChannelCommunicationsSession(socketChannel);
                                dn = null;
                            }
                        } catch (final Exception e) {
                            LOG.error("RemoteSiteListener Unable to accept connection from {} due to {}", socket, e.toString());
                            if (LOG.isDebugEnabled()) {
                                LOG.error("", e);
                            }
                            try {
                                socketChannel.close();
                            } catch (IOException swallow) {
                            }
                            return;
                        }
                        LOG.info("Received connection from {}, User DN: {}", socket.getInetAddress(), dn);
                        final InputStream socketIn;
                        final OutputStream socketOut;
                        try {
                            socketIn = commsSession.getInput().getInputStream();
                            socketOut = commsSession.getOutput().getOutputStream();
                        } catch (final IOException e) {
                            LOG.error("Connection dropped from {} before any data was transmitted", peerUri);
                            try {
                                commsSession.close();
                            } catch (final IOException ioe) {
                            }
                            return;
                        }
                        final DataInputStream dis = new DataInputStream(socketIn);
                        final DataOutputStream dos = new DataOutputStream(socketOut);
                        ServerProtocol protocol = null;
                        Peer peer = null;
                        try {
                            // ensure that we are communicating with another NiFi
                            LOG.debug("Verifying magic bytes...");
                            verifyMagicBytes(dis, peerUri);
                            LOG.debug("Receiving Server Protocol Negotiation");
                            protocol = RemoteResourceFactory.receiveServerProtocolNegotiation(dis, dos);
                            protocol.setRootProcessGroup(rootGroup.get());
                            protocol.setNodeInformant(nodeInformant);
                            final PeerDescription description = new PeerDescription(clientHostName, clientPort, sslContext != null);
                            peer = new Peer(description, commsSession, peerUri, "nifi://localhost:" + getPort());
                            LOG.debug("Handshaking....");
                            protocol.handshake(peer);
                            if (!protocol.isHandshakeSuccessful()) {
                                LOG.error("Handshake failed with {}; closing connection", peer);
                                try {
                                    peer.close();
                                } catch (final IOException e) {
                                    LOG.warn("Failed to close {} due to {}", peer, e);
                                }
                                // no need to shutdown protocol because we failed to perform handshake
                                return;
                            }
                            commsSession.setTimeout((int) protocol.getRequestExpiration());
                            LOG.info("Successfully negotiated ServerProtocol {} Version {} with {}", new Object[] { protocol.getResourceName(), protocol.getVersionNegotiator().getVersion(), peer });
                            try {
                                while (!protocol.isShutdown()) {
                                    LOG.trace("Getting Protocol Request Type...");
                                    int timeoutCount = 0;
                                    RequestType requestType = null;
                                    while (requestType == null) {
                                        try {
                                            requestType = protocol.getRequestType(peer);
                                        } catch (final SocketTimeoutException e) {
                                            // Give the timeout a bit longer (twice as long) to receive the Request Type,
                                            // in order to attempt to receive more data without shutting down the socket if we don't
                                            // have to.
                                            LOG.debug("{} Timed out waiting to receive RequestType using {} with {}", new Object[] { this, protocol, peer });
                                            timeoutCount++;
                                            requestType = null;
                                            if (timeoutCount >= 2) {
                                                throw e;
                                            }
                                        }
                                    }
                                    handleRequest(protocol, peer, requestType);
                                }
                                LOG.debug("Finished communicating with {} ({})", peer, protocol);
                            } catch (final Exception e) {
                                LOG.error("Unable to communicate with remote instance {} ({}) due to {}; closing connection", peer, protocol, e.toString());
                                if (LOG.isDebugEnabled()) {
                                    LOG.error("", e);
                                }
                            }
                        } catch (final IOException e) {
                            LOG.error("Unable to communicate with remote instance {} due to {}; closing connection", peer, e.toString());
                            if (LOG.isDebugEnabled()) {
                                LOG.error("", e);
                            }
                        } catch (final Throwable t) {
                            LOG.error("Handshake failed when communicating with {}; closing connection. Reason for failure: {}", peerUri, t.toString());
                            if (LOG.isDebugEnabled()) {
                                LOG.error("", t);
                            }
                        } finally {
                            LOG.trace("Cleaning up");
                            try {
                                if (protocol != null && peer != null) {
                                    protocol.shutdown(peer);
                                }
                            } catch (final Exception protocolException) {
                                LOG.warn("Failed to shutdown protocol due to {}", protocolException.toString());
                            }
                            try {
                                if (peer != null) {
                                    peer.close();
                                }
                            } catch (final Exception peerException) {
                                LOG.warn("Failed to close peer due to {}; some resources may not be appropriately cleaned up", peerException.toString());
                            }
                            LOG.trace("Finished cleaning up");
                        }
                    }
                });
                thread.setName("Site-to-Site Worker Thread-" + (threadCount++));
                LOG.debug("Handing connection to {}", thread);
                thread.start();
                threads.add(thread);
                threads.removeIf(t -> !t.isAlive());
            }
            for (Thread thread : threads) {
                if (thread != null) {
                    thread.interrupt();
                }
            }
        }
    });
    listenerThread.setName("Site-to-Site Listener");
    listenerThread.start();
}

Example 8 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class StandardRootGroupPort method onTrigger.

private void onTrigger(final ProcessContext context, final ProcessSession session, final FlowFileRequest flowFileRequest) {
    final ServerProtocol protocol = flowFileRequest.getProtocol();
    final BlockingQueue<ProcessingResult> responseQueue = flowFileRequest.getResponseQueue();
    if (flowFileRequest.isExpired()) {
        final String message = String.format("%s Cannot service request from %s because the request has timed out", this, flowFileRequest.getPeer());
        logger.warn(message);
        eventReporter.reportEvent(Severity.WARNING, CATEGORY, message);
        responseQueue.add(new ProcessingResult(new RequestExpiredException()));
        return;
    }
    final Peer peer = flowFileRequest.getPeer();
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final String sourceDn = commsSession.getUserDn();
    logger.debug("{} Servicing request for {} (DN={})", this, peer, sourceDn);
    final PortAuthorizationResult authorizationResult = checkUserAuthorization(sourceDn);
    if (!authorizationResult.isAuthorized()) {
        final String message = String.format("%s Cannot service request from %s (DN=%s) because peer is not authorized to communicate with this port: %s", this, flowFileRequest.getPeer(), flowFileRequest.getPeer().getCommunicationsSession().getUserDn(), authorizationResult.getExplanation());
        logger.error(message);
        eventReporter.reportEvent(Severity.ERROR, CATEGORY, message);
        responseQueue.add(new ProcessingResult(new NotAuthorizedException(authorizationResult.getExplanation())));
        return;
    }
    final FlowFileCodec codec = protocol.getPreNegotiatedCodec();
    if (codec == null) {
        responseQueue.add(new ProcessingResult(new BadRequestException("None of the supported FlowFile Codecs supplied is compatible with this instance")));
        return;
    }
    final int transferCount;
    try {
        if (getConnectableType() == ConnectableType.INPUT_PORT) {
            transferCount = receiveFlowFiles(context, session, codec, flowFileRequest);
        } else {
            transferCount = transferFlowFiles(context, session, codec, flowFileRequest);
        }
    } catch (final IOException e) {
        session.rollback();
        responseQueue.add(new ProcessingResult(e));
        return;
    } catch (final Exception e) {
        session.rollback();
        responseQueue.add(new ProcessingResult(e));
        return;
    }
    // TODO: Comfirm this. Session.commit here is not required since it has been committed inside receiveFlowFiles/transferFlowFiles.
    // session.commit();
    responseQueue.add(new ProcessingResult(transferCount));
}

Example 9 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class StandardRootGroupPort method shutdown.

@Override
public void shutdown() {
    super.shutdown();
    requestLock.lock();
    try {
        this.shutdown = true;
        for (final FlowFileRequest request : activeRequests) {
            final CommunicationsSession commsSession = request.getPeer().getCommunicationsSession();
            if (commsSession != null) {
                commsSession.interrupt();
            }
        }
    } finally {
        requestLock.unlock();
    }
}

Example 10 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class SocketFlowFileServerProtocol method doHandshake.

@Override
protected HandshakeProperties doHandshake(Peer peer) throws IOException, HandshakeException {
    HandshakeProperties confirmed = new HandshakeProperties();
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    confirmed.setCommsIdentifier(dis.readUTF());
    if (versionNegotiator.getVersion() >= 3) {
        String transitUriPrefix = dis.readUTF();
        if (!transitUriPrefix.endsWith("/")) {
            transitUriPrefix = transitUriPrefix + "/";
        }
        confirmed.setTransitUriPrefix(transitUriPrefix);
    }
    final Map<String, String> properties = new HashMap<>();
    final int numProperties = dis.readInt();
    for (int i = 0; i < numProperties; i++) {
        final String propertyName = dis.readUTF();
        final String propertyValue = dis.readUTF();
        properties.put(propertyName, propertyValue);
    }
    // evaluate the properties received
    boolean responseWritten = false;
    try {
        validateHandshakeRequest(confirmed, peer, properties);
    } catch (HandshakeException e) {
        ResponseCode handshakeResult = e.getResponseCode();
        if (handshakeResult.containsMessage()) {
            handshakeResult.writeResponse(dos, e.getMessage());
        } else {
            handshakeResult.writeResponse(dos);
        }
        switch(handshakeResult) {
            case UNAUTHORIZED:
            case PORT_NOT_IN_VALID_STATE:
            case PORTS_DESTINATION_FULL:
                responseWritten = true;
                break;
            default:
                throw e;
        }
    }
    // send "OK" response
    if (!responseWritten) {
        ResponseCode.PROPERTIES_OK.writeResponse(dos);
    }
    return confirmed;
}