Examples with CommunicationsSession org.apache.nifi.remote.protocol.CommunicationsSession used on opensource projects

Example 1 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class StandardRootGroupPort method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) {
    final FlowFileRequest flowFileRequest;
    try {
        flowFileRequest = requestQueue.poll(100, TimeUnit.MILLISECONDS);
    } catch (final InterruptedException ie) {
        return;
    }
    if (flowFileRequest == null) {
        return;
    }
    flowFileRequest.setServiceBegin();
    requestLock.lock();
    try {
        if (shutdown) {
            final CommunicationsSession commsSession = flowFileRequest.getPeer().getCommunicationsSession();
            if (commsSession != null) {
                commsSession.interrupt();
            }
        }
        activeRequests.add(flowFileRequest);
    } finally {
        requestLock.unlock();
    }
    final ProcessSession session = sessionFactory.createSession();
    try {
        onTrigger(context, session, flowFileRequest);
    // we leave the session open, because we send it back to the caller of #receiveFlowFile or #transmitFlowFile,
    // so that they can perform appropriate actions to commit or rollback the transaction.
    } catch (final TransmissionDisabledException e) {
        session.rollback();
    } catch (final Exception e) {
        logger.error("{} Failed to process data due to {}", new Object[] { this, e });
        if (logger.isDebugEnabled()) {
            logger.error("", e);
        }
        session.rollback();
    } finally {
        requestLock.lock();
        try {
            activeRequests.remove(flowFileRequest);
        } finally {
            requestLock.unlock();
        }
    }
}

Example 2 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class SocketFlowFileServerProtocol method sendPeerList.

@Override
public void sendPeerList(final Peer peer, final Optional<ClusterNodeInformation> clusterNodeInfo, final NodeInformation self) throws IOException {
    if (!handshakeCompleted) {
        throw new IllegalStateException("Handshake has not been completed");
    }
    if (shutdown) {
        throw new IllegalStateException("Protocol is shutdown");
    }
    logger.debug("{} Sending Peer List to {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    logger.debug("{} Advertising Remote Input host name {}", this, peer);
    List<NodeInformation> nodeInfos;
    if (clusterNodeInfo.isPresent()) {
        nodeInfos = new ArrayList<>(clusterNodeInfo.get().getNodeInformation());
    } else {
        nodeInfos = Collections.singletonList(self);
    }
    // determine how many nodes have Site-to-site enabled
    int numPeers = 0;
    for (final NodeInformation nodeInfo : nodeInfos) {
        if (nodeInfo.getSiteToSitePort() != null) {
            numPeers++;
        }
    }
    dos.writeInt(numPeers);
    for (final NodeInformation nodeInfo : nodeInfos) {
        if (nodeInfo.getSiteToSitePort() == null) {
            continue;
        }
        dos.writeUTF(nodeInfo.getSiteToSiteHostname());
        dos.writeInt(nodeInfo.getSiteToSitePort());
        dos.writeBoolean(nodeInfo.isSiteToSiteSecure());
        dos.writeInt(nodeInfo.getTotalFlowFiles());
    }
    logger.info("Sending list of {} peers back to client {}", numPeers, peer);
    dos.flush();
}

Example 3 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class TestStandardRemoteGroupPort method testSendRaw.

@Test
public void testSendRaw() throws Exception {
    setupMock(SiteToSiteTransportProtocol.RAW, TransferDirection.SEND);
    setupMockProcessSession();
    final String peerUrl = "nifi://node1.example.com:9090";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 9090, true);
    try (final SocketChannel socketChannel = SocketChannel.open()) {
        final CommunicationsSession commsSession = new SocketChannelCommunicationsSession(socketChannel);
        commsSession.setUserDn("nifi.node1.example.com");
        final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
        doReturn(peer).when(transaction).getCommunicant();
        final MockFlowFile flowFile = processSession.createFlowFile("0123456789".getBytes());
        sessionState.getFlowFileQueue().offer(flowFile);
        port.onTrigger(processContext, processSession);
        // Assert provenance.
        final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
        assertEquals(1, provenanceEvents.size());
        final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
        assertEquals(ProvenanceEventType.SEND, provenanceEvent.getEventType());
        assertEquals(peerUrl + "/" + flowFile.getAttribute(CoreAttributes.UUID.key()), provenanceEvent.getTransitUri());
        assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
    }
}

Example 4 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class TestStandardRemoteGroupPort method testReceiveRaw.

@Test
public void testReceiveRaw() throws Exception {
    setupMock(SiteToSiteTransportProtocol.RAW, TransferDirection.RECEIVE);
    setupMockProcessSession();
    final String peerUrl = "nifi://node1.example.com:9090";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 9090, true);
    try (final SocketChannel socketChannel = SocketChannel.open()) {
        final CommunicationsSession commsSession = new SocketChannelCommunicationsSession(socketChannel);
        commsSession.setUserDn("nifi.node1.example.com");
        final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
        doReturn(peer).when(transaction).getCommunicant();
        final String sourceFlowFileUuid = "flowfile-uuid";
        final Map<String, String> attributes = new HashMap<>();
        attributes.put(CoreAttributes.UUID.key(), sourceFlowFileUuid);
        final byte[] dataPacketContents = "DataPacket Contents".getBytes();
        final ByteArrayInputStream dataPacketInputStream = new ByteArrayInputStream(dataPacketContents);
        final DataPacket dataPacket = new StandardDataPacket(attributes, dataPacketInputStream, dataPacketContents.length);
        // Return null when it gets called second time.
        doReturn(dataPacket).doReturn(null).when(this.transaction).receive();
        port.onTrigger(processContext, processSession);
        // Assert provenance.
        final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
        assertEquals(1, provenanceEvents.size());
        final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
        assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
        assertEquals(peerUrl + "/" + sourceFlowFileUuid, provenanceEvent.getTransitUri());
        assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
        // Assert received flow files.
        processSession.assertAllFlowFilesTransferred(Relationship.ANONYMOUS);
        final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(Relationship.ANONYMOUS);
        assertEquals(1, flowFiles.size());
        final MockFlowFile flowFile = flowFiles.get(0);
        flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_HOST.key(), peer.getHost());
        flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_ADDRESS.key(), peer.getHost() + ":" + peer.getPort());
    }
}

Example 5 with CommunicationsSession

use of org.apache.nifi.remote.protocol.CommunicationsSession in project nifi by apache.

the class HttpClient method createTransaction.

@Override
public Transaction createTransaction(final TransferDirection direction) throws HandshakeException, PortNotRunningException, ProtocolException, UnknownPortException, IOException {
    final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
    PeerStatus peerStatus;
    while ((peerStatus = peerSelector.getNextPeerStatus(direction)) != null) {
        logger.debug("peerStatus={}", peerStatus);
        final CommunicationsSession commSession = new HttpCommunicationsSession();
        final String nodeApiUrl = resolveNodeApiUrl(peerStatus.getPeerDescription());
        final StringBuilder clusterUrls = new StringBuilder();
        config.getUrls().forEach(url -> {
            if (clusterUrls.length() > 0) {
                clusterUrls.append(",");
                clusterUrls.append(url);
            }
        });
        final Peer peer = new Peer(peerStatus.getPeerDescription(), commSession, nodeApiUrl, clusterUrls.toString());
        final int penaltyMillis = (int) config.getPenalizationPeriod(TimeUnit.MILLISECONDS);
        String portId = config.getPortIdentifier();
        if (StringUtils.isEmpty(portId)) {
            portId = siteInfoProvider.getPortIdentifier(config.getPortName(), direction);
            if (StringUtils.isEmpty(portId)) {
                peer.close();
                throw new IOException("Failed to determine the identifier of port " + config.getPortName());
            }
        }
        final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter());
        apiClient.setBaseUrl(peer.getUrl());
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        apiClient.setCompress(config.isUseCompression());
        apiClient.setRequestExpirationMillis(config.getIdleConnectionExpiration(TimeUnit.MILLISECONDS));
        apiClient.setBatchCount(config.getPreferredBatchCount());
        apiClient.setBatchSize(config.getPreferredBatchSize());
        apiClient.setBatchDurationMillis(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
        final String transactionUrl;
        try {
            transactionUrl = apiClient.initiateTransaction(direction, portId);
            commSession.setUserDn(apiClient.getTrustedPeerDn());
        } catch (final Exception e) {
            apiClient.close();
            logger.warn("Penalizing a peer {} due to {}", peer, e.toString());
            peerSelector.penalize(peer, penaltyMillis);
            // Following exceptions will be thrown even if we tried other peers, so throw it.
            if (e instanceof UnknownPortException || e instanceof PortNotRunningException || e instanceof HandshakeException) {
                throw e;
            }
            logger.debug("Continue trying other peers...");
            continue;
        }
        // We found a valid peer to communicate with.
        final Integer transactionProtocolVersion = apiClient.getTransactionProtocolVersion();
        final HttpClientTransaction transaction = new HttpClientTransaction(transactionProtocolVersion, peer, direction, config.isUseCompression(), portId, penaltyMillis, config.getEventReporter()) {

            @Override
            protected void close() throws IOException {
                try {
                    super.close();
                } finally {
                    activeTransactions.remove(this);
                }
            }
        };
        try {
            transaction.initialize(apiClient, transactionUrl);
        } catch (final Exception e) {
            transaction.error();
            throw e;
        }
        activeTransactions.add(transaction);
        return transaction;
    }
    logger.info("Couldn't find a valid peer to communicate with.");
    return null;
}