Examples with ClusterNodeInformation org.apache.nifi.remote.cluster.ClusterNodeInformation used on opensource projects

Example 1 with ClusterNodeInformation

use of org.apache.nifi.remote.cluster.ClusterNodeInformation in project nifi by apache.

the class SocketFlowFileServerProtocol method sendPeerList.

@Override
public void sendPeerList(final Peer peer, final Optional<ClusterNodeInformation> clusterNodeInfo, final NodeInformation self) throws IOException {
    if (!handshakeCompleted) {
        throw new IllegalStateException("Handshake has not been completed");
    }
    if (shutdown) {
        throw new IllegalStateException("Protocol is shutdown");
    }
    logger.debug("{} Sending Peer List to {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    logger.debug("{} Advertising Remote Input host name {}", this, peer);
    List<NodeInformation> nodeInfos;
    if (clusterNodeInfo.isPresent()) {
        nodeInfos = new ArrayList<>(clusterNodeInfo.get().getNodeInformation());
    } else {
        nodeInfos = Collections.singletonList(self);
    }
    // determine how many nodes have Site-to-site enabled
    int numPeers = 0;
    for (final NodeInformation nodeInfo : nodeInfos) {
        if (nodeInfo.getSiteToSitePort() != null) {
            numPeers++;
        }
    }
    dos.writeInt(numPeers);
    for (final NodeInformation nodeInfo : nodeInfos) {
        if (nodeInfo.getSiteToSitePort() == null) {
            continue;
        }
        dos.writeUTF(nodeInfo.getSiteToSiteHostname());
        dos.writeInt(nodeInfo.getSiteToSitePort());
        dos.writeBoolean(nodeInfo.isSiteToSiteSecure());
        dos.writeInt(nodeInfo.getTotalFlowFiles());
    }
    logger.info("Sending list of {} peers back to client {}", numPeers, peer);
    dos.flush();
}

Example 2 with ClusterNodeInformation

use of org.apache.nifi.remote.cluster.ClusterNodeInformation in project nifi by apache.

the class ClusterCoordinatorNodeInformant method getNodeInformation.

@Override
public ClusterNodeInformation getNodeInformation() {
    final List<NodeInformation> nodeInfoCollection;
    try {
        nodeInfoCollection = clusterCoordinator.getClusterWorkload().entrySet().stream().map(entry -> {
            final NodeIdentifier nodeId = entry.getKey();
            final NodeInformation nodeInfo = new NodeInformation(nodeId.getSiteToSiteAddress(), nodeId.getSiteToSitePort(), nodeId.getSiteToSiteHttpApiPort(), nodeId.getApiPort(), nodeId.isSiteToSiteSecure(), entry.getValue().getFlowFileCount());
            return nodeInfo;
        }).collect(Collectors.toList());
    } catch (IOException e) {
        throw new RuntimeException("Failed to retrieve cluster workload due to " + e, e);
    }
    final ClusterNodeInformation nodeInfo = new ClusterNodeInformation();
    nodeInfo.setNodeInformation(nodeInfoCollection);
    return nodeInfo;
}

Example 3 with ClusterNodeInformation

use of org.apache.nifi.remote.cluster.ClusterNodeInformation in project nifi by apache.

the class SocketRemoteSiteListener method handleRequest.

private void handleRequest(final ServerProtocol protocol, final Peer peer, final RequestType requestType) throws IOException, NotAuthorizedException, BadRequestException, RequestExpiredException {
    LOG.debug("Request type from {} is {}", protocol, requestType);
    switch(requestType) {
        case NEGOTIATE_FLOWFILE_CODEC:
            protocol.negotiateCodec(peer);
            break;
        case RECEIVE_FLOWFILES:
            // peer wants to receive FlowFiles, so we will transfer FlowFiles.
            protocol.getPort().transferFlowFiles(peer, protocol);
            break;
        case SEND_FLOWFILES:
            // Peer wants to send FlowFiles, so we will receive.
            protocol.getPort().receiveFlowFiles(peer, protocol);
            break;
        case REQUEST_PEER_LIST:
            final Optional<ClusterNodeInformation> nodeInfo = (nodeInformant == null) ? Optional.empty() : Optional.of(nodeInformant.getNodeInformation());
            String remoteInputHostVal = nifiProperties.getRemoteInputHost();
            if (remoteInputHostVal == null) {
                remoteInputHostVal = InetAddress.getLocalHost().getHostName();
            }
            final Boolean isSiteToSiteSecure = nifiProperties.isSiteToSiteSecure();
            final Integer apiPort = isSiteToSiteSecure ? nifiProperties.getSslPort() : nifiProperties.getPort();
            final NodeInformation self = new NodeInformation(remoteInputHostVal, nifiProperties.getRemoteInputPort(), nifiProperties.getRemoteInputHttpPort(), // Avoid potential NullPointerException.
            apiPort != null ? apiPort : 0, isSiteToSiteSecure, // TotalFlowFiles doesn't matter if it's a standalone NiFi.
            0);
            protocol.sendPeerList(peer, nodeInfo, self);
            break;
        case SHUTDOWN:
            protocol.shutdown(peer);
            break;
    }
}

Example 4 with ClusterNodeInformation

use of org.apache.nifi.remote.cluster.ClusterNodeInformation in project nifi by apache.

the class TestSocketFlowFileServerProtocol method testSendPeerListCluster.

@Test
public void testSendPeerListCluster() throws Exception {
    final SocketFlowFileServerProtocol protocol = getDefaultSocketFlowFileServerProtocol();
    final List<NodeInformation> nodeInfoList = new ArrayList<>();
    final ClusterNodeInformation clusterNodeInformation = new ClusterNodeInformation();
    clusterNodeInformation.setNodeInformation(nodeInfoList);
    final Optional<ClusterNodeInformation> clusterNodeInfo = Optional.of(clusterNodeInformation);
    for (int i = 0; i < 3; i++) {
        final String siteToSiteHostname = String.format("node%d.example.com", i);
        final Integer siteToSitePort = 8081;
        final Integer siteToSiteHttpPort = null;
        final int apiPort = 8080;
        final boolean isSiteToSiteSecure = true;
        final int numOfQueuedFlowFiles = 100 + i;
        final NodeInformation nodeInformation = new NodeInformation(siteToSiteHostname, siteToSitePort, siteToSiteHttpPort, apiPort, isSiteToSiteSecure, numOfQueuedFlowFiles);
        nodeInfoList.add(nodeInformation);
    }
    final NodeInformation self = nodeInfoList.get(0);
    final HandshakeProperties handshakeProperties = new HandshakeProperties();
    handshakeProperties.setCommsIdentifier("communication-identifier");
    handshakeProperties.setTransitUriPrefix("uri-prefix");
    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final Peer peer = getDefaultPeer(handshakeProperties, outputStream);
    protocol.handshake(peer);
    protocol.sendPeerList(peer, clusterNodeInfo, self);
    try (final DataInputStream dis = new DataInputStream(new ByteArrayInputStream(outputStream.toByteArray()))) {
        final Response handshakeResponse = Response.read(dis);
        assertEquals(ResponseCode.PROPERTIES_OK, handshakeResponse.getCode());
        final int numPeers = dis.readInt();
        assertEquals(nodeInfoList.size(), numPeers);
        for (int i = 0; i < nodeInfoList.size(); i++) {
            final NodeInformation node = nodeInfoList.get(i);
            assertEquals(node.getSiteToSiteHostname(), dis.readUTF());
            assertEquals(node.getSiteToSitePort().intValue(), dis.readInt());
            assertEquals(node.isSiteToSiteSecure(), dis.readBoolean());
            assertEquals(node.getTotalFlowFiles(), dis.readInt());
        }
    }
}

Example 5 with ClusterNodeInformation

use of org.apache.nifi.remote.cluster.ClusterNodeInformation in project nifi by apache.

the class TestSocketFlowFileServerProtocol method testSendPeerListStandalone.

@Test
public void testSendPeerListStandalone() throws Exception {
    final SocketFlowFileServerProtocol protocol = getDefaultSocketFlowFileServerProtocol();
    final Optional<ClusterNodeInformation> clusterNodeInfo = Optional.empty();
    final String siteToSiteHostname = "node1.example.com";
    final Integer siteToSitePort = 8081;
    final Integer siteToSiteHttpPort = null;
    final int apiPort = 8080;
    final boolean isSiteToSiteSecure = true;
    final int numOfQueuedFlowFiles = 100;
    final NodeInformation self = new NodeInformation(siteToSiteHostname, siteToSitePort, siteToSiteHttpPort, apiPort, isSiteToSiteSecure, numOfQueuedFlowFiles);
    final HandshakeProperties handshakeProperties = new HandshakeProperties();
    handshakeProperties.setCommsIdentifier("communication-identifier");
    handshakeProperties.setTransitUriPrefix("uri-prefix");
    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final Peer peer = getDefaultPeer(handshakeProperties, outputStream);
    protocol.handshake(peer);
    protocol.sendPeerList(peer, clusterNodeInfo, self);
    try (final DataInputStream dis = new DataInputStream(new ByteArrayInputStream(outputStream.toByteArray()))) {
        final Response handshakeResponse = Response.read(dis);
        assertEquals(ResponseCode.PROPERTIES_OK, handshakeResponse.getCode());
        final int numPeers = dis.readInt();
        assertEquals(1, numPeers);
        assertEquals(siteToSiteHostname, dis.readUTF());
        assertEquals(siteToSitePort.intValue(), dis.readInt());
        assertEquals(isSiteToSiteSecure, dis.readBoolean());
        assertEquals(numOfQueuedFlowFiles, dis.readInt());
    }
}