Examples with PeerDescription org.apache.nifi.remote.PeerDescription used on opensource projects

Example 1 with PeerDescription

use of org.apache.nifi.remote.PeerDescription in project nifi by apache.

the class DataTransferResource method constructPeer.

private Peer constructPeer(final HttpServletRequest req, final InputStream inputStream, final OutputStream outputStream, final String portId, final String transactionId) {
    String clientHostName = req.getRemoteHost();
    try {
        // req.getRemoteHost returns IP address, try to resolve hostname to be consistent with RAW protocol.
        final InetAddress clientAddress = InetAddress.getByName(clientHostName);
        clientHostName = clientAddress.getHostName();
    } catch (UnknownHostException e) {
        logger.info("Failed to resolve client hostname {}, due to {}", clientHostName, e.getMessage());
    }
    final int clientPort = req.getRemotePort();
    final PeerDescription peerDescription = new PeerDescription(clientHostName, clientPort, req.isSecure());
    final NiFiUser user = NiFiUserUtils.getNiFiUser();
    final String userDn = user == null ? null : user.getIdentity();
    final HttpServerCommunicationsSession commSession = new HttpServerCommunicationsSession(inputStream, outputStream, transactionId, userDn);
    boolean useCompression = false;
    final String useCompressionStr = req.getHeader(HANDSHAKE_PROPERTY_USE_COMPRESSION);
    if (!isEmpty(useCompressionStr) && Boolean.valueOf(useCompressionStr)) {
        useCompression = true;
    }
    final String requestExpiration = req.getHeader(HANDSHAKE_PROPERTY_REQUEST_EXPIRATION);
    final String batchCount = req.getHeader(HANDSHAKE_PROPERTY_BATCH_COUNT);
    final String batchSize = req.getHeader(HANDSHAKE_PROPERTY_BATCH_SIZE);
    final String batchDuration = req.getHeader(HANDSHAKE_PROPERTY_BATCH_DURATION);
    commSession.putHandshakeParam(HandshakeProperty.PORT_IDENTIFIER, portId);
    commSession.putHandshakeParam(HandshakeProperty.GZIP, String.valueOf(useCompression));
    if (!isEmpty(requestExpiration)) {
        commSession.putHandshakeParam(REQUEST_EXPIRATION_MILLIS, requestExpiration);
    }
    if (!isEmpty(batchCount)) {
        commSession.putHandshakeParam(BATCH_COUNT, batchCount);
    }
    if (!isEmpty(batchSize)) {
        commSession.putHandshakeParam(BATCH_SIZE, batchSize);
    }
    if (!isEmpty(batchDuration)) {
        commSession.putHandshakeParam(BATCH_DURATION, batchDuration);
    }
    if (peerDescription.isSecure()) {
        final NiFiUser nifiUser = NiFiUserUtils.getNiFiUser();
        logger.debug("initiating peer, nifiUser={}", nifiUser);
        commSession.setUserDn(nifiUser.getIdentity());
    }
    // TODO: Followed how SocketRemoteSiteListener define peerUrl and clusterUrl, but it can be more meaningful values, especially for clusterUrl.
    final String peerUrl = "nifi://" + clientHostName + ":" + clientPort;
    final String clusterUrl = "nifi://localhost:" + req.getLocalPort();
    return new Peer(peerDescription, commSession, peerUrl, clusterUrl);
}

Example 2 with PeerDescription

use of org.apache.nifi.remote.PeerDescription in project nifi by apache.

the class HttpClient method fetchRemotePeerStatuses.

@Override
public Set<PeerStatus> fetchRemotePeerStatuses(PeerDescription peerDescription) throws IOException {
    // Each node should has the same URL structure and network reach-ability with the proxy configuration.
    try (final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter())) {
        final String scheme = peerDescription.isSecure() ? "https" : "http";
        apiClient.setBaseUrl(scheme, peerDescription.getHostname(), peerDescription.getPort());
        final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        final Collection<PeerDTO> peers = apiClient.getPeers();
        if (peers == null || peers.size() == 0) {
            throw new IOException("Couldn't get any peer to communicate with. " + apiClient.getBaseUrl() + " returned zero peers.");
        }
        // was added in NiFi 1.0.0, which means that peer-to-peer queries are always allowed.
        return peers.stream().map(p -> new PeerStatus(new PeerDescription(p.getHostname(), p.getPort(), p.isSecure()), p.getFlowFileCount(), true)).collect(Collectors.toSet());
    }
}

Example 3 with PeerDescription

use of org.apache.nifi.remote.PeerDescription in project nifi by apache.

the class SocketClientProtocol method getPeerStatuses.

@Override
public Set<PeerStatus> getPeerStatuses(final Peer peer) throws IOException {
    if (!handshakeComplete) {
        throw new IllegalStateException("Handshake has not been performed");
    }
    logger.debug("{} Get Peer Statuses from {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    final boolean queryPeersForOtherPeers = getVersionNegotiator().getVersion() >= 6;
    RequestType.REQUEST_PEER_LIST.writeRequestType(dos);
    dos.flush();
    final int numPeers = dis.readInt();
    final Set<PeerStatus> peers = new HashSet<>(numPeers);
    for (int i = 0; i < numPeers; i++) {
        final String hostname = dis.readUTF();
        final int port = dis.readInt();
        final boolean secure = dis.readBoolean();
        final int flowFileCount = dis.readInt();
        peers.add(new PeerStatus(new PeerDescription(hostname, port, secure), flowFileCount, queryPeersForOtherPeers));
    }
    logger.debug("{} Received {} Peer Statuses from {}", this, peers.size(), peer);
    return peers;
}

Example 4 with PeerDescription

use of org.apache.nifi.remote.PeerDescription in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForOutput.

@Test
public void testFormulateDestinationListForOutput() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 1111, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("HasLots", 2222, true), 10240, true));
    collection.add(new PeerStatus(new PeerDescription("HasLittle", 3333, true), 1024, true));
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 4444, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 5555, true), 4096, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    assertTrue("HasLots should send lots", selectedCounts.get("HasLots") > selectedCounts.get("HasMedium"));
    assertTrue("HasMedium should send medium", selectedCounts.get("HasMedium") > selectedCounts.get("HasLittle"));
}

Example 5 with PeerDescription

use of org.apache.nifi.remote.PeerDescription in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForOutputEven.

@Test
public void testFormulateDestinationListForOutputEven() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("Node1", 1111, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node2", 2222, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node3", 3333, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node4", 4444, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node5", 5555, true), 4096, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    int consecutiveSamePeerCount = 0;
    PeerStatus previousPeer = null;
    for (PeerStatus peer : destinations) {
        if (previousPeer != null && peer.getPeerDescription().equals(previousPeer.getPeerDescription())) {
            consecutiveSamePeerCount++;
            // The same peer shouldn't be used consecutively (number of nodes - 1) times or more.
            if (consecutiveSamePeerCount >= (collection.size() - 1)) {
                fail("The same peer is returned consecutively too frequently.");
            }
        } else {
            consecutiveSamePeerCount = 0;
        }
        previousPeer = peer;
    }
}