Examples with PeerStatus org.apache.nifi.remote.PeerStatus used on opensource projects

Example 1 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class HttpClient method fetchRemotePeerStatuses.

@Override
public Set<PeerStatus> fetchRemotePeerStatuses(PeerDescription peerDescription) throws IOException {
    // Each node should has the same URL structure and network reach-ability with the proxy configuration.
    try (final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter())) {
        final String scheme = peerDescription.isSecure() ? "https" : "http";
        apiClient.setBaseUrl(scheme, peerDescription.getHostname(), peerDescription.getPort());
        final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        final Collection<PeerDTO> peers = apiClient.getPeers();
        if (peers == null || peers.size() == 0) {
            throw new IOException("Couldn't get any peer to communicate with. " + apiClient.getBaseUrl() + " returned zero peers.");
        }
        // was added in NiFi 1.0.0, which means that peer-to-peer queries are always allowed.
        return peers.stream().map(p -> new PeerStatus(new PeerDescription(p.getHostname(), p.getPort(), p.isSecure()), p.getFlowFileCount(), true)).collect(Collectors.toSet());
    }
}

Example 2 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class HttpClient method createTransaction.

@Override
public Transaction createTransaction(final TransferDirection direction) throws HandshakeException, PortNotRunningException, ProtocolException, UnknownPortException, IOException {
    final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
    PeerStatus peerStatus;
    while ((peerStatus = peerSelector.getNextPeerStatus(direction)) != null) {
        logger.debug("peerStatus={}", peerStatus);
        final CommunicationsSession commSession = new HttpCommunicationsSession();
        final String nodeApiUrl = resolveNodeApiUrl(peerStatus.getPeerDescription());
        final StringBuilder clusterUrls = new StringBuilder();
        config.getUrls().forEach(url -> {
            if (clusterUrls.length() > 0) {
                clusterUrls.append(",");
                clusterUrls.append(url);
            }
        });
        final Peer peer = new Peer(peerStatus.getPeerDescription(), commSession, nodeApiUrl, clusterUrls.toString());
        final int penaltyMillis = (int) config.getPenalizationPeriod(TimeUnit.MILLISECONDS);
        String portId = config.getPortIdentifier();
        if (StringUtils.isEmpty(portId)) {
            portId = siteInfoProvider.getPortIdentifier(config.getPortName(), direction);
            if (StringUtils.isEmpty(portId)) {
                peer.close();
                throw new IOException("Failed to determine the identifier of port " + config.getPortName());
            }
        }
        final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter());
        apiClient.setBaseUrl(peer.getUrl());
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        apiClient.setCompress(config.isUseCompression());
        apiClient.setRequestExpirationMillis(config.getIdleConnectionExpiration(TimeUnit.MILLISECONDS));
        apiClient.setBatchCount(config.getPreferredBatchCount());
        apiClient.setBatchSize(config.getPreferredBatchSize());
        apiClient.setBatchDurationMillis(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
        final String transactionUrl;
        try {
            transactionUrl = apiClient.initiateTransaction(direction, portId);
            commSession.setUserDn(apiClient.getTrustedPeerDn());
        } catch (final Exception e) {
            apiClient.close();
            logger.warn("Penalizing a peer {} due to {}", peer, e.toString());
            peerSelector.penalize(peer, penaltyMillis);
            // Following exceptions will be thrown even if we tried other peers, so throw it.
            if (e instanceof UnknownPortException || e instanceof PortNotRunningException || e instanceof HandshakeException) {
                throw e;
            }
            logger.debug("Continue trying other peers...");
            continue;
        }
        // We found a valid peer to communicate with.
        final Integer transactionProtocolVersion = apiClient.getTransactionProtocolVersion();
        final HttpClientTransaction transaction = new HttpClientTransaction(transactionProtocolVersion, peer, direction, config.isUseCompression(), portId, penaltyMillis, config.getEventReporter()) {

            @Override
            protected void close() throws IOException {
                try {
                    super.close();
                } finally {
                    activeTransactions.remove(this);
                }
            }
        };
        try {
            transaction.initialize(apiClient, transactionUrl);
        } catch (final Exception e) {
            transaction.error();
            throw e;
        }
        activeTransactions.add(transaction);
        return transaction;
    }
    logger.info("Couldn't find a valid peer to communicate with.");
    return null;
}

Example 3 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class SocketClientProtocol method getPeerStatuses.

@Override
public Set<PeerStatus> getPeerStatuses(final Peer peer) throws IOException {
    if (!handshakeComplete) {
        throw new IllegalStateException("Handshake has not been performed");
    }
    logger.debug("{} Get Peer Statuses from {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    final DataOutputStream dos = new DataOutputStream(commsSession.getOutput().getOutputStream());
    final boolean queryPeersForOtherPeers = getVersionNegotiator().getVersion() >= 6;
    RequestType.REQUEST_PEER_LIST.writeRequestType(dos);
    dos.flush();
    final int numPeers = dis.readInt();
    final Set<PeerStatus> peers = new HashSet<>(numPeers);
    for (int i = 0; i < numPeers; i++) {
        final String hostname = dis.readUTF();
        final int port = dis.readInt();
        final boolean secure = dis.readBoolean();
        final int flowFileCount = dis.readInt();
        peers.add(new PeerStatus(new PeerDescription(hostname, port, secure), flowFileCount, queryPeersForOtherPeers));
    }
    logger.debug("{} Received {} Peer Statuses from {}", this, peers.size(), peer);
    return peers;
}

Example 4 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForOutput.

@Test
public void testFormulateDestinationListForOutput() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 1111, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("HasLots", 2222, true), 10240, true));
    collection.add(new PeerStatus(new PeerDescription("HasLittle", 3333, true), 1024, true));
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 4444, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 5555, true), 4096, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    assertTrue("HasLots should send lots", selectedCounts.get("HasLots") > selectedCounts.get("HasMedium"));
    assertTrue("HasMedium should send medium", selectedCounts.get("HasMedium") > selectedCounts.get("HasLittle"));
}

Example 5 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForOutputEven.

@Test
public void testFormulateDestinationListForOutputEven() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("Node1", 1111, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node2", 2222, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node3", 3333, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node4", 4444, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("Node5", 5555, true), 4096, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    int consecutiveSamePeerCount = 0;
    PeerStatus previousPeer = null;
    for (PeerStatus peer : destinations) {
        if (previousPeer != null && peer.getPeerDescription().equals(previousPeer.getPeerDescription())) {
            consecutiveSamePeerCount++;
            // The same peer shouldn't be used consecutively (number of nodes - 1) times or more.
            if (consecutiveSamePeerCount >= (collection.size() - 1)) {
                fail("The same peer is returned consecutively too frequently.");
            }
        } else {
            consecutiveSamePeerCount = 0;
        }
        previousPeer = peer;
    }
}