Examples with PeerStatus org.apache.nifi.remote.PeerStatus used on opensource projects

Example 6 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForOutputHugeDifference.

@Test
public void testFormulateDestinationListForOutputHugeDifference() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("HasLittle", 1111, true), 500, true));
    collection.add(new PeerStatus(new PeerDescription("HasLots", 2222, true), 50000, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    assertTrue("HasLots should send lots", selectedCounts.get("HasLots") > selectedCounts.get("HasLittle"));
}

Example 7 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForInputPortsHugeDifference.

@Test
public void testFormulateDestinationListForInputPortsHugeDifference() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("HasLots", 1111, true), 500, true));
    collection.add(new PeerStatus(new PeerDescription("HasLittle", 2222, true), 50000, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    assertTrue("HasLots should get little", selectedCounts.get("HasLots") < selectedCounts.get("HasLittle"));
}

Example 8 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class TestPeerSelector method testFormulateDestinationListForInputPorts.

@Test
public void testFormulateDestinationListForInputPorts() throws IOException {
    final Set<PeerStatus> collection = new HashSet<>();
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 1111, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("HasLittle", 2222, true), 10240, true));
    collection.add(new PeerStatus(new PeerDescription("HasLots", 3333, true), 1024, true));
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 4444, true), 4096, true));
    collection.add(new PeerStatus(new PeerDescription("HasMedium", 5555, true), 4096, true));
    PeerStatusProvider peerStatusProvider = Mockito.mock(PeerStatusProvider.class);
    PeerSelector peerSelector = new PeerSelector(peerStatusProvider, null);
    final List<PeerStatus> destinations = peerSelector.formulateDestinationList(collection, TransferDirection.RECEIVE);
    final Map<String, Integer> selectedCounts = calculateAverageSelectedCount(collection, destinations);
    logger.info("selectedCounts={}", selectedCounts);
    assertTrue("HasLots should get little", selectedCounts.get("HasLots") < selectedCounts.get("HasMedium"));
    assertTrue("HasMedium should get medium", selectedCounts.get("HasMedium") < selectedCounts.get("HasLittle"));
}

Example 9 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class PeerSelector method refreshPeers.

public void refreshPeers() {
    final PeerStatusCache existingCache = peerStatusCache;
    if (existingCache != null && (existingCache.getTimestamp() + PEER_CACHE_MILLIS > systemTime.currentTimeMillis())) {
        return;
    }
    try {
        final Set<PeerStatus> statuses = fetchRemotePeerStatuses();
        persistPeerStatuses(statuses);
        peerStatusCache = new PeerStatusCache(statuses);
        logger.info("{} Successfully refreshed Peer Status; remote instance consists of {} peers", this, statuses.size());
    } catch (Exception e) {
        warn(logger, eventReporter, "{} Unable to refresh Remote Group's peers due to {}", this, e.getMessage());
        if (logger.isDebugEnabled()) {
            logger.debug("", e);
        }
    }
}

Example 10 with PeerStatus

use of org.apache.nifi.remote.PeerStatus in project nifi by apache.

the class PeerSelector method formulateDestinationList.

List<PeerStatus> formulateDestinationList(final Set<PeerStatus> statuses, final TransferDirection direction) {
    final int numDestinations = Math.max(128, statuses.size());
    final Map<PeerStatus, Integer> entryCountMap = new HashMap<>();
    long totalFlowFileCount = 0L;
    for (final PeerStatus nodeInfo : statuses) {
        totalFlowFileCount += nodeInfo.getFlowFileCount();
    }
    int totalEntries = 0;
    for (final PeerStatus nodeInfo : statuses) {
        final int flowFileCount = nodeInfo.getFlowFileCount();
        // don't allow any node to get more than 80% of the data
        final double percentageOfFlowFiles = Math.min(0.8D, ((double) flowFileCount / (double) totalFlowFileCount));
        final double relativeWeighting = (direction == TransferDirection.SEND) ? (1 - percentageOfFlowFiles) : percentageOfFlowFiles;
        final int entries = Math.max(1, (int) (numDestinations * relativeWeighting));
        entryCountMap.put(nodeInfo, Math.max(1, entries));
        totalEntries += entries;
    }
    final List<PeerStatus> destinations = new ArrayList<>(totalEntries);
    for (int i = 0; i < totalEntries; i++) {
        destinations.add(null);
    }
    for (final Map.Entry<PeerStatus, Integer> entry : entryCountMap.entrySet()) {
        final PeerStatus nodeInfo = entry.getKey();
        final int numEntries = entry.getValue();
        int skipIndex = numEntries;
        for (int i = 0; i < numEntries; i++) {
            int n = (skipIndex * i);
            while (true) {
                final int index = n % destinations.size();
                PeerStatus status = destinations.get(index);
                if (status == null) {
                    status = new PeerStatus(nodeInfo.getPeerDescription(), nodeInfo.getFlowFileCount(), nodeInfo.isQueryForPeers());
                    destinations.set(index, status);
                    break;
                } else {
                    n++;
                }
            }
        }
    }
    // Shuffle destinations to provide better distribution.
    // Without this, same host will be used continuously, especially when remote peers have the same number of queued files.
    // Use Random(0) to provide consistent result for unit testing. Randomness is not important to shuffle destinations.
    Collections.shuffle(destinations, new Random(0));
    final StringBuilder distributionDescription = new StringBuilder();
    distributionDescription.append("New Weighted Distribution of Nodes:");
    for (final Map.Entry<PeerStatus, Integer> entry : entryCountMap.entrySet()) {
        final double percentage = entry.getValue() * 100D / destinations.size();
        distributionDescription.append("\n").append(entry.getKey()).append(" will receive ").append(percentage).append("% of data");
    }
    logger.info(distributionDescription.toString());
    // Jumble the list of destinations.
    return destinations;
}