Examples with IndexedPriorityQueue io.prestosql.execution.resourcegroups.IndexedPriorityQueue used on opensource projects

Example 1 with IndexedPriorityQueue

use of io.prestosql.execution.resourcegroups.IndexedPriorityQueue in project hetu-core by openlookeng.

the class SimpleNodeSelector method equateDistribution.

/**
 * The method tries to make the distribution of splits more uniform. All nodes are arranged into a maxHeap and a minHeap
 * based on the number of splits that are assigned to them. Splits are redistributed, one at a time, from a maxNode to a
 * minNode until we have as uniform a distribution as possible.
 *
 * @param assignment the node-splits multimap after the first and the second stage
 * @param assignmentStats required to obtain info regarding splits assigned to a node outside the current batch of assignment
 * @param nodeMap to get a list of all nodes to which splits can be assigned
 */
private void equateDistribution(Multimap<InternalNode, Split> assignment, NodeAssignmentStats assignmentStats, NodeMap nodeMap) {
    if (assignment.isEmpty()) {
        return;
    }
    Collection<InternalNode> allNodes = nodeMap.getNodesByHostAndPort().values();
    if (allNodes.size() < 2) {
        return;
    }
    IndexedPriorityQueue<InternalNode> maxNodes = new IndexedPriorityQueue<>();
    for (InternalNode node : assignment.keySet()) {
        maxNodes.addOrUpdate(node, assignmentStats.getTotalSplitCount(node));
    }
    IndexedPriorityQueue<InternalNode> minNodes = new IndexedPriorityQueue<>();
    for (InternalNode node : allNodes) {
        minNodes.addOrUpdate(node, Long.MAX_VALUE - assignmentStats.getTotalSplitCount(node));
    }
    while (true) {
        if (maxNodes.isEmpty()) {
            return;
        }
        // fetch min and max node
        InternalNode maxNode = maxNodes.poll();
        InternalNode minNode = minNodes.poll();
        if (assignmentStats.getTotalSplitCount(maxNode) - assignmentStats.getTotalSplitCount(minNode) <= 1) {
            return;
        }
        // move split from max to min
        redistributeSplit(assignment, maxNode, minNode, nodeMap.getNodesByHost());
        assignmentStats.removeAssignedSplit(maxNode);
        assignmentStats.addAssignedSplit(minNode);
        // add max back into maxNodes only if it still has assignments
        if (assignment.containsKey(maxNode)) {
            maxNodes.addOrUpdate(maxNode, assignmentStats.getTotalSplitCount(maxNode));
        }
        // Add or update both the Priority Queues with the updated node priorities
        maxNodes.addOrUpdate(minNode, assignmentStats.getTotalSplitCount(minNode));
        minNodes.addOrUpdate(minNode, Long.MAX_VALUE - assignmentStats.getTotalSplitCount(minNode));
        minNodes.addOrUpdate(maxNode, Long.MAX_VALUE - assignmentStats.getTotalSplitCount(maxNode));
    }
}