Examples with PriorityQueue java.util.PriorityQueue used on opensource projects

Example 46 with PriorityQueue

use of java.util.PriorityQueue in project asterixdb by apache.

the class Scheduler method scheduleLocalSlots.

/**
     * Schedule data-local slots to each machine.
     *
     * @param splits
     *            The HDFS file splits.
     * @param workloads
     *            The current capacity of each machine.
     * @param locations
     *            The result schedule.
     * @param slots
     *            The maximum slots of each machine.
     * @param random
     *            The random generator.
     * @param scheduled
     *            Indicate which slot is scheduled.
     * @throws IOException
     * @throws UnknownHostException
     */
private void scheduleLocalSlots(InputSplit[] splits, int[] workloads, String[] locations, int slots, Random random, boolean[] scheduled, final Map<String, IntWritable> locationToNumSplits) throws IOException, UnknownHostException {
    /** scheduling candidates will be ordered inversely according to their popularity */
    PriorityQueue<String> scheduleCadndiates = new PriorityQueue<String>(3, new Comparator<String>() {

        @Override
        public int compare(String s1, String s2) {
            return locationToNumSplits.get(s1).compareTo(locationToNumSplits.get(s2));
        }
    });
    for (int i = 0; i < splits.length; i++) {
        if (scheduled[i]) {
            continue;
        }
        /**
             * get the location of all the splits
             */
        String[] locs = splits[i].getLocations();
        if (locs.length > 0) {
            scheduleCadndiates.clear();
            for (int j = 0; j < locs.length; j++) {
                scheduleCadndiates.add(locs[j]);
            }
            for (String candidate : scheduleCadndiates) {
                /**
                     * get all the IP addresses from the name
                     */
                InetAddress[] allIps = InetAddress.getAllByName(candidate);
                /**
                     * iterate overa all ips
                     */
                for (InetAddress ip : allIps) {
                    /**
                         * if the node controller exists
                         */
                    if (ipToNcMapping.get(ip.getHostAddress()) != null) {
                        /**
                             * set the ncs
                             */
                        List<String> dataLocations = ipToNcMapping.get(ip.getHostAddress());
                        int arrayPos = random.nextInt(dataLocations.size());
                        String nc = dataLocations.get(arrayPos);
                        int pos = ncNameToIndex.get(nc);
                        /**
                             * check if the node is already full
                             */
                        if (workloads[pos] < slots) {
                            locations[i] = nc;
                            workloads[pos]++;
                            scheduled[i] = true;
                            break;
                        }
                    }
                }
                /**
                     * break the loop for data-locations if the schedule has
                     * already been found
                     */
                if (scheduled[i] == true) {
                    break;
                }
            }
        }
    }
}

Example 47 with PriorityQueue

use of java.util.PriorityQueue in project drill by apache.

the class MapRDBGroupScan method applyAssignments.

/**
   *
   * @param incomingEndpoints
   */
@Override
public void applyAssignments(List<DrillbitEndpoint> incomingEndpoints) {
    watch.reset();
    watch.start();
    final int numSlots = incomingEndpoints.size();
    Preconditions.checkArgument(numSlots <= regionsToScan.size(), String.format("Incoming endpoints %d is greater than number of scan regions %d", numSlots, regionsToScan.size()));
    /*
     * Minimum/Maximum number of assignment per slot
     */
    final int minPerEndpointSlot = (int) Math.floor((double) regionsToScan.size() / numSlots);
    final int maxPerEndpointSlot = (int) Math.ceil((double) regionsToScan.size() / numSlots);
    /*
     * initialize (endpoint index => HBaseSubScanSpec list) map
     */
    endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots);
    /*
     * another map with endpoint (hostname => corresponding index list) in 'incomingEndpoints' list
     */
    Map<String, Queue<Integer>> endpointHostIndexListMap = Maps.newHashMap();
    /*
     * Initialize these two maps
     */
    for (int i = 0; i < numSlots; ++i) {
        endpointFragmentMapping.put(i, new ArrayList<MapRDBSubScanSpec>(maxPerEndpointSlot));
        String hostname = incomingEndpoints.get(i).getAddress();
        Queue<Integer> hostIndexQueue = endpointHostIndexListMap.get(hostname);
        if (hostIndexQueue == null) {
            hostIndexQueue = Lists.newLinkedList();
            endpointHostIndexListMap.put(hostname, hostIndexQueue);
        }
        hostIndexQueue.add(i);
    }
    Set<Entry<TabletFragmentInfo, String>> regionsToAssignSet = Sets.newHashSet(regionsToScan.entrySet());
    /*
     * First, we assign regions which are hosted on region servers running on drillbit endpoints
     */
    for (Iterator<Entry<TabletFragmentInfo, String>> regionsIterator = regionsToAssignSet.iterator(); regionsIterator.hasNext(); ) /*nothing*/
    {
        Entry<TabletFragmentInfo, String> regionEntry = regionsIterator.next();
        /*
       * Test if there is a drillbit endpoint which is also an HBase RegionServer that hosts the current HBase region
       */
        Queue<Integer> endpointIndexlist = endpointHostIndexListMap.get(regionEntry.getValue());
        if (endpointIndexlist != null) {
            Integer slotIndex = endpointIndexlist.poll();
            List<MapRDBSubScanSpec> endpointSlotScanList = endpointFragmentMapping.get(slotIndex);
            endpointSlotScanList.add(getSubScanSpec(regionEntry.getKey()));
            // add to the tail of the slot list, to add more later in round robin fashion
            endpointIndexlist.offer(slotIndex);
            // this region has been assigned
            regionsIterator.remove();
        }
    }
    /*
     * Build priority queues of slots, with ones which has tasks lesser than 'minPerEndpointSlot' and another which have more.
     */
    PriorityQueue<List<MapRDBSubScanSpec>> minHeap = new PriorityQueue<List<MapRDBSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR);
    PriorityQueue<List<MapRDBSubScanSpec>> maxHeap = new PriorityQueue<List<MapRDBSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR_REV);
    for (List<MapRDBSubScanSpec> listOfScan : endpointFragmentMapping.values()) {
        if (listOfScan.size() <= minPerEndpointSlot) {
            minHeap.offer(listOfScan);
        } else if (listOfScan.size() > minPerEndpointSlot) {
            maxHeap.offer(listOfScan);
        }
    }
    /*
     * Now, let's process any regions which remain unassigned and assign them to slots with minimum number of assignments.
     */
    if (regionsToAssignSet.size() > 0) {
        for (Entry<TabletFragmentInfo, String> regionEntry : regionsToAssignSet) {
            List<MapRDBSubScanSpec> smallestList = minHeap.poll();
            smallestList.add(getSubScanSpec(regionEntry.getKey()));
            if (smallestList.size() < maxPerEndpointSlot) {
                minHeap.offer(smallestList);
            }
        }
    }
    /*
     * While there are slots with lesser than 'minPerEndpointSlot' unit work, balance from those with more.
     */
    while (minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) {
        List<MapRDBSubScanSpec> smallestList = (List<MapRDBSubScanSpec>) minHeap.poll();
        List<MapRDBSubScanSpec> largestList = (List<MapRDBSubScanSpec>) maxHeap.poll();
        smallestList.add(largestList.remove(largestList.size() - 1));
        if (largestList.size() > minPerEndpointSlot) {
            maxHeap.offer(largestList);
        }
        if (smallestList.size() < minPerEndpointSlot) {
            minHeap.offer(smallestList);
        }
    }
    /* no slot should be empty at this point */
    assert (minHeap.peek() == null || minHeap.peek().size() > 0) : String.format("Unable to assign tasks to some endpoints.\nEndpoints: {}.\nAssignment Map: {}.", incomingEndpoints, endpointFragmentMapping.toString());
    logger.debug("Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}", watch.elapsed(TimeUnit.NANOSECONDS) / 1000, incomingEndpoints, endpointFragmentMapping.toString());
}

Example 48 with PriorityQueue

use of java.util.PriorityQueue in project drill by apache.

the class MongoGroupScan method applyAssignments.

@Override
public void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException {
    logger.debug("Incoming endpoints :" + endpoints);
    watch.reset();
    watch.start();
    final int numSlots = endpoints.size();
    int totalAssignmentsTobeDone = chunksMapping.size();
    Preconditions.checkArgument(numSlots <= totalAssignmentsTobeDone, String.format("Incoming endpoints %d is greater than number of chunks %d", numSlots, totalAssignmentsTobeDone));
    final int minPerEndpointSlot = (int) Math.floor((double) totalAssignmentsTobeDone / numSlots);
    final int maxPerEndpointSlot = (int) Math.ceil((double) totalAssignmentsTobeDone / numSlots);
    endpointFragmentMapping = Maps.newHashMapWithExpectedSize(numSlots);
    Map<String, Queue<Integer>> endpointHostIndexListMap = Maps.newHashMap();
    for (int i = 0; i < numSlots; ++i) {
        endpointFragmentMapping.put(i, new ArrayList<MongoSubScanSpec>(maxPerEndpointSlot));
        String hostname = endpoints.get(i).getAddress();
        Queue<Integer> hostIndexQueue = endpointHostIndexListMap.get(hostname);
        if (hostIndexQueue == null) {
            hostIndexQueue = Lists.newLinkedList();
            endpointHostIndexListMap.put(hostname, hostIndexQueue);
        }
        hostIndexQueue.add(i);
    }
    Set<Entry<String, List<ChunkInfo>>> chunksToAssignSet = Sets.newHashSet(chunksInverseMapping.entrySet());
    for (Iterator<Entry<String, List<ChunkInfo>>> chunksIterator = chunksToAssignSet.iterator(); chunksIterator.hasNext(); ) {
        Entry<String, List<ChunkInfo>> chunkEntry = chunksIterator.next();
        Queue<Integer> slots = endpointHostIndexListMap.get(chunkEntry.getKey());
        if (slots != null) {
            for (ChunkInfo chunkInfo : chunkEntry.getValue()) {
                Integer slotIndex = slots.poll();
                List<MongoSubScanSpec> subScanSpecList = endpointFragmentMapping.get(slotIndex);
                subScanSpecList.add(buildSubScanSpecAndGet(chunkInfo));
                slots.offer(slotIndex);
            }
            chunksIterator.remove();
        }
    }
    PriorityQueue<List<MongoSubScanSpec>> minHeap = new PriorityQueue<List<MongoSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR);
    PriorityQueue<List<MongoSubScanSpec>> maxHeap = new PriorityQueue<List<MongoSubScanSpec>>(numSlots, LIST_SIZE_COMPARATOR_REV);
    for (List<MongoSubScanSpec> listOfScan : endpointFragmentMapping.values()) {
        if (listOfScan.size() < minPerEndpointSlot) {
            minHeap.offer(listOfScan);
        } else if (listOfScan.size() > minPerEndpointSlot) {
            maxHeap.offer(listOfScan);
        }
    }
    if (chunksToAssignSet.size() > 0) {
        for (Entry<String, List<ChunkInfo>> chunkEntry : chunksToAssignSet) {
            for (ChunkInfo chunkInfo : chunkEntry.getValue()) {
                List<MongoSubScanSpec> smallestList = minHeap.poll();
                smallestList.add(buildSubScanSpecAndGet(chunkInfo));
                minHeap.offer(smallestList);
            }
        }
    }
    while (minHeap.peek() != null && minHeap.peek().size() < minPerEndpointSlot) {
        List<MongoSubScanSpec> smallestList = minHeap.poll();
        List<MongoSubScanSpec> largestList = maxHeap.poll();
        smallestList.add(largestList.remove(largestList.size() - 1));
        if (largestList.size() > minPerEndpointSlot) {
            maxHeap.offer(largestList);
        }
        if (smallestList.size() < minPerEndpointSlot) {
            minHeap.offer(smallestList);
        }
    }
    logger.debug("Built assignment map in {} µs.\nEndpoints: {}.\nAssignment Map: {}", watch.elapsed(TimeUnit.NANOSECONDS) / 1000, endpoints, endpointFragmentMapping.toString());
}

Example 49 with PriorityQueue

use of java.util.PriorityQueue in project ignite by apache.

the class OptimizedObjectStreamSelfTest method testPriorityQueue.

/**
     * @throws Exception If failed.
     */
public void testPriorityQueue() throws Exception {
    Queue<Integer> queue = new PriorityQueue<>();
    for (int i = 0; i < 100; i++) queue.add(i);
    Queue<Integer> newQueue = marshalUnmarshal(queue);
    assertEquals(queue.size(), newQueue.size());
    Integer i;
    while ((i = newQueue.poll()) != null) assertEquals(queue.poll(), i);
}

Example 50 with PriorityQueue

use of java.util.PriorityQueue in project lucene-solr by apache.

the class GroupOperation method init.

private void init(StreamComparator streamComparator, int size) {
    this.size = size;
    this.streamComparator = streamComparator;
    this.comp = new ReverseComp(streamComparator);
    this.priorityQueue = new PriorityQueue(size, this.comp);
}