Examples with PriorityQueue java.util.PriorityQueue used on opensource projects

Example 1 with PriorityQueue

use of java.util.PriorityQueue in project flink by apache.

the class TimestampedFileInputSplitTest method testPriorityQ.

@Test
public void testPriorityQ() {
    TimestampedFileInputSplit richFirstSplit = new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);
    TimestampedFileInputSplit richSecondSplit = new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 0, 100, null);
    TimestampedFileInputSplit richThirdSplit = new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);
    TimestampedFileInputSplit richForthSplit = new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);
    TimestampedFileInputSplit richFifthSplit = new TimestampedFileInputSplit(11, 1, new Path("test/test3"), 0, 100, null);
    Queue<TimestampedFileInputSplit> pendingSplits = new PriorityQueue<>();
    pendingSplits.add(richSecondSplit);
    pendingSplits.add(richForthSplit);
    pendingSplits.add(richFirstSplit);
    pendingSplits.add(richFifthSplit);
    pendingSplits.add(richFifthSplit);
    pendingSplits.add(richThirdSplit);
    List<TimestampedFileInputSplit> actualSortedSplits = new ArrayList<>();
    while (true) {
        actualSortedSplits.add(pendingSplits.poll());
        if (pendingSplits.isEmpty()) {
            break;
        }
    }
    List<TimestampedFileInputSplit> expectedSortedSplits = new ArrayList<>();
    expectedSortedSplits.add(richFirstSplit);
    expectedSortedSplits.add(richThirdSplit);
    expectedSortedSplits.add(richSecondSplit);
    expectedSortedSplits.add(richForthSplit);
    expectedSortedSplits.add(richFifthSplit);
    expectedSortedSplits.add(richFifthSplit);
    Assert.assertArrayEquals(expectedSortedSplits.toArray(), actualSortedSplits.toArray());
}

Example 2 with PriorityQueue

use of java.util.PriorityQueue in project hadoop by apache.

the class QuorumJournalManager method selectInputStreams.

@Override
public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk, boolean onlyDurableTxns) throws IOException {
    QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk);
    Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams");
    LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
    final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
    for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) {
        AsyncLogger logger = e.getKey();
        RemoteEditLogManifest manifest = e.getValue();
        long committedTxnId = manifest.getCommittedTxnId();
        for (RemoteEditLog remoteLog : manifest.getLogs()) {
            URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());
            long endTxId = remoteLog.getEndTxId();
            // than committedTxnId. This ensures the consistency.
            if (onlyDurableTxns && inProgressOk) {
                endTxId = Math.min(endTxId, committedTxnId);
            }
            EditLogInputStream elis = EditLogFileInputStream.fromUrl(connectionFactory, url, remoteLog.getStartTxId(), endTxId, remoteLog.isInProgress());
            allStreams.add(elis);
        }
    }
    JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}

Example 3 with PriorityQueue

use of java.util.PriorityQueue in project hadoop by apache.

the class AbstractPreemptableResourceCalculator method computeFixpointAllocation.

/**
   * Given a set of queues compute the fix-point distribution of unassigned
   * resources among them. As pending request of a queue are exhausted, the
   * queue is removed from the set and remaining capacity redistributed among
   * remaining queues. The distribution is weighted based on guaranteed
   * capacity, unless asked to ignoreGuarantee, in which case resources are
   * distributed uniformly.
   *
   * @param totGuarant
   *          total guaranteed resource
   * @param qAlloc
   *          List of child queues
   * @param unassigned
   *          Unassigned resource per queue
   * @param ignoreGuarantee
   *          ignore guarantee per queue.
   */
protected void computeFixpointAllocation(Resource totGuarant, Collection<TempQueuePerPartition> qAlloc, Resource unassigned, boolean ignoreGuarantee) {
    // Prior to assigning the unused resources, process each queue as follows:
    // If current > guaranteed, idealAssigned = guaranteed + untouchable extra
    // Else idealAssigned = current;
    // Subtract idealAssigned resources from unassigned.
    // If the queue has all of its needs met (that is, if
    // idealAssigned >= current + pending), remove it from consideration.
    // Sort queues from most under-guaranteed to most over-guaranteed.
    TQComparator tqComparator = new TQComparator(rc, totGuarant);
    PriorityQueue<TempQueuePerPartition> orderedByNeed = new PriorityQueue<>(10, tqComparator);
    for (Iterator<TempQueuePerPartition> i = qAlloc.iterator(); i.hasNext(); ) {
        TempQueuePerPartition q = i.next();
        Resource used = q.getUsed();
        if (Resources.greaterThan(rc, totGuarant, used, q.getGuaranteed())) {
            q.idealAssigned = Resources.add(q.getGuaranteed(), q.untouchableExtra);
        } else {
            q.idealAssigned = Resources.clone(used);
        }
        Resources.subtractFrom(unassigned, q.idealAssigned);
        // If idealAssigned < (allocated + used + pending), q needs more
        // resources, so
        // add it to the list of underserved queues, ordered by need.
        Resource curPlusPend = Resources.add(q.getUsed(), q.pending);
        if (Resources.lessThan(rc, totGuarant, q.idealAssigned, curPlusPend)) {
            orderedByNeed.add(q);
        }
    }
    // left
    while (!orderedByNeed.isEmpty() && Resources.greaterThan(rc, totGuarant, unassigned, Resources.none())) {
        Resource wQassigned = Resource.newInstance(0, 0);
        // we compute normalizedGuarantees capacity based on currently active
        // queues
        resetCapacity(unassigned, orderedByNeed, ignoreGuarantee);
        // For each underserved queue (or set of queues if multiple are equally
        // underserved), offer its share of the unassigned resources based on its
        // normalized guarantee. After the offer, if the queue is not satisfied,
        // place it back in the ordered list of queues, recalculating its place
        // in the order of most under-guaranteed to most over-guaranteed. In this
        // way, the most underserved queue(s) are always given resources first.
        Collection<TempQueuePerPartition> underserved = getMostUnderservedQueues(orderedByNeed, tqComparator);
        for (Iterator<TempQueuePerPartition> i = underserved.iterator(); i.hasNext(); ) {
            TempQueuePerPartition sub = i.next();
            Resource wQavail = Resources.multiplyAndNormalizeUp(rc, unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1));
            Resource wQidle = sub.offer(wQavail, rc, totGuarant, isReservedPreemptionCandidatesSelector);
            Resource wQdone = Resources.subtract(wQavail, wQidle);
            if (Resources.greaterThan(rc, totGuarant, wQdone, Resources.none())) {
                // The queue is still asking for more. Put it back in the priority
                // queue, recalculating its order based on need.
                orderedByNeed.add(sub);
            }
            Resources.addTo(wQassigned, wQdone);
        }
        Resources.subtractFrom(unassigned, wQassigned);
    }
    // all queues are added to list.
    while (!orderedByNeed.isEmpty()) {
        TempQueuePerPartition q1 = orderedByNeed.remove();
        context.addPartitionToUnderServedQueues(q1.queueName, q1.partition);
    }
}

Example 4 with PriorityQueue

use of java.util.PriorityQueue in project hadoop by apache.

the class FifoIntraQueuePreemptionPlugin method createTempAppForResCalculation.

private PriorityQueue<TempAppPerPartition> createTempAppForResCalculation(String partition, Collection<FiCaSchedulerApp> apps, TAPriorityComparator taComparator) {
    PriorityQueue<TempAppPerPartition> orderedByPriority = new PriorityQueue<>(100, taComparator);
    // have an internal temp app structure to store intermediate data(priority)
    for (FiCaSchedulerApp app : apps) {
        Resource used = app.getAppAttemptResourceUsage().getUsed(partition);
        Resource amUsed = null;
        if (!app.isWaitingForAMContainer()) {
            amUsed = app.getAMResource(partition);
        }
        Resource pending = app.getTotalPendingRequestsPerPartition().get(partition);
        Resource reserved = app.getAppAttemptResourceUsage().getReserved(partition);
        used = (used == null) ? Resources.createResource(0, 0) : used;
        amUsed = (amUsed == null) ? Resources.createResource(0, 0) : amUsed;
        pending = (pending == null) ? Resources.createResource(0, 0) : pending;
        reserved = (reserved == null) ? Resources.createResource(0, 0) : reserved;
        HashSet<String> partitions = new HashSet<String>(app.getAppAttemptResourceUsage().getNodePartitionsSet());
        partitions.addAll(app.getTotalPendingRequestsPerPartition().keySet());
        // Create TempAppPerQueue for further calculation.
        TempAppPerPartition tmpApp = new TempAppPerPartition(app, Resources.clone(used), Resources.clone(amUsed), Resources.clone(reserved), Resources.clone(pending));
        // Set ideal allocation of app as 0.
        tmpApp.idealAssigned = Resources.createResource(0, 0);
        orderedByPriority.add(tmpApp);
    }
    return orderedByPriority;
}

Example 5 with PriorityQueue

use of java.util.PriorityQueue in project storm by apache.

the class DirectoryCleaner method deleteOldestWhileTooLarge.

/**
     * If totalSize of files exceeds the either the per-worker quota or global quota,
     * Logviewer deletes oldest inactive log files in a worker directory or in all worker dirs.
     * We use the parameter for_per_dir to switch between the two deletion modes.
     * @param dirs the list of directories to be scanned for deletion
     * @param quota the per-dir quota or the total quota for the all directories
     * @param for_per_dir if true, deletion happens for a single dir; otherwise, for all directories globally
     * @param active_dirs only for global deletion, we want to skip the active logs in active_dirs
     * @return number of files deleted
     */
public int deleteOldestWhileTooLarge(List<File> dirs, long quota, boolean for_per_dir, Set<String> active_dirs) throws IOException {
    // max number of files to delete for every round
    final int PQ_SIZE = 1024;
    // max rounds of scanning the dirs
    final int MAX_ROUNDS = 512;
    long totalSize = 0;
    int deletedFiles = 0;
    for (File dir : dirs) {
        try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
            for (Path path : stream) {
                File file = path.toFile();
                totalSize += file.length();
            }
        }
    }
    long toDeleteSize = totalSize - quota;
    if (toDeleteSize <= 0) {
        return deletedFiles;
    }
    Comparator<File> comparator = new Comparator<File>() {

        public int compare(File f1, File f2) {
            if (f1.lastModified() > f2.lastModified()) {
                return -1;
            } else {
                return 1;
            }
        }
    };
    // the oldest pq_size files in this directory will be placed in PQ, with the newest at the root
    PriorityQueue<File> pq = new PriorityQueue<File>(PQ_SIZE, comparator);
    int round = 0;
    while (toDeleteSize > 0) {
        LOG.debug("To delete size is {}, start a new round of deletion, round: {}", toDeleteSize, round);
        for (File dir : dirs) {
            try (DirectoryStream<Path> stream = getStreamForDirectory(dir)) {
                for (Path path : stream) {
                    File file = path.toFile();
                    if (for_per_dir) {
                        if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
                            // skip active log files
                            continue;
                        }
                    } else {
                        // for global cleanup
                        if (active_dirs.contains(dir.getCanonicalPath())) {
                            // for an active worker's dir, make sure for the last "/"
                            if (ACTIVE_LOG_PATTERN.matcher(file.getName()).matches()) {
                                // skip active log files
                                continue;
                            }
                        } else {
                            if (META_LOG_PATTERN.matcher(file.getName()).matches()) {
                                // skip yaml and pid files
                                continue;
                            }
                        }
                    }
                    if (pq.size() < PQ_SIZE) {
                        pq.offer(file);
                    } else {
                        if (file.lastModified() < pq.peek().lastModified()) {
                            pq.poll();
                            pq.offer(file);
                        }
                    }
                }
            }
        }
        // need to reverse the order of elements in PQ to delete files from oldest to newest
        Stack<File> stack = new Stack<File>();
        while (!pq.isEmpty()) {
            File file = pq.poll();
            stack.push(file);
        }
        while (!stack.isEmpty() && toDeleteSize > 0) {
            File file = stack.pop();
            toDeleteSize -= file.length();
            LOG.info("Delete file: {}, size: {}, lastModified: {}", file.getCanonicalPath(), file.length(), file.lastModified());
            file.delete();
            deletedFiles++;
        }
        pq.clear();
        round++;
        if (round >= MAX_ROUNDS) {
            if (for_per_dir) {
                LOG.warn("Reach the MAX_ROUNDS: {} during per-dir deletion, you may have too many files in " + "a single directory : {}, will delete the rest files in next interval.", MAX_ROUNDS, dirs.get(0).getCanonicalPath());
            } else {
                LOG.warn("Reach the MAX_ROUNDS: {} during global deletion, you may have too many files, " + "will delete the rest files in next interval.", MAX_ROUNDS);
            }
            break;
        }
    }
    return deletedFiles;
}