Examples with Locality org.apache.flink.runtime.jobmanager.scheduler.Locality used on opensource projects

Example 1 with Locality

use of org.apache.flink.runtime.jobmanager.scheduler.Locality in project flink by apache.

the class LocationPreferenceSlotSelectionStrategy method selectWithLocationPreference.

@Nonnull
private Optional<SlotInfoAndLocality> selectWithLocationPreference(@Nonnull Collection<SlotInfoAndResources> availableSlots, @Nonnull Collection<TaskManagerLocation> locationPreferences, @Nonnull ResourceProfile resourceProfile) {
    // we build up two indexes, one for resource id and one for host names of the preferred
    // locations.
    final Map<ResourceID, Integer> preferredResourceIDs = new HashMap<>(locationPreferences.size());
    final Map<String, Integer> preferredFQHostNames = new HashMap<>(locationPreferences.size());
    for (TaskManagerLocation locationPreference : locationPreferences) {
        preferredResourceIDs.merge(locationPreference.getResourceID(), 1, Integer::sum);
        preferredFQHostNames.merge(locationPreference.getFQDNHostname(), 1, Integer::sum);
    }
    SlotInfoAndResources bestCandidate = null;
    Locality bestCandidateLocality = Locality.UNKNOWN;
    double bestCandidateScore = Double.NEGATIVE_INFINITY;
    for (SlotInfoAndResources candidate : availableSlots) {
        if (candidate.getRemainingResources().isMatching(resourceProfile)) {
            // this gets candidate is local-weigh
            int localWeigh = preferredResourceIDs.getOrDefault(candidate.getSlotInfo().getTaskManagerLocation().getResourceID(), 0);
            // this gets candidate is host-local-weigh
            int hostLocalWeigh = preferredFQHostNames.getOrDefault(candidate.getSlotInfo().getTaskManagerLocation().getFQDNHostname(), 0);
            double candidateScore = calculateCandidateScore(localWeigh, hostLocalWeigh, candidate.getTaskExecutorUtilization());
            if (candidateScore > bestCandidateScore) {
                bestCandidateScore = candidateScore;
                bestCandidate = candidate;
                bestCandidateLocality = localWeigh > 0 ? Locality.LOCAL : hostLocalWeigh > 0 ? Locality.HOST_LOCAL : Locality.NON_LOCAL;
            }
        }
    }
    // at the end of the iteration, we return the candidate with best possible locality or null.
    return bestCandidate != null ? Optional.of(SlotInfoAndLocality.of(bestCandidate.getSlotInfo(), bestCandidateLocality)) : Optional.empty();
}

Example 2 with Locality

use of org.apache.flink.runtime.jobmanager.scheduler.Locality in project flink by apache.

the class SlotSharingGroupAssignment method getSlotForTaskInternal.

private Tuple2<SharedSlot, Locality> getSlotForTaskInternal(AbstractID groupId, Iterable<TaskManagerLocation> preferredLocations, boolean localOnly) {
    // check if there is anything at all in this group assignment
    if (allSlots.isEmpty()) {
        return null;
    }
    // get the available slots for the group
    Map<ResourceID, List<SharedSlot>> slotsForGroup = availableSlotsPerJid.get(groupId);
    if (slotsForGroup == null) {
        // we have a new group, so all slots are available
        slotsForGroup = new LinkedHashMap<>();
        availableSlotsPerJid.put(groupId, slotsForGroup);
        for (SharedSlot availableSlot : allSlots) {
            putIntoMultiMap(slotsForGroup, availableSlot.getTaskManagerID(), availableSlot);
        }
    } else if (slotsForGroup.isEmpty()) {
        // the group exists, but nothing is available for that group
        return null;
    }
    // check whether we can schedule the task to a preferred location
    boolean didNotGetPreferred = false;
    if (preferredLocations != null) {
        for (TaskManagerLocation location : preferredLocations) {
            // set the flag that we failed a preferred location. If one will be found,
            // we return early anyways and skip the flag evaluation
            didNotGetPreferred = true;
            SharedSlot slot = removeFromMultiMap(slotsForGroup, location.getResourceID());
            if (slot != null && slot.isAlive()) {
                return new Tuple2<>(slot, Locality.LOCAL);
            }
        }
    }
    // if we want only local assignments, exit now with a "not found" result
    if (didNotGetPreferred && localOnly) {
        return null;
    }
    Locality locality = didNotGetPreferred ? Locality.NON_LOCAL : Locality.UNCONSTRAINED;
    // schedule the task to any available location
    SharedSlot slot;
    while ((slot = pollFromMultiMap(slotsForGroup)) != null) {
        if (slot.isAlive()) {
            return new Tuple2<>(slot, locality);
        }
    }
    // nothing available after all, all slots were dead
    return null;
}