Examples with NormalizedResourceOffer org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer used on opensource projects

Example 1 with NormalizedResourceOffer

use of org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer in project storm by apache.

the class NodeSorter method sortObjectResourcesCommon.

/**
 * Sort objects by the following three criteria.
 *
 * <li>
 *     The number executors of the topology that needs to be scheduled is already on the object (node or rack)
 *     in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on
 *     the same object (node or rack) as the existing executors of the topology.
 * </li>
 *
 * <li>
 *     The subordinate/subservient resource availability percentage of a rack in descending order We calculate the
 *     resource availability percentage by dividing the resource availability of the object (node or rack) by the
 *     resource availability of the entire rack or cluster depending on if object references a node or a rack.
 *     How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is
 *     requested by the executor that the sorting is being done for and pulls it down if it is not.
 *     By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned
 *     above will be ranked after racks that have more balanced resource availability and nodes or racks that have
 *     resources that are not requested will be ranked below . So we will be less likely to pick a rack that
 *     have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
 *     This is similar to logic used {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 * </li>
 *
 * <li>
 *     The tie between two nodes with same resource availability is broken by using the node with lower minimum
 *     percentage used. This comparison was used in {@link #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)}
 *     but here it is made subservient to modified resource availbility used in
 *     {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 *
 * </li>
 *
 * @param allResources         contains all individual ObjectResources as well as cumulative stats
 * @param exec                 executor for which the sorting is done
 * @param existingScheduleFunc a function to get existing executors already scheduled on this object
 * @return a sorted list of ObjectResources
 */
private List<ObjectResourcesItem> sortObjectResourcesCommon(final ObjectResourcesSummary allResources, final ExecutorDetails exec, final ExistingScheduleFunc existingScheduleFunc) {
    // Copy and modify allResources
    ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources);
    final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall();
    final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null;
    affinityBasedAllResources.getObjectResources().forEach(x -> {
        x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources);
        if (requestedResources != null) {
            // negate unrequested resources
            x.availableResources.updateForRareResourceAffinity(requestedResources);
        }
        x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources);
        LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id));
    });
    // Use the following comparator to return a sorted set
    List<ObjectResourcesItem> sortedObjectResources = new ArrayList();
    Comparator<ObjectResourcesItem> comparator = (o1, o2) -> {
        int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
        int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
        if (execsScheduled1 > execsScheduled2) {
            return -1;
        } else if (execsScheduled1 < execsScheduled2) {
            return 1;
        }
        double o1Avg = o1.avgResourcePercent;
        double o2Avg = o2.avgResourcePercent;
        if (o1Avg > o2Avg) {
            return -1;
        } else if (o1Avg < o2Avg) {
            return 1;
        }
        if (o1.minResourcePercent > o2.minResourcePercent) {
            return -1;
        } else if (o1.minResourcePercent < o2.minResourcePercent) {
            return 1;
        }
        return o1.id.compareTo(o2.id);
    };
    sortedObjectResources.addAll(affinityBasedAllResources.getObjectResources());
    sortedObjectResources.sort(comparator);
    LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
    return sortedObjectResources;
}

Example 2 with NormalizedResourceOffer

use of org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer in project storm by apache.

the class NodeSorterHostProximity method sortObjectResourcesCommon.

/**
 * Sort objects by the following three criteria.
 *
 * <li>
 *     The number executors of the topology that needs to be scheduled is already on the object (node or rack)
 *     in descending order. The reasoning to sort based on criterion 1 is so we schedule the rest of a topology on
 *     the same object (node or rack) as the existing executors of the topology.
 * </li>
 *
 * <li>
 *     The subordinate/subservient resource availability percentage of a rack in descending order We calculate the
 *     resource availability percentage by dividing the resource availability of the object (node or rack) by the
 *     resource availability of the entire rack or cluster depending on if object references a node or a rack.
 *     How this differs from the DefaultResourceAwareStrategy is that the percentage boosts the node or rack if it is
 *     requested by the executor that the sorting is being done for and pulls it down if it is not.
 *     By doing this calculation, objects (node or rack) that have exhausted or little of one of the resources mentioned
 *     above will be ranked after racks that have more balanced resource availability and nodes or racks that have
 *     resources that are not requested will be ranked below . So we will be less likely to pick a rack that
 *     have a lot of one resource but a low amount of another and have a lot of resources that are not requested by the executor.
 *     This is similar to logic used {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 * </li>
 *
 * <li>
 *     The tie between two nodes with same resource availability is broken by using the node with lower minimum
 *     percentage used. This comparison was used in {@link #sortObjectResourcesDefault(ObjectResourcesSummary, ExistingScheduleFunc)}
 *     but here it is made subservient to modified resource availbility used in
 *     {@link #sortObjectResourcesGeneric(ObjectResourcesSummary, ExecutorDetails, ExistingScheduleFunc)}.
 *
 * </li>
 *
 * @param allResources         contains all individual ObjectResources as well as cumulative stats
 * @param exec                 executor for which the sorting is done
 * @param existingScheduleFunc a function to get existing executors already scheduled on this object
 * @return an {@link Iterable} of sorted {@link ObjectResourcesItem}
 */
private Iterable<ObjectResourcesItem> sortObjectResourcesCommon(final ObjectResourcesSummary allResources, final ExecutorDetails exec, final ExistingScheduleFunc existingScheduleFunc) {
    // Copy and modify allResources
    ObjectResourcesSummary affinityBasedAllResources = new ObjectResourcesSummary(allResources);
    final NormalizedResourceOffer availableResourcesOverall = allResources.getAvailableResourcesOverall();
    final NormalizedResourceRequest requestedResources = (exec != null) ? topologyDetails.getTotalResources(exec) : null;
    affinityBasedAllResources.getObjectResources().forEach(x -> {
        if (requestedResources != null) {
            // negate unrequested resources
            x.availableResources.updateForRareResourceAffinity(requestedResources);
        }
        x.minResourcePercent = availableResourcesOverall.calculateMinPercentageUsedBy(x.availableResources);
        x.avgResourcePercent = availableResourcesOverall.calculateAveragePercentageUsedBy(x.availableResources);
        LOG.trace("for {}: minResourcePercent={}, avgResourcePercent={}, numExistingSchedule={}", x.id, x.minResourcePercent, x.avgResourcePercent, existingScheduleFunc.getNumExistingSchedule(x.id));
    });
    // Use the following comparator to sort
    Comparator<ObjectResourcesItem> comparator = (o1, o2) -> {
        int execsScheduled1 = existingScheduleFunc.getNumExistingSchedule(o1.id);
        int execsScheduled2 = existingScheduleFunc.getNumExistingSchedule(o2.id);
        if (execsScheduled1 > execsScheduled2) {
            return -1;
        } else if (execsScheduled1 < execsScheduled2) {
            return 1;
        }
        double o1Avg = o1.avgResourcePercent;
        double o2Avg = o2.avgResourcePercent;
        if (o1Avg > o2Avg) {
            return -1;
        } else if (o1Avg < o2Avg) {
            return 1;
        }
        if (o1.minResourcePercent > o2.minResourcePercent) {
            return -1;
        } else if (o1.minResourcePercent < o2.minResourcePercent) {
            return 1;
        }
        return o1.id.compareTo(o2.id);
    };
    TreeSet<ObjectResourcesItem> sortedObjectResources = new TreeSet(comparator);
    sortedObjectResources.addAll(affinityBasedAllResources.getObjectResources());
    LOG.debug("Sorted Object Resources: {}", sortedObjectResources);
    return sortedObjectResources;
}

Example 3 with NormalizedResourceOffer

use of org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer in project storm by apache.

the class RasNode method getTotalAvailableResources.

/**
 * Gets all available resources for this node.
 *
 * @return All of the available resources.
 */
public NormalizedResourceOffer getTotalAvailableResources() {
    if (sup != null) {
        NormalizedResourceOffer availableResources = new NormalizedResourceOffer(sup.getTotalResources());
        if (availableResources.remove(cluster.getAllScheduledResourcesForNode(sup.getId()), cluster.getResourceMetrics())) {
            if (!loggedUnderageUsage) {
                LOG.error("Resources on {} became negative and was clamped to 0 {}.", hostname, availableResources);
                loggedUnderageUsage = true;
            }
        }
        return availableResources;
    } else {
        return new NormalizedResourceOffer();
    }
}

Example 4 with NormalizedResourceOffer

use of org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer in project storm by apache.

the class RasNode method couldEverFit.

/**
 * Is there any possibility that exec could ever fit on this node.
 * @param exec the executor to schedule
 * @param td the topology the executor is a part of
 * @return true if there is the possibility it might fit, no guarantee that it will, or false if there is no
 *     way it would ever fit.
 */
public boolean couldEverFit(ExecutorDetails exec, TopologyDetails td) {
    if (!isAlive) {
        return false;
    }
    NormalizedResourceOffer avail = getTotalAvailableResources();
    NormalizedResourceRequest requestedResources = td.getTotalResources(exec);
    return avail.couldFit(cluster.getMinWorkerCpu(), requestedResources);
}

Example 5 with NormalizedResourceOffer

use of org.apache.storm.scheduler.resource.normalization.NormalizedResourceOffer in project storm by apache.

the class SupervisorHeartbeat method buildSupervisorInfo.

private Map<String, SupervisorInfo> buildSupervisorInfo(Map<String, Object> conf, Supervisor supervisor, Map<String, Object> validatedNumaMap) {
    List metaDatas = (List) supervisor.getiSupervisor().getMetadata();
    List<Long> allPortList = new ArrayList<>();
    if (metaDatas != null) {
        for (Object data : metaDatas) {
            Integer port = ObjectReader.getInt(data);
            if (port != null) {
                allPortList.add(port.longValue());
            }
        }
    }
    List<Long> allUsedPorts = new ArrayList<>();
    allUsedPorts.addAll(supervisor.getCurrAssignment().get().keySet());
    Map<String, Double> totalSupervisorResources = mkSupervisorCapacities(conf);
    NormalizedResourceOffer totalSupervisorNormalizedResources = new NormalizedResourceOffer(totalSupervisorResources);
    Map<String, SupervisorInfo> result = new HashMap();
    if (validatedNumaMap != null) {
        for (Map.Entry<String, Object> numaMapEntry : validatedNumaMap.entrySet()) {
            SupervisorInfo supervisorInfo = new SupervisorInfo();
            supervisorInfo.set_time_secs(Time.currentTimeSecs());
            supervisorInfo.set_hostname(supervisor.getHostName());
            supervisorInfo.set_assignment_id(supervisor.getAssignmentId() + ServerConstants.NUMA_ID_SEPARATOR + numaMapEntry.getKey());
            supervisorInfo.set_server_port(supervisor.getThriftServerPort());
            Map<String, Object> numaMap = (Map<String, Object>) numaMapEntry.getValue();
            List numaPortList = ((List<Integer>) numaMap.get(ServerConstants.NUMA_PORTS)).stream().map(e -> e.longValue()).collect(Collectors.toList());
            List<Long> usedNumaPorts = ListUtils.intersection(numaPortList, allUsedPorts);
            supervisorInfo.set_used_ports(usedNumaPorts);
            supervisorInfo.set_meta(numaPortList);
            allPortList = ListUtils.subtract(allPortList, numaPortList);
            allUsedPorts = ListUtils.subtract(allUsedPorts, usedNumaPorts);
            supervisorInfo.set_scheduler_meta((Map<String, String>) conf.get(DaemonConfig.SUPERVISOR_SCHEDULER_META));
            supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime());
            supervisorInfo.set_version(supervisor.getStormVersion());
            Map<String, Double> supervisorCapacitiesFromNumaMap = mkSupervisorCapacitiesFromNumaMap(numaMap);
            NormalizedResourceOffer numaNormalizedResources = new NormalizedResourceOffer(supervisorCapacitiesFromNumaMap);
            totalSupervisorNormalizedResources.remove(numaNormalizedResources);
            supervisorInfo.set_resources_map(supervisorCapacitiesFromNumaMap);
            result.put(supervisor.getId() + ServerConstants.NUMA_ID_SEPARATOR + numaMapEntry.getKey(), supervisorInfo);
        }
    }
    if (totalSupervisorNormalizedResources.getTotalCpu() > 0 && totalSupervisorNormalizedResources.getTotalMemoryMb() > 0 && !allPortList.isEmpty()) {
        SupervisorInfo supervisorInfo = new SupervisorInfo();
        supervisorInfo.set_time_secs(Time.currentTimeSecs());
        supervisorInfo.set_hostname(supervisor.getHostName());
        supervisorInfo.set_assignment_id(supervisor.getAssignmentId());
        supervisorInfo.set_server_port(supervisor.getThriftServerPort());
        supervisorInfo.set_used_ports(allUsedPorts);
        supervisorInfo.set_meta(allPortList);
        supervisorInfo.set_scheduler_meta((Map<String, String>) conf.get(DaemonConfig.SUPERVISOR_SCHEDULER_META));
        supervisorInfo.set_uptime_secs(supervisor.getUpTime().upTime());
        supervisorInfo.set_version(supervisor.getStormVersion());
        supervisorInfo.set_resources_map(totalSupervisorNormalizedResources.toNormalizedMap());
        result.put(supervisor.getId(), supervisorInfo);
    }
    return result;
}