Examples with DataTransferTimeCalculator edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator used on opensource projects

Example 1 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method findBestWorkerNode.

/**
 * This method finds the worker node which has better network parameters (bandwidth/latency)
 * or it will take lesser time for the data transfer if there is any.
 */
private static List<DataTransferTimeCalculator> findBestWorkerNode(Map<String, List<DataTransferTimeCalculator>> workerPlanMap) {
    List<DataTransferTimeCalculator> cal = new ArrayList<>();
    for (Map.Entry<String, List<DataTransferTimeCalculator>> entry : workerPlanMap.entrySet()) {
        String key = entry.getKey();
        List<DataTransferTimeCalculator> value = entry.getValue();
        for (DataTransferTimeCalculator aValue : value) {
            cal.add(new DataTransferTimeCalculator(aValue.getNodeName(), aValue.getRequiredDataTransferTime(), key));
        }
    }
    return cal;
}

Example 2 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method nonAttributeBasedAllocation.

private Map<Integer, List<TaskInstanceId>> nonAttributeBasedAllocation(Map<String, Integer> parallelTaskMap, WorkerPlan workerPlan) {
    List<DataTransferTimeCalculator> workerNodeList = getWorkerNodeList(workerPlan);
    int instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(config);
    int containerIndex = Integer.parseInt(workerNodeList.get(0).getNodeName());
    for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
        String task = e.getKey();
        int taskParallelism = e.getValue();
        for (int taskIndex = 0, maxTaskObject = 0; taskIndex < taskParallelism; taskIndex++) {
            dataLocalityAwareAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
            maxTaskObject++;
            if (maxTaskObject == instancesPerContainer) {
                ++containerIndex;
            }
        }
        containerIndex = 0;
        gTaskId++;
    }
    return dataLocalityAwareAllocation;
}

Example 3 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method attributeBasedAllocation.

private Map<Integer, List<TaskInstanceId>> attributeBasedAllocation(Map<String, Integer> parallelTaskMap, ComputeGraph graph, WorkerPlan workerPlan) {
    List<DataTransferTimeCalculator> workerNodeList = getWorkerNodeList(workerPlan);
    int containerIndex = Integer.parseInt(workerNodeList.get(0).getNodeName());
    int instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
        String task = e.getKey();
        int taskParallelism = e.getValue();
        for (int taskIndex = 0, maxTaskObject = 0; taskIndex < taskParallelism; taskIndex++) {
            dataLocalityAwareAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
            maxTaskObject++;
            if (maxTaskObject == instancesPerContainer) {
                ++containerIndex;
            }
        }
        containerIndex = 0;
        gTaskId++;
    }
    return dataLocalityAwareAllocation;
}

Example 4 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method dataLocalityStreamingSchedulingAlgorithm.

/**
 * This method is primarily responsible for generating the container and task instance map which
 * is based on the task graph, its configuration, and the allocated worker plan.
 */
private Map<Integer, List<TaskInstanceId>> dataLocalityStreamingSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers, WorkerPlan workerPlan) {
    TaskAttributes taskAttributes = new TaskAttributes();
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Maximum task instances can be accommodated to the container
    int instancesPerContainer;
    if (!graph.getGraphConstraints().isEmpty()) {
        instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    } else {
        instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(this.config);
    }
    // Total container capacity
    int containerCapacity = instancesPerContainer * numberOfContainers;
    int localIndex = 0;
    int containerIndex = 0;
    int totalInstances;
    // Total task instances in the taskgraph
    if (!graph.getNodeConstraints().isEmpty()) {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet, graph.getNodeConstraints());
    } else {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet);
    }
    // Map to hold the allocation of task instances into the containers/workers
    Map<Integer, List<TaskInstanceId>> dataAwareAllocationMap = new HashMap<>();
    // To check the containers can hold all the parallel task instances.
    if (containerCapacity >= totalInstances) {
        LOG.info("Task scheduling could be performed for the container capacity of " + containerCapacity + " and " + totalInstances + " task instances");
        for (int i = 0; i < numberOfContainers; i++) {
            dataAwareAllocationMap.put(i, new ArrayList<>());
        }
    } else {
        throw new TaskSchedulerException("Task scheduling couldn't be performed for the container " + "capacity of " + containerCapacity + " and " + totalInstances + " task instances");
    }
    // Parallel Task Map for the complete task graph
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    Map<String, Integer> parallelTaskMap;
    if (!graph.getNodeConstraints().isEmpty()) {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet, graph.getNodeConstraints());
    } else {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet);
    }
    /*This loop allocate the task instances to the respective container, before allocation
    it will check whether the container has reached maximum task instance size */
    for (Map.Entry<String, Integer> aTaskEntrySet : parallelTaskMap.entrySet()) {
        for (Vertex vertex : taskVertexSet) {
            if (aTaskEntrySet.getKey().equals(vertex.getName())) {
                int totalTaskInstances = vertex.getParallelism();
                int maxContainerTaskObjectSize = 0;
                List<DataTransferTimeCalculator> calList = dTTimecalculatorList(localIndex, workerPlan, dataAwareAllocationMap, containerIndex, instancesPerContainer);
                for (int i = 0; i < totalTaskInstances; i++) {
                    containerIndex = Integer.parseInt(Collections.min(calList).getNodeName().trim());
                    if (maxContainerTaskObjectSize < instancesPerContainer) {
                        dataAwareAllocationMap.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                        ++maxContainerTaskObjectSize;
                    } else {
                        throw new TaskSchedulerException("Task Scheduling couldn't be possible for the " + "present configuration, please check the number of workers, " + "maximum instances per worker");
                    }
                }
                globalTaskIndex++;
                localIndex++;
            }
        }
    }
    return dataAwareAllocationMap;
}

Example 5 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method findBestWorkerNode.

/**
 * This method chooses the data node which takes minimal data transfer time.
 *
 * @return List
 */
private static List<DataTransferTimeCalculator> findBestWorkerNode(Map<String, List<DataTransferTimeCalculator>> workerPlanMap) {
    List<DataTransferTimeCalculator> cal = new ArrayList<>();
    for (Map.Entry<String, List<DataTransferTimeCalculator>> entry : workerPlanMap.entrySet()) {
        String key = entry.getKey();
        List<DataTransferTimeCalculator> value = entry.getValue();
        cal.add(new DataTransferTimeCalculator(Collections.min(value).getNodeName(), Collections.min(value).getRequiredDataTransferTime(), key));
    }
    return cal;
}