use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.
the class DataLocalityBatchTaskScheduler method findBestWorkerNode.
/**
* This method finds the worker node which has better network parameters (bandwidth/latency)
* or it will take lesser time for the data transfer if there is any.
*/
private static List<DataTransferTimeCalculator> findBestWorkerNode(Map<String, List<DataTransferTimeCalculator>> workerPlanMap) {
List<DataTransferTimeCalculator> cal = new ArrayList<>();
for (Map.Entry<String, List<DataTransferTimeCalculator>> entry : workerPlanMap.entrySet()) {
String key = entry.getKey();
List<DataTransferTimeCalculator> value = entry.getValue();
for (DataTransferTimeCalculator aValue : value) {
cal.add(new DataTransferTimeCalculator(aValue.getNodeName(), aValue.getRequiredDataTransferTime(), key));
}
}
return cal;
}
use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.
the class DataLocalityBatchTaskScheduler method nonAttributeBasedAllocation.
private Map<Integer, List<TaskInstanceId>> nonAttributeBasedAllocation(Map<String, Integer> parallelTaskMap, WorkerPlan workerPlan) {
List<DataTransferTimeCalculator> workerNodeList = getWorkerNodeList(workerPlan);
int instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(config);
int containerIndex = Integer.parseInt(workerNodeList.get(0).getNodeName());
for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
String task = e.getKey();
int taskParallelism = e.getValue();
for (int taskIndex = 0, maxTaskObject = 0; taskIndex < taskParallelism; taskIndex++) {
dataLocalityAwareAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
maxTaskObject++;
if (maxTaskObject == instancesPerContainer) {
++containerIndex;
}
}
containerIndex = 0;
gTaskId++;
}
return dataLocalityAwareAllocation;
}
use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.
the class DataLocalityBatchTaskScheduler method attributeBasedAllocation.
private Map<Integer, List<TaskInstanceId>> attributeBasedAllocation(Map<String, Integer> parallelTaskMap, ComputeGraph graph, WorkerPlan workerPlan) {
List<DataTransferTimeCalculator> workerNodeList = getWorkerNodeList(workerPlan);
int containerIndex = Integer.parseInt(workerNodeList.get(0).getNodeName());
int instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
for (Map.Entry<String, Integer> e : parallelTaskMap.entrySet()) {
String task = e.getKey();
int taskParallelism = e.getValue();
for (int taskIndex = 0, maxTaskObject = 0; taskIndex < taskParallelism; taskIndex++) {
dataLocalityAwareAllocation.get(containerIndex).add(new TaskInstanceId(task, gTaskId, taskIndex));
maxTaskObject++;
if (maxTaskObject == instancesPerContainer) {
++containerIndex;
}
}
containerIndex = 0;
gTaskId++;
}
return dataLocalityAwareAllocation;
}
use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.
the class DataLocalityStreamingTaskScheduler method dataLocalityStreamingSchedulingAlgorithm.
/**
* This method is primarily responsible for generating the container and task instance map which
* is based on the task graph, its configuration, and the allocated worker plan.
*/
private Map<Integer, List<TaskInstanceId>> dataLocalityStreamingSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers, WorkerPlan workerPlan) {
TaskAttributes taskAttributes = new TaskAttributes();
Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
// Maximum task instances can be accommodated to the container
int instancesPerContainer;
if (!graph.getGraphConstraints().isEmpty()) {
instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
} else {
instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(this.config);
}
// Total container capacity
int containerCapacity = instancesPerContainer * numberOfContainers;
int localIndex = 0;
int containerIndex = 0;
int totalInstances;
// Total task instances in the taskgraph
if (!graph.getNodeConstraints().isEmpty()) {
totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet, graph.getNodeConstraints());
} else {
totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet);
}
// Map to hold the allocation of task instances into the containers/workers
Map<Integer, List<TaskInstanceId>> dataAwareAllocationMap = new HashMap<>();
// To check the containers can hold all the parallel task instances.
if (containerCapacity >= totalInstances) {
LOG.info("Task scheduling could be performed for the container capacity of " + containerCapacity + " and " + totalInstances + " task instances");
for (int i = 0; i < numberOfContainers; i++) {
dataAwareAllocationMap.put(i, new ArrayList<>());
}
} else {
throw new TaskSchedulerException("Task scheduling couldn't be performed for the container " + "capacity of " + containerCapacity + " and " + totalInstances + " task instances");
}
// Parallel Task Map for the complete task graph
TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
orderedTaskSet.addAll(taskVertexSet);
Map<String, Integer> parallelTaskMap;
if (!graph.getNodeConstraints().isEmpty()) {
parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet, graph.getNodeConstraints());
} else {
parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet);
}
/*This loop allocate the task instances to the respective container, before allocation
it will check whether the container has reached maximum task instance size */
for (Map.Entry<String, Integer> aTaskEntrySet : parallelTaskMap.entrySet()) {
for (Vertex vertex : taskVertexSet) {
if (aTaskEntrySet.getKey().equals(vertex.getName())) {
int totalTaskInstances = vertex.getParallelism();
int maxContainerTaskObjectSize = 0;
List<DataTransferTimeCalculator> calList = dTTimecalculatorList(localIndex, workerPlan, dataAwareAllocationMap, containerIndex, instancesPerContainer);
for (int i = 0; i < totalTaskInstances; i++) {
containerIndex = Integer.parseInt(Collections.min(calList).getNodeName().trim());
if (maxContainerTaskObjectSize < instancesPerContainer) {
dataAwareAllocationMap.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
++maxContainerTaskObjectSize;
} else {
throw new TaskSchedulerException("Task Scheduling couldn't be possible for the " + "present configuration, please check the number of workers, " + "maximum instances per worker");
}
}
globalTaskIndex++;
localIndex++;
}
}
}
return dataAwareAllocationMap;
}
use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.
the class DataLocalityStreamingTaskScheduler method findBestWorkerNode.
/**
* This method chooses the data node which takes minimal data transfer time.
*
* @return List
*/
private static List<DataTransferTimeCalculator> findBestWorkerNode(Map<String, List<DataTransferTimeCalculator>> workerPlanMap) {
List<DataTransferTimeCalculator> cal = new ArrayList<>();
for (Map.Entry<String, List<DataTransferTimeCalculator>> entry : workerPlanMap.entrySet()) {
String key = entry.getKey();
List<DataTransferTimeCalculator> value = entry.getValue();
cal.add(new DataTransferTimeCalculator(Collections.min(value).getNodeName(), Collections.min(value).getRequiredDataTransferTime(), key));
}
return cal;
}
Aggregations