Examples with DataTransferTimeCalculator edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator used on opensource projects

Example 6 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method calculateDistance.

/**
 * It calculates the distance between the data nodes and the worker nodes.
 */
private Map<String, List<DataTransferTimeCalculator>> calculateDistance(List<String> datanodesList, WorkerPlan workerPlan, int taskIndex) {
    Map<String, List<DataTransferTimeCalculator>> workerPlanMap = new HashMap<>();
    Worker worker;
    double workerBandwidth;
    double workerLatency;
    double calculateDistance = 0.0;
    double datanodeBandwidth;
    double datanodeLatency;
    for (String nodesList : datanodesList) {
        ArrayList<DataTransferTimeCalculator> calculatedVal = new ArrayList<>();
        for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
            worker = workerPlan.getWorker(i);
            DataTransferTimeCalculator calculateDataTransferTime = new DataTransferTimeCalculator(nodesList, calculateDistance);
            if (worker.getProperty(Context.TWISTER2_BANDWIDTH) != null && worker.getProperty(Context.TWISTER2_LATENCY) != null) {
                workerBandwidth = (double) worker.getProperty(Context.TWISTER2_BANDWIDTH);
                workerLatency = (double) worker.getProperty(Context.TWISTER2_LATENCY);
            } else {
                workerBandwidth = TaskSchedulerContext.containerInstanceBandwidth(config);
                workerLatency = TaskSchedulerContext.containerInstanceLatency(config);
            }
            // Right now using the default configuration values
            datanodeBandwidth = TaskSchedulerContext.datanodeInstanceBandwidth(config);
            datanodeLatency = TaskSchedulerContext.datanodeInstanceLatency(config);
            // Calculate the distance between worker nodes and data nodes.
            calculateDistance = Math.abs((2 * workerBandwidth * workerLatency) - (2 * datanodeBandwidth * datanodeLatency));
            // (use this formula to calculate the data transfer time)
            // calculateDistance = File Size / Bandwidth;
            calculateDataTransferTime.setRequiredDataTransferTime(calculateDistance);
            calculateDataTransferTime.setNodeName(worker.getId() + "");
            calculateDataTransferTime.setTaskIndex(taskIndex);
            calculatedVal.add(calculateDataTransferTime);
        }
        workerPlanMap.put(nodesList, calculatedVal);
    }
    return workerPlanMap;
}

Example 7 with DataTransferTimeCalculator

use of edu.iu.dsc.tws.tsched.utils.DataTransferTimeCalculator in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method dTTimecalculatorList.

/**
 * Data transfer time calculator list
 */
private List<DataTransferTimeCalculator> dTTimecalculatorList(int index, WorkerPlan workerPlan, Map<Integer, List<TaskInstanceId>> map, int containerIndex, int maxTaskPerContainer) {
    List<String> inputDataList = getInputFilesList();
    Map<String, List<DataTransferTimeCalculator>> workerDatanodeDistanceMap;
    List<DataTransferTimeCalculator> dataTransferTimeCalculatorList = null;
    List<String> datanodesList;
    /*If the index is zero, simply calculate the distance between the worker node and the
    datanodes. Else, if the index values is greater than 0, check the container has reached
    the maximum task instances per container. If it is yes, then calculationList the container to
    the allocatedWorkers list which will not be considered for the next scheduling cycle.*/
    DataNodeLocatorUtils dataNodeLocatorUtils = new DataNodeLocatorUtils(config);
    if (inputDataList.size() > 0) {
        if (index == 0) {
            datanodesList = dataNodeLocatorUtils.findDataNodesLocation(inputDataList);
            workerDatanodeDistanceMap = distanceCalculator(datanodesList, workerPlan, index, allocatedWorkers);
            dataTransferTimeCalculatorList = findBestWorkerNode(workerDatanodeDistanceMap);
        } else {
            datanodesList = dataNodeLocatorUtils.findDataNodesLocation(inputDataList);
            Worker worker = workerPlan.getWorker(containerIndex);
            if (map.get(containerIndex).size() >= maxTaskPerContainer) {
                allocatedWorkers.add(worker.getId());
            }
            workerDatanodeDistanceMap = distanceCalculator(datanodesList, workerPlan, index, allocatedWorkers);
            dataTransferTimeCalculatorList = findBestWorkerNode(workerDatanodeDistanceMap);
        }
    }
    return dataTransferTimeCalculatorList;
}