Examples with Worker edu.iu.dsc.tws.api.compute.schedule.elements.Worker used on opensource projects

Example 11 with Worker

use of edu.iu.dsc.tws.api.compute.schedule.elements.Worker in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method calculateDistance.

/**
 * It calculates the distance between the data nodes and the worker nodes.
 */
private Map<String, List<DataTransferTimeCalculator>> calculateDistance(List<String> datanodesList, WorkerPlan workerPlan, int taskIndex) {
    Map<String, List<DataTransferTimeCalculator>> workerPlanMap = new HashMap<>();
    Worker worker;
    double workerBandwidth;
    double workerLatency;
    double calculateDistance = 0.0;
    double datanodeBandwidth;
    double datanodeLatency;
    for (String nodesList : datanodesList) {
        ArrayList<DataTransferTimeCalculator> calculatedVal = new ArrayList<>();
        for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
            worker = workerPlan.getWorker(i);
            DataTransferTimeCalculator calculateDataTransferTime = new DataTransferTimeCalculator(nodesList, calculateDistance);
            if (worker.getProperty(Context.TWISTER2_BANDWIDTH) != null && worker.getProperty(Context.TWISTER2_LATENCY) != null) {
                workerBandwidth = (double) worker.getProperty(Context.TWISTER2_BANDWIDTH);
                workerLatency = (double) worker.getProperty(Context.TWISTER2_LATENCY);
            } else {
                workerBandwidth = TaskSchedulerContext.containerInstanceBandwidth(config);
                workerLatency = TaskSchedulerContext.containerInstanceLatency(config);
            }
            // Right now using the default configuration values
            datanodeBandwidth = TaskSchedulerContext.datanodeInstanceBandwidth(config);
            datanodeLatency = TaskSchedulerContext.datanodeInstanceLatency(config);
            // Calculate the distance between worker nodes and data nodes.
            calculateDistance = Math.abs((2 * workerBandwidth * workerLatency) - (2 * datanodeBandwidth * datanodeLatency));
            // (use this formula to calculate the data transfer time)
            // calculateDistance = File Size / Bandwidth;
            calculateDataTransferTime.setRequiredDataTransferTime(calculateDistance);
            calculateDataTransferTime.setNodeName(worker.getId() + "");
            calculateDataTransferTime.setTaskIndex(taskIndex);
            calculatedVal.add(calculateDataTransferTime);
        }
        workerPlanMap.put(nodesList, calculatedVal);
    }
    return workerPlanMap;
}

Example 12 with Worker

use of edu.iu.dsc.tws.api.compute.schedule.elements.Worker in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method dTTimecalculatorList.

/**
 * Data transfer time calculator list
 */
private List<DataTransferTimeCalculator> dTTimecalculatorList(int index, WorkerPlan workerPlan, Map<Integer, List<TaskInstanceId>> map, int containerIndex, int maxTaskPerContainer) {
    List<String> inputDataList = getInputFilesList();
    Map<String, List<DataTransferTimeCalculator>> workerDatanodeDistanceMap;
    List<DataTransferTimeCalculator> dataTransferTimeCalculatorList = null;
    List<String> datanodesList;
    /*If the index is zero, simply calculate the distance between the worker node and the
    datanodes. Else, if the index values is greater than 0, check the container has reached
    the maximum task instances per container. If it is yes, then calculationList the container to
    the allocatedWorkers list which will not be considered for the next scheduling cycle.*/
    DataNodeLocatorUtils dataNodeLocatorUtils = new DataNodeLocatorUtils(config);
    if (inputDataList.size() > 0) {
        if (index == 0) {
            datanodesList = dataNodeLocatorUtils.findDataNodesLocation(inputDataList);
            workerDatanodeDistanceMap = distanceCalculator(datanodesList, workerPlan, index, allocatedWorkers);
            dataTransferTimeCalculatorList = findBestWorkerNode(workerDatanodeDistanceMap);
        } else {
            datanodesList = dataNodeLocatorUtils.findDataNodesLocation(inputDataList);
            Worker worker = workerPlan.getWorker(containerIndex);
            if (map.get(containerIndex).size() >= maxTaskPerContainer) {
                allocatedWorkers.add(worker.getId());
            }
            workerDatanodeDistanceMap = distanceCalculator(datanodesList, workerPlan, index, allocatedWorkers);
            dataTransferTimeCalculatorList = findBestWorkerNode(workerDatanodeDistanceMap);
        }
    }
    return dataTransferTimeCalculatorList;
}

Example 13 with Worker

use of edu.iu.dsc.tws.api.compute.schedule.elements.Worker in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method createWorkPlan2.

private WorkerPlan createWorkPlan2(int workers) {
    WorkerPlan plan = new WorkerPlan();
    for (int i = workers - 1; i >= 0; i--) {
        Worker w = new Worker(i);
        w.addProperty("bandwidth", 1000.0);
        w.addProperty("latency", 0.1);
        plan.addWorker(w);
    }
    return plan;
}

Example 14 with Worker

use of edu.iu.dsc.tws.api.compute.schedule.elements.Worker in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method createWorkPlan.

private WorkerPlan createWorkPlan(int workers) {
    WorkerPlan plan = new WorkerPlan();
    for (int i = 0; i < workers; i++) {
        Worker w = new Worker(i);
        w.addProperty("bandwidth", 1000.0);
        w.addProperty("latency", 0.1);
        plan.addWorker(w);
    }
    return plan;
}

Example 15 with Worker

use of edu.iu.dsc.tws.api.compute.schedule.elements.Worker in project twister2 by DSC-SPIDAL.

the class DataLocalityTaskSchedulerTest method createWorkPlan.

private WorkerPlan createWorkPlan(int workers) {
    WorkerPlan plan = new WorkerPlan();
    for (int i = 0; i < workers; i++) {
        Worker w = new Worker(i);
        w.addProperty("bandwidth", 1000.0);
        w.addProperty("latency", 0.1);
        plan.addWorker(w);
    }
    return plan;
}