Examples with TaskSchedulerException edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException used on opensource projects

Example 1 with TaskSchedulerException

use of edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method getInputFilesList.

private List<String> getInputFilesList() {
    List<String> inputDataList = new ArrayList<>();
    String directory = null;
    if (config.get(DataObjectConstants.DINPUT_DIRECTORY) != null) {
        directory = String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY));
    }
    final Path path = new Path(directory);
    final FileSystem fileSystem;
    try {
        fileSystem = FileSystemUtils.get(path);
        if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_HDFS_FILESYSTEM)) {
            final FileStatus pathFile = fileSystem.getFileStatus(path);
            inputDataList.add(String.valueOf(pathFile.getPath()));
        } else if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_LOCAL_FILESYSTEM)) {
            for (FileStatus file : fileSystem.listFiles(path)) {
                String filename = String.valueOf(file.getPath());
                if (filename != null) {
                    inputDataList.add(filename);
                }
            }
        }
    } catch (IOException e) {
        throw new TaskSchedulerException("Not able to get the input files", e);
    }
    return inputDataList;
}

Example 2 with TaskSchedulerException

use of edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method dataLocalityStreamingSchedulingAlgorithm.

/**
 * This method is primarily responsible for generating the container and task instance map which
 * is based on the task graph, its configuration, and the allocated worker plan.
 */
private Map<Integer, List<TaskInstanceId>> dataLocalityStreamingSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers, WorkerPlan workerPlan) {
    TaskAttributes taskAttributes = new TaskAttributes();
    Set<Vertex> taskVertexSet = graph.getTaskVertexSet();
    // Maximum task instances can be accommodated to the container
    int instancesPerContainer;
    if (!graph.getGraphConstraints().isEmpty()) {
        instancesPerContainer = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
    } else {
        instancesPerContainer = TaskSchedulerContext.defaultTaskInstancesPerContainer(this.config);
    }
    // Total container capacity
    int containerCapacity = instancesPerContainer * numberOfContainers;
    int localIndex = 0;
    int containerIndex = 0;
    int totalInstances;
    // Total task instances in the taskgraph
    if (!graph.getNodeConstraints().isEmpty()) {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet, graph.getNodeConstraints());
    } else {
        totalInstances = taskAttributes.getTotalNumberOfInstances(taskVertexSet);
    }
    // Map to hold the allocation of task instances into the containers/workers
    Map<Integer, List<TaskInstanceId>> dataAwareAllocationMap = new HashMap<>();
    // To check the containers can hold all the parallel task instances.
    if (containerCapacity >= totalInstances) {
        LOG.info("Task scheduling could be performed for the container capacity of " + containerCapacity + " and " + totalInstances + " task instances");
        for (int i = 0; i < numberOfContainers; i++) {
            dataAwareAllocationMap.put(i, new ArrayList<>());
        }
    } else {
        throw new TaskSchedulerException("Task scheduling couldn't be performed for the container " + "capacity of " + containerCapacity + " and " + totalInstances + " task instances");
    }
    // Parallel Task Map for the complete task graph
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    Map<String, Integer> parallelTaskMap;
    if (!graph.getNodeConstraints().isEmpty()) {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet, graph.getNodeConstraints());
    } else {
        parallelTaskMap = taskAttributes.getParallelTaskMap(taskVertexSet);
    }
    /*This loop allocate the task instances to the respective container, before allocation
    it will check whether the container has reached maximum task instance size */
    for (Map.Entry<String, Integer> aTaskEntrySet : parallelTaskMap.entrySet()) {
        for (Vertex vertex : taskVertexSet) {
            if (aTaskEntrySet.getKey().equals(vertex.getName())) {
                int totalTaskInstances = vertex.getParallelism();
                int maxContainerTaskObjectSize = 0;
                List<DataTransferTimeCalculator> calList = dTTimecalculatorList(localIndex, workerPlan, dataAwareAllocationMap, containerIndex, instancesPerContainer);
                for (int i = 0; i < totalTaskInstances; i++) {
                    containerIndex = Integer.parseInt(Collections.min(calList).getNodeName().trim());
                    if (maxContainerTaskObjectSize < instancesPerContainer) {
                        dataAwareAllocationMap.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                        ++maxContainerTaskObjectSize;
                    } else {
                        throw new TaskSchedulerException("Task Scheduling couldn't be possible for the " + "present configuration, please check the number of workers, " + "maximum instances per worker");
                    }
                }
                globalTaskIndex++;
                localIndex++;
            }
        }
    }
    return dataAwareAllocationMap;
}

Example 3 with TaskSchedulerException

use of edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException in project twister2 by DSC-SPIDAL.

the class UserDefinedTaskScheduler method userDefinedSchedulingAlgorithm.

/**
 * This method retrieves the parallel task map and the total number of task instances for the task
 * vertex set. Then, it will allocate the instances into the number of containers allocated for
 * the task in a round robin fashion.
 *
 * The user could write their own type of allocations into the available workers using their
 * own scheduling algorithm.
 */
private static Map<Integer, List<TaskInstanceId>> userDefinedSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) {
    Map<Integer, List<TaskInstanceId>> userDefinedAllocation = new LinkedHashMap<>();
    for (int i = 0; i < numberOfContainers; i++) {
        userDefinedAllocation.put(i, new ArrayList<>());
    }
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    TaskAttributes taskAttributes = new TaskAttributes();
    int globalTaskIndex = 0;
    for (Vertex vertex : taskVertexSet) {
        int totalTaskInstances;
        if (!graph.getNodeConstraints().isEmpty()) {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex, graph.getNodeConstraints());
        } else {
            totalTaskInstances = taskAttributes.getTotalNumberOfInstances(vertex);
        }
        if (!graph.getNodeConstraints().isEmpty()) {
            int instancesPerWorker = taskAttributes.getInstancesPerWorker(graph.getGraphConstraints());
            int maxTaskInstancesPerContainer = 0;
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                if (maxTaskInstancesPerContainer < instancesPerWorker) {
                    userDefinedAllocation.get(containerIndex).add(new TaskInstanceId(vertex.getName(), globalTaskIndex, i));
                    ++maxTaskInstancesPerContainer;
                } else {
                    throw new TaskSchedulerException("Task Scheduling couldn't be possible for the present" + "configuration, please check the number of workers, " + "maximum instances per worker");
                }
            }
        } else {
            String task = vertex.getName();
            int containerIndex;
            for (int i = 0; i < totalTaskInstances; i++) {
                containerIndex = i % numberOfContainers;
                userDefinedAllocation.get(containerIndex).add(new TaskInstanceId(task, globalTaskIndex, i));
            }
        }
        globalTaskIndex++;
    }
    return userDefinedAllocation;
}

Example 4 with TaskSchedulerException

use of edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method addInstance.

public TaskSchedulePlanBuilder addInstance(Integer containerId, String taskName) throws TaskSchedulerException {
    initContainer(containerId);
    int taskId = taskIds.isEmpty() ? 1 : taskIds.last() + 1;
    int taskIndex = taskIndexes.get(taskName) != null ? taskIndexes.get(taskName).last() + 1 : 0;
    TaskInstanceId taskInstanceId = new TaskInstanceId(taskName, taskId, taskIndex);
    Resource resource = TaskScheduleUtils.getResourceRequirement(taskName, this.taskRamMap, this.instanceDefaultResourceValue, this.containerMaximumResourceValue, this.requestedContainerPadding);
    try {
        addToContainer(containers.get(containerId), new TaskInstancePlan(taskInstanceId, resource), taskIndexes, taskIds);
    } catch (TaskSchedulerException e) {
        throw new TaskSchedulerException(String.format("Insufficient container resources to add instance %s with resources %s to container %d.", taskInstanceId, resource, containerId), e);
    }
    LOG.info("Task id, index, name:" + taskId + "\t" + taskIndex + "\t" + taskName + "\tadded to Container:" + containers.get(containerId));
    return this;
}

Example 5 with TaskSchedulerException

use of edu.iu.dsc.tws.api.compute.exceptions.TaskSchedulerException in project twister2 by DSC-SPIDAL.

the class TaskSchedulePlanBuilder method initContainers.

/**
 * This method first initialize the container map values, task index values, and task id sets.
 */
private void initContainers() {
    assertResourceSettings();
    Map<Integer, Container> containerMap = this.containers;
    HashMap<String, TreeSet<Integer>> taskindexes = this.taskIndexes;
    TreeSet<Integer> taskids = this.taskIds;
    if (taskindexes == null) {
        taskindexes = new HashMap<>();
    }
    if (taskids == null) {
        taskids = new TreeSet<>();
    }
    if (containerMap == null) {
        if (this.previousTaskSchedulePlan == null) {
            containerMap = new HashMap<>();
            for (int containerId = 1; containerId <= numberOfContainers; containerId++) {
                containerMap.put(containerId, new Container(containerId, this.containerMaximumResourceValue, this.requestedContainerPadding));
            }
        } else {
            try {
                // containerMap = getContainers(this.previousTaskSchedulePlan);
                containerMap = getContainers(this.previousTaskSchedulePlan, this.requestedContainerPadding, taskindexes, taskids);
            } catch (TaskSchedulerException e) {
                throw new TaskSchedulerException("Could not initialize containers using existing packing plan", e);
            }
        }
    }
    if (this.numberOfContainers > containerMap.size()) {
        List<Scorer<Container>> scorers = new ArrayList<>();
        scorers.add(new ContainerIdScorer());
        List<Container> sortedContainers = sortContainers(scorers, containerMap.values());
        int nextContainerId = sortedContainers.get(sortedContainers.size() - 1).getContainerId() + 1;
        Resource capacity = containerMap.get(sortedContainers.get(0).getContainerId()).getResource();
        for (int i = 0; i < numberOfContainers - containerMap.size(); i++) {
            containerMap.put(nextContainerId, new Container(nextContainerId, capacity, this.requestedContainerPadding));
            nextContainerId++;
        }
    }
    this.taskIds = taskids;
    this.taskIndexes = taskindexes;
    this.containers = containerMap;
}