Examples with ResourceContainer edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer used on opensource projects

Example 1 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class WordCountJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    BasicJob.BasicJobBuilder jobBuilder = BasicJob.newBuilder();
    jobBuilder.setName("streaming-wordcount");
    jobBuilder.setContainerClass(WordCountContainer.class.getName());
    jobBuilder.setRequestResource(new ResourceContainer(2, 1024), 4);
    jobBuilder.setConfig(jobConfig);
    // now submit the job
    Twister2Submitter.submitContainerJob(jobBuilder.build(), config);
}

Example 2 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class WordCountUtils method createWordCountPlan.

/**
 * Let assume we have 2 tasks per container and one additional for first container,
 * which will be the destination
 * @param plan the resource plan from scheduler
 * @return task plan
 */
public static TaskPlan createWordCountPlan(Config cfg, ResourcePlan plan, int noOfTasks) {
    int noOfProcs = plan.noOfContainers();
    LOG.log(Level.INFO, "No of containers: " + noOfProcs);
    Map<Integer, Set<Integer>> executorToGraphNodes = new HashMap<>();
    Map<Integer, Set<Integer>> groupsToExeuctors = new HashMap<>();
    int thisExecutor = plan.getThisId();
    List<ResourceContainer> containers = plan.getContainers();
    Map<String, List<ResourceContainer>> containersPerNode = new HashMap<>();
    for (ResourceContainer c : containers) {
        String name = (String) c.getProperty(SchedulerContext.WORKER_NAME);
        List<ResourceContainer> containerList;
        if (!containersPerNode.containsKey(name)) {
            containerList = new ArrayList<>();
            containersPerNode.put(name, containerList);
        } else {
            containerList = containersPerNode.get(name);
        }
        containerList.add(c);
    }
    int taskPerExecutor = noOfTasks / noOfProcs;
    for (int i = 0; i < noOfProcs; i++) {
        Set<Integer> nodesOfExecutor = new HashSet<>();
        for (int j = 0; j < taskPerExecutor; j++) {
            nodesOfExecutor.add(i * taskPerExecutor + j);
        }
        executorToGraphNodes.put(i, nodesOfExecutor);
    }
    int i = 0;
    // we take each container as an executor
    for (Map.Entry<String, List<ResourceContainer>> e : containersPerNode.entrySet()) {
        Set<Integer> executorsOfGroup = new HashSet<>();
        for (ResourceContainer c : e.getValue()) {
            executorsOfGroup.add(c.getId());
        }
        groupsToExeuctors.put(i, executorsOfGroup);
        i++;
    }
    LOG.fine(String.format("%d Executor To Graph: %s", plan.getThisId(), executorToGraphNodes));
    LOG.fine(String.format("%d Groups to executors: %s", plan.getThisId(), groupsToExeuctors));
    // and reduce task in 0th process
    return new TaskPlan(executorToGraphNodes, groupsToExeuctors, thisExecutor);
}

Example 3 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class MPIProcess method addContainers.

private static void addContainers(Config cfg, ResourcePlan resourcePlan, Map<Integer, String> processes) throws MPIException {
    int size = MPI.COMM_WORLD.getSize();
    for (int i = 0; i < size; i++) {
        ResourceContainer resourceContainer = new ResourceContainer(i);
        resourceContainer.addProperty(SchedulerContext.WORKER_NAME, processes.get(i));
        resourcePlan.addContainer(resourceContainer);
    }
}

Example 4 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class Utils method createReduceTaskPlan.

/**
 * Let assume we have 2 tasks per container and one additional for first container,
 * which will be the destination
 * @param plan the resource plan from scheduler
 * @return task plan
 */
public static TaskPlan createReduceTaskPlan(Config cfg, ResourcePlan plan, int noOfTasks) {
    int noOfProcs = plan.noOfContainers();
    LOG.log(Level.INFO, "No of containers: " + noOfProcs);
    Map<Integer, Set<Integer>> executorToGraphNodes = new HashMap<>();
    Map<Integer, Set<Integer>> groupsToExeuctors = new HashMap<>();
    int thisExecutor = plan.getThisId();
    List<ResourceContainer> containers = plan.getContainers();
    Map<String, List<ResourceContainer>> containersPerNode = new HashMap<>();
    for (ResourceContainer c : containers) {
        String name = (String) c.getProperty(SchedulerContext.WORKER_NAME);
        List<ResourceContainer> containerList;
        if (!containersPerNode.containsKey(name)) {
            containerList = new ArrayList<>();
            containersPerNode.put(name, containerList);
        } else {
            containerList = containersPerNode.get(name);
        }
        containerList.add(c);
    }
    int taskPerExecutor = noOfTasks / noOfProcs;
    for (int i = 0; i < noOfProcs; i++) {
        Set<Integer> nodesOfExecutor = new HashSet<>();
        for (int j = 0; j < taskPerExecutor; j++) {
            nodesOfExecutor.add(i * taskPerExecutor + j);
        }
        if (i == 0) {
            nodesOfExecutor.add(noOfTasks);
        }
        executorToGraphNodes.put(i, nodesOfExecutor);
    }
    int i = 0;
    // we take each container as an executor
    for (Map.Entry<String, List<ResourceContainer>> e : containersPerNode.entrySet()) {
        Set<Integer> executorsOfGroup = new HashSet<>();
        for (ResourceContainer c : e.getValue()) {
            executorsOfGroup.add(c.getId());
        }
        groupsToExeuctors.put(i, executorsOfGroup);
        i++;
    }
    String print = printMap(executorToGraphNodes);
    LOG.fine("Executor To Graph: " + print);
    print = printMap(groupsToExeuctors);
    LOG.fine("Groups to executors: " + print);
    // and reduce task in 0th process
    return new TaskPlan(executorToGraphNodes, groupsToExeuctors, thisExecutor);
}

Example 5 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class BasicGatherBatchTestJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    // build the job
    BasicJob basicJob = BasicJob.newBuilder().setName("basic-gather-batched").setContainerClass(BasicGatherBatchTestCommunication.class.getName()).setRequestResource(new ResourceContainer(2, 1024), 4).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitContainerJob(basicJob, config);
}