Examples with ResourceContainer edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer used on opensource projects

Example 21 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class MemoryManagerKeyedGatherJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    // build the job
    BasicJob basicJob = BasicJob.newBuilder().setName("basic-gather-batch-MM").setContainerClass(BasicMemoryManagerKeyedGatherCommunication.class.getName()).setRequestResource(new ResourceContainer(2, 1024), 4).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitContainerJob(basicJob, config);
}

Example 22 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class MultiTaskGraphJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    // build the job
    BasicJob basicJob = BasicJob.newBuilder().setName("multitaskgraph-example").setContainerClass(MultiTaskGraphExample.class.getName()).setRequestResource(new ResourceContainer(2, 1024), 4).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitContainerJob(basicJob, config);
}

Example 23 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class KMeansJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    BasicJob.BasicJobBuilder jobBuilder = BasicJob.newBuilder();
    jobBuilder.setName("basic-wordcount");
    jobBuilder.setContainerClass(KMeansContainer.class.getName());
    jobBuilder.setRequestResource(new ResourceContainer(2, 1024), 4);
    jobBuilder.setConfig(jobConfig);
    // now submit the job
    Twister2Submitter.submitContainerJob(jobBuilder.build(), config);
}

Example 24 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class KMeansUtils method createWordCountPlan.

/**
 * Let assume we have 2 tasks per container and one additional for first container,
 * which will be the destination
 * @param plan the resource plan from scheduler
 * @return task plan
 */
public static TaskPlan createWordCountPlan(Config cfg, ResourcePlan plan, int noOfTasks) {
    int noOfProcs = plan.noOfContainers();
    LOG.log(Level.INFO, "No of containers: " + noOfProcs);
    Map<Integer, Set<Integer>> executorToGraphNodes = new HashMap<>();
    Map<Integer, Set<Integer>> groupsToExeuctors = new HashMap<>();
    int thisExecutor = plan.getThisId();
    List<ResourceContainer> containers = plan.getContainers();
    Map<String, List<ResourceContainer>> containersPerNode = new HashMap<>();
    for (ResourceContainer c : containers) {
        String name = (String) c.getProperty(SchedulerContext.WORKER_NAME);
        List<ResourceContainer> containerList;
        if (!containersPerNode.containsKey(name)) {
            containerList = new ArrayList<>();
            containersPerNode.put(name, containerList);
        } else {
            containerList = containersPerNode.get(name);
        }
        containerList.add(c);
    }
    int taskPerExecutor = noOfTasks / noOfProcs;
    for (int i = 0; i < noOfProcs; i++) {
        Set<Integer> nodesOfExecutor = new HashSet<>();
        for (int j = 0; j < taskPerExecutor; j++) {
            nodesOfExecutor.add(i * taskPerExecutor + j);
        }
        executorToGraphNodes.put(i, nodesOfExecutor);
    }
    int i = 0;
    // we take each container as an executor
    for (Map.Entry<String, List<ResourceContainer>> e : containersPerNode.entrySet()) {
        Set<Integer> executorsOfGroup = new HashSet<>();
        for (ResourceContainer c : e.getValue()) {
            executorsOfGroup.add(c.getId());
        }
        groupsToExeuctors.put(i, executorsOfGroup);
        i++;
    }
    String print = printMap(executorToGraphNodes);
    LOG.fine("Executor To Graph: " + print);
    print = printMap(groupsToExeuctors);
    LOG.fine("Groups to executors: " + print);
    // and reduce task in 0th process
    return new TaskPlan(executorToGraphNodes, groupsToExeuctors, thisExecutor);
}

Example 25 with ResourceContainer

use of edu.iu.dsc.tws.rsched.spi.resource.ResourceContainer in project twister2 by DSC-SPIDAL.

the class BasicAuroraJob method main.

public static void main(String[] args) {
    // first load the configurations from command line and config files
    Config config = ResourceAllocator.loadConfig(new HashMap<>());
    System.out.println("read config values: " + config.size());
    System.out.println(config);
    int cpus = Integer.parseInt(AuroraContext.cpusPerContainer(config));
    int ramMegaBytes = AuroraContext.ramPerContainer(config) / (1024 * 1024);
    int diskMegaBytes = AuroraContext.diskPerContainer(config) / (1024 * 1024);
    int containers = Integer.parseInt(AuroraContext.numberOfContainers(config));
    String jobName = SchedulerContext.jobName(config);
    ResourceContainer resourceContainer = new ResourceContainer(cpus, ramMegaBytes, diskMegaBytes);
    // build JobConfig
    JobConfig jobConfig = new JobConfig();
    String containerClass = SchedulerContext.containerClass(config);
    // build the job
    BasicJob basicJob = BasicJob.newBuilder().setName(jobName).setContainerClass(containerClass).setRequestResource(resourceContainer, containers).setConfig(jobConfig).build();
    // now submit the job
    Twister2Submitter.submitContainerJob(basicJob, config);
    // now terminate the job
    terminateJob(config);
// jobWriteTest(basicJob);
// jobReadTest();
}