Examples with ComputeGraph edu.iu.dsc.tws.api.compute.graph.ComputeGraph used on opensource projects

Example 16 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method testUniqueSchedules.

@Test
public void testUniqueSchedules() {
    int parallel = 256;
    ComputeGraph graph = createGraph(parallel);
    RoundRobinTaskScheduler scheduler = new RoundRobinTaskScheduler();
    scheduler.initialize(Config.newBuilder().build());
    WorkerPlan workerPlan = createWorkPlan(parallel);
    for (int i = 0; i < 1; i++) {
        TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
        TaskSchedulePlan plan2 = scheduler.schedule(graph, workerPlan);
        Assert.assertEquals(plan1.getContainers().size(), plan2.getContainers().size());
        Map<Integer, WorkerSchedulePlan> map2 = plan2.getContainersMap();
        for (WorkerSchedulePlan workerSchedulePlan : plan1.getContainers()) {
            WorkerSchedulePlan p2 = map2.get(workerSchedulePlan.getContainerId());
            Assert.assertTrue(containerEquals(workerSchedulePlan, p2));
        }
    }
}

Example 17 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method createGraphWithComputeTaskAndConstraints.

private ComputeGraph createGraphWithComputeTaskAndConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute testCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", testCompute, parallel);
    ComputeConnection sinkConnection = builder.addCompute("sink", testSink, parallel);
    computeConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    sinkConnection.direct("compute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    builder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "2");
    ComputeGraph graph = builder.build();
    return graph;
}

Example 18 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method testUniqueSchedules1.

@Test
public void testUniqueSchedules1() {
    int parallel = 4;
    int workers = 2;
    ComputeGraph graph = createGraph(parallel);
    DataLocalityBatchTaskScheduler scheduler = new DataLocalityBatchTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    WorkerPlan workerPlan2 = createWorkPlan2(workers);
    for (int i = 0; i < 10; i++) {
        TaskSchedulePlan plan2 = scheduler.schedule(graph, workerPlan2);
        Assert.assertEquals(plan1.getContainers().size(), plan2.getContainers().size());
        Map<Integer, WorkerSchedulePlan> containersMap = plan2.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            Assert.assertEquals(containerPlanTaskInstances.size() / graph.getTaskVertexSet().size(), TaskSchedulerContext.defaultTaskInstancesPerContainer(config));
        }
    }
}

Example 19 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskSchedulerTest method createGraphWithMultipleComputeTaskAndConstraints.

private ComputeGraph createGraphWithMultipleComputeTaskAndConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute firstTestCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestComputeChild secondTestCompute = new TaskSchedulerClassTest.TestComputeChild();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection firstComputeConnection = builder.addCompute("firstcompute", firstTestCompute, parallel);
    ComputeConnection secondComputeConnection = builder.addCompute("secondcompute", secondTestCompute, parallel);
    ComputeConnection sinkConnection = builder.addCompute("sink", testSink, parallel);
    firstComputeConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    secondComputeConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    sinkConnection.allreduce("firstcompute").viaEdge("freduce").withReductionFunction(new TaskSchedulerClassTest.Aggregator()).withDataType(MessageTypes.OBJECT);
    sinkConnection.allreduce("secondcompute").viaEdge("sreduce").withReductionFunction(new TaskSchedulerClassTest.Aggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    builder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "2");
    ComputeGraph graph = builder.build();
    return graph;
}

Example 20 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataLocalityTaskSchedulerTest method testUniqueSchedules1.

@Test
public void testUniqueSchedules1() {
    int parallel = 2;
    int workers = 2;
    ComputeGraph graph = createGraph(parallel);
    DataLocalityStreamingTaskScheduler scheduler = new DataLocalityStreamingTaskScheduler();
    Config config = getConfig();
    scheduler.initialize(config, 1);
    generateData(config);
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    WorkerPlan workerPlan2 = createWorkPlan2(workers);
    for (int i = 0; i < 10; i++) {
        TaskSchedulePlan plan2 = scheduler.schedule(graph, workerPlan2);
        Assert.assertEquals(plan1.getContainers().size(), plan2.getContainers().size());
        Map<Integer, WorkerSchedulePlan> containersMap = plan2.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            Assert.assertEquals(containerPlanTaskInstances.size(), TaskSchedulerContext.defaultTaskInstancesPerContainer(config));
        }
    }
}