Examples with ComputeGraph edu.iu.dsc.tws.api.compute.graph.ComputeGraph used on opensource projects

Example 26 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerTest method createGraphWithDifferentParallelismInput.

private ComputeGraph createGraphWithDifferentParallelismInput(int parallel, String graphName, String... inputKey) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource(inputKey[0]);
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink(inputKey[0]);
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection sinkConnection = builder.addCompute("sink", testSink, parallel);
    sinkConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    graph.setGraphName(graphName);
    return graph;
}

Example 27 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerTest method testUniqueSchedules4.

@Test
public void testUniqueSchedules4() {
    int parallel = 400;
    int workers = 200;
    ComputeGraph[] graph = new ComputeGraph[3];
    Arrays.setAll(graph, i -> createGraph(parallel, "graph" + i));
    BatchTaskScheduler scheduler = new BatchTaskScheduler();
    scheduler.initialize(Config.newBuilder().build());
    WorkerPlan workerPlan = createWorkPlan(workers);
    Map<String, TaskSchedulePlan> taskSchedulePlanMap = scheduler.schedule(workerPlan, graph[0], graph[1], graph[2]);
    for (Map.Entry<String, TaskSchedulePlan> taskSchedulePlanEntry : taskSchedulePlanMap.entrySet()) {
        TaskSchedulePlan taskSchedulePlan = taskSchedulePlanEntry.getValue();
        Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            Assert.assertEquals(containerPlanTaskInstances.size(), (parallel / workers) * graph[0].getTaskVertexSet().size());
        }
    }
}

Example 28 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerTest method createGraphWithComputeTaskAndConstraints.

private ComputeGraph createGraphWithComputeTaskAndConstraints(int parallel, String graphName) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute testCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection computeConnection = builder.addCompute("compute", testCompute, parallel);
    ComputeConnection sinkConnection = builder.addCompute("sink", testSink, parallel);
    computeConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    sinkConnection.direct("compute").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    builder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "8");
    ComputeGraph graph = builder.build();
    graph.setGraphName(graphName);
    return graph;
}

Example 29 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinBatchTaskSchedulerTest method createGraph.

private ComputeGraph createGraph(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection sinkConnection = builder.addCompute("sink", testSink, parallel);
    sinkConnection.direct("source").viaEdge(Context.TWISTER2_DIRECT_EDGE).withDataType(MessageTypes.OBJECT);
    builder.setMode(OperationMode.BATCH);
    ComputeGraph graph = builder.build();
    return graph;
}

Example 30 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class Twister2StormWorker method execute.

@Override
public void execute() {
    ComputeGraph graph = this.buildTopology().getT2ComputeGraph();
    ExecutionPlan executionPlan = taskExecutor.plan(graph);
    taskExecutor.execute(graph, executionPlan);
}