Examples with ComputeGraph edu.iu.dsc.tws.api.compute.graph.ComputeGraph used on opensource projects

Example 66 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class SingleSourceShortestPathWorker method execute.

@Override
public void execute() {
    SsspParameters ssspParameters = SsspParameters.build(config);
    int parallelismValue = ssspParameters.getParallelismValue();
    int dsize = ssspParameters.getDsize();
    String dataDirectory = ssspParameters.getDatapointDirectory();
    String soruceVertex = ssspParameters.getSourcevertex();
    sourceVertexGlobal = soruceVertex;
    /* First Graph to partition and read the partitioned adjacency list datas **/
    // Build the first taskgraph
    ComputeGraph datapointsTaskGraph = buildDataPointsTG(dataDirectory, dsize, parallelismValue, soruceVertex, config);
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(datapointsTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(datapointsTaskGraph, firstGraphExecutionPlan);
    // Retrieve the output of the first task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // Build the second taskgraph
    ComputeGraph graphInitialValueTaskGraph = buildSsspInitialTG(dataDirectory, dsize, parallelismValue, soruceVertex, config);
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(graphInitialValueTaskGraph);
    // Actual execution for the second taskgraph
    taskExecutor.execute(graphInitialValueTaskGraph, secondGraphExecutionPlan);
    // Retrieve the output of the second task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    /* Third Graph to do the actual calculation **/
    ComputeGraph sssptaskgraph = buildComputationSsspTG(parallelismValue, config);
    IExecutor ex = taskExecutor.createExecution(sssptaskgraph);
    int itr = 0;
    while (globaliterationStatus) {
        ex.execute(false);
        itr++;
    }
    ex.close();
    taskExecutor.close();
    if (workerId == 1) {
        System.out.println("Tatol iteration: " + itr);
    }
}

Example 67 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class TaskGraphBuildTest method createGraph.

private ComputeGraph createGraph() {
    TestSource testSource = new TestSource();
    TestSink1 testCompute = new TestSink1();
    TestSink2 testSink = new TestSink2();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(getConfig());
    computeGraphBuilder.addSource("source", testSource, 4);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("compute", testCompute, 4);
    computeConnection.partition("source").viaEdge(TaskConfigurations.DEFAULT_EDGE).withDataType(MessageTypes.OBJECT);
    ComputeConnection rc = computeGraphBuilder.addCompute("sink", testSink, 1);
    rc.allreduce("compute").viaEdge(TaskConfigurations.DEFAULT_EDGE).withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    ComputeGraph graph = computeGraphBuilder.build();
    return graph;
}

Example 68 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method testUniqueSchedules4.

@Test
public void testUniqueSchedules4() {
    int parallel = 16;
    int workers = 2;
    ComputeGraph graph = createGraphWithComputeTaskAndConstraints(parallel);
    RoundRobinTaskScheduler scheduler = new RoundRobinTaskScheduler();
    scheduler.initialize(Config.newBuilder().build());
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        Assert.assertEquals(containerPlanTaskInstances.size(), Integer.parseInt(graph.getGraphConstraints().get(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER)));
    }
}

Example 69 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method createGraphWithGraphConstraints.

private ComputeGraph createGraphWithGraphConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    builder.addSource("source", testSource, parallel);
    ComputeConnection c = builder.addCompute("sink", testSink, parallel);
    c.reduce("source").viaEdge("edge").withOperation(Op.SUM, MessageTypes.INTEGER_ARRAY);
    builder.setMode(OperationMode.STREAMING);
    builder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "16");
    ComputeGraph graph = builder.build();
    return graph;
}

Example 70 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class TaskScheduler method generateTaskSchedulePlan.

private TaskSchedulePlan generateTaskSchedulePlan(String className) {
    Class<?> taskSchedulerClass;
    Method method;
    TaskSchedulePlan taskSchedulePlan;
    try {
        taskSchedulerClass = getClass().getClassLoader().loadClass(className);
        Object newInstance = taskSchedulerClass.newInstance();
        method = taskSchedulerClass.getMethod("initialize", new Class<?>[] { Config.class });
        method.invoke(newInstance, config);
        method = taskSchedulerClass.getMethod("schedule", new Class<?>[] { ComputeGraph.class, WorkerPlan.class });
        taskSchedulePlan = (TaskSchedulePlan) method.invoke(newInstance, computeGraph, workerPlan);
    } catch (InvocationTargetException | IllegalAccessException | NoSuchMethodException | InstantiationException | ClassNotFoundException | TaskSchedulerException e) {
        throw new Twister2RuntimeException(e);
    }
    if (taskSchedulePlan != null) {
        Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
        for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
            Integer integer = entry.getKey();
            WorkerSchedulePlan workerSchedulePlan = entry.getValue();
            Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
            LOG.fine("Task Details for Container Id:" + integer);
            for (TaskInstancePlan ip : containerPlanTaskInstances) {
                LOG.fine("Task Id:" + ip.getTaskId() + "\tTask Index" + ip.getTaskIndex() + "\tTask Name:" + ip.getTaskName());
            }
        }
    }
    return taskSchedulePlan;
}