Examples with ComputeGraph edu.iu.dsc.tws.api.compute.graph.ComputeGraph used on opensource projects

Example 11 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class BasicComputation method execute.

@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    WorkerParameter workerParameter = WorkerParameter.build(config);
    parallelism = workerParameter.getParallelismValue();
    graphsize = workerParameter.getDsize();
    sourceVertexGlobal = workerParameter.getSourcevertex();
    dataDirectory = workerParameter.getDatapointDirectory();
    iterations = workerParameter.getIterations();
    /* First Graph to partition and read the partitioned data points **/
    ComputeGraph graphpartitionTaskGraph = graphpartition();
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan = taskExecutor.plan(graphpartitionTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(graphpartitionTaskGraph, executionPlan);
    // Retrieve the output of the first task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // the second task graph for assign initial pagerank values for vertex.
    ComputeGraph graphInitializationTaskGraph = graphInitialization();
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan1 = taskExecutor.plan(graphInitializationTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(graphInitializationTaskGraph, executionPlan1);
    // Retrieve the output of the first task graph
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // third task graph for computations
    ComputeGraph computationTaskgraph = computation();
    IExecutor ex = taskExecutor.createExecution(computationTaskgraph);
    int unlimitedItr = 0;
    long startime = System.currentTimeMillis();
    if (iterations != 0) {
        for (int i = 0; i < iterations; i++) {
            ex.execute(i == iterations - 1);
        }
    } else {
        while (globaliterationStatus) {
            ex.execute(false);
            unlimitedItr++;
        }
        ex.closeExecution();
    }
    taskExecutor.close();
    long endTime = System.currentTimeMillis();
    if (workerId == 0) {
        System.out.println("total number of iterations : " + unlimitedItr);
        System.out.println("computation time: " + (endTime - startime));
    }
}

Example 12 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class PageRankWorker method execute.

// private static double danglingNodeValues;
@Override
public void execute() {
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    PageRankWorkerParameters pageRankWorkerParameters = PageRankWorkerParameters.build(config);
    int parallelismValue = pageRankWorkerParameters.getParallelismValue();
    int dsize = pageRankWorkerParameters.getDsize();
    String dataDirectory = pageRankWorkerParameters.getDatapointDirectory();
    int iterations = pageRankWorkerParameters.getIterations();
    graphsize = dsize;
    /* First Graph to partition and read the partitioned data points **/
    ComputeGraph datapointsTaskGraph = buildDataPointsTG(dataDirectory, dsize, parallelismValue, config);
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan = taskExecutor.plan(datapointsTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(datapointsTaskGraph, executionPlan);
    /* the out of the first graph would like below
    * task Id: 0
    {1=[3, 4], 2=[3, 4, 5]}*/
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // the second task graph for assign initial pagerank values for vertex.
    ComputeGraph graphInitialValueTaskGraph = buildGraphInitialValueTG(dataDirectory, dsize, parallelismValue, config);
    // Get the execution plan for the first task graph
    ExecutionPlan executionPlan1 = taskExecutor.plan(graphInitialValueTaskGraph);
    // Actual execution for the first taskgraph
    taskExecutor.execute(graphInitialValueTaskGraph, executionPlan1);
    /* the output of second graph should like below
      initiate the pagerank value
    * {1=0.25, 2=0.25}
     */
    // +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // third task graph for computations
    ComputeGraph pageranktaskgraph = buildComputationTG(parallelismValue, config);
    IExecutor ex = taskExecutor.createExecution(pageranktaskgraph);
    // Perform the iterations from 0 to 'n' number of iterations
    long startime = System.currentTimeMillis();
    for (int i = 0; i < iterations; i++) {
        ex.execute(i == iterations - 1);
    }
    taskExecutor.close();
    long endTime = System.currentTimeMillis();
    if (workerId == 0) {
        DataObject<Object> graphInitialPagerankValue = taskExecutor.getOutput("InitialValue");
        DataPartition<?> finaloutput = graphInitialPagerankValue.getPartition(workerId);
        HashMap<String, Double> finalone = (HashMap<String, Double>) finaloutput.getConsumer().next();
        System.out.println(finalone);
        LOG.info("Final output After " + iterations + "iterations ");
        Iterator it = finalone.entrySet().iterator();
        Double recivedFinalDanglingValue = finalone.get("danglingvalues");
        double cummulativepagerankvalue = 0.0;
        int num = 0;
        System.out.println(graphsize);
        while (it.hasNext()) {
            Map.Entry pair = (Map.Entry) it.next();
            if (!pair.getKey().equals("danglingvalues")) {
                double finalPagerankValue = (double) pair.getValue() + ((0.85 * recivedFinalDanglingValue) / graphsize);
                System.out.print("Vertex Id: " + pair.getKey());
                System.out.printf(" and it's pagerank value: %.15f \n", finalPagerankValue);
                cummulativepagerankvalue += finalPagerankValue;
                num += 1;
            }
            // avoids a ConcurrentModificationException
            it.remove();
        }
        System.out.println(recivedFinalDanglingValue);
        System.out.println(num);
        System.out.println(cummulativepagerankvalue);
        System.out.println(cummulativepagerankvalue + ((graphsize - num) * ((((double) 1 / graphsize) * 0.15) + (0.85 * (recivedFinalDanglingValue / graphsize)))));
        System.out.println("computation time: " + (endTime - startime));
    }
}

Example 13 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method testUniqueSchedules3.

@Test
public void testUniqueSchedules3() {
    int parallel = 16;
    int workers = 2;
    ComputeGraph graph = createGraphWithGraphConstraints(parallel);
    RoundRobinTaskScheduler scheduler = new RoundRobinTaskScheduler();
    scheduler.initialize(Config.newBuilder().build());
    WorkerPlan workerPlan = createWorkPlan(workers);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    Map<Integer, WorkerSchedulePlan> containersMap = plan1.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        Assert.assertEquals(containerPlanTaskInstances.size(), Integer.parseInt(graph.getGraphConstraints().get(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER)));
    }
}

Example 14 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method testUniqueSchedules2.

@Test
public void testUniqueSchedules2() {
    int parallel = 256;
    ComputeGraph graph = createGraph(parallel);
    RoundRobinTaskScheduler scheduler = new RoundRobinTaskScheduler();
    scheduler.initialize(Config.newBuilder().build());
    WorkerPlan workerPlan = createWorkPlan(parallel);
    TaskSchedulePlan plan1 = scheduler.schedule(graph, workerPlan);
    WorkerPlan workerPlan2 = createWorkPlan2(parallel);
    for (int i = 0; i < 1000; i++) {
        TaskSchedulePlan plan2 = scheduler.schedule(graph, workerPlan2);
        Assert.assertEquals(plan1.getContainers().size(), plan2.getContainers().size());
        Map<Integer, WorkerSchedulePlan> map2 = plan2.getContainersMap();
        for (WorkerSchedulePlan workerSchedulePlan : plan1.getContainers()) {
            WorkerSchedulePlan p2 = map2.get(workerSchedulePlan.getContainerId());
            Assert.assertTrue(containerEquals(workerSchedulePlan, p2));
        }
    }
}

Example 15 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class RoundRobinTaskSchedulerTest method createGraphWithComputeTaskAndConstraints.

private ComputeGraph createGraphWithComputeTaskAndConstraints(int parallel) {
    TaskSchedulerClassTest.TestSource testSource = new TaskSchedulerClassTest.TestSource();
    TaskSchedulerClassTest.TestCompute testCompute = new TaskSchedulerClassTest.TestCompute();
    TaskSchedulerClassTest.TestSink testSink = new TaskSchedulerClassTest.TestSink();
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(Config.newBuilder().build());
    computeGraphBuilder.addSource("source", testSource, parallel);
    ComputeConnection computeConnection = computeGraphBuilder.addCompute("compute", testCompute, parallel);
    ComputeConnection sinkComputeConnection = computeGraphBuilder.addCompute("sink", testSink, parallel);
    computeConnection.direct("source").viaEdge("cdirect-edge").withDataType(MessageTypes.OBJECT);
    sinkComputeConnection.direct("compute").viaEdge("sdirect-edge").withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.STREAMING);
    computeGraphBuilder.addGraphConstraints(Context.TWISTER2_MAX_TASK_INSTANCES_PER_WORKER, "24");
    ComputeGraph taskGraph = computeGraphBuilder.build();
    return taskGraph;
}