Examples with ComputeEnvironment edu.iu.dsc.tws.task.ComputeEnvironment used on opensource projects

Example 1 with ComputeEnvironment

use of edu.iu.dsc.tws.task.ComputeEnvironment in project twister2 by DSC-SPIDAL.

the class TeraSort method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerID = workerEnv.getWorkerId();
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    Config config = workerEnv.getConfig();
    resultsRecorder = new BenchmarkResultsRecorder(config, workerID == 0);
    Timing.setDefaultTimingUnit(TimingUnit.MILLI_SECONDS);
    final String filePath = config.getStringValue(ARG_INPUT_FILE, null);
    final int keySize = config.getIntegerValue(ARG_KEY_SIZE, 10);
    final int valueSize = config.getIntegerValue(ARG_VALUE_SIZE, 90);
    // Sampling Graph : if file based only
    TaskPartitioner taskPartitioner;
    if (filePath != null) {
        ComputeGraphBuilder samplingGraph = ComputeGraphBuilder.newBuilder(config);
        samplingGraph.setMode(OperationMode.BATCH);
        Sampler samplerTask = new Sampler();
        samplingGraph.addSource(TASK_SAMPLER, samplerTask, config.getIntegerValue(ARG_TASKS_SOURCES, 4));
        SamplerReduce samplerReduce = new SamplerReduce();
        samplingGraph.addCompute(TASK_SAMPLER_REDUCE, samplerReduce, config.getIntegerValue(ARG_RESOURCE_INSTANCES, 4)).allreduce(TASK_SAMPLER).viaEdge(EDGE).withReductionFunction(byte[].class, (minMax1, minMax2) -> {
            byte[] min1 = Arrays.copyOfRange(minMax1, 0, keySize);
            byte[] max1 = Arrays.copyOfRange(minMax1, keySize, minMax1.length);
            byte[] min2 = Arrays.copyOfRange(minMax2, 0, keySize);
            byte[] max2 = Arrays.copyOfRange(minMax2, keySize, minMax2.length);
            byte[] newMinMax = new byte[keySize * 2];
            byte[] min = min1;
            byte[] max = max1;
            if (ByteArrayComparator.getInstance().compare(min1, min2) > 0) {
                min = min2;
            }
            if (ByteArrayComparator.getInstance().compare(max1, max2) < 0) {
                max = max2;
            }
            System.arraycopy(min, 0, newMinMax, 0, keySize);
            System.arraycopy(max, 0, newMinMax, keySize, keySize);
            return newMinMax;
        });
        ComputeGraph sampleGraphBuild = samplingGraph.build();
        ExecutionPlan sampleTaskPlan = cEnv.getTaskExecutor().plan(sampleGraphBuild);
        cEnv.getTaskExecutor().execute(sampleGraphBuild, sampleTaskPlan);
        DataObject<byte[]> output = cEnv.getTaskExecutor().getOutput("sample-reduce");
        LOG.info("Sample output received");
        taskPartitioner = new TaskPartitionerForSampledData(output.getPartitions()[0].getConsumer().next(), keySize);
    } else {
        taskPartitioner = new TaskPartitionerForRandom();
    }
    // Sort Graph
    ComputeGraphBuilder teraSortTaskGraph = ComputeGraphBuilder.newBuilder(config);
    teraSortTaskGraph.setMode(OperationMode.BATCH);
    BaseSource dataSource;
    if (filePath == null) {
        dataSource = new RandomDataSource();
    } else {
        dataSource = new FileDataSource();
    }
    teraSortTaskGraph.addSource(TASK_SOURCE, dataSource, config.getIntegerValue(ARG_TASKS_SOURCES, 4));
    Receiver receiver = new Receiver();
    KeyedGatherConfig keyedGatherConfig = teraSortTaskGraph.addCompute(TASK_RECV, receiver, config.getIntegerValue(ARG_TASKS_SINKS, 4)).keyedGather(TASK_SOURCE).viaEdge(EDGE).withDataType(MessageTypes.BYTE_ARRAY).withKeyType(MessageTypes.BYTE_ARRAY).withTaskPartitioner(taskPartitioner).useDisk(true).sortBatchByKey(ByteArrayComparator.getInstance()).groupBatchByKey(false);
    if (config.getBooleanValue(ARG_FIXED_SCHEMA, false)) {
        LOG.info("Using fixed schema feature with message size : " + (keySize + valueSize) + " and key size : " + keySize);
        keyedGatherConfig.withMessageSchema(MessageSchema.ofSize(keySize + valueSize, keySize));
    }
    ComputeGraph computeGraph = teraSortTaskGraph.build();
    ExecutionPlan executionPlan = cEnv.getTaskExecutor().plan(computeGraph);
    cEnv.getTaskExecutor().execute(computeGraph, executionPlan);
    cEnv.close();
    LOG.info("Finished Sorting...");
}

Example 2 with ComputeEnvironment

use of edu.iu.dsc.tws.task.ComputeEnvironment in project twister2 by DSC-SPIDAL.

the class CheckpointingTaskExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment computeEnvironment = ComputeEnvironment.init(workerEnv);
    ComputeGraphBuilder computeGraphBuilder = computeEnvironment.newTaskGraph(OperationMode.STREAMING);
    int parallelism = workerEnv.getConfig().getIntegerValue("parallelism", 1);
    computeGraphBuilder.addSource("source", new SourceTask(), parallelism);
    computeGraphBuilder.addCompute("compute", new ComputeTask(), parallelism).direct("source").viaEdge("so-c").withDataType(MessageTypes.INTEGER);
    computeGraphBuilder.addCompute("sink", new SinkTask(), parallelism).direct("compute").viaEdge("c-si").withDataType(MessageTypes.INTEGER);
    computeEnvironment.buildAndExecute(computeGraphBuilder);
    computeEnvironment.close();
}

Example 3 with ComputeEnvironment

use of edu.iu.dsc.tws.task.ComputeEnvironment in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    long startTime = System.currentTimeMillis();
    LOG.log(Level.FINE, "Task worker starting: " + workerId);
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    // Independent Graph and it has collector
    ComputeGraph firstGraph = buildFirstGraph(2, config);
    // Dependent Graph and it has collector
    ComputeGraph secondGraph = buildSecondGraph(4, config);
    // Dependent Graph and it has receptor to receive the input from second graph or first graph
    ComputeGraph thirdGraph = buildThirdGraph(4, config);
    ComputeGraph[] computeGraphs = new ComputeGraph[] { firstGraph, secondGraph, thirdGraph };
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(firstGraph);
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(secondGraph);
    // Get the execution plan for the third task graph
    ExecutionPlan thirdGraphExecutionPlan = taskExecutor.plan(thirdGraph);
    taskExecutor.execute(firstGraph, firstGraphExecutionPlan);
    taskExecutor.execute(secondGraph, secondGraphExecutionPlan);
    taskExecutor.execute(thirdGraph, thirdGraphExecutionPlan);
    // This is to test all the three graphs as dependent
    /*Map<String, ExecutionPlan> taskExecutionPlan = taskExecutor.plan(computeGraphs);
    for (Map.Entry<String, ExecutionPlan> planEntry : taskExecutionPlan.entrySet()) {
      String graphName = planEntry.getKey();
      if (graphName.equals(computeGraphs[0].getGraphName())) {
        taskExecutor.execute(computeGraphs[0], planEntry.getValue());
      } else if (graphName.equals(computeGraphs[1].getGraphName())) {
        taskExecutor.execute(computeGraphs[1], planEntry.getValue());
      } else {
        taskExecutor.execute(computeGraphs[2], planEntry.getValue());
      }
    }*/
    cEnv.close();
    long endTime = System.currentTimeMillis();
    LOG.info("Total Execution Time: " + (endTime - startTime));
}

Example 4 with ComputeEnvironment

use of edu.iu.dsc.tws.task.ComputeEnvironment in project twister2 by DSC-SPIDAL.

the class IterativeJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    IterativeSourceTask g = new IterativeSourceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder graphBuilder = ComputeGraphBuilder.newBuilder(config);
    graphBuilder.addSource("source", g, 4);
    ComputeConnection computeConnection = graphBuilder.addCompute("sink", r, 4);
    computeConnection.partition("source").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph graph = graphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    IExecutor ex = taskExecutor.createExecution(graph, plan);
    for (int i = 0; i < 10; i++) {
        LOG.info("Starting iteration: " + i);
        taskExecutor.addInput(graph, plan, "source", "input", new DataObjectImpl<>(config));
        // this is a blocking call
        ex.execute();
        DataObject<Object> dataSet = taskExecutor.getOutput(graph, plan, "sink");
        DataPartition<Object>[] values = dataSet.getPartitions();
    }
    ex.closeExecution();
}

Example 5 with ComputeEnvironment

use of edu.iu.dsc.tws.task.ComputeEnvironment in project twister2 by DSC-SPIDAL.

the class MultiStageGraph method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    GeneratorTask g = new GeneratorTask();
    ReduceTask rt = new ReduceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(workerEnv.getConfig());
    builder.addSource("source", g, 4);
    ComputeConnection pc = builder.addCompute("compute", r, 4);
    pc.partition("source").viaEdge("partition-edge").withDataType(MessageTypes.OBJECT);
    ComputeConnection rc = builder.addCompute("sink", rt, 1);
    rc.reduce("compute").viaEdge("compute-edge").withReductionFunction((object1, object2) -> object1);
    builder.setMode(OperationMode.STREAMING);
    ComputeGraph graph = builder.build();
    graph.setGraphName("MultiTaskGraph");
    cEnv.getTaskExecutor().execute(graph, cEnv.getTaskExecutor().plan(graph));
}