Examples with TaskExecutor edu.iu.dsc.tws.task.impl.TaskExecutor used on opensource projects

Example 1 with TaskExecutor

use of edu.iu.dsc.tws.task.impl.TaskExecutor in project twister2 by DSC-SPIDAL.

the class BatchTaskSchedulerExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    long startTime = System.currentTimeMillis();
    LOG.log(Level.FINE, "Task worker starting: " + workerId);
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    // Independent Graph and it has collector
    ComputeGraph firstGraph = buildFirstGraph(2, config);
    // Dependent Graph and it has collector
    ComputeGraph secondGraph = buildSecondGraph(4, config);
    // Dependent Graph and it has receptor to receive the input from second graph or first graph
    ComputeGraph thirdGraph = buildThirdGraph(4, config);
    ComputeGraph[] computeGraphs = new ComputeGraph[] { firstGraph, secondGraph, thirdGraph };
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(firstGraph);
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(secondGraph);
    // Get the execution plan for the third task graph
    ExecutionPlan thirdGraphExecutionPlan = taskExecutor.plan(thirdGraph);
    taskExecutor.execute(firstGraph, firstGraphExecutionPlan);
    taskExecutor.execute(secondGraph, secondGraphExecutionPlan);
    taskExecutor.execute(thirdGraph, thirdGraphExecutionPlan);
    // This is to test all the three graphs as dependent
    /*Map<String, ExecutionPlan> taskExecutionPlan = taskExecutor.plan(computeGraphs);
    for (Map.Entry<String, ExecutionPlan> planEntry : taskExecutionPlan.entrySet()) {
      String graphName = planEntry.getKey();
      if (graphName.equals(computeGraphs[0].getGraphName())) {
        taskExecutor.execute(computeGraphs[0], planEntry.getValue());
      } else if (graphName.equals(computeGraphs[1].getGraphName())) {
        taskExecutor.execute(computeGraphs[1], planEntry.getValue());
      } else {
        taskExecutor.execute(computeGraphs[2], planEntry.getValue());
      }
    }*/
    cEnv.close();
    long endTime = System.currentTimeMillis();
    LOG.info("Total Execution Time: " + (endTime - startTime));
}

Example 2 with TaskExecutor

use of edu.iu.dsc.tws.task.impl.TaskExecutor in project twister2 by DSC-SPIDAL.

the class CDFWRuntime method execute.

/**
 * execute
 */
public boolean execute() {
    Any msg;
    while (true) {
        msg = executeMessageQueue.peek();
        if (msg == null) {
            if (scaleUpRequest.get()) {
                communicator.close();
                List<JobMasterAPI.WorkerInfo> workerInfoList = initSynch(controller);
                // create the channel
                LOG.info("Existing workers calling barrier");
                channel = Network.initializeChannel(config, controller);
                String persistent = null;
                // create the communicator
                communicator = new Communicator(config, channel, persistent);
                taskExecutor = new TaskExecutor(config, workerId, workerInfoList, communicator, null);
            }
            scaleUpRequest.set(false);
            // scaleDownRequest.set(false);
            continue;
        }
        msg = executeMessageQueue.poll();
        if (msg.is(CDFWJobAPI.ExecuteMessage.class)) {
            if (handleExecuteMessage(msg)) {
                return false;
            }
        } else if (msg.is(CDFWJobAPI.CDFWJobCompletedMessage.class)) {
            LOG.log(Level.INFO, workerId + "Received CDFW job completed message. Leaving execution " + "loop");
            break;
        }
    }
    LOG.log(Level.INFO, workerId + " Execution Completed");
    return true;
}

Example 3 with TaskExecutor

use of edu.iu.dsc.tws.task.impl.TaskExecutor in project twister2 by DSC-SPIDAL.

the class CDFWRuntime method reinitialize.

private boolean reinitialize() {
    communicator.close();
    List<JobMasterAPI.WorkerInfo> workerInfoList = null;
    try {
        workerInfoList = controller.getAllWorkers();
    } catch (TimeoutException timeoutException) {
        LOG.log(Level.SEVERE, timeoutException.getMessage(), timeoutException);
    }
    // create the channel
    channel = Network.initializeChannel(config, controller);
    String persistent = null;
    // create the communicator
    communicator = new Communicator(config, channel, persistent);
    taskExecutor = new TaskExecutor(config, workerId, workerInfoList, communicator, null);
    return true;
}

Example 4 with TaskExecutor

use of edu.iu.dsc.tws.task.impl.TaskExecutor in project twister2 by DSC-SPIDAL.

the class IterativeJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    IterativeSourceTask g = new IterativeSourceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder graphBuilder = ComputeGraphBuilder.newBuilder(config);
    graphBuilder.addSource("source", g, 4);
    ComputeConnection computeConnection = graphBuilder.addCompute("sink", r, 4);
    computeConnection.partition("source").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph graph = graphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    IExecutor ex = taskExecutor.createExecution(graph, plan);
    for (int i = 0; i < 10; i++) {
        LOG.info("Starting iteration: " + i);
        taskExecutor.addInput(graph, plan, "source", "input", new DataObjectImpl<>(config));
        // this is a blocking call
        ex.execute();
        DataObject<Object> dataSet = taskExecutor.getOutput(graph, plan, "sink");
        DataPartition<Object>[] values = dataSet.getPartitions();
    }
    ex.closeExecution();
}

Example 5 with TaskExecutor

use of edu.iu.dsc.tws.task.impl.TaskExecutor in project twister2 by DSC-SPIDAL.

the class ConstraintTaskExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    long startTime = System.currentTimeMillis();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    String dinput = String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY));
    int dimension = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.DIMENSIONS)));
    int parallelismValue = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.PARALLELISM_VALUE)));
    int dsize = Integer.parseInt(String.valueOf(config.get(DataObjectConstants.DSIZE)));
    DataGenerator dataGenerator = new DataGenerator(config, workerId);
    dataGenerator.generate(new Path(dinput), dsize, dimension);
    ComputeGraph firstGraph = buildFirstGraph(parallelismValue, config, dinput, dsize, dimension, "firstgraphpoints", "1");
    ComputeGraph secondGraph = buildSecondGraph(parallelismValue, config, dimension, "firstgraphpoints", "1");
    // Get the execution plan for the first task graph
    ExecutionPlan firstGraphExecutionPlan = taskExecutor.plan(firstGraph);
    taskExecutor.execute(firstGraph, firstGraphExecutionPlan);
    DataObject<Object> firstGraphObject = taskExecutor.getOutput("firstsink");
    // Get the execution plan for the second task graph
    ExecutionPlan secondGraphExecutionPlan = taskExecutor.plan(secondGraph);
    taskExecutor.addInput("firstgraphpoints", firstGraphObject);
    taskExecutor.execute(secondGraph, secondGraphExecutionPlan);
    long endTime = System.currentTimeMillis();
    LOG.info("Total Execution Time: " + (endTime - startTime));
}