Examples with ExecutionPlan edu.iu.dsc.tws.api.compute.executor.ExecutionPlan used on opensource projects

Example 1 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class TaskUtils method execute.

public static void execute(Config config, int workerID, ComputeGraph graph, IWorkerController workerController) {
    RoundRobinTaskScheduler roundRobinTaskScheduler = new RoundRobinTaskScheduler();
    roundRobinTaskScheduler.initialize(config);
    List<JobMasterAPI.WorkerInfo> workerList = null;
    try {
        workerList = workerController.getAllWorkers();
    } catch (TimeoutException timeoutException) {
        LOG.log(Level.SEVERE, timeoutException.getMessage(), timeoutException);
        return;
    }
    WorkerPlan workerPlan = createWorkerPlan(workerList);
    TaskSchedulePlan taskSchedulePlan = roundRobinTaskScheduler.schedule(graph, workerPlan);
    TWSChannel network = Network.initializeChannel(config, workerController);
    ExecutionPlanBuilder executionPlanBuilder = new ExecutionPlanBuilder(workerID, workerList, new Communicator(config, network), workerController.getCheckpointingClient());
    ExecutionPlan plan = executionPlanBuilder.build(config, graph, taskSchedulePlan);
    ExecutorFactory executor = new ExecutorFactory(config, workerID, network);
    executor.getExecutor(config, plan, graph.getOperationMode()).execute();
}

Example 2 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class TaskUtils method executeBatch.

public static void executeBatch(Config config, int workerID, ComputeGraph graph, IWorkerController workerController) {
    RoundRobinTaskScheduler roundRobinTaskScheduler = new RoundRobinTaskScheduler();
    roundRobinTaskScheduler.initialize(config);
    WorkerPlan workerPlan = null;
    List<JobMasterAPI.WorkerInfo> workerList = null;
    try {
        workerList = workerController.getAllWorkers();
    } catch (TimeoutException timeoutException) {
        LOG.log(Level.SEVERE, timeoutException.getMessage(), timeoutException);
        return;
    }
    workerPlan = createWorkerPlan(workerList);
    TaskSchedulePlan taskSchedulePlan = roundRobinTaskScheduler.schedule(graph, workerPlan);
    TWSChannel network = Network.initializeChannel(config, workerController);
    ExecutionPlanBuilder executionPlanBuilder = new ExecutionPlanBuilder(workerID, workerList, new Communicator(config, network), workerController.getCheckpointingClient());
    ExecutionPlan plan = executionPlanBuilder.build(config, graph, taskSchedulePlan);
    ExecutorFactory executor = new ExecutorFactory(config, workerID, network);
    executor.getExecutor(config, plan, graph.getOperationMode()).execute();
}

Example 3 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdOnlineRunner method streamData.

private void streamData() {
    ComputeGraph streamingTrainingTG = buildStreamingTrainingTG();
    ExecutionPlan executionPlan = taskExecutor.plan(streamingTrainingTG);
    taskExecutor.addInput(streamingTrainingTG, executionPlan, Constants.SimpleGraphConfig.ITERATIVE_STREAMING_DATASTREAMER_SOURCE, Constants.SimpleGraphConfig.INPUT_DATA, trainingDoubleDataPointObject);
    taskExecutor.addInput(streamingTrainingTG, executionPlan, Constants.SimpleGraphConfig.ITERATIVE_STREAMING_DATASTREAMER_SOURCE, Constants.SimpleGraphConfig.INPUT_WEIGHT_VECTOR, inputDoubleWeightvectorObject);
    taskExecutor.addInput(streamingTrainingTG, executionPlan, "window-sink", Constants.SimpleGraphConfig.TEST_DATA, testingDoubleDataPointObject);
    taskExecutor.execute(streamingTrainingTG, executionPlan);
// currentDataPoint = taskExecutor.getOutput(streamingTrainingTG,
// executionPlan,
// Constants.SimpleGraphConfig.ITERATIVE_STREAMING_SVM_COMPUTE);
// double[] dp = currentDataPoint.getPartitions(0).getConsumer().next();
// LOG.info(String.format("DataPoint[%d] : %s", count++, Arrays.toString(dp)));
}

Example 4 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdOnlineRunner method loadTrainingData.

private void loadTrainingData() {
    ComputeGraph trainingDFTG = TGUtils.buildTrainingDataPointsTG(this.dataStreamerParallelism, this.svmJobParameters, this.config, this.operationMode);
    ExecutionPlan trainingEP = taskExecutor.plan(trainingDFTG);
    taskExecutor.execute(trainingDFTG, trainingEP);
    trainingDoubleDataPointObject = taskExecutor.getOutput(trainingDFTG, trainingEP, Constants.SimpleGraphConfig.DATA_OBJECT_SINK);
    for (int i = 0; i < trainingDoubleDataPointObject.getPartitions().length; i++) {
        double[][] datapoints = trainingDoubleDataPointObject.getPartitions()[i].getConsumer().next();
        LOG.info(String.format("Training Datapoints : %d,%d", datapoints.length, datapoints[0].length));
        int randomIndex = new Random().nextInt(this.svmJobParameters.getSamples() / dataStreamerParallelism - 1);
        LOG.info(String.format("Random DataPoint[%d] : %s", randomIndex, Arrays.toString(datapoints[randomIndex])));
    }
}

Example 5 with ExecutionPlan

use of edu.iu.dsc.tws.api.compute.executor.ExecutionPlan in project twister2 by DSC-SPIDAL.

the class SvmSgdOnlineRunner method loadTestingData.

private void loadTestingData() {
    ComputeGraph testingDFTG = TGUtils.buildTestingDataPointsTG(this.dataStreamerParallelism, this.svmJobParameters, this.config, this.operationMode);
    ExecutionPlan testingEP = taskExecutor.plan(testingDFTG);
    taskExecutor.execute(testingDFTG, testingEP);
    testingDoubleDataPointObject = taskExecutor.getOutput(testingDFTG, testingEP, Constants.SimpleGraphConfig.DATA_OBJECT_SINK_TESTING);
    for (int i = 0; i < testingDoubleDataPointObject.getPartitions().length; i++) {
        double[][] datapoints = testingDoubleDataPointObject.getPartitions()[i].getConsumer().next();
        LOG.info(String.format("Partition[%d] Testing Datapoints : %d,%d", i, datapoints.length, datapoints[0].length));
        int randomIndex = new Random().nextInt(this.svmJobParameters.getTestingSamples() / dataStreamerParallelism - 1);
        LOG.info(String.format("Random DataPoint[%d] : %s", randomIndex, Arrays.toString(datapoints[randomIndex])));
    }
}