Examples with ComputeGraph edu.iu.dsc.tws.api.compute.graph.ComputeGraph used on opensource projects

Example 51 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method batchSchedulingAlgorithm.

private Map<Integer, List<TaskInstanceId>> batchSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) throws TaskSchedulerException {
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    IntStream.range(0, numberOfContainers).forEach(i1 -> batchTaskAllocation.put(i1, new ArrayList<>()));
    int globalTaskIndex = 0;
    if (dependentGraphs) {
        for (Vertex vertex : taskVertexSet) {
            INode iNode = vertex.getTask();
            if (iNode instanceof Receptor) {
                validateReceptor(graph, vertex);
            }
            dependentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
            globalTaskIndex++;
        }
    } else {
        for (Vertex vertex : taskVertexSet) {
            INode iNode = vertex.getTask();
            if (iNode instanceof Collector) {
                ((Collector) iNode).getCollectibleNames().forEach(key -> collectibleNameMap.put(key, vertex.getParallelism()));
            } else if (iNode instanceof Receptor) {
                ((Receptor) iNode).getReceivableNames().forEach(key -> receivableNameMap.put(key, vertex.getParallelism()));
                validateParallelism();
            }
            independentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
            globalTaskIndex++;
        }
    }
    return batchTaskAllocation;
}

Example 52 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class CDFWRuntime method handleExecuteMessage.

private boolean handleExecuteMessage(Any msg) {
    ISenderToDriver senderToDriver = JMWorkerAgent.getJMWorkerAgent().getDriverAgent();
    CDFWJobAPI.ExecuteMessage executeMessage;
    ExecutionPlan executionPlan;
    CDFWJobAPI.ExecuteCompletedMessage completedMessage = null;
    try {
        executeMessage = msg.unpack(CDFWJobAPI.ExecuteMessage.class);
        // get the subgraph from the map
        CDFWJobAPI.SubGraph subGraph = executeMessage.getGraph();
        ComputeGraph taskGraph = (ComputeGraph) serializer.deserialize(subGraph.getGraphSerialized().toByteArray());
        if (taskGraph == null) {
            LOG.severe(workerId + " Unable to find the subgraph " + subGraph.getName());
            return true;
        }
        // use the taskexecutor to create the execution plan
        executionPlan = taskExecutor.plan(taskGraph);
        taskExecutor.execute(taskGraph, executionPlan);
        // reuse the task executor execute
        completedMessage = CDFWJobAPI.ExecuteCompletedMessage.newBuilder().setSubgraphName(subGraph.getName()).build();
        if (!senderToDriver.sendToDriver(completedMessage)) {
            LOG.severe("Unable to send the subgraph completed message :" + completedMessage);
        }
    } catch (InvalidProtocolBufferException e) {
        LOG.log(Level.SEVERE, "Unable to unpack received message ", e);
    }
    return false;
}

Example 53 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class IterativeJob method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    ComputeEnvironment cEnv = ComputeEnvironment.init(workerEnv);
    TaskExecutor taskExecutor = cEnv.getTaskExecutor();
    int workerId = workerEnv.getWorkerId();
    Config config = workerEnv.getConfig();
    LOG.log(Level.INFO, "Task worker starting: " + workerId);
    IterativeSourceTask g = new IterativeSourceTask();
    PartitionTask r = new PartitionTask();
    ComputeGraphBuilder graphBuilder = ComputeGraphBuilder.newBuilder(config);
    graphBuilder.addSource("source", g, 4);
    ComputeConnection computeConnection = graphBuilder.addCompute("sink", r, 4);
    computeConnection.partition("source").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph graph = graphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    IExecutor ex = taskExecutor.createExecution(graph, plan);
    for (int i = 0; i < 10; i++) {
        LOG.info("Starting iteration: " + i);
        taskExecutor.addInput(graph, plan, "source", "input", new DataObjectImpl<>(config));
        // this is a blocking call
        ex.execute();
        DataObject<Object> dataSet = taskExecutor.getOutput(graph, plan, "sink");
        DataPartition<Object>[] values = dataSet.getPartitions();
    }
    ex.closeExecution();
}

Example 54 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class SvmSgdRunner method initializeExecute.

/**
 * Initializing the execute method
 */
public void initializeExecute() {
    ComputeGraphBuilder builder = ComputeGraphBuilder.newBuilder(config);
    this.operationMode = this.svmJobParameters.isStreaming() ? OperationMode.STREAMING : OperationMode.BATCH;
    DataStreamer dataStreamer = new DataStreamer(this.operationMode, svmJobParameters.isDummy(), this.binaryBatchModel);
    SVMCompute svmCompute = new SVMCompute(this.binaryBatchModel, this.operationMode);
    SVMReduce svmReduce = new SVMReduce(this.operationMode);
    builder.addSource(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE, dataStreamer, dataStreamerParallelism);
    ComputeConnection svmComputeConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_COMPUTE, svmCompute, svmComputeParallelism);
    ComputeConnection svmReduceConnection = builder.addCompute(Constants.SimpleGraphConfig.SVM_REDUCE, svmReduce, reduceParallelism);
    svmComputeConnection.direct(Constants.SimpleGraphConfig.DATASTREAMER_SOURCE).viaEdge(Constants.SimpleGraphConfig.DATA_EDGE).withDataType(MessageTypes.OBJECT);
    svmReduceConnection.reduce(Constants.SimpleGraphConfig.SVM_COMPUTE).viaEdge(Constants.SimpleGraphConfig.REDUCE_EDGE).withReductionFunction(new ReduceAggregator()).withDataType(MessageTypes.OBJECT);
    builder.setMode(operationMode);
    ComputeGraph graph = builder.build();
    ExecutionPlan plan = taskExecutor.plan(graph);
    taskExecutor.execute(graph, plan);
    LOG.info("Task Graph Executed !!! ");
    if (operationMode.equals(OperationMode.BATCH)) {
        DataObject<double[]> dataSet = taskExecutor.getOutput(graph, plan, Constants.SimpleGraphConfig.SVM_REDUCE);
        DataPartition<double[]> values = dataSet.getPartitions()[0];
        DataPartitionConsumer<double[]> dataPartitionConsumer = values.getConsumer();
        // LOG.info("Final Receive  : " + dataPartitionConsumer.hasNext());
        while (dataPartitionConsumer.hasNext()) {
            LOG.info("Final Aggregated Values Are:" + Arrays.toString(dataPartitionConsumer.next()));
        }
    }
}

Example 55 with ComputeGraph

use of edu.iu.dsc.tws.api.compute.graph.ComputeGraph in project twister2 by DSC-SPIDAL.

the class DataParallelWorker method execute.

@Override
public void execute() {
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    String inputDirectory = config.getStringValue(Constants.ARGS_INPUT_DIRECTORY);
    boolean shared = config.getBooleanValue(Constants.ARGS_SHARED_FILE_SYSTEM);
    int numFiles = config.getIntegerValue(Constants.ARGS_NUMBER_OF_FILES, 4);
    int size = config.getIntegerValue(Constants.ARGS_SIZE, 1000);
    int parallel = config.getIntegerValue(Constants.ARGS_PARALLEL, 2);
    if (!shared && workerId == 0) {
        try {
            DataGenerator.generateData("txt", new Path(inputDirectory), numFiles, size, 10);
        } catch (IOException e) {
            throw new RuntimeException("Failed to create data: " + inputDirectory);
        }
    }
    DataParallelTask task = new DataParallelTask();
    computeGraphBuilder.addSource("map", task, parallel);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    ComputeGraph computeGraph = computeGraphBuilder.build();
    ExecutionPlan plan = taskExecutor.plan(computeGraph);
    taskExecutor.execute(computeGraph, plan);
}