Examples with INodeInstance edu.iu.dsc.tws.api.compute.executor.INodeInstance used on opensource projects

Example 11 with INodeInstance

use of edu.iu.dsc.tws.api.compute.executor.INodeInstance in project twister2 by DSC-SPIDAL.

the class StreamingAllSharingExecutor method scheduleWaitFor.

private CommunicationWorker[] scheduleWaitFor(Map<Integer, INodeInstance> nodes) {
    BlockingQueue<INodeInstance> tasks;
    tasks = new ArrayBlockingQueue<>(nodes.size() * 2);
    tasks.addAll(nodes.values());
    CommunicationWorker[] workers = new CommunicationWorker[numThreads];
    workers[0] = new CommunicationWorker(tasks);
    doneSignal = new CountDownLatch(numThreads - 1);
    for (int i = 1; i < numThreads; i++) {
        workers[i] = new CommunicationWorker(tasks);
        threads.submit(workers[i]);
    }
    return workers;
}

Example 12 with INodeInstance

use of edu.iu.dsc.tws.api.compute.executor.INodeInstance in project twister2 by DSC-SPIDAL.

the class StreamingSharingExecutor method cleanUp.

private void cleanUp(Map<Integer, INodeInstance> nodes) {
    // lets wait for thread to finish
    try {
        doneSignal.await();
    } catch (InterruptedException e) {
        throw new RuntimeException("Interrupted", e);
    }
    // clean up the instances
    for (INodeInstance node : nodes.values()) {
        node.close();
    }
    // lets close the operations
    List<IParallelOperation> ops = executionPlan.getParallelOperations();
    for (IParallelOperation op : ops) {
        op.close();
    }
    // execution hook
    executionHook.afterExecution();
    cleanUpCalled = true;
}

Example 13 with INodeInstance

use of edu.iu.dsc.tws.api.compute.executor.INodeInstance in project twister2 by DSC-SPIDAL.

the class StreamingSharingExecutor method schedulerExecution.

private void schedulerExecution(Map<Integer, INodeInstance> nodes) {
    BlockingQueue<INodeInstance> tasks;
    tasks = new ArrayBlockingQueue<>(nodes.size() * 2);
    tasks.addAll(nodes.values());
    for (INodeInstance node : tasks) {
        node.prepare(config);
    }
    doneSignal = new CountDownLatch(numThreads);
    for (int i = 0; i < numThreads; i++) {
        threads.execute(new StreamWorker(tasks));
    }
}

Example 14 with INodeInstance

use of edu.iu.dsc.tws.api.compute.executor.INodeInstance in project twister2 by DSC-SPIDAL.

the class TaskExecutor method addInput.

/**
 * Add input to the the task instances
 *
 * @param graph task graph
 * @param plan execution plan
 * @param taskName task name
 * @param inputKey inputkey
 * @param input input
 * @deprecated Inputs are automatically handled now
 */
@Deprecated
public void addInput(ComputeGraph graph, ExecutionPlan plan, String taskName, String inputKey, DataObject<?> input) {
    Map<Integer, INodeInstance> nodes = plan.getNodes(taskName);
    if (nodes == null) {
        return;
    }
    for (Map.Entry<Integer, INodeInstance> e : nodes.entrySet()) {
        INodeInstance node = e.getValue();
        INode task = node.getNode();
        if (task instanceof Receptor) {
            ((Receptor) task).add(inputKey, input);
        } else {
            throw new RuntimeException("Cannot add input to non input instance: " + node);
        }
    }
}

Example 15 with INodeInstance

use of edu.iu.dsc.tws.api.compute.executor.INodeInstance in project twister2 by DSC-SPIDAL.

the class TaskExecutor method collectData.

/**
 * This method collects all the output from the provided {@link ExecutionPlan}.
 * The partition IDs will be assigned just before adding the partitions to the {@link DataObject}
 */
public static void collectData(Config cfg, ExecutionPlan executionPlan, Map<String, DataObject> dataMap) {
    Map<Integer, INodeInstance> nodes = executionPlan.getNodes();
    Map<String, DataObject> dataObjectMapForPlan = new HashMap<>();
    if (nodes != null) {
        nodes.forEach((taskId, node) -> {
            INode task = node.getNode();
            if (task instanceof Collector) {
                Set<String> collectibleNames = ((Collector) task).getCollectibleNames();
                collectibleNames.forEach(name -> {
                    DataPartition partition = ((Collector) task).get(name);
                    // if this task outs only one partition and user has implemented no arg get() method
                    if (collectibleNames.size() == 1 && partition == null) {
                        partition = ((Collector) task).get();
                    }
                    if (partition != null) {
                        partition.setId(node.getIndex());
                        dataObjectMapForPlan.computeIfAbsent(name, n -> new DataObjectImpl<>(cfg)).addPartition(partition);
                    } else {
                        LOG.warning(String.format("Task index %d  of task %d returned null for data %s", node.getIndex(), node.getId(), name));
                    }
                });
            }
        });
    }
    dataMap.putAll(dataObjectMapForPlan);
}