Examples with Collector edu.iu.dsc.tws.api.compute.modifiers.Collector used on opensource projects

Example 1 with Collector

use of edu.iu.dsc.tws.api.compute.modifiers.Collector in project twister2 by DSC-SPIDAL.

the class BatchTaskScheduler method batchSchedulingAlgorithm.

private Map<Integer, List<TaskInstanceId>> batchSchedulingAlgorithm(ComputeGraph graph, int numberOfContainers) throws TaskSchedulerException {
    Set<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TreeSet<Vertex> orderedTaskSet = new TreeSet<>(new VertexComparator());
    orderedTaskSet.addAll(taskVertexSet);
    IntStream.range(0, numberOfContainers).forEach(i1 -> batchTaskAllocation.put(i1, new ArrayList<>()));
    int globalTaskIndex = 0;
    if (dependentGraphs) {
        for (Vertex vertex : taskVertexSet) {
            INode iNode = vertex.getTask();
            if (iNode instanceof Receptor) {
                validateReceptor(graph, vertex);
            }
            dependentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
            globalTaskIndex++;
        }
    } else {
        for (Vertex vertex : taskVertexSet) {
            INode iNode = vertex.getTask();
            if (iNode instanceof Collector) {
                ((Collector) iNode).getCollectibleNames().forEach(key -> collectibleNameMap.put(key, vertex.getParallelism()));
            } else if (iNode instanceof Receptor) {
                ((Receptor) iNode).getReceivableNames().forEach(key -> receivableNameMap.put(key, vertex.getParallelism()));
                validateParallelism();
            }
            independentTaskWorkerAllocation(graph, vertex, numberOfContainers, globalTaskIndex);
            globalTaskIndex++;
        }
    }
    return batchTaskAllocation;
}

Example 2 with Collector

use of edu.iu.dsc.tws.api.compute.modifiers.Collector in project twister2 by DSC-SPIDAL.

the class TaskExecutor method collectData.

/**
 * This method collects all the output from the provided {@link ExecutionPlan}.
 * The partition IDs will be assigned just before adding the partitions to the {@link DataObject}
 */
public static void collectData(Config cfg, ExecutionPlan executionPlan, Map<String, DataObject> dataMap) {
    Map<Integer, INodeInstance> nodes = executionPlan.getNodes();
    Map<String, DataObject> dataObjectMapForPlan = new HashMap<>();
    if (nodes != null) {
        nodes.forEach((taskId, node) -> {
            INode task = node.getNode();
            if (task instanceof Collector) {
                Set<String> collectibleNames = ((Collector) task).getCollectibleNames();
                collectibleNames.forEach(name -> {
                    DataPartition partition = ((Collector) task).get(name);
                    // if this task outs only one partition and user has implemented no arg get() method
                    if (collectibleNames.size() == 1 && partition == null) {
                        partition = ((Collector) task).get();
                    }
                    if (partition != null) {
                        partition.setId(node.getIndex());
                        dataObjectMapForPlan.computeIfAbsent(name, n -> new DataObjectImpl<>(cfg)).addPartition(partition);
                    } else {
                        LOG.warning(String.format("Task index %d  of task %d returned null for data %s", node.getIndex(), node.getId(), name));
                    }
                });
            }
        });
    }
    dataMap.putAll(dataObjectMapForPlan);
}