Examples with Vertex edu.iu.dsc.tws.api.compute.graph.Vertex used on opensource projects

Example 16 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class GraphBuilder method addConfiguration.

public GraphBuilder addConfiguration(String taskName, String property, Object value) {
    Vertex v = graph.vertex(taskName);
    if (v == null) {
        throw new RuntimeException("Failed to add configuration to non-existing task: " + taskName);
    }
    v.addConfiguration(property, value);
    return this;
}

Example 17 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class ComputeGraphBuilder method addCompute.

/**
 * Add a compute node to the graph
 *
 * @param name name of the node
 * @param compute implementation of the node
 * @param parallel number of parallel instances
 * @return a compute connection, that can be used to connect this node to other nodes as a child
 */
public ComputeConnection addCompute(String name, ICompute compute, int parallel) {
    Vertex vertex = new Vertex(name, compute, parallel);
    nodes.put(name, vertex);
    this.addFTGatherSink(name);
    return createComputeConnection(name);
}

Example 18 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class ComputeGraphBuilder method addFTGatherSink.

private void addFTGatherSink(String sourceName) {
    if (!CheckpointingContext.isCheckpointingEnabled(cfg) || this.mode.equals(OperationMode.BATCH)) {
        return;
    }
    String ftTaskName = "ft-" + sourceName;
    Vertex vertex = new Vertex(ftTaskName, new CheckpointingSGatherSink(sourceName), 1);
    nodes.put(ftTaskName, vertex);
    ComputeConnection computeConnection = this.createComputeConnection(ftTaskName);
    computeConnection.gather(sourceName).withDataType(MessageTypes.LONG).viaEdge(CheckpointingSGatherSink.FT_GATHER_EDGE);
}

Example 19 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class ComputeGraphBuilder method addSource.

/**
 * Add a source node to the graph
 *
 * @param name name of the node
 * @param source implementation of the node
 * @param parallel parallelism of the node
 * @return a compute connection, that can be used to connect this node to other nodes as a child
 */
public SourceConnection addSource(String name, ISource source, int parallel) {
    Vertex vertex = new Vertex(name, source, parallel);
    nodes.put(name, vertex);
    this.addFTGatherSink(name);
    return createSourceConnection(name);
}

Example 20 with Vertex

use of edu.iu.dsc.tws.api.compute.graph.Vertex in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method schedule.

/**
 * This is the base method for the data locality aware task scheduling for scheduling the batch
 * task instances. It retrieves the task vertex set of the task graph and send the set to the
 * data locality aware scheduling algorithm to allocate the batch task instances which are closer
 * to the data nodes.
 */
@Override
public TaskSchedulePlan schedule(ComputeGraph graph, WorkerPlan workerPlan) {
    LinkedHashMap<Integer, WorkerSchedulePlan> containerPlans = new LinkedHashMap<>();
    for (int i = 0; i < workerPlan.getNumberOfWorkers(); i++) {
        dataLocalityAwareAllocation.put(i, new ArrayList<>());
    }
    LinkedHashSet<Vertex> taskVertexSet = new LinkedHashSet<>(graph.getTaskVertexSet());
    TaskVertexParser taskVertexParser = new TaskVertexParser();
    List<Set<Vertex>> taskVertexList = taskVertexParser.parseVertexSet(graph);
    for (Set<Vertex> vertexSet : taskVertexList) {
        Map<Integer, List<TaskInstanceId>> containerInstanceMap;
        if (vertexSet.size() > 1) {
            containerInstanceMap = dataLocalityBatchSchedulingAlgorithm(graph, vertexSet, workerPlan);
        } else {
            Vertex vertex = vertexSet.iterator().next();
            containerInstanceMap = dataLocalityBatchSchedulingAlgorithm(graph, vertex, workerPlan);
        }
        TaskInstanceMapCalculation instanceMapCalculation = new TaskInstanceMapCalculation(this.instanceRAM, this.instanceCPU, this.instanceDisk);
        Map<Integer, Map<TaskInstanceId, Double>> instancesRamMap = instanceMapCalculation.getInstancesRamMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesDiskMap = instanceMapCalculation.getInstancesDiskMapInContainer(containerInstanceMap, taskVertexSet);
        Map<Integer, Map<TaskInstanceId, Double>> instancesCPUMap = instanceMapCalculation.getInstancesCPUMapInContainer(containerInstanceMap, taskVertexSet);
        for (int containerId : containerInstanceMap.keySet()) {
            double containerRAMValue = TaskSchedulerContext.containerRamPadding(config);
            double containerDiskValue = TaskSchedulerContext.containerDiskPadding(config);
            double containerCpuValue = TaskSchedulerContext.containerCpuPadding(config);
            List<TaskInstanceId> taskTaskInstanceIds = containerInstanceMap.get(containerId);
            Map<TaskInstanceId, TaskInstancePlan> taskInstancePlanMap = new HashMap<>();
            for (TaskInstanceId id : taskTaskInstanceIds) {
                double instanceRAMValue = instancesRamMap.get(containerId).get(id);
                double instanceDiskValue = instancesDiskMap.get(containerId).get(id);
                double instanceCPUValue = instancesCPUMap.get(containerId).get(id);
                Resource instanceResource = new Resource(instanceRAMValue, instanceDiskValue, instanceCPUValue);
                taskInstancePlanMap.put(id, new TaskInstancePlan(id.getTaskName(), id.getTaskId(), id.getTaskIndex(), instanceResource));
                containerRAMValue += instanceRAMValue;
                containerDiskValue += instanceDiskValue;
                containerCpuValue += instanceDiskValue;
            }
            Worker worker = workerPlan.getWorker(containerId);
            Resource containerResource;
            if (worker != null && worker.getCpu() > 0 && worker.getDisk() > 0 && worker.getRam() > 0) {
                containerResource = new Resource((double) worker.getRam(), (double) worker.getDisk(), (double) worker.getCpu());
            } else {
                containerResource = new Resource(containerRAMValue, containerDiskValue, containerCpuValue);
            }
            WorkerSchedulePlan taskWorkerSchedulePlan;
            if (containerPlans.containsKey(containerId)) {
                taskWorkerSchedulePlan = containerPlans.get(containerId);
                taskWorkerSchedulePlan.getTaskInstances().addAll(taskInstancePlanMap.values());
            } else {
                taskWorkerSchedulePlan = new WorkerSchedulePlan(containerId, new HashSet<>(taskInstancePlanMap.values()), containerResource);
                containerPlans.put(containerId, taskWorkerSchedulePlan);
            }
        }
    }
    TaskSchedulePlan taskSchedulePlan = new TaskSchedulePlan(0, new HashSet<>(containerPlans.values()));
    Map<Integer, WorkerSchedulePlan> containersMap = taskSchedulePlan.getContainersMap();
    for (Map.Entry<Integer, WorkerSchedulePlan> entry : containersMap.entrySet()) {
        Integer integer = entry.getKey();
        WorkerSchedulePlan workerSchedulePlan = entry.getValue();
        Set<TaskInstancePlan> containerPlanTaskInstances = workerSchedulePlan.getTaskInstances();
        LOG.fine("Task Details for Container Id:" + integer + "\tsize:" + containerPlanTaskInstances.size());
        for (TaskInstancePlan ip : containerPlanTaskInstances) {
            LOG.fine("TaskId:" + ip.getTaskId() + "\tTask Index" + ip.getTaskIndex() + "\tTask Name:" + ip.getTaskName());
        }
    }
    return taskSchedulePlan;
}