Examples with TaskPlan edu.iu.dsc.tws.comms.core.TaskPlan used on opensource projects

Example 31 with TaskPlan

use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.

the class KMeansUtils method createWordCountPlan.

/**
 * Let assume we have 2 tasks per container and one additional for first container,
 * which will be the destination
 * @param plan the resource plan from scheduler
 * @return task plan
 */
public static TaskPlan createWordCountPlan(Config cfg, ResourcePlan plan, int noOfTasks) {
    int noOfProcs = plan.noOfContainers();
    LOG.log(Level.INFO, "No of containers: " + noOfProcs);
    Map<Integer, Set<Integer>> executorToGraphNodes = new HashMap<>();
    Map<Integer, Set<Integer>> groupsToExeuctors = new HashMap<>();
    int thisExecutor = plan.getThisId();
    List<ResourceContainer> containers = plan.getContainers();
    Map<String, List<ResourceContainer>> containersPerNode = new HashMap<>();
    for (ResourceContainer c : containers) {
        String name = (String) c.getProperty(SchedulerContext.WORKER_NAME);
        List<ResourceContainer> containerList;
        if (!containersPerNode.containsKey(name)) {
            containerList = new ArrayList<>();
            containersPerNode.put(name, containerList);
        } else {
            containerList = containersPerNode.get(name);
        }
        containerList.add(c);
    }
    int taskPerExecutor = noOfTasks / noOfProcs;
    for (int i = 0; i < noOfProcs; i++) {
        Set<Integer> nodesOfExecutor = new HashSet<>();
        for (int j = 0; j < taskPerExecutor; j++) {
            nodesOfExecutor.add(i * taskPerExecutor + j);
        }
        executorToGraphNodes.put(i, nodesOfExecutor);
    }
    int i = 0;
    // we take each container as an executor
    for (Map.Entry<String, List<ResourceContainer>> e : containersPerNode.entrySet()) {
        Set<Integer> executorsOfGroup = new HashSet<>();
        for (ResourceContainer c : e.getValue()) {
            executorsOfGroup.add(c.getId());
        }
        groupsToExeuctors.put(i, executorsOfGroup);
        i++;
    }
    String print = printMap(executorToGraphNodes);
    LOG.fine("Executor To Graph: " + print);
    print = printMap(groupsToExeuctors);
    LOG.fine("Groups to executors: " + print);
    // and reduce task in 0th process
    return new TaskPlan(executorToGraphNodes, groupsToExeuctors, thisExecutor);
}

Example 32 with TaskPlan

use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.

the class WordAggregator method init.

@Override
public void init(Config cfg, DataFlowOperation op, Map<Integer, List<Integer>> expectedIds) {
    TaskPlan plan = op.getTaskPlan();
    this.executor = op.getTaskPlan().getThisExecutor();
    LOG.fine(String.format("%d final expected task ids %s", plan.getThisExecutor(), expectedIds));
}

Example 33 with TaskPlan

use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.

the class BaseBroadcastCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    try {
        this.config = cfg;
        this.resourcePlan = plan;
        this.id = containerId;
        this.status = Status.INIT;
        this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
        // lets create the task plan
        TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
        LOG.log(Level.INFO, "Task plan: " + taskPlan);
        // first get the communication config file
        TWSNetwork network = new TWSNetwork(cfg, taskPlan);
        TWSCommunication channel = network.getDataFlowTWSCommunication();
        Set<Integer> sources = new HashSet<>();
        for (int i = 0; i < NO_OF_TASKS; i++) {
            sources.add(i);
        }
        int dest = NO_OF_TASKS;
        Map<String, Object> newCfg = new HashMap<>();
        LOG.info(String.format("Setting up reduce dataflow operation %d %s", dest, sources));
        // this method calls the init method
        // I think this is wrong
        broadcast = channel.broadCast(newCfg, MessageType.OBJECT, 0, dest, sources, new BCastReceive());
        // the map thread where data is produced
        if (id == 0) {
            Thread mapThread = new Thread(new MapWorker());
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                broadcast.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}

Example 34 with TaskPlan

use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.

the class BaseKeyedReduceCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    this.reduceTask = NO_OF_TASKS / 2;
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS / 2; i++) {
        sources.add(i);
    }
    Set<Integer> destinations = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS / 2; i++) {
        destinations.add(NO_OF_TASKS / 2 + i);
    }
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    // this method calls the init method
    // I think this is wrong
    reduce = channel.keyedReduce(newCfg, MessageType.OBJECT, destinations, sources, destinations, new FinalReduceReceive(), new PartialReduceWorker());
    if (id == 0 || id == 1) {
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
    }
    // we need to progress the communication
    while (true) {
        try {
            // progress the channel
            channel.progress();
            // we should progress the communication directive
            reduce.progress();
            Thread.yield();
        } catch (Throwable t) {
            LOG.severe("Error occurred: " + id);
            t.printStackTrace();
        }
    }
}

Example 35 with TaskPlan

use of edu.iu.dsc.tws.comms.core.TaskPlan in project twister2 by DSC-SPIDAL.

the class BasePartitionMultiByteCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    Set<Integer> dests = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
        dests.add(i);
    }
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up partition dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        Map<Integer, List<Integer>> expectedIds = new HashMap<>();
        for (int i = 0; i < NO_OF_TASKS; i++) {
            expectedIds.put(i, new ArrayList<>());
            for (int j = 0; j < NO_OF_TASKS; j++) {
                if (!(i == j)) {
                    expectedIds.get(i).add(j);
                }
            }
        }
        FinalPartitionReciver finalPartitionRec = new FinalPartitionReciver();
        partition = channel.partition(newCfg, MessageType.MULTI_FIXED_BYTE, MessageType.MULTI_FIXED_BYTE, 2, sources, dests, finalPartitionRec);
        finalPartitionRec.setMap(expectedIds);
        partition.setMemoryMapped(true);
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                partition.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}