Examples with TWSNetwork edu.iu.dsc.tws.comms.core.TWSNetwork used on opensource projects

Example 16 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class SimpleTaskQueue method init.

/**
 * Initialize the container
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    // Creates task an task executor instance to be used in this container
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    // lets create the task plan
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    // we are sending messages from 0th task to 1st task
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int dests = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("-------------------------------------------");
    LOG.info("Setting up reduce dataflow operation");
    LOG.info("-------------------------------------------");
    // this method calls the init method
    // I think this is wrong
    // TODO: Does the task genereate the communication or is it done by a controller for examples
    // the direct comm between task 0 and 1 is it done by the container or the the task
    // TODO: if the task creates the dataflowop does the task progress it or the executor
    // TODO : FOR NOW the dataflowop is created at container and sent to task
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, dests, new PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    if (containerId == 0) {
        // the map thread where data is produced
        LOG.info("-------------------------------------------");
        LOG.log(Level.INFO, "Starting map thread");
        LOG.info("-------------------------------------------");
        LOG.info("-------------------------------------------");
        LOG.log(Level.INFO, "Container Id 0");
        LOG.info("-------------------------------------------");
        taskExecutor.registerTask(new MapWorker(0, direct));
        taskExecutor.submitTask(0);
        taskExecutor.progres();
    } else if (containerId == 1) {
        LOG.info("-------------------------------------------");
        LOG.log(Level.INFO, "Container Id 1 : Receiving End");
        LOG.info("-------------------------------------------");
        ArrayList<Integer> inq = new ArrayList<>();
        inq.add(0);
        taskExecutor.setTaskMessageProcessLimit(100);
        taskExecutor.registerSinkTask(new RecieveWorker(1), inq);
        taskExecutor.progres();
    }
}

Example 17 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class WordCountContainer method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    setupTasks();
    network = new TWSNetwork(cfg, taskPlan);
    channel = network.getDataFlowTWSCommunication();
    // first get the communication config file
    network = new TWSNetwork(cfg, taskPlan);
    channel = network.getDataFlowTWSCommunication();
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    // this method calls the init method
    // I think this is wrong
    keyGather = (MPIDataFlowMultiGather) channel.keyedGather(newCfg, MessageType.OBJECT, destinations, sources, destinations, new GatherMultiBatchFinalReceiver(new WordAggregator()), new GatherMultiBatchPartialReceiver());
    if (id < 2) {
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting thread %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new BatchWordSource(config, keyGather, 1000, new ArrayList<>(destinations), noOfTasksPerExecutor * id + i, 200));
            mapThread.start();
        }
    }
    // we need to progress the communication
    while (true) {
        try {
            // progress the channel
            channel.progress();
            // we should progress the communication directive
            keyGather.progress();
            Thread.yield();
        } catch (Throwable t) {
            LOG.log(Level.SEVERE, "Something bad happened", t);
        }
    }
}

Example 18 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class BaseAllReduceCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS / 2; i++) {
        sources.add(i);
    }
    Set<Integer> destinations = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS / 2; i++) {
        destinations.add(NO_OF_TASKS / 2 + i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        allReduce = channel.allReduce(newCfg, MessageType.OBJECT, 0, 1, sources, destinations, dest, new IndentityFunction(), new FinalReduceReceive(), true);
        if (id == 0 || id == 1) {
            for (int i = 0; i < noOfTasksPerExecutor; i++) {
                // the map thread where data is produced
                LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
                Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
                mapThread.start();
            }
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                allReduce.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}

Example 19 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class BaseLoadBalanceCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    Set<Integer> dests = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        if (i < NO_OF_TASKS / 2) {
            sources.add(i);
        } else {
            dests.add(i);
        }
    }
    LOG.info(String.format("Loadbalance: sources %s destinations: %s", sources, dests));
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    // this method calls the init method
    // I think this is wrong
    loadBalance = channel.loadBalance(newCfg, MessageType.BUFFER, 0, sources, dests, new LoadBalanceReceiver());
    // the map thread where data is produced
    LOG.info("Starting worker: " + id);
    // we need to progress the communication
    try {
        if (id == 0 || id == 1) {
            MPIBuffer data = new MPIBuffer(1024);
            data.setSize(24);
            for (int i = 0; i < 50000; i++) {
                mapFunction(data);
                channel.progress();
                // we should progress the communication directive
                loadBalance.progress();
            }
            while (true) {
                channel.progress();
                // we should progress the communication directive
                loadBalance.progress();
            }
        } else {
            while (true) {
                channel.progress();
                // we should progress the communication directive
                loadBalance.progress();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}

Example 20 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class BasePartitionCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    Set<Integer> dests = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
        dests.add(i);
    }
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up partition dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        Map<Integer, List<Integer>> expectedIds = new HashMap<>();
        for (int i = 0; i < NO_OF_TASKS; i++) {
            expectedIds.put(i, new ArrayList<>());
            for (int j = 0; j < NO_OF_TASKS; j++) {
                if (!(i == j)) {
                    expectedIds.get(i).add(j);
                }
            }
        }
        FinalPartitionReciver finalPartitionRec = new FinalPartitionReciver();
        partition = channel.partition(newCfg, MessageType.BYTE, 2, sources, dests, finalPartitionRec);
        finalPartitionRec.setMap(expectedIds);
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                partition.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}