Examples with GatherMultiBatchPartialReceiver edu.iu.dsc.tws.comms.mpi.io.gather.GatherMultiBatchPartialReceiver used on opensource projects

Example 1 with GatherMultiBatchPartialReceiver

use of edu.iu.dsc.tws.comms.mpi.io.gather.GatherMultiBatchPartialReceiver in project twister2 by DSC-SPIDAL.

the class WordCountContainer method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    setupTasks();
    network = new TWSNetwork(cfg, taskPlan);
    channel = network.getDataFlowTWSCommunication();
    // first get the communication config file
    network = new TWSNetwork(cfg, taskPlan);
    channel = network.getDataFlowTWSCommunication();
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    // this method calls the init method
    // I think this is wrong
    keyGather = (MPIDataFlowMultiGather) channel.keyedGather(newCfg, MessageType.OBJECT, destinations, sources, destinations, new GatherMultiBatchFinalReceiver(new WordAggregator()), new GatherMultiBatchPartialReceiver());
    if (id < 2) {
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting thread %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new BatchWordSource(config, keyGather, 1000, new ArrayList<>(destinations), noOfTasksPerExecutor * id + i, 200));
            mapThread.start();
        }
    }
    // we need to progress the communication
    while (true) {
        try {
            // progress the channel
            channel.progress();
            // we should progress the communication directive
            keyGather.progress();
            Thread.yield();
        } catch (Throwable t) {
            LOG.log(Level.SEVERE, "Something bad happened", t);
        }
    }
}