Examples with ReduceStreamingPartialReceiver edu.iu.dsc.tws.comms.mpi.io.reduce.ReduceStreamingPartialReceiver used on opensource projects

Example 1 with ReduceStreamingPartialReceiver

use of edu.iu.dsc.tws.comms.mpi.io.reduce.ReduceStreamingPartialReceiver in project twister2 by DSC-SPIDAL.

the class BaseReduceHLCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
    }
    int dest = NO_OF_TASKS;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        reduce = channel.reduce(newCfg, MessageType.OBJECT, 0, sources, dest, new ReduceStreamingFinalReceiver(new IdentityFunction(), new FinalReduceReceiver()), new ReduceStreamingPartialReceiver(dest, new IdentityFunction()));
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                reduce.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}

Example 2 with ReduceStreamingPartialReceiver

use of edu.iu.dsc.tws.comms.mpi.io.reduce.ReduceStreamingPartialReceiver in project twister2 by DSC-SPIDAL.

the class MPIDataFlowAllReduce method init.

/**
 * Initialize
 * @param config
 * @param t
 * @param instancePlan
 * @param edge
 */
public void init(Config config, MessageType t, TaskPlan instancePlan, int edge) {
    this.type = t;
    this.executor = instancePlan.getThisExecutor();
    this.taskPlan = instancePlan;
    this.executor = taskPlan.getThisExecutor();
    broadcast = new MPIDataFlowBroadcast(channel, middleTask, destinations, new BCastReceiver(finalReceiver));
    broadcast.init(config, t, instancePlan, broadCastEdge);
    MessageReceiver receiver;
    if (streaming) {
        this.partialReceiver = new ReduceStreamingPartialReceiver(middleTask, reduceFunction);
        receiver = new AllReduceStreamingFinalReceiver(reduceFunction, broadcast, middleTask);
    } else {
        this.partialReceiver = new ReduceBatchPartialReceiver(middleTask, reduceFunction);
        receiver = new AllReduceBatchFinalReceiver(reduceFunction, broadcast);
    }
    reduce = new MPIDataFlowReduce(channel, sources, middleTask, receiver, partialReceiver);
    reduce.init(config, t, instancePlan, reduceEdge);
}