Examples with MessageReceiver edu.iu.dsc.tws.comms.api.MessageReceiver used on opensource projects

Example 1 with MessageReceiver

use of edu.iu.dsc.tws.comms.api.MessageReceiver in project twister2 by DSC-SPIDAL.

the class MultiTaskGraphExample method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new MultiTaskGraphExample.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    direct1 = channel.direct(newCfg, MessageType.OBJECT, 1, sources, destination, new MultiTaskGraphExample.PingPongReceive());
    taskExecutor.initCommunication(channel, direct1);
    MessageReceiver receiver = null;
    gather = channel.gather(newCfg, MessageType.OBJECT, 0, sources, destination, receiver);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask1 = new ReceiveWorker();
    ReceiveWorker sinkTask2 = new ReceiveWorker();
    ReceiveWorker sinkTask3 = new ReceiveWorker();
    /* Source Task (Task0) ------> SinkTask1 (Task 1)
   *        |               (1)      |
   *        |                        |
   *   (1)  |                        |  (2)
   *        |                        |
   *        |                        |
   *        V               (2)      V
   *   SinkTask2 (Task 2) -----> SinkTask3 (Task 3)
   *
   *   Here, (1) represents Task 1 and Task 2 starts simultaneously (receive input
    *  from source task (Task 0), whereas (2) represents Task 3 receives input
    *  from Task 1 & Task 2.
   */
    dataflowTaskGraph = new DataflowTaskGraphGenerator().generateDataflowGraph(sourceTask, sinkTask1, direct).generateDataflowGraph(sourceTask, sinkTask2, direct1).generateDataflowGraph(sinkTask1, sinkTask3, direct1).generateDataflowGraph(sinkTask2, sinkTask3, direct1);
    if (dataflowTaskGraph != null) {
        dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraph);
        parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    // This code is for moving the explicit scheduling outside of the example program
    if (!parsedTaskSet.isEmpty()) {
        TaskGraphScheduler taskGraphScheduler = new TaskGraphScheduler();
        if (containerId == 0) {
            LOG.log(Level.INFO, "Parsed Job Value:" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        } else if (containerId > 0) {
            Map<Task, ArrayList<Integer>> taskMap = taskGraphScheduler.taskgraphScheduler(parsedTaskSet, containerId);
            taskExecutor.setTaskMessageProcessLimit(10000);
            for (Map.Entry<Task, ArrayList<Integer>> taskEntry : taskMap.entrySet()) {
                taskExecutor.registerSinkTask(taskEntry.getKey(), taskEntry.getValue());
                taskExecutor.progres();
            }
        }
    }
/*if (!parsedTaskSet.isEmpty()) {
      if (containerId == 0) {
        LOG.log(Level.INFO, "Parsed Job Value:" + parsedTaskSet.iterator().next());
        taskExecutor.registerTask(parsedTaskSet.iterator().next());
        taskExecutor.submitTask(0);
        taskExecutor.progres();
      } else if (containerId == 1) {
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            ArrayList<Integer> inq = new ArrayList<>();
            inq.add(0);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq);
            taskExecutor.progres();
            ++index;
          } else if (index > 2) {
            LOG.info("Task Index is greater than 1");
            break;
          }
        }
      } else if (containerId == 2) { //This loop should be modified for the complex task graphs
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            ++index;
          } else if (index == 2) {
            ArrayList<Integer> inq1 = new ArrayList<>();
            inq1.add(0);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq1);
            taskExecutor.progres();
            ++index;
          } else if (index > 2) {
            LOG.info("Task Index is greater than 2");
            break;
          }
        }
      } else if (containerId == 3) { //This loop should be modified for the complex task graphs
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            ++index;
          } else if (index == 2) {
            ++index;
          } else if (index == 3) {
            ArrayList<Integer> inq1 = new ArrayList<>();
            inq1.add(1);
            inq1.add(2);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq1);
            taskExecutor.progres();
            ++index;
          } else if (index > 3) {
            //it would be constructed based on the container value and no.of tasks
            LOG.info("Task Index is greater than 3");
            break;
          }
        }
      }
    }*/
}

Example 2 with MessageReceiver

use of edu.iu.dsc.tws.comms.api.MessageReceiver in project twister2 by DSC-SPIDAL.

the class MPIDataFlowAllReduce method init.

/**
 * Initialize
 * @param config
 * @param t
 * @param instancePlan
 * @param edge
 */
public void init(Config config, MessageType t, TaskPlan instancePlan, int edge) {
    this.type = t;
    this.executor = instancePlan.getThisExecutor();
    this.taskPlan = instancePlan;
    this.executor = taskPlan.getThisExecutor();
    broadcast = new MPIDataFlowBroadcast(channel, middleTask, destinations, new BCastReceiver(finalReceiver));
    broadcast.init(config, t, instancePlan, broadCastEdge);
    MessageReceiver receiver;
    if (streaming) {
        this.partialReceiver = new ReduceStreamingPartialReceiver(middleTask, reduceFunction);
        receiver = new AllReduceStreamingFinalReceiver(reduceFunction, broadcast, middleTask);
    } else {
        this.partialReceiver = new ReduceBatchPartialReceiver(middleTask, reduceFunction);
        receiver = new AllReduceBatchFinalReceiver(reduceFunction, broadcast);
    }
    reduce = new MPIDataFlowReduce(channel, sources, middleTask, receiver, partialReceiver);
    reduce.init(config, t, instancePlan, reduceEdge);
}