Examples with TWSNetwork edu.iu.dsc.tws.comms.core.TWSNetwork used on opensource projects

Example 21 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class BasePartitionKeyedCommunication method init.

@Override
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    this.config = cfg;
    this.resourcePlan = plan;
    this.id = containerId;
    this.status = Status.INIT;
    this.noOfTasksPerExecutor = NO_OF_TASKS / plan.noOfContainers();
    // lets create the task plan
    TaskPlan taskPlan = Utils.createReduceTaskPlan(cfg, plan, NO_OF_TASKS);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    Set<Integer> dests = new HashSet<>();
    for (int i = 0; i < NO_OF_TASKS; i++) {
        sources.add(i);
        dests.add(i);
    }
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up partition dataflow operation");
    try {
        // this method calls the init method
        // I think this is wrong
        Map<Integer, List<Integer>> expectedIds = new HashMap<>();
        for (int i = 0; i < NO_OF_TASKS; i++) {
            expectedIds.put(i, new ArrayList<>());
            for (int j = 0; j < NO_OF_TASKS; j++) {
                if (!(i == j)) {
                    expectedIds.get(i).add(j);
                }
            }
        }
        FinalPartitionReciver finalPartitionRec = new FinalPartitionReciver();
        partition = channel.partition(newCfg, MessageType.INTEGER, MessageType.INTEGER, 2, sources, dests, finalPartitionRec);
        // partition = channel.partition(newCfg, MessageType.INTEGER, MessageType.INTEGER, 2, sources,
        // dests, finalPartitionRec);
        finalPartitionRec.setMap(expectedIds);
        for (int i = 0; i < noOfTasksPerExecutor; i++) {
            // the map thread where data is produced
            LOG.info(String.format("%d Starting %d", id, i + id * noOfTasksPerExecutor));
            Thread mapThread = new Thread(new MapWorker(i + id * noOfTasksPerExecutor));
            mapThread.start();
        }
        // we need to progress the communication
        while (true) {
            try {
                // progress the channel
                channel.progress();
                // we should progress the communication directive
                partition.progress();
                Thread.yield();
            } catch (Throwable t) {
                t.printStackTrace();
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}

Example 22 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class MultiTaskGraphExample method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new MultiTaskGraphExample.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    direct1 = channel.direct(newCfg, MessageType.OBJECT, 1, sources, destination, new MultiTaskGraphExample.PingPongReceive());
    taskExecutor.initCommunication(channel, direct1);
    MessageReceiver receiver = null;
    gather = channel.gather(newCfg, MessageType.OBJECT, 0, sources, destination, receiver);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask1 = new ReceiveWorker();
    ReceiveWorker sinkTask2 = new ReceiveWorker();
    ReceiveWorker sinkTask3 = new ReceiveWorker();
    /* Source Task (Task0) ------> SinkTask1 (Task 1)
   *        |               (1)      |
   *        |                        |
   *   (1)  |                        |  (2)
   *        |                        |
   *        |                        |
   *        V               (2)      V
   *   SinkTask2 (Task 2) -----> SinkTask3 (Task 3)
   *
   *   Here, (1) represents Task 1 and Task 2 starts simultaneously (receive input
    *  from source task (Task 0), whereas (2) represents Task 3 receives input
    *  from Task 1 & Task 2.
   */
    dataflowTaskGraph = new DataflowTaskGraphGenerator().generateDataflowGraph(sourceTask, sinkTask1, direct).generateDataflowGraph(sourceTask, sinkTask2, direct1).generateDataflowGraph(sinkTask1, sinkTask3, direct1).generateDataflowGraph(sinkTask2, sinkTask3, direct1);
    if (dataflowTaskGraph != null) {
        dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraph);
        parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    // This code is for moving the explicit scheduling outside of the example program
    if (!parsedTaskSet.isEmpty()) {
        TaskGraphScheduler taskGraphScheduler = new TaskGraphScheduler();
        if (containerId == 0) {
            LOG.log(Level.INFO, "Parsed Job Value:" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        } else if (containerId > 0) {
            Map<Task, ArrayList<Integer>> taskMap = taskGraphScheduler.taskgraphScheduler(parsedTaskSet, containerId);
            taskExecutor.setTaskMessageProcessLimit(10000);
            for (Map.Entry<Task, ArrayList<Integer>> taskEntry : taskMap.entrySet()) {
                taskExecutor.registerSinkTask(taskEntry.getKey(), taskEntry.getValue());
                taskExecutor.progres();
            }
        }
    }
/*if (!parsedTaskSet.isEmpty()) {
      if (containerId == 0) {
        LOG.log(Level.INFO, "Parsed Job Value:" + parsedTaskSet.iterator().next());
        taskExecutor.registerTask(parsedTaskSet.iterator().next());
        taskExecutor.submitTask(0);
        taskExecutor.progres();
      } else if (containerId == 1) {
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            ArrayList<Integer> inq = new ArrayList<>();
            inq.add(0);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq);
            taskExecutor.progres();
            ++index;
          } else if (index > 2) {
            LOG.info("Task Index is greater than 1");
            break;
          }
        }
      } else if (containerId == 2) { //This loop should be modified for the complex task graphs
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            ++index;
          } else if (index == 2) {
            ArrayList<Integer> inq1 = new ArrayList<>();
            inq1.add(0);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq1);
            taskExecutor.progres();
            ++index;
          } else if (index > 2) {
            LOG.info("Task Index is greater than 2");
            break;
          }
        }
      } else if (containerId == 3) { //This loop should be modified for the complex task graphs
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            ++index;
          } else if (index == 2) {
            ++index;
          } else if (index == 3) {
            ArrayList<Integer> inq1 = new ArrayList<>();
            inq1.add(1);
            inq1.add(2);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq1);
            taskExecutor.progres();
            ++index;
          } else if (index > 3) {
            //it would be constructed based on the container value and no.of tasks
            LOG.info("Task Index is greater than 3");
            break;
          }
        }
      }
    }*/
}

Example 23 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class SimpleCxMultiTaskGraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleCxMultiTaskGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    direct1 = channel.direct(newCfg, MessageType.OBJECT, 1, sources, destination, new SimpleCxMultiTaskGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct1);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask = new ReceiveWorker();
    ReceiveWorker sinkTask1 = new ReceiveWorker();
    // task 0 -> task 1 direct communication channel
    // task 0 -> task 2 create different task edge or communication channel
    dataflowTaskGraph = new DataflowTaskGraphGenerator().generateDataflowGraph(sourceTask, sinkTask, direct).generateDataflowGraph(sourceTask, sinkTask1, direct1);
    if (dataflowTaskGraph != null) {
        dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraph);
        parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
        if (containerId == 0) {
            LOG.log(Level.INFO, "Job In If Loop" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        } else if (containerId >= 1) {
            // This loop should be modified for the complex task graphs
            int index = 0;
            for (Task processedTask : parsedTaskSet) {
                if (index == 0) {
                    ++index;
                } else if (index == 1) {
                    ArrayList<Integer> inq = new ArrayList<>();
                    inq.add(0);
                    taskExecutor.setTaskMessageProcessLimit(10000);
                    taskExecutor.registerSinkTask(processedTask, inq);
                    taskExecutor.progres();
                    ++index;
                } else if (index > 1) {
                    ArrayList<Integer> inq1 = new ArrayList<>();
                    inq1.add(0);
                    taskExecutor.setTaskMessageProcessLimit(10000);
                    taskExecutor.registerSinkTask(processedTask, inq1);
                    taskExecutor.progres();
                    ++index;
                } else if (index > 2) {
                    // it would be constructed based on the container value and no.of tasks
                    LOG.info("Task Index is greater than 2");
                    break;
                }
            }
        }
    }
}

Example 24 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class SimpleTGraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleTGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    // MapWorker sourceTask = new MapWorker(0, direct);
    // ReceiveWorker sinkTask = new ReceiveWorker();
    TMapper tMapper = new TMapper("1");
    TReducer tReducer = new TReducer("2");
    TShuffler tShuffler = new TShuffler("3");
    // Add the real input data files in the array list...
    tMapper.addInputData("mapper1", new ArrayList<>());
    tMapper.addInputData("reducer1", new ArrayList<>());
    // Mention the output data files to be generated in the array list...
    tMapper.addOutputData("mapperOut1", new ArrayList<>());
    tMapper.addOutputData("mapperOut2", new ArrayList<>());
    dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateTGraph(tMapper, tShuffler, new DataflowOperation("Map")).generateTGraph(tShuffler, tReducer, new DataflowOperation("Shuffle"));
    LOG.info("Generated Dataflow Task Graph Vertices:" + dataflowTaskGraphGenerator.getTGraph().getTaskVertexSet());
    LOG.info("Generated Dataflow Task Edges:" + dataflowTaskGraphGenerator.getTGraph().getAllTaskEdges(tMapper, tShuffler).toString());
    LOG.info("Generated Dataflow Task Edges:" + dataflowTaskGraphGenerator.getTGraph().getAllTaskEdges(tShuffler, tReducer).toString());
    if (containerId == 0) {
        Thread mapThread = new Thread(new TMapper("1"));
        LOG.log(Level.INFO, "Starting map thread");
        mapThread.start();
        // we need to progress the communication
        while (true) {
            // progress the channel
            channel.progress();
            // we should progress the communication directive
            direct.progress();
            Thread.yield();
        }
    } else if (containerId == 1) {
        while (status != Status.LOAD_RECEIVE_FINISHED) {
            channel.progress();
            direct.progress();
        }
    }
/*if (dataflowTaskGraphGenerator != null) {
      dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraphGenerator);
      parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
      if (containerId == 0) {
        LOG.info("Job in if loop is::::::::::::" + parsedTaskSet.iterator().next());
        taskExecutor.registerTask(parsedTaskSet.iterator().next());
        //taskExecutor.registerTask(new MapWorker(0, direct));
        taskExecutor.submitTask(0);
        taskExecutor.progres();
        ///dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
      } else if (containerId == 1) {
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            LOG.info("Job in else loop is::::::::::::" + processedTask);
            ArrayList<Integer> inq = new ArrayList<>();
            inq.add(0);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq);
            taskExecutor.progres();
            ///dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
            ++index;
          } else if (index > 1) { //Just for verification
            LOG.info("Task Index is greater than 1");
            LOG.info("Submit the job to pipeline task");
            break;
          }
        }
      }
    }*/
}

Example 25 with TWSNetwork

use of edu.iu.dsc.tws.comms.core.TWSNetwork in project twister2 by DSC-SPIDAL.

the class SimpleTaskQueueWithMM method init.

/**
 * Initialize the container
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    // Creates task an task executor instance to be used in this container
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    // lets create the task plan
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    // first get the communication config file
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    // we are sending messages from 0th task to 1st task
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int dests = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LOG.info("Setting up reduce dataflow operation");
    Path dataPath = new Path("/home/pulasthi/work/twister2/lmdbdatabase");
    MemoryManager memoryManager = new LMDBMemoryManager(dataPath);
    // this method calls the init method
    // I think this is wrong
    // TODO: Does the task genereate the communication or is it done by a controller for examples
    // the direct comm between task 0 and 1 is it done by the container or the the task
    // TODO: if the task creates the dataflowop does the task progress it or the executor
    // TODO : FOR NOW the dataflowop is created at container and sent to task
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, dests, new PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // Memory Manager
    if (containerId == 0) {
        byte[] val = Longs.toByteArray(1231212121213L);
        byte[] val2 = Longs.toByteArray(22222222L);
        ByteBuffer valbuf = ByteBuffer.allocateDirect(8192);
        memoryManager.put(0, "temp", valbuf);
    // memoryManager.put(0, "temp", val);
    // memoryManager.put(0, "temp", val2);
    // the map thread where data is produced
    // LOG.log(Level.INFO, "Starting map thread");
    // SourceTask<Object> mapTask = new MapWorker(0, direct);
    // mapTask.setMemoryManager(memoryManager);
    // taskExecutor.registerTask(mapTask);
    // taskExecutor.submitTask(0);
    // taskExecutor.progres();
    } else if (containerId == 1) {
        byte[] val3 = Longs.toByteArray(3333333L);
        ByteBuffer val3buf = ByteBuffer.wrap(val3);
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        ByteBuffer results = memoryManager.get(0, "temp");
        if (results.limit() == 8192) {
            System.out.println("Correct " + results.limit());
        }
        ByteBuffer valbuf2 = ByteBuffer.allocateDirect(16192);
        memoryManager.put(0, "temp", valbuf2);
        results = memoryManager.get(0, "temp");
        if (results.limit() == 16192) {
            System.out.println("Correct " + results.limit());
        }
        ByteBuffer results2 = memoryManager.get(0, "temp");
        ByteBuffer results3 = memoryManager.get(0, "temp");
        if (results2 == null) {
            System.out.println("Missing key is null");
        }
        if (results3.getLong() == 1231212121213L) {
            System.out.println("Long value is correct");
        }
        memoryManager.append(0, "temp", val3buf);
        ByteBuffer resultsappend = memoryManager.get(0, "temp");
        System.out.println("Long value 1 :" + resultsappend.getLong());
        System.out.println("Long value 1 :" + resultsappend.getLong());
    // ArrayList<Integer> inq = new ArrayList<>();
    // inq.add(0);
    // taskExecutor.setTaskMessageProcessLimit(10000);
    // SinkTask<Object> recTask = new RecieveWorker(1);
    // recTask.setMemoryManager(memoryManager);
    // taskExecutor.registerSinkTask(recTask, inq);
    // taskExecutor.progres();
    }
}