Examples with SinkTask edu.iu.dsc.tws.task.api.SinkTask used on opensource projects

Example 1 with SinkTask

use of edu.iu.dsc.tws.task.api.SinkTask in project twister2 by DSC-SPIDAL.

the class SimpleMultiTaskGraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = SimpleMultiTaskGraph.Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleMultiTaskGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask = new ReceiveWorker(1);
    // later we can add a different task
    ReceiveWorker sinkTask1 = new ReceiveWorker(2);
    dataflowTaskGraph = new DataflowTaskGraphGenerator().generateDataflowGraph(sourceTask, sinkTask, direct).generateDataflowGraph(sinkTask, sinkTask1, direct);
    if (dataflowTaskGraph != null) {
        dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraph);
        parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    // the taskgraph scheduler is constructed...!
    if (!parsedTaskSet.isEmpty() && containerId > 1) {
        List<Task> taskList = new ArrayList<>();
        for (Task processedTasks : parsedTaskSet) {
            taskList.add(processedTasks);
        }
        LOG.info("Submitting Pipeline Task:" + taskList.size());
        taskExecutionOptimizer = new TaskExecutionOptimizer(taskExecutor);
        PipelinedTask pipelinedTask = new PipelinedTask(1, taskList);
        pipelinedTask.execute();
    }
}

Example 2 with SinkTask

use of edu.iu.dsc.tws.task.api.SinkTask in project twister2 by DSC-SPIDAL.

the class SimpleTaskGraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleTaskGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask = new ReceiveWorker();
    dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateDataflowGraph(sourceTask, sinkTask, direct);
    if (dataflowTaskGraphGenerator != null) {
        dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraphGenerator);
        parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
        if (containerId == 0) {
            LOG.info("Job in if loop is::::::::::::" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            // taskExecutor.registerTask(new MapWorker(0, direct));
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        // /dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
        } else if (containerId == 1) {
            int index = 0;
            for (Task processedTask : parsedTaskSet) {
                if (index == 0) {
                    ++index;
                } else if (index == 1) {
                    LOG.info("Job in else loop is::::::::::::" + processedTask);
                    ArrayList<Integer> inq = new ArrayList<>();
                    inq.add(0);
                    taskExecutor.setTaskMessageProcessLimit(10000);
                    taskExecutor.registerSinkTask(processedTask, inq);
                    taskExecutor.progres();
                    // /dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
                    ++index;
                } else if (index > 1) {
                    // Just for verification
                    LOG.info("Task Index is greater than 1");
                    LOG.info("Submit the job to pipeline task");
                    break;
                }
            }
        }
    }
// This scheduling loop will be used in the future, leave it for reference.
/*int index = 0;
    if (!parsedTaskSet.isEmpty()) {
      for (Task processedTask : parsedTaskSet) {
        if (containerId == index) {
          taskExecutor.registerTask(processedTask);
          taskExecutor.submitTask(0);
          taskExecutor.progres();
          index++;
        } else if (index == 1) {
          ArrayList<Integer> inq = new ArrayList<>();
          inq.add(0);
          taskExecutor.setTaskMessageProcessLimit(10000);
          taskExecutor.registerSinkTask(processedTask, inq);
          taskExecutor.progres();
          index++;
        } else if(index > 1){
          List<Task> taskList = new ArrayList<>();
          for (Task processedTasks : parsedTaskSet) {
            taskList.add(processedTasks);
          }
          //This loop should be properly written...!
          taskExecutionOptimizer = new TaskExecutionOptimizer(taskExecutor);
          Map<Integer, List<Task>> taskMap = new HashMap<>();
          taskMap.put(1, taskList);
        }
      }
    }*/
}

Example 3 with SinkTask

use of edu.iu.dsc.tws.task.api.SinkTask in project twister2 by DSC-SPIDAL.

the class SimpleCxMultiTaskGraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleCxMultiTaskGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    direct1 = channel.direct(newCfg, MessageType.OBJECT, 1, sources, destination, new SimpleCxMultiTaskGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct1);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask = new ReceiveWorker();
    ReceiveWorker sinkTask1 = new ReceiveWorker();
    // task 0 -> task 1 direct communication channel
    // task 0 -> task 2 create different task edge or communication channel
    dataflowTaskGraph = new DataflowTaskGraphGenerator().generateDataflowGraph(sourceTask, sinkTask, direct).generateDataflowGraph(sourceTask, sinkTask1, direct1);
    if (dataflowTaskGraph != null) {
        dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraph);
        parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
        if (containerId == 0) {
            LOG.log(Level.INFO, "Job In If Loop" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        } else if (containerId >= 1) {
            // This loop should be modified for the complex task graphs
            int index = 0;
            for (Task processedTask : parsedTaskSet) {
                if (index == 0) {
                    ++index;
                } else if (index == 1) {
                    ArrayList<Integer> inq = new ArrayList<>();
                    inq.add(0);
                    taskExecutor.setTaskMessageProcessLimit(10000);
                    taskExecutor.registerSinkTask(processedTask, inq);
                    taskExecutor.progres();
                    ++index;
                } else if (index > 1) {
                    ArrayList<Integer> inq1 = new ArrayList<>();
                    inq1.add(0);
                    taskExecutor.setTaskMessageProcessLimit(10000);
                    taskExecutor.registerSinkTask(processedTask, inq1);
                    taskExecutor.progres();
                    ++index;
                } else if (index > 2) {
                    // it would be constructed based on the container value and no.of tasks
                    LOG.info("Task Index is greater than 2");
                    break;
                }
            }
        }
    }
}

Example 4 with SinkTask

use of edu.iu.dsc.tws.task.api.SinkTask in project twister2 by DSC-SPIDAL.

the class SimpleTaskgraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleTaskgraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask = new ReceiveWorker();
    // commented this line for separating the communication component
    // dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateDataflowGraph(
    // sourceTask, sinkTask, direct);
    dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateTaskGraph(sourceTask, sinkTask, new DataflowOperation("Map"));
    if (dataflowTaskGraphGenerator != null) {
        taskGraphParser = new TaskGraphParser(dataflowTaskGraphGenerator);
        parsedTaskSet = taskGraphParser.taskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
        if (containerId == 0) {
            LOG.info("Job in if loop is::::::::::::" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            // taskExecutor.registerTask(new MapWorker(0, direct));
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        // /dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
        } else if (containerId == 1) {
            int index = 0;
            for (Task processedTask : parsedTaskSet) {
                if (index == 0) {
                    ++index;
                } else if (index == 1) {
                    LOG.info("Job in else loop is::::::::::::" + processedTask);
                    ArrayList<Integer> inq = new ArrayList<>();
                    inq.add(0);
                    // 10000
                    taskExecutor.setTaskMessageProcessLimit(100);
                    taskExecutor.registerSinkTask(processedTask, inq);
                    taskExecutor.progres();
                    // /dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
                    ++index;
                } else if (index > 1) {
                    // Just for verification
                    LOG.info("Task Index is greater than 1");
                    LOG.info("Submit the job to pipeline task");
                    break;
                }
            }
        }
    }
}