Examples with DataflowOperation edu.iu.dsc.tws.task.taskgraphbuilder.DataflowOperation used on opensource projects

Example 1 with DataflowOperation

use of edu.iu.dsc.tws.task.taskgraphbuilder.DataflowOperation in project twister2 by DSC-SPIDAL.

the class SimpleTGraphExample method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleTGraphExample.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    TMapper tMapper = new TMapper("1");
    TReducer tReducer = new TReducer("2");
    TShuffler tShuffler = new TShuffler("3");
    TReducer tMergeFinal = new TReducer("4");
    // Add the real input data files in the array list...
    tMapper.addInputData("mapper1", new ArrayList<>());
    tMapper.addInputData("mapper2", new ArrayList<>());
    // Add the real input data files in the array list...
    tReducer.addInputData("reducer1", new ArrayList<>());
    tReducer.addInputData("reducer2", new ArrayList<>());
    // Add the real input data files in the array list...
    tShuffler.addInputData("shuffler1", new ArrayList<>());
    tShuffler.addInputData("shuffler2", new ArrayList<>());
    // Add the real input data files in the array list...
    tMergeFinal.addInputData("merge1", new ArrayList<>());
    tMergeFinal.addInputData("merge2", new ArrayList<>());
    // Mention the output data files to be generated in the array list...
    tMapper.addOutputData("mapperOut1", new ArrayList<>());
    tMapper.addOutputData("mapperOut2", new ArrayList<>());
    if (taskGraphFlag >= 0) {
        // just for verification (replace with proper value)
        /*dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator()
          .generateTGraph(tMapper, tShuffler, new DataflowOperation("Map"))
          .generateTGraph(tMapper, tReducer, new DataflowOperation("Shuffle"))
          .generateTGraph(tShuffler, tReducerFinal, new DataflowOperation("finalReduce"))
          .generateTGraph(tReducer, tReducerFinal, new DataflowOperation("finalReduce"));*/
        dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateTGraph(tMapper).generateTGraph(tMapper, tReducer, new DataflowOperation("Reduce")).generateTGraph(tMapper, tShuffler, new DataflowOperation("Shuffle")).generateTGraph(tReducer, tMergeFinal, new DataflowOperation("Merge1")).generateTGraph(tShuffler, tMergeFinal, new DataflowOperation("Merge2"));
        LOG.info("Generated Dataflow Task Graph Vertices:" + dataflowTaskGraphGenerator.getTGraph().getTaskVertexSet());
        if (dataflowTaskGraphGenerator != null) {
            dataflowTGraphParser = new DataflowTGraphParser(dataflowTaskGraphGenerator);
            parsedTaskSet = dataflowTGraphParser.dataflowTGraphParseAndSchedule();
            LOG.info("parsed task set:" + parsedTaskSet);
        }
        // parsedTaskSet = executionGraph.parseTaskGraph(dataflowTaskGraphGenerator);
        if (!parsedTaskSet.isEmpty()) {
            // newly added for testing
            executionGraph = new ExecutionGraph(parsedTaskSet);
            String message = executionGraph.generateExecutionGraph(containerId);
            // String message = executionGraph.generateExecutionGraph(containerId, parsedTaskSet);
            /*TaskExecutorFixedThread taskExecutionGraph =
            executionGraph.generateExecutionGraph(containerId, parsedTaskSet);*/
            LOG.info(message);
        }
    }
// It removes only the first tax vertex in the parsedTaskSet.
// dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
// It is getting concurrent modification exception...!
/*for (TaskGraphMapper processedTask : parsedTaskSet) {
      dataflowTaskGraphGenerator.removeTaskVertex(processedTask);
    }*/
}

Example 2 with DataflowOperation

use of edu.iu.dsc.tws.task.taskgraphbuilder.DataflowOperation in project twister2 by DSC-SPIDAL.

the class SimpleTGraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleTGraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    // MapWorker sourceTask = new MapWorker(0, direct);
    // ReceiveWorker sinkTask = new ReceiveWorker();
    TMapper tMapper = new TMapper("1");
    TReducer tReducer = new TReducer("2");
    TShuffler tShuffler = new TShuffler("3");
    // Add the real input data files in the array list...
    tMapper.addInputData("mapper1", new ArrayList<>());
    tMapper.addInputData("reducer1", new ArrayList<>());
    // Mention the output data files to be generated in the array list...
    tMapper.addOutputData("mapperOut1", new ArrayList<>());
    tMapper.addOutputData("mapperOut2", new ArrayList<>());
    dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateTGraph(tMapper, tShuffler, new DataflowOperation("Map")).generateTGraph(tShuffler, tReducer, new DataflowOperation("Shuffle"));
    LOG.info("Generated Dataflow Task Graph Vertices:" + dataflowTaskGraphGenerator.getTGraph().getTaskVertexSet());
    LOG.info("Generated Dataflow Task Edges:" + dataflowTaskGraphGenerator.getTGraph().getAllTaskEdges(tMapper, tShuffler).toString());
    LOG.info("Generated Dataflow Task Edges:" + dataflowTaskGraphGenerator.getTGraph().getAllTaskEdges(tShuffler, tReducer).toString());
    if (containerId == 0) {
        Thread mapThread = new Thread(new TMapper("1"));
        LOG.log(Level.INFO, "Starting map thread");
        mapThread.start();
        // we need to progress the communication
        while (true) {
            // progress the channel
            channel.progress();
            // we should progress the communication directive
            direct.progress();
            Thread.yield();
        }
    } else if (containerId == 1) {
        while (status != Status.LOAD_RECEIVE_FINISHED) {
            channel.progress();
            direct.progress();
        }
    }
/*if (dataflowTaskGraphGenerator != null) {
      dataflowTaskGraphParser = new DataflowTaskGraphParser(dataflowTaskGraphGenerator);
      parsedTaskSet = dataflowTaskGraphParser.dataflowTaskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
      if (containerId == 0) {
        LOG.info("Job in if loop is::::::::::::" + parsedTaskSet.iterator().next());
        taskExecutor.registerTask(parsedTaskSet.iterator().next());
        //taskExecutor.registerTask(new MapWorker(0, direct));
        taskExecutor.submitTask(0);
        taskExecutor.progres();
        ///dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
      } else if (containerId == 1) {
        int index = 0;
        for (Task processedTask : parsedTaskSet) {
          if (index == 0) {
            ++index;
          } else if (index == 1) {
            LOG.info("Job in else loop is::::::::::::" + processedTask);
            ArrayList<Integer> inq = new ArrayList<>();
            inq.add(0);
            taskExecutor.setTaskMessageProcessLimit(10000);
            taskExecutor.registerSinkTask(processedTask, inq);
            taskExecutor.progres();
            ///dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
            ++index;
          } else if (index > 1) { //Just for verification
            LOG.info("Task Index is greater than 1");
            LOG.info("Submit the job to pipeline task");
            break;
          }
        }
      }
    }*/
}

Example 3 with DataflowOperation

use of edu.iu.dsc.tws.task.taskgraphbuilder.DataflowOperation in project twister2 by DSC-SPIDAL.

the class SimpleTaskgraph method init.

/**
 * Init method to submit the task to the executor
 */
public void init(Config cfg, int containerId, ResourcePlan plan) {
    LOG.log(Level.INFO, "Starting the example with container id: " + plan.getThisId());
    taskExecutor = new TaskExecutorFixedThread();
    this.status = Status.INIT;
    TaskPlan taskPlan = Utils.createTaskPlan(cfg, plan);
    TWSNetwork network = new TWSNetwork(cfg, taskPlan);
    TWSCommunication channel = network.getDataFlowTWSCommunication();
    Set<Integer> sources = new HashSet<>();
    sources.add(0);
    int destination = 1;
    Map<String, Object> newCfg = new HashMap<>();
    LinkedQueue<Message> pongQueue = new LinkedQueue<Message>();
    taskExecutor.registerQueue(0, pongQueue);
    direct = channel.direct(newCfg, MessageType.OBJECT, 0, sources, destination, new SimpleTaskgraph.PingPongReceive());
    taskExecutor.initCommunication(channel, direct);
    // For Dataflow Task Graph Generation call the dataflow task graph generator
    MapWorker sourceTask = new MapWorker(0, direct);
    ReceiveWorker sinkTask = new ReceiveWorker();
    // commented this line for separating the communication component
    // dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateDataflowGraph(
    // sourceTask, sinkTask, direct);
    dataflowTaskGraphGenerator = new DataflowTaskGraphGenerator().generateTaskGraph(sourceTask, sinkTask, new DataflowOperation("Map"));
    if (dataflowTaskGraphGenerator != null) {
        taskGraphParser = new TaskGraphParser(dataflowTaskGraphGenerator);
        parsedTaskSet = taskGraphParser.taskGraphParseAndSchedule();
    }
    if (!parsedTaskSet.isEmpty()) {
        if (containerId == 0) {
            LOG.info("Job in if loop is::::::::::::" + parsedTaskSet.iterator().next());
            taskExecutor.registerTask(parsedTaskSet.iterator().next());
            // taskExecutor.registerTask(new MapWorker(0, direct));
            taskExecutor.submitTask(0);
            taskExecutor.progres();
        // /dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
        } else if (containerId == 1) {
            int index = 0;
            for (Task processedTask : parsedTaskSet) {
                if (index == 0) {
                    ++index;
                } else if (index == 1) {
                    LOG.info("Job in else loop is::::::::::::" + processedTask);
                    ArrayList<Integer> inq = new ArrayList<>();
                    inq.add(0);
                    // 10000
                    taskExecutor.setTaskMessageProcessLimit(100);
                    taskExecutor.registerSinkTask(processedTask, inq);
                    taskExecutor.progres();
                    // /dataflowTaskGraphGenerator.removeTaskVertex(parsedTaskSet.iterator().next());
                    ++index;
                } else if (index > 1) {
                    // Just for verification
                    LOG.info("Task Index is greater than 1");
                    LOG.info("Submit the job to pipeline task");
                    break;
                }
            }
        }
    }
}