Examples with ConnectedSink edu.iu.dsc.tws.task.cdfw.task.ConnectedSink used on opensource projects

Example 1 with ConnectedSink

use of edu.iu.dsc.tws.task.cdfw.task.ConnectedSink in project twister2 by DSC-SPIDAL.

the class ParallelDataFlowsExample method generateSecondJob.

private static DataFlowGraph generateSecondJob(Config config, int parallelismValue, int workers, DataFlowJobConfig jobConfig) {
    ConnectedSource connectedSource = new ConnectedSource("reduce", "first_out");
    ConnectedSink connectedSink = new ConnectedSink();
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", connectedSource, parallelismValue);
    ComputeConnection reduceConn = graphBuilderX.addCompute("sink1", connectedSink, 1);
    reduceConn.reduce("source1").viaEdge("reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("second_graph", batchGraph).setWorkers(workers).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 2 with ConnectedSink

use of edu.iu.dsc.tws.task.cdfw.task.ConnectedSink in project twister2 by DSC-SPIDAL.

the class ParallelDataFlowsExample method generateFirstJob.

private static DataFlowGraph generateFirstJob(Config config, int parallelismValue, int workers, DataFlowJobConfig jobConfig) {
    FirstSourceTask firstSourceTask = new FirstSourceTask();
    ConnectedSink connectedSink = new ConnectedSink("first_out");
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", firstSourceTask, parallelismValue);
    ComputeConnection partitionConnection = graphBuilderX.addCompute("sink1", connectedSink, parallelismValue);
    partitionConnection.partition("source1").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("first_graph", batchGraph).setWorkers(workers).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    return job;
}

Example 3 with ConnectedSink

use of edu.iu.dsc.tws.task.cdfw.task.ConnectedSink in project twister2 by DSC-SPIDAL.

the class TwoDataFlowsExample method runFirstJob.

private static void runFirstJob(Config config, CDFWEnv cdfwEnv, int parallelism, DataFlowJobConfig jobConfig) {
    FirstSourceTask firstSourceTask = new FirstSourceTask();
    ConnectedSink connectedSink = new ConnectedSink("first_out");
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source1", firstSourceTask, parallelism);
    ComputeConnection partitionConnection = graphBuilderX.addCompute("sink1", connectedSink, parallelism);
    partitionConnection.partition("source1").viaEdge("partition").withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("first_graph", batchGraph).setWorkers(4).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}

Example 4 with ConnectedSink

use of edu.iu.dsc.tws.task.cdfw.task.ConnectedSink in project twister2 by DSC-SPIDAL.

the class TwoDataFlowsExample method runSecondJob.

private static void runSecondJob(Config config, CDFWEnv cdfwEnv, int parallelism, DataFlowJobConfig jobConfig) {
    ConnectedSource connectedSource = new ConnectedSource("reduce", "first_out");
    ConnectedSink connectedSink = new ConnectedSink();
    ComputeGraphBuilder graphBuilderX = ComputeGraphBuilder.newBuilder(config);
    graphBuilderX.addSource("source2", connectedSource, parallelism);
    ComputeConnection reduceConn = graphBuilderX.addCompute("sink2", connectedSink, 1);
    reduceConn.reduce("source2").viaEdge("reduce").withReductionFunction(new Aggregator()).withDataType(MessageTypes.OBJECT);
    graphBuilderX.setMode(OperationMode.BATCH);
    ComputeGraph batchGraph = graphBuilderX.build();
    DataFlowGraph job = DataFlowGraph.newSubGraphJob("second_graph", batchGraph).setWorkers(4).addDataFlowJobConfig(jobConfig).setGraphType("non-iterative");
    cdfwEnv.executeDataFlowGraph(job);
}