Examples with StreamApplicationDescriptorImpl org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl used on opensource projects

Example 96 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForJoins.

@Test
public void testTriggerIntervalForJoins() {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamStreamJoin();
    ExecutionPlan plan = planner.plan(graphSpec);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    for (JobConfig config : jobConfigs) {
        System.out.println(config);
    }
}

Example 97 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithStreamTableJoinAndSendToSameTable.

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinAndSendToSameTable() {
    /**
     * A special example of stream-table join where a stream is joined with a table, and the result is
     * sent to the same table. This example is necessary to ensure {@link ExecutionPlanner} does not
     * get stuck traversing the virtual cycle between stream-table-join and send-to-table operator specs
     * indefinitely.
     *
     * The reason such virtual cycle is present is to support computing partitions of intermediate
     * streams participating in stream-table joins. Please, refer to SAMZA SEP-16 for more details.
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde()));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream1.join(table, mock(StreamTableJoinFunction.class)).sendTo(table);
    }, config);
}

Example 98 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJobGraph method testAddSource.

@Test
public void testAddSource() {
    StreamApplicationDescriptorImpl appDesc = mock(StreamApplicationDescriptorImpl.class);
    JobGraph graph = new JobGraph(new MapConfig(), appDesc);
    /**
     * s1 -> 1
     * s2 ->|
     *
     * s3 -> 2
     *   |-> 3
     */
    JobNode n1 = graph.getOrCreateJobNode("1", "1");
    JobNode n2 = graph.getOrCreateJobNode("2", "1");
    JobNode n3 = graph.getOrCreateJobNode("3", "1");
    StreamSpec s1 = genStream();
    StreamSpec s2 = genStream();
    StreamSpec s3 = genStream();
    graph.addInputStream(s1, n1);
    graph.addInputStream(s2, n1);
    graph.addInputStream(s3, n2);
    graph.addInputStream(s3, n3);
    assertTrue(graph.getInputStreams().size() == 3);
    assertTrue(graph.getOrCreateJobNode("1", "1").getInEdges().size() == 2);
    assertTrue(graph.getOrCreateJobNode("2", "1").getInEdges().size() == 1);
    assertTrue(graph.getOrCreateJobNode("3", "1").getInEdges().size() == 1);
    assertTrue(graph.getOrCreateStreamEdge(s1).getSourceNodes().size() == 0);
    assertTrue(graph.getOrCreateStreamEdge(s1).getTargetNodes().size() == 1);
    assertTrue(graph.getOrCreateStreamEdge(s2).getSourceNodes().size() == 0);
    assertTrue(graph.getOrCreateStreamEdge(s2).getTargetNodes().size() == 1);
    assertTrue(graph.getOrCreateStreamEdge(s3).getSourceNodes().size() == 0);
    assertTrue(graph.getOrCreateStreamEdge(s3).getTargetNodes().size() == 2);
}

Example 99 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJobGraph method createGraph1.

/**
 * graph1 is the example graph from wikipedia
 *
 * 5   7   3
 * | / | / |
 * v   v   |
 * 11  8   |
 * | \X   /
 * v v \v
 * 2 9 10
 */
private void createGraph1() {
    StreamApplicationDescriptorImpl appDesc = mock(StreamApplicationDescriptorImpl.class);
    graph1 = new JobGraph(new MapConfig(), appDesc);
    JobNode n2 = graph1.getOrCreateJobNode("2", "1");
    JobNode n3 = graph1.getOrCreateJobNode("3", "1");
    JobNode n5 = graph1.getOrCreateJobNode("5", "1");
    JobNode n7 = graph1.getOrCreateJobNode("7", "1");
    JobNode n8 = graph1.getOrCreateJobNode("8", "1");
    JobNode n9 = graph1.getOrCreateJobNode("9", "1");
    JobNode n10 = graph1.getOrCreateJobNode("10", "1");
    JobNode n11 = graph1.getOrCreateJobNode("11", "1");
    graph1.addInputStream(genStream(), n5);
    graph1.addInputStream(genStream(), n7);
    graph1.addInputStream(genStream(), n3);
    graph1.addIntermediateStream(genStream(), n5, n11);
    graph1.addIntermediateStream(genStream(), n7, n11);
    graph1.addIntermediateStream(genStream(), n7, n8);
    graph1.addIntermediateStream(genStream(), n3, n8);
    graph1.addIntermediateStream(genStream(), n11, n2);
    graph1.addIntermediateStream(genStream(), n11, n9);
    graph1.addIntermediateStream(genStream(), n8, n9);
    graph1.addIntermediateStream(genStream(), n11, n10);
    graph1.addOutputStream(genStream(), n2);
    graph1.addOutputStream(genStream(), n9);
    graph1.addOutputStream(genStream(), n10);
}

Example 100 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJobGraph method createGraph2.

/**
 * graph2 is a graph with a loop
 * 1 -> 2 -> 3 -> 4 -> 5 -> 7
 *      |<---6 <--|    <>
 */
private void createGraph2() {
    StreamApplicationDescriptorImpl appDesc = mock(StreamApplicationDescriptorImpl.class);
    graph2 = new JobGraph(new MapConfig(), appDesc);
    JobNode n1 = graph2.getOrCreateJobNode("1", "1");
    JobNode n2 = graph2.getOrCreateJobNode("2", "1");
    JobNode n3 = graph2.getOrCreateJobNode("3", "1");
    JobNode n4 = graph2.getOrCreateJobNode("4", "1");
    JobNode n5 = graph2.getOrCreateJobNode("5", "1");
    JobNode n6 = graph2.getOrCreateJobNode("6", "1");
    JobNode n7 = graph2.getOrCreateJobNode("7", "1");
    graph2.addInputStream(genStream(), n1);
    graph2.addIntermediateStream(genStream(), n1, n2);
    graph2.addIntermediateStream(genStream(), n2, n3);
    graph2.addIntermediateStream(genStream(), n3, n4);
    graph2.addIntermediateStream(genStream(), n4, n5);
    graph2.addIntermediateStream(genStream(), n4, n6);
    graph2.addIntermediateStream(genStream(), n6, n2);
    graph2.addIntermediateStream(genStream(), n5, n5);
    graph2.addIntermediateStream(genStream(), n5, n7);
    graph2.addOutputStream(genStream(), n7);
}