Examples with StreamApplicationDescriptorImpl org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl used on opensource projects

Example 81 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithInvalidStreamTableJoin.

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
    /**
     * Example stream-table join that is invalid due to disagreement in partition count
     * between the 2 input streams.
     *
     *    input1 (64) -> send-to-table t
     *
     *                   join-table t -> output1 (8)
     *                         |
     *    input2 (16) —————————
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
        MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde()));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream1.sendTo(table);
        messageStream1.join(table, mock(StreamTableJoinFunction.class)).join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2").sendTo(output1);
    }, config);
}

Example 82 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalWhenWindowMsIsConfigured.

@Test
public void testTriggerIntervalWhenWindowMsIsConfigured() {
    Map<String, String> map = new HashMap<>(config);
    map.put(TaskConfig.WINDOW_MS, "2000");
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createSimpleGraph();
    ExecutionPlan plan = planner.plan(graphSpec);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(1, jobConfigs.size());
    assertEquals("2000", jobConfigs.get(0).get(TaskConfig.WINDOW_MS));
}

Example 83 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testCalculateJoinInputPartitions.

@Test
public void testCalculateJoinInputPartitions() {
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamStreamJoin();
    JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
    // Partitions should be the same as input1
    jobGraph.getIntermediateStreams().forEach(edge -> {
        assertEquals(64, edge.getPartitionCount());
    });
}

Example 84 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testHandlesVirtualStreamTableJoinCycles.

@Test
public void testHandlesVirtualStreamTableJoinCycles() {
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamTableJoinAndSendToSameTable();
    // Just make sure planning terminates.
    planner.plan(graphSpec);
}

Example 85 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalWithNoWindowMs.

@Test
public void testTriggerIntervalWithNoWindowMs() {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithJoinAndWindow();
    ExecutionPlan plan = planner.plan(graphSpec);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(1, jobConfigs.size());
    // GCD of 8, 16, 1600 and 252 is 4
    assertEquals("4", jobConfigs.get(0).get(TaskConfig.WINDOW_MS));
}