Examples with StreamApplicationDescriptorImpl org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl used on opensource projects

Example 91 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testFetchExistingStreamPartitions.

@Test
public void testFetchExistingStreamPartitions() {
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamStreamJoin();
    JobGraph jobGraph = planner.createJobGraph(graphSpec);
    planner.setInputAndOutputStreamPartitionCount(jobGraph);
    assertTrue(jobGraph.getOrCreateStreamEdge(input1Spec).getPartitionCount() == 64);
    assertTrue(jobGraph.getOrCreateStreamEdge(input2Spec).getPartitionCount() == 16);
    assertTrue(jobGraph.getOrCreateStreamEdge(input3Spec).getPartitionCount() == 32);
    assertTrue(jobGraph.getOrCreateStreamEdge(output1Spec).getPartitionCount() == 8);
    assertTrue(jobGraph.getOrCreateStreamEdge(output2Spec).getPartitionCount() == 16);
    jobGraph.getIntermediateStreamEdges().forEach(edge -> {
        assertTrue(edge.getPartitionCount() == -1);
    });
}

Example 92 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testMaxPartitionLimit.

@Test
public void testMaxPartitionLimit() {
    int partitionLimit = IntermediateStreamManager.MAX_INFERRED_PARTITIONS;
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input4Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        input1.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").map(kv -> kv).sendTo(output1);
    }, config);
    JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
    // Partitions should be the same as input1
    jobGraph.getIntermediateStreams().forEach(edge -> {
        // max of input1 and output1
        assertEquals(partitionLimit, edge.getPartitionCount());
    });
}

Example 93 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testRejectsInvalidStreamTableJoin.

@Test(expected = SamzaException.class)
public void testRejectsInvalidStreamTableJoin() {
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithInvalidStreamTableJoin();
    planner.plan(graphSpec);
}

Example 94 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForWindowsAndJoins.

@Test
public void testTriggerIntervalForWindowsAndJoins() {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithJoinAndWindow();
    ExecutionPlan plan = planner.plan(graphSpec);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(1, jobConfigs.size());
    // GCD of 8, 16, 1600 and 252 is 4
    assertEquals("4", jobConfigs.get(0).get(TaskConfig.WINDOW_MS));
}

Example 95 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testRejectsInvalidStreamTableJoinWithSideInputs.

@Test(expected = SamzaException.class)
public void testRejectsInvalidStreamTableJoinWithSideInputs() {
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithInvalidStreamTableJoinWithSideInputs();
    planner.plan(graphSpec);
}