use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method createStreamGraphWithStreamTableJoinWithSideInputs.
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() {
/**
* Example stream-table join where table t is configured with input1 (64) as a side-input stream.
*
* join-table t -> output1 (8)
* |
* input2 (16) -> partitionBy ("64") __|
*/
return new StreamApplicationDescriptorImpl(appDesc -> {
MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde())).withSideInputs(Arrays.asList("input1")).withSideInputsProcessor(mock(SideInputsProcessor.class));
Table table = appDesc.getTable(tableDescriptor);
messageStream2.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").join(table, mock(StreamTableJoinFunction.class)).sendTo(output1);
}, config);
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testTriggerIntervalForStatelessOperators.
@Test
public void testTriggerIntervalForStatelessOperators() {
Map<String, String> map = new HashMap<>(config);
map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
Config cfg = new MapConfig(map);
ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
StreamApplicationDescriptorImpl graphSpec = createSimpleGraph();
ExecutionPlan plan = planner.plan(graphSpec);
List<JobConfig> jobConfigs = plan.getJobConfigs();
assertEquals(1, jobConfigs.size());
assertFalse(jobConfigs.get(0).containsKey(TaskConfig.WINDOW_MS));
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testDefaultPartitions.
@Test
public void testDefaultPartitions() {
Map<String, String> map = new HashMap<>(config);
map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
Config cfg = new MapConfig(map);
ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
StreamApplicationDescriptorImpl graphSpec = createSimpleGraph();
JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
// Partitions should be the same as input1
jobGraph.getIntermediateStreams().forEach(edge -> {
assertTrue(edge.getPartitionCount() == DEFAULT_PARTITIONS);
});
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testRejectsInvalidStreamStreamJoin.
@Test(expected = SamzaException.class)
public void testRejectsInvalidStreamStreamJoin() {
ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithInvalidStreamStreamJoin();
planner.plan(graphSpec);
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testCalculateInStreamPartitionsBehindTables.
@Test
public void testCalculateInStreamPartitionsBehindTables() {
ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamTableJoin();
JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
// Partitions should be the same as input3
jobGraph.getIntermediateStreams().forEach(edge -> {
assertEquals(32, edge.getPartitionCount());
});
}
Aggregations