use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testCalculateIntStreamPartitions.
@Test
public void testCalculateIntStreamPartitions() {
ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
StreamApplicationDescriptorImpl graphSpec = createSimpleGraph();
JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
// Partitions should be the same as input1
jobGraph.getIntermediateStreams().forEach(edge -> {
// max of input1 and output1
assertEquals(64, edge.getPartitionCount());
});
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method createStreamGraphWithInvalidStreamTableJoin.
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
/**
* Example stream-table join that is invalid due to disagreement in partition count
* between the 2 input streams.
*
* input1 (64) -> send-to-table t
*
* join-table t -> output1 (8)
* |
* input2 (16) —————————
*/
return new StreamApplicationDescriptorImpl(appDesc -> {
MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde()));
Table table = appDesc.getTable(tableDescriptor);
messageStream1.sendTo(table);
messageStream1.join(table, mock(StreamTableJoinFunction.class)).join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2").sendTo(output1);
}, config);
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testTriggerIntervalForWindowsAndJoins.
@Test
public void testTriggerIntervalForWindowsAndJoins() {
Map<String, String> map = new HashMap<>(config);
map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
Config cfg = new MapConfig(map);
ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithJoinAndWindow();
ExecutionPlan plan = planner.plan(graphSpec);
List<JobConfig> jobConfigs = plan.getJobConfigs();
assertEquals(1, jobConfigs.size());
// GCD of 8, 16, 1600 and 252 is 4
assertEquals("4", jobConfigs.get(0).get(TaskConfig.WINDOW_MS));
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestExecutionPlanner method testTriggerIntervalForJoins.
@Test
public void testTriggerIntervalForJoins() {
Map<String, String> map = new HashMap<>(config);
map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
Config cfg = new MapConfig(map);
ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamStreamJoin();
ExecutionPlan plan = planner.plan(graphSpec);
List<JobConfig> jobConfigs = plan.getJobConfigs();
for (JobConfig config : jobConfigs) {
System.out.println(config);
}
}
use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.
the class TestJobGraph method createGraph1.
/**
* graph1 is the example graph from wikipedia
*
* 5 7 3
* | / | / |
* v v |
* 11 8 |
* | \X /
* v v \v
* 2 9 10
*/
private void createGraph1() {
StreamApplicationDescriptorImpl appDesc = mock(StreamApplicationDescriptorImpl.class);
graph1 = new JobGraph(new MapConfig(), appDesc);
JobNode n2 = graph1.getOrCreateJobNode("2", "1");
JobNode n3 = graph1.getOrCreateJobNode("3", "1");
JobNode n5 = graph1.getOrCreateJobNode("5", "1");
JobNode n7 = graph1.getOrCreateJobNode("7", "1");
JobNode n8 = graph1.getOrCreateJobNode("8", "1");
JobNode n9 = graph1.getOrCreateJobNode("9", "1");
JobNode n10 = graph1.getOrCreateJobNode("10", "1");
JobNode n11 = graph1.getOrCreateJobNode("11", "1");
graph1.addInputStream(genStream(), n5);
graph1.addInputStream(genStream(), n7);
graph1.addInputStream(genStream(), n3);
graph1.addIntermediateStream(genStream(), n5, n11);
graph1.addIntermediateStream(genStream(), n7, n11);
graph1.addIntermediateStream(genStream(), n7, n8);
graph1.addIntermediateStream(genStream(), n3, n8);
graph1.addIntermediateStream(genStream(), n11, n2);
graph1.addIntermediateStream(genStream(), n11, n9);
graph1.addIntermediateStream(genStream(), n8, n9);
graph1.addIntermediateStream(genStream(), n11, n10);
graph1.addOutputStream(genStream(), n2);
graph1.addOutputStream(genStream(), n9);
graph1.addOutputStream(genStream(), n10);
}
Aggregations