Search in sources :

Example 66 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testCalculateIntStreamPartitions.

@Test
public void testCalculateIntStreamPartitions() {
    ExecutionPlanner planner = new ExecutionPlanner(config, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createSimpleGraph();
    JobGraph jobGraph = (JobGraph) planner.plan(graphSpec);
    // Partitions should be the same as input1
    jobGraph.getIntermediateStreams().forEach(edge -> {
        // max of input1 and output1
        assertEquals(64, edge.getPartitionCount());
    });
}
Also used : StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) Test(org.junit.Test)

Example 67 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithInvalidStreamTableJoin.

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() {
    /**
     * Example stream-table join that is invalid due to disagreement in partition count
     * between the 2 input streams.
     *
     *    input1 (64) -> send-to-table t
     *
     *                   join-table t -> output1 (8)
     *                         |
     *    input2 (16) —————————
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
        MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde()));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream1.sendTo(table);
        messageStream1.join(table, mock(StreamTableJoinFunction.class)).join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2").sendTo(output1);
    }, config);
}
Also used : KVSerde(org.apache.samza.serializers.KVSerde) StringSerde(org.apache.samza.serializers.StringSerde) Table(org.apache.samza.table.Table) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) StreamTableJoinFunction(org.apache.samza.operators.functions.StreamTableJoinFunction) KV(org.apache.samza.operators.KV) TableDescriptor(org.apache.samza.table.descriptors.TableDescriptor) TestLocalTableDescriptor(org.apache.samza.table.descriptors.TestLocalTableDescriptor)

Example 68 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForWindowsAndJoins.

@Test
public void testTriggerIntervalForWindowsAndJoins() {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithJoinAndWindow();
    ExecutionPlan plan = planner.plan(graphSpec);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(1, jobConfigs.size());
    // GCD of 8, 16, 1600 and 252 is 4
    assertEquals("4", jobConfigs.get(0).get(TaskConfig.WINDOW_MS));
}
Also used : HashMap(java.util.HashMap) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) MapConfig(org.apache.samza.config.MapConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) StreamConfig(org.apache.samza.config.StreamConfig) TaskConfig(org.apache.samza.config.TaskConfig) MapConfig(org.apache.samza.config.MapConfig) JobConfig(org.apache.samza.config.JobConfig) Test(org.junit.Test)

Example 69 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForJoins.

@Test
public void testTriggerIntervalForJoins() {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS, String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamStreamJoin();
    ExecutionPlan plan = planner.plan(graphSpec);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    for (JobConfig config : jobConfigs) {
        System.out.println(config);
    }
}
Also used : HashMap(java.util.HashMap) StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) MapConfig(org.apache.samza.config.MapConfig) Config(org.apache.samza.config.Config) JobConfig(org.apache.samza.config.JobConfig) StreamConfig(org.apache.samza.config.StreamConfig) TaskConfig(org.apache.samza.config.TaskConfig) MapConfig(org.apache.samza.config.MapConfig) JobConfig(org.apache.samza.config.JobConfig) Test(org.junit.Test)

Example 70 with StreamApplicationDescriptorImpl

use of org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl in project samza by apache.

the class TestJobGraph method createGraph1.

/**
 * graph1 is the example graph from wikipedia
 *
 * 5   7   3
 * | / | / |
 * v   v   |
 * 11  8   |
 * | \X   /
 * v v \v
 * 2 9 10
 */
private void createGraph1() {
    StreamApplicationDescriptorImpl appDesc = mock(StreamApplicationDescriptorImpl.class);
    graph1 = new JobGraph(new MapConfig(), appDesc);
    JobNode n2 = graph1.getOrCreateJobNode("2", "1");
    JobNode n3 = graph1.getOrCreateJobNode("3", "1");
    JobNode n5 = graph1.getOrCreateJobNode("5", "1");
    JobNode n7 = graph1.getOrCreateJobNode("7", "1");
    JobNode n8 = graph1.getOrCreateJobNode("8", "1");
    JobNode n9 = graph1.getOrCreateJobNode("9", "1");
    JobNode n10 = graph1.getOrCreateJobNode("10", "1");
    JobNode n11 = graph1.getOrCreateJobNode("11", "1");
    graph1.addInputStream(genStream(), n5);
    graph1.addInputStream(genStream(), n7);
    graph1.addInputStream(genStream(), n3);
    graph1.addIntermediateStream(genStream(), n5, n11);
    graph1.addIntermediateStream(genStream(), n7, n11);
    graph1.addIntermediateStream(genStream(), n7, n8);
    graph1.addIntermediateStream(genStream(), n3, n8);
    graph1.addIntermediateStream(genStream(), n11, n2);
    graph1.addIntermediateStream(genStream(), n11, n9);
    graph1.addIntermediateStream(genStream(), n8, n9);
    graph1.addIntermediateStream(genStream(), n11, n10);
    graph1.addOutputStream(genStream(), n2);
    graph1.addOutputStream(genStream(), n9);
    graph1.addOutputStream(genStream(), n10);
}
Also used : StreamApplicationDescriptorImpl(org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl) MapConfig(org.apache.samza.config.MapConfig)

Aggregations

StreamApplicationDescriptorImpl (org.apache.samza.application.descriptors.StreamApplicationDescriptorImpl)104 Test (org.junit.Test)93 MapConfig (org.apache.samza.config.MapConfig)67 Config (org.apache.samza.config.Config)62 StreamConfig (org.apache.samza.config.StreamConfig)39 HashMap (java.util.HashMap)36 Collection (java.util.Collection)32 KVSerde (org.apache.samza.serializers.KVSerde)31 JobConfig (org.apache.samza.config.JobConfig)27 GenericSystemDescriptor (org.apache.samza.system.descriptors.GenericSystemDescriptor)26 SamzaSqlApplicationConfig (org.apache.samza.sql.runner.SamzaSqlApplicationConfig)24 SamzaSqlQueryParser (org.apache.samza.sql.util.SamzaSqlQueryParser)23 SamzaSqlTestConfig (org.apache.samza.sql.util.SamzaSqlTestConfig)23 GenericInputDescriptor (org.apache.samza.system.descriptors.GenericInputDescriptor)23 Duration (java.time.Duration)20 Map (java.util.Map)20 JoinFunction (org.apache.samza.operators.functions.JoinFunction)20 Serde (org.apache.samza.serializers.Serde)20 StringSerde (org.apache.samza.serializers.StringSerde)20 IntegerSerde (org.apache.samza.serializers.IntegerSerde)19