Examples with StringSerde org.apache.samza.serializers.StringSerde used on opensource projects

Example 11 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class TestTableConfigGenerator method testWithSerdes.

@Test
public void testWithSerdes() {
    List<TableDescriptor> descriptors = Arrays.asList(new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())), new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde())));
    Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors));
    JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig);
    assertNotNull(javaTableConfig.getKeySerde("t1"));
    assertNotNull(javaTableConfig.getMsgSerde("t1"));
    assertNotNull(javaTableConfig.getKeySerde("t2"));
    assertNotNull(javaTableConfig.getMsgSerde("t2"));
    MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors));
    javaTableConfig = new JavaTableConfig(tableConfig);
    assertNotNull(javaTableConfig.getTableProviderFactory("t1"));
    assertNotNull(javaTableConfig.getTableProviderFactory("t2"));
}

Example 12 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithStreamTableJoinWithSideInputs.

private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinWithSideInputs() {
    /**
     * Example stream-table join where table t is configured with input1 (64) as a side-input stream.
     *
     *                                   join-table t -> output1 (8)
     *                                        |
     *    input2 (16) -> partitionBy ("64") __|
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde())).withSideInputs(Arrays.asList("input1")).withSideInputsProcessor(mock(SideInputsProcessor.class));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream2.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").join(table, mock(StreamTableJoinFunction.class)).sendTo(output1);
    }, config);
}

Example 13 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class TestJobGraphJsonGenerator method setUp.

@Before
public void setUp() {
    input1Spec = new StreamSpec("input1", "input1", "input-system");
    input2Spec = new StreamSpec("input2", "input2", "input-system");
    outputSpec = new StreamSpec("output", "output", "output-system");
    repartitionSpec = new StreamSpec("jobName-jobId-partition_by-p1", "partition_by-p1", "intermediate-system");
    defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>());
    inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName");
    outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName");
    intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName");
    input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde);
    input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde);
    outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde);
    table1Descriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table1", defaultSerde);
    table2Descriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table2", defaultSerde);
    Map<String, String> configs = new HashMap<>();
    configs.put(JobConfig.JOB_NAME, "jobName");
    configs.put(JobConfig.JOB_ID, "jobId");
    mockConfig = spy(new MapConfig(configs));
    mockJobNode = mock(JobNode.class);
    StreamEdge input1Edge = new StreamEdge(input1Spec, false, false, mockConfig);
    StreamEdge input2Edge = new StreamEdge(input2Spec, false, false, mockConfig);
    StreamEdge outputEdge = new StreamEdge(outputSpec, false, false, mockConfig);
    StreamEdge repartitionEdge = new StreamEdge(repartitionSpec, true, false, mockConfig);
    Map<String, StreamEdge> inputEdges = new HashMap<>();
    inputEdges.put(input1Descriptor.getStreamId(), input1Edge);
    inputEdges.put(input2Descriptor.getStreamId(), input2Edge);
    inputEdges.put(repartitionSpec.getId(), repartitionEdge);
    Map<String, StreamEdge> outputEdges = new HashMap<>();
    outputEdges.put(outputDescriptor.getStreamId(), outputEdge);
    outputEdges.put(repartitionSpec.getId(), repartitionEdge);
    when(mockJobNode.getInEdges()).thenReturn(inputEdges);
    when(mockJobNode.getOutEdges()).thenReturn(outputEdges);
    when(mockJobNode.getConfig()).thenReturn(mockConfig);
    when(mockJobNode.getJobName()).thenReturn("jobName");
    when(mockJobNode.getJobId()).thenReturn("jobId");
    when(mockJobNode.getJobNameAndId()).thenReturn(JobNode.createJobNameAndId("jobName", "jobId"));
    Map<String, TableDescriptor> tables = new HashMap<>();
    tables.put(table1Descriptor.getTableId(), table1Descriptor);
    tables.put(table2Descriptor.getTableId(), table2Descriptor);
    when(mockJobNode.getTables()).thenReturn(tables);
}

Example 14 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class TestExecutionPlanner method createStreamGraphWithInvalidStreamTableJoinWithSideInputs.

private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoinWithSideInputs() {
    /**
     * Example stream-table join that is invalid due to disagreement in partition count between the
     * stream behind table t and another joined stream. Table t is configured with input2 (16) as
     * side-input stream.
     *
     *                   join-table t -> output1 (8)
     *                         |
     *    input1 (64) —————————
     */
    return new StreamApplicationDescriptorImpl(appDesc -> {
        MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor);
        OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor);
        TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor("table-id", new KVSerde(new StringSerde(), new StringSerde())).withSideInputs(Arrays.asList("input2")).withSideInputsProcessor(mock(SideInputsProcessor.class));
        Table table = appDesc.getTable(tableDescriptor);
        messageStream1.join(table, mock(StreamTableJoinFunction.class)).sendTo(output1);
    }, config);
}

Example 15 with StringSerde

use of org.apache.samza.serializers.StringSerde in project samza by apache.

the class ExecutionPlannerTestBase method setUp.

@Before
public void setUp() {
    defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>());
    inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName");
    outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName");
    intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName");
    input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde);
    input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde);
    outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde);
    intermediateInputDescriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-partition_by-p1", defaultSerde).withPhysicalName("jobName-jobId-partition_by-p1");
    intermediateOutputDescriptor = intermediateSystemDescriptor.getOutputDescriptor("jobName-jobId-partition_by-p1", defaultSerde).withPhysicalName("jobName-jobId-partition_by-p1");
    broadcastInputDesriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-broadcast-b1", defaultSerde).withPhysicalName("jobName-jobId-broadcast-b1");
    Map<String, String> configs = new HashMap<>();
    configs.put(JobConfig.JOB_NAME, "jobName");
    configs.put(JobConfig.JOB_ID, "jobId");
    configs.putAll(input1Descriptor.toConfig());
    configs.putAll(input2Descriptor.toConfig());
    configs.putAll(outputDescriptor.toConfig());
    configs.putAll(inputSystemDescriptor.toConfig());
    configs.putAll(outputSystemDescriptor.toConfig());
    configs.putAll(intermediateSystemDescriptor.toConfig());
    configs.put(JobConfig.JOB_DEFAULT_SYSTEM, intermediateSystemDescriptor.getSystemName());
    mockConfig = spy(new MapConfig(configs));
    mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig);
}