Examples with NoOpSerde org.apache.samza.serializers.NoOpSerde used on opensource projects

Example 26 with NoOpSerde

use of org.apache.samza.serializers.NoOpSerde in project samza by apache.

the class TestEventHubsInputDescriptor method testStreamDescriptorContainsKVserde.

@Test
public void testStreamDescriptorContainsKVserde() {
    String systemName = "eventHub";
    String streamId = "input-stream";
    EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName);
    EventHubsInputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor.getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde());
    assertTrue(outputDescriptor.getSerde() instanceof KVSerde);
    assertTrue(((KVSerde) outputDescriptor.getSerde()).getKeySerde() instanceof NoOpSerde);
    assertTrue(((KVSerde) outputDescriptor.getSerde()).getValueSerde() instanceof StringSerde);
}

Example 27 with NoOpSerde

use of org.apache.samza.serializers.NoOpSerde in project samza by apache.

the class EndOfStreamIntegrationTest method testPipeline.

@Test
public void testPipeline() {
    class PipelineApplication implements StreamApplication {

        @Override
        public void describe(StreamApplicationDescriptor appDescriptor) {
            DelegatingSystemDescriptor sd = new DelegatingSystemDescriptor("test");
            GenericInputDescriptor<KV<String, PageView>> isd = sd.getInputDescriptor("PageView", KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()));
            appDescriptor.getInputStream(isd).map(KV::getValue).partitionBy(PageView::getMemberId, pv -> pv, KVSerde.of(new IntegerSerde(), new TestTableData.PageViewJsonSerde()), "p1").sink((m, collector, coordinator) -> {
                RECEIVED.add(m.getValue());
            });
        }
    }
    int numPageViews = 40;
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(new PipelineApplication()).addInputStream(inputDescriptor, TestTableData.generatePartitionedPageViews(numPageViews, 4)).run(Duration.ofSeconds(10));
    assertEquals(RECEIVED.size(), numPageViews);
}

Example 28 with NoOpSerde

use of org.apache.samza.serializers.NoOpSerde in project samza by apache.

the class StreamApplicationIntegrationTest method testStatefulJoinWithLocalTable.

@Test
public void testStatefulJoinWithLocalTable() {
    Random random = new Random();
    List<KV<String, TestTableData.PageView>> pageViews = Arrays.asList(TestTableData.generatePageViews(10)).stream().map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)).collect(Collectors.toList());
    List<KV<String, TestTableData.Profile>> profiles = Arrays.asList(TestTableData.generateProfiles(10)).stream().map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)).collect(Collectors.toList());
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV<String, TestTableData.PageView>> pageViewStreamDesc = isd.getInputDescriptor("PageView", new NoOpSerde<KV<String, TestTableData.PageView>>());
    InMemoryInputDescriptor<KV<String, TestTableData.Profile>> profileStreamDesc = isd.getInputDescriptor("Profile", new NoOpSerde<KV<String, TestTableData.Profile>>()).shouldBootstrap();
    InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd.getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());
    InMemoryOutputDescriptor<String> joinKeysDescriptor = isd.getOutputDescriptor("JoinPageKeys", new NoOpSerde<>());
    TestRunner.of(new PageViewProfileViewJoinApplication()).addInputStream(pageViewStreamDesc, pageViews).addInputStream(profileStreamDesc, profiles).addOutputStream(outputStreamDesc, 1).addOutputStream(joinKeysDescriptor, 1).run(Duration.ofSeconds(2));
    Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
    Assert.assertEquals(10, TestRunner.consumeStream(joinKeysDescriptor, Duration.ofSeconds(1)).get(0).size());
}

Example 29 with NoOpSerde

use of org.apache.samza.serializers.NoOpSerde in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetOutputStreamWithValueSerde.

@Test
public void testGetOutputStreamWithValueSerde() {
    String streamId = "test-stream-1";
    Serde mockValueSerde = mock(Serde.class);
    GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockValueSerde);
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getOutputStream(osd);
    }, getConfig());
    OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
    assertEquals(streamId, outputStreamImpl.getStreamId());
    assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
    assertTrue(outputStreamImpl.getKeySerde() instanceof NoOpSerde);
    assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}

Example 30 with NoOpSerde

use of org.apache.samza.serializers.NoOpSerde in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetInputStreamWithValueSerde.

@Test
public void testGetInputStreamWithValueSerde() {
    String streamId = "test-stream-1";
    Serde mockValueSerde = mock(Serde.class);
    GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
    GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde);
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getInputStream(isd);
    }, getConfig());
    InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId);
    assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
    assertEquals(streamId, inputOpSpec.getStreamId());
    assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId));
    assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde);
    assertEquals(mockValueSerde, inputOpSpec.getValueSerde());
}