Examples with IntegerSerde org.apache.samza.serializers.IntegerSerde used on opensource projects

Example 21 with IntegerSerde

use of org.apache.samza.serializers.IntegerSerde in project samza by apache.

the class TestJoinOperator method getTestJoinStreamGraph.

private StreamApplicationDescriptorImpl getTestJoinStreamGraph(TestJoinFunction joinFn) throws IOException {
    Map<String, String> mapConfig = new HashMap<>();
    mapConfig.put("job.name", "jobName");
    mapConfig.put("job.id", "jobId");
    StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream");
    StreamTestUtils.addStreamConfigs(mapConfig, "inStream2", "insystem", "instream2");
    Config config = new MapConfig(mapConfig);
    return new StreamApplicationDescriptorImpl(appDesc -> {
        IntegerSerde integerSerde = new IntegerSerde();
        KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde);
        GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName");
        GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor1 = sd.getInputDescriptor("inStream", kvSerde);
        GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor2 = sd.getInputDescriptor("inStream2", kvSerde);
        MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor1);
        MessageStream<KV<Integer, Integer>> inStream2 = appDesc.getInputStream(inputDescriptor2);
        inStream.join(inStream2, joinFn, integerSerde, kvSerde, kvSerde, JOIN_TTL, "j1").sink((message, messageCollector, taskCoordinator) -> {
            SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream");
            messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message));
        });
    }, config);
}

Example 22 with IntegerSerde

use of org.apache.samza.serializers.IntegerSerde in project samza by apache.

the class TestStreamApplicationDescriptorImpl method testGetInputStreamWithExpandingSystem.

@Test
public void testGetInputStreamWithExpandingSystem() {
    String streamId = "test-stream-1";
    String expandedStreamId = "expanded-stream";
    AtomicInteger expandCallCount = new AtomicInteger();
    StreamExpander expander = (sg, isd) -> {
        expandCallCount.incrementAndGet();
        InputDescriptor expandedISD = new GenericSystemDescriptor("expanded-system", "mockFactoryClass").getInputDescriptor(expandedStreamId, new IntegerSerde());
        return sg.getInputStream(expandedISD);
    };
    MockExpandingSystemDescriptor sd = new MockExpandingSystemDescriptor("mock-system", expander);
    MockInputDescriptor isd = sd.getInputDescriptor(streamId, new IntegerSerde());
    StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
        appDesc.getInputStream(isd);
    }, getConfig());
    InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(expandedStreamId);
    assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
    assertEquals(1, expandCallCount.get());
    assertFalse(streamAppDesc.getInputOperators().containsKey(streamId));
    assertFalse(streamAppDesc.getInputDescriptors().containsKey(streamId));
    assertTrue(streamAppDesc.getInputDescriptors().containsKey(expandedStreamId));
    assertEquals(expandedStreamId, inputOpSpec.getStreamId());
}

Example 23 with IntegerSerde

use of org.apache.samza.serializers.IntegerSerde in project samza by apache.

the class TestLocalTableEndToEnd method testSendTo.

@Test
public void testSendTo() {
    MyMapFunction mapFn = new MyMapFunction();
    StreamApplication app = appDesc -> {
        Table<KV<Integer, Profile>> table = appDesc.getTable(new InMemoryTableDescriptor<>("t1", KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
        DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor(SYSTEM_NAME);
        GenericInputDescriptor<Profile> isd = ksd.getInputDescriptor(PROFILE_STREAM, new NoOpSerde<>());
        appDesc.getInputStream(isd).map(mapFn).sendTo(table);
    };
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor(SYSTEM_NAME);
    InMemoryInputDescriptor<Profile> profileStreamDesc = isd.getInputDescriptor(PROFILE_STREAM, new NoOpSerde<>());
    int numProfilesPerPartition = 10;
    int numInputPartitions = 4;
    Map<Integer, List<Profile>> inputProfiles = TestTableData.generatePartitionedProfiles(numProfilesPerPartition * numInputPartitions, numInputPartitions);
    TestRunner.of(app).addInputStream(profileStreamDesc, inputProfiles).run(Duration.ofSeconds(10));
    for (int i = 0; i < numInputPartitions; i++) {
        MyMapFunction mapFnCopy = MyMapFunction.getMapFunctionByTask(String.format("Partition %d", i));
        assertEquals(numProfilesPerPartition, mapFnCopy.received.size());
        mapFnCopy.received.forEach(p -> assertNotNull(mapFnCopy.table.get(p.getMemberId())));
    }
}

Example 24 with IntegerSerde

use of org.apache.samza.serializers.IntegerSerde in project samza by apache.

the class EndOfStreamIntegrationTest method testPipeline.

@Test
public void testPipeline() {
    class PipelineApplication implements StreamApplication {

        @Override
        public void describe(StreamApplicationDescriptor appDescriptor) {
            DelegatingSystemDescriptor sd = new DelegatingSystemDescriptor("test");
            GenericInputDescriptor<KV<String, PageView>> isd = sd.getInputDescriptor("PageView", KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()));
            appDescriptor.getInputStream(isd).map(KV::getValue).partitionBy(PageView::getMemberId, pv -> pv, KVSerde.of(new IntegerSerde(), new TestTableData.PageViewJsonSerde()), "p1").sink((m, collector, coordinator) -> {
                RECEIVED.add(m.getValue());
            });
        }
    }
    int numPageViews = 40;
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(new PipelineApplication()).addInputStream(inputDescriptor, TestTableData.generatePartitionedPageViews(numPageViews, 4)).run(Duration.ofSeconds(10));
    assertEquals(RECEIVED.size(), numPageViews);
}