Examples with KV org.apache.samza.operators.KV used on opensource projects

Example 51 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class EndOfStreamIntegrationTest method testPipeline.

@Test
public void testPipeline() {
    class PipelineApplication implements StreamApplication {

        @Override
        public void describe(StreamApplicationDescriptor appDescriptor) {
            DelegatingSystemDescriptor sd = new DelegatingSystemDescriptor("test");
            GenericInputDescriptor<KV<String, PageView>> isd = sd.getInputDescriptor("PageView", KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()));
            appDescriptor.getInputStream(isd).map(KV::getValue).partitionBy(PageView::getMemberId, pv -> pv, KVSerde.of(new IntegerSerde(), new TestTableData.PageViewJsonSerde()), "p1").sink((m, collector, coordinator) -> {
                RECEIVED.add(m.getValue());
            });
        }
    }
    int numPageViews = 40;
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(new PipelineApplication()).addInputStream(inputDescriptor, TestTableData.generatePartitionedPageViews(numPageViews, 4)).run(Duration.ofSeconds(10));
    assertEquals(RECEIVED.size(), numPageViews);
}

Example 52 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class StreamApplicationIntegrationTest method testStatefulJoinWithLocalTable.

@Test
public void testStatefulJoinWithLocalTable() {
    Random random = new Random();
    List<KV<String, TestTableData.PageView>> pageViews = Arrays.asList(TestTableData.generatePageViews(10)).stream().map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)).collect(Collectors.toList());
    List<KV<String, TestTableData.Profile>> profiles = Arrays.asList(TestTableData.generateProfiles(10)).stream().map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)).collect(Collectors.toList());
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV<String, TestTableData.PageView>> pageViewStreamDesc = isd.getInputDescriptor("PageView", new NoOpSerde<KV<String, TestTableData.PageView>>());
    InMemoryInputDescriptor<KV<String, TestTableData.Profile>> profileStreamDesc = isd.getInputDescriptor("Profile", new NoOpSerde<KV<String, TestTableData.Profile>>()).shouldBootstrap();
    InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd.getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());
    InMemoryOutputDescriptor<String> joinKeysDescriptor = isd.getOutputDescriptor("JoinPageKeys", new NoOpSerde<>());
    TestRunner.of(new PageViewProfileViewJoinApplication()).addInputStream(pageViewStreamDesc, pageViews).addInputStream(profileStreamDesc, profiles).addOutputStream(outputStreamDesc, 1).addOutputStream(joinKeysDescriptor, 1).run(Duration.ofSeconds(2));
    Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
    Assert.assertEquals(10, TestRunner.consumeStream(joinKeysDescriptor, Duration.ofSeconds(1)).get(0).size());
}

Example 53 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestContext method testSyncTaskWithMultiplePartitionMultithreadedWithCustomIME.

@Test
public void testSyncTaskWithMultiplePartitionMultithreadedWithCustomIME() throws Exception {
    Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
    Map<Integer, List<KV>> inputPartitionIME = new HashMap<>();
    Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
    genData(inputPartitionData, expectedOutputPartitionData);
    for (Map.Entry<Integer, List<KV>> entry : inputPartitionData.entrySet()) {
        Integer partitionId = entry.getKey();
        List<KV> messages = entry.getValue();
        SystemStreamPartition ssp = new SystemStreamPartition("test", "input", new Partition(partitionId));
        inputPartitionIME.put(partitionId, new ArrayList<>());
        int offset = 0;
        for (KV message : messages) {
            IncomingMessageEnvelope ime = new IncomingMessageEnvelope(ssp, String.valueOf(offset++), message.key, message.getValue());
            inputPartitionIME.get(partitionId).add(KV.of(message.key, ime));
        }
    }
    syncTaskWithMultiplePartitionMultithreadedHelper(inputPartitionIME, expectedOutputPartitionData);
}

Example 54 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestContext method syncTaskWithMultiplePartitionMultithreadedHelper.

void syncTaskWithMultiplePartitionMultithreadedHelper(Map<Integer, List<KV>> inputPartitionData, Map<Integer, List<Integer>> expectedOutputPartitionData) throws Exception {
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV> imid = isd.getInputDescriptor("input", new NoOpSerde<KV>());
    InMemoryOutputDescriptor<Integer> imod = isd.getOutputDescriptor("output", new NoOpSerde<Integer>());
    TestRunner.of(MyStreamTestTask.class).addInputStream(imid, inputPartitionData).addOutputStream(imod, 5).addConfig("job.container.thread.pool.size", "4").addExternalContext(new TestContext(10)).run(Duration.ofSeconds(2));
    StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}

Example 55 with KV

use of org.apache.samza.operators.KV in project samza by apache.

the class TestContext method testSyncTaskWithMultiplePartition.

@Test
public void testSyncTaskWithMultiplePartition() throws Exception {
    Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
    Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
    genData(inputPartitionData, expectedOutputPartitionData);
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV> imid = isd.getInputDescriptor("input", new NoOpSerde<KV>());
    InMemoryOutputDescriptor<Integer> imod = isd.getOutputDescriptor("output", new NoOpSerde<Integer>());
    TestRunner.of(MyStreamTestTask.class).addInputStream(imid, inputPartitionData).addOutputStream(imod, 5).addExternalContext(new TestContext(10)).run(Duration.ofSeconds(2));
    StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}