Examples with InMemorySystemDescriptor org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor used on opensource projects

Example 26 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class EndOfStreamIntegrationTest method testPipeline.

@Test
public void testPipeline() {
    class PipelineApplication implements StreamApplication {

        @Override
        public void describe(StreamApplicationDescriptor appDescriptor) {
            DelegatingSystemDescriptor sd = new DelegatingSystemDescriptor("test");
            GenericInputDescriptor<KV<String, PageView>> isd = sd.getInputDescriptor("PageView", KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()));
            appDescriptor.getInputStream(isd).map(KV::getValue).partitionBy(PageView::getMemberId, pv -> pv, KVSerde.of(new IntegerSerde(), new TestTableData.PageViewJsonSerde()), "p1").sink((m, collector, coordinator) -> {
                RECEIVED.add(m.getValue());
            });
        }
    }
    int numPageViews = 40;
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(new PipelineApplication()).addInputStream(inputDescriptor, TestTableData.generatePartitionedPageViews(numPageViews, 4)).run(Duration.ofSeconds(10));
    assertEquals(RECEIVED.size(), numPageViews);
}

Example 27 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class StreamApplicationIntegrationTest method testStatefulJoinWithLocalTable.

@Test
public void testStatefulJoinWithLocalTable() {
    Random random = new Random();
    List<KV<String, TestTableData.PageView>> pageViews = Arrays.asList(TestTableData.generatePageViews(10)).stream().map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)).collect(Collectors.toList());
    List<KV<String, TestTableData.Profile>> profiles = Arrays.asList(TestTableData.generateProfiles(10)).stream().map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)).collect(Collectors.toList());
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV<String, TestTableData.PageView>> pageViewStreamDesc = isd.getInputDescriptor("PageView", new NoOpSerde<KV<String, TestTableData.PageView>>());
    InMemoryInputDescriptor<KV<String, TestTableData.Profile>> profileStreamDesc = isd.getInputDescriptor("Profile", new NoOpSerde<KV<String, TestTableData.Profile>>()).shouldBootstrap();
    InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd.getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());
    InMemoryOutputDescriptor<String> joinKeysDescriptor = isd.getOutputDescriptor("JoinPageKeys", new NoOpSerde<>());
    TestRunner.of(new PageViewProfileViewJoinApplication()).addInputStream(pageViewStreamDesc, pageViews).addInputStream(profileStreamDesc, profiles).addOutputStream(outputStreamDesc, 1).addOutputStream(joinKeysDescriptor, 1).run(Duration.ofSeconds(2));
    Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
    Assert.assertEquals(10, TestRunner.consumeStream(joinKeysDescriptor, Duration.ofSeconds(1)).get(0).size());
}

Example 28 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class TestAsyncFlatMap method runTest.

private List<PageView> runTest(Map<String, String> configs) {
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM);
    InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd.getInputDescriptor(PAGE_VIEW_STREAM, new NoOpSerde<>());
    InMemoryOutputDescriptor<PageView> outputStreamDesc = isd.getOutputDescriptor(NON_GUEST_PAGE_VIEW_STREAM, new NoOpSerde<>());
    TestRunner.of(new AsyncFlatMapExample()).addInputStream(pageViewStreamDesc, PAGE_VIEWS).addOutputStream(outputStreamDesc, 1).addConfig(new MapConfig(configs)).run(Duration.ofSeconds(10));
    Map<Integer, List<PageView>> result = TestRunner.consumeStream(outputStreamDesc, Duration.ofMillis(1000));
    return result.values().stream().flatMap(List::stream).collect(Collectors.toList());
}

Example 29 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class TestContext method syncTaskWithMultiplePartitionMultithreadedHelper.

void syncTaskWithMultiplePartitionMultithreadedHelper(Map<Integer, List<KV>> inputPartitionData, Map<Integer, List<Integer>> expectedOutputPartitionData) throws Exception {
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV> imid = isd.getInputDescriptor("input", new NoOpSerde<KV>());
    InMemoryOutputDescriptor<Integer> imod = isd.getOutputDescriptor("output", new NoOpSerde<Integer>());
    TestRunner.of(MyStreamTestTask.class).addInputStream(imid, inputPartitionData).addOutputStream(imod, 5).addConfig("job.container.thread.pool.size", "4").addExternalContext(new TestContext(10)).run(Duration.ofSeconds(2));
    StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}

Example 30 with InMemorySystemDescriptor

use of org.apache.samza.test.framework.system.descriptors.InMemorySystemDescriptor in project samza by apache.

the class TestContext method testSyncTaskWithMultiplePartition.

@Test
public void testSyncTaskWithMultiplePartition() throws Exception {
    Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
    Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
    genData(inputPartitionData, expectedOutputPartitionData);
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<KV> imid = isd.getInputDescriptor("input", new NoOpSerde<KV>());
    InMemoryOutputDescriptor<Integer> imod = isd.getOutputDescriptor("output", new NoOpSerde<Integer>());
    TestRunner.of(MyStreamTestTask.class).addInputStream(imid, inputPartitionData).addOutputStream(imod, 5).addExternalContext(new TestContext(10)).run(Duration.ofSeconds(2));
    StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}