Examples with PageView org.apache.samza.test.table.TestTableData.PageView used on opensource projects

Example 11 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class TestLocalTableEndToEnd method testDualStreamTableJoin.

@Test
public void testDualStreamTableJoin() {
    int totalPageViews = 40;
    int partitionCount = 4;
    Map<Integer, List<PageView>> inputPageViews1 = TestTableData.generatePartitionedPageViews(totalPageViews, partitionCount);
    Map<Integer, List<PageView>> inputPageViews2 = TestTableData.generatePartitionedPageViews(totalPageViews, partitionCount);
    // 10 is the max member id for page views
    int numProfiles = 10;
    Map<Integer, List<Profile>> inputProfiles1 = TestTableData.generatePartitionedProfiles(numProfiles, partitionCount);
    Map<Integer, List<Profile>> inputProfiles2 = TestTableData.generatePartitionedProfiles(numProfiles, partitionCount);
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor(SYSTEM_NAME);
    InMemoryInputDescriptor<PageView> pageViewStreamDesc1 = isd.getInputDescriptor(PAGEVIEW_STREAM + "1", new NoOpSerde<>());
    InMemoryInputDescriptor<PageView> pageViewStreamDesc2 = isd.getInputDescriptor(PAGEVIEW_STREAM + "2", new NoOpSerde<>());
    InMemoryInputDescriptor<Profile> profileStreamDesc1 = isd.getInputDescriptor(PROFILE_STREAM + "1", new NoOpSerde<>());
    InMemoryInputDescriptor<Profile> profileStreamDesc2 = isd.getInputDescriptor(PROFILE_STREAM + "2", new NoOpSerde<>());
    TestRunner.of(new DualStreamTableJoinApp()).addInputStream(pageViewStreamDesc1, inputPageViews1).addInputStream(pageViewStreamDesc2, inputPageViews2).addInputStream(profileStreamDesc1, inputProfiles1).addInputStream(profileStreamDesc2, inputProfiles2).run(Duration.ofSeconds(10));
    assertEquals(numProfiles, DualStreamTableJoinApp.sentToProfileTable1.size());
    assertEquals(numProfiles, DualStreamTableJoinApp.sentToProfileTable2.size());
    assertEquals(totalPageViews, DualStreamTableJoinApp.joinedPageViews1.size());
    assertEquals(totalPageViews, DualStreamTableJoinApp.joinedPageViews2.size());
    assertNotNull(DualStreamTableJoinApp.joinedPageViews1.get(0));
    assertNotNull(DualStreamTableJoinApp.joinedPageViews2.get(0));
}

Example 12 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class EndOfStreamIntegrationTest method testPipeline.

@Test
public void testPipeline() {
    class PipelineApplication implements StreamApplication {

        @Override
        public void describe(StreamApplicationDescriptor appDescriptor) {
            DelegatingSystemDescriptor sd = new DelegatingSystemDescriptor("test");
            GenericInputDescriptor<KV<String, PageView>> isd = sd.getInputDescriptor("PageView", KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()));
            appDescriptor.getInputStream(isd).map(KV::getValue).partitionBy(PageView::getMemberId, pv -> pv, KVSerde.of(new IntegerSerde(), new TestTableData.PageViewJsonSerde()), "p1").sink((m, collector, coordinator) -> {
                RECEIVED.add(m.getValue());
            });
        }
    }
    int numPageViews = 40;
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(new PipelineApplication()).addInputStream(inputDescriptor, TestTableData.generatePartitionedPageViews(numPageViews, 4)).run(Duration.ofSeconds(10));
    assertEquals(RECEIVED.size(), numPageViews);
}