Examples with PageView org.apache.samza.test.table.TestTableData.PageView used on opensource projects

Example 1 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class TestLocalTableWithSideInputsEndToEnd method buildExpectedEnrichedPageViews.

private static List<EnrichedPageView> buildExpectedEnrichedPageViews(Map<Integer, List<PageView>> pageViews, Map<Integer, List<Profile>> profiles) {
    ImmutableMap.Builder<Integer, Profile> profilesByMemberIdBuilder = new ImmutableMap.Builder<>();
    profiles.values().stream().flatMap(List::stream).forEach(profile -> profilesByMemberIdBuilder.put(profile.getMemberId(), profile));
    Map<Integer, Profile> profilesByMemberId = profilesByMemberIdBuilder.build();
    ImmutableList.Builder<EnrichedPageView> enrichedPageViewsBuilder = new ImmutableList.Builder<>();
    pageViews.values().stream().flatMap(List::stream).forEach(pageView -> Optional.ofNullable(profilesByMemberId.get(pageView.getMemberId())).ifPresent(profile -> enrichedPageViewsBuilder.add(new EnrichedPageView(pageView.getPageKey(), profile.getMemberId(), profile.getCompany()))));
    return enrichedPageViewsBuilder.build();
}

Example 2 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class TestLocalTableWithSideInputsEndToEnd method runTest.

private <T extends ApplicationDescriptor<?>> void runTest(SamzaApplication<T> app, Map<Integer, List<PageView>> pageViews, Map<Integer, List<Profile>> profiles) throws InterruptedException {
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor(SYSTEM_NAME);
    InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd.getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<>());
    InMemoryInputDescriptor<Profile> profileStreamDesc = isd.getInputDescriptor(PROFILE_STREAM, new NoOpSerde<>());
    InMemoryOutputDescriptor<EnrichedPageView> outputStreamDesc = isd.getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<>());
    TestRunner.of(app).addInputStream(pageViewStreamDesc, pageViews).addInputStream(profileStreamDesc, profiles).addOutputStream(outputStreamDesc, 1).run(Duration.ofSeconds(10));
    List<EnrichedPageView> expectedEnrichedPageViews = buildExpectedEnrichedPageViews(pageViews, profiles);
    StreamAssert.containsInAnyOrder(expectedEnrichedPageViews, outputStreamDesc, Duration.ofSeconds(1));
}

Example 3 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class TestLocalTableWithSideInputsEndToEnd method testLowLevelJoinWithSideInputsTable.

@Test
public void testLowLevelJoinWithSideInputsTable() throws InterruptedException {
    int partitionCount = 4;
    IntegerSerde integerSerde = new IntegerSerde();
    // for low-level, need to pre-partition the input in the same way that the profiles are partitioned
    Map<Integer, List<PageView>> pageViewsPartitionedByMemberId = TestTableData.generatePartitionedPageViews(20, partitionCount).values().stream().flatMap(List::stream).collect(Collectors.groupingBy(pageView -> Math.abs(Arrays.hashCode(integerSerde.toBytes(pageView.getMemberId()))) % partitionCount));
    runTest(new LowLevelPageViewProfileJoin(), pageViewsPartitionedByMemberId, TestTableData.generatePartitionedProfiles(10, partitionCount));
}

Example 4 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class TestContext method testStatefulTaskWithLocalTable.

@Test
public void testStatefulTaskWithLocalTable() {
    List<PageView> pageViews = Arrays.asList(TestTableData.generatePageViews(10));
    List<Profile> profiles = Arrays.asList(TestTableData.generateProfiles(10));
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<TestTableData.PageView> pageViewStreamDesc = isd.getInputDescriptor("PageView", new NoOpSerde<TestTableData.PageView>());
    InMemoryInputDescriptor<TestTableData.Profile> profileStreamDesc = isd.getInputDescriptor("Profile", new NoOpSerde<TestTableData.Profile>()).shouldBootstrap();
    InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd.getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());
    TestRunner.of(new JoinTaskApplication()).addInputStream(pageViewStreamDesc, pageViews).addInputStream(profileStreamDesc, profiles).addOutputStream(outputStreamDesc, 1).run(Duration.ofSeconds(2));
    Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
}

Example 5 with PageView

use of org.apache.samza.test.table.TestTableData.PageView in project samza by apache.

the class TestRemoteTableEndToEnd method testSendToUpdatesWithoutUpdateOptions.

// Test will fail as we use sendTo with KV<K, UpdateMessage> stream without UpdateOptions
@Test(expected = SamzaException.class)
public void testSendToUpdatesWithoutUpdateOptions() throws Exception {
    // max member id for page views is 10
    final String profiles = Base64Serializer.serialize(generateProfiles(10));
    final RateLimiter readRateLimiter = mock(RateLimiter.class, withSettings().serializable());
    final TableRateLimiter.CreditFunction creditFunction = (k, v, args) -> 1;
    final StreamApplication app = appDesc -> {
        final RemoteTableDescriptor joinTableDesc = new RemoteTableDescriptor<Integer, TestTableData.Profile, Void>("profile-table-1").withReadFunction(InMemoryProfileReadFunction.getInMemoryReadFunction(profiles)).withRateLimiter(readRateLimiter, creditFunction, null);
        final RemoteTableDescriptor outputTableDesc = new RemoteTableDescriptor<Integer, EnrichedPageView, EnrichedPageView>("enriched-page-view-table-1").withReadFunction(new NoOpTableReadFunction<>()).withReadRateLimiterDisabled().withWriteFunction(new InMemoryEnrichedPageViewWriteFunction2("testUpdateWithoutUpdateOptions", false)).withWriteRateLimit(1000);
        final Table<KV<Integer, Profile>> outputTable = appDesc.getTable(outputTableDesc);
        final Table<KV<Integer, Profile>> joinTable = appDesc.getTable(joinTableDesc);
        final DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
        final GenericInputDescriptor<PageView> isd = ksd.getInputDescriptor("PageView", new NoOpSerde<>());
        appDesc.getInputStream(isd).map(pv -> new KV<>(pv.getMemberId(), pv)).join(joinTable, new PageViewToProfileJoinFunction()).map(m -> new KV(m.getMemberId(), UpdateMessage.of(m, m))).sendTo(outputTable);
    };
    int numPageViews = 40;
    InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
    InMemoryInputDescriptor<PageView> inputDescriptor = isd.getInputDescriptor("PageView", new NoOpSerde<>());
    TestRunner.of(app).addInputStream(inputDescriptor, TestTableData.generatePartitionedPageViews(numPageViews, 4)).run(Duration.ofSeconds(10));
}