Examples with NexmarkConfiguration org.apache.beam.sdk.nexmark.NexmarkConfiguration used on opensource projects

Example 1 with NexmarkConfiguration

use of org.apache.beam.sdk.nexmark.NexmarkConfiguration in project beam by apache.

the class BoundedSideInputJoinTest method queryMatchesModelStreamingDirect.

@Test
@Category(NeedsRunner.class)
public void queryMatchesModelStreamingDirect() throws Exception {
    NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
    config.sideInputType = NexmarkUtils.SideInputType.DIRECT;
    config.numEventGenerators = 1;
    config.numEvents = 5000;
    config.sideInputRowCount = 10;
    config.sideInputNumShards = 3;
    queryMatchesModel("BoundedSideInputJoinTestStreaming", config, new BoundedSideInputJoin(config), new BoundedSideInputJoinModel(config), true);
}

Example 2 with NexmarkConfiguration

use of org.apache.beam.sdk.nexmark.NexmarkConfiguration in project beam by apache.

the class BoundedSideInputJoinTest method inputOutputSameEvents.

/**
 * A smoke test that the count of input bids and outputs are the same, to help diagnose flakiness
 * in more complex tests.
 */
@Test
@Category(NeedsRunner.class)
public void inputOutputSameEvents() throws Exception {
    NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
    config.sideInputType = NexmarkUtils.SideInputType.DIRECT;
    config.numEventGenerators = 1;
    config.numEvents = 5000;
    config.sideInputRowCount = 10;
    config.sideInputNumShards = 3;
    PCollection<KV<Long, String>> sideInput = NexmarkUtils.prepareSideInput(p, config);
    try {
        PCollection<Event> input = p.apply(NexmarkUtils.batchEventsSource(config));
        PCollection<Bid> justBids = input.apply(NexmarkQueryUtil.JUST_BIDS);
        PCollection<Long> bidCount = justBids.apply("Count Bids", Count.globally());
        NexmarkQueryTransform<Bid> query = new BoundedSideInputJoin(config);
        query.setSideInput(sideInput);
        PCollection<TimestampedValue<Bid>> output = (PCollection<TimestampedValue<Bid>>) input.apply(new NexmarkQuery(config, query));
        PCollection<Long> outputCount = output.apply("Count outputs", Count.globally());
        PAssert.that(PCollectionList.of(bidCount).and(outputCount).apply(Flatten.pCollections())).satisfies(counts -> {
            assertThat(Iterables.size(counts), equalTo(2));
            assertThat(Iterables.get(counts, 0), greaterThan(0L));
            assertThat(Iterables.get(counts, 0), equalTo(Iterables.get(counts, 1)));
            return null;
        });
        p.run();
    } finally {
        NexmarkUtils.cleanUpSideInput(config);
    }
}

Example 3 with NexmarkConfiguration

use of org.apache.beam.sdk.nexmark.NexmarkConfiguration in project beam by apache.

the class BoundedSideInputJoinTest method queryMatchesModelBatchCsv.

@Test
@Category(NeedsRunner.class)
public void queryMatchesModelBatchCsv() throws Exception {
    NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
    config.sideInputType = NexmarkUtils.SideInputType.CSV;
    config.numEventGenerators = 1;
    config.numEvents = 5000;
    config.sideInputRowCount = 10;
    config.sideInputNumShards = 3;
    queryMatchesModel("BoundedSideInputJoinTestBatch", config, new BoundedSideInputJoin(config), new BoundedSideInputJoinModel(config), false);
}

Example 4 with NexmarkConfiguration

use of org.apache.beam.sdk.nexmark.NexmarkConfiguration in project beam by apache.

the class SessionSideInputJoinTest method queryMatchesModelBatchDirect.

@Test
@Category(NeedsRunner.class)
public void queryMatchesModelBatchDirect() throws Exception {
    NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
    config.sideInputType = NexmarkUtils.SideInputType.DIRECT;
    config.numEventGenerators = 1;
    config.numEvents = 5000;
    config.sideInputRowCount = 10;
    config.sideInputNumShards = 3;
    queryMatchesModel("SessionSideInputJoinTestBatch", config, new SessionSideInputJoin(config), new SessionSideInputJoinModel(config), false);
}

Example 5 with NexmarkConfiguration

use of org.apache.beam.sdk.nexmark.NexmarkConfiguration in project beam by apache.

the class BoundedSideInputJoinTest method queryMatchesModelStreamingCsv.

@Test
@Category(NeedsRunner.class)
public void queryMatchesModelStreamingCsv() throws Exception {
    NexmarkConfiguration config = NexmarkConfiguration.DEFAULT.copy();
    config.sideInputType = NexmarkUtils.SideInputType.CSV;
    config.numEventGenerators = 1;
    config.numEvents = 5000;
    config.sideInputRowCount = 10;
    config.sideInputNumShards = 3;
    queryMatchesModel("BoundedSideInputJoinTestStreaming", config, new BoundedSideInputJoin(config), new BoundedSideInputJoinModel(config), true);
}