Examples with GeneratorFunction com.hazelcast.jet.pipeline.test.GeneratorFunction used on opensource projects

Example 6 with GeneratorFunction

use of com.hazelcast.jet.pipeline.test.GeneratorFunction in project hazelcast by hazelcast.

the class OrderedProcessingMergingStagesTest method when_innerJoin2_applied_primary_stream_order_is_preserved.

@Test
public void when_innerJoin2_applied_primary_stream_order_is_preserved() {
    int validatedItemCountPerGenerator = ITEM_COUNT;
    int eventsPerSecondPerGenerator = 5 * ITEM_COUNT;
    int generatorCount = 2;
    // Generate monotonic increasing items that are distinct for each generator.
    GeneratorFunction<Map.Entry<Long, Long>> generator1 = (ts, seq) -> Util.entry(0L, generatorCount * seq);
    GeneratorFunction<Map.Entry<Long, Long>> generator2 = (ts, seq) -> Util.entry(1L, generatorCount * seq + 1);
    StreamStage<Map.Entry<Long, Long>> srcStage = p.readFrom(itemsParallel(eventsPerSecondPerGenerator, Arrays.asList(generator1, generator2))).withIngestionTimestamps().setLocalParallelism(HIGH_LOCAL_PARALLELISM);
    BatchStage<Map.Entry<Long, Long>> batchStage = p.readFrom(TestSources.items(Util.entry(0L, 0L), Util.entry(1L, 0L)));
    BatchStage<Map.Entry<Long, Long>> batchStage2 = p.readFrom(TestSources.items(Util.entry(0L, 0L), Util.entry(1L, 0L)));
    StreamStage<Map.Entry<Long, Long>> joined = srcStage.innerHashJoin2(batchStage, JoinClause.onKeys(Map.Entry::getKey, Map.Entry::getKey), batchStage2, JoinClause.onKeys(Map.Entry::getKey, Map.Entry::getKey), (primary, stg1, stg2) -> primary).setLocalParallelism(HIGH_LOCAL_PARALLELISM);
    joined.groupingKey(Map.Entry::getKey).mapStateful(() -> create(generatorCount), this::orderValidator).writeTo(AssertionSinks.assertCollectedEventually(60, list -> {
        assertTrue("when", validatedItemCountPerGenerator <= list.size());
        assertFalse("There is some reordered items in the list", list.contains(false));
    }));
    Job job = hz.getJet().newJob(p);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: " + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}

Example 7 with GeneratorFunction

use of com.hazelcast.jet.pipeline.test.GeneratorFunction in project hazelcast by hazelcast.

the class OrderedProcessingMergingStagesTest method when_hashJoin_applied_primary_stream_order_is_preserved.

@Test
public void when_hashJoin_applied_primary_stream_order_is_preserved() {
    int validatedItemCountPerGenerator = ITEM_COUNT;
    int eventsPerSecondPerGenerator = 5 * ITEM_COUNT;
    int generatorCount = 2;
    // Generate monotonic increasing items that are distinct for each generator.
    GeneratorFunction<Map.Entry<Long, Long>> generator1 = (ts, seq) -> Util.entry(0L, generatorCount * seq);
    GeneratorFunction<Map.Entry<Long, Long>> generator2 = (ts, seq) -> Util.entry(1L, generatorCount * seq + 1);
    StreamStage<Map.Entry<Long, Long>> srcStage = p.readFrom(itemsParallel(eventsPerSecondPerGenerator, Arrays.asList(generator1, generator2))).withIngestionTimestamps().setLocalParallelism(HIGH_LOCAL_PARALLELISM);
    BatchStage<Map.Entry<Long, Long>> batchStage = p.readFrom(TestSources.items(Util.entry(0L, 0L), Util.entry(1L, 0L)));
    StreamStage<Map.Entry<Long, Long>> joined = srcStage.hashJoin(batchStage, JoinClause.onKeys(Map.Entry::getKey, Map.Entry::getKey), (primary, stage) -> primary).setLocalParallelism(HIGH_LOCAL_PARALLELISM);
    joined.groupingKey(Map.Entry::getKey).mapStateful(() -> create(generatorCount), this::orderValidator).writeTo(AssertionSinks.assertCollectedEventually(60, list -> {
        assertTrue("when", validatedItemCountPerGenerator <= list.size());
        assertFalse("There is some reordered items in the list", list.contains(false));
    }));
    Job job = hz.getJet().newJob(p);
    try {
        job.join();
        fail("Job should have completed with an AssertionCompletedException, but completed normally");
    } catch (CompletionException e) {
        String errorMsg = e.getCause().getMessage();
        assertTrue("Job was expected to complete with AssertionCompletedException, but completed with: " + e.getCause(), errorMsg.contains(AssertionCompletedException.class.getName()));
    }
}