Examples with CoAggregateOperationBuilder com.hazelcast.jet.aggregate.CoAggregateOperationBuilder used on opensource projects

Example 1 with CoAggregateOperationBuilder

use of com.hazelcast.jet.aggregate.CoAggregateOperationBuilder in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation.

@Test
public void when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation() {
    // Given
    FunctionEx<Integer, Integer> keyFn = i -> i % 10;
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input2 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStageWithKey<Integer, Integer> stage0 = batchStageFromList(input0).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = batchStageFromList(input1).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage2Rebalanced = batchStageFromList(input2).rebalance().groupingKey(keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2Rebalanced);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    assertSingleStageAggregation();
    execute();
    Collector<Integer, ?, Map<Integer, Long>> groupAndSum = groupingBy(keyFn, summingLong(i -> i));
    Map<Integer, Long> expectedMap0 = input0.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap1 = input1.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap2 = input2.stream().collect(groupAndSum);
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 2 with CoAggregateOperationBuilder

use of com.hazelcast.jet.aggregate.CoAggregateOperationBuilder in project hazelcast by hazelcast.

the class WindowGroupAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    WindowTestFixture fx = new WindowTestFixture(false);
    // When
    StreamStageWithKey<Entry<String, Integer>, String> stage0 = fx.newSourceStage();
    StreamStageWithKey<Entry<String, Integer>, String> stage1 = fx.newSourceStage();
    WindowGroupAggregateBuilder1<Entry<String, Integer>, String> b = stage0.window(fx.tumblingWinDef).aggregateBuilder();
    Tag<Entry<String, Integer>> tag0_in = b.tag0();
    Tag<Entry<String, Integer>> tag1_in = b.add(stage1);
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<KeyedWindowResult<String, ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfKeyedWinResult(), wr -> String.format("(%04d %s: %04d, %04d)", wr.end(), wr.key(), wr.result().get(tag0), wr.result().get(tag1))));
}

Example 3 with CoAggregateOperationBuilder

use of com.hazelcast.jet.aggregate.CoAggregateOperationBuilder in project hazelcast by hazelcast.

the class BatchAggregateTest method aggregateBuilder_with_complexAggrOp_withOutputFn.

@Test
@SuppressWarnings("ConstantConditions")
public void aggregateBuilder_with_complexAggrOp_withOutputFn() {
    // Given
    BatchStage<Integer> stage1 = batchStageFromInput();
    BatchStage<Integer> stage2 = batchStageFromInput();
    // When
    AggregateBuilder1<Integer> b = batchStageFromInput().aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], Long> aggrOp = agb.build(ibt -> ibt.get(tag0) + ibt.get(tag1) + ibt.get(tag2));
    BatchStage<Long> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(singletonList(3 * input.stream().mapToLong(i -> i).sum()), new ArrayList<>(sinkList));
}

Example 4 with CoAggregateOperationBuilder

use of com.hazelcast.jet.aggregate.CoAggregateOperationBuilder in project hazelcast by hazelcast.

the class BatchAggregateTest method groupAggregateBuilder_withComplexAggrOp.

@Test
public void groupAggregateBuilder_withComplexAggrOp() {
    // Given
    GroupAggregateFixture fx = new GroupAggregateFixture();
    BatchStageWithKey<Integer, Integer> stage0 = fx.srcStage0.groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = fx.srcStage1().groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage2 = fx.srcStage2().groupingKey(fx.keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    execute();
    Map<Integer, Long> expectedMap0 = input.stream().collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap1 = input.stream().map(fx.mapFn1).collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap2 = input.stream().map(fx.mapFn2).collect(groupingBy(fx.keyFn, fx.collectOp));
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 5 with CoAggregateOperationBuilder

use of com.hazelcast.jet.aggregate.CoAggregateOperationBuilder in project hazelcast by hazelcast.

the class WindowAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    CoAggregateFixture fx = new CoAggregateFixture();
    // When
    WindowAggregateBuilder1<Integer> b = fx.stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(fx.newStage());
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<WindowResult<ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfWinResult(), wr -> FORMAT_FN_2.apply(wr.end(), tuple2(wr.result().get(tag0), wr.result().get(tag1)))));
}