Search in sources :

Example 1 with FORMAT_FN_3

use of com.hazelcast.jet.pipeline.BatchAggregateTest.FORMAT_FN_3 in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation.

@Test
public void when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation() {
    // Given
    FunctionEx<Integer, Integer> keyFn = i -> i % 10;
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input2 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStageWithKey<Integer, Integer> stage0 = batchStageFromList(input0).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = batchStageFromList(input1).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage2Rebalanced = batchStageFromList(input2).rebalance().groupingKey(keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2Rebalanced);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    assertSingleStageAggregation();
    execute();
    Collector<Integer, ?, Map<Integer, Long>> groupAndSum = groupingBy(keyFn, summingLong(i -> i));
    Map<Integer, Long> expectedMap0 = input0.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap1 = input1.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap2 = input2.stream().collect(groupAndSum);
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}
Also used : Spliterators.spliteratorUnknownSize(java.util.Spliterators.spliteratorUnknownSize) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) AggregateOperations.coAggregateOperationBuilder(com.hazelcast.jet.aggregate.AggregateOperations.coAggregateOperationBuilder) Collections.singletonList(java.util.Collections.singletonList) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Map(java.util.Map) DAG(com.hazelcast.jet.core.DAG) Collector(java.util.stream.Collector) FunctionEx(com.hazelcast.function.FunctionEx) Tag(com.hazelcast.jet.datamodel.Tag) AggregateOperations(com.hazelcast.jet.aggregate.AggregateOperations) Category(org.junit.experimental.categories.Category) Tuple3.tuple3(com.hazelcast.jet.datamodel.Tuple3.tuple3) List(java.util.List) Stream(java.util.stream.Stream) StreamSupport.stream(java.util.stream.StreamSupport.stream) ItemsByTag(com.hazelcast.jet.datamodel.ItemsByTag) Entry(java.util.Map.Entry) Function.identity(java.util.function.Function.identity) IntStream(java.util.stream.IntStream) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) Function(java.util.function.Function) FORMAT_FN_3(com.hazelcast.jet.pipeline.BatchAggregateTest.FORMAT_FN_3) JoinClause.joinMapEntries(com.hazelcast.jet.pipeline.JoinClause.joinMapEntries) ArrayList(java.util.ArrayList) JetException(com.hazelcast.jet.JetException) JetAssert.assertFalse(com.hazelcast.jet.core.test.JetAssert.assertFalse) FORMAT_FN(com.hazelcast.jet.pipeline.BatchAggregateTest.FORMAT_FN) AssertionSinks.assertAnyOrder(com.hazelcast.jet.pipeline.test.AssertionSinks.assertAnyOrder) FORMAT_FN_2(com.hazelcast.jet.pipeline.BatchAggregateTest.FORMAT_FN_2) Collectors.summingLong(java.util.stream.Collectors.summingLong) Util.entry(com.hazelcast.jet.Util.entry) Edge(com.hazelcast.jet.core.Edge) Nonnull(javax.annotation.Nonnull) Tuple2(com.hazelcast.jet.datamodel.Tuple2) Tuple3(com.hazelcast.jet.datamodel.Tuple3) IList(com.hazelcast.collection.IList) Iterator(java.util.Iterator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) AggregateOperation1(com.hazelcast.jet.aggregate.AggregateOperation1) CoAggregateOperationBuilder(com.hazelcast.jet.aggregate.CoAggregateOperationBuilder) TestSources(com.hazelcast.jet.pipeline.test.TestSources) Collectors.toList(java.util.stream.Collectors.toList) Tuple2.tuple2(com.hazelcast.jet.datamodel.Tuple2.tuple2) Assert.assertNull(org.junit.Assert.assertNull) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Functions.wholeItem(com.hazelcast.function.Functions.wholeItem) Util(com.hazelcast.jet.Util) Assert.assertEquals(org.junit.Assert.assertEquals) ItemsByTag(com.hazelcast.jet.datamodel.ItemsByTag) Entry(java.util.Map.Entry) CoAggregateOperationBuilder(com.hazelcast.jet.aggregate.CoAggregateOperationBuilder) Collectors.summingLong(java.util.stream.Collectors.summingLong) Map(java.util.Map) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) Test(org.junit.Test)

Aggregations

IList (com.hazelcast.collection.IList)1 FunctionEx (com.hazelcast.function.FunctionEx)1 Functions.wholeItem (com.hazelcast.function.Functions.wholeItem)1 JetException (com.hazelcast.jet.JetException)1 Util (com.hazelcast.jet.Util)1 Util.entry (com.hazelcast.jet.Util.entry)1 LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)1 AggregateOperation (com.hazelcast.jet.aggregate.AggregateOperation)1 AggregateOperation1 (com.hazelcast.jet.aggregate.AggregateOperation1)1 AggregateOperations (com.hazelcast.jet.aggregate.AggregateOperations)1 AggregateOperations.coAggregateOperationBuilder (com.hazelcast.jet.aggregate.AggregateOperations.coAggregateOperationBuilder)1 CoAggregateOperationBuilder (com.hazelcast.jet.aggregate.CoAggregateOperationBuilder)1 DAG (com.hazelcast.jet.core.DAG)1 Edge (com.hazelcast.jet.core.Edge)1 JetAssert.assertFalse (com.hazelcast.jet.core.test.JetAssert.assertFalse)1 ItemsByTag (com.hazelcast.jet.datamodel.ItemsByTag)1 Tag (com.hazelcast.jet.datamodel.Tag)1 Tuple2 (com.hazelcast.jet.datamodel.Tuple2)1 Tuple2.tuple2 (com.hazelcast.jet.datamodel.Tuple2.tuple2)1 Tuple3 (com.hazelcast.jet.datamodel.Tuple3)1