Examples with ItemsByTag com.hazelcast.jet.datamodel.ItemsByTag used on opensource projects

Example 1 with ItemsByTag

use of com.hazelcast.jet.datamodel.ItemsByTag in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation.

@Test
public void when_rebalanceAndGroupAggregateBuilderWithComplexAggrOp_then_singleStageAggregation() {
    // Given
    FunctionEx<Integer, Integer> keyFn = i -> i % 10;
    Iterator<Integer> sequence = IntStream.iterate(0, i -> i + 1).boxed().iterator();
    List<Integer> input0 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input1 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    List<Integer> input2 = streamFromIterator(sequence).limit(itemCount).collect(toList());
    BatchStageWithKey<Integer, Integer> stage0 = batchStageFromList(input0).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = batchStageFromList(input1).groupingKey(keyFn);
    BatchStageWithKey<Integer, Integer> stage2Rebalanced = batchStageFromList(input2).rebalance().groupingKey(keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2Rebalanced);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    assertSingleStageAggregation();
    execute();
    Collector<Integer, ?, Map<Integer, Long>> groupAndSum = groupingBy(keyFn, summingLong(i -> i));
    Map<Integer, Long> expectedMap0 = input0.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap1 = input1.stream().collect(groupAndSum);
    Map<Integer, Long> expectedMap2 = input2.stream().collect(groupAndSum);
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 2 with ItemsByTag

use of com.hazelcast.jet.datamodel.ItemsByTag in project hazelcast by hazelcast.

the class WindowGroupAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    WindowTestFixture fx = new WindowTestFixture(false);
    // When
    StreamStageWithKey<Entry<String, Integer>, String> stage0 = fx.newSourceStage();
    StreamStageWithKey<Entry<String, Integer>, String> stage1 = fx.newSourceStage();
    WindowGroupAggregateBuilder1<Entry<String, Integer>, String> b = stage0.window(fx.tumblingWinDef).aggregateBuilder();
    Tag<Entry<String, Integer>> tag0_in = b.tag0();
    Tag<Entry<String, Integer>> tag1_in = b.add(stage1);
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<KeyedWindowResult<String, ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfKeyedWinResult(), wr -> String.format("(%04d %s: %04d, %04d)", wr.end(), wr.key(), wr.result().get(tag0), wr.result().get(tag1))));
}

Example 3 with ItemsByTag

use of com.hazelcast.jet.datamodel.ItemsByTag in project hazelcast by hazelcast.

the class RebalanceBatchStageTest method when_hashJoinBuilderRebalanceMainStage_then_distributedEdge.

@Test
public void when_hashJoinBuilderRebalanceMainStage_then_distributedEdge() {
    // Given
    List<Integer> input = sequence(itemCount);
    String prefix = "value-";
    BatchStage<Integer> stage0Rebalanced = batchStageFromList(input).rebalance();
    BatchStage<Entry<Integer, String>> enrichingStage = batchStageFromList(input).map(i -> entry(i, prefix + i));
    // When
    HashJoinBuilder<Integer> b = stage0Rebalanced.hashJoinBuilder();
    Tag<String> tag1 = b.add(enrichingStage, joinMapEntries(wholeItem()));
    // Method reference avoided due to JDK bug
    BatchStage<Entry<Integer, ItemsByTag>> joined = b.build((k, v) -> entry(k, v));
    // Then
    joined.writeTo(sink);
    DAG dag = p.toDag();
    Edge stage0ToJoin = dag.getInboundEdges("2-way hash-join-joiner").get(0);
    assertTrue("Rebalancing should make the edge distributed", stage0ToJoin.isDistributed());
    assertNull("Didn't rebalance by key, the edge must not be partitioned", stage0ToJoin.getPartitioner());
    execute();
    Function<Entry<Integer, String>, String> formatFn = e -> String.format("(%04d, %s)", e.getKey(), e.getValue());
    assertEquals(streamToString(input.stream().map(i -> tuple2(i, prefix + i)), formatFn), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> formatFn.apply(entry(e.getKey(), e.getValue().get(tag1)))));
}

Example 4 with ItemsByTag

use of com.hazelcast.jet.datamodel.ItemsByTag in project hazelcast by hazelcast.

the class BatchAggregateTest method groupAggregateBuilder_withComplexAggrOp.

@Test
public void groupAggregateBuilder_withComplexAggrOp() {
    // Given
    GroupAggregateFixture fx = new GroupAggregateFixture();
    BatchStageWithKey<Integer, Integer> stage0 = fx.srcStage0.groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = fx.srcStage1().groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage2 = fx.srcStage2().groupingKey(fx.keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, ItemsByTag>> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    execute();
    Map<Integer, Long> expectedMap0 = input.stream().collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap1 = input.stream().map(fx.mapFn1).collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap2 = input.stream().map(fx.mapFn2).collect(groupingBy(fx.keyFn, fx.collectOp));
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue(), expectedMap1.get(e.getKey()), expectedMap2.get(e.getKey())))), streamToString(this.<Integer, ItemsByTag>sinkStreamOfEntry(), e -> FORMAT_FN_3.apply(e.getKey(), tuple3(e.getValue().get(tag0), e.getValue().get(tag1), e.getValue().get(tag2)))));
}

Example 5 with ItemsByTag

use of com.hazelcast.jet.datamodel.ItemsByTag in project hazelcast-jet by hazelcast.

the class HashJoinP method tryProcess0.

@Override
@SuppressWarnings("unchecked")
protected boolean tryProcess0(@Nonnull Object item) {
    E0 e0 = (E0) item;
    ordinal0consumed = true;
    if (tags.isEmpty()) {
        return tryEmit(keyFns.size() == 2 ? mapToOutputBiFn.apply(e0, lookupJoined(1, e0)) : mapToOutputTriFn.apply(e0, lookupJoined(1, e0), lookupJoined(2, e0)));
    }
    ItemsByTag map = new ItemsByTag();
    for (int i = 1; i < keyFns.size(); i++) {
        map.put(tags.get(i), lookupJoined(i, e0));
    }
    return tryEmit(mapToOutputBiFn.apply(e0, map));
}