Examples with AggregateOperation com.hazelcast.jet.aggregate.AggregateOperation used on opensource projects

Example 6 with AggregateOperation

use of com.hazelcast.jet.aggregate.AggregateOperation in project hazelcast by hazelcast.

the class CreateDagVisitor method onCombineByKey.

public Vertex onCombineByKey(AggregateCombineByKeyPhysicalRel rel) {
    AggregateOperation<?, JetSqlRow> aggregateOperation = rel.aggrOp();
    Vertex vertex = dag.newUniqueVertex("CombineByKey", Processors.combineByKeyP(aggregateOperation, (key, value) -> value));
    connectInput(rel.getInput(), vertex, edge -> edge.distributed().partitioned(entryKey()));
    return vertex;
}

Example 7 with AggregateOperation

use of com.hazelcast.jet.aggregate.AggregateOperation in project hazelcast by hazelcast.

the class CreateDagVisitor method onAggregateByKey.

public Vertex onAggregateByKey(AggregateByKeyPhysicalRel rel) {
    FunctionEx<JetSqlRow, ?> groupKeyFn = rel.groupKeyFn();
    AggregateOperation<?, JetSqlRow> aggregateOperation = rel.aggrOp();
    Vertex vertex = dag.newUniqueVertex("AggregateByKey", Processors.aggregateByKeyP(singletonList(groupKeyFn), aggregateOperation, (key, value) -> value));
    connectInput(rel.getInput(), vertex, edge -> edge.distributed().partitioned(groupKeyFn));
    return vertex;
}

Example 8 with AggregateOperation

use of com.hazelcast.jet.aggregate.AggregateOperation in project hazelcast by hazelcast.

the class BatchAggregateTest method groupAggregateBuilder_withComplexAggrOp_withOutputFn.

@Test
@SuppressWarnings("ConstantConditions")
public void groupAggregateBuilder_withComplexAggrOp_withOutputFn() {
    // Given
    GroupAggregateFixture fx = new GroupAggregateFixture();
    BatchStageWithKey<Integer, Integer> stage0 = fx.srcStage0.groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage1 = fx.srcStage1().groupingKey(fx.keyFn);
    BatchStageWithKey<Integer, Integer> stage2 = fx.srcStage2().groupingKey(fx.keyFn);
    // When
    GroupAggregateBuilder1<Integer, Integer> b = stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(stage1);
    Tag<Integer> tag2_in = b.add(stage2);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<Entry<Integer, Long>> aggregated = b.build(aggrOp).map(e -> {
        ItemsByTag ibt = e.getValue();
        return entry(e.getKey(), ibt.get(tag0) + ibt.get(tag1) + ibt.get(tag2));
    });
    // Then
    aggregated.writeTo(sink);
    execute();
    Map<Integer, Long> expectedMap0 = input.stream().collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap1 = input.stream().map(fx.mapFn1).collect(groupingBy(fx.keyFn, fx.collectOp));
    Map<Integer, Long> expectedMap2 = input.stream().map(fx.mapFn2).collect(groupingBy(fx.keyFn, fx.collectOp));
    assertEquals(streamToString(expectedMap0.entrySet().stream(), e -> FORMAT_FN.apply(entry(e.getKey(), e.getValue() + expectedMap1.get(e.getKey()) + expectedMap2.get(e.getKey())))), streamToString(sinkStreamOfEntry(), FORMAT_FN));
}

Example 9 with AggregateOperation

use of com.hazelcast.jet.aggregate.AggregateOperation in project hazelcast by hazelcast.

the class BatchAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    AggregateBuilderFixture fx = new AggregateBuilderFixture();
    // When
    AggregateBuilder1<Integer> b = batchStageFromInput().aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(fx.stage1);
    Tag<Integer> tag2_in = b.add(fx.stage2);
    CoAggregateOperationBuilder agb = coAggregateOperationBuilder();
    Tag<Long> tag0 = agb.add(tag0_in, SUMMING);
    Tag<Long> tag1 = agb.add(tag1_in, SUMMING);
    Tag<Long> tag2 = agb.add(tag2_in, SUMMING);
    AggregateOperation<Object[], ItemsByTag> aggrOp = agb.build();
    BatchStage<ItemsByTag> aggregated = b.build(aggrOp);
    // Then
    aggregated.writeTo(sink);
    execute();
    long sum0 = input.stream().mapToLong(i -> i).sum();
    assertEquals(singletonList(itemsByTag(tag0, sum0, tag1, FACTOR_1 * sum0, tag2, FACTOR_2 * sum0)), new ArrayList<>(sinkList));
}

Example 10 with AggregateOperation

use of com.hazelcast.jet.aggregate.AggregateOperation in project hazelcast-jet by hazelcast.

the class GrAggBuilder method buildBatch.

@SuppressWarnings("unchecked")
public <A, R, OUT> BatchStage<OUT> buildBatch(@Nonnull AggregateOperation<A, ? extends R> aggrOp, @Nonnull DistributedBiFunction<? super K, ? super R, OUT> mapToOutputFn) {
    List<Transform> upstreamTransforms = upstreamStages.stream().map(s -> s.transform).collect(toList());
    Transform transform = new GroupTransform<>(upstreamTransforms, keyFns, aggrOp, mapToOutputFn);
    pipelineImpl.connect(upstreamTransforms, transform);
    return new BatchStageImpl<>(transform, pipelineImpl);
}