Examples with DistributedFunction com.hazelcast.jet.function.DistributedFunction used on opensource projects

Example 1 with DistributedFunction

use of com.hazelcast.jet.function.DistributedFunction in project hazelcast-jet by hazelcast.

the class SlidingWindowPTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andFinish(LongAccumulator::get);
    DistributedFunction<?, Long> keyFn = t -> KEY;
    DistributedToLongFunction<Entry<Long, Long>> timestampFn = Entry::getKey;
    DistributedSupplier<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, windowDef, operation, TimestampedEntry::new) : combineToSlidingWindowP(windowDef, operation, TimestampedEntry::new);
    // new supplier to save the last supplied instance
    supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}

Example 2 with DistributedFunction

use of com.hazelcast.jet.function.DistributedFunction in project hazelcast-jet by hazelcast.

the class SlidingWindowP_twoStageSnapshotTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andFinish(LongAccumulator::get);
    DistributedSupplier<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((DistributedFunction<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((DistributedToLongFunction<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, ((AggregateOperation1<? super Entry<Long, Long>, LongAccumulator, ?>) aggrOp).withFinishFn(identity()));
    DistributedSupplier<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, TimestampedEntry::new);
    // new supplier to save the last supplied instance
    stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
    stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}

Example 3 with DistributedFunction

use of com.hazelcast.jet.function.DistributedFunction in project hazelcast-jet by hazelcast.

the class GrAggBuilder method buildStream.

@SuppressWarnings("unchecked")
public <A, R, OUT> StreamStage<OUT> buildStream(@Nonnull AggregateOperation<A, ? extends R> aggrOp, @Nonnull KeyedWindowResultFunction<? super K, ? super R, OUT> mapToOutputFn) {
    List<Transform> upstreamTransforms = upstreamStages.stream().map(s -> s.transform).collect(toList());
    JetEventFunctionAdapter fnAdapter = ADAPT_TO_JET_EVENT;
    // Avoided Stream API here due to static typing issues
    List<DistributedFunction<?, ? extends K>> adaptedKeyFns = new ArrayList<>();
    for (DistributedFunction keyFn : keyFns) {
        adaptedKeyFns.add(adaptKeyFn(keyFn));
    }
    Transform transform = new WindowGroupTransform<K, A, R, JetEvent<OUT>>(upstreamTransforms, wDef, adaptedKeyFns, adaptAggregateOperation(aggrOp), fnAdapter.adaptKeyedWindowResultFn(mapToOutputFn));
    pipelineImpl.connect(upstreamTransforms, transform);
    return new StreamStageImpl<>(transform, fnAdapter, pipelineImpl);
}

Example 4 with DistributedFunction

use of com.hazelcast.jet.function.DistributedFunction in project hazelcast-jet by hazelcast.

the class HashJoinTransform method addToDag.

// ---------           ----------           ----------
// | primary |         | joined-1 |         | joined-2 |
// ---------           ----------           ----------
// |                   |                     |
// |              distributed          distributed
// |               broadcast            broadcast
// |                   v                     v
// |             -------------         -------------
// |            | collector-1 |       | collector-2 |
// |             -------------         -------------
// |                   |                     |
// |                 local                 local
// local             broadcast             broadcast
// unicast           prioritized           prioritized
// ordinal 0           ordinal 1             ordinal 2
// \                   |                     |
// ----------------\  |   /----------------/
// v  v  v
// --------
// | joiner |
// --------
@Override
@SuppressWarnings("unchecked")
public void addToDag(Planner p) {
    String namePrefix = p.uniqueVertexName(this.name(), "");
    PlannerVertex primary = p.xform2vertex.get(this.upstream().get(0));
    List keyFns = this.clauses.stream().map(JoinClause::leftKeyFn).collect(toList());
    List<Tag> tags = this.tags;
    DistributedBiFunction mapToOutputBiFn = this.mapToOutputBiFn;
    DistributedTriFunction mapToOutputTriFn = this.mapToOutputTriFn;
    Vertex joiner = p.addVertex(this, namePrefix + "-joiner", localParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn)).v;
    p.dag.edge(from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0));
    String collectorName = namePrefix + "-collector";
    int collectorOrdinal = 1;
    for (Transform fromTransform : tailList(this.upstream())) {
        PlannerVertex fromPv = p.xform2vertex.get(fromTransform);
        JoinClause<?, ?, ?, ?> clause = this.clauses.get(collectorOrdinal - 1);
        DistributedFunction<Object, Object> getKeyFn = (DistributedFunction<Object, Object>) clause.rightKeyFn();
        DistributedFunction<Object, Object> projectFn = (DistributedFunction<Object, Object>) clause.rightProjectFn();
        Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn));
        collector.localParallelism(1);
        p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()).to(collector, 0).distributed().broadcast());
        p.dag.edge(from(collector, 0).to(joiner, collectorOrdinal).broadcast().priority(-1));
        collectorOrdinal++;
    }
}

Example 5 with DistributedFunction

use of com.hazelcast.jet.function.DistributedFunction in project hazelcast-jet by hazelcast.

the class JobRestartWithSnapshotTest method when_nodeDown_then_jobRestartsFromSnapshot.

public void when_nodeDown_then_jobRestartsFromSnapshot(boolean twoStage) throws Exception {
    /* Design of this test:

        It uses random partitioned generator of source events. The events are Map.Entry(partitionId, timestamp).
        For each partition timestamps from 0..elementsInPartition are generated.

        We start the test with two nodes and localParallelism(1) for source. Source instances generate items at
        the same rate of 10 per second: this causes one instance to be twice as fast as the other in terms of
        timestamp. The source processor saves partition offsets similarly to how streamKafka() and streamMap()
        do.

        After some time we shut down one instance. The job restarts from snapshot and all partitions are restored
        to single source processor instance. Partition offsets are very different, so the source is written in a way
        that it emits from the most-behind partition in order to not emit late events from more ahead partitions.

        Local parallelism of InsertWatermarkP is also 1 to avoid the edge case when different instances of
        InsertWatermarkP might initialize with first event in different frame and make them start the no-gap
        emission from different WM, which might cause the SlidingWindowP downstream to miss some of the
        first windows.

        The sink writes to an IMap which is an idempotent sink.

        The resulting contents of the sink map are compared to expected value.
         */
    DAG dag = new DAG();
    SlidingWindowPolicy wDef = SlidingWindowPolicy.tumblingWinPolicy(3);
    AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
    IMap<List<Long>, Long> result = instance1.getMap("result");
    result.clear();
    SequencesInPartitionsMetaSupplier sup = new SequencesInPartitionsMetaSupplier(3, 180);
    Vertex generator = dag.newVertex("generator", throttle(sup, 30)).localParallelism(1);
    Vertex insWm = dag.newVertex("insWm", insertWatermarksP(wmGenParams(entry -> ((Entry<Integer, Integer>) entry).getValue(), limitingLag(0), emitByFrame(wDef), -1))).localParallelism(1);
    Vertex map = dag.newVertex("map", mapP((TimestampedEntry e) -> entry(asList(e.getTimestamp(), (long) (int) e.getKey()), e.getValue())));
    Vertex writeMap = dag.newVertex("writeMap", SinkProcessors.writeMapP("result"));
    if (twoStage) {
        Vertex aggregateStage1 = dag.newVertex("aggregateStage1", Processors.accumulateByFrameP(singletonList((DistributedFunction<? super Object, ?>) t -> ((Entry<Integer, Integer>) t).getKey()), singletonList(t1 -> ((Entry<Integer, Integer>) t1).getValue()), TimestampKind.EVENT, wDef, aggrOp.withFinishFn(identity())));
        Vertex aggregateStage2 = dag.newVertex("aggregateStage2", combineToSlidingWindowP(wDef, aggrOp, TimestampedEntry::new));
        dag.edge(between(insWm, aggregateStage1).partitioned(entryKey())).edge(between(aggregateStage1, aggregateStage2).distributed().partitioned(entryKey())).edge(between(aggregateStage2, map));
    } else {
        Vertex aggregate = dag.newVertex("aggregate", Processors.aggregateToSlidingWindowP(singletonList((DistributedFunction<Object, Integer>) t -> ((Entry<Integer, Integer>) t).getKey()), singletonList(t1 -> ((Entry<Integer, Integer>) t1).getValue()), TimestampKind.EVENT, wDef, aggrOp, TimestampedEntry::new));
        dag.edge(between(insWm, aggregate).distributed().partitioned(entryKey())).edge(between(aggregate, map));
    }
    dag.edge(between(generator, insWm)).edge(between(map, writeMap));
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    config.setSnapshotIntervalMillis(1200);
    Job job = instance1.newJob(dag, config);
    SnapshotRepository snapshotRepository = new SnapshotRepository(instance1);
    int timeout = (int) (MILLISECONDS.toSeconds(config.getSnapshotIntervalMillis()) + 2);
    // wait until we have at least one snapshot
    IMapJet<Long, Object> snapshotsMap = snapshotRepository.getSnapshotMap(job.getId());
    assertTrueEventually(() -> assertTrue("No snapshot produced", snapshotsMap.entrySet().stream().anyMatch(en -> en.getValue() instanceof SnapshotRecord && ((SnapshotRecord) en.getValue()).isSuccessful())), timeout);
    waitForNextSnapshot(snapshotsMap, timeout);
    // wait a little more to emit something, so that it will be overwritten in the sink map
    Thread.sleep(300);
    instance2.shutdown();
    // Now the job should detect member shutdown and restart from snapshot.
    // Let's wait until the next snapshot appears.
    waitForNextSnapshot(snapshotsMap, (int) (MILLISECONDS.toSeconds(config.getSnapshotIntervalMillis()) + 10));
    waitForNextSnapshot(snapshotsMap, timeout);
    job.join();
    // compute expected result
    Map<List<Long>, Long> expectedMap = new HashMap<>();
    for (long partition = 0; partition < sup.numPartitions; partition++) {
        long cnt = 0;
        for (long value = 1; value <= sup.elementsInPartition; value++) {
            cnt++;
            if (value % wDef.frameSize() == 0) {
                expectedMap.put(asList(value, partition), cnt);
                cnt = 0;
            }
        }
        if (cnt > 0) {
            expectedMap.put(asList(wDef.higherFrameTs(sup.elementsInPartition - 1), partition), cnt);
        }
    }
    // check expected result
    if (!expectedMap.equals(result)) {
        System.out.println("All expected entries: " + expectedMap.entrySet().stream().map(Object::toString).collect(joining(", ")));
        System.out.println("All actual entries: " + result.entrySet().stream().map(Object::toString).collect(joining(", ")));
        System.out.println("Non-received expected items: " + expectedMap.keySet().stream().filter(key -> !result.containsKey(key)).map(Object::toString).collect(joining(", ")));
        System.out.println("Received non-expected items: " + result.entrySet().stream().filter(entry -> !expectedMap.containsKey(entry.getKey())).map(Object::toString).collect(joining(", ")));
        System.out.println("Different keys: ");
        for (Entry<List<Long>, Long> rEntry : result.entrySet()) {
            Long expectedValue = expectedMap.get(rEntry.getKey());
            if (expectedValue != null && !expectedValue.equals(rEntry.getValue())) {
                System.out.println("key: " + rEntry.getKey() + ", expected value: " + expectedValue + ", actual value: " + rEntry.getValue());
            }
        }
        System.out.println("-- end of different keys");
        assertEquals(expectedMap, new HashMap<>(result));
    }
    assertTrue("Snapshots map not empty after job finished", snapshotsMap.isEmpty());
}