Examples with KeyedWindowResult com.hazelcast.jet.datamodel.KeyedWindowResult used on opensource projects

Example 1 with KeyedWindowResult

use of com.hazelcast.jet.datamodel.KeyedWindowResult in project hazelcast by hazelcast.

the class Processors_slidingWindowingIntegrationTest method smokeTest.

@Test
public void smokeTest() throws Exception {
    HazelcastInstance instance = createHazelcastInstance();
    SlidingWindowPolicy wDef = slidingWinPolicy(2000, 1000);
    DAG dag = new DAG();
    // to prevent serialization of whole class
    boolean isBatchLocal = isBatch;
    FunctionEx<? super MyEvent, ?> keyFn = MyEvent::getKey;
    ToLongFunctionEx<? super MyEvent> timestampFn = MyEvent::getTimestamp;
    List<MyEvent> inputData = singletonList(new MyEvent(10, "a", 1L));
    Vertex source = dag.newVertex("source", () -> new EmitListP(inputData, isBatchLocal)).localParallelism(1);
    Vertex insertPP = dag.newVertex("insertWmP", insertWatermarksP(eventTimePolicy(timestampFn, limitingLag(0), wDef.frameSize(), wDef.frameOffset(), 0))).localParallelism(1);
    Vertex sink = dag.newVertex("sink", SinkProcessors.writeListP("sink"));
    dag.edge(between(source, insertPP).isolated());
    AggregateOperation<LongAccumulator, Long> counting = counting();
    if (singleStageProcessor) {
        Vertex slidingWin = dag.newVertex("slidingWin", Processors.aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, wDef, 0L, counting, KeyedWindowResult::new));
        dag.edge(between(insertPP, slidingWin).partitioned(MyEvent::getKey).distributed()).edge(between(slidingWin, sink));
    } else {
        Vertex accumulateByFrame = dag.newVertex("accumulateByFrame", Processors.accumulateByFrameP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, wDef, counting.withIdentityFinish()));
        Vertex slidingWin = dag.newVertex("slidingWin", combineToSlidingWindowP(wDef, counting, KeyedWindowResult::new));
        dag.edge(between(insertPP, accumulateByFrame).partitioned(keyFn)).edge(between(accumulateByFrame, slidingWin).partitioned(entryKey()).distributed()).edge(between(slidingWin, sink));
    }
    Job job = instance.getJet().newJob(dag);
    if (isBatch) {
        job.join();
    }
    IList<MyEvent> sinkList = instance.getList("sink");
    List<KeyedWindowResult<String, Long>> expectedOutput = asList(new KeyedWindowResult<>(-1000, 1000, "a", 1L), new KeyedWindowResult<>(0, 2000, "a", 1L));
    assertTrueEventually(() -> assertEquals(streamToString(expectedOutput.stream()), streamToString(new ArrayList<>(sinkList).stream())), 5);
    // wait a little more and make sure, that there are no more frames
    Thread.sleep(1000);
    assertEquals(expectedOutput.size(), sinkList.size());
}

Example 2 with KeyedWindowResult

use of com.hazelcast.jet.datamodel.KeyedWindowResult in project hazelcast by hazelcast.

the class WindowGroupAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    WindowTestFixture fx = new WindowTestFixture(false);
    // When
    StreamStageWithKey<Entry<String, Integer>, String> stage0 = fx.newSourceStage();
    StreamStageWithKey<Entry<String, Integer>, String> stage1 = fx.newSourceStage();
    WindowGroupAggregateBuilder1<Entry<String, Integer>, String> b = stage0.window(fx.tumblingWinDef).aggregateBuilder();
    Tag<Entry<String, Integer>> tag0_in = b.tag0();
    Tag<Entry<String, Integer>> tag1_in = b.add(stage1);
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<KeyedWindowResult<String, ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfKeyedWinResult(), wr -> String.format("(%04d %s: %04d, %04d)", wr.end(), wr.key(), wr.result().get(tag0), wr.result().get(tag1))));
}

Example 3 with KeyedWindowResult

use of com.hazelcast.jet.datamodel.KeyedWindowResult in project hazelcast by hazelcast.

the class SessionWindowPTest method when_batchProcessing_then_flushEverything.

@Test
public void when_batchProcessing_then_flushEverything() {
    // Given
    List<Object> inbox = new ArrayList<>(eventsWithKey("a"));
    // This watermark will cause the first session to be emitted, but not the second.
    // The second session will be emitted in complete()
    inbox.add(new Watermark(25));
    verifyProcessor(supplier).input(inbox).expectOutput(asList(new KeyedWindowResult<>(1, 22, "a", 3L, false), new Watermark(25), new KeyedWindowResult<>(30, 50, "a", 3L, false)));
}

Example 4 with KeyedWindowResult

use of com.hazelcast.jet.datamodel.KeyedWindowResult in project hazelcast by hazelcast.

the class SlidingWindowPTest method before.

@Before
public void before() {
    SlidingWindowPolicy winPolicy = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andExportFinish(LongAccumulator::get);
    FunctionEx<?, Long> keyFn = t -> KEY;
    ToLongFunctionEx<Entry<Long, Long>> timestampFn = Entry::getKey;
    SupplierEx<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, winPolicy, 0L, operation, KeyedWindowResult::new) : combineToSlidingWindowP(winPolicy, operation, KeyedWindowResult::new);
    // new supplier to save the last supplied instance
    supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}

Example 5 with KeyedWindowResult

use of com.hazelcast.jet.datamodel.KeyedWindowResult in project hazelcast by hazelcast.

the class SlidingWindowP_twoStageSnapshotTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> aggrOp = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(LongAccumulator::subtract).andExportFinish(LongAccumulator::get);
    SupplierEx<Processor> procSupplier1 = Processors.accumulateByFrameP(singletonList((FunctionEx<? super Entry<Long, Long>, ?>) t -> KEY), singletonList((ToLongFunctionEx<? super Entry<Long, Long>>) Entry::getKey), TimestampKind.EVENT, windowDef, aggrOp.withIdentityFinish());
    SupplierEx<Processor> procSupplier2 = combineToSlidingWindowP(windowDef, aggrOp, KeyedWindowResult::new);
    // new supplier to save the last supplied instance
    stage1Supplier = () -> lastSuppliedStage1Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier1.get();
    stage2Supplier = () -> lastSuppliedStage2Processor = (SlidingWindowP<?, ?, ?, ?>) procSupplier2.get();
}