Examples with WindowResult com.hazelcast.jet.datamodel.WindowResult used on opensource projects

Example 6 with WindowResult

use of com.hazelcast.jet.datamodel.WindowResult in project hazelcast by hazelcast.

the class StreamSourceStageTest method when_sparseItemsWithIngestionTimestamps_then_windowIsNotEmittedTooEarly.

@Test
@Category(NightlyTest.class)
public void when_sparseItemsWithIngestionTimestamps_then_windowIsNotEmittedTooEarly() {
    IList<WindowResult<Long>> sinkList = instance.getList(randomMapName());
    IMap<Integer, Integer> map = instance.getMap(randomMapName());
    Pipeline p = Pipeline.create();
    p.readFrom(Sources.mapJournal(map, START_FROM_OLDEST)).withIngestionTimestamps().window(WindowDefinition.session(15_000)).aggregate(AggregateOperations.counting()).writeTo(Sinks.list(sinkList));
    long start = System.nanoTime();
    Job job = instance.getJet().newJob(p);
    assertEquals(0, sinkList.size());
    map.put(5, 5);
    assertTrueEventually(() -> assertEquals(1, sinkList.size()), 30);
    assertTrue(System.nanoTime() - start > SECONDS.toNanos(15));
    job.cancel();
}

Example 7 with WindowResult

use of com.hazelcast.jet.datamodel.WindowResult in project hazelcast by hazelcast.

the class SourceBuilder_TopologyChangeTest method testTopologyChange.

private void testTopologyChange(Supplier<HazelcastInstance> secondMemberSupplier, Consumer<HazelcastInstance> changeTopologyFn, boolean assertMonotonicity) {
    stateRestored = false;
    StreamSource<Integer> source = SourceBuilder.timestampedStream("src", ctx -> new NumberGeneratorContext()).<Integer>fillBufferFn((src, buffer) -> {
        long expectedCount = NANOSECONDS.toMillis(System.nanoTime() - src.startTime);
        expectedCount = Math.min(expectedCount, src.current + 100);
        while (src.current < expectedCount) {
            buffer.add(src.current, src.current);
            src.current++;
        }
    }).createSnapshotFn(src -> {
        System.out.println("Will save " + src.current + " to snapshot");
        return src;
    }).restoreSnapshotFn((src, states) -> {
        stateRestored = true;
        assert states.size() == 1;
        src.restore(states.get(0));
        System.out.println("Restored " + src.current + " from snapshot");
    }).build();
    Config config = smallInstanceConfig();
    // restart sooner after member add
    config.getJetConfig().setScaleUpDelayMillis(1000);
    HazelcastInstance hz = createHazelcastInstance(config);
    HazelcastInstance possibleSecondNode = secondMemberSupplier.get();
    long windowSize = 100;
    IList<WindowResult<Long>> result = hz.getList("result-" + UuidUtil.newUnsecureUuidString());
    Pipeline p = Pipeline.create();
    p.readFrom(source).withNativeTimestamps(0).window(tumbling(windowSize)).aggregate(AggregateOperations.counting()).peek().writeTo(Sinks.list(result));
    Job job = hz.getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE).setSnapshotIntervalMillis(500));
    assertTrueEventually(() -> assertFalse("result list is still empty", result.isEmpty()));
    assertJobStatusEventually(job, JobStatus.RUNNING);
    JobRepository jr = new JobRepository(hz);
    waitForFirstSnapshot(jr, job.getId(), 10, false);
    assertFalse(stateRestored);
    changeTopologyFn.accept(possibleSecondNode);
    assertTrueEventually(() -> assertTrue("restoreSnapshotFn was not called", stateRestored));
    // wait until more results are added
    int oldSize = result.size();
    assertTrueEventually(() -> assertTrue("no more results added to the list", result.size() > oldSize));
    cancelAndJoin(job);
    // results should contain sequence of results, each with count=windowSize, monotonic, if job was
    // allowed to terminate gracefully
    Iterator<WindowResult<Long>> iterator = result.iterator();
    for (int i = 0; i < result.size(); i++) {
        WindowResult<Long> next = iterator.next();
        assertEquals(windowSize, (long) next.result());
        if (assertMonotonicity) {
            assertEquals(i * windowSize, next.start());
        }
    }
}

Example 8 with WindowResult

use of com.hazelcast.jet.datamodel.WindowResult in project hazelcast by hazelcast.

the class StreamSourceStageTest method when_sparseItemsWithIngestionTimestamps_then_noExtraLatency.

@Test
public void when_sparseItemsWithIngestionTimestamps_then_noExtraLatency() {
    IList<WindowResult<Long>> sinkList = instance.getList(randomMapName());
    IMap<Integer, Integer> map = instance.getMap(randomMapName());
    Pipeline p = Pipeline.create();
    p.readFrom(Sources.mapJournal(map, START_FROM_OLDEST)).withIngestionTimestamps().window(WindowDefinition.tumbling(1)).aggregate(AggregateOperations.counting()).writeTo(Sinks.list(sinkList));
    Job job = instance.getJet().newJob(p);
    assertEquals(0, sinkList.size());
    map.put(3, 3);
    assertTrueEventually(() -> assertEquals(1, sinkList.size()), 10);
    map.put(4, 4);
    assertTrueEventually(() -> assertEquals(2, sinkList.size()), 10);
    job.cancel();
}

Example 9 with WindowResult

use of com.hazelcast.jet.datamodel.WindowResult in project hazelcast by hazelcast.

the class WindowAggregateTest method aggregateBuilder_withComplexAggrOp.

@Test
public void aggregateBuilder_withComplexAggrOp() {
    // Given
    CoAggregateFixture fx = new CoAggregateFixture();
    // When
    WindowAggregateBuilder1<Integer> b = fx.stage0.aggregateBuilder();
    Tag<Integer> tag0_in = b.tag0();
    Tag<Integer> tag1_in = b.add(fx.newStage());
    CoAggregateOperationBuilder b2 = coAggregateOperationBuilder();
    Tag<Long> tag0 = b2.add(tag0_in, SUMMING);
    Tag<Long> tag1 = b2.add(tag1_in, SUMMING);
    StreamStage<WindowResult<ItemsByTag>> aggregated = b.build(b2.build());
    // Then
    aggregated.writeTo(sink);
    execute();
    assertEquals(fx.expectedString2, streamToString(this.<ItemsByTag>sinkStreamOfWinResult(), wr -> FORMAT_FN_2.apply(wr.end(), tuple2(wr.result().get(tag0), wr.result().get(tag1)))));
}

Example 10 with WindowResult

use of com.hazelcast.jet.datamodel.WindowResult in project hazelcast by hazelcast.

the class SourceBuilderTest method testFaultTolerance.

private void testFaultTolerance(StreamSource<?> source) {
    long windowSize = 100;
    IList<WindowResult<Long>> result = hz().getList("result-" + UuidUtil.newUnsecureUuidString());
    Pipeline p = Pipeline.create();
    p.readFrom(source).withNativeTimestamps(0).window(tumbling(windowSize)).aggregate(AggregateOperations.counting()).peek().writeTo(Sinks.list(result));
    Job job = hz().getJet().newJob(p, new JobConfig().setProcessingGuarantee(EXACTLY_ONCE));
    assertTrueEventually(() -> assertFalse("result list is still empty", result.isEmpty()));
    // restart the job
    job.restart();
    assertJobStatusEventually(job, JobStatus.RUNNING);
    // wait until more results are added
    int oldSize = result.size();
    assertTrueEventually(() -> assertTrue("no more results added to the list", result.size() > oldSize));
    cancelAndJoin(job);
    // results should contain a monotonic sequence of results, each with count=windowSize
    Iterator<WindowResult<Long>> iterator = result.iterator();
    for (int i = 0; i < result.size(); i++) {
        WindowResult<Long> next = iterator.next();
        assertEquals(windowSize, (long) next.result());
        assertEquals(i * windowSize, next.start());
    }
}