Search in sources :

Example 6 with TimestampedEntry

use of com.hazelcast.jet.datamodel.TimestampedEntry in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_windowFirst_aggregate2.

@Test
public void testSliding_windowFirst_aggregate2() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "taz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "tuu");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStageWithGrouping<Entry<Long, String>, Character> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).groupingKey(entry -> entry.getValue().charAt(0)).aggregate2(stage1, toTwoBags()).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> {
        assertEquals(listToString(asList(new TimestampedEntry<>(2, 'f', TwoBags.twoBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")))), new TimestampedEntry<>(4, 't', TwoBags.twoBags(asList(entry(2L, "taz")), asList(entry(2L, "tuu")))))), listToString(instance.getHazelcastInstance().getList("sink")));
    }, 5);
}
Also used : JetInstance(com.hazelcast.jet.JetInstance) GroupProperty(com.hazelcast.spi.properties.GroupProperty) RunWith(org.junit.runner.RunWith) EventJournalConfig(com.hazelcast.config.EventJournalConfig) TestUtil.set(com.hazelcast.jet.core.TestUtil.set) HashSet(java.util.HashSet) ThreeBags(com.hazelcast.jet.datamodel.ThreeBags) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) SlidingWindowDef(com.hazelcast.jet.pipeline.SlidingWindowDef) Util.entry(com.hazelcast.jet.Util.entry) Arrays.asList(java.util.Arrays.asList) WindowGroupAggregateBuilder(com.hazelcast.jet.pipeline.WindowGroupAggregateBuilder) StageWithGroupingAndWindow(com.hazelcast.jet.pipeline.StageWithGroupingAndWindow) IList(com.hazelcast.core.IList) Before(org.junit.Before) AggregateOperations.toThreeBags(com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags) JetConfig(com.hazelcast.jet.config.JetConfig) WindowDefinition(com.hazelcast.jet.pipeline.WindowDefinition) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) AggregateOperations.toTwoBags(com.hazelcast.jet.aggregate.AggregateOperations.toTwoBags) AggregateOperations.toSet(com.hazelcast.jet.aggregate.AggregateOperations.toSet) Tag(com.hazelcast.jet.datamodel.Tag) Sinks(com.hazelcast.jet.pipeline.Sinks) Test(org.junit.Test) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) ParallelTest(com.hazelcast.test.annotation.ParallelTest) Category(org.junit.experimental.categories.Category) Sources(com.hazelcast.jet.pipeline.Sources) IMap(com.hazelcast.core.IMap) JournalInitialPosition(com.hazelcast.jet.pipeline.JournalInitialPosition) StreamStageWithGrouping(com.hazelcast.jet.pipeline.StreamStageWithGrouping) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) Entry(java.util.Map.Entry) TwoBags(com.hazelcast.jet.datamodel.TwoBags) Assert.assertEquals(org.junit.Assert.assertEquals) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) WindowResult(com.hazelcast.jet.datamodel.WindowResult) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) Entry(java.util.Map.Entry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Example 7 with TimestampedEntry

use of com.hazelcast.jet.datamodel.TimestampedEntry in project hazelcast-jet by hazelcast.

the class WindowGroupTransform_IntegrationTest method testSliding_windowFirst_aggregate3_with_aggregateBuilder.

@Test
public void testSliding_windowFirst_aggregate3_with_aggregateBuilder() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "taz");
    map.put(10L, "flush-item");
    IMap<Long, String> map1 = instance.getMap("source1");
    // key is timestamp
    map1.put(0L, "faa");
    map1.put(2L, "tuu");
    map1.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source2");
    // key is timestamp
    map2.put(0L, "fzz");
    map2.put(2L, "tcc");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStageWithGrouping<Entry<Long, String>, Character> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    StreamStageWithGrouping<Entry<Long, String>, Character> stage2 = p.drawFrom(Sources.<Long, String>mapJournal("source2", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).groupingKey(entry -> entry.getValue().charAt(0));
    WindowGroupAggregateBuilder<Entry<Long, String>, Character> b = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).groupingKey(entry -> entry.getValue().charAt(0)).aggregateBuilder();
    Tag<Entry<Long, String>> tag0 = b.tag0();
    Tag<Entry<Long, String>> tag1 = b.add(stage1);
    Tag<Entry<Long, String>> tag2 = b.add(stage2);
    b.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish)).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> {
        assertEquals(listToString(asList(new TimestampedEntry<>(2, 'f', ThreeBags.threeBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")), asList(entry(0L, "fzz")))), new TimestampedEntry<>(4, 't', ThreeBags.threeBags(asList(entry(2L, "taz")), asList(entry(2L, "tuu")), asList(entry(2L, "tcc")))))), listToString(instance.getHazelcastInstance().getList("sink")));
    }, 5);
}
Also used : JetInstance(com.hazelcast.jet.JetInstance) GroupProperty(com.hazelcast.spi.properties.GroupProperty) RunWith(org.junit.runner.RunWith) EventJournalConfig(com.hazelcast.config.EventJournalConfig) TestUtil.set(com.hazelcast.jet.core.TestUtil.set) HashSet(java.util.HashSet) ThreeBags(com.hazelcast.jet.datamodel.ThreeBags) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) SlidingWindowDef(com.hazelcast.jet.pipeline.SlidingWindowDef) Util.entry(com.hazelcast.jet.Util.entry) Arrays.asList(java.util.Arrays.asList) WindowGroupAggregateBuilder(com.hazelcast.jet.pipeline.WindowGroupAggregateBuilder) StageWithGroupingAndWindow(com.hazelcast.jet.pipeline.StageWithGroupingAndWindow) IList(com.hazelcast.core.IList) Before(org.junit.Before) AggregateOperations.toThreeBags(com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags) JetConfig(com.hazelcast.jet.config.JetConfig) WindowDefinition(com.hazelcast.jet.pipeline.WindowDefinition) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) AggregateOperations.toTwoBags(com.hazelcast.jet.aggregate.AggregateOperations.toTwoBags) AggregateOperations.toSet(com.hazelcast.jet.aggregate.AggregateOperations.toSet) Tag(com.hazelcast.jet.datamodel.Tag) Sinks(com.hazelcast.jet.pipeline.Sinks) Test(org.junit.Test) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) ParallelTest(com.hazelcast.test.annotation.ParallelTest) Category(org.junit.experimental.categories.Category) Sources(com.hazelcast.jet.pipeline.Sources) IMap(com.hazelcast.core.IMap) JournalInitialPosition(com.hazelcast.jet.pipeline.JournalInitialPosition) StreamStageWithGrouping(com.hazelcast.jet.pipeline.StreamStageWithGrouping) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) Entry(java.util.Map.Entry) TwoBags(com.hazelcast.jet.datamodel.TwoBags) Assert.assertEquals(org.junit.Assert.assertEquals) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) WindowResult(com.hazelcast.jet.datamodel.WindowResult) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) Entry(java.util.Map.Entry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) ThreeBags(com.hazelcast.jet.datamodel.ThreeBags) AggregateOperations.toThreeBags(com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Example 8 with TimestampedEntry

use of com.hazelcast.jet.datamodel.TimestampedEntry in project hazelcast-jet by hazelcast.

the class SlidingWindowP_CoGroupTest method test.

@Test
public void test() {
    DistributedSupplier supplier = Processors.aggregateToSlidingWindowP(asList(entryKey(), entryKey()), asList(t -> 1L, t -> 1L), TimestampKind.FRAME, tumblingWinPolicy(1), AggregateOperations.toTwoBags(), TimestampedEntry::new);
    Entry<String, String> entry1 = entry("k1", "a");
    Entry<String, String> entry2 = entry("k2", "b");
    Entry<String, String> entry3 = entry("k1", "c");
    Entry<String, String> entry4 = entry("k3", "d");
    Entry<String, String> entry5 = entry("k1", "e");
    TestSupport.verifyProcessor(supplier).inputs(asList(asList(entry1, entry2), asList(entry3, entry4, entry5))).expectOutput(asList(new TimestampedEntry<>(1, "k1", twoBags(singletonList(entry1), asList(entry3, entry5))), new TimestampedEntry<>(1, "k2", twoBags(singletonList(entry2), emptyList())), new TimestampedEntry<>(1, "k3", twoBags(emptyList(), singletonList(entry4)))));
}
Also used : DistributedSupplier(com.hazelcast.jet.function.DistributedSupplier) Collections.emptyList(java.util.Collections.emptyList) RunWith(org.junit.runner.RunWith) AggregateOperations(com.hazelcast.jet.aggregate.AggregateOperations) Processors(com.hazelcast.jet.core.processor.Processors) Test(org.junit.Test) TimestampKind(com.hazelcast.jet.core.TimestampKind) ParallelTest(com.hazelcast.test.annotation.ParallelTest) Category(org.junit.experimental.categories.Category) DistributedFunctions.entryKey(com.hazelcast.jet.function.DistributedFunctions.entryKey) Collections.singletonList(java.util.Collections.singletonList) TwoBags.twoBags(com.hazelcast.jet.datamodel.TwoBags.twoBags) TestSupport(com.hazelcast.jet.core.test.TestSupport) Util.entry(com.hazelcast.jet.Util.entry) Arrays.asList(java.util.Arrays.asList) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) Entry(java.util.Map.Entry) SlidingWindowPolicy.tumblingWinPolicy(com.hazelcast.jet.core.SlidingWindowPolicy.tumblingWinPolicy) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) DistributedSupplier(com.hazelcast.jet.function.DistributedSupplier) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Example 9 with TimestampedEntry

use of com.hazelcast.jet.datamodel.TimestampedEntry in project hazelcast-jet by hazelcast.

the class SlidingWindowP_failoverTest method init.

private void init(ProcessingGuarantee guarantee) {
    SlidingWindowPolicy wDef = SlidingWindowPolicy.tumblingWinPolicy(1);
    AggregateOperation1<Object, LongAccumulator, Long> aggrOp = counting();
    p = new SlidingWindowP<>(singletonList(entryKey()), singletonList((DistributedToLongFunction<Entry<?, Long>>) Entry::getValue), wDef, aggrOp, TimestampedEntry::new, true);
    Outbox outbox = new TestOutbox(128);
    Context context = new TestProcessorContext().setProcessingGuarantee(guarantee);
    p.init(outbox, context);
}
Also used : LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext) Context(com.hazelcast.jet.core.Processor.Context) Entry(java.util.Map.Entry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) Outbox(com.hazelcast.jet.core.Outbox) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext)

Aggregations

TimestampedEntry (com.hazelcast.jet.datamodel.TimestampedEntry)9 Entry (java.util.Map.Entry)9 Util.entry (com.hazelcast.jet.Util.entry)7 Arrays.asList (java.util.Arrays.asList)7 Before (org.junit.Before)7 Test (org.junit.Test)7 RunWith (org.junit.runner.RunWith)7 ParallelTest (com.hazelcast.test.annotation.ParallelTest)6 Category (org.junit.experimental.categories.Category)6 IMap (com.hazelcast.core.IMap)5 JetInstance (com.hazelcast.jet.JetInstance)5 AggregateOperation (com.hazelcast.jet.aggregate.AggregateOperation)5 JetConfig (com.hazelcast.jet.config.JetConfig)5 Assert.assertEquals (org.junit.Assert.assertEquals)5 EventJournalConfig (com.hazelcast.config.EventJournalConfig)4 IList (com.hazelcast.core.IList)4 LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)4 AggregateOperations.toSet (com.hazelcast.jet.aggregate.AggregateOperations.toSet)4 AggregateOperations.toThreeBags (com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags)4 AggregateOperations.toTwoBags (com.hazelcast.jet.aggregate.AggregateOperations.toTwoBags)4