Search in sources :

Example 1 with DistributedSupplier

use of com.hazelcast.jet.function.DistributedSupplier in project hazelcast-jet by hazelcast.

the class SlidingWindowPTest method before.

@Before
public void before() {
    SlidingWindowPolicy windowDef = slidingWinPolicy(4, 1);
    AggregateOperation1<Entry<?, Long>, LongAccumulator, Long> operation = AggregateOperation.withCreate(LongAccumulator::new).andAccumulate((LongAccumulator acc, Entry<?, Long> item) -> acc.add(item.getValue())).andCombine(LongAccumulator::add).andDeduct(hasDeduct ? LongAccumulator::subtract : null).andFinish(LongAccumulator::get);
    DistributedFunction<?, Long> keyFn = t -> KEY;
    DistributedToLongFunction<Entry<Long, Long>> timestampFn = Entry::getKey;
    DistributedSupplier<Processor> procSupplier = singleStageProcessor ? aggregateToSlidingWindowP(singletonList(keyFn), singletonList(timestampFn), TimestampKind.EVENT, windowDef, operation, TimestampedEntry::new) : combineToSlidingWindowP(windowDef, operation, TimestampedEntry::new);
    // new supplier to save the last supplied instance
    supplier = () -> lastSuppliedProcessor = (SlidingWindowP) procSupplier.get();
}
Also used : LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Collections.shuffle(java.util.Collections.shuffle) Arrays(java.util.Arrays) DistributedToLongFunction(com.hazelcast.jet.function.DistributedToLongFunction) RunWith(org.junit.runner.RunWith) Parameters(org.junit.runners.Parameterized.Parameters) Processor(com.hazelcast.jet.core.Processor) TimestampKind(com.hazelcast.jet.core.TimestampKind) TestSupport.verifyProcessor(com.hazelcast.jet.core.test.TestSupport.verifyProcessor) SlidingWindowPolicy.slidingWinPolicy(com.hazelcast.jet.core.SlidingWindowPolicy.slidingWinPolicy) ArrayList(java.util.ArrayList) Collections.singletonList(java.util.Collections.singletonList) Watermark(com.hazelcast.jet.core.Watermark) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Processors.aggregateToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.aggregateToSlidingWindowP) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Util.entry(com.hazelcast.jet.Util.entry) Arrays.asList(java.util.Arrays.asList) After(org.junit.After) DistributedFunction(com.hazelcast.jet.function.DistributedFunction) Processors.combineToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.combineToSlidingWindowP) ExpectedException(org.junit.rules.ExpectedException) Parameterized(org.junit.runners.Parameterized) Before(org.junit.Before) DistributedSupplier(com.hazelcast.jet.function.DistributedSupplier) LongStream(java.util.stream.LongStream) Parameter(org.junit.runners.Parameterized.Parameter) Collection(java.util.Collection) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) AggregateOperation1(com.hazelcast.jet.aggregate.AggregateOperation1) ParallelTest(com.hazelcast.test.annotation.ParallelTest) Category(org.junit.experimental.categories.Category) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) Rule(org.junit.Rule) LongAccumulator(com.hazelcast.jet.accumulator.LongAccumulator) Entry(java.util.Map.Entry) HazelcastParametersRunnerFactory(com.hazelcast.test.HazelcastParametersRunnerFactory) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) Entry(java.util.Map.Entry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) SlidingWindowPolicy(com.hazelcast.jet.core.SlidingWindowPolicy) Processor(com.hazelcast.jet.core.Processor) TestSupport.verifyProcessor(com.hazelcast.jet.core.test.TestSupport.verifyProcessor) Processors.aggregateToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.aggregateToSlidingWindowP) Processors.combineToSlidingWindowP(com.hazelcast.jet.core.processor.Processors.combineToSlidingWindowP) Before(org.junit.Before)

Example 2 with DistributedSupplier

use of com.hazelcast.jet.function.DistributedSupplier in project hazelcast-jet by hazelcast.

the class SlidingWindowP_CoGroupTest method test.

@Test
public void test() {
    DistributedSupplier supplier = Processors.aggregateToSlidingWindowP(asList(entryKey(), entryKey()), asList(t -> 1L, t -> 1L), TimestampKind.FRAME, tumblingWinPolicy(1), AggregateOperations.toTwoBags(), TimestampedEntry::new);
    Entry<String, String> entry1 = entry("k1", "a");
    Entry<String, String> entry2 = entry("k2", "b");
    Entry<String, String> entry3 = entry("k1", "c");
    Entry<String, String> entry4 = entry("k3", "d");
    Entry<String, String> entry5 = entry("k1", "e");
    TestSupport.verifyProcessor(supplier).inputs(asList(asList(entry1, entry2), asList(entry3, entry4, entry5))).expectOutput(asList(new TimestampedEntry<>(1, "k1", twoBags(singletonList(entry1), asList(entry3, entry5))), new TimestampedEntry<>(1, "k2", twoBags(singletonList(entry2), emptyList())), new TimestampedEntry<>(1, "k3", twoBags(emptyList(), singletonList(entry4)))));
}
Also used : DistributedSupplier(com.hazelcast.jet.function.DistributedSupplier) Collections.emptyList(java.util.Collections.emptyList) RunWith(org.junit.runner.RunWith) AggregateOperations(com.hazelcast.jet.aggregate.AggregateOperations) Processors(com.hazelcast.jet.core.processor.Processors) Test(org.junit.Test) TimestampKind(com.hazelcast.jet.core.TimestampKind) ParallelTest(com.hazelcast.test.annotation.ParallelTest) Category(org.junit.experimental.categories.Category) DistributedFunctions.entryKey(com.hazelcast.jet.function.DistributedFunctions.entryKey) Collections.singletonList(java.util.Collections.singletonList) TwoBags.twoBags(com.hazelcast.jet.datamodel.TwoBags.twoBags) TestSupport(com.hazelcast.jet.core.test.TestSupport) Util.entry(com.hazelcast.jet.Util.entry) Arrays.asList(java.util.Arrays.asList) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) Entry(java.util.Map.Entry) SlidingWindowPolicy.tumblingWinPolicy(com.hazelcast.jet.core.SlidingWindowPolicy.tumblingWinPolicy) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) TimestampedEntry(com.hazelcast.jet.datamodel.TimestampedEntry) DistributedSupplier(com.hazelcast.jet.function.DistributedSupplier) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Aggregations

Util.entry (com.hazelcast.jet.Util.entry)2 TimestampKind (com.hazelcast.jet.core.TimestampKind)2 TimestampedEntry (com.hazelcast.jet.datamodel.TimestampedEntry)2 DistributedSupplier (com.hazelcast.jet.function.DistributedSupplier)2 ParallelTest (com.hazelcast.test.annotation.ParallelTest)2 Arrays.asList (java.util.Arrays.asList)2 Collections.singletonList (java.util.Collections.singletonList)2 Entry (java.util.Map.Entry)2 Test (org.junit.Test)2 Category (org.junit.experimental.categories.Category)2 RunWith (org.junit.runner.RunWith)2 LongAccumulator (com.hazelcast.jet.accumulator.LongAccumulator)1 AggregateOperation (com.hazelcast.jet.aggregate.AggregateOperation)1 AggregateOperation1 (com.hazelcast.jet.aggregate.AggregateOperation1)1 AggregateOperations (com.hazelcast.jet.aggregate.AggregateOperations)1 Processor (com.hazelcast.jet.core.Processor)1 SlidingWindowPolicy (com.hazelcast.jet.core.SlidingWindowPolicy)1 SlidingWindowPolicy.slidingWinPolicy (com.hazelcast.jet.core.SlidingWindowPolicy.slidingWinPolicy)1 SlidingWindowPolicy.tumblingWinPolicy (com.hazelcast.jet.core.SlidingWindowPolicy.tumblingWinPolicy)1 Watermark (com.hazelcast.jet.core.Watermark)1