Search in sources :

Example 61 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class ApexWindowedStreamImpl method count.

@Override
public <STREAM extends WindowedStream<Tuple.WindowedTuple<Long>>> STREAM count(Option... opts) {
    Function.MapFunction<T, Tuple<Long>> kVMap = new Function.MapFunction<T, Tuple<Long>>() {

        @Override
        public Tuple<Long> f(T input) {
            if (input instanceof Tuple.TimestampedTuple) {
                return new Tuple.TimestampedTuple<>(((Tuple.TimestampedTuple) input).getTimestamp(), 1L);
            } else {
                return new Tuple.TimestampedTuple<>(System.currentTimeMillis(), 1L);
            }
        }
    };
    WindowedStream<Tuple<Long>> innerstream = map(kVMap);
    WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createWindowedOperator(new SumLong());
    return innerstream.addOperator(windowedOperator, windowedOperator.input, windowedOperator.output, opts);
}
Also used : Function(org.apache.apex.malhar.lib.function.Function) MutableLong(org.apache.commons.lang3.mutable.MutableLong) SumLong(org.apache.apex.malhar.lib.window.accumulation.SumLong) SumLong(org.apache.apex.malhar.lib.window.accumulation.SumLong) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Tuple(org.apache.apex.malhar.lib.window.Tuple)

Example 62 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WindowedOperatorTest method testImplicitWatermarks.

@Test
public void testImplicitWatermarks() {
    WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
    CollectorTestSink controlSink = new CollectorTestSink();
    windowedOperator.controlOutput.setSink(controlSink);
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
    windowedOperator.setAllowedLateness(Duration.millis(1000));
    windowedOperator.setImplicitWatermarkGenerator(new FixedDiffEventTimeWatermarkGen(100));
    windowedOperator.setup(testMeta.operatorContext);
    windowedOperator.beginWindow(1);
    windowedOperator.endWindow();
    Assert.assertEquals("We should get no watermark tuple", 0, controlSink.getCount(false));
    windowedOperator.beginWindow(2);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
    windowedOperator.endWindow();
    Assert.assertEquals("We should get one watermark tuple", 1, controlSink.getCount(false));
    Assert.assertEquals("Check Watermark value", ((ControlTuple.Watermark) controlSink.collectedTuples.get(0)).getTimestamp(), BASE);
    windowedOperator.beginWindow(3);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 900L, 4L));
    windowedOperator.endWindow();
    Assert.assertEquals("We should get two watermark tuples", 2, controlSink.getCount(false));
    Assert.assertEquals("Check Watermark value", ((ControlTuple.Watermark) controlSink.collectedTuples.get(1)).getTimestamp(), BASE + 800);
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong) MutableLong(org.apache.commons.lang3.mutable.MutableLong) FixedDiffEventTimeWatermarkGen(org.apache.apex.malhar.lib.window.impl.FixedDiffEventTimeWatermarkGen) CollectorTestSink(org.apache.apex.malhar.lib.testbench.CollectorTestSink) Test(org.junit.Test)

Example 63 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WindowedOperatorTest method testTrigger.

private void testTrigger(TriggerOption.AccumulationMode accumulationMode) {
    WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
    TriggerOption triggerOption = new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000));
    switch(accumulationMode) {
        case ACCUMULATING:
            triggerOption.accumulatingFiredPanes();
            break;
        case ACCUMULATING_AND_RETRACTING:
            triggerOption.accumulatingAndRetractingFiredPanes();
            break;
        case DISCARDING:
            triggerOption.discardingFiredPanes();
            break;
        default:
            throw new RuntimeException("Unknown accumulation mode: " + accumulationMode);
    }
    windowedOperator.setTriggerOption(triggerOption);
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
    CollectorTestSink sink = new CollectorTestSink();
    windowedOperator.output.setSink(sink);
    windowedOperator.setup(testMeta.operatorContext);
    windowedOperator.beginWindow(1);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 200L, 3L));
    windowedOperator.endWindow();
    Assert.assertTrue("No trigger should be fired yet", sink.collectedTuples.isEmpty());
    windowedOperator.beginWindow(2);
    windowedOperator.endWindow();
    Assert.assertTrue("No trigger should be fired yet", sink.collectedTuples.isEmpty());
    windowedOperator.beginWindow(3);
    windowedOperator.endWindow();
    Assert.assertEquals("There should be exactly one tuple for the time trigger", 1, sink.collectedTuples.size());
    Assert.assertEquals(5L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
    sink.collectedTuples.clear();
    windowedOperator.beginWindow(4);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 400L, 4L));
    windowedOperator.endWindow();
    Assert.assertTrue("No trigger should be fired yet", sink.collectedTuples.isEmpty());
    windowedOperator.beginWindow(5);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 300L, 5L));
    windowedOperator.endWindow();
    switch(accumulationMode) {
        case ACCUMULATING:
            Assert.assertEquals("There should be exactly one tuple for the time trigger", 1, sink.collectedTuples.size());
            Assert.assertEquals(14L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
            break;
        case DISCARDING:
            Assert.assertEquals("There should be exactly one tuple for the time trigger", 1, sink.collectedTuples.size());
            Assert.assertEquals(9L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
            break;
        case ACCUMULATING_AND_RETRACTING:
            Assert.assertEquals("There should be exactly two tuples for the time trigger", 2, sink.collectedTuples.size());
            Assert.assertEquals(-5L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
            Assert.assertEquals(14L, ((Tuple<Long>) sink.collectedTuples.get(1)).getValue().longValue());
            break;
        default:
            throw new RuntimeException("Unknown accumulation mode: " + accumulationMode);
    }
    windowedOperator.teardown();
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong) MutableLong(org.apache.commons.lang3.mutable.MutableLong) CollectorTestSink(org.apache.apex.malhar.lib.testbench.CollectorTestSink)

Example 64 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WindowedOperatorTest method createDefaultKeyedWindowedOperator.

private KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> createDefaultKeyedWindowedOperator(boolean forSession) {
    KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
    if (useSpillable) {
        sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
        // TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
        windowStateStorage = new InMemoryWindowedStorage<>();
        if (forSession) {
            SpillableSessionWindowedStorage<String, MutableLong> sws = new SpillableSessionWindowedStorage<>();
            sws.setSpillableComplexComponent(sccImpl);
            keyedDataStorage = sws;
        } else {
            SpillableWindowedKeyedStorage<String, MutableLong> kds = new SpillableWindowedKeyedStorage<>();
            kds.setSpillableComplexComponent(sccImpl);
            keyedDataStorage = kds;
        }
        SpillableWindowedKeyedStorage<String, Long> krs = new SpillableWindowedKeyedStorage<>();
        krs.setSpillableComplexComponent(sccImpl);
        keyedRetractionStorage = krs;
        windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
    } else {
        windowStateStorage = new InMemoryWindowedStorage<>();
        if (forSession) {
            keyedDataStorage = new InMemorySessionWindowedStorage<>();
        } else {
            keyedDataStorage = new InMemoryWindowedKeyedStorage<>();
        }
        keyedRetractionStorage = new InMemoryWindowedKeyedStorage<>();
    }
    windowedOperator.setDataStorage(keyedDataStorage);
    windowedOperator.setRetractionStorage(keyedRetractionStorage);
    windowedOperator.setWindowStateStorage(windowStateStorage);
    windowedOperator.setAccumulation(new SumAccumulation());
    return windowedOperator;
}
Also used : SpillableSessionWindowedStorage(org.apache.apex.malhar.lib.window.impl.SpillableSessionWindowedStorage) SpillableWindowedKeyedStorage(org.apache.apex.malhar.lib.window.impl.SpillableWindowedKeyedStorage) KeyedWindowedOperatorImpl(org.apache.apex.malhar.lib.window.impl.KeyedWindowedOperatorImpl) MutableLong(org.apache.commons.lang3.mutable.MutableLong) MutableLong(org.apache.commons.lang3.mutable.MutableLong) SpillableComplexComponentImpl(org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl)

Example 65 with MutableLong

use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.

the class WindowedOperatorTest method testWatermarkAndAllowedLateness.

@Test
public void testWatermarkAndAllowedLateness() {
    WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
    CollectorTestSink controlSink = new CollectorTestSink();
    windowedOperator.controlOutput.setSink(controlSink);
    windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
    windowedOperator.setAllowedLateness(Duration.millis(1000));
    windowedOperator.setup(testMeta.operatorContext);
    windowedOperator.beginWindow(1);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
    Assert.assertEquals("There should be exactly one window in the storage", 1, plainDataStorage.size());
    Assert.assertEquals("There should be exactly one window in the storage", 1, windowStateStorage.size());
    Map.Entry<Window, WindowState> entry = windowStateStorage.entries().iterator().next();
    Window window = entry.getKey();
    WindowState windowState = entry.getValue();
    Assert.assertEquals(-1, windowState.watermarkArrivalTime);
    Assert.assertEquals(2L, plainDataStorage.get(window).longValue());
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 200L, 3L));
    Assert.assertEquals(5L, plainDataStorage.get(window).longValue());
    windowedOperator.processWatermark(new WatermarkImpl(BASE + 1200));
    windowedOperator.endWindow();
    Assert.assertTrue(windowState.watermarkArrivalTime >= 0);
    Assert.assertEquals("We should get one watermark tuple", 1, controlSink.getCount(false));
    windowedOperator.beginWindow(2);
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 900L, 4L));
    Assert.assertEquals("Late but not too late", 9L, plainDataStorage.get(window).longValue());
    windowedOperator.processWatermark(new WatermarkImpl(BASE + 3000));
    windowedOperator.endWindow();
    Assert.assertEquals("We should get two watermark tuples", 2, controlSink.getCount(false));
    windowedOperator.beginWindow(3);
    // this tuple should be dropped
    windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 120L, 5L));
    Assert.assertEquals("The window should be dropped because it's too late", 0, plainDataStorage.size());
    Assert.assertEquals("The window should be dropped because it's too late", 0, windowStateStorage.size());
    windowedOperator.endWindow();
    windowedOperator.teardown();
}
Also used : MutableLong(org.apache.commons.lang3.mutable.MutableLong) WatermarkImpl(org.apache.apex.malhar.lib.window.impl.WatermarkImpl) MutableLong(org.apache.commons.lang3.mutable.MutableLong) HashMap(java.util.HashMap) Map(java.util.Map) TreeMap(java.util.TreeMap) CollectorTestSink(org.apache.apex.malhar.lib.testbench.CollectorTestSink) Test(org.junit.Test)

Aggregations

MutableLong (org.apache.commons.lang3.mutable.MutableLong)66 Test (org.junit.Test)45 IOException (java.io.IOException)11 Query (org.apache.apex.malhar.lib.appdata.schemas.Query)6 CollectorTestSink (org.apache.apex.malhar.lib.testbench.CollectorTestSink)5 PageCache (org.neo4j.io.pagecache.PageCache)4 File (java.io.File)3 Map (java.util.Map)3 KeyedWindowedOperatorImpl (org.apache.apex.malhar.lib.window.impl.KeyedWindowedOperatorImpl)3 MutableBoolean (org.apache.commons.lang3.mutable.MutableBoolean)3 ParseException (java.text.ParseException)2 ArrayList (java.util.ArrayList)2 TreeMap (java.util.TreeMap)2 ConsoleOutputOperator (org.apache.apex.malhar.lib.io.ConsoleOutputOperator)2 SpillableComplexComponentImpl (org.apache.apex.malhar.lib.state.spillable.SpillableComplexComponentImpl)2 Tuple (org.apache.apex.malhar.lib.window.Tuple)2 WindowOption (org.apache.apex.malhar.lib.window.WindowOption)2 WindowState (org.apache.apex.malhar.lib.window.WindowState)2 SumLong (org.apache.apex.malhar.lib.window.accumulation.SumLong)2 WindowedOperatorImpl (org.apache.apex.malhar.lib.window.impl.WindowedOperatorImpl)2