use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.
the class ApexWindowedStreamImpl method count.
@Override
public <STREAM extends WindowedStream<Tuple.WindowedTuple<Long>>> STREAM count(Option... opts) {
Function.MapFunction<T, Tuple<Long>> kVMap = new Function.MapFunction<T, Tuple<Long>>() {
@Override
public Tuple<Long> f(T input) {
if (input instanceof Tuple.TimestampedTuple) {
return new Tuple.TimestampedTuple<>(((Tuple.TimestampedTuple) input).getTimestamp(), 1L);
} else {
return new Tuple.TimestampedTuple<>(System.currentTimeMillis(), 1L);
}
}
};
WindowedStream<Tuple<Long>> innerstream = map(kVMap);
WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createWindowedOperator(new SumLong());
return innerstream.addOperator(windowedOperator, windowedOperator.input, windowedOperator.output, opts);
}
use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.
the class WindowedOperatorTest method testImplicitWatermarks.
@Test
public void testImplicitWatermarks() {
WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
CollectorTestSink controlSink = new CollectorTestSink();
windowedOperator.controlOutput.setSink(controlSink);
windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
windowedOperator.setAllowedLateness(Duration.millis(1000));
windowedOperator.setImplicitWatermarkGenerator(new FixedDiffEventTimeWatermarkGen(100));
windowedOperator.setup(testMeta.operatorContext);
windowedOperator.beginWindow(1);
windowedOperator.endWindow();
Assert.assertEquals("We should get no watermark tuple", 0, controlSink.getCount(false));
windowedOperator.beginWindow(2);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
windowedOperator.endWindow();
Assert.assertEquals("We should get one watermark tuple", 1, controlSink.getCount(false));
Assert.assertEquals("Check Watermark value", ((ControlTuple.Watermark) controlSink.collectedTuples.get(0)).getTimestamp(), BASE);
windowedOperator.beginWindow(3);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 900L, 4L));
windowedOperator.endWindow();
Assert.assertEquals("We should get two watermark tuples", 2, controlSink.getCount(false));
Assert.assertEquals("Check Watermark value", ((ControlTuple.Watermark) controlSink.collectedTuples.get(1)).getTimestamp(), BASE + 800);
}
use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.
the class WindowedOperatorTest method testTrigger.
private void testTrigger(TriggerOption.AccumulationMode accumulationMode) {
WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
TriggerOption triggerOption = new TriggerOption().withEarlyFiringsAtEvery(Duration.millis(1000));
switch(accumulationMode) {
case ACCUMULATING:
triggerOption.accumulatingFiredPanes();
break;
case ACCUMULATING_AND_RETRACTING:
triggerOption.accumulatingAndRetractingFiredPanes();
break;
case DISCARDING:
triggerOption.discardingFiredPanes();
break;
default:
throw new RuntimeException("Unknown accumulation mode: " + accumulationMode);
}
windowedOperator.setTriggerOption(triggerOption);
windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
CollectorTestSink sink = new CollectorTestSink();
windowedOperator.output.setSink(sink);
windowedOperator.setup(testMeta.operatorContext);
windowedOperator.beginWindow(1);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 200L, 3L));
windowedOperator.endWindow();
Assert.assertTrue("No trigger should be fired yet", sink.collectedTuples.isEmpty());
windowedOperator.beginWindow(2);
windowedOperator.endWindow();
Assert.assertTrue("No trigger should be fired yet", sink.collectedTuples.isEmpty());
windowedOperator.beginWindow(3);
windowedOperator.endWindow();
Assert.assertEquals("There should be exactly one tuple for the time trigger", 1, sink.collectedTuples.size());
Assert.assertEquals(5L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
sink.collectedTuples.clear();
windowedOperator.beginWindow(4);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 400L, 4L));
windowedOperator.endWindow();
Assert.assertTrue("No trigger should be fired yet", sink.collectedTuples.isEmpty());
windowedOperator.beginWindow(5);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 300L, 5L));
windowedOperator.endWindow();
switch(accumulationMode) {
case ACCUMULATING:
Assert.assertEquals("There should be exactly one tuple for the time trigger", 1, sink.collectedTuples.size());
Assert.assertEquals(14L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
break;
case DISCARDING:
Assert.assertEquals("There should be exactly one tuple for the time trigger", 1, sink.collectedTuples.size());
Assert.assertEquals(9L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
break;
case ACCUMULATING_AND_RETRACTING:
Assert.assertEquals("There should be exactly two tuples for the time trigger", 2, sink.collectedTuples.size());
Assert.assertEquals(-5L, ((Tuple<Long>) sink.collectedTuples.get(0)).getValue().longValue());
Assert.assertEquals(14L, ((Tuple<Long>) sink.collectedTuples.get(1)).getValue().longValue());
break;
default:
throw new RuntimeException("Unknown accumulation mode: " + accumulationMode);
}
windowedOperator.teardown();
}
use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.
the class WindowedOperatorTest method createDefaultKeyedWindowedOperator.
private KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> createDefaultKeyedWindowedOperator(boolean forSession) {
KeyedWindowedOperatorImpl<String, Long, MutableLong, Long> windowedOperator = new KeyedWindowedOperatorImpl<>();
if (useSpillable) {
sccImpl = new SpillableComplexComponentImpl(testMeta.timeStore);
// TODO: We don't yet support Spillable data structures for window state storage because SpillableMapImpl does not yet support iterating over all keys.
windowStateStorage = new InMemoryWindowedStorage<>();
if (forSession) {
SpillableSessionWindowedStorage<String, MutableLong> sws = new SpillableSessionWindowedStorage<>();
sws.setSpillableComplexComponent(sccImpl);
keyedDataStorage = sws;
} else {
SpillableWindowedKeyedStorage<String, MutableLong> kds = new SpillableWindowedKeyedStorage<>();
kds.setSpillableComplexComponent(sccImpl);
keyedDataStorage = kds;
}
SpillableWindowedKeyedStorage<String, Long> krs = new SpillableWindowedKeyedStorage<>();
krs.setSpillableComplexComponent(sccImpl);
keyedRetractionStorage = krs;
windowedOperator.addComponent("SpillableComplexComponent", sccImpl);
} else {
windowStateStorage = new InMemoryWindowedStorage<>();
if (forSession) {
keyedDataStorage = new InMemorySessionWindowedStorage<>();
} else {
keyedDataStorage = new InMemoryWindowedKeyedStorage<>();
}
keyedRetractionStorage = new InMemoryWindowedKeyedStorage<>();
}
windowedOperator.setDataStorage(keyedDataStorage);
windowedOperator.setRetractionStorage(keyedRetractionStorage);
windowedOperator.setWindowStateStorage(windowStateStorage);
windowedOperator.setAccumulation(new SumAccumulation());
return windowedOperator;
}
use of org.apache.commons.lang3.mutable.MutableLong in project apex-malhar by apache.
the class WindowedOperatorTest method testWatermarkAndAllowedLateness.
@Test
public void testWatermarkAndAllowedLateness() {
WindowedOperatorImpl<Long, MutableLong, Long> windowedOperator = createDefaultWindowedOperator();
CollectorTestSink controlSink = new CollectorTestSink();
windowedOperator.controlOutput.setSink(controlSink);
windowedOperator.setWindowOption(new WindowOption.TimeWindows(Duration.millis(1000)));
windowedOperator.setAllowedLateness(Duration.millis(1000));
windowedOperator.setup(testMeta.operatorContext);
windowedOperator.beginWindow(1);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 100L, 2L));
Assert.assertEquals("There should be exactly one window in the storage", 1, plainDataStorage.size());
Assert.assertEquals("There should be exactly one window in the storage", 1, windowStateStorage.size());
Map.Entry<Window, WindowState> entry = windowStateStorage.entries().iterator().next();
Window window = entry.getKey();
WindowState windowState = entry.getValue();
Assert.assertEquals(-1, windowState.watermarkArrivalTime);
Assert.assertEquals(2L, plainDataStorage.get(window).longValue());
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 200L, 3L));
Assert.assertEquals(5L, plainDataStorage.get(window).longValue());
windowedOperator.processWatermark(new WatermarkImpl(BASE + 1200));
windowedOperator.endWindow();
Assert.assertTrue(windowState.watermarkArrivalTime >= 0);
Assert.assertEquals("We should get one watermark tuple", 1, controlSink.getCount(false));
windowedOperator.beginWindow(2);
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 900L, 4L));
Assert.assertEquals("Late but not too late", 9L, plainDataStorage.get(window).longValue());
windowedOperator.processWatermark(new WatermarkImpl(BASE + 3000));
windowedOperator.endWindow();
Assert.assertEquals("We should get two watermark tuples", 2, controlSink.getCount(false));
windowedOperator.beginWindow(3);
// this tuple should be dropped
windowedOperator.processTuple(new Tuple.TimestampedTuple<>(BASE + 120L, 5L));
Assert.assertEquals("The window should be dropped because it's too late", 0, plainDataStorage.size());
Assert.assertEquals("The window should be dropped because it's too late", 0, windowStateStorage.size());
windowedOperator.endWindow();
windowedOperator.teardown();
}
Aggregations