Examples with WindowImpl edu.snu.mist.core.operators.window.WindowImpl used on opensource projects

Example 1 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class ApplyStatefulWindowOperatorTest method testApplyStatefulWindowOperator.

/**
 * Test ApplyStatefulWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testApplyStatefulWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window1 = new WindowImpl<>(0L, 100L);
    final WindowImpl<Integer> window2 = new WindowImpl<>(100L, 100L);
    window1.putData(new MistDataEvent(10, 10));
    window1.putData(new MistDataEvent(20, 20));
    window1.putData(new MistDataEvent(30, 30));
    window1.putData(new MistDataEvent(15, 90));
    window2.putData(new MistDataEvent(10, 110));
    window2.putData(new MistDataEvent(20, 120));
    final MistDataEvent dataEvent1 = new MistDataEvent(window1, 90L);
    final MistDataEvent dataEvent2 = new MistDataEvent(window2, 190L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(201L);
    // the state finding maximum integer value among received inputs
    final ApplyStatefulFunction<Integer, Integer> applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulWindowOperator<Integer, Integer> applyStatefulWindowOperator = new ApplyStatefulWindowOperator<>(applyStatefulFunction);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // check that the operator processes a window input properly
    applyStatefulWindowOperator.processLeftData(dataEvent1);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals(30, ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    // check that the operator processes another window separately
    applyStatefulWindowOperator.processLeftData(dataEvent2);
    Assert.assertEquals(2, result.size());
    Assert.assertTrue(result.get(1).isData());
    Assert.assertEquals(20, ((MistDataEvent) result.get(1)).getValue());
    Assert.assertEquals(190L, result.get(1).getTimestamp());
    // check that the operator processes watermark event well
    applyStatefulWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(watermarkEvent, result.get(2));
}

Example 2 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testCountWindowOperatorGetState.

/**
 * Test getting state of the CountWindowOperator.
 */
@Test
public void testCountWindowOperatorGetState() throws InterruptedException {
    final int windowSize = 5;
    final int emissionInterval = 3;
    // Generate the current CountWindowOperator.
    final CountWindowOperator<Integer> countWindowOperator = new CountWindowOperator<>(windowSize, emissionInterval);
    countWindowOperator.processLeftData(d1);
    countWindowOperator.processLeftData(d2);
    // Generate the expected CountWindowOperator's state.
    final Window expectedWindow1 = new WindowImpl<>(1L, emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d1);
    expectedWindow1.putData(d2);
    final Window expectedWindow2 = new WindowImpl<>(2L, windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = 2L + emissionInterval;
    final long expectedCount = 3L;
    // Get the current CountWindowOperator's state.
    final Map<String, Object> operatorState = countWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (LinkedList<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    final long count = (long) operatorState.get("count");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
    Assert.assertEquals(expectedCount, count);
}

Example 3 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testTimeWindowOperatorSetState.

/**
 * Test setting state of the TimeWindowOperator.
 */
@Test
public void testTimeWindowOperatorSetState() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    // Generate a new state and set it to a new TimeWindowOperator.
    final Window expectedWindow1 = new WindowImpl<>(d4.getTimestamp(), emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d4);
    expectedWindow1.putWatermark(w2);
    final Window expectedWindow2 = new WindowImpl<>(d4.getTimestamp(), windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d4);
    expectedWindow2.putWatermark(w2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = d4.getTimestamp() + emissionInterval;
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("windowQueue", expectedWindowQueue);
    loadStateMap.put("windowCreationPoint", expectedWindowCreationPoint);
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    timeWindowOperator.setState(loadStateMap);
    // Get the current TimeWindowOperator's state.
    final Map<String, Object> operatorState = timeWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    // Compare the original and the set operator.
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    timeWindowOperator.processLeftData(d10);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(4);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d4.getTimestamp(), emissionInterval, w2.getTimestamp());
    Assert.assertEquals(result.get(1), w2);
}

Example 4 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testTimeWindowOperatorGetState.

/**
 * Test getting state of the TimeWindowOperator.
 */
@Test
public void testTimeWindowOperatorGetState() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    // Generate the current TimeWindowOperator.
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    timeWindowOperator.processLeftData(d4);
    timeWindowOperator.processLeftWatermark(w2);
    // Generate the expected TimeWindowOperator's state.
    final Window expectedWindow1 = new WindowImpl<>(d4.getTimestamp(), emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d4);
    expectedWindow1.putWatermark(w2);
    final Window expectedWindow2 = new WindowImpl<>(d4.getTimestamp(), windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d4);
    expectedWindow2.putWatermark(w2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = d4.getTimestamp() + emissionInterval;
    // Get the current TimeWindowOperator's state.
    final Map<String, Object> operatorState = timeWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
}

Example 5 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class JoinOperatorTest method testAggregateWindowOperator.

/**
 * Tests JoinOperator.
 * It joins a pair of inputs in two streams that has same key.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("Hello", 1), null), 10));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("MIST", 2), null), 20));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 3000L)), 30));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("SNUCMS", 3), null), 40));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 4000L)), 50));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(2, 5000L)), 60));
    final MistDataEvent dataEvent = new MistDataEvent(window, 60L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // predicate that tests whether two input data have same key or not
    final MISTBiPredicate<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinPredicate = (tuple1, tuple2) -> tuple1.get(1).equals(tuple2.get(0));
    final JoinOperator<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinOperator = new JoinOperator<>(joinPredicate);
    // expected pairs
    // {Hello, 1} and {1, 3000L}
    // {Hello, 1} and {1, 4000L}
    // {MIST, 2} and {2, 5000L}
    final List<MistEvent> result = new LinkedList<>();
    joinOperator.setOutputEmitter(new OutputBufferEmitter(result));
    joinOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertTrue(((MistDataEvent) result.get(0)).getValue() instanceof WindowData);
    final WindowData windowData = (WindowData) ((MistDataEvent) result.get(0)).getValue();
    Assert.assertEquals(0L, windowData.getStart());
    Assert.assertEquals(99L, windowData.getEnd());
    final Collection<Tuple2<Tuple2<String, Integer>, Tuple2<Integer, Long>>> dataCollection = windowData.getDataCollection();
    final Iterator iterator = dataCollection.iterator();
    Assert.assertEquals(3, dataCollection.size());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 3000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 4000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("MIST", 2), new Tuple2<>(2, 5000L)), iterator.next());
    Assert.assertEquals(60L, result.get(0).getTimestamp());
    joinOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}