Examples with TimeWindowOperator edu.snu.mist.core.operators.window.TimeWindowOperator used on opensource projects

Example 1 with TimeWindowOperator

use of edu.snu.mist.core.operators.window.TimeWindowOperator in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testHoppingTimeWindowOperator.

/**
 * Test TimeWindowOperator creating hopping window.
 * It receives some continuous data stream and groups them as a collection.
 */
@Test
public void testHoppingTimeWindowOperator() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 750;
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (1000)Window1------------------(1499):
    // (1750)Window2------------------(2249):
    // d1-----------------------d2---------------d3-w1------------d4-w2-----------------w3------w4:
    // expected results:
    // d1, d2 in Window1
    // d4, w3 in Window2
    timeWindowOperator.processLeftData(d1);
    timeWindowOperator.processLeftData(d2);
    Assert.assertEquals(0, result.size());
    timeWindowOperator.processLeftData(d3);
    timeWindowOperator.processLeftWatermark(w1);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d1.getTimestamp(), windowSize, d2.getTimestamp());
    timeWindowOperator.processLeftData(d4);
    timeWindowOperator.processLeftWatermark(w2);
    timeWindowOperator.processLeftWatermark(w3);
    Assert.assertEquals(1, result.size());
    timeWindowOperator.processLeftWatermark(w4);
    Assert.assertEquals(3, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(4);
    OperatorTestUtils.checkWindowData(result.get(1), expectedResult2, d1.getTimestamp() + emissionInterval, windowSize, w3.getTimestamp());
    Assert.assertEquals(w3, result.get(2));
}

Example 2 with TimeWindowOperator

use of edu.snu.mist.core.operators.window.TimeWindowOperator in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testTimeWindowOperatorSetState.

/**
 * Test setting state of the TimeWindowOperator.
 */
@Test
public void testTimeWindowOperatorSetState() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    // Generate a new state and set it to a new TimeWindowOperator.
    final Window expectedWindow1 = new WindowImpl<>(d4.getTimestamp(), emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d4);
    expectedWindow1.putWatermark(w2);
    final Window expectedWindow2 = new WindowImpl<>(d4.getTimestamp(), windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d4);
    expectedWindow2.putWatermark(w2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = d4.getTimestamp() + emissionInterval;
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("windowQueue", expectedWindowQueue);
    loadStateMap.put("windowCreationPoint", expectedWindowCreationPoint);
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    timeWindowOperator.setState(loadStateMap);
    // Get the current TimeWindowOperator's state.
    final Map<String, Object> operatorState = timeWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    // Compare the original and the set operator.
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    timeWindowOperator.processLeftData(d10);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(4);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d4.getTimestamp(), emissionInterval, w2.getTimestamp());
    Assert.assertEquals(result.get(1), w2);
}

Example 3 with TimeWindowOperator

use of edu.snu.mist.core.operators.window.TimeWindowOperator in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testTimeWindowOperatorGetState.

/**
 * Test getting state of the TimeWindowOperator.
 */
@Test
public void testTimeWindowOperatorGetState() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    // Generate the current TimeWindowOperator.
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    timeWindowOperator.processLeftData(d4);
    timeWindowOperator.processLeftWatermark(w2);
    // Generate the expected TimeWindowOperator's state.
    final Window expectedWindow1 = new WindowImpl<>(d4.getTimestamp(), emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d4);
    expectedWindow1.putWatermark(w2);
    final Window expectedWindow2 = new WindowImpl<>(d4.getTimestamp(), windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d4);
    expectedWindow2.putWatermark(w2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = d4.getTimestamp() + emissionInterval;
    // Get the current TimeWindowOperator's state.
    final Map<String, Object> operatorState = timeWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
}

Example 4 with TimeWindowOperator

use of edu.snu.mist.core.operators.window.TimeWindowOperator in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testSlidingTimeWindowOperator.

/**
 * Test TimeWindowOperator creating sliding window.
 * It receives some continuous data stream and groups them as a collection.
 */
@SuppressWarnings("unchecked")
@Test
public void testSlidingTimeWindowOperator() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (1000)Window1-----(1249):
    // (1000)Window2----------------------(1499):
    // (1250)Window3----------------(1749):
    // (1500)Window4----------------------(1999):
    // (1750)Window5-----------------------------: (will not be emitted)
    // d1--------------------------d2-----------d3-w1-------------w2-----------------w3:
    // expected results:
    // d1 in Window1
    // d1, d2 in Window2
    // d2, d3, w1 in Window3
    // d3, w2 in Window4
    timeWindowOperator.processLeftData(d1);
    Assert.assertEquals(0, result.size());
    timeWindowOperator.processLeftData(d2);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d1.getTimestamp(), emissionInterval, d1.getTimestamp());
    timeWindowOperator.processLeftData(d3);
    timeWindowOperator.processLeftWatermark(w1);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(1);
    expectedResult2.add(2);
    OperatorTestUtils.checkWindowData(result.get(1), expectedResult2, d1.getTimestamp(), windowSize, d2.getTimestamp());
    timeWindowOperator.processLeftWatermark(w2);
    Assert.assertEquals(4, result.size());
    final Collection<Integer> expectedResult3 = new LinkedList<>();
    expectedResult3.add(2);
    expectedResult3.add(3);
    OperatorTestUtils.checkWindowData(result.get(2), expectedResult3, d1.getTimestamp() + emissionInterval, windowSize, d3.getTimestamp());
    Assert.assertEquals(w1, result.get(3));
    timeWindowOperator.processLeftWatermark(w3);
    Assert.assertEquals(6, result.size());
    final Collection<Integer> expectedResult4 = new LinkedList<>();
    expectedResult4.add(3);
    OperatorTestUtils.checkWindowData(result.get(4), expectedResult4, d1.getTimestamp() + 2 * emissionInterval, windowSize, w2.getTimestamp());
    Assert.assertEquals(w2, result.get(5));
}