Examples with MistEvent edu.snu.mist.core.MistEvent used on opensource projects

Example 26 with MistEvent

use of edu.snu.mist.core.MistEvent in project mist by snuspl.

the class ApplyStatefulOperatorTest method testApplyStatefulOperatorGetState.

/**
 * Test getting state of the ApplyStatefulOperator.
 */
@Test
public void testApplyStatefulOperatorGetState() throws InterruptedException {
    // Generate the current ApplyStatefulOperator.
    final ApplyStatefulFunction applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulOperator<Integer, Integer> applyStatefulOperator = new ApplyStatefulOperator<>(applyStatefulFunction);
    final MistDataEvent data10 = new MistDataEvent(10, 0L);
    final MistDataEvent data20 = new MistDataEvent(20, 1L);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulOperator.setOutputEmitter(new OutputBufferEmitter(result));
    applyStatefulOperator.processLeftData(data10);
    applyStatefulOperator.processLeftData(data20);
    // Generate the expected ApplyStatefulOperator's state.
    final int expectedApplyStatefulOperatorState = 20;
    // Get the current ApplyStatefulOperator's state.
    final Map<String, Object> operatorState = applyStatefulOperator.getStateSnapshot();
    final int applyStatefulOperatorState = (int) operatorState.get("applyStatefulFunctionState");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedApplyStatefulOperatorState, applyStatefulOperatorState);
}

Example 27 with MistEvent

use of edu.snu.mist.core.MistEvent in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testSlidingTimeWindowOperator.

/**
 * Test TimeWindowOperator creating sliding window.
 * It receives some continuous data stream and groups them as a collection.
 */
@SuppressWarnings("unchecked")
@Test
public void testSlidingTimeWindowOperator() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (1000)Window1-----(1249):
    // (1000)Window2----------------------(1499):
    // (1250)Window3----------------(1749):
    // (1500)Window4----------------------(1999):
    // (1750)Window5-----------------------------: (will not be emitted)
    // d1--------------------------d2-----------d3-w1-------------w2-----------------w3:
    // expected results:
    // d1 in Window1
    // d1, d2 in Window2
    // d2, d3, w1 in Window3
    // d3, w2 in Window4
    timeWindowOperator.processLeftData(d1);
    Assert.assertEquals(0, result.size());
    timeWindowOperator.processLeftData(d2);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d1.getTimestamp(), emissionInterval, d1.getTimestamp());
    timeWindowOperator.processLeftData(d3);
    timeWindowOperator.processLeftWatermark(w1);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(1);
    expectedResult2.add(2);
    OperatorTestUtils.checkWindowData(result.get(1), expectedResult2, d1.getTimestamp(), windowSize, d2.getTimestamp());
    timeWindowOperator.processLeftWatermark(w2);
    Assert.assertEquals(4, result.size());
    final Collection<Integer> expectedResult3 = new LinkedList<>();
    expectedResult3.add(2);
    expectedResult3.add(3);
    OperatorTestUtils.checkWindowData(result.get(2), expectedResult3, d1.getTimestamp() + emissionInterval, windowSize, d3.getTimestamp());
    Assert.assertEquals(w1, result.get(3));
    timeWindowOperator.processLeftWatermark(w3);
    Assert.assertEquals(6, result.size());
    final Collection<Integer> expectedResult4 = new LinkedList<>();
    expectedResult4.add(3);
    OperatorTestUtils.checkWindowData(result.get(4), expectedResult4, d1.getTimestamp() + 2 * emissionInterval, windowSize, w2.getTimestamp());
    Assert.assertEquals(w2, result.get(5));
}

Example 28 with MistEvent

use of edu.snu.mist.core.MistEvent in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testSlidingCountWindowOperator.

/**
 * Test CountWindowOperator creating sliding window.
 * It receives some continuous data stream and groups them as a collection.
 */
@Test
public void testSlidingCountWindowOperator() throws InterruptedException {
    final int windowSize = 5;
    final int emissionInterval = 3;
    final CountWindowOperator<Integer> countWindowOperator = new CountWindowOperator<>(windowSize, emissionInterval);
    final List<MistEvent> result = new LinkedList<>();
    countWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (1)Window1(3):
    // (2)Window2--------(6):
    // (5)Window3-------(9):
    // (8)Window4--------------: (will not be emitted)
    // d1----d2--d3--d4-w1-d5--d6--d7-w2-d8-d9:
    // expected results:
    // d1, d2, d3 in Window1
    // d2, d3, d4, d5, d6, w1 in Window2
    // d5, d6, d7, d8, d9, w2 in Window3
    countWindowOperator.processLeftData(d1);
    countWindowOperator.processLeftData(d2);
    Assert.assertEquals(0, result.size());
    countWindowOperator.processLeftData(d3);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    expectedResult1.add(3);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, 1L, emissionInterval, d3.getTimestamp());
    countWindowOperator.processLeftData(d4);
    countWindowOperator.processLeftWatermark(w1);
    countWindowOperator.processLeftData(d5);
    Assert.assertEquals(1, result.size());
    countWindowOperator.processLeftData(d6);
    Assert.assertEquals(3, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(2);
    expectedResult2.add(3);
    expectedResult2.add(4);
    expectedResult2.add(5);
    expectedResult2.add(6);
    OperatorTestUtils.checkWindowData(result.get(1), expectedResult2, 2L, windowSize, d6.getTimestamp());
    Assert.assertEquals(w1, result.get(2));
    countWindowOperator.processLeftData(d7);
    countWindowOperator.processLeftWatermark(w2);
    countWindowOperator.processLeftData(d8);
    Assert.assertEquals(3, result.size());
    countWindowOperator.processLeftData(d9);
    Assert.assertEquals(5, result.size());
    final Collection<Integer> expectedResult3 = new LinkedList<>();
    expectedResult3.add(5);
    expectedResult3.add(6);
    expectedResult3.add(7);
    expectedResult3.add(8);
    expectedResult3.add(9);
    OperatorTestUtils.checkWindowData(result.get(3), expectedResult3, 5L, windowSize, d9.getTimestamp());
    Assert.assertEquals(w2, result.get(4));
}

Example 29 with MistEvent

use of edu.snu.mist.core.MistEvent in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testCountWindowOperatorSetState.

/**
 * Test setting state of the CountWindowOperator.
 */
@Test
public void testCountWindowOperatorSetState() throws InterruptedException {
    final int windowSize = 5;
    final int emissionInterval = 3;
    // Generate a new state and set it to a new CountWindowOperator.
    final Window expectedWindow1 = new WindowImpl<>(1L, emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d1);
    expectedWindow1.putData(d2);
    final Window expectedWindow2 = new WindowImpl<>(2L, windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = 2L + emissionInterval;
    final long expectedCount = 3L;
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("windowQueue", expectedWindowQueue);
    loadStateMap.put("windowCreationPoint", expectedWindowCreationPoint);
    loadStateMap.put("count", expectedCount);
    final CountWindowOperator<Integer> countWindowOperator = new CountWindowOperator<>(windowSize, emissionInterval);
    countWindowOperator.setState(loadStateMap);
    // Compare the original and the set operator.
    final Map<String, Object> operatorState = countWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    final long count = (long) operatorState.get("count");
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
    Assert.assertEquals(expectedCount, count);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    countWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    countWindowOperator.processLeftData(d3);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    expectedResult1.add(3);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, 1L, emissionInterval, d3.getTimestamp());
}

Example 30 with MistEvent

use of edu.snu.mist.core.MistEvent in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperator.

/**
 * Test StateTransitionOperator.
 * Final states are "1" and "4".
 */
@Test
public void testStateTransitionOperator() throws InterruptedException {
    // input events
    // expected state transitions: 0 -- 1 -- 0 -- 3 -- 4, "1" and "4" are emitted.
    final Map<String, Integer> value1 = new HashMap<>();
    value1.put("number", 1);
    final MistDataEvent data1 = new MistDataEvent(value1, 0L);
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 1L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(7L);
    final Map<String, Integer> value3 = new HashMap<>();
    value3.put("number", 3);
    final MistDataEvent data3 = new MistDataEvent(value3, 10L);
    final Map<String, Integer> value4 = new HashMap<>();
    value4.put("number", 4);
    final MistDataEvent data4 = new MistDataEvent(value4, 11L);
    // make a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("1");
    finalSet.add("4");
    // make a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list0 = new ArrayList<>();
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 1), "1"));
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 3), "3"));
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "0"));
    final Collection<Tuple2<MISTPredicate, String>> list3 = new ArrayList<>();
    list3.add(new Tuple2<>(new RuleBasedEQPredicate("number", 4), "4"));
    stateTable.put("0", list0);
    stateTable.put("1", list1);
    stateTable.put("3", list3);
    // make a state transition operator
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    final List<MistEvent> result = new LinkedList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    stateTransitionOperator.processLeftData(data1);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(data1, result.get(0));
    Assert.assertEquals("1", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data2);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals("0", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
    Assert.assertEquals("0", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data3);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals("3", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data4);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(data4, result.get(2));
    Assert.assertEquals("4", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
}