Examples with WindowImpl edu.snu.mist.core.operators.window.WindowImpl used on opensource projects

Example 6 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class JoinOperatorTest method testAggregateWindowOperator.

/**
 * Tests JoinOperator.
 * It joins a pair of inputs in two streams that has same key.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("Hello", 1), null), 10));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("MIST", 2), null), 20));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 3000L)), 30));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("SNUCMS", 3), null), 40));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 4000L)), 50));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(2, 5000L)), 60));
    final MistDataEvent dataEvent = new MistDataEvent(window, 60L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // predicate that tests whether two input data have same key or not
    final MISTBiPredicate<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinPredicate = (tuple1, tuple2) -> tuple1.get(1).equals(tuple2.get(0));
    final JoinOperator<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinOperator = new JoinOperator<>(joinPredicate);
    // expected pairs
    // {Hello, 1} and {1, 3000L}
    // {Hello, 1} and {1, 4000L}
    // {MIST, 2} and {2, 5000L}
    final List<MistEvent> result = new LinkedList<>();
    joinOperator.setOutputEmitter(new OutputBufferEmitter(result));
    joinOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertTrue(((MistDataEvent) result.get(0)).getValue() instanceof WindowData);
    final WindowData windowData = (WindowData) ((MistDataEvent) result.get(0)).getValue();
    Assert.assertEquals(0L, windowData.getStart());
    Assert.assertEquals(99L, windowData.getEnd());
    final Collection<Tuple2<Tuple2<String, Integer>, Tuple2<Integer, Long>>> dataCollection = windowData.getDataCollection();
    final Iterator iterator = dataCollection.iterator();
    Assert.assertEquals(3, dataCollection.size());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 3000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 4000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("MIST", 2), new Tuple2<>(2, 5000L)), iterator.next());
    Assert.assertEquals(60L, result.get(0).getTimestamp());
    joinOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}

Example 7 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class StateSerializerTest method testOtherStates.

@Test
public void testOtherStates() {
    // States to be serialized.
    final Map<String, Integer> testMap = new HashMap<>();
    testMap.put("Cheeseburgers", 6);
    testMap.put("Drinks", 3);
    final MistWatermarkEvent testWatermarkEvent = new MistWatermarkEvent(98L);
    final Window<String> testWindow = new WindowImpl<>(100L);
    final Queue<Window<String>> testQueue = new LinkedList<>();
    testQueue.add(testWindow);
    // The map that contains the above states.
    final Map<String, Object> testStateMap = new HashMap<>();
    testStateMap.put("testMap", testMap);
    testStateMap.put("testWatermarkEvent", testWatermarkEvent);
    testStateMap.put("testWindow", testWindow);
    testStateMap.put("testQueue", testQueue);
    // Serialize and deserialize the stateMap.
    final Map<String, Object> serializedStateMap = StateSerializer.serializeStateMap(testStateMap);
    final Map<String, Object> deserializedStateMap = StateSerializer.deserializeStateMap(serializedStateMap);
    final Map<String, Integer> deserializedMap = (Map<String, Integer>) deserializedStateMap.get("testMap");
    final MistWatermarkEvent deserializedWatermarkEvent = (MistWatermarkEvent) deserializedStateMap.get("testWatermarkEvent");
    final Window<String> deserializedWindow = (Window<String>) deserializedStateMap.get("testWindow");
    final Queue<Window<String>> deserializedQueue = (Queue<Window<String>>) deserializedStateMap.get("testQueue");
    // Compare the results.
    Assert.assertEquals(testMap, deserializedMap);
    Assert.assertEquals(testWatermarkEvent, deserializedWatermarkEvent);
    Assert.assertEquals(testWindow, deserializedWindow);
    Assert.assertEquals(testQueue, deserializedQueue);
}

Example 8 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class SessionWindowOperatorTest method testSessionWindowOperatorGetState.

/**
 * Test getting state of the SessionWindowOperator.
 */
@Test
public void testSessionWindowOperatorGetState() throws InterruptedException {
    final int sessionInterval = 500;
    // Generate the current SessionWindowOperator.
    final SessionWindowOperator<Integer> sessionWindowOperator = new SessionWindowOperator<>(sessionInterval);
    sessionWindowOperator.processLeftData(d6);
    sessionWindowOperator.processLeftWatermark(w3);
    // Generate the expected SessionWindowOperator's state.
    final Window expectedCurrentWindow = new WindowImpl<>(d6.getTimestamp(), Long.MAX_VALUE, new LinkedList<>());
    expectedCurrentWindow.putData(d6);
    expectedCurrentWindow.putWatermark(w3);
    expectedCurrentWindow.setEnd(w3.getTimestamp());
    final long expectedLatestDataTimestamp = d6.getTimestamp();
    final boolean expectedStartedNewWindow = true;
    // Get the current SessionWindowOperator's state.
    final Map<String, Object> operatorState = sessionWindowOperator.getStateSnapshot();
    final Window<Integer> currentWindow = (Window<Integer>) operatorState.get("currentWindow");
    final long latestDataTimestamp = (long) operatorState.get("latestDataTimestamp");
    final boolean startedNewWindow = (boolean) operatorState.get("startedNewWindow");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedCurrentWindow, currentWindow);
    Assert.assertEquals(expectedLatestDataTimestamp, latestDataTimestamp);
    Assert.assertEquals(expectedStartedNewWindow, startedNewWindow);
}

Example 9 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class AggregateWindowOperatorTest method testAggregateWindowOperator.

/**
 * Test AggregateWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(10, 10));
    window.putData(new MistDataEvent(20, 20));
    window.putData(new MistDataEvent(15, 30));
    window.putData(new MistDataEvent(30, 90));
    final MistDataEvent dataEvent = new MistDataEvent(window, 90L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // functions that dealing with input WindowData
    final MISTFunction<WindowData<Integer>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", " + windowData.getStart() + ", " + windowData.getEnd();
    };
    final AggregateWindowOperator<Integer, String> aggregateWindowOperator = new AggregateWindowOperator<>(aggregateFunc);
    final List<MistEvent> result = new LinkedList<>();
    aggregateWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    aggregateWindowOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals("[10, 20, 15, 30], 0, 99", ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    aggregateWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}

Example 10 with WindowImpl

use of edu.snu.mist.core.operators.window.WindowImpl in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testCountWindowOperatorSetState.

/**
 * Test setting state of the CountWindowOperator.
 */
@Test
public void testCountWindowOperatorSetState() throws InterruptedException {
    final int windowSize = 5;
    final int emissionInterval = 3;
    // Generate a new state and set it to a new CountWindowOperator.
    final Window expectedWindow1 = new WindowImpl<>(1L, emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d1);
    expectedWindow1.putData(d2);
    final Window expectedWindow2 = new WindowImpl<>(2L, windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = 2L + emissionInterval;
    final long expectedCount = 3L;
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("windowQueue", expectedWindowQueue);
    loadStateMap.put("windowCreationPoint", expectedWindowCreationPoint);
    loadStateMap.put("count", expectedCount);
    final CountWindowOperator<Integer> countWindowOperator = new CountWindowOperator<>(windowSize, emissionInterval);
    countWindowOperator.setState(loadStateMap);
    // Compare the original and the set operator.
    final Map<String, Object> operatorState = countWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    final long count = (long) operatorState.get("count");
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
    Assert.assertEquals(expectedCount, count);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    countWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    countWindowOperator.processLeftData(d3);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    expectedResult1.add(3);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, 1L, emissionInterval, d3.getTimestamp());
}