Examples with MistWatermarkEvent edu.snu.mist.core.MistWatermarkEvent used on opensource projects

Example 6 with MistWatermarkEvent

use of edu.snu.mist.core.MistWatermarkEvent in project mist by snuspl.

the class PunctuatedEventGenerator method emitData.

@Override
public void emitData(final I input) {
    if (isWatermark.test(input)) {
        latestWatermarkTimestamp = parseTimestamp.apply(input);
        outputEmitter.emitWatermark(new MistWatermarkEvent(latestWatermarkTimestamp));
    } else {
        MistDataEvent newInputEvent = generateEvent(input);
        if (newInputEvent != null) {
            outputEmitter.emitData(newInputEvent);
        }
    }
}

Example 7 with MistWatermarkEvent

use of edu.snu.mist.core.MistWatermarkEvent in project mist by snuspl.

the class StateSerializerTest method testOtherStates.

@Test
public void testOtherStates() {
    // States to be serialized.
    final Map<String, Integer> testMap = new HashMap<>();
    testMap.put("Cheeseburgers", 6);
    testMap.put("Drinks", 3);
    final MistWatermarkEvent testWatermarkEvent = new MistWatermarkEvent(98L);
    final Window<String> testWindow = new WindowImpl<>(100L);
    final Queue<Window<String>> testQueue = new LinkedList<>();
    testQueue.add(testWindow);
    // The map that contains the above states.
    final Map<String, Object> testStateMap = new HashMap<>();
    testStateMap.put("testMap", testMap);
    testStateMap.put("testWatermarkEvent", testWatermarkEvent);
    testStateMap.put("testWindow", testWindow);
    testStateMap.put("testQueue", testQueue);
    // Serialize and deserialize the stateMap.
    final Map<String, Object> serializedStateMap = StateSerializer.serializeStateMap(testStateMap);
    final Map<String, Object> deserializedStateMap = StateSerializer.deserializeStateMap(serializedStateMap);
    final Map<String, Integer> deserializedMap = (Map<String, Integer>) deserializedStateMap.get("testMap");
    final MistWatermarkEvent deserializedWatermarkEvent = (MistWatermarkEvent) deserializedStateMap.get("testWatermarkEvent");
    final Window<String> deserializedWindow = (Window<String>) deserializedStateMap.get("testWindow");
    final Queue<Window<String>> deserializedQueue = (Queue<Window<String>>) deserializedStateMap.get("testQueue");
    // Compare the results.
    Assert.assertEquals(testMap, deserializedMap);
    Assert.assertEquals(testWatermarkEvent, deserializedWatermarkEvent);
    Assert.assertEquals(testWindow, deserializedWindow);
    Assert.assertEquals(testQueue, deserializedQueue);
}

Example 8 with MistWatermarkEvent

use of edu.snu.mist.core.MistWatermarkEvent in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperator.

/**
 * Test StateTransitionOperator.
 * Final states are "1" and "4".
 */
@Test
public void testStateTransitionOperator() throws InterruptedException {
    // input events
    // expected state transitions: 0 -- 1 -- 0 -- 3 -- 4, "1" and "4" are emitted.
    final Map<String, Integer> value1 = new HashMap<>();
    value1.put("number", 1);
    final MistDataEvent data1 = new MistDataEvent(value1, 0L);
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 1L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(7L);
    final Map<String, Integer> value3 = new HashMap<>();
    value3.put("number", 3);
    final MistDataEvent data3 = new MistDataEvent(value3, 10L);
    final Map<String, Integer> value4 = new HashMap<>();
    value4.put("number", 4);
    final MistDataEvent data4 = new MistDataEvent(value4, 11L);
    // make a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("1");
    finalSet.add("4");
    // make a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list0 = new ArrayList<>();
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 1), "1"));
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 3), "3"));
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "0"));
    final Collection<Tuple2<MISTPredicate, String>> list3 = new ArrayList<>();
    list3.add(new Tuple2<>(new RuleBasedEQPredicate("number", 4), "4"));
    stateTable.put("0", list0);
    stateTable.put("1", list1);
    stateTable.put("3", list3);
    // make a state transition operator
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    final List<MistEvent> result = new LinkedList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    stateTransitionOperator.processLeftData(data1);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(data1, result.get(0));
    Assert.assertEquals("1", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data2);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals("0", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
    Assert.assertEquals("0", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data3);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals("3", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data4);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(data4, result.get(2));
    Assert.assertEquals("4", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
}

Example 9 with MistWatermarkEvent

use of edu.snu.mist.core.MistWatermarkEvent in project mist by snuspl.

the class UnionOperatorTest method testUnionOperator.

/**
 * Test union operation.
 * It merges two streams into one. (ordered by timestamp)
 */
@Test
public void testUnionOperator() throws InjectionException {
    // input stream events
    final MistDataEvent a = new MistDataEvent("a", 1L);
    final MistDataEvent b = new MistDataEvent("b", 3L);
    final MistDataEvent c = new MistDataEvent("c", 5L);
    final MistWatermarkEvent lw1 = new MistWatermarkEvent(9L);
    final MistWatermarkEvent lw2 = new MistWatermarkEvent(10L);
    final MistDataEvent d = new MistDataEvent("d", 1L);
    final MistDataEvent e = new MistDataEvent("e", 2L);
    final MistDataEvent f = new MistDataEvent("f", 4L);
    final MistWatermarkEvent rw1 = new MistWatermarkEvent(3L);
    final MistDataEvent g = new MistDataEvent("g", 6L);
    final MistWatermarkEvent rw2 = new MistWatermarkEvent(11L);
    final UnionOperator unionOperator = new UnionOperator();
    final List<MistEvent> result = new LinkedList<>();
    unionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // Test:
    // * Left: ----------W:10--W:9--c----b--------a---->
    // * Right -W:11 --g--------------W:3--f--e--d------>
    // Merged stream: ----g--c--f--b--e--d--a-->
    unionOperator.processLeftData(a);
    Assert.assertEquals(0, result.size());
    unionOperator.processRightData(d);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(a, result.get(0));
    Assert.assertEquals(d, result.get(1));
    unionOperator.processRightData(e);
    unionOperator.processRightData(f);
    Assert.assertEquals(2, result.size());
    unionOperator.processLeftData(b);
    Assert.assertEquals(4, result.size());
    Assert.assertEquals(e, result.get(2));
    Assert.assertEquals(b, result.get(3));
    unionOperator.processRightWatermark(rw1);
    Assert.assertEquals(5, result.size());
    Assert.assertEquals(rw1, result.get(4));
    unionOperator.processLeftData(c);
    Assert.assertEquals(6, result.size());
    Assert.assertEquals(f, result.get(5));
    unionOperator.processLeftWatermark(lw1);
    Assert.assertEquals(6, result.size());
    unionOperator.processLeftWatermark(lw2);
    Assert.assertEquals(6, result.size());
    unionOperator.processRightData(g);
    Assert.assertEquals(8, result.size());
    Assert.assertEquals(c, result.get(6));
    Assert.assertEquals(g, result.get(7));
    unionOperator.processRightWatermark(rw2);
    Assert.assertEquals(9, result.size());
    Assert.assertEquals(lw2, result.get(8));
}

Example 10 with MistWatermarkEvent

use of edu.snu.mist.core.MistWatermarkEvent in project mist by snuspl.

the class AggregateWindowOperatorTest method testAggregateWindowOperator.

/**
 * Test AggregateWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(10, 10));
    window.putData(new MistDataEvent(20, 20));
    window.putData(new MistDataEvent(15, 30));
    window.putData(new MistDataEvent(30, 90));
    final MistDataEvent dataEvent = new MistDataEvent(window, 90L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // functions that dealing with input WindowData
    final MISTFunction<WindowData<Integer>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", " + windowData.getStart() + ", " + windowData.getEnd();
    };
    final AggregateWindowOperator<Integer, String> aggregateWindowOperator = new AggregateWindowOperator<>(aggregateFunc);
    final List<MistEvent> result = new LinkedList<>();
    aggregateWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    aggregateWindowOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals("[10, 20, 15, 30], 0, 99", ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    aggregateWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}