Examples with OutputBufferEmitter edu.snu.mist.core.utils.OutputBufferEmitter used on opensource projects

Example 16 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class SessionWindowOperatorTest method testSessionWindowOperator.

/**
 * Test whether SessionWindowOperator creates windows properly.
 * It receives some continuous data stream and groups them as a collection.
 */
@Test
public void testSessionWindowOperator() throws InterruptedException {
    final int sessionInterval = 500;
    final SessionWindowOperator<Integer> sessionWindowOperator = new SessionWindowOperator<>(sessionInterval);
    final List<MistEvent> result = new LinkedList<>();
    sessionWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (200)Window1-(1200):
    // (1750)Window2-----(2000):
    // (2700)Window3-(3100):
    // (3700)Window5---: (will not be emitted)
    // d1-----d2-----w1---------d3--------d4-------d5-------d6----w2------w3-------w4:
    // expected results:
    // d1, d2, w1 in Window1
    // d3, d4, d5 in Window2
    // d6, w3 in Window3
    sessionWindowOperator.processLeftData(d1);
    sessionWindowOperator.processLeftData(d2);
    sessionWindowOperator.processLeftWatermark(w1);
    Assert.assertEquals(0, result.size());
    sessionWindowOperator.processLeftData(d3);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d1.getTimestamp(), w1.getTimestamp() - d1.getTimestamp() + 1, w1.getTimestamp());
    Assert.assertEquals(w1, result.get(1));
    sessionWindowOperator.processLeftData(d4);
    sessionWindowOperator.processLeftData(d5);
    Assert.assertEquals(2, result.size());
    sessionWindowOperator.processLeftData(d6);
    Assert.assertEquals(3, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(3);
    expectedResult2.add(4);
    expectedResult2.add(5);
    OperatorTestUtils.checkWindowData(result.get(2), expectedResult2, d3.getTimestamp(), d5.getTimestamp() - d3.getTimestamp() + 1, d5.getTimestamp());
    sessionWindowOperator.processLeftWatermark(w2);
    sessionWindowOperator.processLeftWatermark(w3);
    Assert.assertEquals(3, result.size());
    sessionWindowOperator.processLeftWatermark(w4);
    Assert.assertEquals(5, result.size());
    final Collection<Integer> expectedResult3 = new LinkedList<>();
    expectedResult3.add(6);
    OperatorTestUtils.checkWindowData(result.get(3), expectedResult3, d6.getTimestamp(), w3.getTimestamp() - d6.getTimestamp() + 1, w3.getTimestamp());
    Assert.assertEquals(w3, result.get(4));
}

Example 17 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class StatelessOperatorTest method testStatelessOperator.

private void testStatelessOperator(final List<MistDataEvent> inputStream, final List<MistEvent> expected, final Operator operator) {
    final List<MistEvent> result = new LinkedList<>();
    operator.setOutputEmitter(new OutputBufferEmitter(result));
    inputStream.stream().forEach(operator::processLeftData);
    LOG.info("expected: " + expected);
    LOG.info("result: " + result);
    Assert.assertEquals(expected, result);
}

Example 18 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperatorSetState.

/**
 * Test setting state of StateTransitionOperator.
 */
@Test
public void testStateTransitionOperatorSetState() throws InterruptedException {
    // Generate a new state and set it to StateTransitionOperator.
    final String newFunctionState = "1";
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("stateTransitionOperatorState", newFunctionState);
    // generate a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("2");
    finalSet.add("3");
    // generate a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "2"));
    final Collection<Tuple2<MISTPredicate, String>> list2 = new ArrayList<>();
    list2.add(new Tuple2<>(new RuleBasedEQPredicate("number", 3), "3"));
    stateTable.put("1", list1);
    stateTable.put("2", list2);
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    stateTransitionOperator.setState(loadStateMap);
    // Get the current StateTransitionOperator's state.
    final Map<String, Object> operatorState = stateTransitionOperator.getStateSnapshot();
    final String stateTransitionOperatorState = (String) operatorState.get("stateTransitionOperatorState");
    // Compare the original and the set operator
    Assert.assertEquals(newFunctionState, stateTransitionOperatorState);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // expected result: 2--3
    // generate input data event
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 0L);
    final Map<String, Integer> value3 = new HashMap<>();
    value3.put("number", 3);
    final MistDataEvent data3 = new MistDataEvent(value3, 1L);
    stateTransitionOperator.processLeftData(data2);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(data2, result.get(0));
    Assert.assertEquals("2", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
    stateTransitionOperator.processLeftData(data3);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(data3, result.get(1));
    Assert.assertEquals("3", stateTransitionOperator.getStateSnapshot().get("stateTransitionOperatorState"));
}

Example 19 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class UnionOperatorTest method testUnionOperator.

/**
 * Test union operation.
 * It merges two streams into one. (ordered by timestamp)
 */
@Test
public void testUnionOperator() throws InjectionException {
    // input stream events
    final MistDataEvent a = new MistDataEvent("a", 1L);
    final MistDataEvent b = new MistDataEvent("b", 3L);
    final MistDataEvent c = new MistDataEvent("c", 5L);
    final MistWatermarkEvent lw1 = new MistWatermarkEvent(9L);
    final MistWatermarkEvent lw2 = new MistWatermarkEvent(10L);
    final MistDataEvent d = new MistDataEvent("d", 1L);
    final MistDataEvent e = new MistDataEvent("e", 2L);
    final MistDataEvent f = new MistDataEvent("f", 4L);
    final MistWatermarkEvent rw1 = new MistWatermarkEvent(3L);
    final MistDataEvent g = new MistDataEvent("g", 6L);
    final MistWatermarkEvent rw2 = new MistWatermarkEvent(11L);
    final UnionOperator unionOperator = new UnionOperator();
    final List<MistEvent> result = new LinkedList<>();
    unionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // Test:
    // * Left: ----------W:10--W:9--c----b--------a---->
    // * Right -W:11 --g--------------W:3--f--e--d------>
    // Merged stream: ----g--c--f--b--e--d--a-->
    unionOperator.processLeftData(a);
    Assert.assertEquals(0, result.size());
    unionOperator.processRightData(d);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(a, result.get(0));
    Assert.assertEquals(d, result.get(1));
    unionOperator.processRightData(e);
    unionOperator.processRightData(f);
    Assert.assertEquals(2, result.size());
    unionOperator.processLeftData(b);
    Assert.assertEquals(4, result.size());
    Assert.assertEquals(e, result.get(2));
    Assert.assertEquals(b, result.get(3));
    unionOperator.processRightWatermark(rw1);
    Assert.assertEquals(5, result.size());
    Assert.assertEquals(rw1, result.get(4));
    unionOperator.processLeftData(c);
    Assert.assertEquals(6, result.size());
    Assert.assertEquals(f, result.get(5));
    unionOperator.processLeftWatermark(lw1);
    Assert.assertEquals(6, result.size());
    unionOperator.processLeftWatermark(lw2);
    Assert.assertEquals(6, result.size());
    unionOperator.processRightData(g);
    Assert.assertEquals(8, result.size());
    Assert.assertEquals(c, result.get(6));
    Assert.assertEquals(g, result.get(7));
    unionOperator.processRightWatermark(rw2);
    Assert.assertEquals(9, result.size());
    Assert.assertEquals(lw2, result.get(8));
}

Example 20 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class AggregateWindowOperatorTest method testAggregateWindowOperator.

/**
 * Test AggregateWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(10, 10));
    window.putData(new MistDataEvent(20, 20));
    window.putData(new MistDataEvent(15, 30));
    window.putData(new MistDataEvent(30, 90));
    final MistDataEvent dataEvent = new MistDataEvent(window, 90L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // functions that dealing with input WindowData
    final MISTFunction<WindowData<Integer>, String> aggregateFunc = (windowData) -> {
        return windowData.getDataCollection().toString() + ", " + windowData.getStart() + ", " + windowData.getEnd();
    };
    final AggregateWindowOperator<Integer, String> aggregateWindowOperator = new AggregateWindowOperator<>(aggregateFunc);
    final List<MistEvent> result = new LinkedList<>();
    aggregateWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    aggregateWindowOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals("[10, 20, 15, 30], 0, 99", ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    aggregateWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}