Examples with OutputBufferEmitter edu.snu.mist.core.utils.OutputBufferEmitter used on opensource projects

Example 11 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class ReduceByKeyOperatorTest method testReduceByKeyOperatorSetState.

/**
 * Test setting state of the ReduceByKeyOperator.
 */
@Test
@SuppressWarnings("unchecked")
public void testReduceByKeyOperatorSetState() throws InterruptedException {
    // Generate a new state and set it to the state of a new ReduceByKeyWindowOperator.
    final Map<String, Integer> expectedOperatorState = new HashMap<>();
    expectedOperatorState.put("a", 3);
    expectedOperatorState.put("b", 2);
    expectedOperatorState.put("c", 1);
    expectedOperatorState.put("d", 1);
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("reduceByKeyState", expectedOperatorState);
    final int keyIndex = 0;
    final MISTBiFunction<Integer, Integer, Integer> wordCountFunc = (oldVal, val) -> oldVal + val;
    final ReduceByKeyOperator reduceByKeyOperator = new ReduceByKeyOperator(keyIndex, wordCountFunc);
    reduceByKeyOperator.setState(loadStateMap);
    // Compare the original and the set operator.
    final Map<String, Object> operatorStateMap = reduceByKeyOperator.getStateSnapshot();
    final Map<String, Integer> operatorState = (Map<String, Integer>) operatorStateMap.get("reduceByKeyState");
    Assert.assertEquals(expectedOperatorState, operatorState);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    reduceByKeyOperator.setOutputEmitter(new OutputBufferEmitter(result));
    reduceByKeyOperator.setState(operatorStateMap);
    final List<MistDataEvent> inputStream = ImmutableList.of(createTupleEvent("a", 1, 10L));
    inputStream.stream().forEach(reduceByKeyOperator::processLeftData);
    final List<MistEvent> expected = new LinkedList<>();
    final Map<String, Integer> o1 = new HashMap<>();
    o1.put("a", 4);
    o1.put("b", 2);
    o1.put("c", 1);
    o1.put("d", 1);
    expected.add(new MistDataEvent(o1, 10L));
    Assert.assertEquals(expected, result);
}

Example 12 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class ReduceByKeyOperatorTest method testReduceByKeyOperator.

/**
 * Test whether reduceByKeyOperator generates correct outputs.
 * Input: a list of tuples: ("a", 1), ("b", 1), ("c", 1), ("a", 1), ("d", 1), ("a", 1), ("b", 1)
 * Expected outputs: word counts:
 *  {"a": 1},
 *  {"a": 1, "b": 1},
 *  {"a": 1, "b": 1, "c": 1}
 *  {"a": 2, "b": 1, "c": 1}
 *  {"a": 2, "b": 1, "c": 1, "d": 1}
 *  {"a": 3, "b": 1, "c": 1, "d": 1}
 *  {"a": 3, "b": 2, "c": 1, "d": 1}
 */
@Test
public void testReduceByKeyOperator() throws InjectionException {
    // input stream
    final List<MistDataEvent> inputStream = ImmutableList.of(createTupleEvent("a", 1, 1L), createTupleEvent("b", 1, 2L), createTupleEvent("c", 1, 3L), createTupleEvent("a", 1, 4L), createTupleEvent("d", 1, 5L), createTupleEvent("a", 1, 6L), createTupleEvent("b", 1, 7L));
    // expected output
    final Map<String, Integer> o0 = new HashMap<>();
    o0.put("a", 1);
    final Map<String, Integer> o1 = new HashMap<>(o0);
    o1.put("b", 1);
    final Map<String, Integer> o2 = new HashMap<>(o1);
    o2.put("c", 1);
    final Map<String, Integer> o3 = new HashMap<>(o2);
    o3.put("a", 2);
    final Map<String, Integer> o4 = new HashMap<>(o3);
    o4.put("d", 1);
    final Map<String, Integer> o5 = new HashMap<>(o4);
    o5.put("a", 3);
    final Map<String, Integer> o6 = new HashMap<>(o5);
    o6.put("b", 2);
    final List<MistEvent> expectedStream = ImmutableList.of(new MistDataEvent(o0, 1L), new MistDataEvent(o1, 2L), new MistDataEvent(o2, 3L), new MistDataEvent(o3, 4L), new MistDataEvent(o4, 5L), new MistDataEvent(o5, 6L), new MistDataEvent(o6, 7L));
    // Set the key index of tuple
    final int keyIndex = 0;
    // Reduce function for word count
    final MISTBiFunction<Integer, Integer, Integer> wordCountFunc = (oldVal, val) -> oldVal + val;
    final ReduceByKeyOperator<String, Integer> wcOperator = new ReduceByKeyOperator<>(keyIndex, wordCountFunc);
    // output test
    final List<MistEvent> result = new LinkedList<>();
    wcOperator.setOutputEmitter(new OutputBufferEmitter(result));
    inputStream.stream().forEach(wcOperator::processLeftData);
    LOG.info("expected: " + expectedStream);
    LOG.info("result: " + result);
    Assert.assertEquals(expectedStream, result);
}

Example 13 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperatorGetState.

/**
 * Test getting state of StateTransitionOperator.
 */
@Test
public void testStateTransitionOperatorGetState() throws InterruptedException {
    // generate input data event
    final Map<String, Integer> value1 = new HashMap<>();
    value1.put("number", 1);
    final MistDataEvent data1 = new MistDataEvent(value1, 0L);
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 1L);
    // generate a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("1");
    finalSet.add("2");
    // generate a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list0 = new ArrayList<>();
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 1), "1"));
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "2"));
    stateTable.put("0", list0);
    stateTable.put("1", list1);
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    final List<MistEvent> result = new ArrayList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    stateTransitionOperator.processLeftData(data1);
    stateTransitionOperator.processLeftData(data2);
    // Generate the expected state
    final String expectedOperatorState = "2";
    // Get the
    final Map<String, Object> operatorState = stateTransitionOperator.getStateSnapshot();
    final String stateTransitionOperatorState = (String) operatorState.get("stateTransitionOperatorState");
    Assert.assertEquals(expectedOperatorState, stateTransitionOperatorState);
}

Example 14 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class ApplyStatefulWindowOperatorTest method testApplyStatefulWindowOperator.

/**
 * Test ApplyStatefulWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testApplyStatefulWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window1 = new WindowImpl<>(0L, 100L);
    final WindowImpl<Integer> window2 = new WindowImpl<>(100L, 100L);
    window1.putData(new MistDataEvent(10, 10));
    window1.putData(new MistDataEvent(20, 20));
    window1.putData(new MistDataEvent(30, 30));
    window1.putData(new MistDataEvent(15, 90));
    window2.putData(new MistDataEvent(10, 110));
    window2.putData(new MistDataEvent(20, 120));
    final MistDataEvent dataEvent1 = new MistDataEvent(window1, 90L);
    final MistDataEvent dataEvent2 = new MistDataEvent(window2, 190L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(201L);
    // the state finding maximum integer value among received inputs
    final ApplyStatefulFunction<Integer, Integer> applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulWindowOperator<Integer, Integer> applyStatefulWindowOperator = new ApplyStatefulWindowOperator<>(applyStatefulFunction);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // check that the operator processes a window input properly
    applyStatefulWindowOperator.processLeftData(dataEvent1);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals(30, ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    // check that the operator processes another window separately
    applyStatefulWindowOperator.processLeftData(dataEvent2);
    Assert.assertEquals(2, result.size());
    Assert.assertTrue(result.get(1).isData());
    Assert.assertEquals(20, ((MistDataEvent) result.get(1)).getValue());
    Assert.assertEquals(190L, result.get(1).getTimestamp());
    // check that the operator processes watermark event well
    applyStatefulWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(watermarkEvent, result.get(2));
}

Example 15 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class JoinOperatorTest method testAggregateWindowOperator.

/**
 * Tests JoinOperator.
 * It joins a pair of inputs in two streams that has same key.
 */
@Test
public void testAggregateWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window = new WindowImpl<>(0L, 100L);
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("Hello", 1), null), 10));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("MIST", 2), null), 20));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 3000L)), 30));
    window.putData(new MistDataEvent(new Tuple2<>(new Tuple2<>("SNUCMS", 3), null), 40));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(1, 4000L)), 50));
    window.putData(new MistDataEvent(new Tuple2<>(null, new Tuple2<>(2, 5000L)), 60));
    final MistDataEvent dataEvent = new MistDataEvent(window, 60L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(101L);
    // predicate that tests whether two input data have same key or not
    final MISTBiPredicate<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinPredicate = (tuple1, tuple2) -> tuple1.get(1).equals(tuple2.get(0));
    final JoinOperator<Tuple2<String, Integer>, Tuple2<Integer, Long>> joinOperator = new JoinOperator<>(joinPredicate);
    // expected pairs
    // {Hello, 1} and {1, 3000L}
    // {Hello, 1} and {1, 4000L}
    // {MIST, 2} and {2, 5000L}
    final List<MistEvent> result = new LinkedList<>();
    joinOperator.setOutputEmitter(new OutputBufferEmitter(result));
    joinOperator.processLeftData(dataEvent);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertTrue(((MistDataEvent) result.get(0)).getValue() instanceof WindowData);
    final WindowData windowData = (WindowData) ((MistDataEvent) result.get(0)).getValue();
    Assert.assertEquals(0L, windowData.getStart());
    Assert.assertEquals(99L, windowData.getEnd());
    final Collection<Tuple2<Tuple2<String, Integer>, Tuple2<Integer, Long>>> dataCollection = windowData.getDataCollection();
    final Iterator iterator = dataCollection.iterator();
    Assert.assertEquals(3, dataCollection.size());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 3000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("Hello", 1), new Tuple2<>(1, 4000L)), iterator.next());
    Assert.assertEquals(new Tuple2<>(new Tuple2<>("MIST", 2), new Tuple2<>(2, 5000L)), iterator.next());
    Assert.assertEquals(60L, result.get(0).getTimestamp());
    joinOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(watermarkEvent, result.get(1));
}