Examples with OutputBufferEmitter edu.snu.mist.core.utils.OutputBufferEmitter used on opensource projects

Example 6 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class CepOperatorTest method testCepOperatorNDRContiguity.

/**
 * Test for cep operator with non-deterministic contiguity.
 * Pattern: A(2 or more, ndr contiguity)
 * Input: A1, A2, A3
 * Result: A1-A2, A1-A2-A3, A1-A3, A2-A3
 */
@Test
public void testCepOperatorNDRContiguity() {
    final CepEventPattern<CepExampleClass> event1 = new CepEventPattern.Builder<CepExampleClass>().setName("first").setCondition(conditionA).setClass(exampleClassType).setContiguity(ndrContiguity).setNOrMore(2).setInnerContiguity(ndrContiguity).build();
    final long exampleWindowTime = 1000L;
    final List<CepEventPattern<CepExampleClass>> exampleEventSequence = new ArrayList<>();
    exampleEventSequence.add(event1);
    final CepExampleClass value1 = new CepExampleClass("A", 1);
    final CepExampleClass value2 = new CepExampleClass("A", 2);
    final CepExampleClass value3 = new CepExampleClass("A", 3);
    final MistDataEvent data1 = new MistDataEvent(value1, 1L);
    final MistDataEvent data2 = new MistDataEvent(value2, 2L);
    final MistDataEvent data3 = new MistDataEvent(value3, 3L);
    final CepOperator cepOperator = new CepOperator(exampleEventSequence, exampleWindowTime);
    final List<MistEvent> result = new LinkedList<>();
    cepOperator.setOutputEmitter(new OutputBufferEmitter(result));
    cepOperator.processLeftData(data1);
    cepOperator.processLeftData(data2);
    cepOperator.processLeftData(data3);
    // 4
    Assert.assertEquals(4, result.size());
    // A1-A2
    final CepExampleClass a01 = getCepExampleClass(result, 0, "first", 0);
    final CepExampleClass a02 = getCepExampleClass(result, 0, "first", 1);
    Assert.assertEquals("A", a01.getName());
    Assert.assertEquals(1, a01.getAge());
    Assert.assertEquals("A", a02.getName());
    Assert.assertEquals(2, a02.getAge());
    // A1-A2-A3
    final CepExampleClass a11 = getCepExampleClass(result, 1, "first", 0);
    final CepExampleClass a12 = getCepExampleClass(result, 1, "first", 1);
    final CepExampleClass a13 = getCepExampleClass(result, 1, "first", 2);
    Assert.assertEquals("A", a11.getName());
    Assert.assertEquals(1, a11.getAge());
    Assert.assertEquals("A", a12.getName());
    Assert.assertEquals(2, a12.getAge());
    Assert.assertEquals("A", a13.getName());
    Assert.assertEquals(3, a13.getAge());
    // A1-A3
    final CepExampleClass a22 = getCepExampleClass(result, 2, "first", 0);
    final CepExampleClass a23 = getCepExampleClass(result, 2, "first", 1);
    Assert.assertEquals("A", a22.getName());
    Assert.assertEquals(1, a22.getAge());
    Assert.assertEquals("A", a23.getName());
    Assert.assertEquals(3, a23.getAge());
    // A2-A3
    final CepExampleClass a31 = getCepExampleClass(result, 3, "first", 0);
    final CepExampleClass a33 = getCepExampleClass(result, 3, "first", 1);
    Assert.assertEquals("A", a31.getName());
    Assert.assertEquals(2, a31.getAge());
    Assert.assertEquals("A", a33.getName());
    Assert.assertEquals(3, a33.getAge());
}

Example 7 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testHoppingTimeWindowOperator.

/**
 * Test TimeWindowOperator creating hopping window.
 * It receives some continuous data stream and groups them as a collection.
 */
@Test
public void testHoppingTimeWindowOperator() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 750;
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (1000)Window1------------------(1499):
    // (1750)Window2------------------(2249):
    // d1-----------------------d2---------------d3-w1------------d4-w2-----------------w3------w4:
    // expected results:
    // d1, d2 in Window1
    // d4, w3 in Window2
    timeWindowOperator.processLeftData(d1);
    timeWindowOperator.processLeftData(d2);
    Assert.assertEquals(0, result.size());
    timeWindowOperator.processLeftData(d3);
    timeWindowOperator.processLeftWatermark(w1);
    Assert.assertEquals(1, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d1.getTimestamp(), windowSize, d2.getTimestamp());
    timeWindowOperator.processLeftData(d4);
    timeWindowOperator.processLeftWatermark(w2);
    timeWindowOperator.processLeftWatermark(w3);
    Assert.assertEquals(1, result.size());
    timeWindowOperator.processLeftWatermark(w4);
    Assert.assertEquals(3, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(4);
    OperatorTestUtils.checkWindowData(result.get(1), expectedResult2, d1.getTimestamp() + emissionInterval, windowSize, w3.getTimestamp());
    Assert.assertEquals(w3, result.get(2));
}

Example 8 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testHoppingCountWindowOperator.

/**
 * Test CountWindowOperator creating hopping window.
 * It receives some continuous data stream and groups them as a collection.
 */
@Test
public void testHoppingCountWindowOperator() throws InterruptedException {
    final int windowSize = 3;
    final int emissionInterval = 5;
    final CountWindowOperator<Integer> countWindowOperator = new CountWindowOperator<>(windowSize, emissionInterval);
    final List<MistEvent> result = new LinkedList<>();
    countWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // (1)Window1-(3):
    // (6)Window2--(8):
    // d1--d2-w1--d3--d4--d5--d6-w2-d7-w3-d8:
    // expected results:
    // d1, d2, d3, w1 in Window1
    // d6, d7, d8, w3 in Window2
    countWindowOperator.processLeftData(d1);
    countWindowOperator.processLeftData(d2);
    countWindowOperator.processLeftWatermark(w1);
    Assert.assertEquals(0, result.size());
    countWindowOperator.processLeftData(d3);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(1);
    expectedResult1.add(2);
    expectedResult1.add(3);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, 1L, windowSize, d3.getTimestamp());
    Assert.assertEquals(w1, result.get(1));
    countWindowOperator.processLeftData(d4);
    countWindowOperator.processLeftData(d5);
    countWindowOperator.processLeftData(d6);
    countWindowOperator.processLeftWatermark(w2);
    countWindowOperator.processLeftData(d7);
    countWindowOperator.processLeftWatermark(w3);
    Assert.assertEquals(2, result.size());
    countWindowOperator.processLeftData(d8);
    Assert.assertEquals(4, result.size());
    final Collection<Integer> expectedResult2 = new LinkedList<>();
    expectedResult2.add(6);
    expectedResult2.add(7);
    expectedResult2.add(8);
    OperatorTestUtils.checkWindowData(result.get(2), expectedResult2, emissionInterval + 1L, windowSize, d8.getTimestamp());
    Assert.assertEquals(w3, result.get(3));
}

Example 9 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class FixedSizeWindowOperatorTest method testTimeWindowOperatorSetState.

/**
 * Test setting state of the TimeWindowOperator.
 */
@Test
public void testTimeWindowOperatorSetState() throws InterruptedException {
    final int windowSize = 500;
    final int emissionInterval = 250;
    // Generate a new state and set it to a new TimeWindowOperator.
    final Window expectedWindow1 = new WindowImpl<>(d4.getTimestamp(), emissionInterval, new LinkedList<Integer>());
    expectedWindow1.putData(d4);
    expectedWindow1.putWatermark(w2);
    final Window expectedWindow2 = new WindowImpl<>(d4.getTimestamp(), windowSize, new LinkedList<Integer>());
    expectedWindow2.putData(d4);
    expectedWindow2.putWatermark(w2);
    final Queue<Window<Integer>> expectedWindowQueue = new LinkedList<>();
    expectedWindowQueue.add(expectedWindow1);
    expectedWindowQueue.add(expectedWindow2);
    final long expectedWindowCreationPoint = d4.getTimestamp() + emissionInterval;
    final Map<String, Object> loadStateMap = new HashMap<>();
    loadStateMap.put("windowQueue", expectedWindowQueue);
    loadStateMap.put("windowCreationPoint", expectedWindowCreationPoint);
    final TimeWindowOperator<Integer> timeWindowOperator = new TimeWindowOperator<>(windowSize, emissionInterval);
    timeWindowOperator.setState(loadStateMap);
    // Get the current TimeWindowOperator's state.
    final Map<String, Object> operatorState = timeWindowOperator.getStateSnapshot();
    final Queue<Window<Integer>> windowQueue = (Queue<Window<Integer>>) operatorState.get("windowQueue");
    final long windowCreationPoint = (long) operatorState.get("windowCreationPoint");
    // Compare the original and the set operator.
    Assert.assertEquals(expectedWindowQueue, windowQueue);
    Assert.assertEquals(expectedWindowCreationPoint, windowCreationPoint);
    // Test if the operator can properly process data.
    final List<MistEvent> result = new LinkedList<>();
    timeWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    timeWindowOperator.processLeftData(d10);
    Assert.assertEquals(2, result.size());
    final Collection<Integer> expectedResult1 = new LinkedList<>();
    expectedResult1.add(4);
    OperatorTestUtils.checkWindowData(result.get(0), expectedResult1, d4.getTimestamp(), emissionInterval, w2.getTimestamp());
    Assert.assertEquals(result.get(1), w2);
}

Example 10 with OutputBufferEmitter

use of edu.snu.mist.core.utils.OutputBufferEmitter in project mist by snuspl.

the class ReduceByKeyOperatorTest method testReduceByKeyOperatorGetState.

/**
 * Test getting state of the ReduceByKeyOperator.
 */
@Test
@SuppressWarnings("unchecked")
public void testReduceByKeyOperatorGetState() throws InterruptedException {
    // Generate the current ReduceByKeyOperator state.
    final List<MistDataEvent> inputStream = ImmutableList.of(createTupleEvent("a", 1, 1L), createTupleEvent("b", 1, 2L), createTupleEvent("c", 1, 3L), createTupleEvent("a", 1, 4L), createTupleEvent("d", 1, 5L), createTupleEvent("a", 1, 6L), createTupleEvent("b", 1, 7L));
    // Generate the expected ReduceByKeyOperator state.
    final Map<String, Integer> expectedOperatorState = new HashMap<>();
    expectedOperatorState.put("a", 3);
    expectedOperatorState.put("b", 2);
    expectedOperatorState.put("c", 1);
    expectedOperatorState.put("d", 1);
    final int keyIndex = 0;
    final MISTBiFunction<Integer, Integer, Integer> wordCountFunc = (oldVal, val) -> oldVal + val;
    final ReduceByKeyOperator<String, Integer> wcOperator = new ReduceByKeyOperator<>(keyIndex, wordCountFunc);
    final List<MistEvent> result = new LinkedList<>();
    wcOperator.setOutputEmitter(new OutputBufferEmitter(result));
    inputStream.stream().forEach(wcOperator::processLeftData);
    // Get the current ReduceByKeyOperator's state.
    final Map<String, Object> operatorStateMap = wcOperator.getStateSnapshot();
    final Map<String, Integer> operatorState = (Map<String, Integer>) operatorStateMap.get("reduceByKeyState");
    // Compare the expected and original operator's state.
    Assert.assertEquals(expectedOperatorState, operatorState);
}