Examples with MistDataEvent edu.snu.mist.core.MistDataEvent used on opensource projects

Example 1 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class FixedSizeWindowOperator method emitElapsedWindow.

/**
 * Checks whether the window emission count is elapsed, and emits some windows if so.
 * @param currentEventPoint the point of received event
 */
protected void emitElapsedWindow(final long currentEventPoint) {
    // Checks the window emission time is elapsed
    while (!windowQueue.isEmpty() && ((Window) windowQueue.peek()).getEnd() < currentEventPoint) {
        final Window<T> window = windowQueue.poll();
        outputEmitter.emitData(new MistDataEvent(window, window.getLatestTimestamp()));
        final MistWatermarkEvent latestWatermark = window.getLatestWatermark();
        if (latestWatermark.getTimestamp() != 0L) {
            outputEmitter.emitWatermark(latestWatermark);
        }
    }
}

Example 2 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class OperatorOutputEmitter method emitData.

@Override
public void emitData(final MistDataEvent output, final int index) {
    // Optimization: do not create new MistEvent and reuse it if it has one downstream operator chain.
    if (nextOperators.size() == 1) {
        for (final Map.Entry<ExecutionVertex, MISTEdge> nextChain : nextOperators.entrySet()) {
            final MISTEdge edge = nextChain.getValue();
            final int edgeIndex = edge.getIndex();
            if (edgeIndex == index) {
                // send the data only if the index of this edge is equal to the target index
                sendData(output, edge.getDirection(), nextChain.getKey());
            }
        }
    } else {
        for (final Map.Entry<ExecutionVertex, MISTEdge> nextChain : nextOperators.entrySet()) {
            final MISTEdge edge = nextChain.getValue();
            final int edgeIndex = edge.getIndex();
            if (edgeIndex == index) {
                // send the data only if the index of this edge is equal to the target index
                final MistDataEvent event = new MistDataEvent(output.getValue(), output.getTimestamp());
                sendData(event, edge.getDirection(), nextChain.getKey());
            }
        }
    }
}

Example 3 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class OperatorOutputEmitter method emitData.

/**
 * This method emits the outputs to next OperatorChains.
 * If the Executor of the current OperatorChain is same as that of next OperatorChain,
 * the OutputEmitter directly forwards outputs of the current OperatorChain
 * as inputs of the next OperatorChain.
 * Otherwise, the OutputEmitter submits a job to the Executor of the next OperatorChain.
 * @param output an output
 */
@Override
public void emitData(final MistDataEvent output) {
    // Optimization: do not create new MistEvent and reuse it if it has one downstream operator chain.
    if (nextOperators.size() == 1) {
        for (final Map.Entry<ExecutionVertex, MISTEdge> nextChain : nextOperators.entrySet()) {
            final Direction direction = nextChain.getValue().getDirection();
            sendData(output, direction, nextChain.getKey());
        }
    } else {
        for (final Map.Entry<ExecutionVertex, MISTEdge> nextChain : nextOperators.entrySet()) {
            final MistDataEvent event = new MistDataEvent(output.getValue(), output.getTimestamp());
            final Direction direction = nextChain.getValue().getDirection();
            sendData(event, direction, nextChain.getKey());
        }
    }
}

Example 4 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class ApplyStatefulOperatorTest method testApplyStatefulOperator.

/**
 * Test ApplyStatefulOperator.
 * It calculates the maximum value through some inputs.
 */
@Test
public void testApplyStatefulOperator() {
    // input events
    // expected results: 10--20--20--30--watermark--
    final MistDataEvent data10 = new MistDataEvent(10, 0L);
    final MistDataEvent data20 = new MistDataEvent(20, 1L);
    final MistDataEvent data15 = new MistDataEvent(15, 2L);
    final MistDataEvent data30 = new MistDataEvent(30, 3L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(4L);
    // the state managing function finding maximum integer value among received inputs
    final ApplyStatefulFunction applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulOperator<Integer, Integer> applyStatefulOperator = new ApplyStatefulOperator<>(applyStatefulFunction);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulOperator.setOutputEmitter(new OutputBufferEmitter(result));
    applyStatefulOperator.processLeftData(data10);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(data10, result.get(0));
    applyStatefulOperator.processLeftData(data20);
    Assert.assertEquals(2, result.size());
    Assert.assertEquals(data20, result.get(1));
    applyStatefulOperator.processLeftData(data15);
    Assert.assertEquals(3, result.size());
    Assert.assertTrue(result.get(2) instanceof MistDataEvent);
    Assert.assertEquals(20, ((MistDataEvent) result.get(2)).getValue());
    Assert.assertEquals(2L, result.get(2).getTimestamp());
    applyStatefulOperator.processLeftData(data30);
    Assert.assertEquals(4, result.size());
    Assert.assertEquals(data30, result.get(3));
    applyStatefulOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(5, result.size());
    Assert.assertEquals(watermarkEvent, result.get(4));
}

Example 5 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class ApplyStatefulWindowOperatorTest method testApplyStatefulWindowOperator.

/**
 * Test ApplyStatefulWindowOperator.
 * It calculates the maximum value from the collection.
 */
@Test
public void testApplyStatefulWindowOperator() {
    // input stream events
    final WindowImpl<Integer> window1 = new WindowImpl<>(0L, 100L);
    final WindowImpl<Integer> window2 = new WindowImpl<>(100L, 100L);
    window1.putData(new MistDataEvent(10, 10));
    window1.putData(new MistDataEvent(20, 20));
    window1.putData(new MistDataEvent(30, 30));
    window1.putData(new MistDataEvent(15, 90));
    window2.putData(new MistDataEvent(10, 110));
    window2.putData(new MistDataEvent(20, 120));
    final MistDataEvent dataEvent1 = new MistDataEvent(window1, 90L);
    final MistDataEvent dataEvent2 = new MistDataEvent(window2, 190L);
    final MistWatermarkEvent watermarkEvent = new MistWatermarkEvent(201L);
    // the state finding maximum integer value among received inputs
    final ApplyStatefulFunction<Integer, Integer> applyStatefulFunction = new FindMaxIntFunction();
    final ApplyStatefulWindowOperator<Integer, Integer> applyStatefulWindowOperator = new ApplyStatefulWindowOperator<>(applyStatefulFunction);
    final List<MistEvent> result = new LinkedList<>();
    applyStatefulWindowOperator.setOutputEmitter(new OutputBufferEmitter(result));
    // check that the operator processes a window input properly
    applyStatefulWindowOperator.processLeftData(dataEvent1);
    Assert.assertEquals(1, result.size());
    Assert.assertTrue(result.get(0).isData());
    Assert.assertEquals(30, ((MistDataEvent) result.get(0)).getValue());
    Assert.assertEquals(90L, result.get(0).getTimestamp());
    // check that the operator processes another window separately
    applyStatefulWindowOperator.processLeftData(dataEvent2);
    Assert.assertEquals(2, result.size());
    Assert.assertTrue(result.get(1).isData());
    Assert.assertEquals(20, ((MistDataEvent) result.get(1)).getValue());
    Assert.assertEquals(190L, result.get(1).getTimestamp());
    // check that the operator processes watermark event well
    applyStatefulWindowOperator.processLeftWatermark(watermarkEvent);
    Assert.assertEquals(3, result.size());
    Assert.assertEquals(watermarkEvent, result.get(2));
}