Examples with MistDataEvent edu.snu.mist.core.MistDataEvent used on opensource projects

Example 16 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class StatelessOperatorTest method testFlatMapOperation.

/**
 * Test flatMap operation.
 * It splits the string by space.
 */
@Test
public void testFlatMapOperation() throws InjectionException {
    // input stream
    final List<MistDataEvent> inputStream = ImmutableList.of(new MistDataEvent("a b c", 1L), new MistDataEvent("b c d", 2L), new MistDataEvent("d e f", 3L));
    // expected output
    final List<MistEvent> expectedStream = ImmutableList.of(new MistDataEvent("a", 1L), new MistDataEvent("b", 1L), new MistDataEvent("c", 1L), new MistDataEvent("b", 2L), new MistDataEvent("c", 2L), new MistDataEvent("d", 2L), new MistDataEvent("d", 3L), new MistDataEvent("e", 3L), new MistDataEvent("f", 3L));
    // map function: splits the string by space.
    final MISTFunction<String, List<String>> flatMapFunc = (mapInput) -> Arrays.asList(mapInput.split(" "));
    final FlatMapOperator<String, String> flatMapOperator = new FlatMapOperator<>(flatMapFunc);
    testStatelessOperator(inputStream, expectedStream, flatMapOperator);
}

Example 17 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class UnionOperator method drainUntilTimestamp.

/**
 * Emits events which have less timestamp than the required timestamp from the queue .
 * @param queue queue
 * @param timestamp timestamp
 */
private void drainUntilTimestamp(final Queue<MistEvent> queue, final long timestamp) {
    // The events in the queue is ordered by timestamp, so just peeks one by one
    while (!queue.isEmpty() && peekTimestamp(queue) <= timestamp) {
        final MistDataEvent event = (MistDataEvent) queue.poll();
        outputEmitter.emitData(event);
    }
}

Example 18 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class UnionOperator method drainUntilMinimumWatermark.

/**
 * Emits events which have less timestamp than the minimum watermark
 * calculated from the leftUpstreamQueue, currentLeftWatermark, rightUpstreamQueue, and currentRightWatermark.
 * This method is not thread-safe now. Therefore, only one event processor have to process union operation at once.
 */
private void drainUntilMinimumWatermark() {
    final long leftWatermarkTimestamp = latestLeftWatermark.getTimestamp();
    final long rightWatermarkTimestamp = latestRightWatermark.getTimestamp();
    final long timestamp = getMinimumWatermark(leftWatermarkTimestamp, rightWatermarkTimestamp);
    if (LOG.isLoggable(Level.FINE)) {
        LOG.log(Level.FINE, "{0} drains inputs until timestamp {1}", new Object[] { this.getClass().getName(), timestamp });
    }
    // The events in the queue is ordered by timestamp, so just peeks one by one
    while (!leftUpstreamQueue.isEmpty() && !rightUpstreamQueue.isEmpty()) {
        // The left and right checkpoint events do not have to be ordered because they are the same.
        if (leftUpstreamQueue.peek().isCheckpoint()) {
            final MistCheckpointEvent event = (MistCheckpointEvent) leftUpstreamQueue.poll();
            outputEmitter.emitCheckpoint(event);
        } else if (rightUpstreamQueue.peek().isCheckpoint()) {
            final MistCheckpointEvent event = (MistCheckpointEvent) rightUpstreamQueue.poll();
            outputEmitter.emitCheckpoint(event);
        } else {
            // Pick minimum timestamp from left and right queue.
            long leftTs = peekTimestamp(leftUpstreamQueue);
            long rightTs = peekTimestamp(rightUpstreamQueue);
            // End of the drain
            if (leftTs > timestamp && rightTs > timestamp) {
                return;
            }
            if (leftTs <= rightTs) {
                final MistDataEvent event = (MistDataEvent) leftUpstreamQueue.poll();
                outputEmitter.emitData(event);
            } else {
                final MistDataEvent event = (MistDataEvent) rightUpstreamQueue.poll();
                outputEmitter.emitData(event);
            }
        }
    }
    // If one of the upstreamQueues is not empty, then drain events from the queue
    if (!(leftUpstreamQueue.isEmpty() && rightUpstreamQueue.isEmpty())) {
        if (leftUpstreamQueue.isEmpty()) {
            // Drain from right upstream queue.
            drainUntilTimestamp(rightUpstreamQueue, timestamp);
        } else {
            // Drain from left upstream queue.
            drainUntilTimestamp(leftUpstreamQueue, timestamp);
        }
    }
    emitWatermarkUntilTimestamp(leftWatermarkTimestamp, rightWatermarkTimestamp, timestamp);
}

Example 19 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class PunctuatedEventGenerator method emitData.

@Override
public void emitData(final I input) {
    if (isWatermark.test(input)) {
        latestWatermarkTimestamp = parseTimestamp.apply(input);
        outputEmitter.emitWatermark(new MistWatermarkEvent(latestWatermarkTimestamp));
    } else {
        MistDataEvent newInputEvent = generateEvent(input);
        if (newInputEvent != null) {
            outputEmitter.emitData(newInputEvent);
        }
    }
}

Example 20 with MistDataEvent

use of edu.snu.mist.core.MistDataEvent in project mist by snuspl.

the class CepOperator method emit.

/**
 * Emit the event stack which is in final state.
 * @param input      current mist data event
 * @param eventStack event stack
 */
private void emit(final MistDataEvent input, final EventStack<T> eventStack) {
    final Map<String, List<T>> output = new HashMap<>();
    final long timeStamp = input.getTimestamp();
    // Check whether current event stack satisfies the loop condition.
    final int finalStateIndex = eventStack.getStack().peek().getIndex();
    final EventStackEntry<T> finalEntry = eventStack.getStack().peek();
    final int times = finalEntry.getlist().size();
    final CepEventPattern<T> finalState = eventPatternList.get(finalStateIndex);
    if (!finalState.isRepeated() || (times >= finalState.getMinRepetition() && (finalState.getMaxRepetition() == -1 || times <= finalState.getMaxRepetition()))) {
        // Make an output data.
        for (final EventStackEntry<T> iterEntry : eventStack.getStack()) {
            output.put(eventPatternList.get(iterEntry.getIndex()).getEventPatternName(), iterEntry.getlist());
        }
        if (LOG.isLoggable(Level.FINE)) {
            LOG.log(Level.FINE, "{0} processes {1} to {2}", new Object[] { this.getClass().getName(), input, output });
        }
        outputEmitter.emitData(new MistDataEvent(output, timeStamp));
        eventStack.setAlreadyEmitted();
    }
}