Examples with StreamElement org.apache.flink.streaming.runtime.streamrecord.StreamElement used on opensource projects

Example 6 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class StreamSourceContextIdleDetectionTests method testManualWatermarkContext.

/**
	 * Test scenario (idleTimeout = 100):
	 * (1) Start from 0 as initial time.
	 * (2) As soon as time reaches 100, status should have been toggled to IDLE.
	 * (3) After some arbitrary time (until 300), the status should remain IDLE.
	 * (4) Emit a record at 310. Status should become ACTIVE. This should fire a idleness detection at 410.
	 * (5) Emit another record at 320 (which is before the next check). This should make the idleness check pass.
	 * (6) Advance time to 410 and trigger idleness detection.
	 *     The status should still be ACTIVE due to step (5). Another idleness detection should be fired at 510.
	 * (7) Advance time to 510 and trigger idleness detection. Since no records were collected in-between the two
	 *     idleness detections, status should have been toggle back to IDLE.
	 *
	 * Inline comments will refer to the corresponding tested steps in the scenario.
	 */
@Test
public void testManualWatermarkContext() throws Exception {
    long idleTimeout = 100;
    long initialTime = 0;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(initialTime);
    final List<StreamElement> output = new ArrayList<>();
    MockStreamStatusMaintainer mockStreamStatusMaintainer = new MockStreamStatusMaintainer();
    SourceFunction.SourceContext<String> context = StreamSourceContexts.getSourceContext(TimeCharacteristic.EventTime, processingTimeService, new Object(), mockStreamStatusMaintainer, new CollectorOutput<String>(output), 0, idleTimeout);
    // -------------------------- begin test scenario --------------------------
    // corresponds to step (2) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + idleTimeout);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    // corresponds to step (3) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 2 * idleTimeout);
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
    // corresponds to step (4) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            break;
        case EMIT_WATERMARK:
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            break;
    }
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
    // corresponds to step (5) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 3 * idleTimeout + 2 * idleTimeout / 10);
    switch(testMethod) {
        case COLLECT:
            context.collect("msg");
            break;
        case COLLECT_WITH_TIMESTAMP:
            context.collectWithTimestamp("msg", processingTimeService.getCurrentProcessingTime());
            break;
        case EMIT_WATERMARK:
            context.emitWatermark(new Watermark(processingTimeService.getCurrentProcessingTime()));
            break;
    }
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
    // corresponds to step (6) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 4 * idleTimeout + idleTimeout / 10);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isActive());
    // corresponds to step (7) of scenario (please see method-level Javadoc comment)
    processingTimeService.setCurrentTime(initialTime + 5 * idleTimeout + idleTimeout / 10);
    assertTrue(mockStreamStatusMaintainer.getStreamStatus().isIdle());
}

Example 7 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class StreamSourceOperatorTest method testLatencyMarkEmission.

/**
	 * Test that latency marks are emitted
	 */
@Test
public void testLatencyMarkEmission() throws Exception {
    final List<StreamElement> output = new ArrayList<>();
    final long maxProcessingTime = 100L;
    final long latencyMarkInterval = 10L;
    final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
    testProcessingTimeService.setCurrentTime(0L);
    final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
    // regular stream source operator
    final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));
    // emit latency marks every 10 milliseconds.
    setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, latencyMarkInterval, testProcessingTimeService);
    // run and wait to be stopped
    operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output));
    int numberLatencyMarkers = (int) (maxProcessingTime / latencyMarkInterval) + 1;
    assertEquals(// + 1 is the final watermark element
    numberLatencyMarkers + 1, output.size());
    long timestamp = 0L;
    int i = 0;
    // and that its only latency markers + a final watermark
    for (; i < output.size() - 1; i++) {
        StreamElement se = output.get(i);
        Assert.assertTrue(se.isLatencyMarker());
        Assert.assertEquals(-1, se.asLatencyMarker().getVertexID());
        Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());
        Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp);
        timestamp += latencyMarkInterval;
    }
    Assert.assertTrue(output.get(i).isWatermark());
}

Example 8 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class AbstractKeyedCEPPatternOperator method restoreState.

//////////////////////			Backwards Compatibility			//////////////////////
@Override
public void restoreState(FSDataInputStream in) throws Exception {
    // this is the flag indicating if we have udf
    // state to restore (not needed here)
    in.read();
    DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(in);
    InternalWatermarkCallbackService<KEY> watermarkCallbackService = getInternalWatermarkCallbackService();
    if (migratingFromOldKeyedOperator) {
        int numberEntries = inputView.readInt();
        for (int i = 0; i < numberEntries; i++) {
            watermarkCallbackService.registerKeyForWatermarkCallback(keySerializer.deserialize(inputView));
        }
    } else {
        final ObjectInputStream ois = new ObjectInputStream(in);
        // retrieve the NFA
        @SuppressWarnings("unchecked") NFA<IN> nfa = (NFA<IN>) ois.readObject();
        // retrieve the elements that were pending in the priority queue
        MultiplexingStreamRecordSerializer<IN> recordSerializer = new MultiplexingStreamRecordSerializer<>(inputSerializer);
        PriorityQueue<StreamRecord<IN>> priorityQueue = priorityQueueFactory.createPriorityQueue();
        int entries = ois.readInt();
        for (int i = 0; i < entries; i++) {
            StreamElement streamElement = recordSerializer.deserialize(inputView);
            priorityQueue.offer(streamElement.<IN>asRecord());
        }
        // finally register the retrieved state with the new keyed state.
        setCurrentKey((byte) 0);
        nfaOperatorState.update(nfa);
        priorityQueueOperatorState.update(priorityQueue);
        if (!isProcessingTime) {
            // this is relevant only for event/ingestion time
            // need to work around type restrictions
            InternalWatermarkCallbackService rawWatermarkCallbackService = (InternalWatermarkCallbackService) watermarkCallbackService;
            rawWatermarkCallbackService.registerKeyForWatermarkCallback((byte) 0);
        }
        ois.close();
    }
}

Example 9 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class StreamSourceOperatorTest method testAutomaticWatermarkContext.

@Test
public void testAutomaticWatermarkContext() throws Exception {
    // regular stream source operator
    final StoppableStreamSource<String, InfiniteSource<String>> operator = new StoppableStreamSource<>(new InfiniteSource<String>());
    long watermarkInterval = 10;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(0);
    setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, 0, processingTimeService);
    final List<StreamElement> output = new ArrayList<>();
    StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, operator.getContainingTask().getProcessingTimeService(), operator.getContainingTask().getCheckpointLock(), operator.getContainingTask().getStreamStatusMaintainer(), new CollectorOutput<String>(output), operator.getExecutionConfig().getAutoWatermarkInterval(), -1);
    for (long i = 1; i < 100; i += watermarkInterval) {
        processingTimeService.setCurrentTime(i);
    }
    assertTrue(output.size() == 9);
    long nextWatermark = 0;
    for (StreamElement el : output) {
        nextWatermark += watermarkInterval;
        Watermark wm = (Watermark) el;
        assertTrue(wm.getTimestamp() == nextWatermark);
    }
}

Example 10 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class AsyncWaitOperator method snapshotState.

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
    super.snapshotState(context);
    ListState<StreamElement> partitionableState = getOperatorStateBackend().getOperatorState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
    partitionableState.clear();
    Collection<StreamElementQueueEntry<?>> values = queue.values();
    try {
        for (StreamElementQueueEntry<?> value : values) {
            partitionableState.add(value.getStreamElement());
        }
        // add the pending stream element queue entry if the stream element queue is currently full
        if (pendingStreamElementQueueEntry != null) {
            partitionableState.add(pendingStreamElementQueueEntry.getStreamElement());
        }
    } catch (Exception e) {
        partitionableState.clear();
        throw new Exception("Could not add stream element queue entries to operator state " + "backend of operator " + getOperatorName() + '.', e);
    }
}