Examples with StreamElement org.apache.flink.streaming.runtime.streamrecord.StreamElement used on opensource projects

Example 16 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class StreamSourceOperatorTest method testAutomaticWatermarkContext.

@Test
public void testAutomaticWatermarkContext() throws Exception {
    // regular stream source operator
    final StoppableStreamSource<String, InfiniteSource<String>> operator = new StoppableStreamSource<>(new InfiniteSource<String>());
    long watermarkInterval = 10;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(0);
    setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, 0, processingTimeService);
    final List<StreamElement> output = new ArrayList<>();
    StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, operator.getContainingTask().getProcessingTimeService(), operator.getContainingTask().getCheckpointLock(), operator.getContainingTask().getStreamStatusMaintainer(), new CollectorOutput<String>(output), operator.getExecutionConfig().getAutoWatermarkInterval(), -1);
    for (long i = 1; i < 100; i += watermarkInterval) {
        processingTimeService.setCurrentTime(i);
    }
    assertTrue(output.size() == 9);
    long nextWatermark = 0;
    for (StreamElement el : output) {
        nextWatermark += watermarkInterval;
        Watermark wm = (Watermark) el;
        assertTrue(wm.getTimestamp() == nextWatermark);
    }
}

Example 17 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class MultiInputSortingDataInputsTest method watermarkPropagation.

@Test
@SuppressWarnings("unchecked")
public void watermarkPropagation() throws Exception {
    CollectingDataOutput<Object> collectingDataOutput = new CollectingDataOutput<>();
    List<StreamElement> elements1 = Arrays.asList(new StreamRecord<>(2, 3), new Watermark(3), new StreamRecord<>(3, 3), new Watermark(7));
    List<StreamElement> elements2 = Arrays.asList(new StreamRecord<>(0, 3), new Watermark(1), new StreamRecord<>(1, 3), new Watermark(3));
    CollectionDataInput<Integer> dataInput1 = new CollectionDataInput<>(elements1, 0);
    CollectionDataInput<Integer> dataInput2 = new CollectionDataInput<>(elements2, 1);
    KeySelector<Integer, Integer> keySelector = value -> value;
    try (MockEnvironment environment = MockEnvironment.builder().build()) {
        SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(new DummyInvokable(), new StreamTaskInput[] { dataInput1, dataInput2 }, new KeySelector[] { keySelector, keySelector }, new TypeSerializer[] { new IntSerializer(), new IntSerializer() }, new IntSerializer(), new StreamTaskInput[0], environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new ExecutionConfig());
        StreamTaskInput<?>[] sortingDataInputs = selectableSortingInputs.getSortedInputs();
        try (StreamTaskInput<Object> input1 = (StreamTaskInput<Object>) sortingDataInputs[0];
            StreamTaskInput<Object> input2 = (StreamTaskInput<Object>) sortingDataInputs[1]) {
            MultipleInputSelectionHandler selectionHandler = new MultipleInputSelectionHandler(selectableSortingInputs.getInputSelectable(), 2);
            StreamMultipleInputProcessor processor = new StreamMultipleInputProcessor(selectionHandler, new StreamOneInputProcessor[] { new StreamOneInputProcessor<>(input1, collectingDataOutput, new DummyOperatorChain()), new StreamOneInputProcessor<>(input2, collectingDataOutput, new DummyOperatorChain()) });
            DataInputStatus inputStatus;
            do {
                inputStatus = processor.processInput();
            } while (inputStatus != DataInputStatus.END_OF_INPUT);
        }
    }
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(0, 3), new StreamRecord<>(1, 3), // watermark from the second input
    new Watermark(3), new StreamRecord<>(2, 3), new StreamRecord<>(3, 3), // watermark from the first input
    new Watermark(7))));
}

Example 18 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class MultiInputSortingDataInputsTest method simpleFixedLengthKeySorting.

@Test
@SuppressWarnings("unchecked")
public void simpleFixedLengthKeySorting() throws Exception {
    CollectingDataOutput<Object> collectingDataOutput = new CollectingDataOutput<>();
    List<StreamElement> elements = Arrays.asList(new StreamRecord<>(1, 3), new StreamRecord<>(1, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 3), new StreamRecord<>(1, 2), new StreamRecord<>(2, 2), Watermark.MAX_WATERMARK);
    CollectionDataInput<Integer> dataInput1 = new CollectionDataInput<>(elements, 0);
    CollectionDataInput<Integer> dataInput2 = new CollectionDataInput<>(elements, 1);
    KeySelector<Integer, Integer> keySelector = value -> value;
    try (MockEnvironment environment = MockEnvironment.builder().build()) {
        SelectableSortingInputs selectableSortingInputs = MultiInputSortingDataInput.wrapInputs(new DummyInvokable(), new StreamTaskInput[] { dataInput1, dataInput2 }, new KeySelector[] { keySelector, keySelector }, new TypeSerializer[] { new IntSerializer(), new IntSerializer() }, new IntSerializer(), new StreamTaskInput[0], environment.getMemoryManager(), environment.getIOManager(), true, 1.0, new Configuration(), new ExecutionConfig());
        StreamTaskInput<?>[] sortingDataInputs = selectableSortingInputs.getSortedInputs();
        try (StreamTaskInput<Object> input1 = (StreamTaskInput<Object>) sortingDataInputs[0];
            StreamTaskInput<Object> input2 = (StreamTaskInput<Object>) sortingDataInputs[1]) {
            MultipleInputSelectionHandler selectionHandler = new MultipleInputSelectionHandler(selectableSortingInputs.getInputSelectable(), 2);
            StreamMultipleInputProcessor processor = new StreamMultipleInputProcessor(selectionHandler, new StreamOneInputProcessor[] { new StreamOneInputProcessor<>(input1, collectingDataOutput, new DummyOperatorChain()), new StreamOneInputProcessor<>(input2, collectingDataOutput, new DummyOperatorChain()) });
            DataInputStatus inputStatus;
            do {
                inputStatus = processor.processInput();
            } while (inputStatus != DataInputStatus.END_OF_INPUT);
        }
    }
    assertThat(collectingDataOutput.events, equalTo(Arrays.asList(new StreamRecord<>(1, 1), new StreamRecord<>(1, 1), new StreamRecord<>(1, 2), new StreamRecord<>(1, 2), new StreamRecord<>(1, 3), new StreamRecord<>(1, 3), new StreamRecord<>(2, 1), new StreamRecord<>(2, 1), new StreamRecord<>(2, 2), new StreamRecord<>(2, 2), new StreamRecord<>(2, 3), // max watermark from one of the inputs
    Watermark.MAX_WATERMARK, new StreamRecord<>(2, 3), // max watermark from the other input
    Watermark.MAX_WATERMARK)));
}

Example 19 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class SourceOperatorStreamTaskTest method testSkipExecutionIfFinishedOnRestore.

@Test
public void testSkipExecutionIfFinishedOnRestore() throws Exception {
    TaskStateSnapshot taskStateSnapshot = TaskStateSnapshot.FINISHED_ON_RESTORE;
    LifeCycleMonitorSource testingSource = new LifeCycleMonitorSource(Boundedness.CONTINUOUS_UNBOUNDED, 10);
    SourceOperatorFactory<Integer> sourceOperatorFactory = new SourceOperatorFactory<>(testingSource, WatermarkStrategy.noWatermarks());
    List<Object> output = new ArrayList<>();
    try (StreamTaskMailboxTestHarness<Integer> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(SourceOperatorStreamTask::new, BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, taskStateSnapshot).addAdditionalOutput(new RecordOrEventCollectingResultPartitionWriter<StreamElement>(output, new StreamElementSerializer<>(IntSerializer.INSTANCE)) {

        @Override
        public void notifyEndOfData(StopMode mode) throws IOException {
            broadcastEvent(new EndOfData(mode), false);
        }
    }).setupOperatorChain(sourceOperatorFactory).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        testHarness.getStreamTask().invoke();
        testHarness.processAll();
        assertThat(output, contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
        LifeCycleMonitorSourceReader sourceReader = (LifeCycleMonitorSourceReader) ((SourceOperator<?, ?>) testHarness.getStreamTask().getMainOperator()).getSourceReader();
        sourceReader.getLifeCycleMonitor().assertCallTimes(0, LifeCyclePhase.values());
    }
}

Example 20 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class StreamSourceOperatorWatermarksTest method testAutomaticWatermarkContext.

@Test
public void testAutomaticWatermarkContext() throws Exception {
    // regular stream source operator
    final StreamSource<String, InfiniteSource<String>> operator = new StreamSource<>(new InfiniteSource<>());
    long watermarkInterval = 10;
    TestProcessingTimeService processingTimeService = new TestProcessingTimeService();
    processingTimeService.setCurrentTime(0);
    MockStreamTask<?, ?> task = setupSourceOperator(operator, TimeCharacteristic.IngestionTime, watermarkInterval, processingTimeService);
    final List<StreamElement> output = new ArrayList<>();
    StreamSourceContexts.getSourceContext(TimeCharacteristic.IngestionTime, processingTimeService, task.getCheckpointLock(), new CollectorOutput<String>(output), operator.getExecutionConfig().getAutoWatermarkInterval(), -1, true);
    for (long i = 1; i < 100; i += watermarkInterval) {
        processingTimeService.setCurrentTime(i);
    }
    assertEquals(9, output.size());
    long nextWatermark = 0;
    for (StreamElement el : output) {
        nextWatermark += watermarkInterval;
        Watermark wm = (Watermark) el;
        assertEquals(wm.getTimestamp(), nextWatermark);
    }
}