Examples with EndOfData org.apache.flink.runtime.io.network.api.EndOfData used on opensource projects

Example 1 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class SortMergeResultPartition method notifyEndOfData.

@Override
public void notifyEndOfData(StopMode mode) throws IOException {
    if (!hasNotifiedEndOfUserRecords) {
        broadcastEvent(new EndOfData(mode), false);
        hasNotifiedEndOfUserRecords = true;
    }
}

Example 2 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class ResultPartitionTest method testWaitForAllRecordProcessed.

@Test
public void testWaitForAllRecordProcessed() throws IOException {
    // Creates a result partition with 2 channels.
    BufferWritingResultPartition bufferWritingResultPartition = createResultPartition(ResultPartitionType.PIPELINED_BOUNDED);
    bufferWritingResultPartition.notifyEndOfData(StopMode.DRAIN);
    CompletableFuture<Void> allRecordsProcessedFuture = bufferWritingResultPartition.getAllDataProcessedFuture();
    assertFalse(allRecordsProcessedFuture.isDone());
    for (ResultSubpartition resultSubpartition : bufferWritingResultPartition.subpartitions) {
        assertEquals(1, resultSubpartition.getTotalNumberOfBuffersUnsafe());
        Buffer nextBuffer = ((PipelinedSubpartition) resultSubpartition).pollBuffer().buffer();
        assertFalse(nextBuffer.isBuffer());
        assertEquals(new EndOfData(StopMode.DRAIN), EventSerializer.fromBuffer(nextBuffer, getClass().getClassLoader()));
    }
    for (int i = 0; i < bufferWritingResultPartition.subpartitions.length; ++i) {
        ((PipelinedSubpartition) bufferWritingResultPartition.subpartitions[i]).acknowledgeAllDataProcessed();
        if (i < bufferWritingResultPartition.subpartitions.length - 1) {
            assertFalse(allRecordsProcessedFuture.isDone());
        } else {
            assertTrue(allRecordsProcessedFuture.isDone());
            assertFalse(allRecordsProcessedFuture.isCompletedExceptionally());
        }
    }
}

Example 3 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class MultipleInputStreamTaskTest method testSkipExecutionsIfFinishedOnRestore.

@Test
public void testSkipExecutionsIfFinishedOnRestore() throws Exception {
    OperatorID nonSourceOperatorId = new OperatorID();
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(nonSourceOperatorId, new LifeCycleMonitorMultipleInputOperatorFactory()).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        testHarness.processElement(Watermark.MAX_WATERMARK, 0);
        testHarness.processElement(Watermark.MAX_WATERMARK, 1);
        testHarness.processElement(Watermark.MAX_WATERMARK, 2);
        testHarness.waitForTaskCompletion();
        assertThat(testHarness.getOutput(), contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
    }
}

Example 4 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class SourceTaskTerminationTest method stopWithSavepointStreamTaskTestHelper.

private void stopWithSavepointStreamTaskTestHelper(final boolean shouldTerminate) throws Exception {
    final long syncSavepointId = 34L;
    try (StreamTaskMailboxTestHarness<Long> srcTaskTestHarness = getSourceStreamTaskTestHarness()) {
        final StreamTask<Long, ?> srcTask = srcTaskTestHarness.getStreamTask();
        srcTaskTestHarness.processAll();
        // step by step let the source thread emit elements
        emitAndVerifyWatermarkAndElement(srcTaskTestHarness, 1L);
        emitAndVerifyWatermarkAndElement(srcTaskTestHarness, 2L);
        srcTaskTestHarness.processUntil(srcTask.triggerCheckpointAsync(new CheckpointMetaData(31L, 900), CheckpointOptions.forCheckpointWithDefaultLocation())::isDone);
        verifyCheckpointBarrier(srcTaskTestHarness.getOutput(), 31L);
        emitAndVerifyWatermarkAndElement(srcTaskTestHarness, 3L);
        srcTaskTestHarness.processUntil(srcTask.triggerCheckpointAsync(new CheckpointMetaData(syncSavepointId, 900), new CheckpointOptions(shouldTerminate ? SavepointType.terminate(SavepointFormatType.CANONICAL) : SavepointType.suspend(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()))::isDone);
        if (shouldTerminate) {
            // if we are in TERMINATE mode, we expect the source task
            // to emit MAX_WM before the SYNC_SAVEPOINT barrier.
            verifyWatermark(srcTaskTestHarness.getOutput(), Watermark.MAX_WATERMARK);
        }
        verifyEvent(srcTaskTestHarness.getOutput(), new EndOfData(shouldTerminate ? StopMode.DRAIN : StopMode.NO_DRAIN));
        verifyCheckpointBarrier(srcTaskTestHarness.getOutput(), syncSavepointId);
        waitForSynchronousSavepointIdToBeSet(srcTask);
        assertTrue(srcTask.getSynchronousSavepointId().isPresent());
        srcTaskTestHarness.processUntil(srcTask.notifyCheckpointCompleteAsync(syncSavepointId)::isDone);
        srcTaskTestHarness.waitForTaskCompletion();
    }
}

Example 5 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class StreamTestSingleInputGate method setupInputChannels.

private TestInputChannel[] setupInputChannels() {
    TestInputChannel[] inputChannels = new TestInputChannel[numInputChannels];
    for (int i = 0; i < numInputChannels; i++) {
        final int channelIndex = i;
        final DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128);
        final SerializationDelegate<StreamElement> delegate = new SerializationDelegate<>(new StreamElementSerializer<T>(serializer));
        inputQueues[channelIndex] = new ConcurrentLinkedQueue<>();
        inputChannels[channelIndex] = new TestInputChannel(inputGate, i);
        final BufferAndAvailabilityProvider answer = () -> {
            ConcurrentLinkedQueue<InputValue<Object>> inputQueue = inputQueues[channelIndex];
            InputValue<Object> input;
            Buffer.DataType nextType;
            synchronized (inputQueue) {
                input = inputQueue.poll();
                nextType = !inputQueue.isEmpty() ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.NONE;
            }
            if (input != null && input.isStreamEnd()) {
                inputChannels[channelIndex].setReleased();
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), nextType, 0, 0));
            } else if (input != null && input.isDataEnd()) {
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(new EndOfData(StopMode.DRAIN), false), nextType, 0, 0));
            } else if (input != null && input.isStreamRecord()) {
                StreamElement inputElement = input.getStreamRecord();
                delegate.setInstance(inputElement);
                ByteBuffer serializedRecord = RecordWriter.serializeRecord(dataOutputSerializer, delegate);
                BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
                BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
                bufferBuilder.appendAndCommit(serializedRecord);
                bufferBuilder.finish();
                bufferBuilder.close();
                // Call getCurrentBuffer to ensure size is set
                return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextType, 0, 0));
            } else if (input != null && input.isEvent()) {
                AbstractEvent event = input.getEvent();
                if (event instanceof EndOfPartitionEvent) {
                    inputChannels[channelIndex].setReleased();
                }
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(event, false), nextType, 0, 0));
            } else {
                return Optional.empty();
            }
        };
        inputChannels[channelIndex].addBufferAndAvailability(answer);
    }
    return inputChannels;
}