Search in sources :

Example 16 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testStopWithSavepointDrainWaitsForSourcesFinish.

/**
 * In this scenario:
 *
 * <ul>
 *   <li>Network inputs are processed until CheckpointBarriers for synchronous savepoint are
 *       processed.
 *   <li>RPC for stop-with-savepoint comes for sources
 *   <li>Sources keep being invoked until they return END_OF_DATA
 *   <li>Synchronous savepoint is triggered
 * </ul>
 */
@Test
public void testStopWithSavepointDrainWaitsForSourcesFinish() throws Exception {
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyExecutionConfig(applyObjectReuse(objectReuse)).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory(4, true)).build()) {
        testHarness.setAutoProcess(false);
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        CheckpointBarrier barrier = createStopWithSavepointDrainBarrier();
        testHarness.processElement(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP), 0);
        testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0);
        testHarness.processEvent(barrier, 0);
        testHarness.processElement(new StreamRecord<>(47d, TimestampAssigner.NO_TIMESTAMP), 1);
        testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1);
        testHarness.processEvent(barrier, 1);
        addSourceRecords(testHarness, 1, Boundedness.CONTINUOUS_UNBOUNDED, 1, 2);
        addSourceRecords(testHarness, 2, Boundedness.CONTINUOUS_UNBOUNDED, 3, 4);
        testHarness.processAll();
        Future<Boolean> checkpointFuture = testHarness.getStreamTask().triggerCheckpointAsync(metaData, barrier.getCheckpointOptions());
        processSingleStepUntil(testHarness, checkpointFuture::isDone);
        expectedOutput.add(new StreamRecord<>("3", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("1", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("4", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("2", TimestampAssigner.NO_TIMESTAMP));
        ArrayList<Object> actualOutput = new ArrayList<>(testHarness.getOutput());
        assertThat(actualOutput.subList(0, expectedOutput.size()), containsInAnyOrder(expectedOutput.toArray()));
        assertThat(actualOutput.subList(actualOutput.size() - 3, actualOutput.size()), contains(new StreamRecord<>("FINISH"), new EndOfData(StopMode.DRAIN), barrier));
    }
}
Also used : MockSource(org.apache.flink.api.connector.source.mocks.MockSource) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) ArrayList(java.util.ArrayList) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) ArrayDeque(java.util.ArrayDeque) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) MapToStringMultipleInputOperatorFactory(org.apache.flink.streaming.runtime.tasks.MultipleInputStreamTaskTest.MapToStringMultipleInputOperatorFactory) Test(org.junit.Test)

Example 17 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testTriggerCheckpointWithFinishedChannelsAndSourceChain.

private void testTriggerCheckpointWithFinishedChannelsAndSourceChain(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyStreamConfig(config -> {
            config.setCheckpointingEnabled(true);
            config.setUnalignedCheckpointsEnabled(checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable());
        }).modifyExecutionConfig(applyObjectReuse(objectReuse)).setCheckpointResponder(checkpointResponder).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MultipleInputStreamTaskTest.LifeCycleTrackingMockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MultipleInputStreamTaskTest.LifeCycleTrackingMockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addAdditionalOutput(partitionWriters).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(4)).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            testHarness.getStreamTask().getCheckpointBarrierHandler().get();
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            testHarness.processAll();
            // The checkpoint 2 would be aligned after received all the EndOfPartitionEvent.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 1, 0);
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after all the inputs have received EndOfPartition.
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 4 would be triggered successfully.
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Each result partition should have emitted 2 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(3, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) RecordOrEventCollectingResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.RecordOrEventCollectingResultPartitionWriter) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) MapToStringMultipleInputOperatorFactory(org.apache.flink.streaming.runtime.tasks.MultipleInputStreamTaskTest.MapToStringMultipleInputOperatorFactory)

Example 18 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testSkipExecutionsIfFinishedOnRestoreWithSourceChained.

@Test
public void testSkipExecutionsIfFinishedOnRestoreWithSourceChained() throws Exception {
    OperatorID firstSourceOperatorId = new OperatorID();
    OperatorID secondSourceOperatorId = new OperatorID();
    OperatorID nonSourceOperatorId = new OperatorID();
    List<Object> output = new ArrayList<>();
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.INT_TYPE_INFO).addAdditionalOutput(new RecordOrEventCollectingResultPartitionWriter<StreamElement>(output, new StreamElementSerializer<>(IntSerializer.INSTANCE)) {

        @Override
        public void notifyEndOfData(StopMode mode) throws IOException {
            broadcastEvent(new EndOfData(mode), false);
        }
    }).addSourceInput(firstSourceOperatorId, new SourceOperatorFactory<>(new SourceOperatorStreamTaskTest.LifeCycleMonitorSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(secondSourceOperatorId, new SourceOperatorFactory<>(new SourceOperatorStreamTaskTest.LifeCycleMonitorSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(nonSourceOperatorId, new LifeCycleMonitorMultipleInputOperatorFactory()).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        testHarness.processElement(Watermark.MAX_WATERMARK);
        assertThat(output, is(empty()));
        testHarness.waitForTaskCompletion();
        assertThat(output, contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
        for (StreamOperatorWrapper<?, ?> wrapper : testHarness.getStreamTask().operatorChain.getAllOperators()) {
            if (wrapper.getStreamOperator() instanceof SourceOperator<?, ?>) {
                SourceOperatorStreamTaskTest.LifeCycleMonitorSourceReader sourceReader = (SourceOperatorStreamTaskTest.LifeCycleMonitorSourceReader) ((SourceOperator<?, ?>) wrapper.getStreamOperator()).getSourceReader();
                sourceReader.getLifeCycleMonitor().assertCallTimes(0, LifeCyclePhase.values());
            }
        }
    }
}
Also used : ArrayList(java.util.ArrayList) StopMode(org.apache.flink.runtime.io.network.api.StopMode) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) SourceOperator(org.apache.flink.streaming.api.operators.SourceOperator) RecordOrEventCollectingResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.RecordOrEventCollectingResultPartitionWriter) Test(org.junit.Test)

Example 19 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class MultipleInputStreamTaskTest method testTriggeringCheckpointWithFinishedChannels.

private void testTriggeringCheckpointWithFinishedChannels(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).addAdditionalOutput(partitionWriters).setCheckpointResponder(checkpointResponder).modifyStreamConfig(config -> {
            config.setCheckpointingEnabled(true);
            config.setUnalignedCheckpointsEnabled(checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable());
        }).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            testHarness.getStreamTask().getCheckpointBarrierHandler().get();
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after all the inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1, 0);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 2, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 1, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 2, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 6, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) TaskIOMetricGroup(org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup) Arrays(java.util.Arrays) TestCheckpointResponder(org.apache.flink.runtime.taskmanager.TestCheckpointResponder) SharedObjects(org.apache.flink.testutils.junit.SharedObjects) NoMoreSplitsEvent(org.apache.flink.runtime.source.event.NoMoreSplitsEvent) AbstractStreamOperatorFactory(org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory) Duration(java.time.Duration) Map(java.util.Map) WatermarkStatus(org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) BoundedOneInput(org.apache.flink.streaming.api.operators.BoundedOneInput) Serializable(java.io.Serializable) StopMode(org.apache.flink.runtime.io.network.api.StopMode) MetricNames(org.apache.flink.runtime.metrics.MetricNames) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Matchers.contains(org.hamcrest.Matchers.contains) WatermarkMetricOperator(org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTest.WatermarkMetricOperator) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) Boundedness(org.apache.flink.api.connector.source.Boundedness) Counter(org.apache.flink.metrics.Counter) RunWith(org.junit.runner.RunWith) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) TimestampAssigner(org.apache.flink.api.common.eventtime.TimestampAssigner) DataOutputView(org.apache.flink.core.memory.DataOutputView) AbstractInput(org.apache.flink.streaming.api.operators.AbstractInput) ArrayList(java.util.ArrayList) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) InternalOperatorMetricGroup(org.apache.flink.runtime.metrics.groups.InternalOperatorMetricGroup) Gauge(org.apache.flink.metrics.Gauge) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) TestHarnessUtil(org.apache.flink.streaming.util.TestHarnessUtil) Before(org.junit.Before) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) SourceReader(org.apache.flink.api.connector.source.SourceReader) Parameter(org.junit.runners.Parameterized.Parameter) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) InterceptingTaskMetricGroup(org.apache.flink.runtime.metrics.util.InterceptingTaskMetricGroup) AddSplitEvent(org.apache.flink.runtime.source.event.AddSplitEvent) StreamMultipleInputProcessor(org.apache.flink.streaming.runtime.io.StreamMultipleInputProcessor) AbstractStreamOperator(org.apache.flink.streaming.api.operators.AbstractStreamOperator) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) JobID(org.apache.flink.api.common.JobID) UnregisteredMetricGroups(org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups) StreamTaskFinalCheckpointsTest.processMailTillCheckpointSucceeds(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.processMailTillCheckpointSucceeds) Assert(org.junit.Assert) ArrayDeque(java.util.ArrayDeque) Assert.assertEquals(org.junit.Assert.assertEquals) Input(org.apache.flink.streaming.api.operators.Input) LifeCycleMonitorMultipleInputOperatorFactory(org.apache.flink.streaming.runtime.tasks.MultipleInputStreamTaskChainedSourcesCheckpointingTest.LifeCycleMonitorMultipleInputOperatorFactory) WatermarkGenerator(org.apache.flink.api.common.eventtime.WatermarkGenerator) SavepointType(org.apache.flink.runtime.checkpoint.SavepointType) StringSerializer(org.apache.flink.api.common.typeutils.base.StringSerializer) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) IntSerializer(org.apache.flink.api.common.typeutils.base.IntSerializer) TypeSerializerSnapshot(org.apache.flink.api.common.typeutils.TypeSerializerSnapshot) Parameterized(org.junit.runners.Parameterized) SourceReaderContext(org.apache.flink.api.connector.source.SourceReaderContext) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) InterceptingOperatorMetricGroup(org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup) BoundedMultiInput(org.apache.flink.streaming.api.operators.BoundedMultiInput) MockSourceReader(org.apache.flink.api.connector.source.mocks.MockSourceReader) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) MockSourceSplitSerializer(org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) MultipleInputStreamOperator(org.apache.flink.streaming.api.operators.MultipleInputStreamOperator) List(java.util.List) SerializedValue(org.apache.flink.util.SerializedValue) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) CancelCheckpointMarker(org.apache.flink.runtime.io.network.api.CancelCheckpointMarker) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) CheckpointResponder(org.apache.flink.runtime.taskmanager.CheckpointResponder) CheckpointMetrics(org.apache.flink.runtime.checkpoint.CheckpointMetrics) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData) Parameters(org.junit.runners.Parameterized.Parameters) CoreMatchers.not(org.hamcrest.CoreMatchers.not) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) AbstractStreamOperatorV2(org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2) Watermark(org.apache.flink.streaming.api.watermark.Watermark) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Metric(org.apache.flink.metrics.Metric) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) MockSourceSplit(org.apache.flink.api.connector.source.mocks.MockSourceSplit) TaskMetricGroup(org.apache.flink.runtime.metrics.groups.TaskMetricGroup) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) MockSource(org.apache.flink.api.connector.source.mocks.MockSource) OperatorMetricGroup(org.apache.flink.metrics.groups.OperatorMetricGroup) DataInputView(org.apache.flink.core.memory.DataInputView) SharedReference(org.apache.flink.testutils.junit.SharedReference) StreamOperatorParameters(org.apache.flink.streaming.api.operators.StreamOperatorParameters) WatermarkOutput(org.apache.flink.api.common.eventtime.WatermarkOutput) IsMapContaining(org.hamcrest.collection.IsMapContaining) Consumer(java.util.function.Consumer) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) Rule(org.junit.Rule) PartitionTestUtils(org.apache.flink.runtime.io.network.partition.PartitionTestUtils) LatencyMarker(org.apache.flink.streaming.runtime.streamrecord.LatencyMarker) Collections(java.util.Collections) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData)

Example 20 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class EventSerializer method fromSerializedEvent.

public static AbstractEvent fromSerializedEvent(ByteBuffer buffer, ClassLoader classLoader) throws IOException {
    if (buffer.remaining() < 4) {
        throw new IOException("Incomplete event");
    }
    final ByteOrder bufferOrder = buffer.order();
    buffer.order(ByteOrder.BIG_ENDIAN);
    try {
        final int type = buffer.getInt();
        if (type == END_OF_PARTITION_EVENT) {
            return EndOfPartitionEvent.INSTANCE;
        } else if (type == CHECKPOINT_BARRIER_EVENT) {
            return deserializeCheckpointBarrier(buffer);
        } else if (type == END_OF_SUPERSTEP_EVENT) {
            return EndOfSuperstepEvent.INSTANCE;
        } else if (type == END_OF_CHANNEL_STATE_EVENT) {
            return EndOfChannelStateEvent.INSTANCE;
        } else if (type == END_OF_USER_RECORDS_EVENT) {
            return new EndOfData(StopMode.values()[buffer.get()]);
        } else if (type == CANCEL_CHECKPOINT_MARKER_EVENT) {
            long id = buffer.getLong();
            return new CancelCheckpointMarker(id);
        } else if (type == ANNOUNCEMENT_EVENT) {
            int sequenceNumber = buffer.getInt();
            AbstractEvent announcedEvent = fromSerializedEvent(buffer, classLoader);
            return new EventAnnouncement(announcedEvent, sequenceNumber);
        } else if (type == VIRTUAL_CHANNEL_SELECTOR_EVENT) {
            return new SubtaskConnectionDescriptor(buffer.getInt(), buffer.getInt());
        } else if (type == OTHER_EVENT) {
            try {
                final DataInputDeserializer deserializer = new DataInputDeserializer(buffer);
                final String className = deserializer.readUTF();
                final Class<? extends AbstractEvent> clazz;
                try {
                    clazz = classLoader.loadClass(className).asSubclass(AbstractEvent.class);
                } catch (ClassNotFoundException e) {
                    throw new IOException("Could not load event class '" + className + "'.", e);
                } catch (ClassCastException e) {
                    throw new IOException("The class '" + className + "' is not a valid subclass of '" + AbstractEvent.class.getName() + "'.", e);
                }
                final AbstractEvent event = InstantiationUtil.instantiate(clazz, AbstractEvent.class);
                event.read(deserializer);
                return event;
            } catch (Exception e) {
                throw new IOException("Error while deserializing or instantiating event.", e);
            }
        } else {
            throw new IOException("Corrupt byte stream for event");
        }
    } finally {
        buffer.order(bufferOrder);
    }
}
Also used : EventAnnouncement(org.apache.flink.runtime.io.network.api.EventAnnouncement) CancelCheckpointMarker(org.apache.flink.runtime.io.network.api.CancelCheckpointMarker) IOException(java.io.IOException) ByteOrder(java.nio.ByteOrder) AbstractEvent(org.apache.flink.runtime.event.AbstractEvent) IOException(java.io.IOException) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) SubtaskConnectionDescriptor(org.apache.flink.runtime.io.network.api.SubtaskConnectionDescriptor) DataInputDeserializer(org.apache.flink.core.memory.DataInputDeserializer)

Aggregations

EndOfData (org.apache.flink.runtime.io.network.api.EndOfData)21 Test (org.junit.Test)12 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)6 CheckpointBarrier (org.apache.flink.runtime.io.network.api.CheckpointBarrier)6 ResultPartitionWriter (org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter)6 ResultPartition (org.apache.flink.runtime.io.network.partition.ResultPartition)6 CompletingCheckpointResponder (org.apache.flink.streaming.util.CompletingCheckpointResponder)6 ArrayList (java.util.ArrayList)5 CheckpointMetaData (org.apache.flink.runtime.checkpoint.CheckpointMetaData)5 SourceOperatorFactory (org.apache.flink.streaming.api.operators.SourceOperatorFactory)5 IOException (java.io.IOException)4 EndOfPartitionEvent (org.apache.flink.runtime.io.network.api.EndOfPartitionEvent)4 StopMode (org.apache.flink.runtime.io.network.api.StopMode)4 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)4 Duration (java.time.Duration)3 ArrayDeque (java.util.ArrayDeque)3 Collections (java.util.Collections)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 JobID (org.apache.flink.api.common.JobID)3 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)3