use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.
the class StreamTaskFinalCheckpointsTest method testReportOperatorsFinishedInCheckpoint.
@Test
public void testReportOperatorsFinishedInCheckpoint() throws Exception {
ResultPartition[] partitionWriters = new ResultPartition[2];
try {
for (int i = 0; i < partitionWriters.length; ++i) {
partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
partitionWriters[i].setup();
}
final CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO, 1).addAdditionalOutput(partitionWriters).setCheckpointResponder(checkpointResponder).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).setupOperatorChain(new StatefulOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
// Trigger the first checkpoint before we call operators' finish method.
CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2);
processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
assertFalse(testHarness.getTaskStateManager().getJobManagerTaskStateSnapshotsByCheckpointId().get(2L).isTaskFinished());
// Trigger the first checkpoint after we call operators' finish method.
// The checkpoint is added to the mailbox and will be processed in the
// mailbox loop after call operators' finish method in the afterInvoke()
// method.
testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
checkpointFuture = triggerCheckpoint(testHarness, 4);
checkpointFuture.thenAccept((ignored) -> {
for (ResultPartition resultPartition : partitionWriters) {
resultPartition.onSubpartitionAllDataProcessed(0);
}
});
testHarness.processAll();
testHarness.finishProcessing();
assertTrue(checkpointFuture.isDone());
testHarness.getTaskStateManager().getWaitForReportLatch().await();
assertTrue(testHarness.getTaskStateManager().getJobManagerTaskStateSnapshotsByCheckpointId().get(4L).isTaskFinished());
}
} finally {
for (ResultPartitionWriter writer : partitionWriters) {
if (writer != null) {
writer.close();
}
}
}
}
use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.
the class SourceOperatorStreamTaskTest method testEmittingMaxWatermarkAfterReadingAllRecords.
@Test
public void testEmittingMaxWatermarkAfterReadingAllRecords() throws Exception {
try (StreamTaskMailboxTestHarness<Integer> testHarness = createTestHarness()) {
testHarness.processAll();
testHarness.finishProcessing();
Queue<Object> expectedOutput = new LinkedList<>();
expectedOutput.add(Watermark.MAX_WATERMARK);
expectedOutput.add(new EndOfData(StopMode.DRAIN));
assertThat(testHarness.getOutput().toArray(), equalTo(expectedOutput.toArray()));
}
}
use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.
the class TwoInputStreamTaskTest method testSkipExecutionsIfFinishedOnRestore.
@Test
public void testSkipExecutionsIfFinishedOnRestore() throws Exception {
OperatorID nonSourceOperatorId = new OperatorID();
try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(nonSourceOperatorId, new TestFinishedOnRestoreStreamOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
testHarness.processElement(Watermark.MAX_WATERMARK, 0);
testHarness.processElement(Watermark.MAX_WATERMARK, 1);
testHarness.waitForTaskCompletion();
assertThat(testHarness.getOutput(), contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
}
}
use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.
the class BoundedBlockingResultPartition method notifyEndOfData.
@Override
public void notifyEndOfData(StopMode mode) throws IOException {
if (!hasNotifiedEndOfUserRecords) {
broadcastEvent(new EndOfData(mode), false);
hasNotifiedEndOfUserRecords = true;
}
}
use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.
the class PipelinedResultPartition method notifyEndOfData.
@Override
public void notifyEndOfData(StopMode mode) throws IOException {
synchronized (lock) {
if (!hasNotifiedEndOfUserRecords) {
broadcastEvent(new EndOfData(mode), false);
hasNotifiedEndOfUserRecords = true;
}
}
}
Aggregations