Examples with InputStatus org.apache.flink.core.io.InputStatus used on opensource projects

Example 6 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class SourceReaderBaseTest method testMultipleSplitsWithDifferentFinishingMoments.

@Test
void testMultipleSplitsWithDifferentFinishingMoments() throws Exception {
    FutureCompletingBlockingQueue<RecordsWithSplitIds<int[]>> elementsQueue = new FutureCompletingBlockingQueue<>();
    MockSplitReader mockSplitReader = MockSplitReader.newBuilder().setNumRecordsPerSplitPerFetch(2).setSeparatedFinishedRecord(false).setBlockingFetch(false).build();
    MockSourceReader reader = new MockSourceReader(elementsQueue, () -> mockSplitReader, getConfig(), new TestingReaderContext());
    reader.start();
    List<MockSourceSplit> splits = Arrays.asList(getSplit(0, 10, Boundedness.BOUNDED), getSplit(1, 12, Boundedness.BOUNDED));
    reader.addSplits(splits);
    reader.notifyNoMoreSplits();
    while (true) {
        InputStatus status = reader.pollNext(new TestingReaderOutput<>());
        if (status == InputStatus.END_OF_INPUT) {
            break;
        }
        if (status == InputStatus.NOTHING_AVAILABLE) {
            reader.isAvailable().get();
        }
    }
}

Example 7 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class SourceReaderBaseTest method testExceptionInSplitReader.

@Test
void testExceptionInSplitReader() {
    assertThatThrownBy(() -> {
        final String errMsg = "Testing Exception";
        FutureCompletingBlockingQueue<RecordsWithSplitIds<int[]>> elementsQueue = new FutureCompletingBlockingQueue<>();
        // called.
        try (MockSourceReader reader = new MockSourceReader(elementsQueue, () -> new SplitReader<int[], MockSourceSplit>() {

            @Override
            public RecordsWithSplitIds<int[]> fetch() {
                throw new RuntimeException(errMsg);
            }

            @Override
            public void handleSplitsChanges(SplitsChange<MockSourceSplit> splitsChanges) {
            }

            @Override
            public void wakeUp() {
            }

            @Override
            public void close() {
            }
        }, getConfig(), new TestingReaderContext())) {
            ValidatingSourceOutput output = new ValidatingSourceOutput();
            reader.addSplits(Collections.singletonList(getSplit(0, NUM_RECORDS_PER_SPLIT, Boundedness.CONTINUOUS_UNBOUNDED)));
            reader.notifyNoMoreSplits();
            // two polls.
            while (true) {
                InputStatus inputStatus = reader.pollNext(output);
                assertThat(inputStatus).isNotEqualTo(InputStatus.END_OF_INPUT);
                // Add a sleep to avoid tight loop.
                Thread.sleep(1);
            }
        }
    }).isInstanceOf(RuntimeException.class).hasMessage("One or more fetchers have encountered exception");
}

Example 8 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class SourceReaderBaseTest method testMultipleSplitsWithSeparatedFinishedRecord.

@Test
void testMultipleSplitsWithSeparatedFinishedRecord() throws Exception {
    FutureCompletingBlockingQueue<RecordsWithSplitIds<int[]>> elementsQueue = new FutureCompletingBlockingQueue<>();
    MockSplitReader mockSplitReader = MockSplitReader.newBuilder().setNumRecordsPerSplitPerFetch(2).setSeparatedFinishedRecord(true).setBlockingFetch(false).build();
    MockSourceReader reader = new MockSourceReader(elementsQueue, () -> mockSplitReader, getConfig(), new TestingReaderContext());
    reader.start();
    List<MockSourceSplit> splits = Arrays.asList(getSplit(0, 10, Boundedness.BOUNDED), getSplit(1, 10, Boundedness.BOUNDED));
    reader.addSplits(splits);
    reader.notifyNoMoreSplits();
    while (true) {
        InputStatus status = reader.pollNext(new TestingReaderOutput<>());
        if (status == InputStatus.END_OF_INPUT) {
            break;
        }
        if (status == InputStatus.NOTHING_AVAILABLE) {
            reader.isAvailable().get();
        }
    }
}

Example 9 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class HybridSourceReader method pollNext.

@Override
public InputStatus pollNext(ReaderOutput output) throws Exception {
    if (currentReader == null) {
        return InputStatus.NOTHING_AVAILABLE;
    }
    InputStatus status = currentReader.pollNext(output);
    if (status == InputStatus.END_OF_INPUT) {
        // trap END_OF_INPUT unless all sources have finished
        LOG.info("End of input subtask={} sourceIndex={} {}", readerContext.getIndexOfSubtask(), currentSourceIndex, currentReader);
        // Signal the coordinator that this reader has consumed all input and the
        // next source can potentially be activated.
        readerContext.sendSourceEventToCoordinator(new SourceReaderFinishedEvent(currentSourceIndex));
        if (!isFinalSource) {
            // availability future after receiving more splits to resume.
            if (availabilityFuture.isDone()) {
                // reset to avoid continued polling
                availabilityFuture = new CompletableFuture();
            }
            return InputStatus.NOTHING_AVAILABLE;
        }
    }
    return status;
}