Examples with TestingSourceSplit org.apache.flink.connector.base.source.reader.mocks.TestingSourceSplit used on opensource projects

Example 1 with TestingSourceSplit

use of org.apache.flink.connector.base.source.reader.mocks.TestingSourceSplit in project flink by apache.

the class SplitFetcherManagerTest method testExceptionPropagation.

// the final modifier is important so that '@SafeVarargs' is accepted on Java 8
@SuppressWarnings("FinalPrivateMethod")
@SafeVarargs
private final void testExceptionPropagation(final RecordsWithSplitIds<Integer>... fetchesBeforeError) throws Exception {
    final IOException testingException = new IOException("test");
    final FutureCompletingBlockingQueue<RecordsWithSplitIds<Integer>> queue = new FutureCompletingBlockingQueue<>(10);
    final AwaitingReader<Integer, TestingSourceSplit> reader = new AwaitingReader<>(testingException, fetchesBeforeError);
    final SplitFetcherManager<Integer, TestingSourceSplit> fetcher = createFetcher("testSplit", queue, reader);
    reader.awaitAllRecordsReturned();
    drainQueue(queue);
    assertFalse(queue.getAvailabilityFuture().isDone());
    reader.triggerThrowException();
    // await the error propagation
    queue.getAvailabilityFuture().get();
    try {
        fetcher.checkErrors();
        fail("expected exception");
    } catch (Exception e) {
        assertSame(testingException, e.getCause().getCause());
    } finally {
        fetcher.close(20_000L);
    }
}

Example 2 with TestingSourceSplit

use of org.apache.flink.connector.base.source.reader.mocks.TestingSourceSplit in project flink by apache.

the class SplitFetcherTest method testFetcherNotIdleAfterSplitAdded.

@Test
public void testFetcherNotIdleAfterSplitAdded() {
    final SplitFetcher<Object, TestingSourceSplit> fetcher = createFetcher(new TestingSplitReader<>());
    final TestingSourceSplit split = new TestingSourceSplit("test-split");
    fetcher.addSplits(Collections.singletonList(split));
    assertFalse(fetcher.isIdle());
    // need to loop here because the internal wakeup flag handling means we need multiple loops
    while (fetcher.assignedSplits().isEmpty()) {
        fetcher.runOnce();
        assertFalse(fetcher.isIdle());
    }
}

Example 3 with TestingSourceSplit

use of org.apache.flink.connector.base.source.reader.mocks.TestingSourceSplit in project flink by apache.

the class SourceReaderBaseTest method createReaderAndAwaitAvailable.

// ------------------------------------------------------------------------
// Testing Setup Helpers
// ------------------------------------------------------------------------
private static <E> SourceReader<E, ?> createReaderAndAwaitAvailable(final String splitId, final RecordsWithSplitIds<E> records) throws Exception {
    final FutureCompletingBlockingQueue<RecordsWithSplitIds<E>> elementsQueue = new FutureCompletingBlockingQueue<>();
    final SourceReader<E, TestingSourceSplit> reader = new SingleThreadMultiplexSourceReaderBase<E, E, TestingSourceSplit, TestingSourceSplit>(elementsQueue, () -> new TestingSplitReader<>(records), new PassThroughRecordEmitter<>(), new Configuration(), new TestingReaderContext()) {

        @Override
        public void notifyCheckpointComplete(long checkpointId) {
        }

        @Override
        protected void onSplitFinished(Map<String, TestingSourceSplit> finishedSplitIds) {
        }

        @Override
        protected TestingSourceSplit initializedState(TestingSourceSplit split) {
            return split;
        }

        @Override
        protected TestingSourceSplit toSplitType(String splitId, TestingSourceSplit splitState) {
            return splitState;
        }
    };
    reader.start();
    final List<TestingSourceSplit> splits = Collections.singletonList(new TestingSourceSplit(splitId));
    reader.addSplits(splits);
    reader.isAvailable().get();
    return reader;
}

Example 4 with TestingSourceSplit

use of org.apache.flink.connector.base.source.reader.mocks.TestingSourceSplit in project flink by apache.

the class SerdeUtilsTest method testSerdeSplitAssignments.

@Test
public void testSerdeSplitAssignments() throws IOException {
    final Map<Integer, Set<TestingSourceSplit>> splitAssignments = new HashMap<>();
    final HashSet<TestingSourceSplit> splitsForReader0 = new HashSet<>();
    splitsForReader0.add(new TestingSourceSplit("split-0"));
    splitsForReader0.add(new TestingSourceSplit("split-1"));
    splitsForReader0.add(new TestingSourceSplit("split-2"));
    final HashSet<TestingSourceSplit> splitsForReader1 = new HashSet<>();
    splitsForReader1.add(new TestingSourceSplit("split-3"));
    splitsForReader1.add(new TestingSourceSplit("split-4"));
    splitsForReader1.add(new TestingSourceSplit("split-5"));
    splitAssignments.put(READER0, splitsForReader0);
    splitAssignments.put(READER1, splitsForReader1);
    final byte[] serializedSplitAssignments = SerdeUtils.serializeSplitAssignments(splitAssignments, new TestingSourceSplitSerializer());
    final Map<Integer, HashSet<TestingSourceSplit>> deseredSplitAssignments = SerdeUtils.deserializeSplitAssignments(serializedSplitAssignments, new TestingSourceSplitSerializer(), HashSet::new);
    assertEquals(splitAssignments, deseredSplitAssignments);
}