Examples with TestSourceContext org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext used on opensource projects

Example 1 with TestSourceContext

use of org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext in project flink by apache.

the class AbstractFetcherTest method testConcurrentPartitionsDiscoveryAndLoopFetching.

@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
    // test data
    final KafkaTopicPartition testPartition = new KafkaTopicPartition("test", 42);
    // ----- create the test fetcher -----
    SourceContext<String> sourceContext = new TestSourceContext<>();
    Map<KafkaTopicPartition, Long> partitionsWithInitialOffsets = Collections.singletonMap(testPartition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    final OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
    final OneShotLatch stateIterationBlockLatch = new OneShotLatch();
    final TestFetcher<String> fetcher = new TestFetcher<>(sourceContext, partitionsWithInitialOffsets, null, /* watermark strategy */
    new TestProcessingTimeService(), 10, fetchLoopWaitLatch, stateIterationBlockLatch);
    // ----- run the fetcher -----
    final CheckedThread checkedThread = new CheckedThread() {

        @Override
        public void go() throws Exception {
            fetcher.runFetchLoop();
        }
    };
    checkedThread.start();
    // wait until state iteration begins before adding discovered partitions
    fetchLoopWaitLatch.await();
    fetcher.addDiscoveredPartitions(Collections.singletonList(testPartition));
    stateIterationBlockLatch.trigger();
    checkedThread.sync();
}

Example 2 with TestSourceContext

use of org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext in project flink by apache.

the class AbstractFetcherTest method testIgnorePartitionStateSentinelInSnapshot.

@Test
public void testIgnorePartitionStateSentinelInSnapshot() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 2), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    originalPartitions.put(new KafkaTopicPartition(testTopic, 3), KafkaTopicPartitionStateSentinel.EARLIEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, null, /* watermark strategy */
    new TestProcessingTimeService(), 0);
    synchronized (sourceContext.getCheckpointLock()) {
        HashMap<KafkaTopicPartition, Long> currentState = fetcher.snapshotCurrentState();
        fetcher.commitInternalOffsetsToKafka(currentState, new KafkaCommitCallback() {

            @Override
            public void onSuccess() {
            }

            @Override
            public void onException(Throwable cause) {
                throw new RuntimeException("Callback failed", cause);
            }
        });
        assertTrue(fetcher.getLastCommittedOffsets().isPresent());
        assertEquals(Collections.emptyMap(), fetcher.getLastCommittedOffsets().get());
    }
}

Example 3 with TestSourceContext

use of org.apache.flink.streaming.connectors.kafka.testutils.TestSourceContext in project flink by apache.

the class AbstractFetcherTest method testSkipCorruptedRecord.

// ------------------------------------------------------------------------
// Record emitting tests
// ------------------------------------------------------------------------
@Test
public void testSkipCorruptedRecord() throws Exception {
    final String testTopic = "test topic name";
    Map<KafkaTopicPartition, Long> originalPartitions = new HashMap<>();
    originalPartitions.put(new KafkaTopicPartition(testTopic, 1), KafkaTopicPartitionStateSentinel.LATEST_OFFSET);
    TestSourceContext<Long> sourceContext = new TestSourceContext<>();
    TestFetcher<Long> fetcher = new TestFetcher<>(sourceContext, originalPartitions, null, /* watermark strategy */
    new TestProcessingTimeService(), 0);
    final KafkaTopicPartitionState<Long, Object> partitionStateHolder = fetcher.subscribedPartitionStates().get(0);
    emitRecord(fetcher, 1L, partitionStateHolder, 1L);
    emitRecord(fetcher, 2L, partitionStateHolder, 2L);
    assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(2L, partitionStateHolder.getOffset());
    // emit no records
    fetcher.emitRecordsWithTimestamps(emptyQueue(), partitionStateHolder, 3L, Long.MIN_VALUE);
    assertEquals(2L, sourceContext.getLatestElement().getValue().longValue());
    assertEquals(3L, // the offset in state still should have advanced
    partitionStateHolder.getOffset());
}