Examples with TestConsumer org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer used on opensource projects

Example 16 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV2StartingPositionAtSequenceNumber.

@Test
public void testToSdkV2StartingPositionAtSequenceNumber() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = emptyShard();
    RecordPublisher publisher = createRecordPublisher(kinesis, StartingPosition.restartFromSequenceNumber(AGGREGATED_SEQUENCE_NUMBER));
    publisher.run(new TestConsumer());
    assertEquals(DUMMY_SEQUENCE, kinesis.getStartingPositionForSubscription(0).sequenceNumber());
    assertEquals(AT_SEQUENCE_NUMBER, kinesis.getStartingPositionForSubscription(0).type());
}

Example 17 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testShardConsumerRetriesIfLimitExceededExceptionThrownFromSubscription.

@Test
public void testShardConsumerRetriesIfLimitExceededExceptionThrownFromSubscription() throws Exception {
    LimitExceededException exception = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(exception);
    RecordPublisher recordPublisher = createRecordPublisher(kinesis);
    TestConsumer consumer = new TestConsumer();
    RecordPublisherRunResult result = recordPublisher.run(consumer);
    // An exception is thrown after the 5th record in each subscription, therefore we expect to
    // receive 5 records
    assertEquals(5, consumer.getRecordBatches().size());
    assertEquals(1, kinesis.getNumberOfSubscribeToShardInvocations());
    // INCOMPLETE is returned to indicate the shard is not complete
    assertEquals(INCOMPLETE, result);
}

Example 18 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testSubscribeToShardBacksOffAttemptIncreases.

@Test
public void testSubscribeToShardBacksOffAttemptIncreases() throws Exception {
    LimitExceededException retryableError = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(retryableError);
    FanOutRecordPublisherConfiguration configuration = createConfiguration();
    FullJitterBackoff backoff = mock(FullJitterBackoff.class);
    FanOutRecordPublisher recordPublisher = new FanOutRecordPublisher(latest(), "arn", createDummyStreamShardHandle(), kinesis, configuration, backoff);
    recordPublisher.run(new TestConsumer());
    recordPublisher.run(new TestConsumer());
    recordPublisher.run(new TestConsumer());
    verify(backoff).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(1));
    verify(backoff).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(2));
    verify(backoff).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(3));
    verify(backoff, never()).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(0));
    verify(backoff, never()).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(4));
}

Example 19 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV2StartingPositionAfterSequenceNumber.

@Test
public void testToSdkV2StartingPositionAfterSequenceNumber() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = emptyShard();
    RecordPublisher publisher = createRecordPublisher(kinesis, StartingPosition.continueFromSequenceNumber(SEQUENCE_NUMBER));
    publisher.run(new TestConsumer());
    assertEquals(DUMMY_SEQUENCE, kinesis.getStartingPositionForSubscription(0).sequenceNumber());
    assertEquals(AFTER_SEQUENCE_NUMBER, kinesis.getStartingPositionForSubscription(0).type());
}

Example 20 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testSubscribeToShardBacksOffForRetryableError.

@Test
public void testSubscribeToShardBacksOffForRetryableError() throws Exception {
    LimitExceededException retryableError = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(retryableError);
    FanOutRecordPublisherConfiguration configuration = createConfiguration();
    FullJitterBackoff backoff = mock(FullJitterBackoff.class);
    when(backoff.calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), anyInt())).thenReturn(100L);
    new FanOutRecordPublisher(latest(), "arn", createDummyStreamShardHandle(), kinesis, configuration, backoff).run(new TestConsumer());
    verify(backoff).calculateFullJitterBackoff(EXPECTED_SUBSCRIBE_TO_SHARD_BASE, EXPECTED_SUBSCRIBE_TO_SHARD_MAX, EXPECTED_SUBSCRIBE_TO_SHARD_POW, 1);
    verify(backoff).sleep(100L);
}