Examples with LimitExceededException software.amazon.awssdk.services.kinesis.model.LimitExceededException used on opensource projects

Example 1 with LimitExceededException

use of software.amazon.awssdk.services.kinesis.model.LimitExceededException in project beam by apache.

the class SimplifiedKinesisClient method describeStreamSummary.

private StreamDescriptionSummary describeStreamSummary(final String streamName) throws IOException, InterruptedException {
    // DescribeStreamSummary has limits that can be hit fairly easily if we are attempting
    // to configure multiple KinesisIO inputs in the same account. Retry up to
    // DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS times if we end up hitting that limit.
    // 
    // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
    // to implement the retry policy.
    FluentBackoff retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS).withInitialBackoff(DESCRIBE_STREAM_SUMMARY_INITIAL_BACKOFF);
    BackOff backoff = retryBackoff.backoff();
    Sleeper sleeper = Sleeper.DEFAULT;
    DescribeStreamSummaryRequest request = DescribeStreamSummaryRequest.builder().streamName(streamName).build();
    while (true) {
        try {
            return kinesis.describeStreamSummary(request).streamDescriptionSummary();
        } catch (LimitExceededException exc) {
            if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
            }
        }
    }
}

Example 2 with LimitExceededException

use of software.amazon.awssdk.services.kinesis.model.LimitExceededException in project flink by apache.

the class FanOutRecordPublisherTest method testSubscribeToShardFailsWhenMaxRetriesExceeded.

@Test
public void testSubscribeToShardFailsWhenMaxRetriesExceeded() throws Exception {
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("Maximum retries exceeded for SubscribeToShard. Failed 3 times.");
    Properties efoProperties = createEfoProperties();
    efoProperties.setProperty(SUBSCRIBE_TO_SHARD_RETRIES, String.valueOf(EXPECTED_SUBSCRIBE_TO_SHARD_RETRIES));
    FanOutRecordPublisherConfiguration configuration = new FanOutRecordPublisherConfiguration(efoProperties, emptyList());
    LimitExceededException retryableError = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(retryableError);
    FullJitterBackoff backoff = mock(FullJitterBackoff.class);
    FanOutRecordPublisher recordPublisher = new FanOutRecordPublisher(latest(), "arn", createDummyStreamShardHandle(), kinesis, configuration, backoff);
    int count = 0;
    while (recordPublisher.run(new TestConsumer()) == INCOMPLETE) {
        if (++count > EXPECTED_SUBSCRIBE_TO_SHARD_RETRIES) {
            break;
        }
    }
}

Example 3 with LimitExceededException

use of software.amazon.awssdk.services.kinesis.model.LimitExceededException in project flink by apache.

the class FanOutRecordPublisherTest method testShardConsumerRetriesIfLimitExceededExceptionThrownFromSubscription.

@Test
public void testShardConsumerRetriesIfLimitExceededExceptionThrownFromSubscription() throws Exception {
    LimitExceededException exception = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(exception);
    RecordPublisher recordPublisher = createRecordPublisher(kinesis);
    TestConsumer consumer = new TestConsumer();
    RecordPublisherRunResult result = recordPublisher.run(consumer);
    // An exception is thrown after the 5th record in each subscription, therefore we expect to
    // receive 5 records
    assertEquals(5, consumer.getRecordBatches().size());
    assertEquals(1, kinesis.getNumberOfSubscribeToShardInvocations());
    // INCOMPLETE is returned to indicate the shard is not complete
    assertEquals(INCOMPLETE, result);
}

Example 4 with LimitExceededException

use of software.amazon.awssdk.services.kinesis.model.LimitExceededException in project flink by apache.

the class FanOutRecordPublisherTest method testSubscribeToShardBacksOffAttemptIncreases.

@Test
public void testSubscribeToShardBacksOffAttemptIncreases() throws Exception {
    LimitExceededException retryableError = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(retryableError);
    FanOutRecordPublisherConfiguration configuration = createConfiguration();
    FullJitterBackoff backoff = mock(FullJitterBackoff.class);
    FanOutRecordPublisher recordPublisher = new FanOutRecordPublisher(latest(), "arn", createDummyStreamShardHandle(), kinesis, configuration, backoff);
    recordPublisher.run(new TestConsumer());
    recordPublisher.run(new TestConsumer());
    recordPublisher.run(new TestConsumer());
    verify(backoff).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(1));
    verify(backoff).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(2));
    verify(backoff).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(3));
    verify(backoff, never()).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(0));
    verify(backoff, never()).calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), eq(4));
}

Example 5 with LimitExceededException

use of software.amazon.awssdk.services.kinesis.model.LimitExceededException in project flink by apache.

the class FanOutRecordPublisherTest method testSubscribeToShardBacksOffForRetryableError.

@Test
public void testSubscribeToShardBacksOffForRetryableError() throws Exception {
    LimitExceededException retryableError = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(retryableError);
    FanOutRecordPublisherConfiguration configuration = createConfiguration();
    FullJitterBackoff backoff = mock(FullJitterBackoff.class);
    when(backoff.calculateFullJitterBackoff(anyLong(), anyLong(), anyDouble(), anyInt())).thenReturn(100L);
    new FanOutRecordPublisher(latest(), "arn", createDummyStreamShardHandle(), kinesis, configuration, backoff).run(new TestConsumer());
    verify(backoff).calculateFullJitterBackoff(EXPECTED_SUBSCRIBE_TO_SHARD_BASE, EXPECTED_SUBSCRIBE_TO_SHARD_MAX, EXPECTED_SUBSCRIBE_TO_SHARD_POW, 1);
    verify(backoff).sleep(100L);
}