Examples with LimitExceededException com.amazonaws.services.kinesis.model.LimitExceededException used on opensource projects

Example 1 with LimitExceededException

use of com.amazonaws.services.kinesis.model.LimitExceededException in project flink by apache.

the class KinesisProxy method describeStream.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException {
    final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    describeStreamRequest.setExclusiveStartShardId(startShardId);
    DescribeStreamResult describeStreamResult = null;
    // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
    int attemptCount = 0;
    while (describeStreamResult == null) {
        // retry until we get a result
        try {
            describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
            LOG.warn(String.format("Got LimitExceededException when describing stream %s. " + "Backing off for %d millis.", streamName, backoffMillis));
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceNotFoundException re) {
            throw new RuntimeException("Error while getting stream details", re);
        }
    }
    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
        if (LOG.isWarnEnabled()) {
            LOG.warn(String.format("The status of stream %s is %s ; result of the current " + "describeStream operation will not contain any shard information.", streamName, streamStatus));
        }
    }
    return describeStreamResult;
}

Example 2 with LimitExceededException

use of com.amazonaws.services.kinesis.model.LimitExceededException in project flink by apache.

the class KinesisProxy method listShards.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's
 *     infos will not appear in result)
 * @return the result of the describe stream operation
 */
private ListShardsResult listShards(String streamName, @Nullable String startShardId, @Nullable String startNextToken) throws InterruptedException {
    final ListShardsRequest listShardsRequest = new ListShardsRequest();
    if (startNextToken == null) {
        listShardsRequest.setExclusiveStartShardId(startShardId);
        listShardsRequest.setStreamName(streamName);
    } else {
        // Note the nextToken returned by AWS expires within 300 sec.
        listShardsRequest.setNextToken(startNextToken);
    }
    ListShardsResult listShardsResults = null;
    // Call ListShards, with full-jitter backoff (if we get LimitExceededException).
    int retryCount = 0;
    // are taken up.
    while (retryCount <= listShardsMaxRetries && listShardsResults == null) {
        // retry until we get a result
        try {
            listShardsResults = kinesisClient.listShards(listShardsRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
            LOG.warn("Got LimitExceededException when listing shards from stream " + streamName + ". Backing off for " + backoffMillis + " millis.");
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceInUseException reInUse) {
            if (LOG.isWarnEnabled()) {
                // List Shards will throw an exception if stream in not in active state. Return
                // and re-use previous state available.
                LOG.info("The stream is currently not in active state. Reusing the older state " + "for the time being");
                break;
            }
        } catch (ResourceNotFoundException reNotFound) {
            throw new RuntimeException("Stream not found. Error while getting shard list.", reNotFound);
        } catch (InvalidArgumentException inArg) {
            throw new RuntimeException("Invalid Arguments to listShards.", inArg);
        } catch (ExpiredNextTokenException expiredToken) {
            LOG.warn("List Shards has an expired token. Reusing the previous state.");
            break;
        } catch (SdkClientException ex) {
            if (retryCount < listShardsMaxRetries && isRecoverableSdkClientException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
                LOG.warn("Got SdkClientException when listing shards from stream {}. Backing off for {} millis.", streamName, backoffMillis);
                BACKOFF.sleep(backoffMillis);
            } else {
                // (otherwise would return null result and keep trying forever)
                throw ex;
            }
        }
    }
    // https://github.com/lyft/kinesalite/pull/4
    if (startShardId != null && listShardsResults != null) {
        List<Shard> shards = listShardsResults.getShards();
        shards.removeIf(shard -> StreamShardHandle.compareShardIds(shard.getShardId(), startShardId) <= 0);
    }
    return listShardsResults;
}

Example 3 with LimitExceededException

use of com.amazonaws.services.kinesis.model.LimitExceededException in project beam by apache.

the class AmazonKinesisMock method listShards.

@Override
public ListShardsResult listShards(ListShardsRequest listShardsRequest) {
    if (expectedListShardsLimitExceededException) {
        throw new LimitExceededException("ListShards rate limit exceeded");
    }
    ListShardsResult result = new ListShardsResult();
    List<Shard> shards = IntStream.range(0, shardedData.size()).boxed().map(i -> new Shard().withShardId(Integer.toString(i))).collect(Collectors.toList());
    result.setShards(shards);
    HttpResponse response = new HttpResponse(null, null);
    response.setStatusCode(200);
    result.setSdkHttpMetadata(SdkHttpMetadata.from(response));
    return result;
}

Example 4 with LimitExceededException

use of com.amazonaws.services.kinesis.model.LimitExceededException in project beam by apache.

the class SimplifiedKinesisClient method describeStreamSummary.

private StreamDescriptionSummary describeStreamSummary(final String streamName) throws IOException, InterruptedException {
    // DescribeStreamSummary has limits that can be hit fairly easily if we are attempting
    // to configure multiple KinesisIO inputs in the same account. Retry up to
    // DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS times if we end up hitting that limit.
    // 
    // Only pass the wrapped exception up once that limit is reached. Use FluentBackoff
    // to implement the retry policy.
    FluentBackoff retryBackoff = FluentBackoff.DEFAULT.withMaxRetries(DESCRIBE_STREAM_SUMMARY_MAX_ATTEMPTS).withInitialBackoff(DESCRIBE_STREAM_SUMMARY_INITIAL_BACKOFF);
    BackOff backoff = retryBackoff.backoff();
    Sleeper sleeper = Sleeper.DEFAULT;
    DescribeStreamSummaryRequest request = new DescribeStreamSummaryRequest();
    request.setStreamName(streamName);
    while (true) {
        try {
            return kinesis.describeStreamSummary(request).getStreamDescriptionSummary();
        } catch (LimitExceededException exc) {
            if (!BackOffUtils.next(sleeper, backoff)) {
                throw exc;
            }
        }
    }
}

Example 5 with LimitExceededException

use of com.amazonaws.services.kinesis.model.LimitExceededException in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForTimestampWithRetriedDescribeStreamSummaryCallAfterStreamCreationTimestamp.

@Test
public void shouldListAllShardsForTimestampWithRetriedDescribeStreamSummaryCallAfterStreamCreationTimestamp() throws TransientKinesisException {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    int hoursDifference = 1;
    int retentionPeriodHours = hoursDifference * 3;
    Instant streamCreationTimestamp = CURRENT_TIMESTAMP.minus(Duration.standardHours(retentionPeriodHours));
    Instant startingPointTimestamp = streamCreationTimestamp.plus(Duration.standardHours(hoursDifference));
    when(currentInstantSupplier.get()).thenReturn(CURRENT_TIMESTAMP);
    when(kinesis.describeStreamSummary(new DescribeStreamSummaryRequest().withStreamName(STREAM))).thenThrow(new LimitExceededException("Fake Exception: Limit exceeded")).thenReturn(new DescribeStreamSummaryResult().withStreamDescriptionSummary(new StreamDescriptionSummary().withRetentionPeriodHours(retentionPeriodHours).withStreamCreationTimestamp(streamCreationTimestamp.toDate())));
    ShardFilter shardFilter = new ShardFilter().withType(ShardFilterType.AT_TIMESTAMP).withTimestamp(startingPointTimestamp.toDate());
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withShardFilter(shardFilter).withMaxResults(1_000))).thenReturn(new ListShardsResult().withShards(shard1, shard2, shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(startingPointTimestamp));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}