Examples with ExpiredNextTokenException com.amazonaws.services.kinesis.model.ExpiredNextTokenException used on opensource projects

Example 1 with ExpiredNextTokenException

use of com.amazonaws.services.kinesis.model.ExpiredNextTokenException in project flink by apache.

the class KinesisProxy method listShards.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's
 *     infos will not appear in result)
 * @return the result of the describe stream operation
 */
private ListShardsResult listShards(String streamName, @Nullable String startShardId, @Nullable String startNextToken) throws InterruptedException {
    final ListShardsRequest listShardsRequest = new ListShardsRequest();
    if (startNextToken == null) {
        listShardsRequest.setExclusiveStartShardId(startShardId);
        listShardsRequest.setStreamName(streamName);
    } else {
        // Note the nextToken returned by AWS expires within 300 sec.
        listShardsRequest.setNextToken(startNextToken);
    }
    ListShardsResult listShardsResults = null;
    // Call ListShards, with full-jitter backoff (if we get LimitExceededException).
    int retryCount = 0;
    // are taken up.
    while (retryCount <= listShardsMaxRetries && listShardsResults == null) {
        // retry until we get a result
        try {
            listShardsResults = kinesisClient.listShards(listShardsRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
            LOG.warn("Got LimitExceededException when listing shards from stream " + streamName + ". Backing off for " + backoffMillis + " millis.");
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceInUseException reInUse) {
            if (LOG.isWarnEnabled()) {
                // List Shards will throw an exception if stream in not in active state. Return
                // and re-use previous state available.
                LOG.info("The stream is currently not in active state. Reusing the older state " + "for the time being");
                break;
            }
        } catch (ResourceNotFoundException reNotFound) {
            throw new RuntimeException("Stream not found. Error while getting shard list.", reNotFound);
        } catch (InvalidArgumentException inArg) {
            throw new RuntimeException("Invalid Arguments to listShards.", inArg);
        } catch (ExpiredNextTokenException expiredToken) {
            LOG.warn("List Shards has an expired token. Reusing the previous state.");
            break;
        } catch (SdkClientException ex) {
            if (retryCount < listShardsMaxRetries && isRecoverableSdkClientException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
                LOG.warn("Got SdkClientException when listing shards from stream {}. Backing off for {} millis.", streamName, backoffMillis);
                BACKOFF.sleep(backoffMillis);
            } else {
                // (otherwise would return null result and keep trying forever)
                throw ex;
            }
        }
    }
    // https://github.com/lyft/kinesalite/pull/4
    if (startShardId != null && listShardsResults != null) {
        List<Shard> shards = listShardsResults.getShards();
        shards.removeIf(shard -> StreamShardHandle.compareShardIds(shard.getShardId(), startShardId) <= 0);
    }
    return listShardsResults;
}