Examples with ResourceNotFoundException com.amazonaws.services.kinesis.model.ResourceNotFoundException used on opensource projects

Example 1 with ResourceNotFoundException

use of com.amazonaws.services.kinesis.model.ResourceNotFoundException in project flink by apache.

the class KinesisPubsubClient method createTopic.

public void createTopic(String stream, int shards, Properties props) throws Exception {
    try {
        kinesisClient.describeStream(stream);
        kinesisClient.deleteStream(stream);
    } catch (ResourceNotFoundException rnfe) {
    // expected when stream doesn't exist
    }
    kinesisClient.createStream(stream, shards);
    Deadline deadline = Deadline.fromNow(Duration.ofSeconds(5));
    while (deadline.hasTimeLeft()) {
        try {
            // sleep for a bit for stream to be created
            Thread.sleep(250);
            if (kinesisClient.describeStream(stream).getStreamDescription().getShards().size() != shards) {
                // not fully created yet
                continue;
            }
            break;
        } catch (ResourceNotFoundException rnfe) {
        // not ready yet
        }
    }
}

Example 2 with ResourceNotFoundException

use of com.amazonaws.services.kinesis.model.ResourceNotFoundException in project flink by apache.

the class KinesisProxy method describeStream.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException {
    final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    describeStreamRequest.setExclusiveStartShardId(startShardId);
    DescribeStreamResult describeStreamResult = null;
    // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
    int attemptCount = 0;
    while (describeStreamResult == null) {
        // retry until we get a result
        try {
            describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
            LOG.warn(String.format("Got LimitExceededException when describing stream %s. " + "Backing off for %d millis.", streamName, backoffMillis));
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceNotFoundException re) {
            throw new RuntimeException("Error while getting stream details", re);
        }
    }
    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
        if (LOG.isWarnEnabled()) {
            LOG.warn(String.format("The status of stream %s is %s ; result of the current " + "describeStream operation will not contain any shard information.", streamName, streamStatus));
        }
    }
    return describeStreamResult;
}

Example 3 with ResourceNotFoundException

use of com.amazonaws.services.kinesis.model.ResourceNotFoundException in project flink by apache.

the class KinesisProxy method listShards.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's
 *     infos will not appear in result)
 * @return the result of the describe stream operation
 */
private ListShardsResult listShards(String streamName, @Nullable String startShardId, @Nullable String startNextToken) throws InterruptedException {
    final ListShardsRequest listShardsRequest = new ListShardsRequest();
    if (startNextToken == null) {
        listShardsRequest.setExclusiveStartShardId(startShardId);
        listShardsRequest.setStreamName(streamName);
    } else {
        // Note the nextToken returned by AWS expires within 300 sec.
        listShardsRequest.setNextToken(startNextToken);
    }
    ListShardsResult listShardsResults = null;
    // Call ListShards, with full-jitter backoff (if we get LimitExceededException).
    int retryCount = 0;
    // are taken up.
    while (retryCount <= listShardsMaxRetries && listShardsResults == null) {
        // retry until we get a result
        try {
            listShardsResults = kinesisClient.listShards(listShardsRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
            LOG.warn("Got LimitExceededException when listing shards from stream " + streamName + ". Backing off for " + backoffMillis + " millis.");
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceInUseException reInUse) {
            if (LOG.isWarnEnabled()) {
                // List Shards will throw an exception if stream in not in active state. Return
                // and re-use previous state available.
                LOG.info("The stream is currently not in active state. Reusing the older state " + "for the time being");
                break;
            }
        } catch (ResourceNotFoundException reNotFound) {
            throw new RuntimeException("Stream not found. Error while getting shard list.", reNotFound);
        } catch (InvalidArgumentException inArg) {
            throw new RuntimeException("Invalid Arguments to listShards.", inArg);
        } catch (ExpiredNextTokenException expiredToken) {
            LOG.warn("List Shards has an expired token. Reusing the previous state.");
            break;
        } catch (SdkClientException ex) {
            if (retryCount < listShardsMaxRetries && isRecoverableSdkClientException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
                LOG.warn("Got SdkClientException when listing shards from stream {}. Backing off for {} millis.", streamName, backoffMillis);
                BACKOFF.sleep(backoffMillis);
            } else {
                // (otherwise would return null result and keep trying forever)
                throw ex;
            }
        }
    }
    // https://github.com/lyft/kinesalite/pull/4
    if (startShardId != null && listShardsResults != null) {
        List<Shard> shards = listShardsResults.getShards();
        shards.removeIf(shard -> StreamShardHandle.compareShardIds(shard.getShardId(), startShardId) <= 0);
    }
    return listShardsResults;
}

Example 4 with ResourceNotFoundException

use of com.amazonaws.services.kinesis.model.ResourceNotFoundException in project voltdb by VoltDB.

the class KinesisStreamImporterConfig method discoverShards.

/**
     * connect to kinesis stream to discover the shards on the stream
     *
     * @param regionName The region name where the stream resides
     * @param streamName The kinesis stream name
     * @param accessKey The user access key
     * @param secretKey The user secret key
     * @param appName  The name of stream application
     * @return a list of shards
     */
public static List<Shard> discoverShards(String regionName, String streamName, String accessKey, String secretKey, String appName) {
    try {
        Region region = RegionUtils.getRegion(regionName);
        if (region != null) {
            final AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
            AmazonKinesis kinesisClient = new AmazonKinesisClient(credentials, getClientConfigWithUserAgent(appName));
            kinesisClient.setRegion(region);
            DescribeStreamResult result = kinesisClient.describeStream(streamName);
            if (!"ACTIVE".equals(result.getStreamDescription().getStreamStatus())) {
                throw new IllegalArgumentException("Kinesis stream " + streamName + " is not active.");
            }
            return result.getStreamDescription().getShards();
        }
    } catch (ResourceNotFoundException e) {
        LOGGER.warn("Kinesis stream " + streamName + " does not exist.", e);
    } catch (Exception e) {
        LOGGER.warn("Error found while describing the kinesis stream " + streamName, e);
    }
    return null;
}

Example 5 with ResourceNotFoundException

use of com.amazonaws.services.kinesis.model.ResourceNotFoundException in project spring-integration-aws by spring-projects.

the class KinesisMessageDrivenChannelAdapter method populateShardsForStream.

private void populateShardsForStream(final String stream, final CountDownLatch shardsGatherLatch) {
    this.dispatcherExecutor.execute(() -> {
        try {
            int describeStreamRetries = 0;
            List<Shard> shardsToConsume = new ArrayList<>();
            String exclusiveStartShardId = null;
            while (true) {
                DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream).withExclusiveStartShardId(exclusiveStartShardId);
                DescribeStreamResult describeStreamResult = null;
                // Call DescribeStream, with backoff and retries (if we get LimitExceededException).
                try {
                    describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
                } catch (Exception e) {
                    logger.info("Got an exception when describing stream [" + stream + "]. " + "Backing off for [" + this.describeStreamBackoff + "] millis.", e);
                }
                if (describeStreamResult == null || !StreamStatus.ACTIVE.toString().equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
                    if (describeStreamRetries++ > this.describeStreamRetries) {
                        ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException("The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
                        resourceNotFoundException.setServiceName("Kinesis");
                        throw resourceNotFoundException;
                    }
                    try {
                        Thread.sleep(this.describeStreamBackoff);
                        continue;
                    } catch (InterruptedException e) {
                        Thread.interrupted();
                        throw new IllegalStateException("The [describeStream] thread for the stream [" + stream + "] has been interrupted.", e);
                    }
                }
                List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
                try {
                    for (Shard shard : shards) {
                        String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber();
                        if (endingSequenceNumber != null) {
                            String key = buildCheckpointKeyForShard(stream, shard.getShardId());
                            String checkpoint = this.checkpointStore.get(key);
                            boolean skipClosedShard = checkpoint != null && new BigInteger(endingSequenceNumber).compareTo(new BigInteger(checkpoint)) <= 0;
                            if (logger.isTraceEnabled()) {
                                logger.trace("The shard [" + shard + "] in stream [" + stream + "] is closed CLOSED with endingSequenceNumber [" + endingSequenceNumber + "].\nThe last processed checkpoint is [" + checkpoint + "]." + (skipClosedShard ? "\nThe shard will be skipped." : ""));
                            }
                            if (skipClosedShard) {
                                // Skip CLOSED shard which has been read before according a checkpoint
                                continue;
                            }
                        }
                        shardsToConsume.add(shard);
                    }
                } catch (Exception e) {
                    logger.info("Got an exception when processing shards in stream [" + stream + "].\n" + "Retrying...", e);
                    continue;
                }
                if (describeStreamResult.getStreamDescription().getHasMoreShards()) {
                    exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
                    describeStreamRetries = 0;
                } else {
                    break;
                }
            }
            for (Shard shard : shardsToConsume) {
                KinesisShardOffset shardOffset = new KinesisShardOffset(this.streamInitialSequence);
                shardOffset.setShard(shard.getShardId());
                shardOffset.setStream(stream);
                boolean addedOffset;
                synchronized (this.shardOffsets) {
                    addedOffset = this.shardOffsets.add(shardOffset);
                }
                if (addedOffset && shardsGatherLatch == null && this.active) {
                    populateConsumer(shardOffset);
                }
            }
        } finally {
            if (shardsGatherLatch != null) {
                shardsGatherLatch.countDown();
            }
            this.inResharding.remove(stream);
        }
    });
}