Examples with ListShardsRequest com.amazonaws.services.kinesis.model.ListShardsRequest used on opensource projects

Example 1 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project flink by apache.

the class KinesisProxy method listShards.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's
 *     infos will not appear in result)
 * @return the result of the describe stream operation
 */
private ListShardsResult listShards(String streamName, @Nullable String startShardId, @Nullable String startNextToken) throws InterruptedException {
    final ListShardsRequest listShardsRequest = new ListShardsRequest();
    if (startNextToken == null) {
        listShardsRequest.setExclusiveStartShardId(startShardId);
        listShardsRequest.setStreamName(streamName);
    } else {
        // Note the nextToken returned by AWS expires within 300 sec.
        listShardsRequest.setNextToken(startNextToken);
    }
    ListShardsResult listShardsResults = null;
    // Call ListShards, with full-jitter backoff (if we get LimitExceededException).
    int retryCount = 0;
    // are taken up.
    while (retryCount <= listShardsMaxRetries && listShardsResults == null) {
        // retry until we get a result
        try {
            listShardsResults = kinesisClient.listShards(listShardsRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
            LOG.warn("Got LimitExceededException when listing shards from stream " + streamName + ". Backing off for " + backoffMillis + " millis.");
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceInUseException reInUse) {
            if (LOG.isWarnEnabled()) {
                // List Shards will throw an exception if stream in not in active state. Return
                // and re-use previous state available.
                LOG.info("The stream is currently not in active state. Reusing the older state " + "for the time being");
                break;
            }
        } catch (ResourceNotFoundException reNotFound) {
            throw new RuntimeException("Stream not found. Error while getting shard list.", reNotFound);
        } catch (InvalidArgumentException inArg) {
            throw new RuntimeException("Invalid Arguments to listShards.", inArg);
        } catch (ExpiredNextTokenException expiredToken) {
            LOG.warn("List Shards has an expired token. Reusing the previous state.");
            break;
        } catch (SdkClientException ex) {
            if (retryCount < listShardsMaxRetries && isRecoverableSdkClientException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
                LOG.warn("Got SdkClientException when listing shards from stream {}. Backing off for {} millis.", streamName, backoffMillis);
                BACKOFF.sleep(backoffMillis);
            } else {
                // (otherwise would return null result and keep trying forever)
                throw ex;
            }
        }
    }
    // https://github.com/lyft/kinesalite/pull/4
    if (startShardId != null && listShardsResults != null) {
        List<Shard> shards = listShardsResults.getShards();
        shards.removeIf(shard -> StreamShardHandle.compareShardIds(shard.getShardId(), startShardId) <= 0);
    }
    return listShardsResults;
}

Example 2 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project beam by apache.

the class AmazonKinesisMock method listShards.

@Override
public ListShardsResult listShards(ListShardsRequest listShardsRequest) {
    if (expectedListShardsLimitExceededException) {
        throw new LimitExceededException("ListShards rate limit exceeded");
    }
    ListShardsResult result = new ListShardsResult();
    List<Shard> shards = IntStream.range(0, shardedData.size()).boxed().map(i -> new Shard().withShardId(Integer.toString(i))).collect(Collectors.toList());
    result.setShards(shards);
    HttpResponse response = new HttpResponse(null, null);
    response.setStatusCode(200);
    result.setSdkHttpMetadata(SdkHttpMetadata.from(response));
    return result;
}

Example 3 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project beam by apache.

the class SimplifiedKinesisClient method listShards.

private List<Shard> listShards(final String streamName, final ShardFilter shardFilter) throws TransientKinesisException {
    return wrapExceptions(() -> {
        ImmutableList.Builder<Shard> shardsBuilder = ImmutableList.builder();
        String currentNextToken = null;
        do {
            ListShardsRequest request = new ListShardsRequest();
            request.setMaxResults(LIST_SHARDS_MAX_RESULTS);
            if (currentNextToken != null) {
                request.setNextToken(currentNextToken);
            } else {
                request.setStreamName(streamName);
            }
            request.setShardFilter(shardFilter);
            ListShardsResult response = kinesis.listShards(request);
            List<Shard> shards = response.getShards();
            shardsBuilder.addAll(shards);
            currentNextToken = response.getNextToken();
        } while (currentNextToken != null);
        return shardsBuilder.build();
    });
}

Example 4 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForTimestampWithRetriedDescribeStreamSummaryCallAfterStreamCreationTimestamp.

@Test
public void shouldListAllShardsForTimestampWithRetriedDescribeStreamSummaryCallAfterStreamCreationTimestamp() throws TransientKinesisException {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    int hoursDifference = 1;
    int retentionPeriodHours = hoursDifference * 3;
    Instant streamCreationTimestamp = CURRENT_TIMESTAMP.minus(Duration.standardHours(retentionPeriodHours));
    Instant startingPointTimestamp = streamCreationTimestamp.plus(Duration.standardHours(hoursDifference));
    when(currentInstantSupplier.get()).thenReturn(CURRENT_TIMESTAMP);
    when(kinesis.describeStreamSummary(new DescribeStreamSummaryRequest().withStreamName(STREAM))).thenThrow(new LimitExceededException("Fake Exception: Limit exceeded")).thenReturn(new DescribeStreamSummaryResult().withStreamDescriptionSummary(new StreamDescriptionSummary().withRetentionPeriodHours(retentionPeriodHours).withStreamCreationTimestamp(streamCreationTimestamp.toDate())));
    ShardFilter shardFilter = new ShardFilter().withType(ShardFilterType.AT_TIMESTAMP).withTimestamp(startingPointTimestamp.toDate());
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withShardFilter(shardFilter).withMaxResults(1_000))).thenReturn(new ListShardsResult().withShards(shard1, shard2, shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(startingPointTimestamp));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 5 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForTimestampOutsideStreamRetentionAfterStreamCreationTimestamp.

@Test
public void shouldListAllShardsForTimestampOutsideStreamRetentionAfterStreamCreationTimestamp() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    int retentionPeriodHours = 3;
    int startingPointHours = 5;
    int hoursSinceStreamCreation = 6;
    Instant streamCreationTimestamp = CURRENT_TIMESTAMP.minus(Duration.standardHours(hoursSinceStreamCreation));
    Instant startingPointTimestampAfterStreamRetentionTimestamp = CURRENT_TIMESTAMP.minus(Duration.standardHours(startingPointHours));
    when(currentInstantSupplier.get()).thenReturn(CURRENT_TIMESTAMP);
    DescribeStreamSummaryRequest describeStreamRequest = new DescribeStreamSummaryRequest().withStreamName(STREAM);
    when(kinesis.describeStreamSummary(describeStreamRequest)).thenReturn(new DescribeStreamSummaryResult().withStreamDescriptionSummary(new StreamDescriptionSummary().withRetentionPeriodHours(retentionPeriodHours).withStreamCreationTimestamp(streamCreationTimestamp.toDate())));
    ShardFilter shardFilter = new ShardFilter().withType(ShardFilterType.AT_TRIM_HORIZON);
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withShardFilter(shardFilter).withMaxResults(1_000))).thenReturn(new ListShardsResult().withShards(shard1, shard2, shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(startingPointTimestampAfterStreamRetentionTimestamp));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}