Examples with ListShardsRequest com.amazonaws.services.kinesis.model.ListShardsRequest used on opensource projects

Example 11 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project hazelcast by hazelcast.

the class RangeMonitor method listAllShardsRequest.

public static ListShardsRequest listAllShardsRequest(String stream, @Nullable String nextToken, ShardFilterType filterType) {
    ListShardsRequest request = new ListShardsRequest();
    if (nextToken == null) {
        request.setStreamName(stream);
    } else {
        request.setNextToken(nextToken);
    }
    // include all the shards within the retention period of the data stream
    request.setShardFilter(new ShardFilter().withType(filterType));
    return request;
}

Example 12 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project hazelcast by hazelcast.

the class RangeMonitor method listAllShardsAsync.

private Future<ListShardsResult> listAllShardsAsync(String nextToken) {
    ShardFilterType filterType = ShardFilterType.FROM_TRIM_HORIZON;
    // all shards within the retention period (including closed, excluding expired)
    ListShardsRequest request = listAllShardsRequest(streamName, nextToken, filterType);
    return kinesis.listShardsAsync(request);
}

Example 13 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project druid by druid-io.

the class KinesisRecordSupplier method getShards.

/**
 * Use the API listShards which is the recommended way instead of describeStream
 * listShards can return 1000 shards per call and has a limit of 100TPS
 * This makes the method resilient to LimitExceeded exceptions (compared to 100 shards, 10 TPS of describeStream)
 *
 * @param stream name of stream
 * @return Immutable set of shards
 */
public Set<Shard> getShards(String stream) {
    ImmutableSet.Builder<Shard> shards = ImmutableSet.builder();
    ListShardsRequest request = new ListShardsRequest().withStreamName(stream);
    while (true) {
        ListShardsResult result = kinesis.listShards(request);
        shards.addAll(result.getShards());
        String nextToken = result.getNextToken();
        if (nextToken == null) {
            return shards.build();
        }
        request = new ListShardsRequest().withNextToken(nextToken);
    }
}

Example 14 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForExclusiveStartShardId.

@Test
public void shouldListAllShardsForExclusiveStartShardId() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    String exclusiveStartShardId = "exclusiveStartShardId";
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withMaxResults(1_000).withShardFilter(new ShardFilter().withType(ShardFilterType.AFTER_SHARD_ID).withShardId(exclusiveStartShardId)))).thenReturn(new ListShardsResult().withShards(shard1, shard2, shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsFollowingClosedShard(STREAM, exclusiveStartShardId);
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 15 with ListShardsRequest

use of com.amazonaws.services.kinesis.model.ListShardsRequest in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForTrimHorizonWithPagedResults.

@Test
public void shouldListAllShardsForTrimHorizonWithPagedResults() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    ShardFilter shardFilter = new ShardFilter().withType(ShardFilterType.AT_TRIM_HORIZON);
    String nextListShardsToken = "testNextToken";
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withShardFilter(shardFilter).withMaxResults(1_000))).thenReturn(new ListShardsResult().withShards(shard1, shard2).withNextToken(nextListShardsToken));
    when(kinesis.listShards(new ListShardsRequest().withMaxResults(1_000).withShardFilter(shardFilter).withNextToken(nextListShardsToken))).thenReturn(new ListShardsResult().withShards(shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(InitialPositionInStream.TRIM_HORIZON));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}