Examples with Shard software.amazon.awssdk.services.kinesis.model.Shard used on opensource projects

Example 1 with Shard

use of software.amazon.awssdk.services.kinesis.model.Shard in project beam by apache.

the class ShardRecordsIterator method findSuccessiveShardRecordIterators.

List<ShardRecordsIterator> findSuccessiveShardRecordIterators() throws TransientKinesisException {
    List<Shard> shards = kinesis.listShardsFollowingClosedShard(streamName, shardId);
    List<ShardRecordsIterator> successiveShardRecordIterators = new ArrayList<>();
    for (Shard shard : shards) {
        if (shardId.equals(shard.parentShardId())) {
            ShardCheckpoint shardCheckpoint = new ShardCheckpoint(streamName, shard.shardId(), new StartingPoint(InitialPositionInStream.TRIM_HORIZON));
            successiveShardRecordIterators.add(new ShardRecordsIterator(shardCheckpoint, kinesis, watermarkPolicyFactory));
        }
    }
    return successiveShardRecordIterators;
}

Example 2 with Shard

use of software.amazon.awssdk.services.kinesis.model.Shard in project beam by apache.

the class SimplifiedKinesisClient method listShards.

private List<Shard> listShards(final String streamName, final ShardFilter shardFilter) throws TransientKinesisException {
    return wrapExceptions(() -> {
        ImmutableList.Builder<Shard> shardsBuilder = ImmutableList.builder();
        String currentNextToken = null;
        do {
            ListShardsRequest.Builder reqBuilder = ListShardsRequest.builder().maxResults(LIST_SHARDS_MAX_RESULTS).shardFilter(shardFilter);
            if (currentNextToken != null) {
                reqBuilder.nextToken(currentNextToken);
            } else {
                reqBuilder.streamName(streamName);
            }
            ListShardsResponse response = kinesis.listShards(reqBuilder.build());
            shardsBuilder.addAll(response.shards());
            currentNextToken = response.nextToken();
        } while (currentNextToken != null);
        return shardsBuilder.build();
    });
}

Example 3 with Shard

use of software.amazon.awssdk.services.kinesis.model.Shard in project beam by apache.

the class KinesisIOReadTest method testRecords.

private List<List<Record>> testRecords(int shards, int events) {
    final Instant now = DateTime.now().toInstant();
    Function<Integer, List<Record>> dataStream = shard -> range(0, events).mapToObj(off -> record(now, shard, off)).collect(toList());
    return range(0, shards).boxed().map(dataStream).collect(toList());
}

Example 4 with Shard

use of software.amazon.awssdk.services.kinesis.model.Shard in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForTimestampOutsideStreamRetentionAfterStreamCreationTimestamp.

@Test
public void shouldListAllShardsForTimestampOutsideStreamRetentionAfterStreamCreationTimestamp() throws Exception {
    Shard shard1 = Shard.builder().shardId(SHARD_1).build();
    Shard shard2 = Shard.builder().shardId(SHARD_2).build();
    Shard shard3 = Shard.builder().shardId(SHARD_3).build();
    int retentionPeriodHours = 3;
    int startingPointHours = 5;
    int hoursSinceStreamCreation = 6;
    Instant streamCreationTimestamp = CURRENT_TIMESTAMP.minus(Duration.standardHours(hoursSinceStreamCreation));
    Instant startingPointTimestampAfterStreamRetentionTimestamp = CURRENT_TIMESTAMP.minus(Duration.standardHours(startingPointHours));
    when(currentInstantSupplier.get()).thenReturn(CURRENT_TIMESTAMP);
    DescribeStreamSummaryRequest describeStreamRequest = DescribeStreamSummaryRequest.builder().streamName(STREAM).build();
    when(kinesis.describeStreamSummary(describeStreamRequest)).thenReturn(DescribeStreamSummaryResponse.builder().streamDescriptionSummary(StreamDescriptionSummary.builder().retentionPeriodHours(retentionPeriodHours).streamCreationTimestamp(TimeUtil.toJava(streamCreationTimestamp)).build()).build());
    ShardFilter shardFilter = ShardFilter.builder().type(ShardFilterType.AT_TRIM_HORIZON).build();
    when(kinesis.listShards(ListShardsRequest.builder().streamName(STREAM).shardFilter(shardFilter).maxResults(1_000).build())).thenReturn(ListShardsResponse.builder().shards(shard1, shard2, shard3).nextToken(null).build());
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(startingPointTimestampAfterStreamRetentionTimestamp));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 5 with Shard

use of software.amazon.awssdk.services.kinesis.model.Shard in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForLatest.

@Test
public void shouldListAllShardsForLatest() throws Exception {
    Shard shard1 = Shard.builder().shardId(SHARD_1).build();
    Shard shard2 = Shard.builder().shardId(SHARD_2).build();
    Shard shard3 = Shard.builder().shardId(SHARD_3).build();
    when(kinesis.listShards(ListShardsRequest.builder().streamName(STREAM).shardFilter(ShardFilter.builder().type(ShardFilterType.AT_LATEST).build()).maxResults(1_000).build())).thenReturn(ListShardsResponse.builder().shards(shard1, shard2, shard3).nextToken(null).build());
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(InitialPositionInStream.LATEST));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}