Examples with ListShardsResult com.amazonaws.services.kinesis.model.ListShardsResult used on opensource projects

Example 16 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project flink by apache.

the class KinesisProxyTest method testGetShardListWithNewShardsOnSecondRun.

@Test
public void testGetShardListWithNewShardsOnSecondRun() throws Exception {
    // given
    List<String> shardIds = Arrays.asList(KinesisShardIdGenerator.generateFromShardOrder(0), KinesisShardIdGenerator.generateFromShardOrder(1));
    String fakeStreamName = "fake-stream";
    List<Shard> shards = shardIds.stream().map(shardId -> new Shard().withShardId(shardId)).collect(Collectors.toList());
    AmazonKinesis mockClient = mock(AmazonKinesis.class);
    KinesisProxy kinesisProxy = getProxy(mockClient);
    ListShardsResult responseFirst = new ListShardsResult().withShards(shards).withNextToken(null);
    doReturn(responseFirst).when(mockClient).listShards(argThat(initialListShardsRequestMatcher()));
    HashMap<String, String> streamHashMap = createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList(fakeStreamName));
    // when
    GetShardListResult shardListResult = kinesisProxy.getShardList(streamHashMap);
    // then
    Assert.assertTrue(shardListResult.hasRetrievedShards());
    Set<String> expectedStreams = new HashSet<>();
    expectedStreams.add(fakeStreamName);
    Assert.assertEquals(shardListResult.getStreamsWithRetrievedShards(), expectedStreams);
    List<StreamShardHandle> actualShardList = shardListResult.getRetrievedShardListOfStream(fakeStreamName);
    Assert.assertThat(actualShardList, hasSize(2));
    List<StreamShardHandle> expectedStreamShard = IntStream.range(0, actualShardList.size()).mapToObj(i -> new StreamShardHandle(fakeStreamName, new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(i)))).collect(Collectors.toList());
    Assert.assertThat(actualShardList, containsInAnyOrder(expectedStreamShard.toArray(new StreamShardHandle[actualShardList.size()])));
    // given new shards
    ListShardsResult responseSecond = new ListShardsResult().withShards(new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))).withNextToken(null);
    doReturn(responseSecond).when(mockClient).listShards(argThat(initialListShardsRequestMatcher()));
    // when new shards
    GetShardListResult newShardListResult = kinesisProxy.getShardList(streamHashMap);
    // then new shards
    Assert.assertTrue(newShardListResult.hasRetrievedShards());
    Assert.assertEquals(newShardListResult.getStreamsWithRetrievedShards(), expectedStreams);
    List<StreamShardHandle> newActualShardList = newShardListResult.getRetrievedShardListOfStream(fakeStreamName);
    Assert.assertThat(newActualShardList, hasSize(1));
    List<StreamShardHandle> newExpectedStreamShard = Collections.singletonList(new StreamShardHandle(fakeStreamName, new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))));
    Assert.assertThat(newActualShardList, containsInAnyOrder(newExpectedStreamShard.toArray(new StreamShardHandle[newActualShardList.size()])));
}

Example 17 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForExclusiveStartShardId.

@Test
public void shouldListAllShardsForExclusiveStartShardId() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    String exclusiveStartShardId = "exclusiveStartShardId";
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withMaxResults(1_000).withShardFilter(new ShardFilter().withType(ShardFilterType.AFTER_SHARD_ID).withShardId(exclusiveStartShardId)))).thenReturn(new ListShardsResult().withShards(shard1, shard2, shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsFollowingClosedShard(STREAM, exclusiveStartShardId);
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 18 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShardsForTrimHorizonWithPagedResults.

@Test
public void shouldListAllShardsForTrimHorizonWithPagedResults() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    ShardFilter shardFilter = new ShardFilter().withType(ShardFilterType.AT_TRIM_HORIZON);
    String nextListShardsToken = "testNextToken";
    when(kinesis.listShards(new ListShardsRequest().withStreamName(STREAM).withShardFilter(shardFilter).withMaxResults(1_000))).thenReturn(new ListShardsResult().withShards(shard1, shard2).withNextToken(nextListShardsToken));
    when(kinesis.listShards(new ListShardsRequest().withMaxResults(1_000).withShardFilter(shardFilter).withNextToken(nextListShardsToken))).thenReturn(new ListShardsResult().withShards(shard3).withNextToken(null));
    List<Shard> shards = underTest.listShardsAtPoint(STREAM, new StartingPoint(InitialPositionInStream.TRIM_HORIZON));
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 19 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project hazelcast by hazelcast.

the class RangeMonitor method run.

public void run() {
    long currentTimeMs = System.currentTimeMillis();
    if (listShardsResult == null) {
        initShardListing(currentTimeMs);
    } else {
        if (listShardsResult.isDone()) {
            ListShardsResult result;
            try {
                result = KinesisUtil.readResult(listShardsResult);
            } catch (SdkClientException e) {
                dealWithListShardsFailure(e);
                return;
            } catch (Throwable t) {
                throw rethrow(t);
            } finally {
                listShardsResult = null;
            }
            listShardRetryTracker.reset();
            checkForNewShards(currentTimeMs, result);
            nextToken = result.getNextToken();
            if (nextToken == null) {
                checkForExpiredShards(currentTimeMs);
            }
        }
    }
}

Example 20 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project hazelcast by hazelcast.

the class RangeMonitor method checkForNewShards.

private void checkForNewShards(long currentTimeMs, ListShardsResult result) {
    Set<Shard> shards = result.getShards().stream().filter(shard -> KinesisUtil.shardBelongsToRange(shard, memberHashRange)).collect(Collectors.toSet());
    Map<Shard, Integer> newShards = shardTracker.markDetections(shards, currentTimeMs);
    if (!newShards.isEmpty()) {
        logger.info("New shards detected: " + newShards.keySet().stream().map(Shard::getShardId).collect(joining(", ")));
        for (Map.Entry<Shard, Integer> e : newShards.entrySet()) {
            Shard shard = e.getKey();
            int owner = e.getValue();
            shardQueues[owner].addAdded(shard);
        }
    }
}