Examples with ListShardsResult com.amazonaws.services.kinesis.model.ListShardsResult used on opensource projects

Example 1 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project flink by apache.

the class KinesisProxyTest method testGetShardListRetry.

@Test
public void testGetShardListRetry() throws Exception {
    Properties kinesisConsumerConfig = new Properties();
    kinesisConsumerConfig.setProperty(ConsumerConfigConstants.AWS_REGION, "us-east-1");
    Shard shard = new Shard();
    shard.setShardId("fake-shard-000000000000");
    final ListShardsResult expectedResult = new ListShardsResult();
    expectedResult.withShards(shard);
    MutableInt exceptionCount = new MutableInt();
    final Throwable[] retriableExceptions = new Throwable[] { new AmazonKinesisException("attempt1"), new AmazonKinesisException("attempt2") };
    AmazonKinesisClient mockClient = mock(AmazonKinesisClient.class);
    Mockito.when(mockClient.listShards(any())).thenAnswer(new Answer<ListShardsResult>() {

        @Override
        public ListShardsResult answer(InvocationOnMock invocation) throws Throwable {
            if (exceptionCount.intValue() < retriableExceptions.length) {
                exceptionCount.increment();
                throw retriableExceptions[exceptionCount.intValue() - 1];
            }
            return expectedResult;
        }
    });
    KinesisProxy kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
    Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);
    HashMap<String, String> streamNames = new HashMap();
    streamNames.put("fake-stream", null);
    GetShardListResult result = kinesisProxy.getShardList(streamNames);
    assertEquals(retriableExceptions.length, exceptionCount.intValue());
    assertEquals(true, result.hasRetrievedShards());
    assertEquals(shard.getShardId(), result.getLastSeenShardOfStream("fake-stream").getShard().getShardId());
    // test max attempt count exceeded
    int maxRetries = 1;
    exceptionCount.setValue(0);
    kinesisConsumerConfig.setProperty(ConsumerConfigConstants.LIST_SHARDS_RETRIES, String.valueOf(maxRetries));
    kinesisProxy = new KinesisProxy(kinesisConsumerConfig);
    Whitebox.getField(KinesisProxy.class, "kinesisClient").set(kinesisProxy, mockClient);
    try {
        kinesisProxy.getShardList(streamNames);
        Assert.fail("exception expected");
    } catch (SdkClientException ex) {
        assertEquals(retriableExceptions[maxRetries], ex);
    }
    assertEquals(maxRetries + 1, exceptionCount.intValue());
}

Example 2 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project flink by apache.

the class KinesisProxy method getShardsOfStream.

private List<StreamShardHandle> getShardsOfStream(String streamName, @Nullable String lastSeenShardId) throws InterruptedException {
    List<StreamShardHandle> shardsOfStream = new ArrayList<>();
    // List Shards returns just the first 1000 shard entries. In order to read the entire
    // stream,
    // we need to use the returned nextToken to get additional shards.
    ListShardsResult listShardsResult;
    String startShardToken = null;
    do {
        listShardsResult = listShards(streamName, lastSeenShardId, startShardToken);
        if (listShardsResult == null) {
            // In case we have exceptions while retrieving all shards, ensure that incomplete
            // shard list is not returned.
            // Hence clearing the incomplete shard list before returning it.
            shardsOfStream.clear();
            return shardsOfStream;
        }
        List<Shard> shards = listShardsResult.getShards();
        for (Shard shard : shards) {
            shardsOfStream.add(new StreamShardHandle(streamName, shard));
        }
        startShardToken = listShardsResult.getNextToken();
    } while (startShardToken != null);
    return shardsOfStream;
}

Example 3 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project flink by apache.

the class KinesisProxy method listShards.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation (earlier shard's
 *     infos will not appear in result)
 * @return the result of the describe stream operation
 */
private ListShardsResult listShards(String streamName, @Nullable String startShardId, @Nullable String startNextToken) throws InterruptedException {
    final ListShardsRequest listShardsRequest = new ListShardsRequest();
    if (startNextToken == null) {
        listShardsRequest.setExclusiveStartShardId(startShardId);
        listShardsRequest.setStreamName(streamName);
    } else {
        // Note the nextToken returned by AWS expires within 300 sec.
        listShardsRequest.setNextToken(startNextToken);
    }
    ListShardsResult listShardsResults = null;
    // Call ListShards, with full-jitter backoff (if we get LimitExceededException).
    int retryCount = 0;
    // are taken up.
    while (retryCount <= listShardsMaxRetries && listShardsResults == null) {
        // retry until we get a result
        try {
            listShardsResults = kinesisClient.listShards(listShardsRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
            LOG.warn("Got LimitExceededException when listing shards from stream " + streamName + ". Backing off for " + backoffMillis + " millis.");
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceInUseException reInUse) {
            if (LOG.isWarnEnabled()) {
                // List Shards will throw an exception if stream in not in active state. Return
                // and re-use previous state available.
                LOG.info("The stream is currently not in active state. Reusing the older state " + "for the time being");
                break;
            }
        } catch (ResourceNotFoundException reNotFound) {
            throw new RuntimeException("Stream not found. Error while getting shard list.", reNotFound);
        } catch (InvalidArgumentException inArg) {
            throw new RuntimeException("Invalid Arguments to listShards.", inArg);
        } catch (ExpiredNextTokenException expiredToken) {
            LOG.warn("List Shards has an expired token. Reusing the previous state.");
            break;
        } catch (SdkClientException ex) {
            if (retryCount < listShardsMaxRetries && isRecoverableSdkClientException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(listShardsBaseBackoffMillis, listShardsMaxBackoffMillis, listShardsExpConstant, retryCount++);
                LOG.warn("Got SdkClientException when listing shards from stream {}. Backing off for {} millis.", streamName, backoffMillis);
                BACKOFF.sleep(backoffMillis);
            } else {
                // (otherwise would return null result and keep trying forever)
                throw ex;
            }
        }
    }
    // https://github.com/lyft/kinesalite/pull/4
    if (startShardId != null && listShardsResults != null) {
        List<Shard> shards = listShardsResults.getShards();
        shards.removeIf(shard -> StreamShardHandle.compareShardIds(shard.getShardId(), startShardId) <= 0);
    }
    return listShardsResults;
}

Example 4 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project druid by druid-io.

the class KinesisAdminClient method listShards.

private Set<Shard> listShards(String streamName) {
    ListShardsRequest listShardsRequest = new ListShardsRequest().withStreamName(streamName);
    ImmutableSet.Builder<Shard> shards = ImmutableSet.builder();
    while (true) {
        ListShardsResult listShardsResult = amazonKinesis.listShards(listShardsRequest);
        shards.addAll(listShardsResult.getShards());
        String nextToken = listShardsResult.getNextToken();
        if (nextToken == null) {
            return shards.build();
        }
        listShardsRequest = new ListShardsRequest().withNextToken(nextToken);
    }
}

Example 5 with ListShardsResult

use of com.amazonaws.services.kinesis.model.ListShardsResult in project druid by druid-io.

the class KinesisRecordSupplier method getShards.

/**
 * Use the API listShards which is the recommended way instead of describeStream
 * listShards can return 1000 shards per call and has a limit of 100TPS
 * This makes the method resilient to LimitExceeded exceptions (compared to 100 shards, 10 TPS of describeStream)
 *
 * @param stream name of stream
 * @return Immutable set of shards
 */
public Set<Shard> getShards(String stream) {
    ImmutableSet.Builder<Shard> shards = ImmutableSet.builder();
    ListShardsRequest request = new ListShardsRequest().withStreamName(stream);
    while (true) {
        ListShardsResult result = kinesis.listShards(request);
        shards.addAll(result.getShards());
        String nextToken = result.getNextToken();
        if (nextToken == null) {
            return shards.build();
        }
        request = new ListShardsRequest().withNextToken(nextToken);
    }
}