Examples with DescribeStreamResult com.amazonaws.services.kinesis.model.DescribeStreamResult used on opensource projects

Example 1 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShards.

@Test
public void shouldListAllShards() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    given(kinesis.describeStream(STREAM, null)).willReturn(new DescribeStreamResult().withStreamDescription(new StreamDescription().withShards(shard1, shard2).withHasMoreShards(true)));
    given(kinesis.describeStream(STREAM, SHARD_2)).willReturn(new DescribeStreamResult().withStreamDescription(new StreamDescription().withShards(shard3).withHasMoreShards(false)));
    List<Shard> shards = underTest.listShards(STREAM);
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 2 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project storm by apache.

the class KinesisConnection method getShardsForStream.

List<Shard> getShardsForStream(String stream) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(stream);
    List<Shard> shards = new ArrayList<>();
    String exclusiveStartShardId = null;
    do {
        describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
        DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        shards.addAll(describeStreamResult.getStreamDescription().getShards());
        if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
            exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
        } else {
            exclusiveStartShardId = null;
        }
    } while (exclusiveStartShardId != null);
    LOG.info("Number of shards for stream " + stream + " are " + shards.size());
    return shards;
}

Example 3 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project flink by apache.

the class ManualExactlyOnceTest method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    LOG.info("Starting exactly once test");
    final String streamName = "flink-test-" + UUID.randomUUID().toString();
    final String accessKey = pt.getRequired("accessKey");
    final String secretKey = pt.getRequired("secretKey");
    final String region = pt.getRequired("region");
    Properties configProps = new Properties();
    configProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
    configProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
    configProps.setProperty(AWSConfigConstants.AWS_REGION, region);
    AmazonKinesis client = AWSUtil.createKinesisClient(configProps);
    // create a stream for the test:
    client.createStream(streamName, 1);
    // wait until stream has been created
    DescribeStreamResult status = client.describeStream(streamName);
    LOG.info("status {}", status);
    while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
        status = client.describeStream(streamName);
        LOG.info("Status of stream {}", status);
        Thread.sleep(1000);
    }
    final Configuration flinkConfig = new Configuration();
    flinkConfig.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("16m"));
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder().setNumberTaskManagers(1).setNumberSlotsPerTaskManager(8).setConfiguration(flinkConfig).build());
    flink.before();
    final int flinkPort = flink.getRestAddres().getPort();
    try {
        final AtomicReference<Throwable> producerError = new AtomicReference<>();
        Thread producerThread = KinesisEventsGeneratorProducerThread.create(TOTAL_EVENT_COUNT, 2, accessKey, secretKey, region, streamName, producerError, flinkPort, flinkConfig);
        producerThread.start();
        final AtomicReference<Throwable> consumerError = new AtomicReference<>();
        Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(TOTAL_EVENT_COUNT, 200, 2, 500, 500, accessKey, secretKey, region, streamName, consumerError, flinkPort, flinkConfig);
        consumerThread.start();
        boolean deadlinePassed = false;
        long deadline = // wait at most for two minutes
        System.currentTimeMillis() + (1000 * 2 * 60);
        // wait until both producer and consumer finishes, or an unexpected error is thrown
        while ((consumerThread.isAlive() || producerThread.isAlive()) && (producerError.get() == null && consumerError.get() == null)) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() >= deadline) {
                LOG.warn("Deadline passed");
                deadlinePassed = true;
                // enough waiting
                break;
            }
        }
        if (producerThread.isAlive()) {
            producerThread.interrupt();
        }
        if (consumerThread.isAlive()) {
            consumerThread.interrupt();
        }
        if (producerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Producer failed", producerError.get());
        }
        if (consumerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Consumer failed", consumerError.get());
        }
        if (!deadlinePassed) {
            LOG.info("+++ TEST passed! +++");
        } else {
            LOG.info("+++ TEST failed! +++");
        }
    } finally {
        client.deleteStream(streamName);
        client.shutdown();
        // stopping flink
        flink.after();
    }
}

Example 4 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project flink by apache.

the class KinesisProxy method describeStream.

/**
 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis
 * stream possess.
 *
 * <p>This method is using a "full jitter" approach described in AWS's article, <a
 * href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and
 * Jitter"</a>. This is necessary because concurrent calls will be made by all parallel
 * subtask's fetcher. This jitter backoff approach will help distribute calls across the
 * fetchers over time.
 *
 * @param streamName the stream to describe
 * @param startShardId which shard to start with for this describe operation
 * @return the result of the describe stream operation
 */
protected DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException {
    final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    describeStreamRequest.setExclusiveStartShardId(startShardId);
    DescribeStreamResult describeStreamResult = null;
    // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
    int attemptCount = 0;
    while (describeStreamResult == null) {
        // retry until we get a result
        try {
            describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = BACKOFF.calculateFullJitterBackoff(describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
            LOG.warn(String.format("Got LimitExceededException when describing stream %s. " + "Backing off for %d millis.", streamName, backoffMillis));
            BACKOFF.sleep(backoffMillis);
        } catch (ResourceNotFoundException re) {
            throw new RuntimeException("Error while getting stream details", re);
        }
    }
    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
        if (LOG.isWarnEnabled()) {
            LOG.warn(String.format("The status of stream %s is %s ; result of the current " + "describeStream operation will not contain any shard information.", streamName, streamStatus));
        }
    }
    return describeStreamResult;
}

Example 5 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project apex-malhar by apache.

the class KinesisTestConsumer method prepareIterator.

public String prepareIterator() {
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);
    List<Shard> shards = null;
    for (int i = 0; i < MAX_TRY_TIMES; ++i) {
        try {
            DescribeStreamResult describeResponse = client.describeStream(describeRequest);
            shards = describeResponse.getStreamDescription().getShards();
            if (shards.isEmpty()) {
                logger.warn("shards is empty");
            } else {
                break;
            }
        } catch (Exception e) {
            logger.error("get Stream description exception: ", e);
            throw new RuntimeException(e);
        }
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        // 
        }
    }
    Shard shId = shards.get(0);
    GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
    iteratorRequest.setStreamName(streamName);
    iteratorRequest.setShardId(shId.getShardId());
    iteratorRequest.setShardIteratorType("TRIM_HORIZON");
    GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
    return iteratorResponse.getShardIterator();
}