Examples with DescribeStreamResult com.amazonaws.services.kinesis.model.DescribeStreamResult used on opensource projects

Example 11 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project kafka-connect-kinesis by jcustenborder.

the class KinesisSourceConnectorTest method start.

@Test
public void start() {
    final DescribeStreamRequest expectedDescribeStreamRequest = new DescribeStreamRequest().withStreamName(TestData.EXPECTED_STREAM_NAME);
    final int SHARD_COUNT = 50;
    List<Shard> shards = new ArrayList<>(SHARD_COUNT);
    for (int i = 0; i < SHARD_COUNT; i++) {
        String shardId = String.format("%03d", i);
        final Shard shard = new Shard().withShardId(shardId);
        shards.add(shard);
    }
    final StreamDescription streamDescription = new StreamDescription().withStreamName(TestData.EXPECTED_STREAM_NAME).withShards(shards);
    final DescribeStreamResult expectedStreamRequest = new DescribeStreamResult().withStreamDescription(streamDescription);
    when(this.kinesisClient.describeStream(any(DescribeStreamRequest.class))).thenReturn(expectedStreamRequest);
    this.connector.start(TestData.settings());
    List<Map<String, String>> taskConfigs = this.connector.taskConfigs(SHARD_COUNT);
    assertEquals(SHARD_COUNT, taskConfigs.size());
    verify(this.kinesisClient, atLeastOnce()).describeStream(expectedDescribeStreamRequest);
}

Example 12 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project apex-malhar by apache.

the class KinesisUtil method getShardList.

/**
 * Get the available shards from the kinesis
 * @param streamName Name of the stream from where the shards to be accessed
 * @return the list of shards from the given stream
 */
public List<Shard> getShardList(String streamName) {
    assert client != null : "Illegal client";
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);
    DescribeStreamResult describeResponse = client.describeStream(describeRequest);
    return describeResponse.getStreamDescription().getShards();
}

Example 13 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project spring-integration-aws by spring-projects.

the class KinesisMessageDrivenChannelAdapter method populateShardsForStream.

private void populateShardsForStream(final String stream, final CountDownLatch shardsGatherLatch) {
    this.dispatcherExecutor.execute(() -> {
        try {
            int describeStreamRetries = 0;
            List<Shard> shardsToConsume = new ArrayList<>();
            String exclusiveStartShardId = null;
            while (true) {
                DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest().withStreamName(stream).withExclusiveStartShardId(exclusiveStartShardId);
                DescribeStreamResult describeStreamResult = null;
                // Call DescribeStream, with backoff and retries (if we get LimitExceededException).
                try {
                    describeStreamResult = this.amazonKinesis.describeStream(describeStreamRequest);
                } catch (Exception e) {
                    logger.info("Got an exception when describing stream [" + stream + "]. " + "Backing off for [" + this.describeStreamBackoff + "] millis.", e);
                }
                if (describeStreamResult == null || !StreamStatus.ACTIVE.toString().equals(describeStreamResult.getStreamDescription().getStreamStatus())) {
                    if (describeStreamRetries++ > this.describeStreamRetries) {
                        ResourceNotFoundException resourceNotFoundException = new ResourceNotFoundException("The stream [" + stream + "] isn't ACTIVE or doesn't exist.");
                        resourceNotFoundException.setServiceName("Kinesis");
                        throw resourceNotFoundException;
                    }
                    try {
                        Thread.sleep(this.describeStreamBackoff);
                        continue;
                    } catch (InterruptedException e) {
                        Thread.interrupted();
                        throw new IllegalStateException("The [describeStream] thread for the stream [" + stream + "] has been interrupted.", e);
                    }
                }
                List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
                try {
                    for (Shard shard : shards) {
                        String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber();
                        if (endingSequenceNumber != null) {
                            String key = buildCheckpointKeyForShard(stream, shard.getShardId());
                            String checkpoint = this.checkpointStore.get(key);
                            boolean skipClosedShard = checkpoint != null && new BigInteger(endingSequenceNumber).compareTo(new BigInteger(checkpoint)) <= 0;
                            if (logger.isTraceEnabled()) {
                                logger.trace("The shard [" + shard + "] in stream [" + stream + "] is closed CLOSED with endingSequenceNumber [" + endingSequenceNumber + "].\nThe last processed checkpoint is [" + checkpoint + "]." + (skipClosedShard ? "\nThe shard will be skipped." : ""));
                            }
                            if (skipClosedShard) {
                                // Skip CLOSED shard which has been read before according a checkpoint
                                continue;
                            }
                        }
                        shardsToConsume.add(shard);
                    }
                } catch (Exception e) {
                    logger.info("Got an exception when processing shards in stream [" + stream + "].\n" + "Retrying...", e);
                    continue;
                }
                if (describeStreamResult.getStreamDescription().getHasMoreShards()) {
                    exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
                    describeStreamRetries = 0;
                } else {
                    break;
                }
            }
            for (Shard shard : shardsToConsume) {
                KinesisShardOffset shardOffset = new KinesisShardOffset(this.streamInitialSequence);
                shardOffset.setShard(shard.getShardId());
                shardOffset.setStream(stream);
                boolean addedOffset;
                synchronized (this.shardOffsets) {
                    addedOffset = this.shardOffsets.add(shardOffset);
                }
                if (addedOffset && shardsGatherLatch == null && this.active) {
                    populateConsumer(shardOffset);
                }
            }
        } finally {
            if (shardsGatherLatch != null) {
                shardsGatherLatch.countDown();
            }
            this.inResharding.remove(stream);
        }
    });
}

Example 14 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project flink by apache.

the class ManualExactlyOnceWithStreamReshardingTest method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    LOG.info("Starting exactly once with stream resharding test");
    final String streamName = "flink-test-" + UUID.randomUUID().toString();
    final String accessKey = pt.getRequired("accessKey");
    final String secretKey = pt.getRequired("secretKey");
    final String region = pt.getRequired("region");
    final Properties configProps = new Properties();
    configProps.setProperty(ConsumerConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
    configProps.setProperty(ConsumerConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
    configProps.setProperty(ConsumerConfigConstants.AWS_REGION, region);
    configProps.setProperty(ConsumerConfigConstants.SHARD_DISCOVERY_INTERVAL_MILLIS, "0");
    final AmazonKinesis client = AWSUtil.createKinesisClient(configProps);
    // the stream is first created with 1 shard
    client.createStream(streamName, 1);
    // wait until stream has been created
    DescribeStreamResult status = client.describeStream(streamName);
    LOG.info("status {}", status);
    while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
        status = client.describeStream(streamName);
        LOG.info("Status of stream {}", status);
        Thread.sleep(1000);
    }
    final Configuration flinkConfig = new Configuration();
    flinkConfig.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("16m"));
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    MiniClusterResource flink = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder().setNumberTaskManagers(1).setNumberSlotsPerTaskManager(8).setConfiguration(flinkConfig).build());
    flink.before();
    final int flinkPort = flink.getRestAddres().getPort();
    try {
        // we have to use a manual generator here instead of the FlinkKinesisProducer
        // because the FlinkKinesisProducer currently has a problem where records will be resent
        // to a shard
        // when resharding happens; this affects the consumer exactly-once validation test and
        // will never pass
        final AtomicReference<Throwable> producerError = new AtomicReference<>();
        Runnable manualGenerate = new Runnable() {

            @Override
            public void run() {
                AmazonKinesis client = AWSUtil.createKinesisClient(configProps);
                int count = 0;
                final int batchSize = 30;
                while (true) {
                    try {
                        Thread.sleep(10);
                        Set<PutRecordsRequestEntry> batch = new HashSet<>();
                        for (int i = count; i < count + batchSize; i++) {
                            if (i >= TOTAL_EVENT_COUNT) {
                                break;
                            }
                            batch.add(new PutRecordsRequestEntry().withData(ByteBuffer.wrap(((i) + "-" + RandomStringUtils.randomAlphabetic(12)).getBytes(ConfigConstants.DEFAULT_CHARSET))).withPartitionKey(UUID.randomUUID().toString()));
                        }
                        count += batchSize;
                        PutRecordsResult result = client.putRecords(new PutRecordsRequest().withStreamName(streamName).withRecords(batch));
                        // and let this test fail
                        if (result.getFailedRecordCount() > 0) {
                            producerError.set(new RuntimeException("The producer has failed records in one of the put batch attempts."));
                            break;
                        }
                        if (count >= TOTAL_EVENT_COUNT) {
                            break;
                        }
                    } catch (Exception e) {
                        producerError.set(e);
                    }
                }
            }
        };
        Thread producerThread = new Thread(manualGenerate);
        producerThread.start();
        final AtomicReference<Throwable> consumerError = new AtomicReference<>();
        Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(TOTAL_EVENT_COUNT, 10000, 2, 500, 500, accessKey, secretKey, region, streamName, consumerError, flinkPort, flinkConfig);
        consumerThread.start();
        // reshard the Kinesis stream while the producer / and consumers are running
        Runnable splitShard = new Runnable() {

            @Override
            public void run() {
                try {
                    // first, split shard in the middle of the hash range
                    Thread.sleep(5000);
                    LOG.info("Splitting shard ...");
                    client.splitShard(streamName, KinesisShardIdGenerator.generateFromShardOrder(0), "170141183460469231731687303715884105727");
                    // wait until the split shard operation finishes updating ...
                    DescribeStreamResult status;
                    Random rand = new Random();
                    do {
                        status = null;
                        while (status == null) {
                            // retry until we get status
                            try {
                                status = client.describeStream(streamName);
                            } catch (LimitExceededException lee) {
                                LOG.warn("LimitExceededException while describing stream ... retrying ...");
                                Thread.sleep(rand.nextInt(1200));
                            }
                        }
                    } while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE"));
                    // then merge again
                    Thread.sleep(7000);
                    LOG.info("Merging shards ...");
                    client.mergeShards(streamName, KinesisShardIdGenerator.generateFromShardOrder(1), KinesisShardIdGenerator.generateFromShardOrder(2));
                } catch (InterruptedException iex) {
                // 
                }
            }
        };
        Thread splitShardThread = new Thread(splitShard);
        splitShardThread.start();
        boolean deadlinePassed = false;
        long deadline = // wait at most for five minutes
        System.currentTimeMillis() + (1000 * 5 * 60);
        // wait until both producer and consumer finishes, or an unexpected error is thrown
        while ((consumerThread.isAlive() || producerThread.isAlive()) && (producerError.get() == null && consumerError.get() == null)) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() >= deadline) {
                LOG.warn("Deadline passed");
                deadlinePassed = true;
                // enough waiting
                break;
            }
        }
        if (producerThread.isAlive()) {
            producerThread.interrupt();
        }
        if (consumerThread.isAlive()) {
            consumerThread.interrupt();
        }
        if (producerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Producer failed", producerError.get());
        }
        if (consumerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Consumer failed", consumerError.get());
        }
        if (!deadlinePassed) {
            LOG.info("+++ TEST passed! +++");
        } else {
            LOG.info("+++ TEST failed! +++");
        }
    } finally {
        client.deleteStream(streamName);
        client.shutdown();
        // stopping flink
        flink.after();
    }
}

Example 15 with DescribeStreamResult

use of com.amazonaws.services.kinesis.model.DescribeStreamResult in project flink by apache.

the class DynamoDBStreamsProxy method getShardsOfStream.

private List<StreamShardHandle> getShardsOfStream(String streamName, @Nullable String lastSeenShardId) throws InterruptedException {
    List<StreamShardHandle> shardsOfStream = new ArrayList<>();
    DescribeStreamResult describeStreamResult;
    do {
        describeStreamResult = describeStream(streamName, lastSeenShardId);
        List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
        for (Shard shard : shards) {
            shardsOfStream.add(new StreamShardHandle(streamName, shard));
        }
        if (shards.size() != 0) {
            lastSeenShardId = shards.get(shards.size() - 1).getShardId();
        }
    } while (describeStreamResult.getStreamDescription().isHasMoreShards());
    return shardsOfStream;
}