Examples with Shard com.amazonaws.services.kinesis.model.Shard used on opensource projects

Example 6 with Shard

use of com.amazonaws.services.kinesis.model.Shard in project flink by apache.

the class KinesisProxy method describeStream.

/**
	 * Get metainfo for a Kinesis stream, which contains information about which shards this Kinesis stream possess.
	 *
	 * This method is using a "full jitter" approach described in AWS's article,
	 * <a href="https://www.awsarchitectureblog.com/2015/03/backoff.html">"Exponential Backoff and Jitter"</a>.
	 * This is necessary because concurrent calls will be made by all parallel subtask's fetcher. This
	 * jitter backoff approach will help distribute calls across the fetchers over time.
	 *
	 * @param streamName the stream to describe
	 * @param startShardId which shard to start with for this describe operation (earlier shard's infos will not appear in result)
	 * @return the result of the describe stream operation
	 */
private DescribeStreamResult describeStream(String streamName, @Nullable String startShardId) throws InterruptedException {
    final DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(streamName);
    describeStreamRequest.setExclusiveStartShardId(startShardId);
    DescribeStreamResult describeStreamResult = null;
    // Call DescribeStream, with full-jitter backoff (if we get LimitExceededException).
    int attemptCount = 0;
    while (describeStreamResult == null) {
        // retry until we get a result
        try {
            describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        } catch (LimitExceededException le) {
            long backoffMillis = fullJitterBackoff(describeStreamBaseBackoffMillis, describeStreamMaxBackoffMillis, describeStreamExpConstant, attemptCount++);
            LOG.warn("Got LimitExceededException when describing stream " + streamName + ". Backing off for " + backoffMillis + " millis.");
            Thread.sleep(backoffMillis);
        } catch (ResourceNotFoundException re) {
            throw new RuntimeException("Error while getting stream details", re);
        }
    }
    String streamStatus = describeStreamResult.getStreamDescription().getStreamStatus();
    if (!(streamStatus.equals(StreamStatus.ACTIVE.toString()) || streamStatus.equals(StreamStatus.UPDATING.toString()))) {
        if (LOG.isWarnEnabled()) {
            LOG.warn("The status of stream " + streamName + " is " + streamStatus + "; result of the current " + "describeStream operation will not contain any shard information.");
        }
    }
    // start shard id in the returned shards list; check if we need to remove these erroneously returned shards
    if (startShardId != null) {
        List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
        Iterator<Shard> shardItr = shards.iterator();
        while (shardItr.hasNext()) {
            if (KinesisStreamShard.compareShardIds(shardItr.next().getShardId(), startShardId) <= 0) {
                shardItr.remove();
            }
        }
    }
    return describeStreamResult;
}

Example 7 with Shard

use of com.amazonaws.services.kinesis.model.Shard in project storm by apache.

the class KinesisConnection method getShardsForStream.

List<Shard> getShardsForStream(String stream) {
    DescribeStreamRequest describeStreamRequest = new DescribeStreamRequest();
    describeStreamRequest.setStreamName(stream);
    List<Shard> shards = new ArrayList<>();
    String exclusiveStartShardId = null;
    do {
        describeStreamRequest.setExclusiveStartShardId(exclusiveStartShardId);
        DescribeStreamResult describeStreamResult = kinesisClient.describeStream(describeStreamRequest);
        shards.addAll(describeStreamResult.getStreamDescription().getShards());
        if (describeStreamResult.getStreamDescription().getHasMoreShards() && shards.size() > 0) {
            exclusiveStartShardId = shards.get(shards.size() - 1).getShardId();
        } else {
            exclusiveStartShardId = null;
        }
    } while (exclusiveStartShardId != null);
    LOG.info("Number of shards for stream " + stream + " are " + shards.size());
    return shards;
}

Example 8 with Shard

use of com.amazonaws.services.kinesis.model.Shard in project beam by apache.

the class SimplifiedKinesisClientTest method shouldListAllShards.

@Test
public void shouldListAllShards() throws Exception {
    Shard shard1 = new Shard().withShardId(SHARD_1);
    Shard shard2 = new Shard().withShardId(SHARD_2);
    Shard shard3 = new Shard().withShardId(SHARD_3);
    given(kinesis.describeStream(STREAM, null)).willReturn(new DescribeStreamResult().withStreamDescription(new StreamDescription().withShards(shard1, shard2).withHasMoreShards(true)));
    given(kinesis.describeStream(STREAM, SHARD_2)).willReturn(new DescribeStreamResult().withStreamDescription(new StreamDescription().withShards(shard3).withHasMoreShards(false)));
    List<Shard> shards = underTest.listShards(STREAM);
    assertThat(shards).containsOnly(shard1, shard2, shard3);
}

Example 9 with Shard

use of com.amazonaws.services.kinesis.model.Shard in project apex-malhar by apache.

the class KinesisConsumer method start.

/**
 * This method is called in the activate method of the operator
 */
public void start() {
    isAlive = true;
    int realNumStream = simpleConsumerThreads.size();
    if (realNumStream == 0) {
        return;
    }
    consumerThreadExecutor = Executors.newFixedThreadPool(realNumStream);
    for (final Shard shd : simpleConsumerThreads) {
        consumerThreadExecutor.submit(new Runnable() {

            @Override
            public void run() {
                logger.debug("Thread " + Thread.currentThread().getName() + " start consuming Records...");
                while (isAlive) {
                    Shard shard = shd;
                    try {
                        List<Record> records = KinesisUtil.getInstance().getRecords(streamName, recordsLimit, shard.getShardId(), getIteratorType(shard.getShardId()), shardPosition.get(shard.getShardId()));
                        if (records == null || records.isEmpty()) {
                            if (shard.getSequenceNumberRange().getEndingSequenceNumber() != null) {
                                closedShards.add(shard);
                                break;
                            }
                            try {
                                Thread.sleep(recordsCheckInterval);
                            } catch (Exception e) {
                                throw new RuntimeException(e);
                            }
                        } else {
                            String seqNo = "";
                            for (Record rc : records) {
                                seqNo = rc.getSequenceNumber();
                                putRecord(shd.getShardId(), rc);
                            }
                            shardPosition.put(shard.getShardId(), seqNo);
                        }
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
                logger.debug("Thread " + Thread.currentThread().getName() + " stop consuming Records...");
            }
        });
    }
}

Example 10 with Shard

use of com.amazonaws.services.kinesis.model.Shard in project apex-malhar by apache.

the class KinesisTestConsumer method prepareIterator.

public String prepareIterator() {
    DescribeStreamRequest describeRequest = new DescribeStreamRequest();
    describeRequest.setStreamName(streamName);
    List<Shard> shards = null;
    for (int i = 0; i < MAX_TRY_TIMES; ++i) {
        try {
            DescribeStreamResult describeResponse = client.describeStream(describeRequest);
            shards = describeResponse.getStreamDescription().getShards();
            if (shards.isEmpty()) {
                logger.warn("shards is empty");
            } else {
                break;
            }
        } catch (Exception e) {
            logger.error("get Stream description exception: ", e);
            throw new RuntimeException(e);
        }
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
        // 
        }
    }
    Shard shId = shards.get(0);
    GetShardIteratorRequest iteratorRequest = new GetShardIteratorRequest();
    iteratorRequest.setStreamName(streamName);
    iteratorRequest.setShardId(shId.getShardId());
    iteratorRequest.setShardIteratorType("TRIM_HORIZON");
    GetShardIteratorResult iteratorResponse = client.getShardIterator(iteratorRequest);
    return iteratorResponse.getShardIterator();
}