Examples with ShardIteratorType com.amazonaws.services.kinesis.model.ShardIteratorType used on opensource projects

Example 1 with ShardIteratorType

use of com.amazonaws.services.kinesis.model.ShardIteratorType in project beam by apache.

the class AmazonKinesisMock method getShardIterator.

@Override
public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest) {
    ShardIteratorType shardIteratorType = ShardIteratorType.fromValue(getShardIteratorRequest.getShardIteratorType());
    String shardIterator;
    if (shardIteratorType == ShardIteratorType.TRIM_HORIZON) {
        shardIterator = String.format("%s:%s", getShardIteratorRequest.getShardId(), 0);
    } else {
        throw new RuntimeException("Not implemented");
    }
    return new GetShardIteratorResult().withShardIterator(shardIterator);
}

Example 2 with ShardIteratorType

use of com.amazonaws.services.kinesis.model.ShardIteratorType in project storm by apache.

the class KinesisRecordsManager method refreshShardIteratorForNewRecords.

private void refreshShardIteratorForNewRecords(String shardId) {
    String shardIterator = null;
    String lastFetchedSequenceNumber = fetchedSequenceNumberPerShard.get(shardId);
    ShardIteratorType shardIteratorType = (lastFetchedSequenceNumber == null ? kinesisConfig.getShardIteratorType() : ShardIteratorType.AFTER_SEQUENCE_NUMBER);
    // Set the shard iterator for last fetched sequence number to start from correct position in shard
    shardIterator = kinesisConnection.getShardIterator(kinesisConfig.getStreamName(), shardId, shardIteratorType, lastFetchedSequenceNumber, kinesisConfig.getTimestamp());
    if (shardIterator != null && !shardIterator.isEmpty()) {
        LOG.warn("Refreshing shard iterator for new records for shardId " + shardId + " with shardIterator " + shardIterator);
        shardIteratorPerShard.put(shardId, shardIterator);
    }
}

Example 3 with ShardIteratorType

use of com.amazonaws.services.kinesis.model.ShardIteratorType in project druid by druid-io.

the class KinesisRecordSupplier method getSequenceNumber.

/**
 * Given a partition and a {@link ShardIteratorType}, create a shard iterator and fetch
 * {@link #GET_SEQUENCE_NUMBER_RECORD_COUNT} records and return the first sequence number from the result set.
 * This method is thread safe as it does not depend on the internal state of the supplier (it doesn't use the
 * {@link PartitionResource} which have been assigned to the supplier), and the Kinesis client is thread safe.
 */
@Nullable
private String getSequenceNumber(StreamPartition<String> partition, ShardIteratorType iteratorEnum) {
    return wrapExceptions(() -> {
        String shardIterator = kinesis.getShardIterator(partition.getStream(), partition.getPartitionId(), iteratorEnum.toString()).getShardIterator();
        long timeoutMillis = System.currentTimeMillis() + fetchSequenceNumberTimeout;
        GetRecordsResult recordsResult = null;
        while (shardIterator != null && System.currentTimeMillis() < timeoutMillis) {
            if (closed) {
                log.info("KinesisRecordSupplier closed while fetching sequenceNumber");
                return null;
            }
            final String currentShardIterator = shardIterator;
            final GetRecordsRequest request = new GetRecordsRequest().withShardIterator(currentShardIterator).withLimit(GET_SEQUENCE_NUMBER_RECORD_COUNT);
            recordsResult = RetryUtils.retry(() -> kinesis.getRecords(request), (throwable) -> {
                if (throwable instanceof ProvisionedThroughputExceededException) {
                    log.warn(throwable, "encountered ProvisionedThroughputExceededException while fetching records, this means " + "that the request rate for the stream is too high, or the requested data is too large for " + "the available throughput. Reduce the frequency or size of your requests. Consider increasing " + "the number of shards to increase throughput.");
                    return true;
                }
                if (throwable instanceof AmazonClientException) {
                    AmazonClientException ase = (AmazonClientException) throwable;
                    return AWSClientUtil.isClientExceptionRecoverable(ase);
                }
                return false;
            }, GET_SEQUENCE_NUMBER_RETRY_COUNT);
            List<Record> records = recordsResult.getRecords();
            if (!records.isEmpty()) {
                return records.get(0).getSequenceNumber();
            }
            shardIterator = recordsResult.getNextShardIterator();
        }
        if (shardIterator == null) {
            log.info("Partition[%s] returned a null shard iterator, is the shard closed?", partition.getPartitionId());
            return KinesisSequenceNumber.END_OF_SHARD_MARKER;
        }
        // if we reach here, it usually means either the shard has no more records, or records have not been
        // added to this shard
        log.warn("timed out while trying to fetch position for shard[%s], millisBehindLatest is [%s], likely no more records in shard", partition.getPartitionId(), recordsResult != null ? recordsResult.getMillisBehindLatest() : "UNKNOWN");
        return null;
    });
}