Examples with GetRecordsRequest com.amazonaws.services.kinesis.model.GetRecordsRequest used on opensource projects

Example 1 with GetRecordsRequest

use of com.amazonaws.services.kinesis.model.GetRecordsRequest in project camel by apache.

the class KinesisConsumer method poll.

@Override
protected int poll() throws Exception {
    GetRecordsRequest req = new GetRecordsRequest().withShardIterator(getShardItertor()).withLimit(getEndpoint().getMaxResultsPerRequest());
    GetRecordsResult result = getClient().getRecords(req);
    Queue<Exchange> exchanges = createExchanges(result.getRecords());
    int processedExchangeCount = processBatch(CastUtils.cast(exchanges));
    // May cache the last successful sequence number, and pass it to the
    // getRecords request. That way, on the next poll, we start from where
    // we left off, however, I don't know what happens to subsequent
    // exchanges when an earlier echangee fails.
    currentShardIterator = result.getNextShardIterator();
    return processedExchangeCount;
}

Example 2 with GetRecordsRequest

use of com.amazonaws.services.kinesis.model.GetRecordsRequest in project kafka-connect-kinesis by jcustenborder.

the class KinesisSourceTask method start.

@Override
public void start(Map<String, String> settings) {
    this.config = new KinesisSourceConnectorConfig(settings);
    this.kinesisClient = this.kinesisClientFactory.create(this.config);
    this.sourcePartition = ImmutableMap.of(RecordConverter.FIELD_SHARD_ID, this.config.kinesisShardId);
    Map<String, Object> lastOffset = this.context.offsetStorageReader().offset(this.sourcePartition);
    GetShardIteratorRequest shardIteratorRequest = new GetShardIteratorRequest().withShardId(this.config.kinesisShardId).withStreamName(this.config.kinesisStreamName);
    if (null != lastOffset && !lastOffset.isEmpty()) {
        String startingSequenceNumber = (String) lastOffset.get(RecordConverter.FIELD_SEQUENCE_NUMBER);
        log.info("start() - Starting iterator after last processed sequence number of '{}'", startingSequenceNumber);
        shardIteratorRequest.setShardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER);
        shardIteratorRequest.setStartingSequenceNumber(startingSequenceNumber);
    } else {
        log.info("start() - Setting Shard Iterator Type to {} for {}", this.config.kinesisPosition, this.config.kinesisShardId);
        shardIteratorRequest.setShardIteratorType(this.config.kinesisPosition);
    }
    GetShardIteratorResult shardIteratorResult = this.kinesisClient.getShardIterator(shardIteratorRequest);
    log.info("start() - Using Shard Iterator {}", shardIteratorResult.getShardIterator());
    this.recordsRequest = new GetRecordsRequest().withLimit(this.config.kinesisRecordLimit).withShardIterator(shardIteratorResult.getShardIterator());
    this.recordConverter = new RecordConverter(this.config);
}

Example 3 with GetRecordsRequest

use of com.amazonaws.services.kinesis.model.GetRecordsRequest in project flink by apache.

the class KinesisProxy method getRecords.

@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
    final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
    getRecordsRequest.setShardIterator(shardIterator);
    getRecordsRequest.setLimit(maxRecordsToGet);
    GetRecordsResult getRecordsResult = null;
    int retryCount = 0;
    while (retryCount <= getRecordsMaxRetries && getRecordsResult == null) {
        try {
            getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
        } catch (SdkClientException ex) {
            if (isRecoverableSdkClientException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, retryCount++);
                LOG.warn("Got recoverable SdkClientException. Backing off for " + backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
                BACKOFF.sleep(backoffMillis);
            } else {
                throw ex;
            }
        }
    }
    if (getRecordsResult == null) {
        throw new RuntimeException("Retries exceeded for getRecords operation - all " + getRecordsMaxRetries + " retry attempts failed.");
    }
    return getRecordsResult;
}

Example 4 with GetRecordsRequest

use of com.amazonaws.services.kinesis.model.GetRecordsRequest in project druid by druid-io.

the class KinesisRecordSupplier method isClosedShardEmpty.

/**
 * Fetches records from the specified shard to determine if it is empty.
 * @param stream to which shard belongs
 * @param shardId of the closed shard
 * @return true if the closed shard is empty, false otherwise.
 */
public boolean isClosedShardEmpty(String stream, String shardId) {
    String shardIterator = kinesis.getShardIterator(stream, shardId, ShardIteratorType.TRIM_HORIZON.toString()).getShardIterator();
    GetRecordsRequest request = new GetRecordsRequest().withShardIterator(shardIterator).withLimit(1);
    GetRecordsResult shardData = kinesis.getRecords(request);
    return shardData.getRecords().isEmpty() && shardData.getNextShardIterator() == null;
}

Example 5 with GetRecordsRequest

use of com.amazonaws.services.kinesis.model.GetRecordsRequest in project druid by druid-io.

the class KinesisRecordSupplier method getRecordsForLag.

private GetRecordsResult getRecordsForLag(String iteratorType, String offsetToUse, StreamPartition<String> partition) {
    String shardIterator = kinesis.getShardIterator(partition.getStream(), partition.getPartitionId(), iteratorType, offsetToUse).getShardIterator();
    GetRecordsResult recordsResult = kinesis.getRecords(new GetRecordsRequest().withShardIterator(shardIterator).withLimit(1));
    return recordsResult;
}