Examples with GetShardIteratorResult com.amazonaws.services.kinesis.model.GetShardIteratorResult used on opensource projects

Example 1 with GetShardIteratorResult

use of com.amazonaws.services.kinesis.model.GetShardIteratorResult in project kafka-connect-kinesis by jcustenborder.

the class KinesisSourceTask method start.

@Override
public void start(Map<String, String> settings) {
    this.config = new KinesisSourceConnectorConfig(settings);
    this.kinesisClient = this.kinesisClientFactory.create(this.config);
    this.sourcePartition = ImmutableMap.of(RecordConverter.FIELD_SHARD_ID, this.config.kinesisShardId);
    Map<String, Object> lastOffset = this.context.offsetStorageReader().offset(this.sourcePartition);
    GetShardIteratorRequest shardIteratorRequest = new GetShardIteratorRequest().withShardId(this.config.kinesisShardId).withStreamName(this.config.kinesisStreamName);
    if (null != lastOffset && !lastOffset.isEmpty()) {
        String startingSequenceNumber = (String) lastOffset.get(RecordConverter.FIELD_SEQUENCE_NUMBER);
        log.info("start() - Starting iterator after last processed sequence number of '{}'", startingSequenceNumber);
        shardIteratorRequest.setShardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER);
        shardIteratorRequest.setStartingSequenceNumber(startingSequenceNumber);
    } else {
        log.info("start() - Setting Shard Iterator Type to {} for {}", this.config.kinesisPosition, this.config.kinesisShardId);
        shardIteratorRequest.setShardIteratorType(this.config.kinesisPosition);
    }
    GetShardIteratorResult shardIteratorResult = this.kinesisClient.getShardIterator(shardIteratorRequest);
    log.info("start() - Using Shard Iterator {}", shardIteratorResult.getShardIterator());
    this.recordsRequest = new GetRecordsRequest().withLimit(this.config.kinesisRecordLimit).withShardIterator(shardIteratorResult.getShardIterator());
    this.recordConverter = new RecordConverter(this.config);
}

Example 2 with GetShardIteratorResult

use of com.amazonaws.services.kinesis.model.GetShardIteratorResult in project kafka-connect-kinesis by jcustenborder.

the class KinesisSourceTaskTest method sourceOffsets.

@Test
public void sourceOffsets() throws InterruptedException {
    final String SEQUENCE_NUMBER = "asdfasdfddsa";
    Map<String, Object> sourceOffset = ImmutableMap.of(RecordConverter.FIELD_SEQUENCE_NUMBER, SEQUENCE_NUMBER);
    when(this.offsetStorageReader.offset(anyMap())).thenReturn(sourceOffset);
    when(this.kinesisClient.getShardIterator(any())).thenReturn(new GetShardIteratorResult().withShardIterator("dfasdfsadfasdf"));
    this.task.start(settings);
    GetRecordsResult recordsResult = new GetRecordsResult().withNextShardIterator("dsfargadsfasdfasda").withRecords(TestData.record()).withMillisBehindLatest(0L);
    when(this.kinesisClient.getRecords(any())).thenReturn(recordsResult);
    List<SourceRecord> records = this.task.poll();
    assertNotNull(records, "records should not be null.");
    assertFalse(records.isEmpty(), "records should not be empty.");
    verify(this.offsetStorageReader, atLeastOnce()).offset(anyMap());
    GetShardIteratorRequest expectedIteratorRequest = new GetShardIteratorRequest().withShardIteratorType(ShardIteratorType.AFTER_SEQUENCE_NUMBER).withShardId(this.config.kinesisShardId).withStreamName(this.config.kinesisStreamName).withStartingSequenceNumber(SEQUENCE_NUMBER);
    verify(this.kinesisClient, atLeastOnce()).getShardIterator(expectedIteratorRequest);
}

Example 3 with GetShardIteratorResult

use of com.amazonaws.services.kinesis.model.GetShardIteratorResult in project storm by apache.

the class KinesisConnection method getShardIterator.

String getShardIterator(String stream, String shardId, ShardIteratorType shardIteratorType, String sequenceNumber, Date timestamp) {
    String shardIterator = "";
    try {
        GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest();
        getShardIteratorRequest.setStreamName(stream);
        getShardIteratorRequest.setShardId(shardId);
        getShardIteratorRequest.setShardIteratorType(shardIteratorType);
        if (shardIteratorType.equals(ShardIteratorType.AFTER_SEQUENCE_NUMBER) || shardIteratorType.equals(ShardIteratorType.AT_SEQUENCE_NUMBER)) {
            getShardIteratorRequest.setStartingSequenceNumber(sequenceNumber);
        } else if (shardIteratorType.equals(ShardIteratorType.AT_TIMESTAMP)) {
            getShardIteratorRequest.setTimestamp(timestamp);
        }
        GetShardIteratorResult getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
        if (getShardIteratorResult != null) {
            shardIterator = getShardIteratorResult.getShardIterator();
        }
    } catch (Exception e) {
        LOG.warn("Exception occured while getting shardIterator for shard " + shardId + " shardIteratorType " + shardIteratorType + " sequence number " + sequenceNumber + " timestamp " + timestamp, e);
    }
    LOG.warn("Returning shardIterator " + shardIterator + " for shardId " + shardId + " shardIteratorType " + shardIteratorType + " sequenceNumber " + sequenceNumber + " timestamp" + timestamp);
    return shardIterator;
}

Example 4 with GetShardIteratorResult

use of com.amazonaws.services.kinesis.model.GetShardIteratorResult in project flink by apache.

the class KinesisProxy method getShardIterator.

private String getShardIterator(GetShardIteratorRequest getShardIteratorRequest) throws InterruptedException {
    GetShardIteratorResult getShardIteratorResult = null;
    int retryCount = 0;
    while (retryCount <= getShardIteratorMaxRetries && getShardIteratorResult == null) {
        try {
            getShardIteratorResult = kinesisClient.getShardIterator(getShardIteratorRequest);
        } catch (AmazonServiceException ex) {
            if (isRecoverableException(ex)) {
                long backoffMillis = BACKOFF.calculateFullJitterBackoff(getShardIteratorBaseBackoffMillis, getShardIteratorMaxBackoffMillis, getShardIteratorExpConstant, retryCount++);
                LOG.warn("Got recoverable AmazonServiceException. Backing off for " + backoffMillis + " millis (" + ex.getClass().getName() + ": " + ex.getMessage() + ")");
                BACKOFF.sleep(backoffMillis);
            } else {
                throw ex;
            }
        }
    }
    if (getShardIteratorResult == null) {
        throw new RuntimeException("Retries exceeded for getShardIterator operation - all " + getShardIteratorMaxRetries + " retry attempts failed.");
    }
    return getShardIteratorResult.getShardIterator();
}

Example 5 with GetShardIteratorResult

use of com.amazonaws.services.kinesis.model.GetShardIteratorResult in project beam by apache.

the class AmazonKinesisMock method getShardIterator.

@Override
public GetShardIteratorResult getShardIterator(GetShardIteratorRequest getShardIteratorRequest) {
    ShardIteratorType shardIteratorType = ShardIteratorType.fromValue(getShardIteratorRequest.getShardIteratorType());
    String shardIterator;
    if (shardIteratorType == ShardIteratorType.TRIM_HORIZON) {
        shardIterator = String.format("%s:%s", getShardIteratorRequest.getShardId(), 0);
    } else {
        throw new RuntimeException("Not implemented");
    }
    return new GetShardIteratorResult().withShardIterator(shardIterator);
}