Examples with GetRecordsResult com.amazonaws.services.kinesis.model.GetRecordsResult used on opensource projects

Example 1 with GetRecordsResult

use of com.amazonaws.services.kinesis.model.GetRecordsResult in project camel by apache.

the class KinesisConsumer method poll.

@Override
protected int poll() throws Exception {
    GetRecordsRequest req = new GetRecordsRequest().withShardIterator(getShardItertor()).withLimit(getEndpoint().getMaxResultsPerRequest());
    GetRecordsResult result = getClient().getRecords(req);
    Queue<Exchange> exchanges = createExchanges(result.getRecords());
    int processedExchangeCount = processBatch(CastUtils.cast(exchanges));
    // May cache the last successful sequence number, and pass it to the
    // getRecords request. That way, on the next poll, we start from where
    // we left off, however, I don't know what happens to subsequent
    // exchanges when an earlier echangee fails.
    currentShardIterator = result.getNextShardIterator();
    return processedExchangeCount;
}

Example 2 with GetRecordsResult

use of com.amazonaws.services.kinesis.model.GetRecordsResult in project storm by apache.

the class KinesisRecordsManager method fetchFailedRecords.

// fetch records from kinesis starting at sequence number for message passed as argument. Any other messages fetched and are in the failed queue will also
// be kept in memory to avoid going to kinesis again for retry
private void fetchFailedRecords(KinesisMessageId kinesisMessageId) {
    // if shard iterator not present for this message, get it
    if (!shardIteratorPerFailedMessage.containsKey(kinesisMessageId)) {
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    }
    String shardIterator = shardIteratorPerFailedMessage.get(kinesisMessageId);
    LOG.debug("Fetching failed records for shard id :" + kinesisMessageId.getShardId() + " at sequence number " + kinesisMessageId.getSequenceNumber() + " using shardIterator " + shardIterator);
    try {
        GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
        if (getRecordsResult != null) {
            List<Record> records = getRecordsResult.getRecords();
            LOG.debug("Records size from fetchFailedRecords is " + records.size());
            // update the shard iterator to next one in case this fetch does not give the message.
            shardIteratorPerFailedMessage.put(kinesisMessageId, getRecordsResult.getNextShardIterator());
            if (records.size() == 0) {
                LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                Thread.sleep(1000);
            } else {
                // add all fetched records to the set of failed records if they are present in failed set
                for (Record record : records) {
                    KinesisMessageId current = new KinesisMessageId(kinesisMessageId.getStreamName(), kinesisMessageId.getShardId(), record.getSequenceNumber());
                    if (failedPerShard.get(kinesisMessageId.getShardId()).contains(new BigInteger(current.getSequenceNumber()))) {
                        failedandFetchedRecords.put(current, record);
                        shardIteratorPerFailedMessage.remove(current);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        LOG.warn("Thread interrupted while sleeping", ie);
    } catch (ExpiredIteratorException ex) {
        LOG.warn("shardIterator for failedRecord " + kinesisMessageId + " has expired. Refreshing shardIterator");
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    } catch (ProvisionedThroughputExceededException pe) {
        try {
            LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Thread interrupted exception", e);
        }
    }
}

Example 3 with GetRecordsResult

use of com.amazonaws.services.kinesis.model.GetRecordsResult in project storm by apache.

the class KinesisRecordsManager method fetchNewRecords.

private void fetchNewRecords() {
    for (Map.Entry<String, LinkedList<Record>> entry : toEmitPerShard.entrySet()) {
        String shardId = entry.getKey();
        try {
            String shardIterator = shardIteratorPerShard.get(shardId);
            LOG.debug("Fetching new records for shard id :" + shardId + " using shardIterator " + shardIterator + " after sequence number " + fetchedSequenceNumberPerShard.get(shardId));
            GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
            if (getRecordsResult != null) {
                List<Record> records = getRecordsResult.getRecords();
                LOG.debug("Records size from fetchNewRecords is " + records.size());
                // update the shard iterator to next one in case this fetch does not give the message.
                shardIteratorPerShard.put(shardId, getRecordsResult.getNextShardIterator());
                if (records.size() == 0) {
                    LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                    Thread.sleep(1000);
                } else {
                    entry.getValue().addAll(records);
                    fetchedSequenceNumberPerShard.put(shardId, records.get(records.size() - 1).getSequenceNumber());
                }
            }
        } catch (InterruptedException ie) {
            LOG.warn("Thread interrupted while sleeping", ie);
        } catch (ExpiredIteratorException ex) {
            LOG.warn("shardIterator for shardId " + shardId + " has expired. Refreshing shardIterator");
            refreshShardIteratorForNewRecords(shardId);
        } catch (ProvisionedThroughputExceededException pe) {
            try {
                LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                LOG.warn("Thread interrupted exception", e);
            }
        }
    }
}

Example 4 with GetRecordsResult

use of com.amazonaws.services.kinesis.model.GetRecordsResult in project flink by apache.

the class KinesisProxy method getRecords.

/**
	 * {@inheritDoc}
	 */
@Override
public GetRecordsResult getRecords(String shardIterator, int maxRecordsToGet) throws InterruptedException {
    final GetRecordsRequest getRecordsRequest = new GetRecordsRequest();
    getRecordsRequest.setShardIterator(shardIterator);
    getRecordsRequest.setLimit(maxRecordsToGet);
    GetRecordsResult getRecordsResult = null;
    int attempt = 0;
    while (attempt <= getRecordsMaxAttempts && getRecordsResult == null) {
        try {
            getRecordsResult = kinesisClient.getRecords(getRecordsRequest);
        } catch (AmazonServiceException ex) {
            if (isRecoverableException(ex)) {
                long backoffMillis = fullJitterBackoff(getRecordsBaseBackoffMillis, getRecordsMaxBackoffMillis, getRecordsExpConstant, attempt++);
                LOG.warn("Got recoverable AmazonServiceException. Backing off for " + backoffMillis + " millis (" + ex.getErrorMessage() + ")");
                Thread.sleep(backoffMillis);
            } else {
                throw ex;
            }
        }
    }
    if (getRecordsResult == null) {
        throw new RuntimeException("Rate Exceeded for getRecords operation - all " + getRecordsMaxAttempts + " retry attempts returned ProvisionedThroughputExceededException.");
    }
    return getRecordsResult;
}

Example 5 with GetRecordsResult

use of com.amazonaws.services.kinesis.model.GetRecordsResult in project flink by apache.

the class ShardConsumer method run.

@SuppressWarnings("unchecked")
@Override
public void run() {
    String nextShardItr;
    try {
        if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_LATEST_SEQUENCE_NUM.get())) {
            // if the shard is already closed, there will be no latest next record to get for this shard
            if (subscribedShard.isClosed()) {
                nextShardItr = null;
            } else {
                nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.LATEST.toString(), null);
            }
        } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get())) {
            nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.TRIM_HORIZON.toString(), null);
        } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) {
            nextShardItr = null;
        } else if (lastSequenceNum.equals(SentinelSequenceNumber.SENTINEL_AT_TIMESTAMP_SEQUENCE_NUM.get())) {
            nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_TIMESTAMP.toString(), initTimestamp);
        } else {
            if (lastSequenceNum.isAggregated()) {
                String itrForLastAggregatedRecord = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AT_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber());
                // get only the last aggregated record
                GetRecordsResult getRecordsResult = getRecords(itrForLastAggregatedRecord, 1);
                List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey());
                long lastSubSequenceNum = lastSequenceNum.getSubSequenceNumber();
                for (UserRecord record : fetchedRecords) {
                    // than our last sequence number; if so, collect the record and update state
                    if (record.getSubSequenceNumber() > lastSubSequenceNum) {
                        deserializeRecordForCollectionAndUpdateState(record);
                    }
                }
                // set the nextShardItr so we can continue iterating in the next while loop
                nextShardItr = getRecordsResult.getNextShardIterator();
            } else {
                // the last record was non-aggregated, so we can simply start from the next record
                nextShardItr = kinesis.getShardIterator(subscribedShard, ShardIteratorType.AFTER_SEQUENCE_NUMBER.toString(), lastSequenceNum.getSequenceNumber());
            }
        }
        while (isRunning()) {
            if (nextShardItr == null) {
                fetcherRef.updateState(subscribedShardStateIndex, SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get());
                // we can close this consumer thread once we've reached the end of the subscribed shard
                break;
            } else {
                if (fetchIntervalMillis != 0) {
                    Thread.sleep(fetchIntervalMillis);
                }
                GetRecordsResult getRecordsResult = getRecords(nextShardItr, maxNumberOfRecordsPerFetch);
                // each of the Kinesis records may be aggregated, so we must deaggregate them before proceeding
                List<UserRecord> fetchedRecords = deaggregateRecords(getRecordsResult.getRecords(), subscribedShard.getShard().getHashKeyRange().getStartingHashKey(), subscribedShard.getShard().getHashKeyRange().getEndingHashKey());
                for (UserRecord record : fetchedRecords) {
                    deserializeRecordForCollectionAndUpdateState(record);
                }
                nextShardItr = getRecordsResult.getNextShardIterator();
            }
        }
    } catch (Throwable t) {
        fetcherRef.stopWithError(t);
    }
}