Examples with ExpiredIteratorException com.amazonaws.services.kinesis.model.ExpiredIteratorException used on opensource projects

Example 1 with ExpiredIteratorException

use of com.amazonaws.services.kinesis.model.ExpiredIteratorException in project storm by apache.

the class KinesisRecordsManager method fetchFailedRecords.

// fetch records from kinesis starting at sequence number for message passed as argument. Any other messages fetched
// and are in the failed queue will also
// be kept in memory to avoid going to kinesis again for retry
private void fetchFailedRecords(KinesisMessageId kinesisMessageId) {
    // if shard iterator not present for this message, get it
    if (!shardIteratorPerFailedMessage.containsKey(kinesisMessageId)) {
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    }
    String shardIterator = shardIteratorPerFailedMessage.get(kinesisMessageId);
    LOG.debug("Fetching failed records for shard id :{} at sequence number {} using shardIterator {}", kinesisMessageId.getShardId(), kinesisMessageId.getSequenceNumber(), shardIterator);
    try {
        GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
        if (getRecordsResult != null) {
            List<Record> records = getRecordsResult.getRecords();
            LOG.debug("Records size from fetchFailedRecords is {}", records.size());
            // update the shard iterator to next one in case this fetch does not give the message.
            shardIteratorPerFailedMessage.put(kinesisMessageId, getRecordsResult.getNextShardIterator());
            if (records.size() == 0) {
                LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                Thread.sleep(1000);
            } else {
                // add all fetched records to the set of failed records if they are present in failed set
                for (Record record : records) {
                    KinesisMessageId current = new KinesisMessageId(kinesisMessageId.getStreamName(), kinesisMessageId.getShardId(), record.getSequenceNumber());
                    if (failedPerShard.get(kinesisMessageId.getShardId()).contains(new BigInteger(current.getSequenceNumber()))) {
                        failedandFetchedRecords.put(current, record);
                        shardIteratorPerFailedMessage.remove(current);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        LOG.warn("Thread interrupted while sleeping", ie);
    } catch (ExpiredIteratorException ex) {
        LOG.warn("shardIterator for failedRecord " + kinesisMessageId + " has expired. Refreshing shardIterator");
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    } catch (ProvisionedThroughputExceededException pe) {
        try {
            LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Thread interrupted exception", e);
        }
    }
}

Example 2 with ExpiredIteratorException

use of com.amazonaws.services.kinesis.model.ExpiredIteratorException in project storm by apache.

the class KinesisRecordsManager method fetchNewRecords.

private void fetchNewRecords() {
    for (Map.Entry<String, LinkedList<Record>> entry : toEmitPerShard.entrySet()) {
        String shardId = entry.getKey();
        try {
            String shardIterator = shardIteratorPerShard.get(shardId);
            LOG.debug("Fetching new records for shard id :{} using shardIterator {} after sequence number {}", shardId, shardIterator, fetchedSequenceNumberPerShard.get(shardId));
            GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
            if (getRecordsResult != null) {
                List<Record> records = getRecordsResult.getRecords();
                LOG.debug("Records size from fetchNewRecords is {}", records.size());
                // update the shard iterator to next one in case this fetch does not give the message.
                shardIteratorPerShard.put(shardId, getRecordsResult.getNextShardIterator());
                if (records.size() == 0) {
                    LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                    Thread.sleep(1000);
                } else {
                    entry.getValue().addAll(records);
                    fetchedSequenceNumberPerShard.put(shardId, records.get(records.size() - 1).getSequenceNumber());
                }
            }
        } catch (InterruptedException ie) {
            LOG.warn("Thread interrupted while sleeping", ie);
        } catch (ExpiredIteratorException ex) {
            LOG.warn("shardIterator for shardId " + shardId + " has expired. Refreshing shardIterator");
            refreshShardIteratorForNewRecords(shardId);
        } catch (ProvisionedThroughputExceededException pe) {
            try {
                LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                LOG.warn("Thread interrupted exception", e);
            }
        }
    }
}

Example 3 with ExpiredIteratorException

use of com.amazonaws.services.kinesis.model.ExpiredIteratorException in project flink by apache.

the class KinesisProxyTest method testIsRecoverableExceptionWithExpiredIteratorException.

@Test
public void testIsRecoverableExceptionWithExpiredIteratorException() {
    final ExpiredIteratorException ex = new ExpiredIteratorException("asdf");
    ex.setErrorType(ErrorType.Client);
    assertFalse(KinesisProxy.isRecoverableException(ex));
}