Examples with ProvisionedThroughputExceededException com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException used on opensource projects

Example 1 with ProvisionedThroughputExceededException

use of com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException in project storm by apache.

the class KinesisRecordsManager method fetchFailedRecords.

// fetch records from kinesis starting at sequence number for message passed as argument. Any other messages fetched
// and are in the failed queue will also
// be kept in memory to avoid going to kinesis again for retry
private void fetchFailedRecords(KinesisMessageId kinesisMessageId) {
    // if shard iterator not present for this message, get it
    if (!shardIteratorPerFailedMessage.containsKey(kinesisMessageId)) {
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    }
    String shardIterator = shardIteratorPerFailedMessage.get(kinesisMessageId);
    LOG.debug("Fetching failed records for shard id :{} at sequence number {} using shardIterator {}", kinesisMessageId.getShardId(), kinesisMessageId.getSequenceNumber(), shardIterator);
    try {
        GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
        if (getRecordsResult != null) {
            List<Record> records = getRecordsResult.getRecords();
            LOG.debug("Records size from fetchFailedRecords is {}", records.size());
            // update the shard iterator to next one in case this fetch does not give the message.
            shardIteratorPerFailedMessage.put(kinesisMessageId, getRecordsResult.getNextShardIterator());
            if (records.size() == 0) {
                LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                Thread.sleep(1000);
            } else {
                // add all fetched records to the set of failed records if they are present in failed set
                for (Record record : records) {
                    KinesisMessageId current = new KinesisMessageId(kinesisMessageId.getStreamName(), kinesisMessageId.getShardId(), record.getSequenceNumber());
                    if (failedPerShard.get(kinesisMessageId.getShardId()).contains(new BigInteger(current.getSequenceNumber()))) {
                        failedandFetchedRecords.put(current, record);
                        shardIteratorPerFailedMessage.remove(current);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        LOG.warn("Thread interrupted while sleeping", ie);
    } catch (ExpiredIteratorException ex) {
        LOG.warn("shardIterator for failedRecord " + kinesisMessageId + " has expired. Refreshing shardIterator");
        refreshShardIteratorForFailedRecord(kinesisMessageId);
    } catch (ProvisionedThroughputExceededException pe) {
        try {
            LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            LOG.warn("Thread interrupted exception", e);
        }
    }
}

Example 2 with ProvisionedThroughputExceededException

use of com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException in project storm by apache.

the class KinesisRecordsManager method fetchNewRecords.

private void fetchNewRecords() {
    for (Map.Entry<String, LinkedList<Record>> entry : toEmitPerShard.entrySet()) {
        String shardId = entry.getKey();
        try {
            String shardIterator = shardIteratorPerShard.get(shardId);
            LOG.debug("Fetching new records for shard id :{} using shardIterator {} after sequence number {}", shardId, shardIterator, fetchedSequenceNumberPerShard.get(shardId));
            GetRecordsResult getRecordsResult = kinesisConnection.fetchRecords(shardIterator);
            if (getRecordsResult != null) {
                List<Record> records = getRecordsResult.getRecords();
                LOG.debug("Records size from fetchNewRecords is {}", records.size());
                // update the shard iterator to next one in case this fetch does not give the message.
                shardIteratorPerShard.put(shardId, getRecordsResult.getNextShardIterator());
                if (records.size() == 0) {
                    LOG.warn("No records returned from kinesis. Hence sleeping for 1 second");
                    Thread.sleep(1000);
                } else {
                    entry.getValue().addAll(records);
                    fetchedSequenceNumberPerShard.put(shardId, records.get(records.size() - 1).getSequenceNumber());
                }
            }
        } catch (InterruptedException ie) {
            LOG.warn("Thread interrupted while sleeping", ie);
        } catch (ExpiredIteratorException ex) {
            LOG.warn("shardIterator for shardId " + shardId + " has expired. Refreshing shardIterator");
            refreshShardIteratorForNewRecords(shardId);
        } catch (ProvisionedThroughputExceededException pe) {
            try {
                LOG.warn("ProvisionedThroughputExceededException occured. Check your limits. Sleeping for 1 second.", pe);
                Thread.sleep(1000);
            } catch (InterruptedException e) {
                LOG.warn("Thread interrupted exception", e);
            }
        }
    }
}

Example 3 with ProvisionedThroughputExceededException

use of com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException in project hazelcast by hazelcast.

the class ShardReader method handleWaitingForRecords.

private Result handleWaitingForRecords() {
    if (recordsResult.isDone()) {
        GetRecordsResult result;
        try {
            result = KinesisUtil.readResult(recordsResult);
        } catch (ProvisionedThroughputExceededException pte) {
            return dealWithThroughputExceeded();
        } catch (SdkClientException sce) {
            return dealWithReadRecordFailure(sce);
        } catch (Throwable t) {
            throw rethrow(t);
        }
        readRecordRetryTracker.reset();
        shardIterator = result.getNextShardIterator();
        data = result.getRecords();
        if (!data.isEmpty()) {
            lastSeenSeqNo = data.get(data.size() - 1).getSequenceNumber();
            millisBehindLatestMetric.set(result.getMillisBehindLatest());
        }
        if (shardIterator == null) {
            state = State.SHARD_CLOSED;
            return data.isEmpty() ? Result.CLOSED : Result.HAS_DATA;
        } else if (!data.isEmpty()) {
            state = State.HAS_DATA_NEED_TO_REQUEST_RECORDS;
            return Result.HAS_DATA;
        } else {
            state = State.NEED_TO_REQUEST_RECORDS;
            return Result.NOTHING;
        }
    } else {
        return Result.NOTHING;
    }
}

Example 4 with ProvisionedThroughputExceededException

use of com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException in project kafka-connect-kinesis by jcustenborder.

the class KinesisSourceTask method poll.

@Override
public List<SourceRecord> poll() throws InterruptedException {
    List<SourceRecord> records;
    try {
        GetRecordsResult recordsResult = this.kinesisClient.getRecords(this.recordsRequest);
        records = new ArrayList<>(recordsResult.getRecords().size());
        log.trace("poll() - {} record(s) returned from shard {}.", this.config.kinesisShardId);
        for (Record record : recordsResult.getRecords()) {
            SourceRecord sourceRecord = this.recordConverter.sourceRecord(this.config.kinesisStreamName, this.config.kinesisShardId, record);
            records.add(sourceRecord);
        }
        log.trace("poll() - Changing shard iterator to {}", recordsResult.getNextShardIterator());
        this.recordsRequest.setShardIterator(recordsResult.getNextShardIterator());
    } catch (ProvisionedThroughputExceededException ex) {
        log.warn("poll() - Throughput exceeded sleeping {} ms", this.config.kinesisThroughputExceededBackoffMs, ex);
        this.time.sleep(this.config.kinesisThroughputExceededBackoffMs);
        return new ArrayList<>();
    }
    if (records.isEmpty()) {
        log.trace("poll() - No records returned. Sleeping {} ms.", this.config.kinesisEmptyRecordsBackoffMs);
        this.time.sleep(this.config.kinesisEmptyRecordsBackoffMs);
    }
    return records;
}

Example 5 with ProvisionedThroughputExceededException

use of com.amazonaws.services.kinesis.model.ProvisionedThroughputExceededException in project kafka-connect-kinesis by jcustenborder.

the class KinesisSourceTaskTest method throughputExceeded.

@Test
public void throughputExceeded() throws InterruptedException {
    final String SEQUENCE_NUMBER = "asdfasdfddsa";
    Map<String, Object> sourceOffset = ImmutableMap.of(RecordConverter.FIELD_SEQUENCE_NUMBER, SEQUENCE_NUMBER);
    when(this.offsetStorageReader.offset(anyMap())).thenReturn(sourceOffset);
    when(this.kinesisClient.getShardIterator(any())).thenReturn(new GetShardIteratorResult().withShardIterator("dfasdfsadfasdf"));
    this.task.start(settings);
    when(this.kinesisClient.getRecords(any())).thenThrow(new ProvisionedThroughputExceededException(""));
    List<SourceRecord> records = this.task.poll();
    assertNotNull(records, "records should not be null");
    assertTrue(records.isEmpty(), "records should be empty.");
    verify(this.task.time, atLeastOnce()).sleep(this.config.kinesisThroughputExceededBackoffMs);
}