Examples with Shard com.amazonaws.services.dynamodbv2.model.Shard used on opensource projects

Example 1 with Shard

use of com.amazonaws.services.dynamodbv2.model.Shard in project camel by apache.

the class ShardIteratorHandler method getShardIterator.

String getShardIterator(String resumeFromSequenceNumber) {
    ShardIteratorType iteratorType = getEndpoint().getIteratorType();
    String sequenceNumber = getEndpoint().getSequenceNumber();
    if (resumeFromSequenceNumber != null) {
        // Reset things as we're in an error condition.
        currentShard = null;
        currentShardIterator = null;
        iteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER;
        sequenceNumber = resumeFromSequenceNumber;
    }
    // either return a cached one or get a new one via a GetShardIterator request.
    if (currentShardIterator == null) {
        ListStreamsResult streamsListResult = getClient().listStreams(new ListStreamsRequest().withTableName(getEndpoint().getTableName()));
        // XXX assumes there is only one stream
        final String streamArn = streamsListResult.getStreams().get(0).getStreamArn();
        DescribeStreamResult streamDescriptionResult = getClient().describeStream(new DescribeStreamRequest().withStreamArn(streamArn));
        shardList.addAll(streamDescriptionResult.getStreamDescription().getShards());
        LOG.trace("Current shard is: {} (in {})", currentShard, shardList);
        if (currentShard == null) {
            currentShard = resolveNewShard(iteratorType, resumeFromSequenceNumber);
        } else {
            currentShard = shardList.nextAfter(currentShard);
        }
        shardList.removeOlderThan(currentShard);
        LOG.trace("Next shard is: {} (in {})", currentShard, shardList);
        GetShardIteratorResult result = getClient().getShardIterator(buildGetShardIteratorRequest(streamArn, iteratorType, sequenceNumber));
        currentShardIterator = result.getShardIterator();
    }
    LOG.trace("Shard Iterator is: {}", currentShardIterator);
    return currentShardIterator;
}

Example 2 with Shard

use of com.amazonaws.services.dynamodbv2.model.Shard in project camel by apache.

the class ShardListTest method nextWithNullReturnsFirstKnownShard.

@Test
public void nextWithNullReturnsFirstKnownShard() throws Exception {
    Shard first = new Shard().withShardId("first_shard");
    Shard second = new Shard().withParentShardId("first_shard").withShardId("second_shard");
    ShardList shards = new ShardList();
    shards.add(first);
    shards.add(second);
    assertThat(shards.nextAfter(first), is(second));
}

Example 3 with Shard

use of com.amazonaws.services.dynamodbv2.model.Shard in project camel by apache.

the class ShardListTest method createShards.

static List<Shard> createShards(String initialParent, String... shardIds) {
    String previous = initialParent;
    List<Shard> result = new ArrayList<>();
    for (String s : shardIds) {
        result.add(new Shard().withShardId(s).withParentShardId(previous));
        previous = s;
    }
    return result;
}

Example 4 with Shard

use of com.amazonaws.services.dynamodbv2.model.Shard in project camel by apache.

the class DdbStreamConsumer method poll.

@Override
protected int poll() throws Exception {
    GetRecordsResult result;
    try {
        GetRecordsRequest req = new GetRecordsRequest().withShardIterator(shardIteratorHandler.getShardIterator(null)).withLimit(getEndpoint().getMaxResultsPerRequest());
        result = getClient().getRecords(req);
    } catch (ExpiredIteratorException e) {
        LOG.warn("Expired Shard Iterator, attempting to resume from " + lastSeenSequenceNumber, e);
        GetRecordsRequest req = new GetRecordsRequest().withShardIterator(shardIteratorHandler.getShardIterator(lastSeenSequenceNumber)).withLimit(getEndpoint().getMaxResultsPerRequest());
        result = getClient().getRecords(req);
    }
    List<Record> records = result.getRecords();
    Queue<Exchange> exchanges = createExchanges(records, lastSeenSequenceNumber);
    int processedExchangeCount = processBatch(CastUtils.cast(exchanges));
    shardIteratorHandler.updateShardIterator(result.getNextShardIterator());
    if (!records.isEmpty()) {
        lastSeenSequenceNumber = records.get(records.size() - 1).getDynamodb().getSequenceNumber();
    }
    return processedExchangeCount;
}

Example 5 with Shard

use of com.amazonaws.services.dynamodbv2.model.Shard in project camel by apache.

the class ShardList method removeOlderThan.

/**
     * Removes shards that are older than the provided shard. Does not remove
     * the provided shard.
     *
     * @param removeBefore
     */
void removeOlderThan(Shard removeBefore) {
    String current = removeBefore.getParentShardId();
    int removedShards = 0;
    while (current != null) {
        Shard s = shards.remove(current);
        if (s == null) {
            current = null;
        } else {
            removedShards++;
            current = s.getParentShardId();
        }
    }
    log.trace("removed {} shards from the store, new size is {}", removedShards, shards.size());
}