use of com.amazonaws.services.dynamodbv2.model.GetShardIteratorResult in project camel by apache.
the class ShardIteratorHandler method getShardIterator.
String getShardIterator(String resumeFromSequenceNumber) {
ShardIteratorType iteratorType = getEndpoint().getIteratorType();
String sequenceNumber = getEndpoint().getSequenceNumber();
if (resumeFromSequenceNumber != null) {
// Reset things as we're in an error condition.
currentShard = null;
currentShardIterator = null;
iteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER;
sequenceNumber = resumeFromSequenceNumber;
}
// either return a cached one or get a new one via a GetShardIterator request.
if (currentShardIterator == null) {
ListStreamsResult streamsListResult = getClient().listStreams(new ListStreamsRequest().withTableName(getEndpoint().getTableName()));
// XXX assumes there is only one stream
final String streamArn = streamsListResult.getStreams().get(0).getStreamArn();
DescribeStreamResult streamDescriptionResult = getClient().describeStream(new DescribeStreamRequest().withStreamArn(streamArn));
shardList.addAll(streamDescriptionResult.getStreamDescription().getShards());
LOG.trace("Current shard is: {} (in {})", currentShard, shardList);
if (currentShard == null) {
currentShard = resolveNewShard(iteratorType, resumeFromSequenceNumber);
} else {
currentShard = shardList.nextAfter(currentShard);
}
shardList.removeOlderThan(currentShard);
LOG.trace("Next shard is: {} (in {})", currentShard, shardList);
GetShardIteratorResult result = getClient().getShardIterator(buildGetShardIteratorRequest(streamArn, iteratorType, sequenceNumber));
currentShardIterator = result.getShardIterator();
}
LOG.trace("Shard Iterator is: {}", currentShardIterator);
return currentShardIterator;
}
use of com.amazonaws.services.dynamodbv2.model.GetShardIteratorResult in project camel by apache.
the class ShardIteratorHandlerTest method setup.
@Before
public void setup() throws Exception {
endpoint.setAmazonDynamoDbStreamsClient(amazonDynamoDBStreams);
undertest = new ShardIteratorHandler(endpoint);
when(amazonDynamoDBStreams.listStreams(any(ListStreamsRequest.class))).thenReturn(new ListStreamsResult().withStreams(new Stream().withStreamArn("arn:aws:dynamodb:region:12345:table/table_name/stream/timestamp")));
when(amazonDynamoDBStreams.describeStream(any(DescribeStreamRequest.class))).thenReturn(new DescribeStreamResult().withStreamDescription(new StreamDescription().withTableName("table_name").withShards(ShardListTest.createShardsWithSequenceNumbers(null, "a", "1", "5", "b", "8", "15", "c", "16", "16", "d", "20", null))));
when(amazonDynamoDBStreams.getShardIterator(any(GetShardIteratorRequest.class))).thenAnswer(new Answer<GetShardIteratorResult>() {
@Override
public GetShardIteratorResult answer(InvocationOnMock invocation) throws Throwable {
return new GetShardIteratorResult().withShardIterator("shard_iterator_" + ((GetShardIteratorRequest) invocation.getArguments()[0]).getShardId() + "_000");
}
});
}
use of com.amazonaws.services.dynamodbv2.model.GetShardIteratorResult in project aws-doc-sdk-examples by awsdocs.
the class StreamsLowLevelDemo method main.
public static void main(String[] args) throws InterruptedException {
AmazonDynamoDB dynamoDBClient = AmazonDynamoDBClientBuilder.standard().withRegion(Regions.US_EAST_2).withCredentials(new DefaultAWSCredentialsProviderChain()).build();
AmazonDynamoDBStreams streamsClient = AmazonDynamoDBStreamsClientBuilder.standard().withRegion(Regions.US_EAST_2).withCredentials(new DefaultAWSCredentialsProviderChain()).build();
// Create a table, with a stream enabled
String tableName = "TestTableForStreams";
ArrayList<AttributeDefinition> attributeDefinitions = new ArrayList<>(Arrays.asList(new AttributeDefinition().withAttributeName("Id").withAttributeType("N")));
ArrayList<KeySchemaElement> keySchema = new ArrayList<>(Arrays.asList(new KeySchemaElement().withAttributeName("Id").withKeyType(// Partition key
KeyType.HASH)));
StreamSpecification streamSpecification = new StreamSpecification().withStreamEnabled(true).withStreamViewType(StreamViewType.NEW_AND_OLD_IMAGES);
CreateTableRequest createTableRequest = new CreateTableRequest().withTableName(tableName).withKeySchema(keySchema).withAttributeDefinitions(attributeDefinitions).withProvisionedThroughput(new ProvisionedThroughput().withReadCapacityUnits(10L).withWriteCapacityUnits(10L)).withStreamSpecification(streamSpecification);
System.out.println("Issuing CreateTable request for " + tableName);
dynamoDBClient.createTable(createTableRequest);
System.out.println("Waiting for " + tableName + " to be created...");
try {
TableUtils.waitUntilActive(dynamoDBClient, tableName);
} catch (AmazonClientException e) {
e.printStackTrace();
}
// Print the stream settings for the table
DescribeTableResult describeTableResult = dynamoDBClient.describeTable(tableName);
String streamArn = describeTableResult.getTable().getLatestStreamArn();
System.out.println("Current stream ARN for " + tableName + ": " + describeTableResult.getTable().getLatestStreamArn());
StreamSpecification streamSpec = describeTableResult.getTable().getStreamSpecification();
System.out.println("Stream enabled: " + streamSpec.getStreamEnabled());
System.out.println("Update view type: " + streamSpec.getStreamViewType());
System.out.println();
// Generate write activity in the table
System.out.println("Performing write activities on " + tableName);
int maxItemCount = 100;
for (Integer i = 1; i <= maxItemCount; i++) {
System.out.println("Processing item " + i + " of " + maxItemCount);
// Write a new item
Map<String, AttributeValue> item = new HashMap<>();
item.put("Id", new AttributeValue().withN(i.toString()));
item.put("Message", new AttributeValue().withS("New item!"));
dynamoDBClient.putItem(tableName, item);
// Update the item
Map<String, AttributeValue> key = new HashMap<>();
key.put("Id", new AttributeValue().withN(i.toString()));
Map<String, AttributeValueUpdate> attributeUpdates = new HashMap<>();
attributeUpdates.put("Message", new AttributeValueUpdate().withAction(AttributeAction.PUT).withValue(new AttributeValue().withS("This item has changed")));
dynamoDBClient.updateItem(tableName, key, attributeUpdates);
// Delete the item
dynamoDBClient.deleteItem(tableName, key);
}
// Get all the shard IDs from the stream. Note that DescribeStream returns
// the shard IDs one page at a time.
String lastEvaluatedShardId = null;
do {
DescribeStreamResult describeStreamResult = streamsClient.describeStream(new DescribeStreamRequest().withStreamArn(streamArn).withExclusiveStartShardId(lastEvaluatedShardId));
List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
for (Shard shard : shards) {
String shardId = shard.getShardId();
System.out.println("Shard: " + shard);
// Get an iterator for the current shard
GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest().withStreamArn(streamArn).withShardId(shardId).withShardIteratorType(ShardIteratorType.TRIM_HORIZON);
GetShardIteratorResult getShardIteratorResult = streamsClient.getShardIterator(getShardIteratorRequest);
String currentShardIter = getShardIteratorResult.getShardIterator();
// Shard iterator is not null until the Shard is sealed (marked as READ_ONLY).
// To prevent running the loop until the Shard is sealed, which will be on average
// 4 hours, we process only the items that were written into DynamoDB and then exit.
int processedRecordCount = 0;
while (currentShardIter != null && processedRecordCount < maxItemCount) {
System.out.println(" Shard iterator: " + currentShardIter.substring(380));
// Use the shard iterator to read the stream records
GetRecordsResult getRecordsResult = streamsClient.getRecords(new GetRecordsRequest().withShardIterator(currentShardIter));
List<Record> records = getRecordsResult.getRecords();
for (Record record : records) {
System.out.println(" " + record.getDynamodb());
}
processedRecordCount += records.size();
currentShardIter = getRecordsResult.getNextShardIterator();
}
}
// If LastEvaluatedShardId is set, then there is
// at least one more page of shard IDs to retrieve
lastEvaluatedShardId = describeStreamResult.getStreamDescription().getLastEvaluatedShardId();
} while (lastEvaluatedShardId != null);
// Delete the table
System.out.println("Deleting the table...");
dynamoDBClient.deleteTable(tableName);
System.out.println("Demo complete");
}
Aggregations