Examples with Stream com.amazonaws.services.dynamodbv2.model.Stream used on opensource projects

Example 1 with Stream

use of com.amazonaws.services.dynamodbv2.model.Stream in project camel by apache.

the class ShardIteratorHandler method getShardIterator.

String getShardIterator(String resumeFromSequenceNumber) {
    ShardIteratorType iteratorType = getEndpoint().getIteratorType();
    String sequenceNumber = getEndpoint().getSequenceNumber();
    if (resumeFromSequenceNumber != null) {
        // Reset things as we're in an error condition.
        currentShard = null;
        currentShardIterator = null;
        iteratorType = ShardIteratorType.AFTER_SEQUENCE_NUMBER;
        sequenceNumber = resumeFromSequenceNumber;
    }
    // either return a cached one or get a new one via a GetShardIterator request.
    if (currentShardIterator == null) {
        ListStreamsResult streamsListResult = getClient().listStreams(new ListStreamsRequest().withTableName(getEndpoint().getTableName()));
        // XXX assumes there is only one stream
        final String streamArn = streamsListResult.getStreams().get(0).getStreamArn();
        DescribeStreamResult streamDescriptionResult = getClient().describeStream(new DescribeStreamRequest().withStreamArn(streamArn));
        shardList.addAll(streamDescriptionResult.getStreamDescription().getShards());
        LOG.trace("Current shard is: {} (in {})", currentShard, shardList);
        if (currentShard == null) {
            currentShard = resolveNewShard(iteratorType, resumeFromSequenceNumber);
        } else {
            currentShard = shardList.nextAfter(currentShard);
        }
        shardList.removeOlderThan(currentShard);
        LOG.trace("Next shard is: {} (in {})", currentShard, shardList);
        GetShardIteratorResult result = getClient().getShardIterator(buildGetShardIteratorRequest(streamArn, iteratorType, sequenceNumber));
        currentShardIterator = result.getShardIterator();
    }
    LOG.trace("Shard Iterator is: {}", currentShardIterator);
    return currentShardIterator;
}

Example 2 with Stream

use of com.amazonaws.services.dynamodbv2.model.Stream in project athenz by yahoo.

the class DynamoDBStatusChecker method check.

@Override
public void check() throws StatusCheckException {
    DynamoDBClientAndCredentials clientAndCreds = null;
    try {
        // Get DynamoDB client and temp credentials (if required)
        DynamoDBClientFetcher dynamoDBClientFetcher = getDynamoDBClientFetcher();
        clientAndCreds = dynamoDBClientFetcher.getDynamoDBClient(null, keyStore);
        AmazonDynamoDB amazonDynamoDB = clientAndCreds.getAmazonDynamoDB();
        // Get list of tables and verify our table appears
        boolean tableFound = amazonDynamoDB.listTables().getTableNames().stream().anyMatch(fetchedTableName -> fetchedTableName.equals(tableName));
        if (!tableFound) {
            throw new StatusCheckException(HttpStatus.SC_OK, "Table named " + tableName + " wasn't found in DynamoDB");
        }
    } catch (StatusCheckException ex) {
        throw ex;
    } catch (Throwable ex) {
        throw new StatusCheckException(ex);
    } finally {
        // Close resources
        if (clientAndCreds != null) {
            try {
                if (clientAndCreds.getAmazonDynamoDB() != null) {
                    clientAndCreds.getAmazonDynamoDB().shutdown();
                }
                if (clientAndCreds.getAwsCredentialsProvider() != null) {
                    clientAndCreds.getAwsCredentialsProvider().close();
                }
            } catch (IOException ignored) {
            }
        }
    }
}

Example 3 with Stream

use of com.amazonaws.services.dynamodbv2.model.Stream in project camel by apache.

the class ShardIteratorHandlerTest method setup.

@Before
public void setup() throws Exception {
    endpoint.setAmazonDynamoDbStreamsClient(amazonDynamoDBStreams);
    undertest = new ShardIteratorHandler(endpoint);
    when(amazonDynamoDBStreams.listStreams(any(ListStreamsRequest.class))).thenReturn(new ListStreamsResult().withStreams(new Stream().withStreamArn("arn:aws:dynamodb:region:12345:table/table_name/stream/timestamp")));
    when(amazonDynamoDBStreams.describeStream(any(DescribeStreamRequest.class))).thenReturn(new DescribeStreamResult().withStreamDescription(new StreamDescription().withTableName("table_name").withShards(ShardListTest.createShardsWithSequenceNumbers(null, "a", "1", "5", "b", "8", "15", "c", "16", "16", "d", "20", null))));
    when(amazonDynamoDBStreams.getShardIterator(any(GetShardIteratorRequest.class))).thenAnswer(new Answer<GetShardIteratorResult>() {

        @Override
        public GetShardIteratorResult answer(InvocationOnMock invocation) throws Throwable {
            return new GetShardIteratorResult().withShardIterator("shard_iterator_" + ((GetShardIteratorRequest) invocation.getArguments()[0]).getShardId() + "_000");
        }
    });
}

Example 4 with Stream

use of com.amazonaws.services.dynamodbv2.model.Stream in project aws-doc-sdk-examples by awsdocs.

the class StreamsAdapterDemo method main.

/**
 * @param args
 */
public static void main(String[] args) throws Exception {
    System.out.println("Starting demo...");
    dynamoDBClient = AmazonDynamoDBClientBuilder.standard().withRegion(awsRegion).build();
    cloudWatchClient = AmazonCloudWatchClientBuilder.standard().withRegion(awsRegion).build();
    dynamoDBStreamsClient = AmazonDynamoDBStreamsClientBuilder.standard().withRegion(awsRegion).build();
    adapterClient = new AmazonDynamoDBStreamsAdapterClient(dynamoDBStreamsClient);
    String srcTable = tablePrefix + "-src";
    String destTable = tablePrefix + "-dest";
    recordProcessorFactory = new StreamsRecordProcessorFactory(dynamoDBClient, destTable);
    setUpTables();
    workerConfig = new KinesisClientLibConfiguration("streams-adapter-demo", streamArn, awsCredentialsProvider, "streams-demo-worker").withMaxRecords(1000).withIdleTimeBetweenReadsInMillis(500).withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON);
    System.out.println("Creating worker for stream: " + streamArn);
    worker = StreamsWorkerFactory.createDynamoDbStreamsWorker(recordProcessorFactory, workerConfig, adapterClient, dynamoDBClient, cloudWatchClient);
    System.out.println("Starting worker...");
    Thread t = new Thread(worker);
    t.start();
    Thread.sleep(25000);
    worker.shutdown();
    t.join();
    if (StreamsAdapterDemoHelper.scanTable(dynamoDBClient, srcTable).getItems().equals(StreamsAdapterDemoHelper.scanTable(dynamoDBClient, destTable).getItems())) {
        System.out.println("Scan result is equal.");
    } else {
        System.out.println("Tables are different!");
    }
    System.out.println("Done.");
    cleanupAndExit(0);
}

Example 5 with Stream

use of com.amazonaws.services.dynamodbv2.model.Stream in project aws-doc-sdk-examples by awsdocs.

the class StreamsLowLevelDemo method main.

public static void main(String[] args) throws InterruptedException {
    AmazonDynamoDB dynamoDBClient = AmazonDynamoDBClientBuilder.standard().withRegion(Regions.US_EAST_2).withCredentials(new DefaultAWSCredentialsProviderChain()).build();
    AmazonDynamoDBStreams streamsClient = AmazonDynamoDBStreamsClientBuilder.standard().withRegion(Regions.US_EAST_2).withCredentials(new DefaultAWSCredentialsProviderChain()).build();
    // Create a table, with a stream enabled
    String tableName = "TestTableForStreams";
    ArrayList<AttributeDefinition> attributeDefinitions = new ArrayList<>(Arrays.asList(new AttributeDefinition().withAttributeName("Id").withAttributeType("N")));
    ArrayList<KeySchemaElement> keySchema = new ArrayList<>(Arrays.asList(new KeySchemaElement().withAttributeName("Id").withKeyType(// Partition key
    KeyType.HASH)));
    StreamSpecification streamSpecification = new StreamSpecification().withStreamEnabled(true).withStreamViewType(StreamViewType.NEW_AND_OLD_IMAGES);
    CreateTableRequest createTableRequest = new CreateTableRequest().withTableName(tableName).withKeySchema(keySchema).withAttributeDefinitions(attributeDefinitions).withProvisionedThroughput(new ProvisionedThroughput().withReadCapacityUnits(10L).withWriteCapacityUnits(10L)).withStreamSpecification(streamSpecification);
    System.out.println("Issuing CreateTable request for " + tableName);
    dynamoDBClient.createTable(createTableRequest);
    System.out.println("Waiting for " + tableName + " to be created...");
    try {
        TableUtils.waitUntilActive(dynamoDBClient, tableName);
    } catch (AmazonClientException e) {
        e.printStackTrace();
    }
    // Print the stream settings for the table
    DescribeTableResult describeTableResult = dynamoDBClient.describeTable(tableName);
    String streamArn = describeTableResult.getTable().getLatestStreamArn();
    System.out.println("Current stream ARN for " + tableName + ": " + describeTableResult.getTable().getLatestStreamArn());
    StreamSpecification streamSpec = describeTableResult.getTable().getStreamSpecification();
    System.out.println("Stream enabled: " + streamSpec.getStreamEnabled());
    System.out.println("Update view type: " + streamSpec.getStreamViewType());
    System.out.println();
    // Generate write activity in the table
    System.out.println("Performing write activities on " + tableName);
    int maxItemCount = 100;
    for (Integer i = 1; i <= maxItemCount; i++) {
        System.out.println("Processing item " + i + " of " + maxItemCount);
        // Write a new item
        Map<String, AttributeValue> item = new HashMap<>();
        item.put("Id", new AttributeValue().withN(i.toString()));
        item.put("Message", new AttributeValue().withS("New item!"));
        dynamoDBClient.putItem(tableName, item);
        // Update the item
        Map<String, AttributeValue> key = new HashMap<>();
        key.put("Id", new AttributeValue().withN(i.toString()));
        Map<String, AttributeValueUpdate> attributeUpdates = new HashMap<>();
        attributeUpdates.put("Message", new AttributeValueUpdate().withAction(AttributeAction.PUT).withValue(new AttributeValue().withS("This item has changed")));
        dynamoDBClient.updateItem(tableName, key, attributeUpdates);
        // Delete the item
        dynamoDBClient.deleteItem(tableName, key);
    }
    // Get all the shard IDs from the stream.  Note that DescribeStream returns
    // the shard IDs one page at a time.
    String lastEvaluatedShardId = null;
    do {
        DescribeStreamResult describeStreamResult = streamsClient.describeStream(new DescribeStreamRequest().withStreamArn(streamArn).withExclusiveStartShardId(lastEvaluatedShardId));
        List<Shard> shards = describeStreamResult.getStreamDescription().getShards();
        for (Shard shard : shards) {
            String shardId = shard.getShardId();
            System.out.println("Shard: " + shard);
            // Get an iterator for the current shard
            GetShardIteratorRequest getShardIteratorRequest = new GetShardIteratorRequest().withStreamArn(streamArn).withShardId(shardId).withShardIteratorType(ShardIteratorType.TRIM_HORIZON);
            GetShardIteratorResult getShardIteratorResult = streamsClient.getShardIterator(getShardIteratorRequest);
            String currentShardIter = getShardIteratorResult.getShardIterator();
            // Shard iterator is not null until the Shard is sealed (marked as READ_ONLY).
            // To prevent running the loop until the Shard is sealed, which will be on average
            // 4 hours, we process only the items that were written into DynamoDB and then exit.
            int processedRecordCount = 0;
            while (currentShardIter != null && processedRecordCount < maxItemCount) {
                System.out.println("    Shard iterator: " + currentShardIter.substring(380));
                // Use the shard iterator to read the stream records
                GetRecordsResult getRecordsResult = streamsClient.getRecords(new GetRecordsRequest().withShardIterator(currentShardIter));
                List<Record> records = getRecordsResult.getRecords();
                for (Record record : records) {
                    System.out.println("        " + record.getDynamodb());
                }
                processedRecordCount += records.size();
                currentShardIter = getRecordsResult.getNextShardIterator();
            }
        }
        // If LastEvaluatedShardId is set, then there is
        // at least one more page of shard IDs to retrieve
        lastEvaluatedShardId = describeStreamResult.getStreamDescription().getLastEvaluatedShardId();
    } while (lastEvaluatedShardId != null);
    // Delete the table
    System.out.println("Deleting the table...");
    dynamoDBClient.deleteTable(tableName);
    System.out.println("Demo complete");
}