Examples with BatchWriteItemResponse software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse used on opensource projects

Example 1 with BatchWriteItemResponse

use of software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse in project para by Erudika.

the class AWSDynamoUtils method batchWrite.

/**
 * Writes multiple items in batch.
 * @param items a map of tables->write requests
 * @param backoff backoff seconds
 */
protected static void batchWrite(Map<String, List<WriteRequest>> items, int backoff) {
    if (items == null || items.isEmpty()) {
        return;
    }
    try {
        logger.debug("batchWrite(): requests {}, backoff {}", items.values().iterator().next().size(), backoff);
        BatchWriteItemResponse result = getClient().batchWriteItem(b -> b.returnConsumedCapacity(ReturnConsumedCapacity.TOTAL).requestItems(items));
        if (result == null) {
            return;
        }
        logger.debug("batchWrite(): success - consumed capacity {}", result.consumedCapacity());
        if (result.unprocessedItems() != null && !result.unprocessedItems().isEmpty()) {
            Thread.sleep((long) backoff * 1000L);
            for (Map.Entry<String, List<WriteRequest>> entry : result.unprocessedItems().entrySet()) {
                logger.warn("UNPROCESSED DynamoDB write requests for keys {} in table {}!", entry.getValue().stream().map(r -> r.getValueForField(Config._KEY, String.class).orElse("")).collect(Collectors.joining(",")), entry.getKey());
            }
            batchWrite(result.unprocessedItems(), backoff * 2);
        }
    } catch (ProvisionedThroughputExceededException ex) {
        logger.warn("Write capacity exceeded for table '{}'. Retrying request in {} seconds.", items.keySet().iterator().next(), backoff);
        try {
            Thread.sleep((long) backoff * 1000L);
            // retry forever
            batchWrite(items, backoff * 2);
        } catch (InterruptedException ie) {
            logger.error(null, ie);
            Thread.currentThread().interrupt();
        }
    } catch (InterruptedException ie) {
        logger.error(null, ie);
        Thread.currentThread().interrupt();
    } catch (Exception e) {
        logger.error("Failed to execute batch write operation on table '{}'", items.keySet().iterator().next(), e);
        throwIfNecessary(e);
    }
}

Example 2 with BatchWriteItemResponse

use of software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse in project aws-sdk-java-v2 by aws.

the class DynamoServiceIntegrationTest method testServiceOperations.

/**
 * Tests that we can call each service operation to create and describe
 * tables, put, update and delete data, and query.
 */
@Test
public void testServiceOperations() throws Exception {
    // Describe all tables
    ListTablesResponse describeTablesResult = dynamo.listTables(ListTablesRequest.builder().build());
    // Describe our new table
    DescribeTableRequest describeTablesRequest = DescribeTableRequest.builder().tableName(tableName).build();
    TableDescription tableDescription = dynamo.describeTable(describeTablesRequest).table();
    assertEquals(tableName, tableDescription.tableName());
    assertNotNull(tableDescription.tableStatus());
    assertEquals(HASH_KEY_NAME, tableDescription.keySchema().get(0).attributeName());
    assertEquals(KeyType.HASH, tableDescription.keySchema().get(0).keyType());
    assertNotNull(tableDescription.provisionedThroughput().numberOfDecreasesToday());
    assertEquals(READ_CAPACITY, tableDescription.provisionedThroughput().readCapacityUnits());
    assertEquals(WRITE_CAPACITY, tableDescription.provisionedThroughput().writeCapacityUnits());
    // Add some data
    int contentLength = 1 * 1024;
    Set<SdkBytes> byteBufferSet = new HashSet<SdkBytes>();
    byteBufferSet.add(SdkBytes.fromByteArray(generateByteArray(contentLength)));
    byteBufferSet.add(SdkBytes.fromByteArray(generateByteArray(contentLength + 1)));
    Map<String, AttributeValue> item = new HashMap<String, AttributeValue>();
    item.put(HASH_KEY_NAME, AttributeValue.builder().s("bar").build());
    item.put("age", AttributeValue.builder().n("30").build());
    item.put("bar", AttributeValue.builder().s("" + System.currentTimeMillis()).build());
    item.put("foos", AttributeValue.builder().ss("bleh", "blah").build());
    item.put("S", AttributeValue.builder().ss("ONE", "TWO").build());
    item.put("blob", AttributeValue.builder().b(SdkBytes.fromByteArray(generateByteArray(contentLength))).build());
    item.put("blobs", AttributeValue.builder().bs(SdkBytes.fromByteArray(generateByteArray(contentLength)), SdkBytes.fromByteArray(generateByteArray(contentLength + 1))).build());
    item.put("BS", AttributeValue.builder().bs(byteBufferSet).build());
    PutItemRequest putItemRequest = PutItemRequest.builder().tableName(tableName).item(item).returnValues(ReturnValue.ALL_OLD.toString()).build();
    PutItemResponse putItemResult = dynamo.putItem(putItemRequest);
    // Get our new item
    GetItemResponse itemResult = dynamo.getItem(GetItemRequest.builder().tableName(tableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("bar").build())).consistentRead(true).build());
    assertNotNull(itemResult.item().get("S").ss());
    assertEquals(2, itemResult.item().get("S").ss().size());
    assertTrue(itemResult.item().get("S").ss().contains("ONE"));
    assertTrue(itemResult.item().get("S").ss().contains("TWO"));
    assertEquals("30", itemResult.item().get("age").n());
    assertNotNull(itemResult.item().get("bar").s());
    assertNotNull(itemResult.item().get("blob").b());
    assertTrue(itemResult.item().get("blob").b().equals(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    assertNotNull(itemResult.item().get("blobs").bs());
    assertEquals(2, itemResult.item().get("blobs").bs().size());
    assertTrue(itemResult.item().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    assertTrue(itemResult.item().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))));
    assertNotNull(itemResult.item().get("BS").bs());
    assertEquals(2, itemResult.item().get("BS").bs().size());
    assertTrue(itemResult.item().get("BS").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    assertTrue(itemResult.item().get("BS").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))));
    // Pause to try and deal with ProvisionedThroughputExceededExceptions
    Thread.sleep(1000 * 5);
    // Add some data into the table with binary hash key
    ByteBuffer byteBuffer = ByteBuffer.allocate(contentLength * 2);
    byteBuffer.put(generateByteArray(contentLength));
    byteBuffer.flip();
    item = new HashMap<String, AttributeValue>();
    item.put(HASH_KEY_NAME, AttributeValue.builder().b(SdkBytes.fromByteBuffer(byteBuffer)).build());
    // Reuse the byteBuffer
    item.put("blob", AttributeValue.builder().b(SdkBytes.fromByteBuffer(byteBuffer)).build());
    item.put("blobs", AttributeValue.builder().bs(SdkBytes.fromByteArray(generateByteArray(contentLength)), SdkBytes.fromByteArray(generateByteArray(contentLength + 1))).build());
    // Reuse the byteBufferSet
    item.put("BS", AttributeValue.builder().bs(byteBufferSet).build());
    putItemRequest = PutItemRequest.builder().tableName(binaryKeyTableName).item(item).returnValues(ReturnValue.ALL_OLD.toString()).build();
    dynamo.putItem(putItemRequest);
    // Get our new item
    itemResult = dynamo.getItem(GetItemRequest.builder().tableName(binaryKeyTableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().b(SdkBytes.fromByteBuffer(byteBuffer)).build())).consistentRead(true).build());
    assertNotNull(itemResult.item().get("blob").b());
    assertEquals(itemResult.item().get("blob").b(), SdkBytes.fromByteArray(generateByteArray(contentLength)));
    assertNotNull(itemResult.item().get("blobs").bs());
    assertEquals(2, itemResult.item().get("blobs").bs().size());
    assertTrue(itemResult.item().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    assertTrue(itemResult.item().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))));
    assertNotNull(itemResult.item().get("BS").bs());
    assertEquals(2, itemResult.item().get("BS").bs().size());
    assertTrue(itemResult.item().get("BS").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    assertTrue(itemResult.item().get("BS").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))));
    // Pause to try and deal with ProvisionedThroughputExceededExceptions
    Thread.sleep(1000 * 5);
    // Load some random data
    System.out.println("Loading data...");
    Random random = new Random();
    for (int i = 0; i < 50; i++) {
        item = new HashMap<String, AttributeValue>();
        item.put(HASH_KEY_NAME, AttributeValue.builder().s("bar-" + System.currentTimeMillis()).build());
        item.put("age", AttributeValue.builder().n(Integer.toString(random.nextInt(100) + 30)).build());
        item.put("bar", AttributeValue.builder().s("" + System.currentTimeMillis()).build());
        item.put("foos", AttributeValue.builder().ss("bleh", "blah").build());
        dynamo.putItem(PutItemRequest.builder().tableName(tableName).item(item).returnValues(ReturnValue.ALL_OLD.toString()).build());
    }
    // Update an item
    Map<String, AttributeValueUpdate> itemUpdates = new HashMap<String, AttributeValueUpdate>();
    itemUpdates.put("1", AttributeValueUpdate.builder().value(AttributeValue.builder().s("¢").build()).action(AttributeAction.PUT.toString()).build());
    itemUpdates.put("foos", AttributeValueUpdate.builder().value(AttributeValue.builder().ss("foo").build()).action(AttributeAction.PUT.toString()).build());
    itemUpdates.put("S", AttributeValueUpdate.builder().value(AttributeValue.builder().ss("THREE").build()).action(AttributeAction.ADD.toString()).build());
    itemUpdates.put("age", AttributeValueUpdate.builder().value(AttributeValue.builder().n("10").build()).action(AttributeAction.ADD.toString()).build());
    itemUpdates.put("blob", AttributeValueUpdate.builder().value(AttributeValue.builder().b(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))).build()).action(AttributeAction.PUT.toString()).build());
    itemUpdates.put("blobs", AttributeValueUpdate.builder().value(AttributeValue.builder().bs(SdkBytes.fromByteArray(generateByteArray(contentLength))).build()).action(AttributeAction.PUT.toString()).build());
    UpdateItemRequest updateItemRequest = UpdateItemRequest.builder().tableName(tableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("bar").build())).attributeUpdates(itemUpdates).returnValues("ALL_NEW").build();
    UpdateItemResponse updateItemResult = dynamo.updateItem(updateItemRequest);
    assertEquals("¢", updateItemResult.attributes().get("1").s());
    assertEquals(1, updateItemResult.attributes().get("foos").ss().size());
    assertTrue(updateItemResult.attributes().get("foos").ss().contains("foo"));
    assertEquals(3, updateItemResult.attributes().get("S").ss().size());
    assertTrue(updateItemResult.attributes().get("S").ss().contains("ONE"));
    assertTrue(updateItemResult.attributes().get("S").ss().contains("TWO"));
    assertTrue(updateItemResult.attributes().get("S").ss().contains("THREE"));
    assertEquals(Integer.toString(30 + 10), updateItemResult.attributes().get("age").n());
    assertEquals(updateItemResult.attributes().get("blob").b(), SdkBytes.fromByteArray(generateByteArray(contentLength + 1)));
    assertEquals(1, updateItemResult.attributes().get("blobs").bs().size());
    assertTrue(updateItemResult.attributes().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    itemUpdates.clear();
    itemUpdates.put("age", AttributeValueUpdate.builder().value(AttributeValue.builder().n("30").build()).action(AttributeAction.PUT.toString()).build());
    itemUpdates.put("blobs", AttributeValueUpdate.builder().value(AttributeValue.builder().bs(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))).build()).action(AttributeAction.ADD.toString()).build());
    updateItemRequest = UpdateItemRequest.builder().tableName(tableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("bar").build())).attributeUpdates(itemUpdates).returnValues("ALL_NEW").build();
    updateItemResult = dynamo.updateItem(updateItemRequest);
    assertEquals("30", updateItemResult.attributes().get("age").n());
    assertEquals(2, updateItemResult.attributes().get("blobs").bs().size());
    assertTrue(updateItemResult.attributes().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength))));
    assertTrue(updateItemResult.attributes().get("blobs").bs().contains(SdkBytes.fromByteArray(generateByteArray(contentLength + 1))));
    // Get an item that doesn't exist.
    GetItemRequest itemsRequest = GetItemRequest.builder().tableName(tableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("3").build())).consistentRead(true).build();
    GetItemResponse itemsResult = dynamo.getItem(itemsRequest);
    assertTrue(itemsResult.item() instanceof SdkAutoConstructMap);
    // Get an item that doesn't have any attributes,
    itemsRequest = GetItemRequest.builder().tableName(tableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("bar").build())).consistentRead(true).attributesToGet("non-existent-attribute").build();
    itemsResult = dynamo.getItem(itemsRequest);
    assertEquals(0, itemsResult.item().size());
    // Scan data
    ScanRequest scanRequest = ScanRequest.builder().tableName(tableName).attributesToGet(HASH_KEY_NAME).build();
    ScanResponse scanResult = dynamo.scan(scanRequest);
    assertTrue(scanResult.count() > 0);
    assertTrue(scanResult.scannedCount() > 0);
    // Try a more advanced Scan query and run it a few times for performance metrics
    System.out.println("Testing Scan...");
    for (int i = 0; i < 10; i++) {
        HashMap<String, Condition> scanFilter = new HashMap<String, Condition>();
        scanFilter.put("age", Condition.builder().attributeValueList(AttributeValue.builder().n("40").build()).comparisonOperator(ComparisonOperator.GT.toString()).build());
        scanRequest = ScanRequest.builder().tableName(tableName).scanFilter(scanFilter).build();
        scanResult = dynamo.scan(scanRequest);
    }
    // Batch write
    HashMap<String, List<WriteRequest>> requestItems = new HashMap<String, List<WriteRequest>>();
    List<WriteRequest> writeRequests = new ArrayList<WriteRequest>();
    HashMap<String, AttributeValue> writeAttributes = new HashMap<String, AttributeValue>();
    writeAttributes.put(HASH_KEY_NAME, AttributeValue.builder().s("" + System.currentTimeMillis()).build());
    writeAttributes.put("bar", AttributeValue.builder().s("" + System.currentTimeMillis()).build());
    writeRequests.add(WriteRequest.builder().putRequest(PutRequest.builder().item(writeAttributes).build()).build());
    writeRequests.add(WriteRequest.builder().deleteRequest(DeleteRequest.builder().key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("toDelete").build())).build()).build());
    requestItems.put(tableName, writeRequests);
    BatchWriteItemResponse batchWriteItem = dynamo.batchWriteItem(BatchWriteItemRequest.builder().requestItems(requestItems).build());
    // assertNotNull(batchWriteItem.itemCollectionMetrics());
    // assertEquals(1, batchWriteItem.itemCollectionMetrics().size());
    // assertEquals(tableName, batchWriteItem.itemCollectionMetrics().entrySet().iterator().next().get);
    // assertNotNull(tableName, batchWriteItem.getResponses().iterator().next().getCapacityUnits());
    assertNotNull(batchWriteItem.unprocessedItems());
    assertTrue(batchWriteItem.unprocessedItems().isEmpty());
    // Delete some data
    DeleteItemRequest deleteItemRequest = DeleteItemRequest.builder().tableName(tableName).key(mapKey(HASH_KEY_NAME, AttributeValue.builder().s("jeep").build())).returnValues(ReturnValue.ALL_OLD.toString()).build();
    DeleteItemResponse deleteItemResult = dynamo.deleteItem(deleteItemRequest);
    // Delete our table
    DeleteTableResponse deleteTable = dynamo.deleteTable(DeleteTableRequest.builder().tableName(tableName).build());
}

Example 3 with BatchWriteItemResponse

use of software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse in project aws-sdk-java-v2 by aws.

the class BatchWriteItemOperationTest method transformResults_multipleUnprocessedOperations_extensionTransformsPutsNotDeletes.

@Test
public void transformResults_multipleUnprocessedOperations_extensionTransformsPutsNotDeletes() {
    BatchWriteItemOperation operation = BatchWriteItemOperation.create(emptyRequest());
    List<WriteRequest> writeRequests1 = Arrays.asList(putRequest(FAKE_ITEM_MAPS.get(0)), deleteRequest(FAKE_ITEM_MAPS.get(1)), deleteRequest(FAKE_ITEM_MAPS.get(2)));
    List<WriteRequest> writeRequests2 = Arrays.asList(deleteRequest(FAKESORT_ITEM_MAPS.get(0)), putRequest(FAKESORT_ITEM_MAPS.get(1)), putRequest(FAKESORT_ITEM_MAPS.get(2)));
    Map<String, List<WriteRequest>> writeRequests = new HashMap<>();
    writeRequests.put(TABLE_NAME, writeRequests1);
    writeRequests.put(TABLE_NAME_2, writeRequests2);
    BatchWriteItemResponse response = BatchWriteItemResponse.builder().unprocessedItems(writeRequests).build();
    // Use the mock extension to transform every item based on table name
    IntStream.range(0, 3).forEach(i -> {
        doReturn(ReadModification.builder().transformedItem(FAKE_ITEM_MAPS.get(i + 3)).build()).when(mockExtension).afterRead(argThat(extensionContext -> extensionContext.operationContext().tableName().equals(TABLE_NAME) && extensionContext.items().equals(FAKE_ITEM_MAPS.get(i))));
        doReturn(ReadModification.builder().transformedItem(FAKESORT_ITEM_MAPS.get(i + 3)).build()).when(mockExtension).afterRead(argThat(extensionContext -> extensionContext.operationContext().tableName().equals(TABLE_NAME_2) && extensionContext.items().equals(FAKESORT_ITEM_MAPS.get(i))));
    });
    BatchWriteResult results = operation.transformResponse(response, mockExtension);
    assertThat(results.unprocessedDeleteItemsForTable(fakeItemMappedTableWithExtension), containsInAnyOrder(FAKE_ITEM_KEYS.get(1), FAKE_ITEM_KEYS.get(2)));
    assertThat(results.unprocessedPutItemsForTable(fakeItemMappedTableWithExtension), containsInAnyOrder(FAKE_ITEMS.get(3)));
    assertThat(results.unprocessedDeleteItemsForTable(fakeItemWithSortMappedTableWithExtension), containsInAnyOrder(FAKESORT_ITEM_KEYS.get(0)));
    assertThat(results.unprocessedPutItemsForTable(fakeItemWithSortMappedTableWithExtension), containsInAnyOrder(FAKESORT_ITEMS.get(4), FAKESORT_ITEMS.get(5)));
}

Example 4 with BatchWriteItemResponse

use of software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse in project aws-sdk-java-v2 by aws.

the class BatchWriteItemOperationTest method transformResults_noUnprocessedOperations.

@Test
public void transformResults_noUnprocessedOperations() {
    BatchWriteItemOperation operation = BatchWriteItemOperation.create(emptyRequest());
    BatchWriteItemResponse response = BatchWriteItemResponse.builder().unprocessedItems(emptyMap()).build();
    BatchWriteResult results = operation.transformResponse(response, mockExtension);
    assertThat(results.unprocessedDeleteItemsForTable(fakeItemMappedTable), is(emptyList()));
    assertThat(results.unprocessedPutItemsForTable(fakeItemMappedTable), is(emptyList()));
}

Example 5 with BatchWriteItemResponse

use of software.amazon.awssdk.services.dynamodb.model.BatchWriteItemResponse in project aws-sdk-java-v2 by aws.

the class BatchWriteItemOperationTest method getServiceCall_makesTheRightCallAndReturnsResponse.

@Test
public void getServiceCall_makesTheRightCallAndReturnsResponse() {
    WriteBatch batch = WriteBatch.builder(FakeItem.class).mappedTableResource(fakeItemMappedTable).addPutItem(r -> r.item(FAKE_ITEMS.get(0))).build();
    BatchWriteItemEnhancedRequest batchWriteItemEnhancedRequest = BatchWriteItemEnhancedRequest.builder().writeBatches(batch).build();
    BatchWriteItemOperation operation = BatchWriteItemOperation.create(batchWriteItemEnhancedRequest);
    WriteRequest writeRequest = WriteRequest.builder().putRequest(PutRequest.builder().item(FAKE_ITEM_MAPS.get(0)).build()).build();
    BatchWriteItemRequest request = BatchWriteItemRequest.builder().requestItems(singletonMap("table", singletonList(writeRequest))).build();
    BatchWriteItemResponse expectedResponse = BatchWriteItemResponse.builder().build();
    when(mockDynamoDbClient.batchWriteItem(any(BatchWriteItemRequest.class))).thenReturn(expectedResponse);
    BatchWriteItemResponse response = operation.serviceCall(mockDynamoDbClient).apply(request);
    assertThat(response, sameInstance(expectedResponse));
    verify(mockDynamoDbClient).batchWriteItem(request);
}