Examples with FailedBatch com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper.FailedBatch used on opensource projects

Example 1 with FailedBatch

use of com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper.FailedBatch in project BridgeServer2 by Sage-Bionetworks.

the class BridgeUtils method ifFailuresThrowException.

/**
 * All batch methods in Dynamo return a list of failures rather than
 * throwing an exception. We should have an exception specifically for
 * these so the caller gets a list of items back, but for now, convert
 * to a generic exception;
 * @param failures
 */
public static void ifFailuresThrowException(List<FailedBatch> failures) {
    if (!failures.isEmpty()) {
        List<String> messages = Lists.newArrayList();
        for (FailedBatch failure : failures) {
            String message = failure.getException().getMessage();
            messages.add(message);
            String ids = Joiner.on("; ").join(failure.getUnprocessedItems().keySet());
            messages.add(ids);
        }
        throw new BridgeServiceException(Joiner.on(", ").join(messages));
    }
}

Example 2 with FailedBatch

use of com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper.FailedBatch in project aws-sdk-android by aws-amplify.

the class DynamoDBMapperTest method testWriteOneBatchWithEntityTooLarge.

@Test
public void testWriteOneBatchWithEntityTooLarge() {
    Map<String, List<WriteRequest>> batchMap = new HashMap<String, List<WriteRequest>>();
    List<WriteRequest> batchList = new ArrayList<WriteRequest>();
    WriteRequest wr1 = new WriteRequest();
    WriteRequest wr2 = new WriteRequest();
    WriteRequest wr3 = new WriteRequest();
    batchList.add(wr1);
    batchList.add(wr2);
    batchList.add(wr3);
    batchMap.put("testTable", batchList);
    EasyMock.reset(mockClient);
    AmazonServiceException ase = new AmazonServiceException("TestException");
    ase.setErrorCode("Request entity too large");
    BatchWriteItemResult mockResult = EasyMock.createMock(BatchWriteItemResult.class);
    EasyMock.reset(mockResult);
    EasyMock.expect(mockResult.getUnprocessedItems()).andReturn(new HashMap<String, List<WriteRequest>>()).times(2);
    // Will cause batches to be split and re-tried
    EasyMock.expect(mockClient.batchWriteItem(anyObject(BatchWriteItemRequest.class))).andThrow(ase);
    EasyMock.expect(mockClient.batchWriteItem(anyObject(BatchWriteItemRequest.class))).andReturn(mockResult);
    EasyMock.expect(mockClient.batchWriteItem(anyObject(BatchWriteItemRequest.class))).andReturn(mockResult);
    EasyMock.replay(mockClient, mockResult);
    List<FailedBatch> result = mapper.writeOneBatch(batchMap);
    assertEquals(result.size(), 0);
    EasyMock.verify(mockClient);
}

Example 3 with FailedBatch

use of com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper.FailedBatch in project aws-sdk-android by aws-amplify.

the class DynamoDBMapper method callUntilCompletion.

/**
 * Continue trying to process the batch until it finishes or an exception
 * occurs.
 */
private FailedBatch callUntilCompletion(Map<String, List<WriteRequest>> batch) {
    BatchWriteItemResult result = null;
    int retries = 0;
    FailedBatch failedBatch = null;
    while (true) {
        try {
            result = db.batchWriteItem(applyBatchOperationUserAgent(new BatchWriteItemRequest().withRequestItems(batch)));
        } catch (final Exception e) {
            failedBatch = new FailedBatch();
            failedBatch.setUnprocessedItems(batch);
            failedBatch.setException(e);
            return failedBatch;
        }
        retries++;
        batch = result.getUnprocessedItems();
        if (batch.size() > 0) {
            pauseExponentially(retries);
        } else {
            break;
        }
    }
    return failedBatch;
}

Example 4 with FailedBatch

use of com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper.FailedBatch in project aws-sdk-android by aws-amplify.

the class DynamoDBMapper method batchWrite.

/**
 * Saves and deletes the objects given using one or more calls to the
 * {@link AmazonDynamoDB#batchWriteItem(BatchWriteItemRequest)} API.
 * <p>
 * This method fails to save the batch if the size of an individual object
 * in the batch exceeds 400 KB. For more information on batch restrictions
 * see, http://docs.aws.amazon
 * .com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html
 * </p>
 *
 * @param objectsToWrite A list of objects to save to DynamoDB. <b>No
 *            version checks are performed</b>, as required by the
 *            {@link AmazonDynamoDB#batchWriteItem(BatchWriteItemRequest)}
 *            API.
 * @param objectsToDelete A list of objects to delete from DynamoDB. <b>No
 *            version checks are performed</b>, as required by the
 *            {@link AmazonDynamoDB#batchWriteItem(BatchWriteItemRequest)}
 *            API.
 * @param config Only {@link DynamoDBMapperConfig#getTableNameOverride()} is
 *            considered; if specified, all objects in the two parameter
 *            lists will be considered to belong to the given table
 *            override. In particular, this method <b>always acts as if
 *            SaveBehavior.CLOBBER was specified</b> regardless of the value
 *            of the config parameter.
 * @return A list of failed batches which includes the unprocessed items and
 *         the exceptions causing the failure.
 */
public List<FailedBatch> batchWrite(List<? extends Object> objectsToWrite, List<? extends Object> objectsToDelete, DynamoDBMapperConfig config) {
    config = mergeConfig(config);
    final List<FailedBatch> totalFailedBatches = new LinkedList<FailedBatch>();
    final HashMap<String, List<WriteRequest>> requestItems = new HashMap<String, List<WriteRequest>>();
    final ItemConverter converter = getConverter(config);
    final List<ValueUpdate> inMemoryUpdates = new LinkedList<ValueUpdate>();
    for (final Object toWrite : objectsToWrite) {
        final Class<?> clazz = toWrite.getClass();
        final String tableName = getTableName(clazz, toWrite, config);
        final Map<String, AttributeValue> attributeValues = new HashMap<String, AttributeValue>();
        // Look at every getter and construct a value object for it
        for (final Method method : reflector.getRelevantGetters(clazz)) {
            final Object getterResult = ReflectionUtils.safeInvoke(method, toWrite);
            final String attributeName = reflector.getAttributeName(method);
            AttributeValue currentValue = null;
            if (getterResult == null && reflector.isAssignableKey(method)) {
                currentValue = getAutoGeneratedKeyAttributeValue(converter, method);
                inMemoryUpdates.add(new ValueUpdate(method, currentValue, toWrite, converter));
            } else {
                currentValue = converter.convert(method, getterResult);
            }
            if (currentValue != null) {
                attributeValues.put(attributeName, currentValue);
            }
        }
        if (!requestItems.containsKey(tableName)) {
            requestItems.put(tableName, new LinkedList<WriteRequest>());
        }
        final AttributeTransformer.Parameters<?> parameters = toParameters(attributeValues, clazz, tableName, config);
        requestItems.get(tableName).add(new WriteRequest().withPutRequest(new PutRequest().withItem(transformAttributes(parameters))));
    }
    for (final Object toDelete : objectsToDelete) {
        final Class<?> clazz = toDelete.getClass();
        final String tableName = getTableName(clazz, toDelete, config);
        final Map<String, AttributeValue> key = getKey(converter, toDelete);
        if (!requestItems.containsKey(tableName)) {
            requestItems.put(tableName, new LinkedList<WriteRequest>());
        }
        requestItems.get(tableName).add(new WriteRequest().withDeleteRequest(new DeleteRequest().withKey(key)));
    }
    // Break into chunks of 25 items and make service requests to DynamoDB
    while (!requestItems.isEmpty()) {
        final HashMap<String, List<WriteRequest>> batch = new HashMap<String, List<WriteRequest>>();
        int i = 0;
        final Iterator<Entry<String, List<WriteRequest>>> tableIter = requestItems.entrySet().iterator();
        while (tableIter.hasNext() && i < MAX_ITEMS_PER_BATCH) {
            final Entry<String, List<WriteRequest>> tableRequest = tableIter.next();
            batch.put(tableRequest.getKey(), new LinkedList<WriteRequest>());
            final Iterator<WriteRequest> writeRequestIter = tableRequest.getValue().iterator();
            while (writeRequestIter.hasNext() && i++ < MAX_ITEMS_PER_BATCH) {
                final WriteRequest writeRequest = writeRequestIter.next();
                batch.get(tableRequest.getKey()).add(writeRequest);
                writeRequestIter.remove();
            }
            // remove it from the parent iterator.
            if (!writeRequestIter.hasNext()) {
                tableIter.remove();
            }
        }
        final List<FailedBatch> failedBatches = writeOneBatch(batch);
        if (failedBatches != null) {
            totalFailedBatches.addAll(failedBatches);
            // If contains throttling exception, we do a backoff
            if (containsThrottlingException(failedBatches)) {
                try {
                    Thread.sleep(THREAD_SLEEP_TWO_SECONDS);
                } catch (final InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new AmazonClientException(e.getMessage(), e);
                }
            }
        }
    }
    // Once the entire batch is processed, update assigned keys in memory
    for (final ValueUpdate update : inMemoryUpdates) {
        update.apply();
    }
    return totalFailedBatches;
}

Example 5 with FailedBatch

use of com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper.FailedBatch in project BridgeServer2 by Sage-Bionetworks.

the class DynamoSurveyDao method saveSurvey.

private Survey saveSurvey(Survey survey) {
    deleteAllElements(survey.getGuid(), survey.getCreatedOn());
    List<DynamoSurveyElement> dynamoElements = Lists.newArrayList();
    for (int i = 0; i < survey.getElements().size(); i++) {
        SurveyElement element = survey.getElements().get(i);
        element.setSurveyKeyComponents(survey.getGuid(), survey.getCreatedOn());
        element.setOrder(i);
        if (element.getGuid() == null) {
            element.setGuid(generateGuid());
        }
        reconcileRules(element);
        dynamoElements.add((DynamoSurveyElement) element);
    }
    List<FailedBatch> failures = surveyElementMapper.batchSave(dynamoElements);
    BridgeUtils.ifFailuresThrowException(failures);
    try {
        surveyMapper.save(survey);
    } catch (ConditionalCheckFailedException throwable) {
        // At this point, delete the elements you just created... compensating transaction
        deleteAllElements(survey.getGuid(), survey.getCreatedOn());
        throw new ConcurrentModificationException(survey);
    } catch (Throwable t) {
        throw new BridgeServiceException(t);
    }
    return survey;
}