Search in sources :

Example 1 with ChangeMessageVisibilityBatchResult

use of com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchResult in project beam by apache.

the class SqsUnboundedReader method extendBatch.

/**
 * BLOCKING Extend the visibility timeout for messages from SQS with the given {@code
 * receiptHandles}.
 */
void extendBatch(long nowMsSinceEpoch, List<String> receiptHandles, int extensionSec) throws IOException {
    int retries = 0;
    int numMessages = receiptHandles.size();
    Map<String, String> pendingReceipts = IntStream.range(0, receiptHandles.size()).boxed().collect(toMap(Object::toString, receiptHandles::get));
    List<String> errorMessages = new ArrayList<>();
    while (!pendingReceipts.isEmpty()) {
        if (retries >= BATCH_OPERATION_MAX_RETIRES) {
            throw new IOException("Failed to extend visibility timeout for " + pendingReceipts.size() + " messages after " + retries + " retries: " + String.join(", ", errorMessages));
        }
        List<ChangeMessageVisibilityBatchRequestEntry> entries = pendingReceipts.entrySet().stream().map(r -> new ChangeMessageVisibilityBatchRequestEntry(r.getKey(), r.getValue()).withVisibilityTimeout(extensionSec)).collect(Collectors.toList());
        ChangeMessageVisibilityBatchResult result = sqsClient.changeMessageVisibilityBatch(source.getRead().queueUrl(), entries);
        // Retry errors except invalid handles
        Set<BatchResultErrorEntry> retryErrors = result.getFailed().stream().filter(e -> !e.getCode().equals("ReceiptHandleIsInvalid")).collect(Collectors.toSet());
        pendingReceipts.keySet().retainAll(retryErrors.stream().map(BatchResultErrorEntry::getId).collect(Collectors.toSet()));
        errorMessages = retryErrors.stream().map(BatchResultErrorEntry::getMessage).collect(Collectors.toList());
        retries += 1;
    }
    numExtendedDeadlines.add(nowMsSinceEpoch, numMessages);
}
Also used : AmazonSQSClientBuilder(com.amazonaws.services.sqs.AmazonSQSClientBuilder) Arrays(java.util.Arrays) MessageSystemAttributeName(com.amazonaws.services.sqs.model.MessageSystemAttributeName) UnboundedSource(org.apache.beam.sdk.io.UnboundedSource) LoggerFactory(org.slf4j.LoggerFactory) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) BatchResultErrorEntry(com.amazonaws.services.sqs.model.BatchResultErrorEntry) ReceiveMessageResult(com.amazonaws.services.sqs.model.ReceiveMessageResult) ChangeMessageVisibilityBatchResult(com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchResult) Collectors.toMap(java.util.stream.Collectors.toMap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) GetQueueAttributesRequest(com.amazonaws.services.sqs.model.GetQueueAttributesRequest) Min(org.apache.beam.sdk.transforms.Min) Set(java.util.Set) Sum(org.apache.beam.sdk.transforms.Sum) Collectors(java.util.stream.Collectors) MovingFunction(org.apache.beam.sdk.util.MovingFunction) ChangeMessageVisibilityBatchRequestEntry(com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry) Objects(java.util.Objects) List(java.util.List) Max(org.apache.beam.sdk.transforms.Max) AmazonSQS(com.amazonaws.services.sqs.AmazonSQS) Queue(java.util.Queue) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) IntStream(java.util.stream.IntStream) MessageAttributeValue(com.amazonaws.services.sqs.model.MessageAttributeValue) Combine(org.apache.beam.sdk.transforms.Combine) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) Duration(org.joda.time.Duration) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BucketingFunction(org.apache.beam.sdk.util.BucketingFunction) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) DeleteMessageBatchResult(com.amazonaws.services.sqs.model.DeleteMessageBatchResult) EvictingQueue(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.EvictingQueue) DeleteMessageBatchRequestEntry(com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry) ReceiveMessageRequest(com.amazonaws.services.sqs.model.ReceiveMessageRequest) NoSuchElementException(java.util.NoSuchElementException) QueueAttributeName(com.amazonaws.services.sqs.model.QueueAttributeName) Logger(org.slf4j.Logger) UTF_8(java.nio.charset.StandardCharsets.UTF_8) Lists(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Lists) IOException(java.io.IOException) Message(com.amazonaws.services.sqs.model.Message) CheckpointMark(org.apache.beam.sdk.io.UnboundedSource.CheckpointMark) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) Instant(org.joda.time.Instant) ArrayDeque(java.util.ArrayDeque) Collections(java.util.Collections) ArrayList(java.util.ArrayList) ChangeMessageVisibilityBatchRequestEntry(com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry) BatchResultErrorEntry(com.amazonaws.services.sqs.model.BatchResultErrorEntry) IOException(java.io.IOException) ChangeMessageVisibilityBatchResult(com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchResult)

Aggregations

AmazonSQS (com.amazonaws.services.sqs.AmazonSQS)1 AmazonSQSClientBuilder (com.amazonaws.services.sqs.AmazonSQSClientBuilder)1 BatchResultErrorEntry (com.amazonaws.services.sqs.model.BatchResultErrorEntry)1 ChangeMessageVisibilityBatchRequestEntry (com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry)1 ChangeMessageVisibilityBatchResult (com.amazonaws.services.sqs.model.ChangeMessageVisibilityBatchResult)1 DeleteMessageBatchRequestEntry (com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry)1 DeleteMessageBatchResult (com.amazonaws.services.sqs.model.DeleteMessageBatchResult)1 GetQueueAttributesRequest (com.amazonaws.services.sqs.model.GetQueueAttributesRequest)1 Message (com.amazonaws.services.sqs.model.Message)1 MessageAttributeValue (com.amazonaws.services.sqs.model.MessageAttributeValue)1 MessageSystemAttributeName (com.amazonaws.services.sqs.model.MessageSystemAttributeName)1 QueueAttributeName (com.amazonaws.services.sqs.model.QueueAttributeName)1 ReceiveMessageRequest (com.amazonaws.services.sqs.model.ReceiveMessageRequest)1 ReceiveMessageResult (com.amazonaws.services.sqs.model.ReceiveMessageResult)1 IOException (java.io.IOException)1 UTF_8 (java.nio.charset.StandardCharsets.UTF_8)1 ArrayDeque (java.util.ArrayDeque)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1