Search in sources :

Example 1 with Message

use of software.amazon.awssdk.services.sqs.model.Message in project beam by apache.

the class SqsUnboundedReader method extendBatch.

/**
 * BLOCKING. Extend the SQS visibility timeout for messages in {@code messages} as {@link KV} of
 * message id, receipt handle.
 */
void extendBatch(long nowMsSinceEpoch, List<KV<String, String>> messages, int extensionSec) throws IOException {
    int retries = 0;
    Function<KV<String, String>, ChangeMessageVisibilityBatchRequestEntry> buildEntry = kv -> ChangeMessageVisibilityBatchRequestEntry.builder().visibilityTimeout(extensionSec).id(kv.getKey()).receiptHandle(kv.getValue()).build();
    Map<String, ChangeMessageVisibilityBatchRequestEntry> pendingExtends = messages.stream().collect(toMap(KV::getKey, buildEntry));
    while (!pendingExtends.isEmpty()) {
        if (retries >= BATCH_OPERATION_MAX_RETIRES) {
            throw new IOException("Failed to extend visibility timeout for " + messages.size() + " messages after " + retries + " retries");
        }
        ChangeMessageVisibilityBatchResponse response = sqsClient.changeMessageVisibilityBatch(ChangeMessageVisibilityBatchRequest.builder().queueUrl(queueUrl()).entries(pendingExtends.values()).build());
        Map<Boolean, Set<String>> failures = response.failed().stream().collect(partitioningBy(this::isHandleInvalid, mapping(e -> e.id(), toSet())));
        // Keep failed IDs only, but discard invalid (expired) receipt handles
        pendingExtends.keySet().retainAll(failures.getOrDefault(FALSE, ImmutableSet.of()));
        // redelivery
        if (extensionSec > 0) {
            numExtendedDeadlines.add(nowMsSinceEpoch, response.successful().size());
            Set<String> invalidMsgIds = failures.getOrDefault(TRUE, ImmutableSet.of());
            if (invalidMsgIds.size() > 0) {
                // consider invalid (expired) messages no longer in flight
                numLateDeadlines.add(nowMsSinceEpoch, invalidMsgIds.size());
                for (String msgId : invalidMsgIds) {
                    inFlight.remove(msgId);
                }
                LOG.warn("Failed to extend visibility timeout for {} messages with expired receipt handles.", invalidMsgIds.size());
            }
        }
        retries += 1;
    }
}
Also used : ChangeMessageVisibilityBatchResponse(software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchResponse) ChangeMessageVisibilityBatchRequestEntry(software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry) SENT_TIMESTAMP(software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName.SENT_TIMESTAMP) UnboundedSource(org.apache.beam.sdk.io.UnboundedSource) LoggerFactory(org.slf4j.LoggerFactory) Streams.mapWithIndex(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams.mapWithIndex) ChangeMessageVisibilityBatchRequest(software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchRequest) ImmutableSet(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet) Collectors.toMap(java.util.stream.Collectors.toMap) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) ReceiveMessageRequest(software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest) Collectors.toSet(java.util.stream.Collectors.toSet) Min(org.apache.beam.sdk.transforms.Min) Set(java.util.Set) ClientBuilderFactory(org.apache.beam.sdk.io.aws2.common.ClientBuilderFactory) Sum(org.apache.beam.sdk.transforms.Sum) SqsClient(software.amazon.awssdk.services.sqs.SqsClient) MovingFunction(org.apache.beam.sdk.util.MovingFunction) List(java.util.List) Max(org.apache.beam.sdk.transforms.Max) Collections2.transform(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Collections2.transform) DeleteMessageBatchResponse(software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResponse) AwsOptions(org.apache.beam.sdk.io.aws2.options.AwsOptions) VisibleForTesting(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting) Function.identity(java.util.function.Function.identity) Queue(java.util.Queue) TRUE(java.lang.Boolean.TRUE) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) Collectors.partitioningBy(java.util.stream.Collectors.partitioningBy) KV(org.apache.beam.sdk.values.KV) Combine(org.apache.beam.sdk.transforms.Combine) Duration(org.joda.time.Duration) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BucketingFunction(org.apache.beam.sdk.util.BucketingFunction) Function(java.util.function.Function) DeleteMessageBatchRequest(software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequest) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) Message(software.amazon.awssdk.services.sqs.model.Message) Collectors.mapping(java.util.stream.Collectors.mapping) NoSuchElementException(java.util.NoSuchElementException) FALSE(java.lang.Boolean.FALSE) Logger(org.slf4j.Logger) UTF_8(java.nio.charset.StandardCharsets.UTF_8) FunctionWithIndex(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Streams.FunctionWithIndex) IOException(java.io.IOException) ReceiveMessageResponse(software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse) BatchResultErrorEntry(software.amazon.awssdk.services.sqs.model.BatchResultErrorEntry) Collectors.toList(java.util.stream.Collectors.toList) CheckpointMark(org.apache.beam.sdk.io.UnboundedSource.CheckpointMark) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) Instant(org.joda.time.Instant) VISIBILITY_TIMEOUT(software.amazon.awssdk.services.sqs.model.QueueAttributeName.VISIBILITY_TIMEOUT) ClientConfiguration(org.apache.beam.sdk.io.aws2.common.ClientConfiguration) Clock(java.time.Clock) ArrayDeque(java.util.ArrayDeque) DeleteMessageBatchRequestEntry(software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequestEntry) ImmutableSet(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableSet) Collectors.toSet(java.util.stream.Collectors.toSet) Set(java.util.Set) HashSet(java.util.HashSet) KV(org.apache.beam.sdk.values.KV) IOException(java.io.IOException) ChangeMessageVisibilityBatchResponse(software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchResponse) ChangeMessageVisibilityBatchRequestEntry(software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityBatchRequestEntry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean)

Example 2 with Message

use of software.amazon.awssdk.services.sqs.model.Message in project aws-doc-sdk-examples by awsdocs.

the class SendReceiveMessages method getMessages.

public String getMessages() {
    List attr = new ArrayList<String>();
    attr.add("Name");
    SqsClient sqsClient = getClient();
    try {
        GetQueueUrlRequest getQueueRequest = GetQueueUrlRequest.builder().queueName(QUEUE_NAME).build();
        String queueUrl = sqsClient.getQueueUrl(getQueueRequest).queueUrl();
        // Receive messages from the queue
        ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder().queueUrl(queueUrl).maxNumberOfMessages(10).messageAttributeNames(attr).build();
        List<Message> messages = sqsClient.receiveMessage(receiveRequest).messages();
        com.example.Message myMessage;
        List allMessages = new ArrayList<com.example.Message>();
        // Push the messages to a list
        for (Message m : messages) {
            myMessage = new com.example.Message();
            myMessage.setBody(m.body());
            Map map = m.messageAttributes();
            MessageAttributeValue val = (MessageAttributeValue) map.get("Name");
            myMessage.setName(val.stringValue());
            allMessages.add(myMessage);
        }
        return convertToString(toXml(allMessages));
    } catch (SqsException e) {
        e.getStackTrace();
    }
    return "";
}
Also used : ReceiveMessageRequest(software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest) Message(software.amazon.awssdk.services.sqs.model.Message) SqsClient(software.amazon.awssdk.services.sqs.SqsClient)

Example 3 with Message

use of software.amazon.awssdk.services.sqs.model.Message in project aws-doc-sdk-examples by awsdocs.

the class VideoDetect method getLabelJob.

public static void getLabelJob(RekognitionClient rekClient, SqsClient sqs, String queueUrl) {
    List<Message> messages = null;
    ReceiveMessageRequest messageRequest = ReceiveMessageRequest.builder().queueUrl(queueUrl).build();
    try {
        messages = sqs.receiveMessage(messageRequest).messages();
        if (!messages.isEmpty()) {
            for (Message message : messages) {
                String notification = message.body();
                // Get the status and job id from the notification
                ObjectMapper mapper = new ObjectMapper();
                JsonNode jsonMessageTree = mapper.readTree(notification);
                JsonNode messageBodyText = jsonMessageTree.get("Message");
                ObjectMapper operationResultMapper = new ObjectMapper();
                JsonNode jsonResultTree = operationResultMapper.readTree(messageBodyText.textValue());
                JsonNode operationJobId = jsonResultTree.get("JobId");
                JsonNode operationStatus = jsonResultTree.get("Status");
                System.out.println("Job found in JSON is " + operationJobId);
                DeleteMessageRequest deleteMessageRequest = DeleteMessageRequest.builder().queueUrl(queueUrl).build();
                String jobId = operationJobId.textValue();
                if (startJobId.compareTo(jobId) == 0) {
                    System.out.println("Job id: " + operationJobId);
                    System.out.println("Status : " + operationStatus.toString());
                    if (operationStatus.asText().equals("SUCCEEDED"))
                        GetResultsLabels(rekClient);
                    else
                        System.out.println("Video analysis failed");
                    sqs.deleteMessage(deleteMessageRequest);
                } else {
                    System.out.println("Job received was not job " + startJobId);
                    sqs.deleteMessage(deleteMessageRequest);
                }
            }
        }
    } catch (RekognitionException e) {
        e.getMessage();
        System.exit(1);
    } catch (JsonMappingException e) {
        e.printStackTrace();
    } catch (JsonProcessingException e) {
        e.printStackTrace();
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : Message(software.amazon.awssdk.services.sqs.model.Message) ReceiveMessageRequest(software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest) RekognitionException(software.amazon.awssdk.services.rekognition.model.RekognitionException) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) DeleteMessageRequest(software.amazon.awssdk.services.sqs.model.DeleteMessageRequest) JsonNode(com.fasterxml.jackson.databind.JsonNode) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) RekognitionException(software.amazon.awssdk.services.rekognition.model.RekognitionException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException)

Example 4 with Message

use of software.amazon.awssdk.services.sqs.model.Message in project aws-doc-sdk-examples by awsdocs.

the class SendReceiveMessages method main.

public static void main(String[] args) {
    SqsClient sqsClient = SqsClient.builder().region(Region.US_WEST_2).build();
    try {
        CreateQueueRequest request = CreateQueueRequest.builder().queueName(QUEUE_NAME).build();
        CreateQueueResponse createResult = sqsClient.createQueue(request);
        GetQueueUrlRequest getQueueRequest = GetQueueUrlRequest.builder().queueName(QUEUE_NAME).build();
        String queueUrl = sqsClient.getQueueUrl(getQueueRequest).queueUrl();
        SendMessageRequest sendMsgRequest = SendMessageRequest.builder().queueUrl(queueUrl).messageBody("hello world").delaySeconds(5).build();
        sqsClient.sendMessage(sendMsgRequest);
        // Send multiple messages to the queue
        SendMessageBatchRequest sendBatchRequest = SendMessageBatchRequest.builder().queueUrl(queueUrl).entries(SendMessageBatchRequestEntry.builder().messageBody("Hello from message 1").id("msg_1").build(), SendMessageBatchRequestEntry.builder().messageBody("Hello from message 2").delaySeconds(10).id("msg_2").build()).build();
        sqsClient.sendMessageBatch(sendBatchRequest);
        // Receive messages from the queue
        ReceiveMessageRequest receiveRequest = ReceiveMessageRequest.builder().queueUrl(queueUrl).build();
        List<Message> messages = sqsClient.receiveMessage(receiveRequest).messages();
        // Print out the messages
        for (Message m : messages) {
            System.out.println("\n" + m.body());
        }
    } catch (QueueNameExistsException e) {
        throw e;
    }
}
Also used : CreateQueueResponse(software.amazon.awssdk.services.sqs.model.CreateQueueResponse) GetQueueUrlRequest(software.amazon.awssdk.services.sqs.model.GetQueueUrlRequest) QueueNameExistsException(software.amazon.awssdk.services.sqs.model.QueueNameExistsException) SendMessageBatchRequest(software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest) ReceiveMessageRequest(software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest) Message(software.amazon.awssdk.services.sqs.model.Message) CreateQueueRequest(software.amazon.awssdk.services.sqs.model.CreateQueueRequest) SqsClient(software.amazon.awssdk.services.sqs.SqsClient) SendMessageRequest(software.amazon.awssdk.services.sqs.model.SendMessageRequest)

Example 5 with Message

use of software.amazon.awssdk.services.sqs.model.Message in project beam by apache.

the class SqsUnboundedReader method pull.

/**
 * BLOCKING. Fetch another batch of messages from SQS.
 */
private void pull() {
    if (inFlight.size() >= MAX_IN_FLIGHT) {
        // messages faster than we can checkpoint them.
        return;
    }
    long requestTimeMsSinceEpoch = now();
    long deadlineMsSinceEpoch = requestTimeMsSinceEpoch + visibilityTimeoutMs;
    final ReceiveMessageRequest receiveMessageRequest = ReceiveMessageRequest.builder().maxNumberOfMessages(MAX_NUMBER_OF_MESSAGES).attributeNamesWithStrings(SENT_TIMESTAMP.toString()).queueUrl(queueUrl()).build();
    final ReceiveMessageResponse receiveMessageResponse = sqsClient.receiveMessage(receiveMessageRequest);
    final List<Message> messages = receiveMessageResponse.messages();
    if (messages == null || messages.isEmpty()) {
        return;
    }
    lastReceivedMsSinceEpoch = requestTimeMsSinceEpoch;
    // Capture the received messages.
    for (Message orgMsg : messages) {
        long msgTimeStamp = Long.parseLong(orgMsg.attributes().get(SENT_TIMESTAMP));
        SqsMessage message = SqsMessage.create(orgMsg.body(), orgMsg.messageId(), orgMsg.receiptHandle(), msgTimeStamp, requestTimeMsSinceEpoch);
        messagesNotYetRead.add(message);
        notYetReadBytes += message.getBody().getBytes(UTF_8).length;
        inFlight.put(message.getMessageId(), new InFlightState(message.getReceiptHandle(), requestTimeMsSinceEpoch, deadlineMsSinceEpoch));
        numReceived++;
        numReceivedRecently.add(requestTimeMsSinceEpoch, 1L);
        minReceivedTimestampMsSinceEpoch.add(requestTimeMsSinceEpoch, msgTimeStamp);
        maxReceivedTimestampMsSinceEpoch.add(requestTimeMsSinceEpoch, msgTimeStamp);
        minUnreadTimestampMsSinceEpoch.add(requestTimeMsSinceEpoch, msgTimeStamp);
    }
}
Also used : ReceiveMessageRequest(software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest) Message(software.amazon.awssdk.services.sqs.model.Message) ReceiveMessageResponse(software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse)

Aggregations

Message (software.amazon.awssdk.services.sqs.model.Message)7 ReceiveMessageRequest (software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest)5 SqsClient (software.amazon.awssdk.services.sqs.SqsClient)3 Then (io.cucumber.java.en.Then)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 JsonMappingException (com.fasterxml.jackson.databind.JsonMappingException)1 JsonNode (com.fasterxml.jackson.databind.JsonNode)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 IOException (java.io.IOException)1 FALSE (java.lang.Boolean.FALSE)1 TRUE (java.lang.Boolean.TRUE)1 UTF_8 (java.nio.charset.StandardCharsets.UTF_8)1 Clock (java.time.Clock)1 ArrayDeque (java.util.ArrayDeque)1 ArrayList (java.util.ArrayList)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 NoSuchElementException (java.util.NoSuchElementException)1