Examples with MessageId org.apache.pulsar.client.api.MessageId used on opensource projects

Example 31 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class RawBatchConverter method rebatchMessage.

/**
 * Take a batched message and a filter, and returns a message with the only the submessages
 * which match the filter. Returns an empty optional if no messages match.
 *
 * This takes ownership of the passes in message, and if the returned optional is not empty,
 * the ownership of that message is returned also.
 */
public static Optional<RawMessage> rebatchMessage(RawMessage msg, BiPredicate<String, MessageId> filter) throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    ByteBuf batchBuffer = PooledByteBufAllocator.DEFAULT.buffer(payload.capacity());
    try {
        int batchSize = metadata.getNumMessagesInBatch();
        int messagesRetained = 0;
        SingleMessageMetadata.Builder emptyMetadataBuilder = SingleMessageMetadata.newBuilder().setCompactedOut(true);
        for (int i = 0; i < batchSize; i++) {
            SingleMessageMetadata.Builder singleMessageMetadataBuilder = SingleMessageMetadata.newBuilder();
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(payload, singleMessageMetadataBuilder, 0, batchSize);
            String key = singleMessageMetadataBuilder.getPartitionKey();
            MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
            if (filter.test(key, id)) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, singleMessagePayload, batchBuffer);
            } else {
                Commands.serializeSingleMessageInBatchWithPayload(emptyMetadataBuilder, Unpooled.EMPTY_BUFFER, batchBuffer);
            }
            singleMessageMetadataBuilder.recycle();
            singleMessagePayload.release();
        }
        emptyMetadataBuilder.recycle();
        if (messagesRetained > 0) {
            ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, batchBuffer);
            return Optional.of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload));
        } else {
            return Optional.empty();
        }
    } finally {
        batchBuffer.release();
        metadata.recycle();
        msg.close();
    }
}

Example 32 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class TwoPhaseCompactor method phaseTwoLoop.

private void phaseTwoLoop(RawReader reader, MessageId to, Map<String, MessageId> latestForKey, LedgerHandle lh, Semaphore outstanding, CompletableFuture<Void> promise) {
    reader.readNextAsync().whenCompleteAsync((m, exception) -> {
        if (exception != null) {
            promise.completeExceptionally(exception);
            return;
        } else if (promise.isDone()) {
            return;
        }
        MessageId id = m.getMessageId();
        Optional<RawMessage> messageToAdd = Optional.empty();
        if (RawBatchConverter.isBatch(m)) {
            try {
                messageToAdd = RawBatchConverter.rebatchMessage(m, (key, subid) -> latestForKey.get(key).equals(subid));
            } catch (IOException ioe) {
                log.info("Error decoding batch for message {}. Whole batch will be included in output", id, ioe);
                messageToAdd = Optional.of(m);
            }
        } else {
            String key = extractKey(m);
            if (latestForKey.get(key).equals(id)) {
                messageToAdd = Optional.of(m);
            } else {
                m.close();
            }
        }
        messageToAdd.ifPresent((toAdd) -> {
            try {
                outstanding.acquire();
                CompletableFuture<Void> addFuture = addToCompactedLedger(lh, toAdd).whenComplete((res, exception2) -> {
                    outstanding.release();
                    if (exception2 != null) {
                        promise.completeExceptionally(exception2);
                    }
                });
                if (to.equals(id)) {
                    addFuture.whenComplete((res, exception2) -> {
                        if (exception2 == null) {
                            promise.complete(null);
                        }
                    });
                }
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
                promise.completeExceptionally(ie);
            }
        });
        phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise);
    }, scheduler);
}

Example 33 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtLeastOnceProcessingFailures.

@Test
public void testAtLeastOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            if (i % 2 == 0) {
                // all messages (i % 2 == 0) will fail to process.
                continue;
            }
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // only 5 succeed messages are acknowledged
        verify(consumerInstance.getConsumer(), times(5)).acknowledgeAsync(any(Message.class));
        assertEquals(5, consumerInstance.getNumMessages());
        for (int i = 0; i < 10; i++) {
            assertEquals(i % 2 == 0, consumerInstance.containMessage(new MessageIdImpl(1L, i, 0)));
        }
    }
}

Example 34 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtLeastOnceProcessing.

@Test
public void testAtLeastOnceProcessing() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(0)).acknowledgeAsync(any(Message.class));
        assertEquals(10, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
    }
}

Example 35 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testAtMostOnceProcessingFailures.

@Test
public void testAtMostOnceProcessingFailures() throws Exception {
    FunctionConfig newFnConfig = FunctionConfig.newBuilder(fnConfig).setProcessingGuarantees(ProcessingGuarantees.ATMOST_ONCE).setClassName(TestFailureFunction.class.getName()).build();
    config.setFunctionConfig(newFnConfig);
    @Cleanup("shutdown") ExecutorService executorService = Executors.newSingleThreadExecutor();
    try (JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, null)) {
        executorService.submit(runnable);
        Pair<String, String> consumerId = Pair.of(newFnConfig.getInputs(0), FunctionConfigUtils.getFullyQualifiedName(newFnConfig));
        ConsumerInstance consumerInstance = mockConsumers.get(consumerId);
        while (null == consumerInstance) {
            TimeUnit.MILLISECONDS.sleep(20);
            consumerInstance = mockConsumers.get(consumerId);
        }
        ProducerInstance producerInstance = mockProducers.values().iterator().next();
        // once we get consumer id, simulate receiving 10 messages from consumer
        for (int i = 0; i < 10; i++) {
            Message msg = mock(Message.class);
            when(msg.getData()).thenReturn(("message-" + i).getBytes(UTF_8));
            when(msg.getMessageId()).thenReturn(new MessageIdImpl(1L, i, 0));
            consumerInstance.addMessage(msg);
            consumerInstance.getConf().getMessageListener().received(consumerInstance.getConsumer(), msg);
        }
        // wait until all the messages are published
        for (int i = 0; i < 10; i++) {
            if (i % 2 == 0) {
                // all messages (i % 2 == 0) will fail to process.
                continue;
            }
            Message msg = producerInstance.msgQueue.take();
            assertEquals("message-" + i + "!", new String(msg.getData(), UTF_8));
            // sequence id is not set for AT_MOST_ONCE processing
            assertEquals(0L, msg.getSequenceId());
        }
        // verify acknowledge before send completes
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
        // complete all the publishes
        synchronized (producerInstance) {
            for (CompletableFuture<MessageId> future : producerInstance.sendFutures) {
                future.complete(mock(MessageId.class));
            }
        }
        // acknowledges count should remain same
        verify(consumerInstance.getConsumer(), times(10)).acknowledgeAsync(any(Message.class));
        assertEquals(0, consumerInstance.getNumMessages());
    }
}