Examples with TopicMessageImpl org.apache.pulsar.client.impl.TopicMessageImpl used on opensource projects

Example 1 with TopicMessageImpl

use of org.apache.pulsar.client.impl.TopicMessageImpl in project pulsar by apache.

the class InterceptorsTest method testConsumerInterceptorWithMultiTopicSubscribe.

@Test
public void testConsumerInterceptorWithMultiTopicSubscribe() throws PulsarClientException {
    ConsumerInterceptor<String> interceptor = new ConsumerInterceptor<String>() {

        @Override
        public void close() {
        }

        @Override
        public Message<String> beforeConsume(Consumer<String> consumer, Message<String> message) {
            MessageImpl<String> msg = (MessageImpl<String>) message;
            msg.getMessageBuilder().addProperty().setKey("beforeConsumer").setValue("1");
            return msg;
        }

        @Override
        public void onAcknowledge(Consumer<String> consumer, MessageId messageId, Throwable cause) {
            log.info("onAcknowledge messageId: {}", messageId, cause);
        }

        @Override
        public void onAcknowledgeCumulative(Consumer<String> consumer, MessageId messageId, Throwable cause) {
            log.info("onAcknowledgeCumulative messageIds: {}", messageId, cause);
        }

        @Override
        public void onNegativeAcksSend(Consumer<String> consumer, Set<MessageId> messageIds) {
        }

        @Override
        public void onAckTimeoutSend(Consumer<String> consumer, Set<MessageId> messageIds) {
        }
    };
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic").create();
    Producer<String> producer1 = pulsarClient.newProducer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic1").create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic", "persistent://my-property/my-ns/my-topic1").subscriptionType(SubscriptionType.Shared).intercept(interceptor).subscriptionName("my-subscription").subscribe();
    producer.newMessage().value("Hello Pulsar!").send();
    producer1.newMessage().value("Hello Pulsar!").send();
    int keyCount = 0;
    for (int i = 0; i < 2; i++) {
        Message<String> received = consumer.receive();
        MessageImpl<String> msg = (MessageImpl<String>) ((TopicMessageImpl<String>) received).getMessage();
        for (KeyValue keyValue : msg.getMessageBuilder().getPropertiesList()) {
            if ("beforeConsumer".equals(keyValue.getKey())) {
                keyCount++;
            }
        }
        consumer.acknowledge(received);
    }
    Assert.assertEquals(2, keyCount);
    producer.close();
    producer1.close();
    consumer.close();
}

Example 2 with TopicMessageImpl

use of org.apache.pulsar.client.impl.TopicMessageImpl in project pulsar by apache.

the class InterceptorsTest method testConsumerInterceptorWithPatternTopicSubscribe.

@Test
public void testConsumerInterceptorWithPatternTopicSubscribe() throws PulsarClientException {
    ConsumerInterceptor<String> interceptor = new ConsumerInterceptor<String>() {

        @Override
        public void close() {
        }

        @Override
        public Message<String> beforeConsume(Consumer<String> consumer, Message<String> message) {
            MessageImpl<String> msg = (MessageImpl<String>) message;
            msg.getMessageBuilder().addProperty().setKey("beforeConsumer").setValue("1");
            return msg;
        }

        @Override
        public void onAcknowledge(Consumer<String> consumer, MessageId messageId, Throwable cause) {
            log.info("onAcknowledge messageId: {}", messageId, cause);
        }

        @Override
        public void onAcknowledgeCumulative(Consumer<String> consumer, MessageId messageId, Throwable cause) {
            log.info("onAcknowledgeCumulative messageIds: {}", messageId, cause);
        }

        @Override
        public void onNegativeAcksSend(Consumer<String> consumer, Set<MessageId> messageIds) {
        }

        @Override
        public void onAckTimeoutSend(Consumer<String> consumer, Set<MessageId> messageIds) {
        }
    };
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic").create();
    Producer<String> producer1 = pulsarClient.newProducer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic1").create();
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topicsPattern("persistent://my-property/my-ns/my-.*").subscriptionType(SubscriptionType.Shared).intercept(interceptor).subscriptionName("my-subscription").subscribe();
    producer.newMessage().value("Hello Pulsar!").send();
    producer1.newMessage().value("Hello Pulsar!").send();
    int keyCount = 0;
    for (int i = 0; i < 2; i++) {
        Message<String> received = consumer.receive();
        MessageImpl<String> msg = (MessageImpl<String>) ((TopicMessageImpl<String>) received).getMessage();
        for (KeyValue keyValue : msg.getMessageBuilder().getPropertiesList()) {
            if ("beforeConsumer".equals(keyValue.getKey())) {
                keyCount++;
            }
        }
        consumer.acknowledge(received);
    }
    Assert.assertEquals(2, keyCount);
    producer.close();
    producer1.close();
    consumer.close();
}

Example 3 with TopicMessageImpl

use of org.apache.pulsar.client.impl.TopicMessageImpl in project pulsar by apache.

the class PulsarSource method buildRecord.

protected Record<T> buildRecord(Consumer<T> consumer, Message<T> message) {
    Schema<T> schema = null;
    if (message instanceof MessageImpl) {
        MessageImpl impl = (MessageImpl) message;
        schema = impl.getSchemaInternal();
    } else if (message instanceof TopicMessageImpl) {
        TopicMessageImpl impl = (TopicMessageImpl) message;
        schema = impl.getSchemaInternal();
    }
    return PulsarRecord.<T>builder().message(message).schema(schema).topicName(message.getTopicName()).ackFunction(() -> {
        try {
            if (pulsarSourceConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
                consumer.acknowledgeCumulativeAsync(message);
            } else {
                consumer.acknowledgeAsync(message);
            }
        } finally {
            // don't need to check if message pooling is set
            // client will automatically check
            message.release();
        }
    }).failFunction(() -> {
        try {
            if (pulsarSourceConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) {
                throw new RuntimeException("Failed to process message: " + message.getMessageId());
            }
            consumer.negativeAcknowledge(message);
        } finally {
            // don't need to check if message pooling is set
            // client will automatically check
            message.release();
        }
    }).build();
}

Example 4 with TopicMessageImpl

use of org.apache.pulsar.client.impl.TopicMessageImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testEncryptionConsumerWithoutCryptoReader.

@Test
public void testEncryptionConsumerWithoutCryptoReader() throws Exception {
    final String encryptionKeyName = "client-rsa.pem";
    final String encryptionKeyVersion = "1.0";
    Map<String, String> metadata = new HashMap<>();
    metadata.put("version", encryptionKeyVersion);
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
            if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
                try {
                    keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
                    keyInfo.setMetadata(metadata);
                    return keyInfo;
                } catch (IOException e) {
                    Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
                }
            } else {
                Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
            }
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName;
            if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
                try {
                    keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
                    keyInfo.setMetadata(metadata);
                    return keyInfo;
                } catch (IOException e) {
                    Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
                }
            } else {
                Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
            }
            return null;
        }
    }
    Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/my-ns/myrsa-topic3").addEncryptionKey(encryptionKeyName).compressionType(CompressionType.LZ4).cryptoKeyReader(new EncKeyReader()).create();
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topicsPattern("persistent://my-property/my-ns/myrsa-topic3").subscriptionName("my-subscriber-name").cryptoFailureAction(ConsumerCryptoFailureAction.CONSUME).subscribe();
    String message = "my-message";
    producer.send(message.getBytes());
    TopicMessageImpl<byte[]> msg = (TopicMessageImpl<byte[]>) consumer.receive(5, TimeUnit.SECONDS);
    String receivedMessage = decryptMessage(msg, encryptionKeyName, new EncKeyReader());
    assertEquals(message, receivedMessage);
    consumer.close();
}

Example 5 with TopicMessageImpl

use of org.apache.pulsar.client.impl.TopicMessageImpl in project pulsar by yahoo.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithPartition.

@Test
public void testSimpleConsumerEventsWithPartition() throws Exception {
    // Resetting ActiveConsumerFailoverDelayTimeMillis else if testActiveConsumerFailoverWithDelay get executed
    // first could cause this test to fail.
    conf.setActiveConsumerFailoverDelayTimeMillis(0);
    restartBroker();
    int numPartitions = 4;
    final String topicName = BrokerTestUtil.newUniqueName("persistent://prop/use/ns-abc/testSimpleConsumerEventsWithPartition");
    final TopicName destName = TopicName.get(topicName);
    final String subName = "sub1";
    final int numMsgs = 100;
    Set<String> uniqueMessages = new HashSet<>();
    admin.topics().createPartitionedTopic(topicName, numPartitions);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover);
    // 1. two consumers on the same subscription
    ActiveInactiveListenerEvent listener1 = new ActiveInactiveListenerEvent();
    ActiveInactiveListenerEvent listener2 = new ActiveInactiveListenerEvent();
    Consumer<byte[]> consumer1 = consumerBuilder.clone().consumerName("1").consumerEventListener(listener1).receiverQueueSize(1).subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2).receiverQueueSize(1).subscribe();
    PersistentTopic topicRef;
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(0).toString()).get();
    PersistentDispatcherSingleActiveConsumer disp0 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(1).toString()).get();
    PersistentDispatcherSingleActiveConsumer disp1 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(2).toString()).get();
    PersistentDispatcherSingleActiveConsumer disp2 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(destName.getPartition(3).toString()).get();
    PersistentDispatcherSingleActiveConsumer disp3 = (PersistentDispatcherSingleActiveConsumer) topicRef.getSubscription(subName).getDispatcher();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        producer.sendAsync(message.getBytes());
    }
    producer.flush();
    // equal distribution between both consumers
    int totalMessages = 0;
    Message<byte[]> msg = null;
    Set<Integer> receivedPtns = Sets.newHashSet();
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer1.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) (((TopicMessageImpl) msg).getInnerMessageId());
        receivedPtns.add(msgId.getPartitionIndex());
    }
    assertTrue(Sets.difference(listener1.activePtns, receivedPtns).isEmpty());
    assertTrue(Sets.difference(listener2.inactivePtns, receivedPtns).isEmpty());
    Assert.assertEquals(totalMessages, numMsgs / 2);
    receivedPtns = Sets.newHashSet();
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        totalMessages++;
        consumer2.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) (((TopicMessageImpl) msg).getInnerMessageId());
        receivedPtns.add(msgId.getPartitionIndex());
    }
    assertTrue(Sets.difference(listener1.inactivePtns, receivedPtns).isEmpty());
    assertTrue(Sets.difference(listener2.activePtns, receivedPtns).isEmpty());
    Assert.assertEquals(totalMessages, numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "1");
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "1");
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "2");
    totalMessages = 0;
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        producer.sendAsync(message.getBytes());
    }
    producer.flush();
    // add a consumer
    for (int i = 0; i < 20; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    Consumer<byte[]> consumer3 = consumerBuilder.clone().consumerName("3").receiverQueueSize(1).subscribe();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    while (true) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "1");
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "3");
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "1");
    uniqueMessages.clear();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        producer.sendAsync(message.getBytes());
    }
    producer.flush();
    // remove a consumer
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        uniqueMessages.add(new String(msg.getData()));
        consumer1.acknowledge(msg);
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    while (true) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        uniqueMessages.add(new String(msg.getData()));
        consumer2.acknowledge(msg);
    }
    while (true) {
        msg = consumer3.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
        uniqueMessages.add(new String(msg.getData()));
        consumer3.acknowledge(msg);
    }
    Assert.assertEquals(uniqueMessages.size(), numMsgs);
    Assert.assertEquals(disp0.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp1.getActiveConsumer().consumerName(), "3");
    Assert.assertEquals(disp2.getActiveConsumer().consumerName(), "2");
    Assert.assertEquals(disp3.getActiveConsumer().consumerName(), "3");
    producer.close();
    consumer2.close();
    consumer3.unsubscribe();
    admin.topics().deletePartitionedTopic(topicName);
}