Examples with MessageRoutingMode org.apache.pulsar.client.api.MessageRoutingMode used on opensource projects

Example 21 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class AdminApi2Test method testPartitionedStatsAggregationByProducerName.

@Test(dataProvider = "topicType")
public void testPartitionedStatsAggregationByProducerName(String topicType) throws Exception {
    conf.setAggregatePublisherStatsByProducerName(true);
    final String topic = topicType + "://prop-xyz/ns1/test-partitioned-stats-aggregation-by-producer-name";
    admin.topics().createPartitionedTopic(topic, 10);
    @Cleanup Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topic).enableLazyStartPartitionedProducers(true).enableBatching(false).messageRoutingMode(MessageRoutingMode.CustomPartition).messageRouter(new MessageRouter() {

        @Override
        public int choosePartition(Message<?> msg, TopicMetadata metadata) {
            return msg.hasKey() ? Integer.parseInt(msg.getKey()) : 0;
        }
    }).accessMode(ProducerAccessMode.Shared).create();
    @Cleanup Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topic).enableLazyStartPartitionedProducers(true).enableBatching(false).messageRoutingMode(MessageRoutingMode.CustomPartition).messageRouter(new MessageRouter() {

        @Override
        public int choosePartition(Message<?> msg, TopicMetadata metadata) {
            return msg.hasKey() ? Integer.parseInt(msg.getKey()) : 5;
        }
    }).accessMode(ProducerAccessMode.Shared).create();
    for (int i = 0; i < 10; i++) {
        producer1.newMessage().key(String.valueOf(i % 5)).value(("message".getBytes(StandardCharsets.UTF_8))).send();
        producer2.newMessage().key(String.valueOf(i % 5 + 5)).value(("message".getBytes(StandardCharsets.UTF_8))).send();
    }
    PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(topic, true);
    assertEquals(topicStats.getPartitions().size(), 10);
    assertEquals(topicStats.getPartitions().values().stream().mapToInt(e -> e.getPublishers().size()).sum(), 10);
    assertEquals(topicStats.getPartitions().values().stream().map(e -> e.getPublishers().get(0).getProducerName()).distinct().count(), 2);
    assertEquals(topicStats.getPublishers().size(), 2);
    topicStats.getPublishers().forEach(p -> assertTrue(p.isSupportsPartialProducer()));
}

Example 22 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class RawReaderTest method testBatchingRebatch.

@Test
public void testBatchingRebatch() throws Exception {
    String topic = "persistent://my-property/my-ns/my-raw-topic";
    try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).messageRoutingMode(MessageRoutingMode.SinglePartition).create()) {
        producer.newMessage().key("key1").value("my-content-1".getBytes()).sendAsync();
        producer.newMessage().key("key2").value("my-content-2".getBytes()).sendAsync();
        producer.newMessage().key("key3").value("my-content-3".getBytes()).send();
    }
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m1 = reader.readNextAsync().get()) {
        RawMessage m2 = RawBatchConverter.rebatchMessage(m1, (key, id) -> key.equals("key2")).get();
        List<ImmutableTriple<MessageId, String, Integer>> idsAndKeys = RawBatchConverter.extractIdsAndKeysAndSize(m2);
        Assert.assertEquals(idsAndKeys.size(), 1);
        Assert.assertEquals(idsAndKeys.get(0).getMiddle(), "key2");
        m2.close();
        Assert.assertEquals(m1.getHeadersAndPayload().refCnt(), 1);
    } finally {
        reader.closeAsync().get();
    }
}

Example 23 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by yahoo.

the class TopicsConsumerImplTest method testAsyncConsumer.

@Test(timeOut = testTimeout)
public void testAsyncConsumer() throws Exception {
    String key = "TopicsConsumerAsyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    TenantInfoImpl tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).enableBatching(false).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).enableBatching(false).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof MultiTopicsConsumerImpl);
    // Asynchronously produce messages
    List<Future<MessageId>> futures = Lists.newArrayList();
    for (int i = 0; i < totalMessages / 3; i++) {
        futures.add(producer1.sendAsync((messagePredicate + "producer1-" + i).getBytes()));
        futures.add(producer2.sendAsync((messagePredicate + "producer2-" + i).getBytes()));
        futures.add(producer3.sendAsync((messagePredicate + "producer3-" + i).getBytes()));
    }
    log.info("Waiting for async publish to complete : {}", futures.size());
    for (Future<MessageId> future : futures) {
        future.get();
    }
    log.info("start async consume");
    CountDownLatch latch = new CountDownLatch(totalMessages);
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.execute(() -> IntStream.range(0, totalMessages).forEach(index -> consumer.receiveAsync().thenAccept(msg -> {
        assertTrue(msg instanceof TopicMessageImpl);
        try {
            consumer.acknowledge(msg);
        } catch (PulsarClientException e1) {
            fail("message acknowledge failed", e1);
        }
        latch.countDown();
        log.info("receive index: {}, latch countDown: {}", index, latch.getCount());
    }).exceptionally(ex -> {
        log.warn("receive index: {}, failed receive message {}", index, ex.getMessage());
        ex.printStackTrace();
        return null;
    })));
    latch.await();
    log.info("success latch wait");
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}

Example 24 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by yahoo.

the class RawReaderTest method testBatchingRebatch.

@Test
public void testBatchingRebatch() throws Exception {
    String topic = "persistent://my-property/my-ns/my-raw-topic";
    try (Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).messageRoutingMode(MessageRoutingMode.SinglePartition).create()) {
        producer.newMessage().key("key1").value("my-content-1".getBytes()).sendAsync();
        producer.newMessage().key("key2").value("my-content-2".getBytes()).sendAsync();
        producer.newMessage().key("key3").value("my-content-3".getBytes()).send();
    }
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m1 = reader.readNextAsync().get()) {
        RawMessage m2 = RawBatchConverter.rebatchMessage(m1, (key, id) -> key.equals("key2")).get();
        List<ImmutableTriple<MessageId, String, Integer>> idsAndKeys = RawBatchConverter.extractIdsAndKeysAndSize(m2);
        Assert.assertEquals(idsAndKeys.size(), 1);
        Assert.assertEquals(idsAndKeys.get(0).getMiddle(), "key2");
        m2.close();
        Assert.assertEquals(m1.getHeadersAndPayload().refCnt(), 1);
    } finally {
        reader.closeAsync().get();
    }
}

Example 25 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by yahoo.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic-xyz";
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        final int index = i;
        producer.sendAsync(content.getBytes()).thenAccept((msgId) -> {
            assertEquals(msgId, new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}