Examples with MessageRoutingMode org.apache.pulsar.client.api.MessageRoutingMode used on opensource projects

Example 1 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by apache.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic-xyz";
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        final int index = i;
        producer.sendAsync(content.getBytes()).thenAccept((msgId) -> {
            assertEquals(msgId, new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 2 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by apache.

the class TopicsConsumerImplTest method testAsyncConsumer.

@Test(timeOut = testTimeout)
public void testAsyncConsumer() throws Exception {
    String key = "TopicsConsumerAsyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    TenantInfoImpl tenantInfo = createDefaultTenantInfo();
    admin.tenants().createTenant("prop", tenantInfo);
    admin.topics().createPartitionedTopic(topicName2, 2);
    admin.topics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).enableBatching(false).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).enableBatching(false).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof MultiTopicsConsumerImpl);
    // Asynchronously produce messages
    List<Future<MessageId>> futures = Lists.newArrayList();
    for (int i = 0; i < totalMessages / 3; i++) {
        futures.add(producer1.sendAsync((messagePredicate + "producer1-" + i).getBytes()));
        futures.add(producer2.sendAsync((messagePredicate + "producer2-" + i).getBytes()));
        futures.add(producer3.sendAsync((messagePredicate + "producer3-" + i).getBytes()));
    }
    log.info("Waiting for async publish to complete : {}", futures.size());
    for (Future<MessageId> future : futures) {
        future.get();
    }
    log.info("start async consume");
    CountDownLatch latch = new CountDownLatch(totalMessages);
    @Cleanup("shutdownNow") ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.execute(() -> IntStream.range(0, totalMessages).forEach(index -> consumer.receiveAsync().thenAccept(msg -> {
        assertTrue(msg instanceof TopicMessageImpl);
        try {
            consumer.acknowledge(msg);
        } catch (PulsarClientException e1) {
            fail("message acknowledge failed", e1);
        }
        latch.countDown();
        log.info("receive index: {}, latch countDown: {}", index, latch.getCount());
    }).exceptionally(ex -> {
        log.warn("receive index: {}, failed receive message {}", index, ex.getMessage());
        ex.printStackTrace();
        return null;
    })));
    latch.await();
    log.info("success latch wait");
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}

Example 3 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by apache.

the class ProducerHandler method getProducerBuilder.

protected ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client) {
    ProducerBuilder<byte[]> builder = client.newProducer().enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition);
    // Set to false to prevent the server thread from being blocked if a lot of messages are pending.
    builder.blockIfQueueFull(false);
    if (queryParams.containsKey("producerName")) {
        builder.producerName(queryParams.get("producerName"));
    }
    if (queryParams.containsKey("initialSequenceId")) {
        builder.initialSequenceId(Long.parseLong(queryParams.get("initialSequenceId")));
    }
    if (queryParams.containsKey("hashingScheme")) {
        builder.hashingScheme(HashingScheme.valueOf(queryParams.get("hashingScheme")));
    }
    if (queryParams.containsKey("sendTimeoutMillis")) {
        builder.sendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS);
    }
    if (queryParams.containsKey("batchingEnabled")) {
        builder.enableBatching(Boolean.parseBoolean(queryParams.get("batchingEnabled")));
    }
    if (queryParams.containsKey("batchingMaxMessages")) {
        builder.batchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages")));
    }
    if (queryParams.containsKey("maxPendingMessages")) {
        builder.maxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages")));
    }
    if (queryParams.containsKey("batchingMaxPublishDelay")) {
        builder.batchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")), TimeUnit.MILLISECONDS);
    }
    if (queryParams.containsKey("messageRoutingMode")) {
        checkArgument(Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(), "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode"));
        MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode"));
        if (!MessageRoutingMode.CustomPartition.equals(routingMode)) {
            builder.messageRoutingMode(routingMode);
        }
    }
    if (queryParams.containsKey("compressionType")) {
        checkArgument(Enums.getIfPresent(CompressionType.class, queryParams.get("compressionType")).isPresent(), "Invalid compressionType %s", queryParams.get("compressionType"));
        builder.compressionType(CompressionType.valueOf(queryParams.get("compressionType")));
    }
    return builder;
}

Example 4 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project pulsar by apache.

the class AbstractWebSocketHandlerTest method producerBuilderTest.

@Test
public void producerBuilderTest() throws IOException {
    String producerV2 = "/ws/v2/producer/persistent/my-property/my-ns/my-topic";
    // the params are all different with the default value
    Map<String, String[]> queryParams = new HashMap<String, String>() {

        {
            put("producerName", "my-producer");
            put("initialSequenceId", "1");
            put("hashingScheme", "Murmur3_32Hash");
            put("sendTimeoutMillis", "30001");
            put("batchingEnabled", "false");
            put("batchingMaxMessages", "1001");
            put("maxPendingMessages", "1001");
            put("batchingMaxPublishDelay", "2");
            put("messageRoutingMode", "RoundRobinPartition");
            put("compressionType", "LZ4");
        }
    }.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> new String[] { entry.getValue() }));
    httpServletRequest = mock(HttpServletRequest.class);
    when(httpServletRequest.getRequestURI()).thenReturn(producerV2);
    when(httpServletRequest.getParameterMap()).thenReturn(queryParams);
    WebSocketService service = mock(WebSocketService.class);
    when(service.isAuthenticationEnabled()).thenReturn(false);
    when(service.isAuthorizationEnabled()).thenReturn(false);
    when(service.getPulsarClient()).thenReturn(newPulsarClient());
    MockedServletUpgradeResponse response = new MockedServletUpgradeResponse(null);
    MockedProducerHandler producerHandler = new MockedProducerHandler(service, httpServletRequest, response);
    assertEquals(response.getStatusCode(), 500);
    assertTrue(response.getMessage().contains("Connection refused"));
    ProducerConfigurationData conf = producerHandler.getConf();
    assertEquals(conf.getProducerName(), "my-producer");
    assertEquals(conf.getInitialSequenceId().longValue(), 1L);
    assertEquals(conf.getHashingScheme(), HashingScheme.Murmur3_32Hash);
    assertEquals(conf.getSendTimeoutMs(), 30001);
    assertFalse(conf.isBatchingEnabled());
    assertEquals(conf.getBatchingMaxMessages(), 1001);
    assertEquals(conf.getMaxPendingMessages(), 1001);
    assertEquals(conf.getMessageRoutingMode(), MessageRoutingMode.RoundRobinPartition);
    assertEquals(conf.getCompressionType(), CompressionType.LZ4);
    producerHandler.clearQueryParams();
    conf = producerHandler.getConf();
    // The default message routing mode is SinglePartition, which is different with ProducerBuilder
    assertEquals(conf.getMessageRoutingMode(), MessageRoutingMode.SinglePartition);
    producerHandler.putQueryParam("messageRoutingMode", "CustomPartition");
    conf = producerHandler.getConf();
    // ProducerHandler doesn't support CustomPartition
    assertEquals(conf.getMessageRoutingMode(), MessageRoutingMode.SinglePartition);
}

Example 5 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-inlong by apache.

the class PulsarUtils method createProducer.

public static Producer<byte[]> createProducer(PulsarClient client, String topic, Configuration configuration) throws PulsarClientException {
    MessageRoutingMode routeMode = Enum.valueOf(MessageRoutingMode.class, PRODUCER_ROUTE_MODE.defaultValue());
    Duration sendTimeout = parseDuration(configuration.getString(PRODUCER_SEND_TIMEOUT));
    int pendingQueueSize = configuration.getInteger(PRODUCER_PENDING_QUEUE_SIZE);
    int pendingSize = configuration.getInteger(PRODUCER_PENDING_SIZE);
    boolean blockIfQueueFull = configuration.getBoolean(PRODUCER_BLOCK_QUEUE_FULL);
    return client.newProducer().topic(topic).messageRoutingMode(routeMode).sendTimeout((int) sendTimeout.toMillis(), TimeUnit.MILLISECONDS).maxPendingMessages(pendingQueueSize).maxPendingMessagesAcrossPartitions(pendingSize).blockIfQueueFull(blockIfQueueFull).create();
}