Examples with MessageRoutingMode org.apache.pulsar.client.api.MessageRoutingMode used on opensource projects

Example 16 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/ns-abc/topic-xyz";
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        final int index = i;
        producer.sendAsync(content.getBytes()).thenAccept((msgId) -> {
            assertEquals(msgId, new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 17 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class ReplicatorRateLimiterTest method testReplicatorRateLimiterMessageNotReceivedAllMessages.

/**
 * verifies dispatch rate for replicators works well for both Message limit and Byte limit .
 *
 * 1. verify topic replicator get configured.
 * 2. namespace setting of replicator dispatchRate, verify consumer in other cluster could not receive all messages.
 *
 * @throws Exception
 */
@Test(dataProvider = "dispatchRateType")
public void testReplicatorRateLimiterMessageNotReceivedAllMessages(DispatchRateType dispatchRateType) throws Exception {
    log.info("--- Starting ReplicatorTest::{} --- ", methodName);
    final String namespace = "pulsar/replicatorbyteandmsg-" + dispatchRateType.toString() + "-" + System.currentTimeMillis();
    final String topicName = "persistent://" + namespace + "/notReceivedAll";
    admin1.namespaces().createNamespace(namespace);
    // 0. set 2 clusters, there will be 1 replicator in each topic
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
    final int messageRate = 100;
    DispatchRate dispatchRate;
    if (DispatchRateType.messageRate.equals(dispatchRateType)) {
        dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(messageRate).dispatchThrottlingRateInByte(-1).ratePeriodInSecond(360).build();
    } else {
        dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(-1).dispatchThrottlingRateInByte(messageRate).ratePeriodInSecond(360).build();
    }
    admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRate);
    @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Producer<byte[]> producer = client1.newProducer().topic(topicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get();
    boolean replicatorUpdated = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getReplicators().values().get(0).getRateLimiter().isPresent()) {
            replicatorUpdated = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(replicatorUpdated);
    if (DispatchRateType.messageRate.equals(dispatchRateType)) {
        Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), messageRate);
    } else {
        Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnByte(), messageRate);
    }
    @Cleanup PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    int numMessages = 500;
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    log.info("Received message number: [{}]", totalReceived.get());
    Assert.assertTrue(totalReceived.get() < messageRate * 2);
    consumer.close();
    producer.close();
}

Example 18 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class ReplicatorRateLimiterTest method testReplicatorRateLimiterMessageReceivedAllMessages.

/**
 * verifies dispatch rate for replicators works well for both Message limit.
 *
 * 1. verify topic replicator get configured.
 * 2. namespace setting of replicator dispatchRate,
 *      verify consumer in other cluster could receive all messages < message limit.
 * 3. verify consumer in other cluster could not receive all messages > message limit.
 *
 * @throws Exception
 */
@Test
public void testReplicatorRateLimiterMessageReceivedAllMessages() throws Exception {
    log.info("--- Starting ReplicatorTest::{} --- ", methodName);
    final String namespace = "pulsar/replicatormsg-" + System.currentTimeMillis();
    final String topicName = "persistent://" + namespace + "/notReceivedAll";
    admin1.namespaces().createNamespace(namespace);
    // 0. set 2 clusters, there will be 1 replicator in each topic
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("r1", "r2"));
    final int messageRate = 100;
    DispatchRate dispatchRate = DispatchRate.builder().dispatchThrottlingRateInMsg(messageRate).dispatchThrottlingRateInByte(-1).ratePeriodInSecond(360).build();
    admin1.namespaces().setReplicatorDispatchRate(namespace, dispatchRate);
    @Cleanup PulsarClient client1 = PulsarClient.builder().serviceUrl(url1.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Producer<byte[]> producer = client1.newProducer().topic(topicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getOrCreateTopic(topicName).get();
    boolean replicatorUpdated = false;
    int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (topic.getReplicators().values().get(0).getRateLimiter().isPresent()) {
            replicatorUpdated = true;
            break;
        } else {
            if (i != retry - 1) {
                Thread.sleep(100);
            }
        }
    }
    Assert.assertTrue(replicatorUpdated);
    Assert.assertEquals(topic.getReplicators().values().get(0).getRateLimiter().get().getDispatchRateOnMsg(), messageRate);
    @Cleanup PulsarClient client2 = PulsarClient.builder().serviceUrl(url2.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final AtomicInteger totalReceived = new AtomicInteger(0);
    Consumer<byte[]> consumer = client2.newConsumer().topic(topicName).subscriptionName("sub2-in-cluster2").messageListener((c1, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        totalReceived.incrementAndGet();
    }).subscribe();
    int numMessages = 50;
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    Thread.sleep(1000);
    log.info("Received message number: [{}]", totalReceived.get());
    Assert.assertEquals(totalReceived.get(), numMessages);
    numMessages = 200;
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        producer.send(new byte[80]);
    }
    Thread.sleep(1000);
    log.info("Received message number: [{}]", totalReceived.get());
    Assert.assertEquals(totalReceived.get(), messageRate);
    consumer.close();
    producer.close();
}

Example 19 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class ProducerHandler method getProducerBuilder.

protected ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client) {
    ProducerBuilder<byte[]> builder = client.newProducer().enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition);
    // Set to false to prevent the server thread from being blocked if a lot of messages are pending.
    builder.blockIfQueueFull(false);
    if (queryParams.containsKey("producerName")) {
        builder.producerName(queryParams.get("producerName"));
    }
    if (queryParams.containsKey("initialSequenceId")) {
        builder.initialSequenceId(Long.parseLong(queryParams.get("initialSequenceId")));
    }
    if (queryParams.containsKey("hashingScheme")) {
        builder.hashingScheme(HashingScheme.valueOf(queryParams.get("hashingScheme")));
    }
    if (queryParams.containsKey("sendTimeoutMillis")) {
        builder.sendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS);
    }
    if (queryParams.containsKey("batchingEnabled")) {
        builder.enableBatching(Boolean.parseBoolean(queryParams.get("batchingEnabled")));
    }
    if (queryParams.containsKey("batchingMaxMessages")) {
        builder.batchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages")));
    }
    if (queryParams.containsKey("maxPendingMessages")) {
        builder.maxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages")));
    }
    if (queryParams.containsKey("batchingMaxPublishDelay")) {
        builder.batchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")), TimeUnit.MILLISECONDS);
    }
    if (queryParams.containsKey("messageRoutingMode")) {
        checkArgument(Enums.getIfPresent(MessageRoutingMode.class, queryParams.get("messageRoutingMode")).isPresent(), "Invalid messageRoutingMode %s", queryParams.get("messageRoutingMode"));
        MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode"));
        if (!MessageRoutingMode.CustomPartition.equals(routingMode)) {
            builder.messageRoutingMode(routingMode);
        }
    }
    if (queryParams.containsKey("compressionType")) {
        checkArgument(Enums.getIfPresent(CompressionType.class, queryParams.get("compressionType")).isPresent(), "Invalid compressionType %s", queryParams.get("compressionType"));
        builder.compressionType(CompressionType.valueOf(queryParams.get("compressionType")));
    }
    return builder;
}

Example 20 with MessageRoutingMode

use of org.apache.pulsar.client.api.MessageRoutingMode in project incubator-pulsar by apache.

the class AbstractWebSocketHandlerTest method producerBuilderTest.

@Test
public void producerBuilderTest() throws IOException {
    String producerV2 = "/ws/v2/producer/persistent/my-property/my-ns/my-topic";
    // the params are all different with the default value
    Map<String, String[]> queryParams = new HashMap<String, String>() {

        {
            put("producerName", "my-producer");
            put("initialSequenceId", "1");
            put("hashingScheme", "Murmur3_32Hash");
            put("sendTimeoutMillis", "30001");
            put("batchingEnabled", "true");
            put("batchingMaxMessages", "1001");
            put("maxPendingMessages", "1001");
            put("batchingMaxPublishDelay", "2");
            put("messageRoutingMode", "RoundRobinPartition");
            put("compressionType", "LZ4");
        }
    }.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> new String[] { entry.getValue() }));
    httpServletRequest = mock(HttpServletRequest.class);
    when(httpServletRequest.getRequestURI()).thenReturn(producerV2);
    when(httpServletRequest.getParameterMap()).thenReturn(queryParams);
    WebSocketService service = mock(WebSocketService.class);
    when(service.isAuthenticationEnabled()).thenReturn(false);
    when(service.isAuthorizationEnabled()).thenReturn(false);
    when(service.getPulsarClient()).thenReturn(newPulsarClient());
    MockedServletUpgradeResponse response = new MockedServletUpgradeResponse(null);
    MockedProducerHandler producerHandler = new MockedProducerHandler(service, httpServletRequest, response);
    assertEquals(response.getStatusCode(), 500);
    assertTrue(response.getMessage().contains("Connection refused"));
    ProducerConfigurationData conf = producerHandler.getConf();
    assertEquals(conf.getProducerName(), "my-producer");
    assertEquals(conf.getInitialSequenceId().longValue(), 1L);
    assertEquals(conf.getHashingScheme(), HashingScheme.Murmur3_32Hash);
    assertEquals(conf.getSendTimeoutMs(), 30001);
    assertTrue(conf.isBatchingEnabled());
    assertEquals(conf.getBatchingMaxMessages(), 1001);
    assertEquals(conf.getMaxPendingMessages(), 1001);
    assertEquals(conf.getBatchingMaxPublishDelayMicros(), 2000);
    assertEquals(conf.getMessageRoutingMode(), MessageRoutingMode.RoundRobinPartition);
    assertEquals(conf.getCompressionType(), CompressionType.LZ4);
    producerHandler.clearQueryParams();
    conf = producerHandler.getConf();
    // By default batching is disabled, which is different with ProducerBuilder
    assertFalse(conf.isBatchingEnabled());
    // The default message routing mode is SinglePartition, which is different with ProducerBuilder
    assertEquals(conf.getMessageRoutingMode(), MessageRoutingMode.SinglePartition);
    producerHandler.putQueryParam("messageRoutingMode", "CustomPartition");
    conf = producerHandler.getConf();
    // ProducerHandler doesn't support CustomPartition
    assertEquals(conf.getMessageRoutingMode(), MessageRoutingMode.SinglePartition);
}