Examples with PropertyAdmin org.apache.pulsar.common.policies.data.PropertyAdmin used on opensource projects

Example 6 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class PerMessageUnAcknowledgedRedeliveryTest method testSharedAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedAckedPartitionedTopic() throws Exception {
    String key = "testSharedAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 15;
    final int numberOfPartitions = 3;
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(50).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).subscriptionType(SubscriptionType.Shared).subscribe();
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message, doesn't ack
    Message<byte[]> message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    long size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    // 5. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 6. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        received++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    assertEquals(received, 5);
    // 7. Simulate ackTimeout
    ((PartitionedConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
    ((PartitionedConsumerImpl<byte[]>) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
    // 8. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 9. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 10);
    Thread.sleep(ackTimeOutMillis);
    // 10. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        redelivered++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 5);
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
}

Example 7 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class RawReaderTest method setup.

@BeforeMethod
@Override
public void setup() throws Exception {
    super.internalSetup();
    admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1:" + BROKER_WEBSERVICE_PORT));
    admin.properties().createProperty("my-property", new PropertyAdmin(Lists.newArrayList("appid1", "appid2"), Sets.newHashSet("use")));
    admin.namespaces().createNamespace("my-property/use/my-ns");
}

Example 8 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testDifferentTopicsNameSubscribe.

// Verify subscribe topics from different namespace should return error.
@Test(timeOut = testTimeout)
public void testDifferentTopicsNameSubscribe() throws Exception {
    String key = "TopicsFromDifferentNamespace";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String topicName1 = "persistent://prop/use/ns-abc1/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc2/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc3/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 2. Create consumer
    try {
        pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).subscribe();
        fail("subscribe for topics from different namespace should fail.");
    } catch (IllegalArgumentException e) {
    // expected for have different namespace
    }
}

Example 9 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testConsumerUnackedRedelivery.

@Test(timeOut = testTimeout)
public void testConsumerUnackedRedelivery() throws Exception {
    String key = "TopicsConsumerRedeliveryTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof TopicsConsumerImpl);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-" + i).getBytes());
        producer2.send((messagePredicate + "producer2-" + i).getBytes());
        producer3.send((messagePredicate + "producer3-" + i).getBytes());
    }
    // 4. Receiver receives the message, not ack, Unacked Message Tracker size should be totalMessages.
    Message<byte[]> message = consumer.receive();
    while (message != null) {
        assertTrue(message instanceof TopicMessageImpl);
        log.debug("Consumer received : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    }
    long size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
    log.debug(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, totalMessages);
    // 5. Blocking call, redeliver should kick in, after receive and ack, Unacked Message Tracker size should be 0.
    message = consumer.receive();
    HashSet<String> hSet = new HashSet<>();
    do {
        assertTrue(message instanceof TopicMessageImpl);
        hSet.add(new String(message.getData()));
        consumer.acknowledge(message);
        log.debug("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
    log.debug(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 0);
    assertEquals(hSet.size(), totalMessages);
    // 6. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-round2" + i).getBytes());
        producer2.send((messagePredicate + "producer2-round2" + i).getBytes());
        producer3.send((messagePredicate + "producer3-round2" + i).getBytes());
    }
    // 7. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        assertTrue(message instanceof TopicMessageImpl);
        received++;
        String data = new String(message.getData());
        log.debug("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
    log.debug(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 0);
    assertEquals(received, totalMessages);
    // 8. Simulate ackTimeout
    ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().toggle();
    ((TopicsConsumerImpl<byte[]>) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
    // 9. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        producer1.send((messagePredicate + "producer1-round3" + i).getBytes());
        producer2.send((messagePredicate + "producer2-round3" + i).getBytes());
        producer3.send((messagePredicate + "producer3-round3" + i).getBytes());
    }
    // 10. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.debug("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
    log.debug(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 30);
    Thread.sleep(ackTimeOutMillis);
    // 11. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        assertTrue(message instanceof TopicMessageImpl);
        redelivered++;
        String data = new String(message.getData());
        log.debug("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 30);
    size = ((TopicsConsumerImpl<byte[]>) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 0);
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}

Example 10 with PropertyAdmin

use of org.apache.pulsar.common.policies.data.PropertyAdmin in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testAsyncConsumer.

@Test(timeOut = testTimeout)
public void testAsyncConsumer() throws Exception {
    String key = "TopicsConsumerAsyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof TopicsConsumerImpl);
    // Asynchronously produce messages
    List<Future<MessageId>> futures = Lists.newArrayList();
    for (int i = 0; i < totalMessages / 3; i++) {
        futures.add(producer1.sendAsync((messagePredicate + "producer1-" + i).getBytes()));
        futures.add(producer2.sendAsync((messagePredicate + "producer2-" + i).getBytes()));
        futures.add(producer3.sendAsync((messagePredicate + "producer3-" + i).getBytes()));
    }
    log.info("Waiting for async publish to complete : {}", futures.size());
    for (Future<MessageId> future : futures) {
        future.get();
    }
    log.info("start async consume");
    CountDownLatch latch = new CountDownLatch(totalMessages);
    ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.execute(() -> IntStream.range(0, totalMessages).forEach(index -> consumer.receiveAsync().thenAccept(msg -> {
        assertTrue(msg instanceof TopicMessageImpl);
        try {
            consumer.acknowledge(msg);
        } catch (PulsarClientException e1) {
            fail("message acknowledge failed", e1);
        }
        latch.countDown();
        log.info("receive index: {}, latch countDown: {}", index, latch.getCount());
    }).exceptionally(ex -> {
        log.warn("receive index: {}, failed receive message {}", index, ex.getMessage());
        ex.printStackTrace();
        return null;
    })));
    latch.await();
    log.info("success latch wait");
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}