Examples with DestinationName com.yahoo.pulsar.common.naming.DestinationName used on opensource projects

Example 36 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testFailures.

@Test(enabled = true)
public void testFailures() throws Exception {
    log.info("--- Starting ReplicatorTest::testFailures ---");
    try {
        // 1. Create a consumer using the reserved consumer id prefix "pulsar.repl."
        final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns/res-cons-id"));
        // Create another consumer using replication prefix as sub id
        MessageConsumer consumer = new MessageConsumer(url2, dest, "pulsar.repl.");
        consumer.close();
    } catch (Exception e) {
    // SUCCESS
    }
}

Example 37 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testConcurrentReplicator.

@Test
public void testConcurrentReplicator() throws Exception {
    log.info("--- Starting ReplicatorTest::testConcurrentReplicator ---");
    final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns1/topic-%d", 0));
    ClientConfiguration conf = new ClientConfiguration();
    conf.setStatsInterval(0, TimeUnit.SECONDS);
    Producer producer = PulsarClient.create(url1.toString(), conf).createProducer(dest.toString());
    producer.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(dest.toString()).get();
    PulsarClientImpl pulsarClient = spy((PulsarClientImpl) pulsar1.getBrokerService().getReplicationClient("r3"));
    final Method startRepl = PersistentTopic.class.getDeclaredMethod("startReplicator", String.class);
    startRepl.setAccessible(true);
    Field replClientField = BrokerService.class.getDeclaredField("replicationClients");
    replClientField.setAccessible(true);
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients = (ConcurrentOpenHashMap<String, PulsarClient>) replClientField.get(pulsar1.getBrokerService());
    replicationClients.put("r3", pulsarClient);
    ExecutorService executor = Executors.newFixedThreadPool(5);
    for (int i = 0; i < 5; i++) {
        executor.submit(() -> {
            try {
                startRepl.invoke(topic, "r3");
            } catch (Exception e) {
                fail("setting replicator failed", e);
            }
        });
    }
    Thread.sleep(3000);
    Mockito.verify(pulsarClient, Mockito.times(1)).createProducerAsync(Mockito.anyString(), Mockito.anyObject(), Mockito.anyString());
}

Example 38 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testResetCursor.

@Test(timeOut = 10000, dataProvider = "subType")
public void testResetCursor(SubscriptionType subType) throws Exception {
    final RetentionPolicies policy = new RetentionPolicies(60, 52 * 1024);
    final DestinationName destName = DestinationName.get("persistent://my-property/use/my-ns/unacked-topic");
    final int warmup = 20;
    final int testSize = 150;
    final List<Message> received = new ArrayList<Message>();
    final ConsumerConfiguration consConfig = new ConsumerConfiguration();
    final String subsId = "sub";
    final NavigableMap<Long, TimestampEntryCount> publishTimeIdMap = new ConcurrentSkipListMap<>();
    consConfig.setSubscriptionType(subType);
    consConfig.setMessageListener((MessageListener) (Consumer consumer, Message msg) -> {
        try {
            synchronized (received) {
                received.add(msg);
            }
            consumer.acknowledge(msg);
            long publishTime = ((MessageImpl) msg).getPublishTime();
            log.info(" publish time is " + publishTime + "," + msg.getMessageId());
            TimestampEntryCount timestampEntryCount = publishTimeIdMap.computeIfAbsent(publishTime, (k) -> new TimestampEntryCount(publishTime));
            timestampEntryCount.incrementAndGet();
        } catch (final PulsarClientException e) {
            log.warn("Failed to ack!");
        }
    });
    admin.namespaces().setRetention(destName.getNamespace(), policy);
    Consumer consumer = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
    final Producer producer = pulsarClient.createProducer(destName.toString());
    log.info("warm up started for " + destName.toString());
    // send warmup msgs
    byte[] msgBytes = new byte[1000];
    for (Integer i = 0; i < warmup; i++) {
        producer.send(msgBytes);
    }
    log.info("warm up finished.");
    // sleep to ensure receiving of msgs
    for (int n = 0; n < 10 && received.size() < warmup; n++) {
        Thread.sleep(100);
    }
    // validate received msgs
    Assert.assertEquals(received.size(), warmup);
    received.clear();
    // publish testSize num of msgs
    log.info("Sending more messages.");
    for (Integer n = 0; n < testSize; n++) {
        producer.send(msgBytes);
        Thread.sleep(1);
    }
    log.info("Sending more messages done.");
    Thread.sleep(3000);
    long begints = publishTimeIdMap.firstEntry().getKey();
    long endts = publishTimeIdMap.lastEntry().getKey();
    // find reset timestamp
    long timestamp = (endts - begints) / 2 + begints;
    timestamp = publishTimeIdMap.floorKey(timestamp);
    NavigableMap<Long, TimestampEntryCount> expectedMessages = new ConcurrentSkipListMap<>();
    expectedMessages.putAll(publishTimeIdMap.tailMap(timestamp, true));
    received.clear();
    log.info("reset cursor to " + timestamp + " for topic " + destName.toString() + " for subs " + subsId);
    log.info("issuing admin operation on " + admin.getServiceUrl().toString());
    List<String> subList = admin.persistentTopics().getSubscriptions(destName.toString());
    for (String subs : subList) {
        log.info("got sub " + subs);
    }
    publishTimeIdMap.clear();
    // reset the cursor to this timestamp
    Assert.assertTrue(subList.contains(subsId));
    admin.persistentTopics().resetCursor(destName.toString(), subsId, timestamp);
    consumer = pulsarClient.subscribe(destName.toString(), subsId, consConfig);
    Thread.sleep(3000);
    int totalExpected = 0;
    for (TimestampEntryCount tec : expectedMessages.values()) {
        totalExpected += tec.numMessages;
    }
    // validate that replay happens after the timestamp
    Assert.assertTrue(publishTimeIdMap.firstEntry().getKey() >= timestamp);
    consumer.close();
    producer.close();
    // validate that expected and received counts match
    int totalReceived = 0;
    for (TimestampEntryCount tec : publishTimeIdMap.values()) {
        totalReceived += tec.numMessages;
    }
    Assert.assertEquals(totalReceived, totalExpected, "did not receive all messages on replay after reset");
}

Example 39 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class PartitionedProducerConsumerTest method testSinglePartitionProducer.

@Test
public void testSinglePartitionProducer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int numPartitions = 4;
    DestinationName dn = DestinationName.get("persistent://my-property/use/my-ns/my-partitionedtopic2");
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    admin.persistentTopics().createPartitionedTopic(dn.toString(), numPartitions);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.SinglePartition);
    Producer producer = pulsarClient.createProducer(dn.toString(), producerConf);
    Consumer consumer = pulsarClient.subscribe(dn.toString(), "my-partitioned-subscriber", conf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        Assert.assertNotNull(msg, "Message should not be null");
        consumer.acknowledge(msg);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    producer.close();
    consumer.unsubscribe();
    consumer.close();
    admin.persistentTopics().deletePartitionedTopic(dn.toString());
    log.info("-- Exiting {} test --", methodName);
}

Example 40 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class PartitionedProducerConsumerTest method testRoundRobinProducer.

@Test
public void testRoundRobinProducer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int numPartitions = 4;
    DestinationName dn = DestinationName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    admin.persistentTopics().createPartitionedTopic(dn.toString(), numPartitions);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(dn.toString(), producerConf);
    Consumer consumer = pulsarClient.subscribe(dn.toString(), "my-partitioned-subscriber", conf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        Assert.assertNotNull(msg, "Message should not be null");
        consumer.acknowledge(msg);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        Assert.assertTrue(messageSet.add(receivedMessage), "Message " + receivedMessage + " already received");
    }
    producer.close();
    consumer.unsubscribe();
    consumer.close();
    admin.persistentTopics().deletePartitionedTopic(dn.toString());
    log.info("-- Exiting {} test --", methodName);
}