Examples with DestinationName com.yahoo.pulsar.common.naming.DestinationName used on opensource projects

Example 66 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class BrokerServiceLookupTest method testPartitionTopicLookup.

/**
     * Create #PartitionedTopic and let it served by multiple brokers which requries 
     * a. tcp partitioned-metadata-lookup
     * b. multiple topic-lookup 
     * c. partitioned producer-consumer
     * 
     * @throws Exception
     */
@Test
public void testPartitionTopicLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int numPartitions = 8;
    DestinationName dn = DestinationName.get("persistent://my-property/use/my-ns/my-partitionedtopic1");
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    admin.persistentTopics().createPartitionedTopic(dn.toString(), numPartitions);
    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(PortManager.nextFreePort());
    conf2.setBrokerServicePortTls(PortManager.nextFreePort());
    conf2.setWebServicePort(PortManager.nextFreePort());
    conf2.setWebServicePortTls(PortManager.nextFreePort());
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(pulsar.getConfiguration().getClusterName());
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().writeLoadReportOnZookeeper();
    LoadManager loadManager1 = spy(pulsar.getLoadManager());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);
    // mock: return Broker2 as a Least-loaded broker when leader receies request
    doReturn(true).when(loadManager1).isCentralized();
    loadManagerField.set(pulsar.getNamespaceService(), loadManager1);
    // mock: redirect request to leader
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), loadManager2);
    /****  broker-2 started ****/
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(dn.toString(), producerConf);
    Consumer consumer = pulsarClient.subscribe(dn.toString(), "my-partitioned-subscriber", conf);
    for (int i = 0; i < 20; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 20; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        Assert.assertNotNull(msg, "Message should not be null");
        consumer.acknowledge(msg);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        Assert.assertTrue(messageSet.add(receivedMessage), "Message " + receivedMessage + " already received");
    }
    producer.close();
    consumer.unsubscribe();
    consumer.close();
    admin.persistentTopics().deletePartitionedTopic(dn.toString());
    pulsar2.close();
    loadManager2 = null;
    log.info("-- Exiting {} test --", methodName);
}

Example 67 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testReplicatorClearBacklog.

@Test
public void testReplicatorClearBacklog() throws Exception {
    // This test is to verify that reset cursor fails on global topic
    SortedSet<String> testDests = new TreeSet<String>();
    final DestinationName dest = DestinationName.get("persistent://pulsar/global/ns/clearBacklogTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = spy(topic.getReplicators().get(topic.getReplicators().keys().get(0)));
    replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
    replicator.clearBacklog().get();
    Thread.sleep(100);
    // for code-coverage
    replicator.updateRates();
    // for code-coverage
    replicator.expireMessages(1);
    ReplicatorStats status = replicator.getStats();
    assertTrue(status.replicationBacklog == 0);
    consumer1.close();
}

Example 68 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testReplicationForBatchMessages.

@Test(enabled = true)
public void testReplicationForBatchMessages() throws Exception {
    log.info("--- Starting ReplicatorTest::testReplicationForBatchMessages ---");
    // Run a set of producer tasks to create the destinations
    SortedSet<String> testDests = new TreeSet<String>();
    List<Future<Void>> results = Lists.newArrayList();
    for (int i = 0; i < 3; i++) {
        final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns/repltopicbatch-%d", i));
        testDests.add(dest.toString());
        results.add(executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                MessageProducer producer1 = new MessageProducer(url1, dest, true);
                log.info("--- Starting producer --- " + url1);
                MessageProducer producer2 = new MessageProducer(url2, dest, true);
                log.info("--- Starting producer --- " + url2);
                MessageProducer producer3 = new MessageProducer(url3, dest, true);
                log.info("--- Starting producer --- " + url3);
                MessageConsumer consumer1 = new MessageConsumer(url1, dest);
                log.info("--- Starting Consumer --- " + url1);
                MessageConsumer consumer2 = new MessageConsumer(url2, dest);
                log.info("--- Starting Consumer --- " + url2);
                MessageConsumer consumer3 = new MessageConsumer(url3, dest);
                log.info("--- Starting Consumer --- " + url3);
                // Produce from cluster1 and consume from the rest
                producer1.produceBatch(10);
                consumer1.receive(10);
                consumer2.receive(10);
                consumer3.receive(10);
                // Produce from cluster2 and consume from the rest
                producer2.produceBatch(10);
                consumer1.receive(10);
                consumer2.receive(10);
                consumer3.receive(10);
                return null;
            }
        }));
    }
    for (Future<Void> result : results) {
        try {
            result.get(5, TimeUnit.SECONDS);
        } catch (Exception e) {
            log.error("exception in getting future result ", e);
            fail(String.format("replication test failed with %s exception", e.getMessage()));
        }
    }
}

Example 69 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testResetCursorNotFail.

@Test(enabled = true)
public void testResetCursorNotFail() throws Exception {
    log.info("--- Starting ReplicatorTest::testResetCursorNotFail ---");
    // This test is to verify that reset cursor fails on global topic
    SortedSet<String> testDests = new TreeSet<String>();
    List<Future<Void>> results = Lists.newArrayList();
    for (int i = 0; i < 1; i++) {
        final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns/resetrepltopic-%d", i));
        testDests.add(dest.toString());
        results.add(executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                MessageProducer producer1 = new MessageProducer(url1, dest);
                log.info("--- Starting producer --- " + url1);
                MessageConsumer consumer1 = new MessageConsumer(url1, dest);
                log.info("--- Starting Consumer --- " + url1);
                // Produce from cluster1 and consume from the rest
                producer1.produce(2);
                consumer1.receive(2);
                producer1.close();
                consumer1.close();
                return null;
            }
        }));
    }
    for (Future<Void> result : results) {
        try {
            result.get();
        } catch (Exception e) {
            log.error("exception in getting future result ", e);
            fail(String.format("replication test failed with %s exception", e.getMessage()));
        }
    }
    admin1.persistentTopics().resetCursor(testDests.first(), "sub-id", System.currentTimeMillis());
}

Example 70 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testCloseReplicatorStartProducer.

/**
     * It verifies that PersistentReplicator considers CursorAlreadyClosedException as non-retriable-read exception and
     * it should closed the producer as cursor is already closed because replicator is already deleted.
     * 
     * @throws Exception
     */
@Test(timeOut = 5000)
public void testCloseReplicatorStartProducer() throws Exception {
    DestinationName dest = DestinationName.get("persistent://pulsar/global/ns1/closeCursor");
    // Producer on r1
    MessageProducer producer1 = new MessageProducer(url1, dest);
    // Consumer on r1
    MessageConsumer consumer1 = new MessageConsumer(url1, dest);
    // Consumer on r2
    MessageConsumer consumer2 = new MessageConsumer(url2, dest);
    // Replicator for r1 -> r2
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = topic.getPersistentReplicator("r2");
    // close the cursor
    Field cursorField = PersistentReplicator.class.getDeclaredField("cursor");
    cursorField.setAccessible(true);
    ManagedCursor cursor = (ManagedCursor) cursorField.get(replicator);
    cursor.close();
    // try to read entries
    CountDownLatch latch = new CountDownLatch(1);
    producer1.produce(10);
    cursor.asyncReadEntriesOrWait(10, new ReadEntriesCallback() {

        @Override
        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            latch.countDown();
            fail("it should have been failed");
        }

        @Override
        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
            assertTrue(exception instanceof CursorAlreadyClosedException);
        }
    }, null);
    // replicator-readException: cursorAlreadyClosed
    replicator.readEntriesFailed(new CursorAlreadyClosedException("Cursor already closed exception"), null);
    // wait replicator producer to be closed
    Thread.sleep(1000);
    // Replicator producer must be closed
    Field producerField = PersistentReplicator.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    ProducerImpl replicatorProducer = (ProducerImpl) producerField.get(replicator);
    assertEquals(replicatorProducer, null);
    producer1.close();
    consumer1.close();
    consumer2.close();
}