Examples with DestinationName com.yahoo.pulsar.common.naming.DestinationName used on opensource projects

Example 31 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testReplicatorProducerClosing.

@Test(priority = 5)
public void testReplicatorProducerClosing() throws Exception {
    log.info("--- Starting ReplicatorTest::testDeleteReplicatorFailure ---");
    final String topicName = "persistent://pulsar/global/ns/repltopicbatch";
    final DestinationName dest = DestinationName.get(topicName);
    MessageProducer producer1 = new MessageProducer(url1, dest);
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(topicName);
    final String replicatorClusterName = topic.getReplicators().keys().get(0);
    PersistentReplicator replicator = topic.getPersistentReplicator(replicatorClusterName);
    pulsar2.close();
    pulsar3.close();
    replicator.disconnect(false);
    Thread.sleep(100);
    Field field = PersistentReplicator.class.getDeclaredField("producer");
    field.setAccessible(true);
    ProducerImpl producer = (ProducerImpl) field.get(replicator);
    assertNull(producer);
}

Example 32 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testReplicatePeekAndSkip.

@Test
public void testReplicatePeekAndSkip() throws Exception {
    SortedSet<String> testDests = new TreeSet<String>();
    final DestinationName dest = DestinationName.get("persistent://pulsar/global/ns/peekAndSeekTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = topic.getReplicators().get(topic.getReplicators().keys().get(0));
    replicator.skipMessages(2);
    CompletableFuture<Entry> result = replicator.peekNthMessage(1);
    Entry entry = result.get(50, TimeUnit.MILLISECONDS);
    assertNull(entry);
    consumer1.close();
}

Example 33 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testConfigChange.

@Test(enabled = true)
public void testConfigChange() throws Exception {
    log.info("--- Starting ReplicatorTest::testConfigChange ---");
    // This test is to verify that the config change on global namespace is successfully applied in broker during
    // runtime.
    // Run a set of producer tasks to create the destinations
    List<Future<Void>> results = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns/topic-%d", i));
        results.add(executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                MessageProducer producer = new MessageProducer(url1, dest);
                log.info("--- Starting producer --- " + url1);
                MessageConsumer consumer = new MessageConsumer(url1, dest);
                log.info("--- Starting Consumer --- " + url1);
                producer.produce(2);
                consumer.receive(2);
                producer.close();
                consumer.close();
                return null;
            }
        }));
    }
    for (Future<Void> result : results) {
        try {
            result.get();
        } catch (Exception e) {
            log.error("exception in getting future result ", e);
            fail(String.format("replication test failed with %s exception", e.getMessage()));
        }
    }
    Thread.sleep(1000L);
    // Make sure that the internal replicators map contains remote cluster info
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients1 = ns1.getReplicationClients();
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients2 = ns2.getReplicationClients();
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients3 = ns3.getReplicationClients();
    Assert.assertNotNull(replicationClients1.get("r2"));
    Assert.assertNotNull(replicationClients1.get("r3"));
    Assert.assertNotNull(replicationClients2.get("r1"));
    Assert.assertNotNull(replicationClients2.get("r3"));
    Assert.assertNotNull(replicationClients3.get("r1"));
    Assert.assertNotNull(replicationClients3.get("r2"));
    // Case 1: Update the global namespace replication configuration to only contains the local cluster itself
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1"));
    // Wait for config changes to be updated.
    Thread.sleep(1000L);
    // Make sure that the internal replicators map still contains remote cluster info
    Assert.assertNotNull(replicationClients1.get("r2"));
    Assert.assertNotNull(replicationClients1.get("r3"));
    Assert.assertNotNull(replicationClients2.get("r1"));
    Assert.assertNotNull(replicationClients2.get("r3"));
    Assert.assertNotNull(replicationClients3.get("r1"));
    Assert.assertNotNull(replicationClients3.get("r2"));
    // Case 2: Update the configuration back
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
    // Wait for config changes to be updated.
    Thread.sleep(1000L);
    // Make sure that the internal replicators map still contains remote cluster info
    Assert.assertNotNull(replicationClients1.get("r2"));
    Assert.assertNotNull(replicationClients1.get("r3"));
    Assert.assertNotNull(replicationClients2.get("r1"));
    Assert.assertNotNull(replicationClients2.get("r3"));
    Assert.assertNotNull(replicationClients3.get("r1"));
    Assert.assertNotNull(replicationClients3.get("r2"));
// Case 3: TODO: Once automatic cleanup is implemented, add tests case to verify auto removal of clusters
}

Example 34 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testReplicationOverrides.

@Test(enabled = false)
public void testReplicationOverrides() throws Exception {
    log.info("--- Starting ReplicatorTest::testReplicationOverrides ---");
    // This test is to verify that the config change on global namespace is successfully applied in broker during
    // runtime.
    // Run a set of producer tasks to create the destinations
    SortedSet<String> testDests = new TreeSet<String>();
    List<Future<Void>> results = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        final DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns/repltopic-%d", i));
        testDests.add(dest.toString());
        results.add(executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                MessageProducer producer1 = new MessageProducer(url1, dest);
                log.info("--- Starting producer --- " + url1);
                MessageProducer producer2 = new MessageProducer(url2, dest);
                log.info("--- Starting producer --- " + url2);
                MessageProducer producer3 = new MessageProducer(url3, dest);
                log.info("--- Starting producer --- " + url3);
                MessageConsumer consumer1 = new MessageConsumer(url1, dest);
                log.info("--- Starting Consumer --- " + url1);
                MessageConsumer consumer2 = new MessageConsumer(url2, dest);
                log.info("--- Starting Consumer --- " + url2);
                MessageConsumer consumer3 = new MessageConsumer(url3, dest);
                log.info("--- Starting Consumer --- " + url3);
                // Produce from cluster1 for this test
                int nr1 = 0;
                int nr2 = 0;
                int nR3 = 0;
                // Produce a message that isn't replicated
                producer1.produce(1, MessageBuilder.create().disableReplication());
                // Produce a message not replicated to r2
                producer1.produce(1, MessageBuilder.create().setReplicationClusters(Lists.newArrayList("r1", "r3")));
                // Produce a default replicated message
                producer1.produce(1);
                consumer1.receive(3);
                consumer2.receive(1);
                if (!consumer2.drained()) {
                    throw new Exception("consumer2 - unexpected message in queue");
                }
                consumer3.receive(2);
                if (!consumer3.drained()) {
                    throw new Exception("consumer3 - unexpected message in queue");
                }
                producer1.close();
                producer2.close();
                producer3.close();
                consumer1.close();
                consumer2.close();
                consumer3.close();
                return null;
            }
        }));
    }
    for (Future<Void> result : results) {
        try {
            result.get();
        } catch (Exception e) {
            log.error("exception in getting future result ", e);
            fail(String.format("replication test failed with %s exception", e.getMessage()));
        }
    }
}

Example 35 with DestinationName

use of com.yahoo.pulsar.common.naming.DestinationName in project pulsar by yahoo.

the class ReplicatorTest method testResumptionAfterBacklogRelaxed.

/**
     * Issue #199
     *
     * It verifies that: if the remote cluster reaches backlog quota limit, replicator temporarily stops and once the
     * backlog drains it should resume replication.
     *
     * @throws Exception
     */
@Test(enabled = true, priority = -1)
public void testResumptionAfterBacklogRelaxed() throws Exception {
    List<RetentionPolicy> policies = Lists.newArrayList();
    policies.add(RetentionPolicy.producer_exception);
    policies.add(RetentionPolicy.producer_request_hold);
    for (RetentionPolicy policy : policies) {
        DestinationName dest = DestinationName.get(String.format("persistent://pulsar/global/ns1/%s", policy));
        // Producer on r1
        MessageProducer producer1 = new MessageProducer(url1, dest);
        // Consumer on r1
        MessageConsumer consumer1 = new MessageConsumer(url1, dest);
        // Consumer on r2
        MessageConsumer consumer2 = new MessageConsumer(url2, dest);
        // Replicator for r1 -> r2
        PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
        PersistentReplicator replicator = topic.getPersistentReplicator("r2");
        // Restrict backlog quota limit to 1
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1, policy));
        // Produce a message to r1, then it will be replicated to r2 and fulfill the backlog.
        producer1.produce(1);
        consumer1.receive(1);
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        // Produce 9 messages to r1, then it will be pended because of the backlog limit excess
        producer1.produce(9);
        consumer1.receive(9);
        Thread.sleep(1000L);
        assertEquals(replicator.getStats().replicationBacklog, 9);
        // Relax backlog quota limit to 1G
        admin1.namespaces().setBacklogQuota("pulsar/global/ns1", new BacklogQuota(1024 * 1024 * 1024, policy));
        Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
        // The messages should be replicated to r2
        assertEquals(replicator.getStats().replicationBacklog, 0);
        consumer2.receive(1);
        consumer2.receive(9);
        if (!consumer2.drained()) {
            throw new Exception("consumer2 - unexpected message in queue");
        }
        producer1.close();
        consumer1.close();
        consumer2.close();
    }
}