Examples with PulsarClient org.apache.pulsar.client.api.PulsarClient used on opensource projects

Example 61 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class ReplicatorTest method testConfigChange.

@Test(enabled = true, timeOut = 30000)
public void testConfigChange() throws Exception {
    log.info("--- Starting ReplicatorTest::testConfigChange ---");
    // This test is to verify that the config change on global namespace is successfully applied in broker during
    // runtime.
    // Run a set of producer tasks to create the topics
    List<Future<Void>> results = Lists.newArrayList();
    for (int i = 0; i < 10; i++) {
        final TopicName dest = TopicName.get(String.format("persistent://pulsar/global/ns/topic-%d", i));
        results.add(executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                MessageProducer producer = new MessageProducer(url1, dest);
                log.info("--- Starting producer --- " + url1);
                MessageConsumer consumer = new MessageConsumer(url1, dest);
                log.info("--- Starting Consumer --- " + url1);
                producer.produce(2);
                consumer.receive(2);
                producer.close();
                consumer.close();
                return null;
            }
        }));
    }
    for (Future<Void> result : results) {
        try {
            result.get();
        } catch (Exception e) {
            log.error("exception in getting future result ", e);
            fail(String.format("replication test failed with %s exception", e.getMessage()));
        }
    }
    Thread.sleep(1000L);
    // Make sure that the internal replicators map contains remote cluster info
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients1 = ns1.getReplicationClients();
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients2 = ns2.getReplicationClients();
    ConcurrentOpenHashMap<String, PulsarClient> replicationClients3 = ns3.getReplicationClients();
    Assert.assertNotNull(replicationClients1.get("r2"));
    Assert.assertNotNull(replicationClients1.get("r3"));
    Assert.assertNotNull(replicationClients2.get("r1"));
    Assert.assertNotNull(replicationClients2.get("r3"));
    Assert.assertNotNull(replicationClients3.get("r1"));
    Assert.assertNotNull(replicationClients3.get("r2"));
    // Case 1: Update the global namespace replication configuration to only contains the local cluster itself
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1"));
    // Wait for config changes to be updated.
    Thread.sleep(1000L);
    // Make sure that the internal replicators map still contains remote cluster info
    Assert.assertNotNull(replicationClients1.get("r2"));
    Assert.assertNotNull(replicationClients1.get("r3"));
    Assert.assertNotNull(replicationClients2.get("r1"));
    Assert.assertNotNull(replicationClients2.get("r3"));
    Assert.assertNotNull(replicationClients3.get("r1"));
    Assert.assertNotNull(replicationClients3.get("r2"));
    // Case 2: Update the configuration back
    admin1.namespaces().setNamespaceReplicationClusters("pulsar/global/ns", Lists.newArrayList("r1", "r2", "r3"));
    // Wait for config changes to be updated.
    Thread.sleep(1000L);
    // Make sure that the internal replicators map still contains remote cluster info
    Assert.assertNotNull(replicationClients1.get("r2"));
    Assert.assertNotNull(replicationClients1.get("r3"));
    Assert.assertNotNull(replicationClients2.get("r1"));
    Assert.assertNotNull(replicationClients2.get("r3"));
    Assert.assertNotNull(replicationClients3.get("r1"));
    Assert.assertNotNull(replicationClients3.get("r2"));
// Case 3: TODO: Once automatic cleanup is implemented, add tests case to verify auto removal of clusters
}

Example 62 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class ReplicatorTest method verifyChecksumAfterReplication.

@Test(timeOut = 30000)
public void verifyChecksumAfterReplication() throws Exception {
    final String topicName = "persistent://pulsar/global/ns/checksumAfterReplication";
    PulsarClient c1 = PulsarClient.builder().serviceUrl(url1.toString()).build();
    Producer<byte[]> p1 = c1.newProducer().topic(topicName).create();
    PulsarClient c2 = PulsarClient.builder().serviceUrl(url2.toString()).build();
    RawReader reader2 = RawReader.create(c2, topicName, "sub").get();
    p1.send("Hello".getBytes());
    RawMessage msg = reader2.readNextAsync().get();
    ByteBuf b = msg.getHeadersAndPayload();
    assertTrue(Commands.hasChecksum(b));
    int parsedChecksum = Commands.readChecksum(b);
    int computedChecksum = Crc32cIntChecksum.computeChecksum(b);
    assertEquals(parsedChecksum, computedChecksum);
    p1.close();
    reader2.closeAsync().get();
}

Example 63 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class ReplicatorTest method testReplicatorOnPartitionedTopic.

/**
 * It verifies that broker should not start replicator for partitioned-topic (topic without -partition postfix)
 *
 * @param isPartitionedTopic
 * @throws Exception
 */
@Test(dataProvider = "partitionedTopic")
public void testReplicatorOnPartitionedTopic(boolean isPartitionedTopic) throws Exception {
    log.info("--- Starting ReplicatorTest::{} --- ", methodName);
    final String namespace = "pulsar/global/partitionedNs-" + isPartitionedTopic;
    final String persistentTopicName = "persistent://" + namespace + "/partTopic-" + isPartitionedTopic;
    final String nonPersistentTopicName = "non-persistent://" + namespace + "/partTopic-" + isPartitionedTopic;
    BrokerService brokerService = pulsar1.getBrokerService();
    admin1.namespaces().createNamespace(namespace);
    admin1.namespaces().setNamespaceReplicationClusters(namespace, Lists.newArrayList("r1", "r2", "r3"));
    if (isPartitionedTopic) {
        admin1.persistentTopics().createPartitionedTopic(persistentTopicName, 5);
        admin1.nonPersistentTopics().createPartitionedTopic(nonPersistentTopicName, 5);
    }
    // load namespace with dummy topic on ns
    PulsarClient client = PulsarClient.builder().serviceUrl(url1.toString()).build();
    client.newProducer().topic("persistent://" + namespace + "/dummyTopic").create();
    // persistent topic test
    try {
        brokerService.getTopic(persistentTopicName).get();
        if (isPartitionedTopic) {
            fail("Topic creation fails with partitioned topic as replicator init fails");
        }
    } catch (Exception e) {
        if (!isPartitionedTopic) {
            fail("Topic creation should not fail without any partitioned topic");
        }
        assertTrue(e.getCause() instanceof NamingException);
    }
    // non-persistent topic test
    try {
        brokerService.getTopic(nonPersistentTopicName).get();
        if (isPartitionedTopic) {
            fail("Topic creation fails with partitioned topic as replicator init fails");
        }
    } catch (Exception e) {
        if (!isPartitionedTopic) {
            fail("Topic creation should not fail without any partitioned topic");
        }
        assertTrue(e.getCause() instanceof NamingException);
    }
}

Example 64 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testProducerQueueFullNonBlocking.

@Test
public void testProducerQueueFullNonBlocking() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyzx";
    final int messages = 10;
    // 1. Producer connect
    PulsarClient client = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
    ProducerImpl<byte[]> producer = (ProducerImpl<byte[]>) client.newProducer().topic(topicName).maxPendingMessages(messages).blockIfQueueFull(false).sendTimeout(1, TimeUnit.SECONDS).create();
    // 2. Stop broker
    cleanup();
    // 2. producer publish messages
    long startTime = System.nanoTime();
    for (int i = 0; i < messages; i++) {
        // Should never block
        producer.sendAsync("msg".getBytes());
    }
    // Verify thread was not blocked
    long delayNs = System.nanoTime() - startTime;
    assertTrue(delayNs < TimeUnit.SECONDS.toNanos(1));
    assertEquals(producer.getPendingQueueSize(), messages);
    // Next send operation must fail and not block
    startTime = System.nanoTime();
    try {
        producer.send("msg".getBytes());
        fail("Send should have failed");
    } catch (PulsarClientException.ProducerQueueIsFullError e) {
    // Expected
    }
    delayNs = System.nanoTime() - startTime;
    assertTrue(delayNs < TimeUnit.SECONDS.toNanos(1));
    assertEquals(producer.getPendingQueueSize(), messages);
    // 4. producer disconnect
    producer.close();
    client.close();
    // 5. Restart broker
    setup();
}

Example 65 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class PersistentTopicTest method testAtomicReplicationRemoval.

/**
 * {@link NonPersistentReplicator.removeReplicator} doesn't remove replicator in atomic way and does in multiple step:
 * 1. disconnect replicator producer
 * <p>
 * 2. close cursor
 * <p>
 * 3. remove from replicator-list.
 * <p>
 *
 * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer.
 *
 * @throws Exception
 */
@Test
public void testAtomicReplicationRemoval() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns-abc/successTopic";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + remoteCluster;
    ConcurrentOpenHashMap<String, Replicator> replicatorMap = topic.getReplicators();
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = spy(new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService));
    replicatorMap.put(remoteReplicatorName, replicator);
    // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed
    Method removeMethod = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeMethod.setAccessible(true);
    removeMethod.invoke(topic, remoteReplicatorName);
    // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the
    // replicator
    when(pulsar.getConfigurationCache().policiesCache().get(AdminResource.path(POLICIES, TopicName.get(globalTopicName).getNamespace()))).thenReturn(Optional.of(new Policies()));
    // try to start replicator again
    topic.startReplProducers();
    // verify: replicator.startProducer is not invoked
    verify(replicator, Mockito.times(0)).startProducer();
    // step-3 : complete the callback to remove replicator from the list
    ArgumentCaptor<DeleteCursorCallback> captor = ArgumentCaptor.forClass(DeleteCursorCallback.class);
    Mockito.verify(ledgerMock).asyncDeleteCursor(anyObject(), captor.capture(), anyObject());
    DeleteCursorCallback callback = captor.getValue();
    callback.deleteCursorComplete(null);
}