Examples with Producer org.apache.pulsar.client.api.Producer used on opensource projects

Example 51 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class BrokerServiceTest method testBrokerServiceNamespaceStats.

@Test
public void testBrokerServiceNamespaceStats() throws Exception {
    final int numBundles = 4;
    final String ns1 = "prop/use/stats1";
    final String ns2 = "prop/use/stats2";
    List<String> nsList = Lists.newArrayList(ns1, ns2);
    List<Producer<byte[]>> producerList = Lists.newArrayList();
    BrokerStats brokerStatsClient = admin.brokerStats();
    for (String ns : nsList) {
        admin.namespaces().createNamespace(ns, numBundles);
        String topic1 = String.format("persistent://%s/topic1", ns);
        producerList.add(pulsarClient.newProducer().topic(topic1).create());
        String topic2 = String.format("persistent://%s/topic2", ns);
        producerList.add(pulsarClient.newProducer().topic(topic2).create());
    }
    rolloverPerIntervalStats();
    JsonObject topicStats = brokerStatsClient.getTopics();
    assertEquals(topicStats.size(), 2, topicStats.toString());
    for (String ns : nsList) {
        JsonObject nsObject = topicStats.getAsJsonObject(ns);
        List<String> topicList = admin.namespaces().getTopics(ns);
        for (String topic : topicList) {
            NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService().getBundle(TopicName.get(topic));
            JsonObject bundleObject = nsObject.getAsJsonObject(bundle.getBundleRange());
            JsonObject topicObject = bundleObject.getAsJsonObject("persistent");
            AtomicBoolean topicPresent = new AtomicBoolean();
            topicObject.entrySet().iterator().forEachRemaining(persistentTopic -> {
                if (persistentTopic.getKey().equals(topic)) {
                    topicPresent.set(true);
                }
            });
            assertTrue(topicPresent.get());
        }
    }
    for (Producer<?> producer : producerList) {
        producer.close();
    }
    for (String ns : nsList) {
        List<String> topics = admin.namespaces().getTopics(ns);
        for (String dest : topics) {
            admin.persistentTopics().delete(dest);
        }
        admin.namespaces().deleteNamespace(ns);
    }
}

Example 52 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class BrokerServiceTest method testTopicFailureShouldNotHaveDeadLock.

/**
 * Verifies brokerService should not have deadlock and successfully remove topic from topicMap on topic-failure and
 * it should not introduce deadlock while performing it.
 */
@Test(timeOut = 3000)
public void testTopicFailureShouldNotHaveDeadLock() {
    final String namespace = "prop/usw/my-ns";
    final String deadLockTestTopic = "persistent://" + namespace + "/deadLockTestTopic";
    // let this broker own this namespace bundle by creating a topic
    try {
        final String successfulTopic = "persistent://" + namespace + "/ownBundleTopic";
        Producer<byte[]> producer = pulsarClient.newProducer().topic(successfulTopic).create();
        producer.close();
    } catch (Exception e) {
        fail(e.getMessage());
    }
    ExecutorService executor = Executors.newSingleThreadExecutor();
    BrokerService service = spy(pulsar.getBrokerService());
    // create topic will fail to get managedLedgerConfig
    CompletableFuture<ManagedLedgerConfig> failedManagedLedgerConfig = new CompletableFuture<>();
    failedManagedLedgerConfig.completeExceptionally(new NullPointerException("failed to peristent policy"));
    doReturn(failedManagedLedgerConfig).when(service).getManagedLedgerConfig(anyObject());
    CompletableFuture<Void> topicCreation = new CompletableFuture<Void>();
    // create topic async and wait on the future completion
    executor.submit(() -> {
        service.getTopic(deadLockTestTopic).thenAccept(topic -> topicCreation.complete(null)).exceptionally(e -> {
            topicCreation.completeExceptionally(e.getCause());
            return null;
        });
    });
    // future-result should be completed with exception
    try {
        topicCreation.get(1, TimeUnit.SECONDS);
    } catch (TimeoutException | InterruptedException e) {
        fail("there is a dead-lock and it should have been prevented");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof NullPointerException);
    } finally {
        executor.shutdownNow();
    }
}

Example 53 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class BrokerServiceTest method testLedgerOpenFailureShouldNotHaveDeadLock.

@Test
public void testLedgerOpenFailureShouldNotHaveDeadLock() throws Exception {
    final String namespace = "prop/usw/my-ns";
    final String deadLockTestTopic = "persistent://" + namespace + "/deadLockTestTopic";
    // let this broker own this namespace bundle by creating a topic
    try {
        final String successfulTopic = "persistent://" + namespace + "/ownBundleTopic";
        Producer<byte[]> producer = pulsarClient.newProducer().topic(successfulTopic).create();
        producer.close();
    } catch (Exception e) {
        fail(e.getMessage());
    }
    ExecutorService executor = Executors.newSingleThreadExecutor();
    BrokerService service = spy(pulsar.getBrokerService());
    // create topic will fail to get managedLedgerConfig
    CompletableFuture<ManagedLedgerConfig> failedManagedLedgerConfig = new CompletableFuture<>();
    failedManagedLedgerConfig.complete(null);
    doReturn(failedManagedLedgerConfig).when(service).getManagedLedgerConfig(anyObject());
    CompletableFuture<Void> topicCreation = new CompletableFuture<Void>();
    // fail managed-ledger future
    Field ledgerField = ManagedLedgerFactoryImpl.class.getDeclaredField("ledgers");
    ledgerField.setAccessible(true);
    @SuppressWarnings("unchecked") ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>> ledgers = (ConcurrentHashMap<String, CompletableFuture<ManagedLedgerImpl>>) ledgerField.get(pulsar.getManagedLedgerFactory());
    CompletableFuture<ManagedLedgerImpl> future = new CompletableFuture<>();
    future.completeExceptionally(new ManagedLedgerException("ledger opening failed"));
    ledgers.put(namespace + "/persistent/deadLockTestTopic", future);
    // create topic async and wait on the future completion
    executor.submit(() -> {
        service.getTopic(deadLockTestTopic).thenAccept(topic -> topicCreation.complete(null)).exceptionally(e -> {
            topicCreation.completeExceptionally(e.getCause());
            return null;
        });
    });
    // future-result should be completed with exception
    try {
        topicCreation.get(1, TimeUnit.SECONDS);
    } catch (TimeoutException | InterruptedException e) {
        fail("there is a dead-lock and it should have been prevented");
    } catch (ExecutionException e) {
        assertTrue(e.getCause() instanceof PersistenceException);
    } finally {
        executor.shutdownNow();
        ledgers.clear();
    }
}

Example 54 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class CompactionTest method testWholeBatchCompactedOut.

@Test
public void testWholeBatchCompactedOut() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";
    // subscribe before sending anything, so that we get all messages
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
    try (Producer producerNormal = pulsarClient.newProducer().topic(topic).create();
        Producer producerBatch = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producerBatch.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-1".getBytes()).build());
        producerBatch.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-2".getBytes()).build());
        producerBatch.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-3".getBytes()).build()).get();
        producerNormal.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-4".getBytes()).build()).get();
    }
    // compact the topic
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        Message message = consumer.receive();
        Assert.assertEquals(message.getKey(), "key1");
        Assert.assertEquals(new String(message.getData()), "my-message-4");
    }
}

Example 55 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class RawReaderTest method testBatchingRebatch.

@Test
public void testBatchingRebatch() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    try (Producer producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-content-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-content-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key3").setContent("my-content-3".getBytes()).build()).get();
    }
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try {
        RawMessage m1 = reader.readNextAsync().get();
        RawMessage m2 = RawBatchConverter.rebatchMessage(m1, (key, id) -> key.equals("key2")).get();
        List<ImmutablePair<MessageId, String>> idsAndKeys = RawBatchConverter.extractIdsAndKeys(m2);
        Assert.assertEquals(idsAndKeys.size(), 1);
        Assert.assertEquals(idsAndKeys.get(0).getRight(), "key2");
        m2.close();
    } finally {
        reader.closeAsync().get();
    }
}