Examples with Producer org.apache.pulsar.client.api.Producer used on opensource projects

Example 1 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class RawReaderTest method testBatchingExtractKeysAndIds.

@Test
public void testBatchingExtractKeysAndIds() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    try (Producer producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-content-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-content-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key3").setContent("my-content-3".getBytes()).build()).get();
    }
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m = reader.readNextAsync().get()) {
        List<ImmutablePair<MessageId, String>> idsAndKeys = RawBatchConverter.extractIdsAndKeys(m);
        Assert.assertEquals(idsAndKeys.size(), 3);
        // assert message ids are in correct order
        Assert.assertTrue(idsAndKeys.get(0).getLeft().compareTo(idsAndKeys.get(1).getLeft()) < 0);
        Assert.assertTrue(idsAndKeys.get(1).getLeft().compareTo(idsAndKeys.get(2).getLeft()) < 0);
        // assert keys are as expected
        Assert.assertEquals(idsAndKeys.get(0).getRight(), "key1");
        Assert.assertEquals(idsAndKeys.get(1).getRight(), "key2");
        Assert.assertEquals(idsAndKeys.get(2).getRight(), "key3");
    } finally {
        reader.closeAsync().get();
    }
}

Example 2 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testAsyncConsumer.

@Test(timeOut = testTimeout)
public void testAsyncConsumer() throws Exception {
    String key = "TopicsConsumerAsyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof TopicsConsumerImpl);
    // Asynchronously produce messages
    List<Future<MessageId>> futures = Lists.newArrayList();
    for (int i = 0; i < totalMessages / 3; i++) {
        futures.add(producer1.sendAsync((messagePredicate + "producer1-" + i).getBytes()));
        futures.add(producer2.sendAsync((messagePredicate + "producer2-" + i).getBytes()));
        futures.add(producer3.sendAsync((messagePredicate + "producer3-" + i).getBytes()));
    }
    log.info("Waiting for async publish to complete : {}", futures.size());
    for (Future<MessageId> future : futures) {
        future.get();
    }
    log.info("start async consume");
    CountDownLatch latch = new CountDownLatch(totalMessages);
    ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.execute(() -> IntStream.range(0, totalMessages).forEach(index -> consumer.receiveAsync().thenAccept(msg -> {
        assertTrue(msg instanceof TopicMessageImpl);
        try {
            consumer.acknowledge(msg);
        } catch (PulsarClientException e1) {
            fail("message acknowledge failed", e1);
        }
        latch.countDown();
        log.info("receive index: {}, latch countDown: {}", index, latch.getCount());
    }).exceptionally(ex -> {
        log.warn("receive index: {}, failed receive message {}", index, ex.getMessage());
        ex.printStackTrace();
        return null;
    })));
    latch.await();
    log.info("success latch wait");
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}

Example 3 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class CompactionTest method testFirstMessageRetained.

@Test
public void testFirstMessageRetained() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";
    // subscribe before sending anything, so that we get all messages
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
    try (Producer producer = pulsarClient.createProducer(topic)) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-3".getBytes()).build()).get();
    }
    // Read messages before compaction to get ids
    List<Message> messages = new ArrayList<>();
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        messages.add(consumer.receive());
        messages.add(consumer.receive());
        messages.add(consumer.receive());
    }
    // compact the topic
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();
    // Check that messages after compaction have same ids
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        Message message1 = consumer.receive();
        Assert.assertEquals(message1.getKey(), "key1");
        Assert.assertEquals(new String(message1.getData()), "my-message-1");
        Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId());
        Message message2 = consumer.receive();
        Assert.assertEquals(message2.getKey(), "key2");
        Assert.assertEquals(new String(message2.getData()), "my-message-3");
        Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId());
    }
}

Example 4 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class CompactionTest method testBatchMessageIdsDontChange.

@Test
public void testBatchMessageIdsDontChange() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";
    // subscribe before sending anything, so that we get all messages
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
    try (Producer producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-3".getBytes()).build()).get();
    }
    // Read messages before compaction to get ids
    List<Message> messages = new ArrayList<>();
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        messages.add(consumer.receive());
        messages.add(consumer.receive());
        messages.add(consumer.receive());
    }
    // Ensure all messages are in same batch
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getLedgerId(), ((BatchMessageIdImpl) messages.get(1).getMessageId()).getLedgerId());
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getLedgerId(), ((BatchMessageIdImpl) messages.get(2).getMessageId()).getLedgerId());
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getEntryId(), ((BatchMessageIdImpl) messages.get(1).getMessageId()).getEntryId());
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getEntryId(), ((BatchMessageIdImpl) messages.get(2).getMessageId()).getEntryId());
    // compact the topic
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();
    // Check that messages after compaction have same ids
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        Message message1 = consumer.receive();
        Assert.assertEquals(message1.getKey(), "key1");
        Assert.assertEquals(new String(message1.getData()), "my-message-1");
        Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId());
        Message message2 = consumer.receive();
        Assert.assertEquals(message2.getKey(), "key2");
        Assert.assertEquals(new String(message2.getData()), "my-message-3");
        Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId());
    }
}

Example 5 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class SchedulerManagerTest method testUpdate.

@Test
public void testUpdate() throws Exception {
    List<Function.FunctionMetaData> functionMetaDataList = new LinkedList<>();
    long version = 5;
    Function.FunctionMetaData function1 = Function.FunctionMetaData.newBuilder().setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1")).setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-1").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(1)).setVersion(version).build();
    Function.FunctionMetaData function2 = Function.FunctionMetaData.newBuilder().setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar1")).setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    // set assignments
    Function.Assignment assignment1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function1).setInstanceId(0).build()).build();
    Map<String, Map<String, Function.Assignment>> currentAssignments = new HashMap<>();
    Map<String, Function.Assignment> assignmentEntry1 = new HashMap<>();
    assignmentEntry1.put(Utils.getFullyQualifiedInstanceId(assignment1.getInstance()), assignment1);
    currentAssignments.put("worker-1", assignmentEntry1);
    doReturn(currentAssignments).when(functionRuntimeManager).getCurrentAssignments();
    // set version
    doReturn(version).when(functionRuntimeManager).getCurrentAssignmentVersion();
    // single node
    List<MembershipManager.WorkerInfo> workerInfoList = new LinkedList<>();
    workerInfoList.add(MembershipManager.WorkerInfo.of("worker-1", "workerHostname-1", 5000));
    doReturn(workerInfoList).when(membershipManager).getCurrentMembership();
    // i am leader
    doReturn(true).when(membershipManager).isLeader();
    callSchedule();
    List<Invocation> invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 1);
    byte[] send = (byte[]) invocations.get(0).getRawArguments()[0];
    Request.AssignmentsUpdate assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    log.info("assignmentsUpdate: {}", assignmentsUpdate);
    Function.Assignment assignment2_1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(0).build()).build();
    Function.Assignment assignment2_2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(1).build()).build();
    Function.Assignment assignment2_3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2).setInstanceId(2).build()).build();
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1).addAssignments(assignment1).addAssignments(assignment2_1).addAssignments(assignment2_2).addAssignments(assignment2_3).build());
    // scale down
    PulsarClient pulsarClient = mock(PulsarClient.class);
    doReturn(producer).when(pulsarClient).createProducer(any(), any());
    Function.FunctionMetaData function2Updated = Function.FunctionMetaData.newBuilder().setPackageLocation(Function.PackageLocationMetaData.newBuilder().setPackagePath("/foo/bar2")).setFunctionConfig(Function.FunctionConfig.newBuilder().setName("func-2").setNamespace("namespace-1").setTenant("tenant-1").setParallelism(3)).setVersion(version).build();
    functionMetaDataList = new LinkedList<>();
    functionMetaDataList.add(function1);
    functionMetaDataList.add(function2Updated);
    doReturn(functionMetaDataList).when(functionMetaDataManager).getAllFunctionMetaData();
    Function.Assignment assignment2Updated1 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Updated).setInstanceId(0).build()).build();
    Function.Assignment assignment2Updated2 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Updated).setInstanceId(1).build()).build();
    Function.Assignment assignment2Updated3 = Function.Assignment.newBuilder().setWorkerId("worker-1").setInstance(Function.Instance.newBuilder().setFunctionMetaData(function2Updated).setInstanceId(2).build()).build();
    callSchedule();
    invocations = getMethodInvocationDetails(producer, Producer.class.getMethod("sendAsync", Object.class));
    Assert.assertEquals(invocations.size(), 2);
    send = (byte[]) invocations.get(1).getRawArguments()[0];
    assignmentsUpdate = Request.AssignmentsUpdate.parseFrom(send);
    Assert.assertEquals(assignmentsUpdate, Request.AssignmentsUpdate.newBuilder().setVersion(version + 1 + 1).addAssignments(assignment1).addAssignments(assignment2Updated1).addAssignments(assignment2Updated2).addAssignments(assignment2Updated3).build());
}