Examples with Producer org.apache.pulsar.client.api.Producer used on opensource projects

Example 26 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessagesMultipleIteration.

/**
 * Verify: iteration of a. message receive w/o acking b. stop receiving msg c. ack msgs d. started receiving msgs
 *
 * 1. Produce total X (1500) messages 2. Consumer consumes messages without acking until stop receiving from broker
 * due to reaching ack-threshold (500) 3. Consumer acks messages after stop getting messages 4. Consumer again tries
 * to consume messages 5. Consumer should be able to complete consuming all 1500 messages in 3 iteration (1500/500)
 *
 * @throws Exception
 */
@Test
public void testConsumerBlockingWithUnAckedMessagesMultipleIteration() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 500;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 1500;
        // receiver consumes messages in iteration after acknowledging broker
        final int totalReceiveIteration = totalProducedMsgs / unAckedMessagesBufferSize;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        int totalReceivedMessages = 0;
        // (2) Receive Messages
        for (int j = 0; j < totalReceiveIteration; j++) {
            Message msg = null;
            List<Message> messages = Lists.newArrayList();
            for (int i = 0; i < totalProducedMsgs; i++) {
                msg = consumer.receive(1, TimeUnit.SECONDS);
                if (msg != null) {
                    messages.add(msg);
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
            // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
            assertEquals(messages.size(), unAckedMessagesBufferSize);
            // start acknowledging messages
            messages.forEach(m -> {
                try {
                    consumer.acknowledge(m);
                } catch (PulsarClientException e) {
                    fail("ack failed", e);
                }
            });
            totalReceivedMessages += messages.size();
        }
        // total received-messages should match to produced messages
        assertEquals(totalReceivedMessages, totalProducedMsgs);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 27 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class MultiConsumersOneOutputTopicProducersTest method setup.

@BeforeMethod
public void setup() throws Exception {
    this.mockClient = mock(PulsarClient.class);
    when(mockClient.createProducer(anyString(), any(ProducerConfiguration.class))).thenAnswer(invocationOnMock -> {
        ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class);
        String producerName = conf.getProducerName();
        synchronized (mockProducers) {
            Producer producer = mockProducers.get(producerName);
            if (null == producer) {
                producer = createMockProducer(producerName);
                mockProducers.put(producerName, producer);
            }
            return producer;
        }
    });
    producers = new MultiConsumersOneOuputTopicProducers(mockClient, TEST_OUTPUT_TOPIC);
    producers.initialize();
}

Example 28 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class MultiConsumersOneOutputTopicProducersTest method testGetCloseProducer.

@Test
public void testGetCloseProducer() throws Exception {
    String srcTopic = "test-src-topic";
    int ptnIdx = 1234;
    Producer producer = producers.getProducer(srcTopic, ptnIdx);
    String producerName = makeProducerName(srcTopic, ptnIdx);
    assertSame(mockProducers.get(producerName), producer);
    verify(mockClient, times(1)).createProducer(eq(TEST_OUTPUT_TOPIC), any(ProducerConfiguration.class));
    assertTrue(producers.getProducers().containsKey(srcTopic));
    assertEquals(1, producers.getProducers().get(srcTopic).size());
    assertTrue(producers.getProducers().get(srcTopic).containsKey(ptnIdx));
    // second get will not create a new producer
    assertSame(mockProducers.get(producerName), producer);
    verify(mockClient, times(1)).createProducer(eq(TEST_OUTPUT_TOPIC), any(ProducerConfiguration.class));
    assertTrue(producers.getProducers().containsKey(srcTopic));
    assertEquals(1, producers.getProducers().get(srcTopic).size());
    assertTrue(producers.getProducers().get(srcTopic).containsKey(ptnIdx));
    // close
    producers.closeProducer(srcTopic, ptnIdx);
    verify(producer, times(1)).closeAsync();
    assertFalse(producers.getProducers().containsKey(srcTopic));
}

Example 29 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class MultiConsumersOneOuputTopicProducers method getProducer.

@Override
public synchronized Producer getProducer(String srcTopicName, int srcTopicPartition) throws PulsarClientException {
    IntObjectMap<Producer> producerMap = producers.get(srcTopicName);
    if (null == producerMap) {
        producerMap = new IntObjectHashMap<>();
        producers.put(srcTopicName, producerMap);
    }
    Producer producer = producerMap.get(srcTopicPartition);
    if (null == producer) {
        producer = createProducer(outputTopic, makeProducerName(srcTopicName, srcTopicPartition));
        producerMap.put(srcTopicPartition, producer);
    }
    return producer;
}

Example 30 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method setup.

@BeforeMethod
public void setup() throws Exception {
    mockProducers.clear();
    mockConsumers.clear();
    fnConfig = FunctionConfig.newBuilder().setAutoAck(true).setClassName(TestFunction.class.getName()).addInputs("test-src-topic").setName("test-function").setOutput("test-output-topic").setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setTenant("test-tenant").setNamespace("test-namespace").build();
    config = new InstanceConfig();
    config.setFunctionId("test-function-id");
    config.setFunctionVersion("v1");
    config.setInstanceId("test-instance-id");
    config.setMaxBufferedTuples(1000);
    config.setFunctionConfig(fnConfig);
    mockClient = mock(PulsarClientImpl.class);
    // mock FunctionCacheManager
    fnCache = mock(FunctionCacheManager.class);
    doNothing().when(fnCache).registerFunctionInstance(anyString(), anyString(), anyList(), anyList());
    doNothing().when(fnCache).unregisterFunctionInstance(anyString(), anyString());
    ClassLoader clsLoader = JavaInstanceRunnableTest.class.getClassLoader();
    when(fnCache.getClassLoader(anyString())).thenReturn(clsLoader);
    // mock producer & consumer
    when(mockClient.createProducer(anyString(), any(ProducerConfiguration.class))).thenAnswer(invocationOnMock -> {
        String topic = invocationOnMock.getArgumentAt(0, String.class);
        ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class);
        String producerName = conf.getProducerName();
        Pair<String, String> pair = Pair.of(topic, producerName);
        ProducerInstance producerInstance = mockProducers.get(pair);
        if (null == producerInstance) {
            Producer producer = mock(Producer.class);
            LinkedBlockingQueue<Message> msgQueue = new LinkedBlockingQueue<>();
            final ProducerInstance instance = new ProducerInstance(producer, msgQueue);
            producerInstance = instance;
            when(producer.getProducerName()).thenReturn(producerName);
            when(producer.getTopic()).thenReturn(topic);
            when(producer.sendAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("producer send message {}", msg);
                CompletableFuture<MessageId> future = new CompletableFuture<>();
                instance.addSendFuture(future);
                msgQueue.put(msg);
                return future;
            });
            when(producer.closeAsync()).thenReturn(FutureUtils.Void());
            mockProducers.put(pair, producerInstance);
        }
        return producerInstance.getProducer();
    });
    when(mockClient.subscribe(anyString(), anyString(), any(ConsumerConfiguration.class))).thenAnswer(invocationOnMock -> {
        String topic = invocationOnMock.getArgumentAt(0, String.class);
        String subscription = invocationOnMock.getArgumentAt(1, String.class);
        ConsumerConfiguration conf = invocationOnMock.getArgumentAt(2, ConsumerConfiguration.class);
        Pair<String, String> pair = Pair.of(topic, subscription);
        ConsumerInstance consumerInstance = mockConsumers.get(pair);
        if (null == consumerInstance) {
            Consumer consumer = mock(Consumer.class);
            ConsumerInstance instance = new ConsumerInstance(consumer, conf);
            consumerInstance = instance;
            when(consumer.getTopic()).thenReturn(topic);
            when(consumer.getSubscription()).thenReturn(subscription);
            when(consumer.acknowledgeAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("Ack message {} : message id = {}", msg, msg.getMessageId());
                instance.removeMessage(msg.getMessageId());
                return FutureUtils.Void();
            });
            when(consumer.acknowledgeCumulativeAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId());
                instance.removeMessagesBefore(msg.getMessageId());
                return FutureUtils.Void();
            });
            when(consumer.closeAsync()).thenAnswer(invocationOnMock1 -> {
                mockConsumers.remove(pair, instance);
                return FutureUtils.Void();
            });
            doAnswer(invocationOnMock1 -> {
                mockConsumers.remove(pair, instance);
                return null;
            }).when(consumer).close();
            mockConsumers.put(pair, consumerInstance);
        }
        return consumerInstance.getConsumer();
    });
    // 
    // Mock State Store
    // 
    StorageClientBuilder mockBuilder = mock(StorageClientBuilder.class);
    when(mockBuilder.withNamespace(anyString())).thenReturn(mockBuilder);
    when(mockBuilder.withSettings(any(StorageClientSettings.class))).thenReturn(mockBuilder);
    this.mockStorageClient = mock(StorageClient.class);
    when(mockBuilder.build()).thenReturn(mockStorageClient);
    StorageAdminClient adminClient = mock(StorageAdminClient.class);
    when(mockBuilder.buildAdmin()).thenReturn(adminClient);
    PowerMockito.mockStatic(StorageClientBuilder.class);
    PowerMockito.when(StorageClientBuilder.newBuilder()).thenReturn(mockBuilder);
    when(adminClient.getStream(anyString(), anyString())).thenReturn(FutureUtils.value(StreamProperties.newBuilder().build()));
    mockTable = mock(Table.class);
    when(mockStorageClient.openTable(anyString())).thenReturn(FutureUtils.value(mockTable));
    // 
    // Mock Function Stats
    // 
    mockFunctionStats = spy(new FunctionStats());
    PowerMockito.whenNew(FunctionStats.class).withNoArguments().thenReturn(mockFunctionStats);
    // Mock message builder
    PowerMockito.mockStatic(MessageBuilder.class);
    PowerMockito.when(MessageBuilder.create()).thenAnswer(invocationOnMock -> {
        Message msg = mock(Message.class);
        MessageBuilder builder = mock(MessageBuilder.class);
        when(builder.setContent(any(byte[].class))).thenAnswer(invocationOnMock1 -> {
            byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class);
            when(msg.getData()).thenReturn(content);
            return builder;
        });
        when(builder.setSequenceId(anyLong())).thenAnswer(invocationOnMock1 -> {
            long seqId = invocationOnMock1.getArgumentAt(0, long.class);
            when(msg.getSequenceId()).thenReturn(seqId);
            return builder;
        });
        when(builder.setProperty(anyString(), anyString())).thenAnswer(invocationOnMock1 -> {
            String key = invocationOnMock1.getArgumentAt(0, String.class);
            String value = invocationOnMock1.getArgumentAt(1, String.class);
            when(msg.getProperty(eq(key))).thenReturn(value);
            return builder;
        });
        when(builder.build()).thenReturn(msg);
        return builder;
    });
}