Examples with Consumer org.apache.pulsar.client.api.Consumer used on opensource projects

Example 16 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessagesMultipleIteration.

/**
 * Verify: iteration of a. message receive w/o acking b. stop receiving msg c. ack msgs d. started receiving msgs
 *
 * 1. Produce total X (1500) messages 2. Consumer consumes messages without acking until stop receiving from broker
 * due to reaching ack-threshold (500) 3. Consumer acks messages after stop getting messages 4. Consumer again tries
 * to consume messages 5. Consumer should be able to complete consuming all 1500 messages in 3 iteration (1500/500)
 *
 * @throws Exception
 */
@Test
public void testConsumerBlockingWithUnAckedMessagesMultipleIteration() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 500;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 1500;
        // receiver consumes messages in iteration after acknowledging broker
        final int totalReceiveIteration = totalProducedMsgs / unAckedMessagesBufferSize;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        int totalReceivedMessages = 0;
        // (2) Receive Messages
        for (int j = 0; j < totalReceiveIteration; j++) {
            Message msg = null;
            List<Message> messages = Lists.newArrayList();
            for (int i = 0; i < totalProducedMsgs; i++) {
                msg = consumer.receive(1, TimeUnit.SECONDS);
                if (msg != null) {
                    messages.add(msg);
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
            // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
            assertEquals(messages.size(), unAckedMessagesBufferSize);
            // start acknowledging messages
            messages.forEach(m -> {
                try {
                    consumer.acknowledge(m);
                } catch (PulsarClientException e) {
                    fail("ack failed", e);
                }
            });
            totalReceivedMessages += messages.size();
        }
        // total received-messages should match to produced messages
        assertEquals(totalReceivedMessages, totalProducedMsgs);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 17 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBroker.

/**
 * Verifies: Broker side throttling:
 *
 * <pre>
 * 1. concurrent_consumer_creation > maxConcurrentLookupRequest at broker
 * 2. few of the consumer creation must fail with TooManyLookupRequestException.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBroker() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS).ioThreads(20).connectionsPerBroker(20).build();
    int newPermits = 1;
    admin.brokers().updateDynamicConfiguration("maxConcurrentLookupRequest", Integer.toString(newPermits));
    // wait config to be updated
    for (int i = 0; i < 5; i++) {
        if (pulsar.getConfiguration().getMaxConcurrentLookupRequest() != newPermits) {
            Thread.sleep(100 + (i * 10));
        } else {
            break;
        }
    }
    List<Consumer<byte[]>> successfulConsumers = Collections.synchronizedList(Lists.newArrayList());
    ExecutorService executor = Executors.newFixedThreadPool(10);
    final int totalConsumers = 20;
    CountDownLatch latch = new CountDownLatch(totalConsumers);
    for (int i = 0; i < totalConsumers; i++) {
        executor.execute(() -> {
            try {
                successfulConsumers.add(pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscriptionType(SubscriptionType.Shared).subscribe());
            } catch (PulsarClientException.TooManyRequestsException e) {
            // ok
            } catch (Exception e) {
                fail("it shouldn't failed");
            }
            latch.countDown();
        });
    }
    latch.await();
    for (Consumer<?> c : successfulConsumers) {
        if (c != null) {
            c.close();
        }
    }
    pulsarClient.close();
    executor.shutdown();
    assertNotEquals(successfulConsumers.size(), totalConsumers);
}

Example 18 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class TopicTerminationTest method testSimpleTerminationMessageListener.

@Test(timeOut = 20000)
public void testSimpleTerminationMessageListener() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    CountDownLatch latch = new CountDownLatch(1);
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").messageListener(new MessageListener<byte[]>() {

        @Override
        public void received(Consumer<byte[]> consumer, Message<byte[]> msg) {
        // do nothing
        }

        @Override
        public void reachedEndOfTopic(Consumer<byte[]> consumer) {
            latch.countDown();
            assertTrue(consumer.hasReachedEndOfTopic());
        }
    }).subscribe();
    /* MessageId msgId1 = */
    producer.send("test-msg-1".getBytes());
    /* MessageId msgId2 = */
    producer.send("test-msg-2".getBytes());
    MessageId msgId3 = producer.send("test-msg-3".getBytes());
    consumer.acknowledgeCumulative(msgId3);
    Thread.sleep(100);
    assertFalse(consumer.hasReachedEndOfTopic());
    MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);
    assertTrue(latch.await(3, TimeUnit.SECONDS));
    assertTrue(consumer.hasReachedEndOfTopic());
}

Example 19 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method setup.

@BeforeMethod
public void setup() throws Exception {
    mockProducers.clear();
    mockConsumers.clear();
    fnConfig = FunctionConfig.newBuilder().setAutoAck(true).setClassName(TestFunction.class.getName()).addInputs("test-src-topic").setName("test-function").setOutput("test-output-topic").setProcessingGuarantees(ProcessingGuarantees.ATLEAST_ONCE).setTenant("test-tenant").setNamespace("test-namespace").build();
    config = new InstanceConfig();
    config.setFunctionId("test-function-id");
    config.setFunctionVersion("v1");
    config.setInstanceId("test-instance-id");
    config.setMaxBufferedTuples(1000);
    config.setFunctionConfig(fnConfig);
    mockClient = mock(PulsarClientImpl.class);
    // mock FunctionCacheManager
    fnCache = mock(FunctionCacheManager.class);
    doNothing().when(fnCache).registerFunctionInstance(anyString(), anyString(), anyList(), anyList());
    doNothing().when(fnCache).unregisterFunctionInstance(anyString(), anyString());
    ClassLoader clsLoader = JavaInstanceRunnableTest.class.getClassLoader();
    when(fnCache.getClassLoader(anyString())).thenReturn(clsLoader);
    // mock producer & consumer
    when(mockClient.createProducer(anyString(), any(ProducerConfiguration.class))).thenAnswer(invocationOnMock -> {
        String topic = invocationOnMock.getArgumentAt(0, String.class);
        ProducerConfiguration conf = invocationOnMock.getArgumentAt(1, ProducerConfiguration.class);
        String producerName = conf.getProducerName();
        Pair<String, String> pair = Pair.of(topic, producerName);
        ProducerInstance producerInstance = mockProducers.get(pair);
        if (null == producerInstance) {
            Producer producer = mock(Producer.class);
            LinkedBlockingQueue<Message> msgQueue = new LinkedBlockingQueue<>();
            final ProducerInstance instance = new ProducerInstance(producer, msgQueue);
            producerInstance = instance;
            when(producer.getProducerName()).thenReturn(producerName);
            when(producer.getTopic()).thenReturn(topic);
            when(producer.sendAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("producer send message {}", msg);
                CompletableFuture<MessageId> future = new CompletableFuture<>();
                instance.addSendFuture(future);
                msgQueue.put(msg);
                return future;
            });
            when(producer.closeAsync()).thenReturn(FutureUtils.Void());
            mockProducers.put(pair, producerInstance);
        }
        return producerInstance.getProducer();
    });
    when(mockClient.subscribe(anyString(), anyString(), any(ConsumerConfiguration.class))).thenAnswer(invocationOnMock -> {
        String topic = invocationOnMock.getArgumentAt(0, String.class);
        String subscription = invocationOnMock.getArgumentAt(1, String.class);
        ConsumerConfiguration conf = invocationOnMock.getArgumentAt(2, ConsumerConfiguration.class);
        Pair<String, String> pair = Pair.of(topic, subscription);
        ConsumerInstance consumerInstance = mockConsumers.get(pair);
        if (null == consumerInstance) {
            Consumer consumer = mock(Consumer.class);
            ConsumerInstance instance = new ConsumerInstance(consumer, conf);
            consumerInstance = instance;
            when(consumer.getTopic()).thenReturn(topic);
            when(consumer.getSubscription()).thenReturn(subscription);
            when(consumer.acknowledgeAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("Ack message {} : message id = {}", msg, msg.getMessageId());
                instance.removeMessage(msg.getMessageId());
                return FutureUtils.Void();
            });
            when(consumer.acknowledgeCumulativeAsync(any(Message.class))).thenAnswer(invocationOnMock1 -> {
                Message msg = invocationOnMock1.getArgumentAt(0, Message.class);
                log.info("Ack message cumulatively message id = {}", msg, msg.getMessageId());
                instance.removeMessagesBefore(msg.getMessageId());
                return FutureUtils.Void();
            });
            when(consumer.closeAsync()).thenAnswer(invocationOnMock1 -> {
                mockConsumers.remove(pair, instance);
                return FutureUtils.Void();
            });
            doAnswer(invocationOnMock1 -> {
                mockConsumers.remove(pair, instance);
                return null;
            }).when(consumer).close();
            mockConsumers.put(pair, consumerInstance);
        }
        return consumerInstance.getConsumer();
    });
    // 
    // Mock State Store
    // 
    StorageClientBuilder mockBuilder = mock(StorageClientBuilder.class);
    when(mockBuilder.withNamespace(anyString())).thenReturn(mockBuilder);
    when(mockBuilder.withSettings(any(StorageClientSettings.class))).thenReturn(mockBuilder);
    this.mockStorageClient = mock(StorageClient.class);
    when(mockBuilder.build()).thenReturn(mockStorageClient);
    StorageAdminClient adminClient = mock(StorageAdminClient.class);
    when(mockBuilder.buildAdmin()).thenReturn(adminClient);
    PowerMockito.mockStatic(StorageClientBuilder.class);
    PowerMockito.when(StorageClientBuilder.newBuilder()).thenReturn(mockBuilder);
    when(adminClient.getStream(anyString(), anyString())).thenReturn(FutureUtils.value(StreamProperties.newBuilder().build()));
    mockTable = mock(Table.class);
    when(mockStorageClient.openTable(anyString())).thenReturn(FutureUtils.value(mockTable));
    // 
    // Mock Function Stats
    // 
    mockFunctionStats = spy(new FunctionStats());
    PowerMockito.whenNew(FunctionStats.class).withNoArguments().thenReturn(mockFunctionStats);
    // Mock message builder
    PowerMockito.mockStatic(MessageBuilder.class);
    PowerMockito.when(MessageBuilder.create()).thenAnswer(invocationOnMock -> {
        Message msg = mock(Message.class);
        MessageBuilder builder = mock(MessageBuilder.class);
        when(builder.setContent(any(byte[].class))).thenAnswer(invocationOnMock1 -> {
            byte[] content = invocationOnMock1.getArgumentAt(0, byte[].class);
            when(msg.getData()).thenReturn(content);
            return builder;
        });
        when(builder.setSequenceId(anyLong())).thenAnswer(invocationOnMock1 -> {
            long seqId = invocationOnMock1.getArgumentAt(0, long.class);
            when(msg.getSequenceId()).thenReturn(seqId);
            return builder;
        });
        when(builder.setProperty(anyString(), anyString())).thenAnswer(invocationOnMock1 -> {
            String key = invocationOnMock1.getArgumentAt(0, String.class);
            String value = invocationOnMock1.getArgumentAt(1, String.class);
            when(msg.getProperty(eq(key))).thenReturn(value);
            return builder;
        });
        when(builder.build()).thenReturn(msg);
        return builder;
    });
}

Example 20 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class JavaInstanceRunnableProcessTest method testSetupJavaInstance.

/**
 * Test the basic run logic of instance.
 */
@Test
public void testSetupJavaInstance() throws Exception {
    JavaInstanceRunnable runnable = new JavaInstanceRunnable(config, fnCache, "test-jar-file", mockClient, TEST_STORAGE_SERVICE_URL);
    runnable.setupJavaInstance();
    // verify
    // 1. verify jar is loaded
    verify(fnCache, times(1)).registerFunctionInstance(eq(config.getFunctionId()), eq(config.getInstanceId()), eq(Arrays.asList("test-jar-file")), eq(Collections.emptyList()));
    verify(fnCache, times(1)).getClassLoader(eq(config.getFunctionId()));
    // 2. verify serde is setup
    for (String inputTopic : fnConfig.getInputsList()) {
        assertTrue(runnable.getInputSerDe().containsKey(inputTopic));
        assertTrue(runnable.getInputSerDe().get(inputTopic) instanceof DefaultSerDe);
        DefaultSerDe serDe = (DefaultSerDe) runnable.getInputSerDe().get(inputTopic);
        assertEquals(String.class, Whitebox.getInternalState(serDe, "type"));
    }
    // 3. verify producers and consumers are setup
    Producers producers = runnable.getOutputProducer();
    assertTrue(producers instanceof SimpleOneOuputTopicProducers);
    assertSame(mockProducers.get(Pair.of(fnConfig.getOutput(), null)).getProducer(), ((SimpleOneOuputTopicProducers) producers).getProducer());
    assertEquals(mockConsumers.size(), runnable.getInputConsumers().size());
    for (Map.Entry<String, Consumer> consumerEntry : runnable.getInputConsumers().entrySet()) {
        String topic = consumerEntry.getKey();
        Consumer mockConsumer = mockConsumers.get(Pair.of(topic, FunctionConfigUtils.getFullyQualifiedName(fnConfig))).getConsumer();
        assertSame(mockConsumer, consumerEntry.getValue());
    }
    // 4. verify state table
    assertSame(mockStorageClient, runnable.getStorageClient());
    assertSame(mockTable, runnable.getStateTable());
    runnable.close();
    // verify close
    for (ConsumerInstance consumer : mockConsumers.values()) {
        verify(consumer.getConsumer(), times(1)).close();
    }
    assertTrue(runnable.getInputConsumers().isEmpty());
    for (ProducerInstance producer : mockProducers.values()) {
        verify(producer.getProducer(), times(1)).close();
    }
    verify(mockTable, times(1)).close();
    verify(mockStorageClient, times(1)).close();
    // function is unregistered
    verify(fnCache, times(1)).unregisterFunctionInstance(eq(config.getFunctionId()), eq(config.getInstanceId()));
}