Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 91 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ClientErrorsTest method producerContinuousRetryAfterSendFail.

private void producerContinuousRetryAfterSendFail(String topic) throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    final AtomicInteger counter = new AtomicInteger(0);
    mockBrokerService.setHandleProducer((ctx, producer) -> {
        int i = counter.incrementAndGet();
        if (i == 1 || i == 5) {
            // succeed on 1st and 5th attempts
            ctx.writeAndFlush(Commands.newProducerSuccess(producer.getRequestId(), "default-producer"));
            return;
        }
        ctx.writeAndFlush(Commands.newError(producer.getRequestId(), ServerError.PersistenceError, "msg"));
    });
    final AtomicInteger msgCounter = new AtomicInteger(0);
    mockBrokerService.setHandleSend((ctx, send, headersAndPayload) -> {
        // fail send once, but succeed later
        if (msgCounter.incrementAndGet() == 1) {
            ctx.writeAndFlush(Commands.newSendError(0, 0, new IllegalStateException("Send Failed")));
            return;
        }
        ctx.writeAndFlush(Commands.newSendReceipt(0, 0, 1, 1));
    });
    try {
        Producer producer = client.createProducer(topic);
        producer.send("message".getBytes());
    } catch (Exception e) {
        fail("Should not fail");
    }
    mockBrokerService.resetHandleProducer();
    mockBrokerService.resetHandleSend();
    client.close();
}

Example 92 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ClientErrorsTest method testProducerReconnect.

// Run this test multiple times to reproduce race conditions on reconnection logic
@Test(invocationCount = 100)
public void testProducerReconnect() throws Exception {
    AtomicInteger numOfConnections = new AtomicInteger();
    AtomicReference<ChannelHandlerContext> channelCtx = new AtomicReference<>();
    AtomicBoolean msgSent = new AtomicBoolean();
    mockBrokerService.setHandleConnect((ctx, connect) -> {
        channelCtx.set(ctx);
        ctx.writeAndFlush(Commands.newConnected(connect));
        if (numOfConnections.incrementAndGet() == 2) {
            // close the cnx immediately when trying to conenct the 2nd time
            ctx.channel().close();
        }
    });
    mockBrokerService.setHandleProducer((ctx, produce) -> {
        ctx.writeAndFlush(Commands.newProducerSuccess(produce.getRequestId(), "default-producer"));
    });
    mockBrokerService.setHandleSend((ctx, sendCmd, headersAndPayload) -> {
        msgSent.set(true);
        ctx.writeAndFlush(Commands.newSendReceipt(0, 0, 1, 1));
    });
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    Producer producer = client.createProducer("persistent://prop/use/ns/t1");
    // close the cnx after creating the producer
    channelCtx.get().channel().close().get();
    producer.send(new byte[0]);
    assertEquals(msgSent.get(), true);
    assertTrue(numOfConnections.get() >= 3);
    mockBrokerService.resetHandleConnect();
    mockBrokerService.resetHandleProducer();
    mockBrokerService.resetHandleSend();
    client.close();
}

Example 93 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ResendRequestTest method testFailoverSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedNormalTopic() throws Exception {
    String key = "testFailoverSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(10);
    conf.setSubscriptionType(SubscriptionType.Failover);
    conf.setConsumerName("consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    conf.setConsumerName("consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1;
    Message message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    } while (message1 != null || message2 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 5. Consumer 1 asks for resend
    consumer1.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    // 6. Consumer 1 acknowledges a few messages
    receivedConsumer1 = receivedConsumer2 = 0;
    for (int i = 0; i < totalMessages / 2; i++) {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            log.info("Consumer 1 Acknowledged: " + new String(message1.getData()));
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    }
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages / 2);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 7. Consumer 1 close
    consumer1.close();
    // 8. Checking if all messages are received by Consumer 2
    message2 = consumer2.receive();
    int acknowledgedMessages = 0;
    int unAcknowledgedMessages = 0;
    boolean flag = true;
    do {
        if (flag) {
            consumer2.acknowledge(message2);
            acknowledgedMessages += 1;
            log.info("Consumer 2 Acknowledged: " + new String(message2.getData()));
        } else {
            unAcknowledgedMessages += 1;
        }
        flag = !flag;
        log.info("Consumer 2 Received: " + new String(message2.getData()));
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + (unAcknowledgedMessages + acknowledgedMessages));
    log.info("Consumer 2 acknowledges = " + acknowledgedMessages);
    assertEquals(unAcknowledgedMessages + acknowledgedMessages, totalMessages - receivedConsumer1);
    // 9 .Consumer 2 asks for a resend
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    message2 = consumer2.receive();
    receivedConsumer2 = 0;
    do {
        receivedConsumer2 += 1;
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    assertEquals(unAcknowledgedMessages, receivedConsumer2);
}

Example 94 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ResendRequestTest method testSharedSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                log.info("Consumer1 acked " + new String(message1.getData()));
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                log.info("Consumer2 acked " + new String(message2.getData()));
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }
    // 6. Check if Messages redelivered again
    message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    messageCount1 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message1 = consumer1.receive(1000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(1000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2 + ackCount1, totalMessages);
}

Example 95 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ResendRequestTest method testFailoverSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedPartitionedTopic() throws Exception {
    String key = "testFailoverSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    consumerConfig.setSubscriptionType(SubscriptionType.Failover);
    consumerConfig.setConsumerName("Consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    consumerConfig.setConsumerName("Consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    Thread.sleep(1000);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }
    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    consumer1.close();
    // 6. Check if Messages redelivered again
    message2 = consumer2.receive();
    messageCount1 = 0;
    do {
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount2 + ackCount1, totalMessages);
}