Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 56 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class MessageIdTest method testCorruptMessageRemove.

/**
     * Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then
     * 1. Client-Producer recomputes checksum with modified data
     * 2. Retry message-send again
     * 3. Broker verifies checksum 
     * 4. client receives send-ack success
     * 
     * @throws Exception
     */
@Test
public void testCorruptMessageRemove() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/retry-topic";
    ProducerConfiguration config = new ProducerConfiguration();
    config.setSendTimeout(10, TimeUnit.MINUTES);
    // 1. producer connect
    Producer prod = pulsarClient.createProducer(topicName, config);
    ProducerImpl producer = spy((ProducerImpl) prod);
    Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
    producerIdField.setAccessible(true);
    long producerId = (long) producerIdField.get(producer);
    // registered spy ProducerImpl
    producer.cnx().registerProducer(producerId, producer);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    // 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
    // checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    // enable checksum at producer
    stopBroker();
    Message msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future = producer.sendAsync(msg);
    // 3. corrupt the message
    // new content would be 'message-3'
    msg.getData()[msg.getData().length - 1] = '2';
    // 4. Restart the broker to have the messages published
    startBroker();
    try {
        future.get();
        fail("send message should have failed with checksum excetion");
    } catch (Exception e) {
        if (e.getCause() instanceof PulsarClientException.ChecksumException) {
        //ok (callback should get checksum exception as message was modified and corrupt)
        } else {
            fail("Callback should have only failed with ChecksumException", e);
        }
    }
    // 5. Verify
    // (5.1) Verify: producer's recoverChecksumError and updateChecksum invoked
    verify(producer, times(1)).recoverChecksumError(any(), anyLong());
    verify(producer, times(1)).verifyLocalBufferIsNotCorrupted(any());
    /**
         * (5.3) verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message
         * is corrupt
         */
    MessageImpl msg2 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
    ByteBuf payload = msg2.getDataBuffer();
    Builder metadataBuilder = ((MessageImpl) msg).getMessageBuilder();
    MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L).build();
    ByteBuf cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
    // (a) create OpSendMsg with message-data : "message-1"
    OpSendMsg op = OpSendMsg.create(((MessageImpl) msg), cmd, 1, null);
    // a.verify: as message is not corrupt: no need to update checksum
    assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
    // (b) corrupt message
    // new content would be 'message-2'
    msg2.getData()[msg2.getData().length - 1] = '2';
    // b. verify: as message is corrupt: update checksum
    assertFalse(producer.verifyLocalBufferIsNotCorrupted(op));
    assertEquals(producer.getPendingQueueSize(), 0);
    // [2] test-recoverChecksumError functionality
    stopBroker();
    MessageImpl msg1 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
    future = producer.sendAsync(msg1);
    ClientCnx cnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient) {
    });
    String exc = "broker is already stopped";
    // when client-try to recover checksum by resending to broker: throw exception as broker is stopped
    doThrow(new IllegalStateException(exc)).when(cnx).ctx();
    try {
        producer.recoverChecksumError(cnx, 1);
        fail("it should call : resendMessages() => which should throw above mocked exception");
    } catch (IllegalStateException e) {
        assertEquals(exc, e.getMessage());
    }
    producer.close();
    consumer.close();
    // clean reference of mocked producer
    producer = null;
}

Example 57 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class CmdProduce method run.

/**
     * Run the producer.
     *
     * @return 0 for success, < 0 otherwise
     * @throws Exception
     */
public int run() throws PulsarClientException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.serviceURL == null || this.serviceURL.isEmpty())
        throw (new ParameterException("Broker URL is not provided."));
    if (this.numTimesProduce <= 0)
        throw (new ParameterException("Number of times need to be positive number."));
    if (messages.size() == 0 && messageFileNames.size() == 0)
        throw (new ParameterException("Please supply message content with either --messages or --files"));
    int totalMessages = (messages.size() + messageFileNames.size()) * numTimesProduce;
    if (totalMessages > MAX_MESSAGES) {
        String msg = "Attempting to send " + totalMessages + " messages. Please do not send more than " + MAX_MESSAGES + " messages";
        throw new ParameterException(msg);
    }
    String topic = this.mainOptions.get(0);
    int numMessagesSent = 0;
    int returnCode = 0;
    try {
        PulsarClient client = PulsarClient.create(this.serviceURL, this.clientConfig);
        Producer producer = client.createProducer(topic);
        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;
        for (int i = 0; i < this.numTimesProduce; i++) {
            List<Message> messages = generateMessages(messageBodies);
            for (Message msg : messages) {
                if (limiter != null)
                    limiter.acquire();
                producer.send(msg);
                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }
    return returnCode;
}

Example 58 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeNormalConsumer.

@Test()
public void zeroQueueSizeNormalConsumer() throws PulsarClientException {
    String key = "nonZeroQueueSizeNormalConsumer";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message
    Message message;
    for (int i = 0; i < totalMessages; i++) {
        assertEquals(consumer.numMessagesInQueue(), 0);
        message = consumer.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 59 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeSharedSubscription.

@Test()
public void zeroQueueSizeSharedSubscription() throws PulsarClientException {
    String key = "zeroQueueSizeSharedSubscription";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    int numOfSubscribers = 4;
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    configuration.setSubscriptionType(SubscriptionType.Shared);
    ConsumerImpl[] consumers = new ConsumerImpl[numOfSubscribers];
    for (int i = 0; i < numOfSubscribers; i++) {
        consumers[i] = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    }
    // 4. Produce Messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 5. Consume messages
    Message message;
    for (int i = 0; i < totalMessages; i++) {
        assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
        message = consumers[i % numOfSubscribers].receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 60 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class StormExample method main.

public static void main(String[] args) throws PulsarClientException {
    ClientConfiguration clientConf = new ClientConfiguration();
    // String authPluginClassName = "com.yahoo.pulsar.client.impl.auth.MyAuthentication";
    // String authParams = "key1:val1,key2:val2";
    // clientConf.setAuthentication(authPluginClassName, authParams);
    String topic1 = "persistent://my-property/use/my-ns/my-topic1";
    String topic2 = "persistent://my-property/use/my-ns/my-topic2";
    String subscriptionName1 = "my-subscriber-name1";
    String subscriptionName2 = "my-subscriber-name2";
    // create spout
    PulsarSpoutConfiguration spoutConf = new PulsarSpoutConfiguration();
    spoutConf.setServiceUrl(serviceUrl);
    spoutConf.setTopic(topic1);
    spoutConf.setSubscriptionName(subscriptionName1);
    spoutConf.setMessageToValuesMapper(messageToValuesMapper);
    PulsarSpout spout = new PulsarSpout(spoutConf, clientConf);
    // create bolt
    PulsarBoltConfiguration boltConf = new PulsarBoltConfiguration();
    boltConf.setServiceUrl(serviceUrl);
    boltConf.setTopic(topic2);
    boltConf.setTupleToMessageMapper(tupleToMessageMapper);
    PulsarBolt bolt = new PulsarBolt(boltConf, clientConf);
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("testSpout", spout);
    builder.setBolt("testBolt", bolt).shuffleGrouping("testSpout");
    Config conf = new Config();
    conf.setNumWorkers(2);
    conf.setDebug(true);
    conf.registerMetricsConsumer(PulsarMetricsConsumer.class);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    PulsarClient pulsarClient = PulsarClient.create(serviceUrl, clientConf);
    // create a consumer on topic2 to receive messages from the bolt when the processing is done
    Consumer consumer = pulsarClient.subscribe(topic2, subscriptionName2);
    // create a producer on topic1 to send messages that will be received by the spout
    Producer producer = pulsarClient.createProducer(topic1);
    for (int i = 0; i < 10; i++) {
        String msg = "msg-" + i;
        producer.send(msg.getBytes());
        LOG.info("Message {} sent", msg);
    }
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        LOG.info("Message {} received", new String(msg.getData()));
    }
    cluster.killTopology("test");
    cluster.shutdown();
}