Examples with Message com.yahoo.pulsar.client.api.Message used on opensource projects

Example 41 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarBoltTest method testNoMessageSend.

@Test
public void testNoMessageSend() throws Exception {
    String msgContent = "message to be dropped";
    Tuple tuple = getMockTuple(msgContent);
    bolt.execute(tuple);
    Assert.assertTrue(mockCollector.acked());
    Message msg = consumer.receive(5, TimeUnit.SECONDS);
    Assert.assertNull(msg);
}

Example 42 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarBoltTest method testTickTuple.

@Test
public void testTickTuple() throws Exception {
    Tuple mockTuple = mock(Tuple.class);
    when(mockTuple.getSourceComponent()).thenReturn(Constants.SYSTEM_COMPONENT_ID);
    when(mockTuple.getSourceStreamId()).thenReturn(Constants.SYSTEM_TICK_STREAM_ID);
    bolt.execute(mockTuple);
    Assert.assertTrue(mockCollector.acked());
    Message msg = consumer.receive(5, TimeUnit.SECONDS);
    Assert.assertNull(msg);
}

Example 43 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarBolt method execute.

@Override
public void execute(Tuple input) {
    // do not send tick tuples since they are used to execute periodic tasks
    if (isTickTuple(input)) {
        collector.ack(input);
        return;
    }
    try {
        if (producer != null) {
            // a message key can be provided in the mapper
            Message msg = pulsarBoltConf.getTupleToMessageMapper().toMessage(input);
            if (msg == null) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("[{}] Cannot send null message, acking the collector", boltId);
                }
                collector.ack(input);
            } else {
                final long messageSizeToBeSent = msg.getData().length;
                producer.sendAsync(msg).handle((r, ex) -> {
                    synchronized (collector) {
                        if (ex != null) {
                            collector.reportError(ex);
                            collector.fail(input);
                            LOG.error("[{}] Message send failed", boltId, ex);
                        } else {
                            collector.ack(input);
                            ++messagesSent;
                            messageSizeSent += messageSizeToBeSent;
                            if (LOG.isDebugEnabled()) {
                                LOG.debug("[{}] Message sent with id {}", boltId, msg.getMessageId());
                            }
                        }
                    }
                    return null;
                });
            }
        }
    } catch (Exception e) {
        LOG.error("[{}] Message processing failed", boltId, e);
        collector.reportError(e);
        collector.fail(input);
    }
}

Example 44 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testUnsupportedBatchMessageConsumer.

/**
     * It verifies that consumer which doesn't support batch-message:
     * <p>
     * 1. broker disconnects that consumer
     * <p>
     * 2. redeliver all those messages to other supported consumer under the same subscription
     * 
     * @param subType
     * @throws Exception
     */
@Test(timeOut = 7000, dataProvider = "subType")
public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int batchMessageDelayMs = 1000;
    final String topicName = "persistent://my-property/use/my-ns/my-topic1";
    final String subscriptionName = "my-subscriber-name" + subType;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(subType);
    ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(20);
    }
    Producer producer = pulsarClient.createProducer(topicName, new ProducerConfiguration());
    Producer batchProducer = pulsarClient.createProducer(topicName, producerConf);
    // update consumer's version to incompatible batch-message version = Version.V3
    Topic topic = pulsar.getBrokerService().getTopic(topicName).get();
    com.yahoo.pulsar.broker.service.Consumer brokerConsumer = topic.getSubscriptions().get(subscriptionName).getConsumers().get(0);
    Field cnxField = com.yahoo.pulsar.broker.service.Consumer.class.getDeclaredField("cnx");
    cnxField.setAccessible(true);
    PulsarHandler cnx = (PulsarHandler) cnxField.get(brokerConsumer);
    Field versionField = PulsarHandler.class.getDeclaredField("remoteEndpointProtocolVersion");
    versionField.setAccessible(true);
    versionField.set(cnx, 3);
    // (1) send non-batch message: consumer should be able to consume
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Set<String> messageSet = Sets.newHashSet();
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        consumer1.acknowledge(msg);
    }
    // Also set clientCnx of the consumer to null so, it avoid reconnection so, other consumer can consume for
    // verification
    consumer1.setClientCnx(null);
    // (2) send batch-message which should not be able to consume: as broker will disconnect the consumer
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        batchProducer.sendAsync(message.getBytes());
    }
    Thread.sleep(batchMessageDelayMs);
    // consumer should have not received any message as it should have been disconnected
    msg = consumer1.receive(2, TimeUnit.SECONDS);
    assertNull(msg);
    // subscrie consumer2 with supporting batch version
    pulsarClient = PulsarClient.create(brokerUrl.toString());
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    messageSet.clear();
    for (int i = 0; i < 10; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        consumer2.acknowledge(msg);
    }
    consumer2.close();
    producer.close();
    batchProducer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 45 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class MessageIdTest method testChecksumVersionComptability.

/**
     * Verifies: different versions of broker-deployment (one broker understands Checksum and other
     * doesn't in that case remove checksum before sending to broker-2)
     * 
     * client first produce message with checksum and then retries to send message due to connection unavailable. But this time, if
     * broker doesn't understand checksum: then client should remove checksum from the message before sending to broker.
     * 
     * 1. stop broker 
     * 2. client compute checksum and add into message 
     * 3. produce 2 messages and corrupt 1 message 
     * 4. start broker with lower version (which doesn't support checksum) 
     * 5. client reconnects to broker and due to incompatibility of version: removes checksum from message 
     * 6. broker doesn't do checksum validation and persist message 
     * 7. client receives ack
     * 
     * @throws Exception
     */
@Test
public void testChecksumVersionComptability() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    // 1. producer connect
    ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
    ProducerImpl producer = spy(prod);
    // return higher version compare to broker : so, it forces client-producer to remove checksum from payload
    doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
    doAnswer(invocationOnMock -> prod.getState()).when(producer).getState();
    doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
    doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    // Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    stopBroker();
    // stop timer to auto-reconnect as let spy-Producer connect to broker manually so, spy-producer object can get
    // mock-value from brokerChecksumSupportedVersion
    ((PulsarClientImpl) pulsarClient).timer().stop();
    ClientCnx mockClientCnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient));
    doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
    prod.setClientCnx(mockClientCnx);
    Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
    Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
    CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
    // corrupt the message
    // new content would be 'message-3'
    msg2.getData()[msg2.getData().length - 1] = '3';
    prod.setClientCnx(null);
    // Restart the broker to have the messages published
    startBroker();
    // grab broker connection with mocked producer which has higher version compare to broker
    prod.grabCnx();
    try {
        // it should not fail: as due to unsupported version of broker: client removes checksum and broker should
        // ignore the checksum validation
        future1.get();
        future2.get();
    } catch (Exception e) {
        e.printStackTrace();
        fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
    }
    ((ConsumerImpl) consumer).grabCnx();
    // We should only receive msg1
    Message msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-1");
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-3");
}