Search in sources :

Example 11 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class ConsumerImpl method redeliverUnacknowledgedMessages.

@Override
public void redeliverUnacknowledgedMessages(Set<MessageIdImpl> messageIds) {
    if (conf.getSubscriptionType() != SubscriptionType.Shared) {
        // We cannot redeliver single messages if subscription type is not Shared
        redeliverUnacknowledgedMessages();
        return;
    }
    ClientCnx cnx = cnx();
    if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getNumber()) {
        int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds);
        Iterable<List<MessageIdImpl>> batches = Iterables.partition(messageIds, MAX_REDELIVER_UNACKNOWLEDGED);
        MessageIdData.Builder builder = MessageIdData.newBuilder();
        batches.forEach(ids -> {
            List<MessageIdData> messageIdDatas = ids.stream().map(messageId -> {
                batchMessageAckTracker.remove(messageId);
                builder.setPartition(messageId.getPartitionIndex());
                builder.setLedgerId(messageId.getLedgerId());
                builder.setEntryId(messageId.getEntryId());
                return builder.build();
            }).collect(Collectors.toList());
            ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdDatas);
            cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise());
            messageIdDatas.forEach(MessageIdData::recycle);
        });
        if (messagesFromQueue > 0) {
            increaseAvailablePermits(cnx, messagesFromQueue);
        }
        builder.recycle();
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Redeliver unacked messages and increase {} permits", subscription, topic, consumerName, messagesFromQueue);
        }
        return;
    }
    if (cnx == null || (getState() == State.Connecting)) {
        log.warn("[{}] Client Connection needs to be establised for redelivery of unacknowledged messages", this);
    } else {
        log.warn("[{}] Reconnecting the client to redeliver the messages.", this);
        cnx.ctx().close();
    }
}
Also used : AtomicIntegerFieldUpdater(java.util.concurrent.atomic.AtomicIntegerFieldUpdater) Iterables(com.google.common.collect.Iterables) Consumer(com.yahoo.pulsar.client.api.Consumer) MessageId(com.yahoo.pulsar.client.api.MessageId) CompressionCodec(com.yahoo.pulsar.common.compression.CompressionCodec) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) PulsarDecoder(com.yahoo.pulsar.common.api.PulsarDecoder) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) SubscriptionType(com.yahoo.pulsar.client.api.SubscriptionType) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ByteBuf(io.netty.buffer.ByteBuf) FutureUtil(com.yahoo.pulsar.client.util.FutureUtil) Commands(com.yahoo.pulsar.common.api.Commands) ExecutorService(java.util.concurrent.ExecutorService) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) Commands.hasChecksum(com.yahoo.pulsar.common.api.Commands.hasChecksum) CompressionType(com.yahoo.pulsar.common.api.proto.PulsarApi.CompressionType) Timeout(io.netty.util.Timeout) Logger(org.slf4j.Logger) Commands.readChecksum(com.yahoo.pulsar.common.api.Commands.readChecksum) Crc32cChecksum.computeChecksum(com.yahoo.pulsar.checksum.utils.Crc32cChecksum.computeChecksum) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) ConsumerConfiguration(com.yahoo.pulsar.client.api.ConsumerConfiguration) Set(java.util.Set) GenericFutureListener(io.netty.util.concurrent.GenericFutureListener) IOException(java.io.IOException) CompressionCodecProvider(com.yahoo.pulsar.common.compression.CompressionCodecProvider) NavigableMap(java.util.NavigableMap) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) TimeUnit(java.util.concurrent.TimeUnit) ProtocolVersion(com.yahoo.pulsar.common.api.proto.PulsarApi.ProtocolVersion) MessageMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata) List(java.util.List) ConcurrentSkipListMap(java.util.concurrent.ConcurrentSkipListMap) PulsarApi(com.yahoo.pulsar.common.api.proto.PulsarApi) AckType(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandAck.AckType) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) MessageIdData(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData) Message(com.yahoo.pulsar.client.api.Message) BitSet(java.util.BitSet) Future(io.netty.util.concurrent.Future) ValidationError(com.yahoo.pulsar.common.api.proto.PulsarApi.CommandAck.ValidationError) MessageIdData(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageIdData) List(java.util.List) ByteBuf(io.netty.buffer.ByteBuf)

Example 12 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdTest method testChecksumVersionComptability.

/**
     * Verifies: different versions of broker-deployment (one broker understands Checksum and other
     * doesn't in that case remove checksum before sending to broker-2)
     * 
     * client first produce message with checksum and then retries to send message due to connection unavailable. But this time, if
     * broker doesn't understand checksum: then client should remove checksum from the message before sending to broker.
     * 
     * 1. stop broker 
     * 2. client compute checksum and add into message 
     * 3. produce 2 messages and corrupt 1 message 
     * 4. start broker with lower version (which doesn't support checksum) 
     * 5. client reconnects to broker and due to incompatibility of version: removes checksum from message 
     * 6. broker doesn't do checksum validation and persist message 
     * 7. client receives ack
     * 
     * @throws Exception
     */
@Test
public void testChecksumVersionComptability() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    // 1. producer connect
    ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
    ProducerImpl producer = spy(prod);
    // return higher version compare to broker : so, it forces client-producer to remove checksum from payload
    doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
    doAnswer(invocationOnMock -> prod.getState()).when(producer).getState();
    doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
    doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    // Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    stopBroker();
    // stop timer to auto-reconnect as let spy-Producer connect to broker manually so, spy-producer object can get
    // mock-value from brokerChecksumSupportedVersion
    ((PulsarClientImpl) pulsarClient).timer().stop();
    ClientCnx mockClientCnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient));
    doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
    prod.setClientCnx(mockClientCnx);
    Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
    Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
    CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
    // corrupt the message
    // new content would be 'message-3'
    msg2.getData()[msg2.getData().length - 1] = '3';
    prod.setClientCnx(null);
    // Restart the broker to have the messages published
    startBroker();
    // grab broker connection with mocked producer which has higher version compare to broker
    prod.grabCnx();
    try {
        // it should not fail: as due to unsupported version of broker: client removes checksum and broker should
        // ignore the checksum validation
        future1.get();
        future2.get();
    } catch (Exception e) {
        e.printStackTrace();
        fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
    }
    ((ConsumerImpl) consumer).grabCnx();
    // We should only receive msg1
    Message msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-1");
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-3");
}
Also used : Consumer(com.yahoo.pulsar.client.api.Consumer) Message(com.yahoo.pulsar.client.api.Message) PulsarAdminException(com.yahoo.pulsar.client.admin.PulsarAdminException) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) MessageId(com.yahoo.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 13 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdTest method testCorruptMessageRemove.

/**
     * Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then
     * 1. Client-Producer recomputes checksum with modified data
     * 2. Retry message-send again
     * 3. Broker verifies checksum 
     * 4. client receives send-ack success
     * 
     * @throws Exception
     */
@Test
public void testCorruptMessageRemove() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/retry-topic";
    ProducerConfiguration config = new ProducerConfiguration();
    config.setSendTimeout(10, TimeUnit.MINUTES);
    // 1. producer connect
    Producer prod = pulsarClient.createProducer(topicName, config);
    ProducerImpl producer = spy((ProducerImpl) prod);
    Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
    producerIdField.setAccessible(true);
    long producerId = (long) producerIdField.get(producer);
    // registered spy ProducerImpl
    producer.cnx().registerProducer(producerId, producer);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    // 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
    // checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    // enable checksum at producer
    stopBroker();
    Message msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future = producer.sendAsync(msg);
    // 3. corrupt the message
    // new content would be 'message-3'
    msg.getData()[msg.getData().length - 1] = '2';
    // 4. Restart the broker to have the messages published
    startBroker();
    try {
        future.get();
        fail("send message should have failed with checksum excetion");
    } catch (Exception e) {
        if (e.getCause() instanceof PulsarClientException.ChecksumException) {
        //ok (callback should get checksum exception as message was modified and corrupt)
        } else {
            fail("Callback should have only failed with ChecksumException", e);
        }
    }
    // 5. Verify
    // (5.1) Verify: producer's recoverChecksumError and updateChecksum invoked
    verify(producer, times(1)).recoverChecksumError(any(), anyLong());
    verify(producer, times(1)).verifyLocalBufferIsNotCorrupted(any());
    /**
         * (5.3) verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message
         * is corrupt
         */
    MessageImpl msg2 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
    ByteBuf payload = msg2.getDataBuffer();
    Builder metadataBuilder = ((MessageImpl) msg).getMessageBuilder();
    MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L).build();
    ByteBuf cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
    // (a) create OpSendMsg with message-data : "message-1"
    OpSendMsg op = OpSendMsg.create(((MessageImpl) msg), cmd, 1, null);
    // a.verify: as message is not corrupt: no need to update checksum
    assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
    // (b) corrupt message
    // new content would be 'message-2'
    msg2.getData()[msg2.getData().length - 1] = '2';
    // b. verify: as message is corrupt: update checksum
    assertFalse(producer.verifyLocalBufferIsNotCorrupted(op));
    assertEquals(producer.getPendingQueueSize(), 0);
    // [2] test-recoverChecksumError functionality
    stopBroker();
    MessageImpl msg1 = (MessageImpl) MessageBuilder.create().setContent("message-1".getBytes()).build();
    future = producer.sendAsync(msg1);
    ClientCnx cnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient) {
    });
    String exc = "broker is already stopped";
    // when client-try to recover checksum by resending to broker: throw exception as broker is stopped
    doThrow(new IllegalStateException(exc)).when(cnx).ctx();
    try {
        producer.recoverChecksumError(cnx, 1);
        fail("it should call : resendMessages() => which should throw above mocked exception");
    } catch (IllegalStateException e) {
        assertEquals(exc, e.getMessage());
    }
    producer.close();
    consumer.close();
    // clean reference of mocked producer
    producer = null;
}
Also used : Message(com.yahoo.pulsar.client.api.Message) OpSendMsg(com.yahoo.pulsar.client.impl.ProducerImpl.OpSendMsg) MessageBuilder(com.yahoo.pulsar.client.api.MessageBuilder) Builder(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata.Builder) ProducerConfiguration(com.yahoo.pulsar.client.api.ProducerConfiguration) DoubleByteBuf(com.yahoo.pulsar.common.api.DoubleByteBuf) ByteBuf(io.netty.buffer.ByteBuf) PulsarAdminException(com.yahoo.pulsar.client.admin.PulsarAdminException) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) Field(java.lang.reflect.Field) MessageMetadata(com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata) Producer(com.yahoo.pulsar.client.api.Producer) Consumer(com.yahoo.pulsar.client.api.Consumer) PulsarClientException(com.yahoo.pulsar.client.api.PulsarClientException) MessageId(com.yahoo.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 14 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class PulsarSpout method fail.

@Override
public void fail(Object msgId) {
    if (msgId instanceof Message) {
        Message msg = (Message) msgId;
        MessageId id = msg.getMessageId();
        LOG.warn("[{}] Error processing message {}", spoutId, id);
        // Since the message processing failed, we put it in the failed messages queue if there are more retries
        // remaining for the message
        MessageRetries messageRetries = pendingMessageRetries.computeIfAbsent(id, (k) -> new MessageRetries());
        if ((failedRetriesTimeoutNano < 0 || (messageRetries.getTimeStamp() + failedRetriesTimeoutNano) > System.nanoTime()) && (maxFailedRetries < 0 || messageRetries.numRetries < maxFailedRetries)) {
            // since we can retry again, we increment retry count and put it in the queue
            LOG.info("[{}] Putting message {} in the retry queue", spoutId, id);
            messageRetries.incrementAndGet();
            pendingMessageRetries.putIfAbsent(id, messageRetries);
            failedMessages.add(msg);
            --pendingAcks;
        } else {
            LOG.warn("[{}] Number of retries limit reached, dropping the message {}", spoutId, id);
            ack(msg);
        }
    }
}
Also used : Message(com.yahoo.pulsar.client.api.Message) MessageId(com.yahoo.pulsar.client.api.MessageId)

Example 15 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdSerialization method testProtobufSerialization2.

@Test
void testProtobufSerialization2() throws Exception {
    MessageId id = new MessageIdImpl(1, 2, -1);
    byte[] serializedId = id.toByteArray();
    assertEquals(MessageId.fromByteArray(serializedId), id);
}
Also used : MessageIdImpl(com.yahoo.pulsar.client.impl.MessageIdImpl) MessageId(com.yahoo.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Aggregations

MessageId (com.yahoo.pulsar.client.api.MessageId)18 Test (org.testng.annotations.Test)14 Consumer (com.yahoo.pulsar.client.api.Consumer)12 Producer (com.yahoo.pulsar.client.api.Producer)11 Message (com.yahoo.pulsar.client.api.Message)10 PulsarClientException (com.yahoo.pulsar.client.api.PulsarClientException)7 PulsarAdminException (com.yahoo.pulsar.client.admin.PulsarAdminException)6 HashSet (java.util.HashSet)5 CompletableFuture (java.util.concurrent.CompletableFuture)5 ProducerConfiguration (com.yahoo.pulsar.client.api.ProducerConfiguration)4 PersistentTopic (com.yahoo.pulsar.broker.service.persistent.PersistentTopic)3 ConsumerConfiguration (com.yahoo.pulsar.client.api.ConsumerConfiguration)3 MessageIdImpl (com.yahoo.pulsar.client.impl.MessageIdImpl)3 MessageMetadata (com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata)2 ByteBuf (io.netty.buffer.ByteBuf)2 IOException (java.io.IOException)2 Future (java.util.concurrent.Future)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)1 Iterables (com.google.common.collect.Iterables)1