Examples with ByteBufPair org.apache.pulsar.common.api.ByteBufPair used on opensource projects

Example 1 with ByteBufPair

use of org.apache.pulsar.common.api.ByteBufPair in project incubator-pulsar by apache.

the class MessageIdTest method testCorruptMessageRemove.

/**
 * Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then 1.
 * Client-Producer recomputes checksum with modified data 2. Retry message-send again 3. Broker verifies checksum 4.
 * client receives send-ack success
 *
 * @throws Exception
 */
@Test
public void testCorruptMessageRemove() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/retry-topic";
    // 1. producer connect
    ProducerImpl<byte[]> prod = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).sendTimeout(10, TimeUnit.MINUTES).create();
    ProducerImpl<byte[]> producer = spy(prod);
    Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
    producerIdField.setAccessible(true);
    long producerId = (long) producerIdField.get(producer);
    // registered spy ProducerImpl
    producer.cnx().registerProducer(producerId, producer);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    // 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
    // checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    // enable checksum at producer
    stopBroker();
    Message<byte[]> msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future = producer.sendAsync(msg);
    // 3. corrupt the message
    // new content would be 'message-3'
    msg.getData()[msg.getData().length - 1] = '2';
    // 4. Restart the broker to have the messages published
    startBroker();
    try {
        future.get();
        fail("send message should have failed with checksum excetion");
    } catch (Exception e) {
        if (e.getCause() instanceof PulsarClientException.ChecksumException) {
        // ok (callback should get checksum exception as message was modified and corrupt)
        } else {
            fail("Callback should have only failed with ChecksumException", e);
        }
    }
    // 5. Verify
    /**
     * verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message is corrupt
     */
    MessageImpl<byte[]> msg2 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes()).build();
    ByteBuf payload = msg2.getDataBuffer();
    Builder metadataBuilder = ((MessageImpl<byte[]>) msg).getMessageBuilder();
    MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L).build();
    ByteBufPair cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
    // (a) create OpSendMsg with message-data : "message-1"
    OpSendMsg op = OpSendMsg.create(((MessageImpl<byte[]>) msg), cmd, 1, null);
    // a.verify: as message is not corrupt: no need to update checksum
    assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
    // (b) corrupt message
    // new content would be 'message-2'
    msg2.getData()[msg2.getData().length - 1] = '2';
    // b. verify: as message is corrupt: update checksum
    assertFalse(producer.verifyLocalBufferIsNotCorrupted(op));
    assertEquals(producer.getPendingQueueSize(), 0);
    // [2] test-recoverChecksumError functionality
    stopBroker();
    MessageImpl<byte[]> msg1 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes()).build();
    future = producer.sendAsync(msg1);
    ClientCnx cnx = spy(new ClientCnx(new ClientConfigurationData(), ((PulsarClientImpl) pulsarClient).eventLoopGroup()));
    String exc = "broker is already stopped";
    // when client-try to recover checksum by resending to broker: throw exception as broker is stopped
    doThrow(new IllegalStateException(exc)).when(cnx).ctx();
    try {
        producer.recoverChecksumError(cnx, 1);
        fail("it should call : resendMessages() => which should throw above mocked exception");
    } catch (IllegalStateException e) {
        assertEquals(exc, e.getMessage());
    }
    producer.close();
    consumer.close();
    // clean reference of mocked producer
    producer = null;
}

Example 2 with ByteBufPair

use of org.apache.pulsar.common.api.ByteBufPair in project incubator-pulsar by apache.

the class ProducerImpl method sendAsync.

public void sendAsync(Message<T> message, SendCallback callback) {
    checkArgument(message instanceof MessageImpl);
    if (!isValidProducerState(callback)) {
        return;
    }
    if (!canEnqueueRequest(callback)) {
        return;
    }
    MessageImpl<T> msg = (MessageImpl<T>) message;
    MessageMetadata.Builder msgMetadata = msg.getMessageBuilder();
    ByteBuf payload = msg.getDataBuffer();
    // If compression is enabled, we are compressing, otherwise it will simply use the same buffer
    int uncompressedSize = payload.readableBytes();
    ByteBuf compressedPayload = payload;
    // batch will be compressed when closed
    if (!isBatchMessagingEnabled()) {
        compressedPayload = compressor.encode(payload);
        payload.release();
    }
    int compressedSize = compressedPayload.readableBytes();
    // batch)
    if (compressedSize > PulsarDecoder.MaxMessageSize) {
        compressedPayload.release();
        String compressedStr = (!isBatchMessagingEnabled() && conf.getCompressionType() != CompressionType.NONE) ? "Compressed" : "";
        callback.sendComplete(new PulsarClientException.InvalidMessageException(format("%s Message payload size %d cannot exceed %d bytes", compressedStr, compressedSize, PulsarDecoder.MaxMessageSize)));
        return;
    }
    if (!msg.isReplicated() && msgMetadata.hasProducerName()) {
        callback.sendComplete(new PulsarClientException.InvalidMessageException("Cannot re-use the same message"));
        compressedPayload.release();
        return;
    }
    try {
        synchronized (this) {
            long sequenceId;
            if (!msgMetadata.hasSequenceId()) {
                sequenceId = msgIdGeneratorUpdater.getAndIncrement(this);
                msgMetadata.setSequenceId(sequenceId);
            } else {
                sequenceId = msgMetadata.getSequenceId();
            }
            if (!msgMetadata.hasPublishTime()) {
                msgMetadata.setPublishTime(System.currentTimeMillis());
                checkArgument(!msgMetadata.hasProducerName());
                msgMetadata.setProducerName(producerName);
                if (conf.getCompressionType() != CompressionType.NONE) {
                    msgMetadata.setCompression(convertCompressionType(conf.getCompressionType()));
                    msgMetadata.setUncompressedSize(uncompressedSize);
                }
            }
            if (isBatchMessagingEnabled()) {
                // batch size and/or max message size
                if (batchMessageContainer.hasSpaceInBatch(msg)) {
                    batchMessageContainer.add(msg, callback);
                    payload.release();
                    if (batchMessageContainer.numMessagesInBatch == maxNumMessagesInBatch || batchMessageContainer.currentBatchSizeBytes >= BatchMessageContainer.MAX_MESSAGE_BATCH_SIZE_BYTES) {
                        batchMessageAndSend();
                    }
                } else {
                    doBatchSendAndAdd(msg, callback, payload);
                }
            } else {
                ByteBuf encryptedPayload = encryptMessage(msgMetadata, compressedPayload);
                ByteBufPair cmd = sendMessage(producerId, sequenceId, 1, msgMetadata.build(), encryptedPayload);
                msgMetadata.recycle();
                final OpSendMsg op = OpSendMsg.create(msg, cmd, sequenceId, callback);
                op.setNumMessagesInBatch(1);
                op.setBatchSizeByte(encryptedPayload.readableBytes());
                pendingMessages.put(op);
                // Read the connection before validating if it's still connected, so that we avoid reading a null
                // value
                ClientCnx cnx = cnx();
                if (isConnected()) {
                    // If we do have a connection, the message is sent immediately, otherwise we'll try again once a
                    // new
                    // connection is established
                    cmd.retain();
                    cnx.ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx, op));
                    stats.updateNumMsgsSent(op.numMessagesInBatch, op.batchSizeByte);
                } else {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName, sequenceId);
                    }
                }
            }
        }
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        semaphore.release();
        callback.sendComplete(new PulsarClientException(ie));
    } catch (PulsarClientException e) {
        semaphore.release();
        callback.sendComplete(e);
    } catch (Throwable t) {
        semaphore.release();
        callback.sendComplete(new PulsarClientException(t));
    }
}

Example 3 with ByteBufPair

use of org.apache.pulsar.common.api.ByteBufPair in project incubator-pulsar by apache.

the class ProducerImpl method verifyLocalBufferIsNotCorrupted.

/**
 * Computes checksum again and verifies it against existing checksum. If checksum doesn't match it means that
 * message is corrupt.
 *
 * @param op
 * @return returns true only if message is not modified and computed-checksum is same as previous checksum else
 *         return false that means that message is corrupted. Returns true if checksum is not present.
 */
protected boolean verifyLocalBufferIsNotCorrupted(OpSendMsg op) {
    ByteBufPair msg = op.cmd;
    if (msg != null) {
        ByteBuf headerFrame = msg.getFirst();
        headerFrame.markReaderIndex();
        try {
            // skip bytes up to checksum index
            // skip [total-size]
            headerFrame.skipBytes(4);
            int cmdSize = (int) headerFrame.readUnsignedInt();
            headerFrame.skipBytes(cmdSize);
            // verify if checksum present
            if (hasChecksum(headerFrame)) {
                int checksum = readChecksum(headerFrame);
                // msg.readerIndex is already at header-payload index, Recompute checksum for headers-payload
                int metadataChecksum = computeChecksum(headerFrame);
                long computedChecksum = resumeChecksum(metadataChecksum, msg.getSecond());
                return checksum == computedChecksum;
            } else {
                log.warn("[{}] [{}] checksum is not present into message with id {}", topic, producerName, op.sequenceId);
            }
        } finally {
            headerFrame.resetReaderIndex();
        }
        return true;
    } else {
        log.warn("[{}] Failed while casting {} into ByteBufPair", producerName, op.cmd.getClass().getName());
        return false;
    }
}

Example 4 with ByteBufPair

use of org.apache.pulsar.common.api.ByteBufPair in project incubator-pulsar by apache.

the class ProducerImpl method batchMessageAndSend.

// must acquire semaphore before enqueuing
private void batchMessageAndSend() {
    if (log.isDebugEnabled()) {
        log.debug("[{}] [{}] Batching the messages from the batch container with {} messages", topic, producerName, batchMessageContainer.numMessagesInBatch);
    }
    OpSendMsg op = null;
    int numMessagesInBatch = 0;
    try {
        if (!batchMessageContainer.isEmpty()) {
            numMessagesInBatch = batchMessageContainer.numMessagesInBatch;
            ByteBuf compressedPayload = batchMessageContainer.getCompressedBatchMetadataAndPayload();
            long sequenceId = batchMessageContainer.sequenceId;
            ByteBuf encryptedPayload = encryptMessage(batchMessageContainer.messageMetadata, compressedPayload);
            ByteBufPair cmd = sendMessage(producerId, sequenceId, batchMessageContainer.numMessagesInBatch, batchMessageContainer.setBatchAndBuild(), encryptedPayload);
            op = OpSendMsg.create(batchMessageContainer.messages, cmd, sequenceId, batchMessageContainer.firstCallback);
            op.setNumMessagesInBatch(batchMessageContainer.numMessagesInBatch);
            op.setBatchSizeByte(batchMessageContainer.currentBatchSizeBytes);
            batchMessageContainer.clear();
            pendingMessages.put(op);
            if (isConnected()) {
                // If we do have a connection, the message is sent immediately, otherwise we'll try again once a new
                // connection is established
                cmd.retain();
                cnx().ctx().channel().eventLoop().execute(WriteInEventLoopCallback.create(this, cnx(), op));
                stats.updateNumMsgsSent(numMessagesInBatch, op.batchSizeByte);
            } else {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Connection is not ready -- sequenceId {}", topic, producerName, sequenceId);
                }
            }
        }
    } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        semaphore.release(numMessagesInBatch);
        if (op != null) {
            op.callback.sendComplete(new PulsarClientException(ie));
        }
    } catch (PulsarClientException e) {
        Thread.currentThread().interrupt();
        semaphore.release(numMessagesInBatch);
        if (op != null) {
            op.callback.sendComplete(e);
        }
    } catch (Throwable t) {
        semaphore.release(numMessagesInBatch);
        log.warn("[{}] [{}] error while closing out batch -- {}", topic, producerName, t);
        if (op != null) {
            op.callback.sendComplete(new PulsarClientException(t));
        }
    }
}

Example 5 with ByteBufPair

use of org.apache.pulsar.common.api.ByteBufPair in project incubator-pulsar by apache.

the class CommandsTest method testChecksumSendCommand.

@Test
public void testChecksumSendCommand() throws Exception {
    // test checksum in send command
    String producerName = "prod-name";
    int sequenceId = 0;
    ByteBuf data = Unpooled.buffer(1024);
    MessageMetadata messageMetadata = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName(producerName).setSequenceId(sequenceId).build();
    int expectedChecksum = computeChecksum(messageMetadata, data);
    ByteBufPair clientCommand = Commands.newSend(1, 0, 1, ChecksumType.Crc32c, messageMetadata, data);
    clientCommand.retain();
    ByteBuf receivedBuf = ByteBufPair.coalesce(clientCommand);
    // skip [total-size]
    receivedBuf.skipBytes(4);
    int cmdSize = (int) receivedBuf.readUnsignedInt();
    receivedBuf.readerIndex(8 + cmdSize);
    int startMessagePos = receivedBuf.readerIndex();
    /**
     * 1. verify checksum and metadataParsing **
     */
    boolean hasChecksum = Commands.hasChecksum(receivedBuf);
    int checksum = Commands.readChecksum(receivedBuf);
    // verify checksum is present
    assertTrue(hasChecksum);
    // verify checksum value
    assertEquals(expectedChecksum, checksum);
    MessageMetadata metadata = Commands.parseMessageMetadata(receivedBuf);
    // verify metadata parsing
    assertEquals(metadata.getProducerName(), producerName);
    /**
     * 2. parseMessageMetadata should skip checksum if present *
     */
    receivedBuf.readerIndex(startMessagePos);
    metadata = Commands.parseMessageMetadata(receivedBuf);
    // verify metadata parsing
    assertEquals(metadata.getProducerName(), producerName);
}