Examples with KeyValue org.apache.pulsar.common.api.proto.KeyValue used on opensource projects

Example 6 with KeyValue

use of org.apache.pulsar.common.api.proto.KeyValue in project pulsar by apache.

the class TopicsImpl method getIndividualMsgsFromBatch.

private List<Message<byte[]>> getIndividualMsgsFromBatch(String topic, String msgId, byte[] data, Map<String, String> properties, MessageMetadata msgMetadataBuilder, BrokerEntryMetadata brokerEntryMetadata) {
    List<Message<byte[]>> ret = new ArrayList<>();
    int batchSize = Integer.parseInt(properties.get(BATCH_HEADER));
    ByteBuf buf = Unpooled.wrappedBuffer(data);
    for (int i = 0; i < batchSize; i++) {
        String batchMsgId = msgId + ":" + i;
        SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
        try {
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadata, i, batchSize);
            if (singleMessageMetadata.getPropertiesCount() > 0) {
                for (KeyValue entry : singleMessageMetadata.getPropertiesList()) {
                    properties.put(entry.getKey(), entry.getValue());
                }
            }
            MessageImpl message = new MessageImpl<>(topic, batchMsgId, properties, singleMessagePayload, Schema.BYTES, msgMetadataBuilder);
            if (brokerEntryMetadata != null) {
                message.setBrokerEntryMetadata(brokerEntryMetadata);
            }
            ret.add(message);
        } catch (Exception ex) {
            log.error("Exception occurred while trying to get BatchMsgId: {}", batchMsgId, ex);
        }
    }
    buf.release();
    return ret;
}

Example 7 with KeyValue

use of org.apache.pulsar.common.api.proto.KeyValue in project pulsar by apache.

the class Commands method newSubscribe.

public static ByteBuf newSubscribe(String topic, String subscription, long consumerId, long requestId, SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageIdData startMessageId, Map<String, String> metadata, boolean readCompacted, boolean isReplicated, InitialPosition subscriptionInitialPosition, long startMessageRollbackDurationInSec, SchemaInfo schemaInfo, boolean createTopicIfDoesNotExist, KeySharedPolicy keySharedPolicy, Map<String, String> subscriptionProperties, long consumerEpoch) {
    BaseCommand cmd = localCmd(Type.SUBSCRIBE);
    CommandSubscribe subscribe = cmd.setSubscribe().setTopic(topic).setSubscription(subscription).setSubType(subType).setConsumerId(consumerId).setConsumerName(consumerName).setRequestId(requestId).setPriorityLevel(priorityLevel).setDurable(isDurable).setReadCompacted(readCompacted).setInitialPosition(subscriptionInitialPosition).setReplicateSubscriptionState(isReplicated).setForceTopicCreation(createTopicIfDoesNotExist).setConsumerEpoch(consumerEpoch);
    if (subscriptionProperties != null && !subscriptionProperties.isEmpty()) {
        List<KeyValue> keyValues = new ArrayList<>();
        subscriptionProperties.forEach((key, value) -> {
            KeyValue keyValue = new KeyValue();
            keyValue.setKey(key);
            keyValue.setValue(value);
            keyValues.add(keyValue);
        });
        subscribe.addAllSubscriptionProperties(keyValues);
    }
    if (keySharedPolicy != null) {
        KeySharedMeta keySharedMeta = subscribe.setKeySharedMeta();
        keySharedMeta.setAllowOutOfOrderDelivery(keySharedPolicy.isAllowOutOfOrderDelivery());
        keySharedMeta.setKeySharedMode(convertKeySharedMode(keySharedPolicy.getKeySharedMode()));
        if (keySharedPolicy instanceof KeySharedPolicy.KeySharedPolicySticky) {
            List<Range> ranges = ((KeySharedPolicy.KeySharedPolicySticky) keySharedPolicy).getRanges();
            for (Range range : ranges) {
                IntRange r = keySharedMeta.addHashRange();
                r.setStart(range.getStart());
                r.setEnd(range.getEnd());
            }
        }
    }
    if (startMessageId != null) {
        subscribe.setStartMessageId().copyFrom(startMessageId);
    }
    if (startMessageRollbackDurationInSec > 0) {
        subscribe.setStartMessageRollbackDurationSec(startMessageRollbackDurationInSec);
    }
    if (!metadata.isEmpty()) {
        metadata.entrySet().forEach(e -> subscribe.addMetadata().setKey(e.getKey()).setValue(e.getValue()));
    }
    if (schemaInfo != null) {
        if (subscribe.hasSchema()) {
            throw new IllegalStateException();
        }
        if (subscribe.setSchema().getPropertiesCount() > 0) {
            throw new IllegalStateException();
        }
        convertSchema(schemaInfo, subscribe.setSchema());
    }
    return serializeWithSize(cmd);
}

Example 8 with KeyValue

use of org.apache.pulsar.common.api.proto.KeyValue in project pulsar by yahoo.

the class InterceptorsTest method testConsumerInterceptorForAcknowledgeCumulative.

@Test
public void testConsumerInterceptorForAcknowledgeCumulative() throws PulsarClientException {
    List<MessageId> ackHolder = new ArrayList<>();
    ConsumerInterceptor<String> interceptor = new ConsumerInterceptor<String>() {

        @Override
        public void close() {
        }

        @Override
        public Message<String> beforeConsume(Consumer<String> consumer, Message<String> message) {
            MessageImpl<String> msg = (MessageImpl<String>) message;
            msg.getMessageBuilder().addProperty().setKey("beforeConsumer").setValue("1");
            return msg;
        }

        @Override
        public void onAcknowledge(Consumer<String> consumer, MessageId messageId, Throwable cause) {
            log.info("onAcknowledge messageId: {}", messageId, cause);
        }

        @Override
        public void onAcknowledgeCumulative(Consumer<String> consumer, MessageId messageId, Throwable cause) {
            long acknowledged = ackHolder.stream().filter(m -> (m.compareTo(messageId) <= 0)).count();
            Assert.assertEquals(acknowledged, 100);
            ackHolder.clear();
            log.info("onAcknowledgeCumulative messageIds: {}", messageId, cause);
        }

        @Override
        public void onNegativeAcksSend(Consumer<String> consumer, Set<MessageId> messageIds) {
        }

        @Override
        public void onAckTimeoutSend(Consumer<String> consumer, Set<MessageId> messageIds) {
        }
    };
    Consumer<String> consumer = pulsarClient.newConsumer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic").subscriptionType(SubscriptionType.Failover).intercept(interceptor).subscriptionName("my-subscription").subscribe();
    Producer<String> producer = pulsarClient.newProducer(Schema.STRING).topic("persistent://my-property/my-ns/my-topic").create();
    for (int i = 0; i < 100; i++) {
        producer.newMessage().value("Hello Pulsar!").send();
    }
    int keyCount = 0;
    for (int i = 0; i < 100; i++) {
        Message<String> received = consumer.receive();
        MessageImpl<String> msg = (MessageImpl<String>) received;
        for (KeyValue keyValue : msg.getMessageBuilder().getPropertiesList()) {
            if ("beforeConsumer".equals(keyValue.getKey())) {
                keyCount++;
            }
        }
        ackHolder.add(received.getMessageId());
        if (i == 99) {
            consumer.acknowledgeCumulative(received);
        }
    }
    Assert.assertEquals(100, keyCount);
    producer.close();
    consumer.close();
}

Example 9 with KeyValue

use of org.apache.pulsar.common.api.proto.KeyValue in project pulsar by yahoo.

the class ConsumerImpl method createEncryptionContext.

/**
 * Create EncryptionContext if message payload is encrypted.
 *
 * @param msgMetadata
 * @return {@link Optional}<{@link EncryptionContext}>
 */
private Optional<EncryptionContext> createEncryptionContext(MessageMetadata msgMetadata) {
    EncryptionContext encryptionCtx = null;
    if (msgMetadata.getEncryptionKeysCount() > 0) {
        encryptionCtx = new EncryptionContext();
        Map<String, EncryptionKey> keys = msgMetadata.getEncryptionKeysList().stream().collect(Collectors.toMap(EncryptionKeys::getKey, e -> new EncryptionKey(e.getValue(), e.getMetadatasList().stream().collect(Collectors.toMap(KeyValue::getKey, KeyValue::getValue)))));
        byte[] encParam = msgMetadata.getEncryptionParam();
        Optional<Integer> batchSize = Optional.ofNullable(msgMetadata.hasNumMessagesInBatch() ? msgMetadata.getNumMessagesInBatch() : null);
        encryptionCtx.setKeys(keys);
        encryptionCtx.setParam(encParam);
        if (msgMetadata.hasEncryptionAlgo()) {
            encryptionCtx.setAlgorithm(msgMetadata.getEncryptionAlgo());
        }
        encryptionCtx.setCompressionType(CompressionCodecProvider.convertFromWireProtocol(msgMetadata.getCompression()));
        encryptionCtx.setUncompressedMessageSize(msgMetadata.getUncompressedSize());
        encryptionCtx.setBatchSize(batchSize);
    }
    return Optional.ofNullable(encryptionCtx);
}

Example 10 with KeyValue

use of org.apache.pulsar.common.api.proto.KeyValue in project pulsar by yahoo.

the class Commands method newSubscribe.

public static ByteBuf newSubscribe(String topic, String subscription, long consumerId, long requestId, SubType subType, int priorityLevel, String consumerName, boolean isDurable, MessageIdData startMessageId, Map<String, String> metadata, boolean readCompacted, boolean isReplicated, InitialPosition subscriptionInitialPosition, long startMessageRollbackDurationInSec, SchemaInfo schemaInfo, boolean createTopicIfDoesNotExist, KeySharedPolicy keySharedPolicy, Map<String, String> subscriptionProperties, long consumerEpoch) {
    BaseCommand cmd = localCmd(Type.SUBSCRIBE);
    CommandSubscribe subscribe = cmd.setSubscribe().setTopic(topic).setSubscription(subscription).setSubType(subType).setConsumerId(consumerId).setConsumerName(consumerName).setRequestId(requestId).setPriorityLevel(priorityLevel).setDurable(isDurable).setReadCompacted(readCompacted).setInitialPosition(subscriptionInitialPosition).setReplicateSubscriptionState(isReplicated).setForceTopicCreation(createTopicIfDoesNotExist).setConsumerEpoch(consumerEpoch);
    if (subscriptionProperties != null && !subscriptionProperties.isEmpty()) {
        List<KeyValue> keyValues = new ArrayList<>();
        subscriptionProperties.forEach((key, value) -> {
            KeyValue keyValue = new KeyValue();
            keyValue.setKey(key);
            keyValue.setValue(value);
            keyValues.add(keyValue);
        });
        subscribe.addAllSubscriptionProperties(keyValues);
    }
    if (keySharedPolicy != null) {
        KeySharedMeta keySharedMeta = subscribe.setKeySharedMeta();
        keySharedMeta.setAllowOutOfOrderDelivery(keySharedPolicy.isAllowOutOfOrderDelivery());
        keySharedMeta.setKeySharedMode(convertKeySharedMode(keySharedPolicy.getKeySharedMode()));
        if (keySharedPolicy instanceof KeySharedPolicy.KeySharedPolicySticky) {
            List<Range> ranges = ((KeySharedPolicy.KeySharedPolicySticky) keySharedPolicy).getRanges();
            for (Range range : ranges) {
                IntRange r = keySharedMeta.addHashRange();
                r.setStart(range.getStart());
                r.setEnd(range.getEnd());
            }
        }
    }
    if (startMessageId != null) {
        subscribe.setStartMessageId().copyFrom(startMessageId);
    }
    if (startMessageRollbackDurationInSec > 0) {
        subscribe.setStartMessageRollbackDurationSec(startMessageRollbackDurationInSec);
    }
    if (!metadata.isEmpty()) {
        metadata.entrySet().forEach(e -> subscribe.addMetadata().setKey(e.getKey()).setValue(e.getValue()));
    }
    if (schemaInfo != null) {
        if (subscribe.hasSchema()) {
            throw new IllegalStateException();
        }
        if (subscribe.setSchema().getPropertiesCount() > 0) {
            throw new IllegalStateException();
        }
        convertSchema(schemaInfo, subscribe.setSchema());
    }
    return serializeWithSize(cmd);
}