Examples with KeyValue org.apache.pulsar.common.api.proto.PulsarApi.KeyValue used on opensource projects

Example 1 with KeyValue

use of org.apache.pulsar.common.api.proto.PulsarApi.KeyValue in project incubator-pulsar by apache.

the class PersistentTopicsImpl method getIndividualMsgsFromBatch.

private List<Message<byte[]>> getIndividualMsgsFromBatch(String msgId, byte[] data, Map<String, String> properties) {
    List<Message<byte[]>> ret = new ArrayList<>();
    int batchSize = Integer.parseInt(properties.get(BATCH_HEADER));
    for (int i = 0; i < batchSize; i++) {
        String batchMsgId = msgId + ":" + i;
        PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata.newBuilder();
        ByteBuf buf = Unpooled.wrappedBuffer(data);
        try {
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadataBuilder, i, batchSize);
            SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.build();
            if (singleMessageMetadata.getPropertiesCount() > 0) {
                for (KeyValue entry : singleMessageMetadata.getPropertiesList()) {
                    properties.put(entry.getKey(), entry.getValue());
                }
            }
            ret.add(new MessageImpl<>(batchMsgId, properties, singleMessagePayload, Schema.IDENTITY));
        } catch (Exception ex) {
            log.error("Exception occured while trying to get BatchMsgId: {}", batchMsgId, ex);
        }
        buf.release();
        singleMessageMetadataBuilder.recycle();
    }
    return ret;
}

Example 2 with KeyValue

use of org.apache.pulsar.common.api.proto.PulsarApi.KeyValue in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalPeekNthMessage.

protected Response internalPeekNthMessage(String subName, int messagePosition, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnTopic(authoritative);
    if (!(getTopicReference(topicName) instanceof PersistentTopic)) {
        log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), topicName, subName);
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Skip messages on a non-persistent topic is not allowed");
    }
    PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = (PersistentSubscription) getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DateFormatter.format(metadata.getPublishTime()));
        }
        if (metadata.hasEventTime()) {
            responseBuilder.header("X-Pulsar-event-time", DateFormatter.format(metadata.getEventTime()));
        }
        if (metadata.hasNumMessagesInBatch()) {
            responseBuilder.header("X-Pulsar-num-batch-message", metadata.getNumMessagesInBatch());
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, topicName, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 3 with KeyValue

use of org.apache.pulsar.common.api.proto.PulsarApi.KeyValue in project incubator-pulsar by apache.

the class MessageCrypto method encrypt.

/*
     * Encrypt the payload using the data key and update message metadata with the keyname & encrypted data key
     *
     * @param encKeys One or more public keys to encrypt data key
     *
     * @param msgMetadata Message Metadata
     *
     * @param payload Message which needs to be encrypted
     *
     * @return encryptedData if success
     */
public synchronized ByteBuf encrypt(Set<String> encKeys, CryptoKeyReader keyReader, MessageMetadata.Builder msgMetadata, ByteBuf payload) throws PulsarClientException {
    if (encKeys.isEmpty()) {
        return payload;
    }
    // Update message metadata with encrypted data key
    for (String keyName : encKeys) {
        if (encryptedDataKeyMap.get(keyName) == null) {
            // Attempt to load the key. This will allow us to load keys as soon as
            // a new key is added to producer config
            addPublicKeyCipher(keyName, keyReader);
        }
        EncryptionKeyInfo keyInfo = encryptedDataKeyMap.get(keyName);
        if (keyInfo != null) {
            if (keyInfo.getMetadata() != null && !keyInfo.getMetadata().isEmpty()) {
                List<KeyValue> kvList = new ArrayList<KeyValue>();
                keyInfo.getMetadata().forEach((key, value) -> {
                    kvList.add(KeyValue.newBuilder().setKey(key).setValue(value).build());
                });
                msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName).setValue(ByteString.copyFrom(keyInfo.getKey())).addAllMetadata(kvList).build());
            } else {
                msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName).setValue(ByteString.copyFrom(keyInfo.getKey())).build());
            }
        } else {
            // We should never reach here.
            log.error("{} Failed to find encrypted Data key for key {}.", logCtx, keyName);
        }
    }
    // Create gcm param
    // TODO: Replace random with counter and periodic refreshing based on timer/counter value
    secureRandom.nextBytes(iv);
    GCMParameterSpec gcmParam = new GCMParameterSpec(tagLen, iv);
    // Update message metadata with encryption param
    msgMetadata.setEncryptionParam(ByteString.copyFrom(iv));
    ByteBuf targetBuf = null;
    try {
        // Encrypt the data
        cipher.init(Cipher.ENCRYPT_MODE, dataKey, gcmParam);
        ByteBuffer sourceNioBuf = payload.nioBuffer(payload.readerIndex(), payload.readableBytes());
        int maxLength = cipher.getOutputSize(payload.readableBytes());
        targetBuf = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength);
        ByteBuffer targetNioBuf = targetBuf.nioBuffer(0, maxLength);
        int bytesStored = cipher.doFinal(sourceNioBuf, targetNioBuf);
        targetBuf.writerIndex(bytesStored);
    } catch (IllegalBlockSizeException | BadPaddingException | InvalidKeyException | InvalidAlgorithmParameterException | ShortBufferException e) {
        targetBuf.release();
        log.error("{} Failed to encrypt message. {}", logCtx, e);
        throw new PulsarClientException.CryptoException(e.getMessage());
    }
    payload.release();
    return targetBuf;
}

Example 4 with KeyValue

use of org.apache.pulsar.common.api.proto.PulsarApi.KeyValue in project incubator-pulsar by apache.

the class MessageCrypto method decrypt.

/*
     * Decrypt the payload using the data key. Keys used to encrypt data key can be retrieved from msgMetadata
     *
     * @param msgMetadata Message Metadata
     *
     * @param payload Message which needs to be decrypted
     *
     * @param keyReader KeyReader implementation to retrieve key value
     *
     * @return decryptedData if success, null otherwise
     */
public ByteBuf decrypt(MessageMetadata msgMetadata, ByteBuf payload, CryptoKeyReader keyReader) {
    // If dataKey is present, attempt to decrypt using the existing key
    if (dataKey != null) {
        ByteBuf decryptedData = getKeyAndDecryptData(msgMetadata, payload);
        // If decryption succeeded, data is non null
        if (decryptedData != null) {
            return decryptedData;
        }
    }
    // dataKey is null or decryption failed. Attempt to regenerate data key
    List<EncryptionKeys> encKeys = msgMetadata.getEncryptionKeysList();
    EncryptionKeys encKeyInfo = encKeys.stream().filter(kbv -> {
        byte[] encDataKey = kbv.getValue().toByteArray();
        List<KeyValue> encKeyMeta = kbv.getMetadataList();
        return decryptDataKey(kbv.getKey(), encDataKey, encKeyMeta, keyReader);
    }).findFirst().orElse(null);
    if (encKeyInfo == null || dataKey == null) {
        // Unable to decrypt data key
        return null;
    }
    return getKeyAndDecryptData(msgMetadata, payload);
}