Examples with EncryptionKeys org.apache.pulsar.common.api.proto.EncryptionKeys used on opensource projects

Example 1 with EncryptionKeys

use of org.apache.pulsar.common.api.proto.EncryptionKeys in project pulsar by apache.

the class TopicsImpl method getMessagesFromHttpResponse.

private List<Message<byte[]>> getMessagesFromHttpResponse(String topic, Response response) throws Exception {
    if (response.getStatus() != Status.OK.getStatusCode()) {
        throw getApiException(response);
    }
    String msgId = response.getHeaderString(MESSAGE_ID);
    // build broker entry metadata if exist
    String brokerEntryTimestamp = response.getHeaderString(BROKER_ENTRY_TIMESTAMP);
    String brokerEntryIndex = response.getHeaderString(BROKER_ENTRY_INDEX);
    BrokerEntryMetadata brokerEntryMetadata;
    if (brokerEntryTimestamp == null && brokerEntryIndex == null) {
        brokerEntryMetadata = null;
    } else {
        brokerEntryMetadata = new BrokerEntryMetadata();
        if (brokerEntryTimestamp != null) {
            brokerEntryMetadata.setBrokerTimestamp(DateFormatter.parse(brokerEntryTimestamp));
        }
        if (brokerEntryIndex != null) {
            brokerEntryMetadata.setIndex(Long.parseLong(brokerEntryIndex));
        }
    }
    MessageMetadata messageMetadata = new MessageMetadata();
    try (InputStream stream = (InputStream) response.getEntity()) {
        byte[] data = new byte[stream.available()];
        stream.read(data);
        Map<String, String> properties = new TreeMap<>();
        MultivaluedMap<String, Object> headers = response.getHeaders();
        Object tmp = headers.getFirst(PUBLISH_TIME);
        if (tmp != null) {
            messageMetadata.setPublishTime(DateFormatter.parse(tmp.toString()));
        }
        tmp = headers.getFirst(EVENT_TIME);
        if (tmp != null) {
            messageMetadata.setEventTime(DateFormatter.parse(tmp.toString()));
        }
        tmp = headers.getFirst(DELIVER_AT_TIME);
        if (tmp != null) {
            messageMetadata.setDeliverAtTime(DateFormatter.parse(tmp.toString()));
        }
        tmp = headers.getFirst("X-Pulsar-null-value");
        if (tmp != null) {
            messageMetadata.setNullValue(Boolean.parseBoolean(tmp.toString()));
        }
        tmp = headers.getFirst(PRODUCER_NAME);
        if (tmp != null) {
            messageMetadata.setProducerName(tmp.toString());
        }
        tmp = headers.getFirst(SEQUENCE_ID);
        if (tmp != null) {
            messageMetadata.setSequenceId(Long.parseLong(tmp.toString()));
        }
        tmp = headers.getFirst(REPLICATED_FROM);
        if (tmp != null) {
            messageMetadata.setReplicatedFrom(tmp.toString());
        }
        tmp = headers.getFirst(PARTITION_KEY);
        if (tmp != null) {
            messageMetadata.setPartitionKey(tmp.toString());
        }
        tmp = headers.getFirst(COMPRESSION);
        if (tmp != null) {
            messageMetadata.setCompression(CompressionType.valueOf(tmp.toString()));
        }
        tmp = headers.getFirst(UNCOMPRESSED_SIZE);
        if (tmp != null) {
            messageMetadata.setUncompressedSize(Integer.parseInt(tmp.toString()));
        }
        tmp = headers.getFirst(ENCRYPTION_ALGO);
        if (tmp != null) {
            messageMetadata.setEncryptionAlgo(tmp.toString());
        }
        tmp = headers.getFirst(PARTITION_KEY_B64_ENCODED);
        if (tmp != null) {
            messageMetadata.setPartitionKeyB64Encoded(Boolean.parseBoolean(tmp.toString()));
        }
        tmp = headers.getFirst(MARKER_TYPE);
        if (tmp != null) {
            messageMetadata.setMarkerType(Integer.parseInt(tmp.toString()));
        }
        tmp = headers.getFirst(TXNID_LEAST_BITS);
        if (tmp != null) {
            messageMetadata.setTxnidLeastBits(Long.parseLong(tmp.toString()));
        }
        tmp = headers.getFirst(TXNID_MOST_BITS);
        if (tmp != null) {
            messageMetadata.setTxnidMostBits(Long.parseLong(tmp.toString()));
        }
        tmp = headers.getFirst(HIGHEST_SEQUENCE_ID);
        if (tmp != null) {
            messageMetadata.setHighestSequenceId(Long.parseLong(tmp.toString()));
        }
        tmp = headers.getFirst(UUID);
        if (tmp != null) {
            messageMetadata.setUuid(tmp.toString());
        }
        tmp = headers.getFirst(NUM_CHUNKS_FROM_MSG);
        if (tmp != null) {
            messageMetadata.setNumChunksFromMsg(Integer.parseInt(tmp.toString()));
        }
        tmp = headers.getFirst(TOTAL_CHUNK_MSG_SIZE);
        if (tmp != null) {
            messageMetadata.setTotalChunkMsgSize(Integer.parseInt(tmp.toString()));
        }
        tmp = headers.getFirst(CHUNK_ID);
        if (tmp != null) {
            messageMetadata.setChunkId(Integer.parseInt(tmp.toString()));
        }
        tmp = headers.getFirst(NULL_PARTITION_KEY);
        if (tmp != null) {
            messageMetadata.setNullPartitionKey(Boolean.parseBoolean(tmp.toString()));
        }
        tmp = headers.getFirst(ENCRYPTION_PARAM);
        if (tmp != null) {
            messageMetadata.setEncryptionParam(Base64.getDecoder().decode(tmp.toString()));
        }
        tmp = headers.getFirst(ORDERING_KEY);
        if (tmp != null) {
            messageMetadata.setOrderingKey(Base64.getDecoder().decode(tmp.toString()));
        }
        tmp = headers.getFirst(SCHEMA_VERSION);
        if (tmp != null) {
            messageMetadata.setSchemaVersion(Base64.getDecoder().decode(tmp.toString()));
        }
        tmp = headers.getFirst(ENCRYPTION_PARAM);
        if (tmp != null) {
            messageMetadata.setEncryptionParam(Base64.getDecoder().decode(tmp.toString()));
        }
        List<Object> tmpList = headers.get(REPLICATED_TO);
        if (tmpList != null) {
            for (Object o : tmpList) {
                messageMetadata.addReplicateTo(o.toString());
            }
        }
        tmpList = headers.get(ENCRYPTION_KEYS);
        if (tmpList != null) {
            for (Object o : tmpList) {
                EncryptionKeys encryptionKey = messageMetadata.addEncryptionKey();
                encryptionKey.parseFrom(Base64.getDecoder().decode(o.toString()));
            }
        }
        tmp = headers.getFirst(BATCH_SIZE_HEADER);
        if (tmp != null) {
            properties.put(BATCH_SIZE_HEADER, (String) tmp);
        }
        for (Entry<String, List<Object>> entry : headers.entrySet()) {
            String header = entry.getKey();
            if (header.contains("X-Pulsar-PROPERTY-")) {
                String keyName = header.substring("X-Pulsar-PROPERTY-".length());
                properties.put(keyName, (String) entry.getValue().get(0));
            }
        }
        tmp = headers.getFirst(BATCH_HEADER);
        if (tmp != null) {
            properties.put(BATCH_HEADER, (String) tmp);
        }
        boolean isEncrypted = false;
        tmp = headers.getFirst("X-Pulsar-Is-Encrypted");
        if (tmp != null) {
            isEncrypted = Boolean.parseBoolean(tmp.toString());
        }
        if (!isEncrypted && response.getHeaderString(BATCH_HEADER) != null) {
            return getIndividualMsgsFromBatch(topic, msgId, data, properties, messageMetadata, brokerEntryMetadata);
        }
        MessageImpl message = new MessageImpl(topic, msgId, properties, Unpooled.wrappedBuffer(data), Schema.BYTES, messageMetadata);
        if (brokerEntryMetadata != null) {
            message.setBrokerEntryMetadata(brokerEntryMetadata);
        }
        return Collections.singletonList(message);
    }
}

Example 2 with EncryptionKeys

use of org.apache.pulsar.common.api.proto.EncryptionKeys in project pulsar by apache.

the class MessageCryptoBc method decrypt.

/*
     * Decrypt the payload using the data key. Keys used to encrypt data key can be retrieved from msgMetadata
     *
     * @param msgMetadata Message Metadata
     *
     * @param payload Message which needs to be decrypted
     *
     * @param keyReader KeyReader implementation to retrieve key value
     *
     * @return true if success, false otherwise
     */
@Override
public boolean decrypt(Supplier<MessageMetadata> messageMetadataSupplier, ByteBuffer payload, ByteBuffer outBuffer, CryptoKeyReader keyReader) {
    MessageMetadata msgMetadata = messageMetadataSupplier.get();
    // If dataKey is present, attempt to decrypt using the existing key
    if (dataKey != null) {
        if (getKeyAndDecryptData(msgMetadata, payload, outBuffer)) {
            return true;
        }
    }
    // dataKey is null or decryption failed. Attempt to regenerate data key
    List<EncryptionKeys> encKeys = msgMetadata.getEncryptionKeysList();
    EncryptionKeys encKeyInfo = encKeys.stream().filter(kbv -> {
        byte[] encDataKey = kbv.getValue();
        List<KeyValue> encKeyMeta = kbv.getMetadatasList();
        return decryptDataKey(kbv.getKey(), encDataKey, encKeyMeta, keyReader);
    }).findFirst().orElse(null);
    if (encKeyInfo == null || dataKey == null) {
        // Unable to decrypt data key
        return false;
    }
    return getKeyAndDecryptData(msgMetadata, payload, outBuffer);
}

Example 3 with EncryptionKeys

use of org.apache.pulsar.common.api.proto.EncryptionKeys in project pulsar by apache.

the class MessageCryptoBc method encrypt.

/*
     * Encrypt the payload using the data key and update message metadata with the keyname & encrypted data key
     *
     * @param encKeys One or more public keys to encrypt data key
     *
     * @param msgMetadata Message Metadata
     *
     * @param payload Message which needs to be encrypted
     *
     * @return encryptedData if success
     */
@Override
public synchronized void encrypt(Set<String> encKeys, CryptoKeyReader keyReader, Supplier<MessageMetadata> messageMetadataBuilderSupplier, ByteBuffer payload, ByteBuffer outBuffer) throws PulsarClientException {
    MessageMetadata msgMetadata = messageMetadataBuilderSupplier.get();
    if (encKeys.isEmpty()) {
        outBuffer.put(payload);
        outBuffer.flip();
        return;
    }
    msgMetadata.clearEncryptionKeys();
    // Update message metadata with encrypted data key
    for (String keyName : encKeys) {
        if (encryptedDataKeyMap.get(keyName) == null) {
            // Attempt to load the key. This will allow us to load keys as soon as
            // a new key is added to producer config
            addPublicKeyCipher(keyName, keyReader);
        }
        EncryptionKeyInfo keyInfo = encryptedDataKeyMap.get(keyName);
        if (keyInfo != null) {
            if (keyInfo.getMetadata() != null && !keyInfo.getMetadata().isEmpty()) {
                EncryptionKeys encKey = msgMetadata.addEncryptionKey().setKey(keyName).setValue(keyInfo.getKey());
                keyInfo.getMetadata().forEach((key, value) -> {
                    encKey.addMetadata().setKey(key).setValue(value);
                });
            } else {
                msgMetadata.addEncryptionKey().setKey(keyName).setValue(keyInfo.getKey());
            }
        } else {
            // We should never reach here.
            log.error("{} Failed to find encrypted Data key for key {}.", logCtx, keyName);
        }
    }
    // Create gcm param
    // TODO: Replace random with counter and periodic refreshing based on timer/counter value
    secureRandom.nextBytes(iv);
    GCMParameterSpec gcmParam = new GCMParameterSpec(tagLen, iv);
    // Update message metadata with encryption param
    msgMetadata.setEncryptionParam(iv);
    try {
        // Encrypt the data
        cipher.init(Cipher.ENCRYPT_MODE, dataKey, gcmParam);
        int maxLength = cipher.getOutputSize(payload.remaining());
        if (outBuffer.remaining() < maxLength) {
            throw new IllegalArgumentException("Outbuffer has not enough space available");
        }
        int bytesStored = cipher.doFinal(payload, outBuffer);
        outBuffer.flip();
        outBuffer.limit(bytesStored);
    } catch (IllegalBlockSizeException | BadPaddingException | InvalidKeyException | InvalidAlgorithmParameterException | ShortBufferException e) {
        log.error("{} Failed to encrypt message. {}", logCtx, e);
        throw new PulsarClientException.CryptoException(e.getMessage());
    }
}

Example 4 with EncryptionKeys

use of org.apache.pulsar.common.api.proto.EncryptionKeys in project pulsar by apache.

the class MessageCryptoBc method getKeyAndDecryptData.

private boolean getKeyAndDecryptData(MessageMetadata msgMetadata, ByteBuffer payload, ByteBuffer targetBuffer) {
    List<EncryptionKeys> encKeys = msgMetadata.getEncryptionKeysList();
    // Go through all keys to retrieve data key from cache
    for (int i = 0; i < encKeys.size(); i++) {
        byte[] msgDataKey = encKeys.get(i).getValue();
        byte[] keyDigest = digest.digest(msgDataKey);
        SecretKey storedSecretKey = dataKeyCache.getIfPresent(ByteBuffer.wrap(keyDigest));
        if (storedSecretKey != null) {
            // call decryptDataKey to refresh the cache and come here again to decrypt.
            if (decryptData(storedSecretKey, msgMetadata, payload, targetBuffer)) {
                // If decryption succeeded, we can already return
                return true;
            }
        } else {
            // First time, entry won't be present in cache
            log.debug("{} Failed to decrypt data or data key is not in cache. Will attempt to refresh", logCtx);
        }
    }
    return false;
}

Example 5 with EncryptionKeys

use of org.apache.pulsar.common.api.proto.EncryptionKeys in project pulsar by apache.

the class ConsumerImpl method createEncryptionContext.

/**
 * Create EncryptionContext if message payload is encrypted.
 *
 * @param msgMetadata
 * @return {@link Optional}<{@link EncryptionContext}>
 */
private Optional<EncryptionContext> createEncryptionContext(MessageMetadata msgMetadata) {
    EncryptionContext encryptionCtx = null;
    if (msgMetadata.getEncryptionKeysCount() > 0) {
        encryptionCtx = new EncryptionContext();
        Map<String, EncryptionKey> keys = msgMetadata.getEncryptionKeysList().stream().collect(Collectors.toMap(EncryptionKeys::getKey, e -> new EncryptionKey(e.getValue(), e.getMetadatasList().stream().collect(Collectors.toMap(KeyValue::getKey, KeyValue::getValue)))));
        byte[] encParam = msgMetadata.getEncryptionParam();
        Optional<Integer> batchSize = Optional.ofNullable(msgMetadata.hasNumMessagesInBatch() ? msgMetadata.getNumMessagesInBatch() : null);
        encryptionCtx.setKeys(keys);
        encryptionCtx.setParam(encParam);
        if (msgMetadata.hasEncryptionAlgo()) {
            encryptionCtx.setAlgorithm(msgMetadata.getEncryptionAlgo());
        }
        encryptionCtx.setCompressionType(CompressionCodecProvider.convertFromWireProtocol(msgMetadata.getCompression()));
        encryptionCtx.setUncompressedMessageSize(msgMetadata.getUncompressedSize());
        encryptionCtx.setBatchSize(batchSize);
    }
    return Optional.ofNullable(encryptionCtx);
}