Examples with MultipartCryptoKeyInputStream org.apache.hadoop.ozone.client.io.MultipartCryptoKeyInputStream used on opensource projects

Example 1 with MultipartCryptoKeyInputStream

use of org.apache.hadoop.ozone.client.io.MultipartCryptoKeyInputStream in project ozone by apache.

the class RpcClient method createInputStream.

private OzoneInputStream createInputStream(OmKeyInfo keyInfo, Function<OmKeyInfo, OmKeyInfo> retryFunction) throws IOException {
    // When Key is not MPU or when Key is MPU and encryption is not enabled
    // Need to revisit for GDP.
    FileEncryptionInfo feInfo = keyInfo.getFileEncryptionInfo();
    if (feInfo == null) {
        LengthInputStream lengthInputStream = KeyInputStream.getFromOmKeyInfo(keyInfo, xceiverClientManager, clientConfig.isChecksumVerify(), retryFunction);
        try {
            Map<String, String> keyInfoMetadata = keyInfo.getMetadata();
            if (Boolean.valueOf(keyInfoMetadata.get(OzoneConsts.GDPR_FLAG))) {
                GDPRSymmetricKey gk = new GDPRSymmetricKey(keyInfoMetadata.get(OzoneConsts.GDPR_SECRET), keyInfoMetadata.get(OzoneConsts.GDPR_ALGORITHM));
                gk.getCipher().init(Cipher.DECRYPT_MODE, gk.getSecretKey());
                return new OzoneInputStream(new CipherInputStream(lengthInputStream, gk.getCipher()));
            }
        } catch (Exception ex) {
            throw new IOException(ex);
        }
        return new OzoneInputStream(lengthInputStream.getWrappedStream());
    } else if (!keyInfo.getLatestVersionLocations().isMultipartKey()) {
        // Regular Key with FileEncryptionInfo
        LengthInputStream lengthInputStream = KeyInputStream.getFromOmKeyInfo(keyInfo, xceiverClientManager, clientConfig.isChecksumVerify(), retryFunction);
        final KeyProvider.KeyVersion decrypted = getDEK(feInfo);
        final CryptoInputStream cryptoIn = new CryptoInputStream(lengthInputStream.getWrappedStream(), OzoneKMSUtil.getCryptoCodec(conf, feInfo), decrypted.getMaterial(), feInfo.getIV());
        return new OzoneInputStream(cryptoIn);
    } else {
        // Multipart Key with FileEncryptionInfo
        List<LengthInputStream> lengthInputStreams = KeyInputStream.getStreamsFromKeyInfo(keyInfo, xceiverClientManager, clientConfig.isChecksumVerify(), retryFunction);
        final KeyProvider.KeyVersion decrypted = getDEK(feInfo);
        List<OzoneCryptoInputStream> cryptoInputStreams = new ArrayList<>();
        for (LengthInputStream lengthInputStream : lengthInputStreams) {
            final OzoneCryptoInputStream ozoneCryptoInputStream = new OzoneCryptoInputStream(lengthInputStream, OzoneKMSUtil.getCryptoCodec(conf, feInfo), decrypted.getMaterial(), feInfo.getIV());
            cryptoInputStreams.add(ozoneCryptoInputStream);
        }
        return new MultipartCryptoKeyInputStream(keyInfo.getKeyName(), cryptoInputStreams);
    }
}

Example 2 with MultipartCryptoKeyInputStream

use of org.apache.hadoop.ozone.client.io.MultipartCryptoKeyInputStream in project ozone by apache.

the class TestOzoneAtRestEncryption method testMultipartUploadWithEncryption.

public void testMultipartUploadWithEncryption(OzoneBucket bucket, int numParts) throws Exception {
    String keyName = "mpu_test_key_" + numParts;
    // Initiate multipart upload
    String uploadID = initiateMultipartUpload(bucket, keyName, RATIS, ONE);
    // Upload Parts
    Map<Integer, String> partsMap = new TreeMap<>();
    List<byte[]> partsData = new ArrayList<>();
    int keySize = 0;
    for (int i = 1; i <= numParts; i++) {
        // Generate random data with different sizes for each part.
        // Adding a random int with a cap at 8K (the default crypto buffer
        // size) to get parts whose last byte does not coincide with crypto
        // buffer boundary.
        int partSize = (MPU_PART_MIN_SIZE * i) + RANDOM.nextInt(DEFAULT_CRYPTO_BUFFER_SIZE - 1) + 1;
        byte[] data = generateRandomData(partSize);
        String partName = uploadPart(bucket, keyName, uploadID, i, data);
        partsMap.put(i, partName);
        partsData.add(data);
        keySize += data.length;
    }
    // Combine the parts data into 1 byte array for verification
    byte[] inputData = new byte[keySize];
    int dataCopied = 0;
    for (int i = 1; i <= numParts; i++) {
        byte[] partBytes = partsData.get(i - 1);
        System.arraycopy(partBytes, 0, inputData, dataCopied, partBytes.length);
        dataCopied += partBytes.length;
    }
    // Complete MPU
    completeMultipartUpload(bucket, keyName, uploadID, partsMap);
    // Create an input stream to read the data
    OzoneInputStream inputStream = bucket.readKey(keyName);
    Assert.assertTrue(inputStream instanceof MultipartCryptoKeyInputStream);
    // Test complete read
    byte[] completeRead = new byte[keySize];
    int bytesRead = inputStream.read(completeRead, 0, keySize);
    Assert.assertEquals(bytesRead, keySize);
    Assert.assertArrayEquals(inputData, completeRead);
    // Read different data lengths and starting from different offsets and
    // verify the data matches.
    Random random = new Random();
    int randomSize = random.nextInt(keySize / 2);
    int randomOffset = random.nextInt(keySize - randomSize);
    int[] readDataSizes = { keySize, keySize / 3 + 1, BLOCK_SIZE, BLOCK_SIZE * 2 + 1, CHUNK_SIZE, CHUNK_SIZE / 4 - 1, DEFAULT_CRYPTO_BUFFER_SIZE, DEFAULT_CRYPTO_BUFFER_SIZE / 2, 1, randomSize };
    int[] readFromPositions = { 0, DEFAULT_CRYPTO_BUFFER_SIZE + 10, CHUNK_SIZE, BLOCK_SIZE - DEFAULT_CRYPTO_BUFFER_SIZE + 1, BLOCK_SIZE, keySize / 3, keySize - 1, randomOffset };
    for (int readDataLen : readDataSizes) {
        for (int readFromPosition : readFromPositions) {
            // Check that offset + buffer size does not exceed the key size
            if (readFromPosition + readDataLen > keySize) {
                continue;
            }
            byte[] readData = new byte[readDataLen];
            inputStream.seek(readFromPosition);
            int actualReadLen = inputStream.read(readData, 0, readDataLen);
            assertReadContent(inputData, readData, readFromPosition);
            Assert.assertEquals(readFromPosition + readDataLen, inputStream.getPos());
            Assert.assertEquals(readDataLen, actualReadLen);
        }
    }
}