Examples with ColumnEncryptionProperties org.apache.parquet.crypto.ColumnEncryptionProperties used on opensource projects

Example 6 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class EncDecProperties method getFileEncryptionProperties.

public static FileEncryptionProperties getFileEncryptionProperties(String[] encryptColumns, ParquetCipher cipher, Boolean encryptFooter) {
    if (encryptColumns.length == 0) {
        return null;
    }
    Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap = new HashMap<>();
    for (String encryptColumn : encryptColumns) {
        ColumnPath columnPath = ColumnPath.fromDotString(encryptColumn);
        ColumnEncryptionProperties columnEncryptionProperties = ColumnEncryptionProperties.builder(columnPath).withKey(COL_KEY).withKeyMetaData(COL_KEY_METADATA).build();
        columnPropertyMap.put(columnPath, columnEncryptionProperties);
    }
    FileEncryptionProperties.Builder encryptionPropertiesBuilder = FileEncryptionProperties.builder(FOOTER_KEY).withFooterKeyMetadata(FOOTER_KEY_METADATA).withAlgorithm(cipher).withEncryptedColumns(columnPropertyMap);
    if (!encryptFooter) {
        encryptionPropertiesBuilder.withPlaintextFooter();
    }
    return encryptionPropertiesBuilder.build();
}

Example 7 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class SchemaCryptoPropertiesFactory method getColumnEncryptionProperties.

private void getColumnEncryptionProperties(String[] path, Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap, Configuration conf) throws ParquetCryptoRuntimeException {
    String pathName = String.join(".", path);
    String columnKeyName = conf.get(PATH_NAME_PREFIX + pathName, null);
    if (columnKeyName != null) {
        ColumnPath columnPath = ColumnPath.get(path);
        ColumnEncryptionProperties colEncProp = ColumnEncryptionProperties.builder(columnPath).withKey(COL_KEY).withKeyMetaData(COL_KEY_METADATA).build();
        columnPropertyMap.put(columnPath, colEncProp);
    }
}

Example 8 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class PropertiesDrivenCryptoFactory method getFileEncryptionProperties.

@Override
public FileEncryptionProperties getFileEncryptionProperties(Configuration fileHadoopConfig, Path tempFilePath, WriteContext fileWriteContext) throws ParquetCryptoRuntimeException {
    String footerKeyId = fileHadoopConfig.getTrimmed(FOOTER_KEY_PROPERTY_NAME);
    String columnKeysStr = fileHadoopConfig.getTrimmed(COLUMN_KEYS_PROPERTY_NAME);
    String uniformKeyId = fileHadoopConfig.getTrimmed(UNIFORM_KEY_PROPERTY_NAME);
    boolean emptyFooterKeyId = stringIsEmpty(footerKeyId);
    boolean emptyColumnKeyIds = stringIsEmpty(columnKeysStr);
    boolean emptyUniformKeyId = stringIsEmpty(uniformKeyId);
    // File shouldn't be encrypted
    if (emptyFooterKeyId && emptyColumnKeyIds && emptyUniformKeyId) {
        LOG.debug("Unencrypted file: {}", tempFilePath);
        return null;
    }
    if (emptyUniformKeyId) {
        // Non-uniform encryption.Must have both footer and column key ids
        if (emptyFooterKeyId) {
            throw new ParquetCryptoRuntimeException("No footer key configured in " + FOOTER_KEY_PROPERTY_NAME);
        }
        if (emptyColumnKeyIds) {
            throw new ParquetCryptoRuntimeException("No column keys configured in " + COLUMN_KEYS_PROPERTY_NAME);
        }
    } else {
        // Uniform encryption. Can't have configuration of footer and column key ids
        if (!emptyFooterKeyId) {
            throw new ParquetCryptoRuntimeException("Uniform encryption. Cant have footer key configured in " + FOOTER_KEY_PROPERTY_NAME);
        }
        if (!emptyColumnKeyIds) {
            throw new ParquetCryptoRuntimeException("Uniform encryption. Cant have column keys configured in " + COLUMN_KEYS_PROPERTY_NAME);
        }
        // Now assign footer key id to uniform key id
        footerKeyId = uniformKeyId;
    }
    FileKeyMaterialStore keyMaterialStore = null;
    boolean keyMaterialInternalStorage = fileHadoopConfig.getBoolean(KeyToolkit.KEY_MATERIAL_INTERNAL_PROPERTY_NAME, KeyToolkit.KEY_MATERIAL_INTERNAL_DEFAULT);
    if (!keyMaterialInternalStorage) {
        if (tempFilePath == null) {
            throw new ParquetCryptoRuntimeException("Output file path cannot be null");
        }
        try {
            keyMaterialStore = new HadoopFSKeyMaterialStore(tempFilePath.getFileSystem(fileHadoopConfig));
            keyMaterialStore.initialize(tempFilePath, fileHadoopConfig, false);
        } catch (IOException e) {
            throw new ParquetCryptoRuntimeException("Failed to get key material store", e);
        }
    }
    FileKeyWrapper keyWrapper = new FileKeyWrapper(fileHadoopConfig, keyMaterialStore);
    String algo = fileHadoopConfig.getTrimmed(ENCRYPTION_ALGORITHM_PROPERTY_NAME, ENCRYPTION_ALGORITHM_DEFAULT);
    ParquetCipher cipher;
    try {
        cipher = ParquetCipher.valueOf(algo);
    } catch (IllegalArgumentException e) {
        throw new ParquetCryptoRuntimeException("Wrong encryption algorithm: " + algo);
    }
    int dekLengthBits = fileHadoopConfig.getInt(KeyToolkit.DATA_KEY_LENGTH_PROPERTY_NAME, KeyToolkit.DATA_KEY_LENGTH_DEFAULT);
    if (Arrays.binarySearch(ACCEPTABLE_DATA_KEY_LENGTHS, dekLengthBits) < 0) {
        throw new ParquetCryptoRuntimeException("Wrong data key length : " + dekLengthBits);
    }
    int dekLength = dekLengthBits / 8;
    byte[] footerKeyBytes = new byte[dekLength];
    RANDOM.nextBytes(footerKeyBytes);
    byte[] footerKeyMetadata = keyWrapper.getEncryptionKeyMetadata(footerKeyBytes, footerKeyId, true);
    boolean plaintextFooter = fileHadoopConfig.getBoolean(PLAINTEXT_FOOTER_PROPERTY_NAME, PLAINTEXT_FOOTER_DEFAULT);
    FileEncryptionProperties.Builder propertiesBuilder = FileEncryptionProperties.builder(footerKeyBytes).withFooterKeyMetadata(footerKeyMetadata).withAlgorithm(cipher);
    if (emptyUniformKeyId) {
        Map<ColumnPath, ColumnEncryptionProperties> encryptedColumns = getColumnEncryptionProperties(dekLength, columnKeysStr, keyWrapper);
        propertiesBuilder = propertiesBuilder.withEncryptedColumns(encryptedColumns);
    }
    if (plaintextFooter) {
        propertiesBuilder = propertiesBuilder.withPlaintextFooter();
    }
    if (null != keyMaterialStore) {
        keyMaterialStore.saveMaterial();
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("File encryption properties for {} - algo: {}; footer key id: {}; uniform key id: {}; " + "" + "plaintext footer: {}; internal key material: {}; encrypted columns: {}", tempFilePath, cipher, footerKeyId, uniformKeyId, plaintextFooter, keyMaterialInternalStorage, columnKeysStr);
    }
    return propertiesBuilder.build();
}