Examples with ColumnEncryptionProperties org.apache.parquet.crypto.ColumnEncryptionProperties used on opensource projects

Example 1 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class SchemaCryptoPropertiesFactory method getFileEncryptionProperties.

@Override
public FileEncryptionProperties getFileEncryptionProperties(Configuration conf, Path tempFilePath, WriteContext fileWriteContext) throws ParquetCryptoRuntimeException {
    MessageType schema = fileWriteContext.getSchema();
    List<String[]> paths = schema.getPaths();
    if (paths == null || paths.isEmpty()) {
        throw new ParquetCryptoRuntimeException("Null or empty fields is found");
    }
    Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap = new HashMap<>();
    for (String[] path : paths) {
        getColumnEncryptionProperties(path, columnPropertyMap, conf);
    }
    if (columnPropertyMap.size() == 0) {
        log.debug("No column is encrypted. Returning null so that Parquet can skip. Empty properties will cause Parquet exception");
        return null;
    }
    /**
     * Why we still need footerKeyMetadata even withEncryptedFooter as false? According to the
     * 'Plaintext Footer' section of
     * https://github.com/apache/parquet-format/blob/encryption/Encryption.md, the plaintext footer
     * is signed in order to prevent tampering with the FileMetaData contents. So footerKeyMetadata
     * is always needed. This signature will be verified if parquet-mr code is with parquet-1178.
     * Otherwise, it will be ignored.
     */
    boolean shouldEncryptFooter = getEncryptFooter(conf);
    FileEncryptionProperties.Builder encryptionPropertiesBuilder = FileEncryptionProperties.builder(FOOTER_KEY).withFooterKeyMetadata(FOOTER_KEY_METADATA).withAlgorithm(getParquetCipherOrDefault(conf)).withEncryptedColumns(columnPropertyMap);
    if (!shouldEncryptFooter) {
        encryptionPropertiesBuilder = encryptionPropertiesBuilder.withPlaintextFooter();
    }
    FileEncryptionProperties encryptionProperties = encryptionPropertiesBuilder.build();
    log.info("FileEncryptionProperties is built with, algorithm:{}, footerEncrypted:{}", encryptionProperties.getAlgorithm(), encryptionProperties.encryptedFooter());
    return encryptionProperties;
}

Example 2 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class TestColumnIndexFiltering method getFileEncryptionProperties.

private static FileEncryptionProperties getFileEncryptionProperties() {
    ColumnEncryptionProperties columnProperties1 = ColumnEncryptionProperties.builder("id").withKey(COLUMN_ENCRYPTION_KEY1).withKeyID(COLUMN_ENCRYPTION_KEY1_ID).build();
    ColumnEncryptionProperties columnProperties2 = ColumnEncryptionProperties.builder("name").withKey(COLUMN_ENCRYPTION_KEY2).withKeyID(COLUMN_ENCRYPTION_KEY2_ID).build();
    Map<ColumnPath, ColumnEncryptionProperties> columnPropertiesMap = new HashMap<>();
    columnPropertiesMap.put(columnProperties1.getPath(), columnProperties1);
    columnPropertiesMap.put(columnProperties2.getPath(), columnProperties2);
    FileEncryptionProperties encryptionProperties = FileEncryptionProperties.builder(FOOTER_ENCRYPTION_KEY).withFooterKeyID(FOOTER_ENCRYPTION_KEY_ID).withEncryptedColumns(columnPropertiesMap).build();
    return encryptionProperties;
}

Example 3 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class PropertiesDrivenCryptoFactory method getColumnEncryptionProperties.

private Map<ColumnPath, ColumnEncryptionProperties> getColumnEncryptionProperties(int dekLength, String columnKeys, FileKeyWrapper keyWrapper) throws ParquetCryptoRuntimeException {
    Map<ColumnPath, ColumnEncryptionProperties> encryptedColumns = new HashMap<ColumnPath, ColumnEncryptionProperties>();
    String[] keyToColumns = columnKeys.split(";");
    for (int i = 0; i < keyToColumns.length; ++i) {
        final String curKeyToColumns = keyToColumns[i].trim();
        if (curKeyToColumns.isEmpty()) {
            continue;
        }
        String[] parts = curKeyToColumns.split(":");
        if (parts.length != 2) {
            throw new ParquetCryptoRuntimeException("Incorrect key to columns mapping in " + COLUMN_KEYS_PROPERTY_NAME + ": [" + curKeyToColumns + "]");
        }
        String columnKeyId = parts[0].trim();
        if (columnKeyId.isEmpty()) {
            throw new ParquetCryptoRuntimeException("Empty key name in " + COLUMN_KEYS_PROPERTY_NAME);
        }
        String columnNamesStr = parts[1].trim();
        String[] columnNames = columnNamesStr.split(",");
        if (0 == columnNames.length) {
            throw new ParquetCryptoRuntimeException("No columns to encrypt defined for key: " + columnKeyId);
        }
        for (int j = 0; j < columnNames.length; ++j) {
            final String columnName = columnNames[j].trim();
            if (columnName.isEmpty()) {
                throw new ParquetCryptoRuntimeException("Empty column name in " + COLUMN_KEYS_PROPERTY_NAME + " for key: " + columnKeyId);
            }
            final ColumnPath columnPath = ColumnPath.fromDotString(columnName);
            if (encryptedColumns.containsKey(columnPath)) {
                throw new ParquetCryptoRuntimeException("Multiple keys defined for the same column: " + columnName);
            }
            byte[] columnKeyBytes = new byte[dekLength];
            RANDOM.nextBytes(columnKeyBytes);
            byte[] columnKeyKeyMetadata = keyWrapper.getEncryptionKeyMetadata(columnKeyBytes, columnKeyId, false);
            ColumnEncryptionProperties cmd = ColumnEncryptionProperties.builder(columnPath).withKey(columnKeyBytes).withKeyMetaData(columnKeyKeyMetadata).build();
            encryptedColumns.put(columnPath, cmd);
        }
    }
    if (encryptedColumns.isEmpty()) {
        throw new ParquetCryptoRuntimeException("No column keys configured in " + COLUMN_KEYS_PROPERTY_NAME);
    }
    return encryptedColumns;
}

Example 4 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class TestEncryptionOptions method getColumnEncryptionPropertiesMap.

private static Map<ColumnPath, ColumnEncryptionProperties> getColumnEncryptionPropertiesMap() {
    Map<ColumnPath, ColumnEncryptionProperties> columnPropertiesMap = new HashMap<>();
    ColumnEncryptionProperties columnPropertiesDouble = ColumnEncryptionProperties.builder(SingleRow.DOUBLE_FIELD_NAME).withKey(COLUMN_ENCRYPTION_KEYS[0]).withKeyID(COLUMN_ENCRYPTION_KEY_IDS[0]).build();
    columnPropertiesMap.put(columnPropertiesDouble.getPath(), columnPropertiesDouble);
    ColumnEncryptionProperties columnPropertiesFloat = ColumnEncryptionProperties.builder(SingleRow.FLOAT_FIELD_NAME).withKey(COLUMN_ENCRYPTION_KEYS[1]).withKeyID(COLUMN_ENCRYPTION_KEY_IDS[1]).build();
    columnPropertiesMap.put(columnPropertiesFloat.getPath(), columnPropertiesFloat);
    ColumnEncryptionProperties columnPropertiesBool = ColumnEncryptionProperties.builder(SingleRow.BOOLEAN_FIELD_NAME).withKey(COLUMN_ENCRYPTION_KEYS[2]).withKeyID(COLUMN_ENCRYPTION_KEY_IDS[2]).build();
    columnPropertiesMap.put(columnPropertiesBool.getPath(), columnPropertiesBool);
    ColumnEncryptionProperties columnPropertiesInt32 = ColumnEncryptionProperties.builder(SingleRow.INT32_FIELD_NAME).withKey(COLUMN_ENCRYPTION_KEYS[3]).withKeyID(COLUMN_ENCRYPTION_KEY_IDS[3]).build();
    columnPropertiesMap.put(columnPropertiesInt32.getPath(), columnPropertiesInt32);
    ColumnEncryptionProperties columnPropertiesBinary = ColumnEncryptionProperties.builder(SingleRow.BINARY_FIELD_NAME).withKey(COLUMN_ENCRYPTION_KEYS[4]).withKeyID(COLUMN_ENCRYPTION_KEY_IDS[4]).build();
    columnPropertiesMap.put(columnPropertiesBinary.getPath(), columnPropertiesBinary);
    ColumnEncryptionProperties columnPropertiesFixed = ColumnEncryptionProperties.builder(SingleRow.FIXED_LENGTH_BINARY_FIELD_NAME).withKey(COLUMN_ENCRYPTION_KEYS[5]).withKeyID(COLUMN_ENCRYPTION_KEY_IDS[5]).build();
    columnPropertiesMap.put(columnPropertiesFixed.getPath(), columnPropertiesFixed);
    return columnPropertiesMap;
}

Example 5 with ColumnEncryptionProperties

use of org.apache.parquet.crypto.ColumnEncryptionProperties in project parquet-mr by apache.

the class EncDecProperties method getFileEncryptionProperties.

public static FileEncryptionProperties getFileEncryptionProperties(String[] encryptColumns, ParquetCipher cipher, Boolean encryptFooter) {
    if (encryptColumns.length == 0) {
        return null;
    }
    Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap = new HashMap<>();
    for (String encryptColumn : encryptColumns) {
        ColumnPath columnPath = ColumnPath.fromDotString(encryptColumn);
        ColumnEncryptionProperties columnEncryptionProperties = ColumnEncryptionProperties.builder(columnPath).withKey(COL_KEY).withKeyMetaData(COL_KEY_METADATA).build();
        columnPropertyMap.put(columnPath, columnEncryptionProperties);
    }
    FileEncryptionProperties.Builder encryptionPropertiesBuilder = FileEncryptionProperties.builder(FOOTER_KEY).withFooterKeyMetadata(FOOTER_KEY_METADATA).withAlgorithm(cipher).withEncryptedColumns(columnPropertyMap);
    if (!encryptFooter) {
        encryptionPropertiesBuilder.withPlaintextFooter();
    }
    return encryptionPropertiesBuilder.build();
}