Examples with ParquetCryptoRuntimeException org.apache.parquet.crypto.ParquetCryptoRuntimeException used on opensource projects

Example 1 with ParquetCryptoRuntimeException

use of org.apache.parquet.crypto.ParquetCryptoRuntimeException in project parquet-mr by apache.

the class EncryptedColumnChunkMetaData method decryptIfNeeded.

@Override
protected void decryptIfNeeded() {
    if (decrypted)
        return;
    if (null == fileDecryptor) {
        throw new ParquetCryptoRuntimeException(path + ". Null File Decryptor");
    }
    // Decrypt the ColumnMetaData
    InternalColumnDecryptionSetup columnDecryptionSetup = fileDecryptor.setColumnCryptoMetadata(path, true, false, columnKeyMetadata, columnOrdinal);
    ColumnMetaData metaData;
    ByteArrayInputStream tempInputStream = new ByteArrayInputStream(encryptedMetadata);
    byte[] columnMetaDataAAD = AesCipher.createModuleAAD(fileDecryptor.getFileAAD(), ModuleType.ColumnMetaData, rowGroupOrdinal, columnOrdinal, -1);
    try {
        metaData = readColumnMetaData(tempInputStream, columnDecryptionSetup.getMetaDataDecryptor(), columnMetaDataAAD);
    } catch (IOException e) {
        throw new ParquetCryptoRuntimeException(path + ". Failed to decrypt column metadata", e);
    }
    decrypted = true;
    shadowColumnChunkMetaData = parquetMetadataConverter.buildColumnChunkMetaData(metaData, path, primitiveType, createdBy);
    this.encodingStats = shadowColumnChunkMetaData.encodingStats;
    this.properties = shadowColumnChunkMetaData.properties;
    if (metaData.isSetBloom_filter_offset()) {
        setBloomFilterOffset(metaData.getBloom_filter_offset());
    }
}

Example 2 with ParquetCryptoRuntimeException

use of org.apache.parquet.crypto.ParquetCryptoRuntimeException in project parquet-mr by apache.

the class VaultClient method executeAndGetResponse.

private String executeAndGetResponse(String endPoint, Request request) {
    Response response = null;
    try {
        response = httpClient.newCall(request).execute();
        final String responseBody = response.body().string();
        if (response.isSuccessful()) {
            return responseBody;
        } else {
            if ((401 == response.code()) || (403 == response.code())) {
                throw new KeyAccessDeniedException(responseBody);
            }
            throw new IOException("Vault call [" + endPoint + "] didn't succeed: " + responseBody);
        }
    } catch (IOException e) {
        throw new ParquetCryptoRuntimeException("Vault call [" + request.url().toString() + endPoint + "] didn't succeed", e);
    } finally {
        if (null != response) {
            response.close();
        }
    }
}

Example 3 with ParquetCryptoRuntimeException

use of org.apache.parquet.crypto.ParquetCryptoRuntimeException in project parquet-mr by apache.

the class SchemaCryptoPropertiesFactory method getFileEncryptionProperties.

@Override
public FileEncryptionProperties getFileEncryptionProperties(Configuration conf, Path tempFilePath, WriteContext fileWriteContext) throws ParquetCryptoRuntimeException {
    MessageType schema = fileWriteContext.getSchema();
    List<String[]> paths = schema.getPaths();
    if (paths == null || paths.isEmpty()) {
        throw new ParquetCryptoRuntimeException("Null or empty fields is found");
    }
    Map<ColumnPath, ColumnEncryptionProperties> columnPropertyMap = new HashMap<>();
    for (String[] path : paths) {
        getColumnEncryptionProperties(path, columnPropertyMap, conf);
    }
    if (columnPropertyMap.size() == 0) {
        log.debug("No column is encrypted. Returning null so that Parquet can skip. Empty properties will cause Parquet exception");
        return null;
    }
    /**
     * Why we still need footerKeyMetadata even withEncryptedFooter as false? According to the
     * 'Plaintext Footer' section of
     * https://github.com/apache/parquet-format/blob/encryption/Encryption.md, the plaintext footer
     * is signed in order to prevent tampering with the FileMetaData contents. So footerKeyMetadata
     * is always needed. This signature will be verified if parquet-mr code is with parquet-1178.
     * Otherwise, it will be ignored.
     */
    boolean shouldEncryptFooter = getEncryptFooter(conf);
    FileEncryptionProperties.Builder encryptionPropertiesBuilder = FileEncryptionProperties.builder(FOOTER_KEY).withFooterKeyMetadata(FOOTER_KEY_METADATA).withAlgorithm(getParquetCipherOrDefault(conf)).withEncryptedColumns(columnPropertyMap);
    if (!shouldEncryptFooter) {
        encryptionPropertiesBuilder = encryptionPropertiesBuilder.withPlaintextFooter();
    }
    FileEncryptionProperties encryptionProperties = encryptionPropertiesBuilder.build();
    log.info("FileEncryptionProperties is built with, algorithm:{}, footerEncrypted:{}", encryptionProperties.getAlgorithm(), encryptionProperties.encryptedFooter());
    return encryptionProperties;
}

Example 4 with ParquetCryptoRuntimeException

use of org.apache.parquet.crypto.ParquetCryptoRuntimeException in project parquet-mr by apache.

the class TestEncryptionOptions method testInteropReadEncryptedParquetFiles.

private void testInteropReadEncryptedParquetFiles(Path root, boolean readOnlyEncrypted, List<SingleRow> data) throws IOException {
    Configuration conf = new Configuration();
    DecryptionConfiguration[] decryptionConfigurations = DecryptionConfiguration.values();
    for (DecryptionConfiguration decryptionConfiguration : decryptionConfigurations) {
        EncryptionConfiguration[] encryptionConfigurations = EncryptionConfiguration.values();
        for (EncryptionConfiguration encryptionConfiguration : encryptionConfigurations) {
            if (readOnlyEncrypted && (EncryptionConfiguration.NO_ENCRYPTION == encryptionConfiguration)) {
                continue;
            }
            Path file = new Path(root, getFileName(encryptionConfiguration));
            LOG.info("==> Decryption configuration {}", decryptionConfiguration);
            FileDecryptionProperties fileDecryptionProperties = decryptionConfiguration.getDecryptionProperties();
            LOG.info("--> Read file {} {}", file.toString(), encryptionConfiguration);
            // Read only the non-encrypted columns
            if ((decryptionConfiguration == DecryptionConfiguration.NO_DECRYPTION) && (encryptionConfiguration == EncryptionConfiguration.ENCRYPT_COLUMNS_PLAINTEXT_FOOTER)) {
                conf.set("parquet.read.schema", Types.buildMessage().required(BOOLEAN).named(SingleRow.BOOLEAN_FIELD_NAME).required(INT32).named(SingleRow.INT32_FIELD_NAME).named("FormatTestObject").toString());
            }
            int rowNum = 0;
            try (ParquetReader<Group> reader = ParquetReader.builder(new GroupReadSupport(), file).withConf(conf).withDecryption(fileDecryptionProperties).build()) {
                for (Group group = reader.read(); group != null; group = reader.read()) {
                    SingleRow rowExpected = data.get(rowNum++);
                    // plaintext columns
                    if (rowExpected.boolean_field != group.getBoolean(SingleRow.BOOLEAN_FIELD_NAME, 0)) {
                        addErrorToErrorCollectorAndLog("Wrong bool", encryptionConfiguration, decryptionConfiguration);
                    }
                    if (rowExpected.int32_field != group.getInteger(SingleRow.INT32_FIELD_NAME, 0)) {
                        addErrorToErrorCollectorAndLog("Wrong int", encryptionConfiguration, decryptionConfiguration);
                    }
                    // encrypted columns
                    if (decryptionConfiguration != DecryptionConfiguration.NO_DECRYPTION) {
                        if (rowExpected.float_field != group.getFloat(SingleRow.FLOAT_FIELD_NAME, 0)) {
                            addErrorToErrorCollectorAndLog("Wrong float", encryptionConfiguration, decryptionConfiguration);
                        }
                        if (rowExpected.double_field != group.getDouble(SingleRow.DOUBLE_FIELD_NAME, 0)) {
                            addErrorToErrorCollectorAndLog("Wrong double", encryptionConfiguration, decryptionConfiguration);
                        }
                    }
                }
            } catch (ParquetCryptoRuntimeException e) {
                checkResult(file.getName(), decryptionConfiguration, e);
            } catch (Exception e) {
                e.printStackTrace();
                addErrorToErrorCollectorAndLog("Unexpected exception: " + e.getClass().getName() + " with message: " + e.getMessage(), encryptionConfiguration, decryptionConfiguration);
            }
            conf.unset("parquet.read.schema");
        }
    }
}

Example 5 with ParquetCryptoRuntimeException

use of org.apache.parquet.crypto.ParquetCryptoRuntimeException in project parquet-mr by apache.

the class TestEncryptionOptions method testReadEncryptedParquetFiles.

private void testReadEncryptedParquetFiles(Path root, List<SingleRow> data) {
    Configuration conf = new Configuration();
    DecryptionConfiguration[] decryptionConfigurations = DecryptionConfiguration.values();
    for (DecryptionConfiguration decryptionConfiguration : decryptionConfigurations) {
        EncryptionConfiguration[] encryptionConfigurations = EncryptionConfiguration.values();
        for (EncryptionConfiguration encryptionConfiguration : encryptionConfigurations) {
            Path file = new Path(root, getFileName(encryptionConfiguration));
            LOG.info("==> Decryption configuration {}", decryptionConfiguration);
            FileDecryptionProperties fileDecryptionProperties = decryptionConfiguration.getDecryptionProperties();
            LOG.info("--> Read file {} {}", file.toString(), encryptionConfiguration);
            // Read only the non-encrypted columns
            if ((decryptionConfiguration == DecryptionConfiguration.NO_DECRYPTION) && (encryptionConfiguration == EncryptionConfiguration.ENCRYPT_COLUMNS_PLAINTEXT_FOOTER)) {
                conf.set("parquet.read.schema", Types.buildMessage().optional(INT32).named(SingleRow.PLAINTEXT_INT32_FIELD_NAME).named("FormatTestObject").toString());
            }
            int rowNum = 0;
            try (ParquetReader<Group> reader = ParquetReader.builder(new GroupReadSupport(), file).withConf(conf).withDecryption(fileDecryptionProperties).build()) {
                for (Group group = reader.read(); group != null; group = reader.read()) {
                    SingleRow rowExpected = data.get(rowNum++);
                    // plaintext columns
                    if (rowExpected.plaintext_int32_field != group.getInteger(SingleRow.PLAINTEXT_INT32_FIELD_NAME, 0)) {
                        addErrorToErrorCollectorAndLog("Wrong int", encryptionConfiguration, decryptionConfiguration);
                    }
                    // encrypted columns
                    if (decryptionConfiguration != DecryptionConfiguration.NO_DECRYPTION) {
                        if (rowExpected.boolean_field != group.getBoolean(SingleRow.BOOLEAN_FIELD_NAME, 0)) {
                            addErrorToErrorCollectorAndLog("Wrong bool", encryptionConfiguration, decryptionConfiguration);
                        }
                        if (rowExpected.int32_field != group.getInteger(SingleRow.INT32_FIELD_NAME, 0)) {
                            addErrorToErrorCollectorAndLog("Wrong int", encryptionConfiguration, decryptionConfiguration);
                        }
                        if (rowExpected.float_field != group.getFloat(SingleRow.FLOAT_FIELD_NAME, 0)) {
                            addErrorToErrorCollectorAndLog("Wrong float", encryptionConfiguration, decryptionConfiguration);
                        }
                        if (rowExpected.double_field != group.getDouble(SingleRow.DOUBLE_FIELD_NAME, 0)) {
                            addErrorToErrorCollectorAndLog("Wrong double", encryptionConfiguration, decryptionConfiguration);
                        }
                        if ((null != rowExpected.ba_field) && !Arrays.equals(rowExpected.ba_field, group.getBinary(SingleRow.BINARY_FIELD_NAME, 0).getBytes())) {
                            addErrorToErrorCollectorAndLog("Wrong byte array", encryptionConfiguration, decryptionConfiguration);
                        }
                        if (!Arrays.equals(rowExpected.flba_field, group.getBinary(SingleRow.FIXED_LENGTH_BINARY_FIELD_NAME, 0).getBytes())) {
                            addErrorToErrorCollectorAndLog("Wrong fixed-length byte array", encryptionConfiguration, decryptionConfiguration);
                        }
                    }
                }
            } catch (ParquetCryptoRuntimeException e) {
                checkResult(file.getName(), decryptionConfiguration, e);
            } catch (Exception e) {
                e.printStackTrace();
                addErrorToErrorCollectorAndLog("Unexpected exception: " + e.getClass().getName() + " with message: " + e.getMessage(), encryptionConfiguration, decryptionConfiguration);
            }
            conf.unset("parquet.read.schema");
        }
    }
}