Examples with InternalColumnEncryptionSetup org.apache.parquet.crypto.InternalColumnEncryptionSetup used on opensource projects

Example 1 with InternalColumnEncryptionSetup

use of org.apache.parquet.crypto.InternalColumnEncryptionSetup in project parquet-mr by apache.

the class ParquetFileWriter method serializeOffsetIndexes.

private static void serializeOffsetIndexes(List<List<OffsetIndex>> offsetIndexes, List<BlockMetaData> blocks, PositionOutputStream out, InternalFileEncryptor fileEncryptor) throws IOException {
    LOG.debug("{}: offset indexes", out.getPos());
    for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
        BlockMetaData block = blocks.get(bIndex);
        List<ColumnChunkMetaData> columns = block.getColumns();
        List<OffsetIndex> blockOffsetIndexes = offsetIndexes.get(bIndex);
        for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
            OffsetIndex offsetIndex = blockOffsetIndexes.get(cIndex);
            if (offsetIndex == null) {
                continue;
            }
            ColumnChunkMetaData column = columns.get(cIndex);
            BlockCipher.Encryptor offsetIndexEncryptor = null;
            byte[] offsetIndexAAD = null;
            if (null != fileEncryptor) {
                InternalColumnEncryptionSetup columnEncryptionSetup = fileEncryptor.getColumnSetup(column.getPath(), false, cIndex);
                if (columnEncryptionSetup.isEncrypted()) {
                    offsetIndexEncryptor = columnEncryptionSetup.getMetaDataEncryptor();
                    offsetIndexAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.OffsetIndex, block.getOrdinal(), columnEncryptionSetup.getOrdinal(), -1);
                }
            }
            long offset = out.getPos();
            Util.writeOffsetIndex(ParquetMetadataConverter.toParquetOffsetIndex(offsetIndex), out, offsetIndexEncryptor, offsetIndexAAD);
            column.setOffsetIndexReference(new IndexReference(offset, (int) (out.getPos() - offset)));
        }
    }
}

Example 2 with InternalColumnEncryptionSetup

use of org.apache.parquet.crypto.InternalColumnEncryptionSetup in project parquet-mr by apache.

the class ParquetFileWriter method serializeBloomFilters.

private static void serializeBloomFilters(List<Map<String, BloomFilter>> bloomFilters, List<BlockMetaData> blocks, PositionOutputStream out, InternalFileEncryptor fileEncryptor) throws IOException {
    LOG.debug("{}: bloom filters", out.getPos());
    for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
        BlockMetaData block = blocks.get(bIndex);
        List<ColumnChunkMetaData> columns = block.getColumns();
        Map<String, BloomFilter> blockBloomFilters = bloomFilters.get(bIndex);
        if (blockBloomFilters.isEmpty())
            continue;
        for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
            ColumnChunkMetaData column = columns.get(cIndex);
            BloomFilter bloomFilter = blockBloomFilters.get(column.getPath().toDotString());
            if (bloomFilter == null) {
                continue;
            }
            long offset = out.getPos();
            column.setBloomFilterOffset(offset);
            BlockCipher.Encryptor bloomFilterEncryptor = null;
            byte[] bloomFilterHeaderAAD = null;
            byte[] bloomFilterBitsetAAD = null;
            if (null != fileEncryptor) {
                InternalColumnEncryptionSetup columnEncryptionSetup = fileEncryptor.getColumnSetup(column.getPath(), false, cIndex);
                if (columnEncryptionSetup.isEncrypted()) {
                    bloomFilterEncryptor = columnEncryptionSetup.getMetaDataEncryptor();
                    int columnOrdinal = columnEncryptionSetup.getOrdinal();
                    bloomFilterHeaderAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.BloomFilterHeader, block.getOrdinal(), columnOrdinal, -1);
                    bloomFilterBitsetAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.BloomFilterBitset, block.getOrdinal(), columnOrdinal, -1);
                }
            }
            Util.writeBloomFilterHeader(ParquetMetadataConverter.toBloomFilterHeader(bloomFilter), out, bloomFilterEncryptor, bloomFilterHeaderAAD);
            ByteArrayOutputStream tempOutStream = new ByteArrayOutputStream();
            bloomFilter.writeTo(tempOutStream);
            byte[] serializedBitset = tempOutStream.toByteArray();
            if (null != bloomFilterEncryptor) {
                serializedBitset = bloomFilterEncryptor.encrypt(serializedBitset, bloomFilterBitsetAAD);
            }
            out.write(serializedBitset);
        }
    }
}

Example 3 with InternalColumnEncryptionSetup

use of org.apache.parquet.crypto.InternalColumnEncryptionSetup in project parquet-mr by apache.

the class ParquetFileWriter method serializeColumnIndexes.

private static void serializeColumnIndexes(List<List<ColumnIndex>> columnIndexes, List<BlockMetaData> blocks, PositionOutputStream out, InternalFileEncryptor fileEncryptor) throws IOException {
    LOG.debug("{}: column indexes", out.getPos());
    for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
        BlockMetaData block = blocks.get(bIndex);
        List<ColumnChunkMetaData> columns = block.getColumns();
        List<ColumnIndex> blockColumnIndexes = columnIndexes.get(bIndex);
        for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
            ColumnChunkMetaData column = columns.get(cIndex);
            org.apache.parquet.format.ColumnIndex columnIndex = ParquetMetadataConverter.toParquetColumnIndex(column.getPrimitiveType(), blockColumnIndexes.get(cIndex));
            if (columnIndex == null) {
                continue;
            }
            BlockCipher.Encryptor columnIndexEncryptor = null;
            byte[] columnIndexAAD = null;
            if (null != fileEncryptor) {
                InternalColumnEncryptionSetup columnEncryptionSetup = fileEncryptor.getColumnSetup(column.getPath(), false, cIndex);
                if (columnEncryptionSetup.isEncrypted()) {
                    columnIndexEncryptor = columnEncryptionSetup.getMetaDataEncryptor();
                    columnIndexAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.ColumnIndex, block.getOrdinal(), columnEncryptionSetup.getOrdinal(), -1);
                }
            }
            long offset = out.getPos();
            Util.writeColumnIndex(columnIndex, out, columnIndexEncryptor, columnIndexAAD);
            column.setColumnIndexReference(new IndexReference(offset, (int) (out.getPos() - offset)));
        }
    }
}

Example 4 with InternalColumnEncryptionSetup

use of org.apache.parquet.crypto.InternalColumnEncryptionSetup in project drill by apache.

the class ParquetFileWriter method serializeBloomFilters.

private static void serializeBloomFilters(List<Map<String, BloomFilter>> bloomFilters, List<BlockMetaData> blocks, PositionOutputStream out, InternalFileEncryptor fileEncryptor) throws IOException {
    LOG.debug("{}: bloom filters", out.getPos());
    for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
        BlockMetaData block = blocks.get(bIndex);
        List<ColumnChunkMetaData> columns = block.getColumns();
        Map<String, BloomFilter> blockBloomFilters = bloomFilters.get(bIndex);
        if (blockBloomFilters.isEmpty()) {
            continue;
        }
        for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
            ColumnChunkMetaData column = columns.get(cIndex);
            BloomFilter bloomFilter = blockBloomFilters.get(column.getPath().toDotString());
            if (bloomFilter == null) {
                continue;
            }
            long offset = out.getPos();
            column.setBloomFilterOffset(offset);
            BlockCipher.Encryptor bloomFilterEncryptor = null;
            byte[] bloomFilterHeaderAAD = null;
            byte[] bloomFilterBitsetAAD = null;
            if (null != fileEncryptor) {
                InternalColumnEncryptionSetup columnEncryptionSetup = fileEncryptor.getColumnSetup(column.getPath(), false, cIndex);
                if (columnEncryptionSetup.isEncrypted()) {
                    bloomFilterEncryptor = columnEncryptionSetup.getMetaDataEncryptor();
                    int columnOrdinal = columnEncryptionSetup.getOrdinal();
                    bloomFilterHeaderAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.BloomFilterHeader, block.getOrdinal(), columnOrdinal, -1);
                    bloomFilterBitsetAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.BloomFilterBitset, block.getOrdinal(), columnOrdinal, -1);
                }
            }
            Util.writeBloomFilterHeader(ParquetMetadataConverter.toBloomFilterHeader(bloomFilter), out, bloomFilterEncryptor, bloomFilterHeaderAAD);
            ByteArrayOutputStream tempOutStream = new ByteArrayOutputStream();
            bloomFilter.writeTo(tempOutStream);
            byte[] serializedBitset = tempOutStream.toByteArray();
            if (null != bloomFilterEncryptor) {
                serializedBitset = bloomFilterEncryptor.encrypt(serializedBitset, bloomFilterBitsetAAD);
            }
            out.write(serializedBitset);
        }
    }
}

Example 5 with InternalColumnEncryptionSetup

use of org.apache.parquet.crypto.InternalColumnEncryptionSetup in project drill by apache.

the class ParquetFileWriter method serializeOffsetIndexes.

private static void serializeOffsetIndexes(List<List<OffsetIndex>> offsetIndexes, List<BlockMetaData> blocks, PositionOutputStream out, InternalFileEncryptor fileEncryptor) throws IOException {
    LOG.debug("{}: offset indexes", out.getPos());
    for (int bIndex = 0, bSize = blocks.size(); bIndex < bSize; ++bIndex) {
        BlockMetaData block = blocks.get(bIndex);
        List<ColumnChunkMetaData> columns = block.getColumns();
        List<OffsetIndex> blockOffsetIndexes = offsetIndexes.get(bIndex);
        for (int cIndex = 0, cSize = columns.size(); cIndex < cSize; ++cIndex) {
            OffsetIndex offsetIndex = blockOffsetIndexes.get(cIndex);
            if (offsetIndex == null) {
                continue;
            }
            ColumnChunkMetaData column = columns.get(cIndex);
            BlockCipher.Encryptor offsetIndexEncryptor = null;
            byte[] offsetIndexAAD = null;
            if (null != fileEncryptor) {
                InternalColumnEncryptionSetup columnEncryptionSetup = fileEncryptor.getColumnSetup(column.getPath(), false, cIndex);
                if (columnEncryptionSetup.isEncrypted()) {
                    offsetIndexEncryptor = columnEncryptionSetup.getMetaDataEncryptor();
                    offsetIndexAAD = AesCipher.createModuleAAD(fileEncryptor.getFileAAD(), ModuleType.OffsetIndex, block.getOrdinal(), columnEncryptionSetup.getOrdinal(), -1);
                }
            }
            long offset = out.getPos();
            Util.writeOffsetIndex(ParquetMetadataConverter.toParquetOffsetIndex(offsetIndex), out, offsetIndexEncryptor, offsetIndexAAD);
            column.setOffsetIndexReference(new IndexReference(offset, (int) (out.getPos() - offset)));
        }
    }
}