Search in sources :

Example 1 with WriterEncryptionGroup

use of com.facebook.presto.orc.WriterEncryptionGroup in project presto by prestodb.

the class OrcFileWriterFactory method createDwrfEncryption.

private Optional<DwrfWriterEncryption> createDwrfEncryption(Optional<EncryptionInformation> encryptionInformation, List<String> fileColumnNames, List<Type> types) {
    if (!encryptionInformation.isPresent()) {
        return Optional.empty();
    }
    if (!encryptionInformation.get().getDwrfEncryptionMetadata().isPresent()) {
        return Optional.empty();
    }
    DwrfEncryptionMetadata dwrfEncryptionMetadata = encryptionInformation.get().getDwrfEncryptionMetadata().get();
    List<OrcType> orcTypes = OrcType.createOrcRowType(0, fileColumnNames, types);
    Map<String, Integer> columnNamesMap = IntStream.range(0, fileColumnNames.size()).boxed().collect(toImmutableMap(fileColumnNames::get, index -> index));
    Map<Integer, Slice> keyMap = dwrfEncryptionMetadata.toKeyMap(orcTypes, columnNamesMap);
    ImmutableListMultimap.Builder<Slice, Integer> encryptionGroupsBuilder = ImmutableListMultimap.builder();
    keyMap.entrySet().stream().forEach(entry -> encryptionGroupsBuilder.put(entry.getValue(), entry.getKey()));
    ImmutableListMultimap<Slice, Integer> encryptionGroups = encryptionGroupsBuilder.build();
    List<WriterEncryptionGroup> writerEncryptionGroups = encryptionGroups.keySet().stream().map(key -> new WriterEncryptionGroup(encryptionGroups.get(key), key)).collect(toImmutableList());
    return Optional.of(new DwrfWriterEncryption(toKeyProvider(dwrfEncryptionMetadata.getEncryptionProvider()), writerEncryptionGroups));
}
Also used : HdfsOrcDataSource(com.facebook.presto.hive.orc.HdfsOrcDataSource) DateTimeZone(org.joda.time.DateTimeZone) FileSystem(org.apache.hadoop.fs.FileSystem) HiveSessionProperties.getDwrfWriterStripeCacheeMaxSize(com.facebook.presto.hive.HiveSessionProperties.getDwrfWriterStripeCacheeMaxSize) HIVE_WRITE_VALIDATION_FAILED(com.facebook.presto.hive.HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED) DataSink(com.facebook.presto.common.io.DataSink) HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory) OrcConf(org.apache.orc.OrcConf) HiveSessionProperties.getOrcOptimizedWriterMinStripeSize(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMinStripeSize) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) HiveSessionProperties.getOrcOptimizedWriterValidateMode(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterValidateMode) OrcDataSource(com.facebook.presto.orc.OrcDataSource) Splitter(com.google.common.base.Splitter) ENGLISH(java.util.Locale.ENGLISH) WriterEncryptionGroup(com.facebook.presto.orc.WriterEncryptionGroup) CRYPTO_SERVICE(com.facebook.presto.orc.metadata.KeyProvider.CRYPTO_SERVICE) HIVE_UNSUPPORTED_FORMAT(com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT) KeyProvider(com.facebook.presto.orc.metadata.KeyProvider) META_TABLE_COLUMN_TYPES(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMN_TYPES) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) HiveSessionProperties.isDwrfWriterStripeCacheEnabled(com.facebook.presto.hive.HiveSessionProperties.isDwrfWriterStripeCacheEnabled) HiveSessionProperties.isExecutionBasedMemoryAccountingEnabled(com.facebook.presto.hive.HiveSessionProperties.isExecutionBasedMemoryAccountingEnabled) ConnectorSession(com.facebook.presto.spi.ConnectorSession) ORC(com.facebook.presto.orc.OrcEncoding.ORC) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) DWRF(com.facebook.presto.orc.OrcEncoding.DWRF) CompressionKind(com.facebook.presto.orc.metadata.CompressionKind) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Optional(java.util.Optional) OrcOutputFormat(org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat) HiveType.toHiveTypes(com.facebook.presto.hive.HiveType.toHiveTypes) IntStream(java.util.stream.IntStream) HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize) Slice(io.airlift.slice.Slice) HiveSessionProperties.getOrcMaxMergeDistance(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxMergeDistance) Flatten(org.weakref.jmx.Flatten) Callable(java.util.concurrent.Callable) META_TABLE_COLUMNS(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_COLUMNS) DwrfWriterEncryption(com.facebook.presto.orc.DwrfWriterEncryption) DataSinkFactory(com.facebook.presto.hive.datasink.DataSinkFactory) PrestoException(com.facebook.presto.spi.PrestoException) Supplier(java.util.function.Supplier) UNKNOWN(com.facebook.presto.orc.metadata.KeyProvider.UNKNOWN) Inject(javax.inject.Inject) MetastoreUtil(com.facebook.presto.hive.metastore.MetastoreUtil) Managed(org.weakref.jmx.Managed) TypeManager(com.facebook.presto.common.type.TypeManager) HiveSessionProperties.getOrcMaxBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize) Objects.requireNonNull(java.util.Objects.requireNonNull) OrcType(com.facebook.presto.orc.metadata.OrcType) OrcWriterStats(com.facebook.presto.orc.OrcWriterStats) Type(com.facebook.presto.common.type.Type) DwrfEncryptionProvider(com.facebook.presto.orc.DwrfEncryptionProvider) OrcDataSourceId(com.facebook.presto.orc.OrcDataSourceId) OrcEncoding(com.facebook.presto.orc.OrcEncoding) Properties(java.util.Properties) DefaultOrcWriterFlushPolicy(com.facebook.presto.orc.DefaultOrcWriterFlushPolicy) HiveSessionProperties.getOrcStreamBufferSize(com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize) IOException(java.io.IOException) HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows(com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows) JobConf(org.apache.hadoop.mapred.JobConf) Collectors.toList(java.util.stream.Collectors.toList) HiveSessionProperties.getOrcStringStatisticsLimit(com.facebook.presto.hive.HiveSessionProperties.getOrcStringStatisticsLimit) HIVE_WRITER_OPEN_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR) WriterEncryptionGroup(com.facebook.presto.orc.WriterEncryptionGroup) OrcType(com.facebook.presto.orc.metadata.OrcType) Slice(io.airlift.slice.Slice) DwrfWriterEncryption(com.facebook.presto.orc.DwrfWriterEncryption) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap)

Aggregations

DataSink (com.facebook.presto.common.io.DataSink)1 Type (com.facebook.presto.common.type.Type)1 TypeManager (com.facebook.presto.common.type.TypeManager)1 HIVE_UNSUPPORTED_FORMAT (com.facebook.presto.hive.HiveErrorCode.HIVE_UNSUPPORTED_FORMAT)1 HIVE_WRITER_OPEN_ERROR (com.facebook.presto.hive.HiveErrorCode.HIVE_WRITER_OPEN_ERROR)1 HIVE_WRITE_VALIDATION_FAILED (com.facebook.presto.hive.HiveErrorCode.HIVE_WRITE_VALIDATION_FAILED)1 HiveSessionProperties.getDwrfWriterStripeCacheeMaxSize (com.facebook.presto.hive.HiveSessionProperties.getDwrfWriterStripeCacheeMaxSize)1 HiveSessionProperties.getOrcMaxBufferSize (com.facebook.presto.hive.HiveSessionProperties.getOrcMaxBufferSize)1 HiveSessionProperties.getOrcMaxMergeDistance (com.facebook.presto.hive.HiveSessionProperties.getOrcMaxMergeDistance)1 HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory (com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxDictionaryMemory)1 HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows (com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxStripeRows)1 HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize (com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMaxStripeSize)1 HiveSessionProperties.getOrcOptimizedWriterMinStripeSize (com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterMinStripeSize)1 HiveSessionProperties.getOrcOptimizedWriterValidateMode (com.facebook.presto.hive.HiveSessionProperties.getOrcOptimizedWriterValidateMode)1 HiveSessionProperties.getOrcStreamBufferSize (com.facebook.presto.hive.HiveSessionProperties.getOrcStreamBufferSize)1 HiveSessionProperties.getOrcStringStatisticsLimit (com.facebook.presto.hive.HiveSessionProperties.getOrcStringStatisticsLimit)1 HiveSessionProperties.isDwrfWriterStripeCacheEnabled (com.facebook.presto.hive.HiveSessionProperties.isDwrfWriterStripeCacheEnabled)1 HiveSessionProperties.isExecutionBasedMemoryAccountingEnabled (com.facebook.presto.hive.HiveSessionProperties.isExecutionBasedMemoryAccountingEnabled)1 HiveType.toHiveTypes (com.facebook.presto.hive.HiveType.toHiveTypes)1 DataSinkFactory (com.facebook.presto.hive.datasink.DataSinkFactory)1