Examples with DwrfSequenceEncoding com.facebook.presto.orc.metadata.DwrfSequenceEncoding used on opensource projects

Example 1 with DwrfSequenceEncoding

use of com.facebook.presto.orc.metadata.DwrfSequenceEncoding in project presto by prestodb.

the class MapFlatBatchStreamReader method getIntegerKeyBlockTemplate.

private Block getIntegerKeyBlockTemplate(Collection<DwrfSequenceEncoding> sequenceEncodings) {
    Type keyType;
    switch(keyOrcType) {
        case BYTE:
            keyType = TinyintType.TINYINT;
            break;
        case SHORT:
            keyType = SmallintType.SMALLINT;
            break;
        case INT:
            keyType = IntegerType.INTEGER;
            break;
        case LONG:
            keyType = BigintType.BIGINT;
            break;
        default:
            throw new IllegalArgumentException("Unsupported flat map key type: " + keyOrcType);
    }
    BlockBuilder blockBuilder = keyType.createBlockBuilder(null, sequenceEncodings.size());
    for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) {
        keyType.writeLong(blockBuilder, sequenceEncoding.getKey().getIntKey());
    }
    return blockBuilder.build();
}

Example 2 with DwrfSequenceEncoding

use of com.facebook.presto.orc.metadata.DwrfSequenceEncoding in project presto by prestodb.

the class MapFlatBatchStreamReader method startStripe.

@Override
public void startStripe(Stripe stripe) throws IOException {
    presentStreamSource = missingStreamSource(BooleanInputStream.class);
    inMapStreamSources.clear();
    valueStreamDescriptors.clear();
    valueStreamReaders.clear();
    ColumnEncoding encoding = stripe.getColumnEncodings().get(baseValueStreamDescriptor.getStreamId());
    SortedMap<Integer, DwrfSequenceEncoding> additionalSequenceEncodings = Collections.emptySortedMap();
    // encoding or encoding.getAdditionalSequenceEncodings() may not be present when every map is empty or null
    if (encoding != null && encoding.getAdditionalSequenceEncodings().isPresent()) {
        additionalSequenceEncodings = encoding.getAdditionalSequenceEncodings().get();
    }
    // The ColumnEncoding with sequence ID 0 doesn't have any data associated with it
    for (int sequence : additionalSequenceEncodings.keySet()) {
        inMapStreamSources.add(missingStreamSource(BooleanInputStream.class));
        StreamDescriptor valueStreamDescriptor = copyStreamDescriptorWithSequence(baseValueStreamDescriptor, sequence);
        valueStreamDescriptors.add(valueStreamDescriptor);
        BatchStreamReader valueStreamReader = BatchStreamReaders.createStreamReader(type.getValueType(), valueStreamDescriptor, hiveStorageTimeZone, options, systemMemoryContext);
        valueStreamReader.startStripe(stripe);
        valueStreamReaders.add(valueStreamReader);
    }
    keyBlockTemplate = getKeyBlockTemplate(additionalSequenceEncodings.values());
    readOffset = 0;
    nextBatchSize = 0;
    presentStream = null;
    rowGroupOpen = false;
}

Example 3 with DwrfSequenceEncoding

use of com.facebook.presto.orc.metadata.DwrfSequenceEncoding in project presto by prestodb.

the class MapFlatBatchStreamReader method getSliceKeysBlockTemplate.

private Block getSliceKeysBlockTemplate(Collection<DwrfSequenceEncoding> sequenceEncodings) {
    int bytes = 0;
    for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) {
        bytes += sequenceEncoding.getKey().getBytesKey().size();
    }
    VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes);
    for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) {
        Slice key = Slices.wrappedBuffer(sequenceEncoding.getKey().getBytesKey().toByteArray());
        builder.writeBytes(key, 0, key.length());
        builder.closeEntry();
    }
    return builder.build();
}

Example 4 with DwrfSequenceEncoding

use of com.facebook.presto.orc.metadata.DwrfSequenceEncoding in project presto by prestodb.

the class StripeReader method addIncludedStreams.

/**
 * Add streams that are in includedOrcColumns to the includedStreams map,
 * and return whether there were any rowGroupDictionaries
 */
private boolean addIncludedStreams(Map<Integer, ColumnEncoding> columnEncodings, List<Stream> streams, Map<StreamId, Stream> includedStreams) {
    boolean hasRowGroupDictionary = false;
    for (Stream stream : streams) {
        if (includedOrcColumns.contains(stream.getColumn())) {
            includedStreams.put(new StreamId(stream), stream);
            if (stream.getStreamKind() == StreamKind.IN_DICTIONARY) {
                ColumnEncoding columnEncoding = columnEncodings.get(stream.getColumn());
                if (columnEncoding.getColumnEncodingKind() == DICTIONARY) {
                    hasRowGroupDictionary = true;
                }
                Optional<SortedMap<Integer, DwrfSequenceEncoding>> additionalSequenceEncodings = columnEncoding.getAdditionalSequenceEncodings();
                if (additionalSequenceEncodings.isPresent() && additionalSequenceEncodings.get().values().stream().map(DwrfSequenceEncoding::getValueEncoding).anyMatch(encoding -> encoding.getColumnEncodingKind() == DICTIONARY)) {
                    hasRowGroupDictionary = true;
                }
            }
        }
    }
    return hasRowGroupDictionary;
}

Example 5 with DwrfSequenceEncoding

use of com.facebook.presto.orc.metadata.DwrfSequenceEncoding in project presto by prestodb.

the class MapFlatSelectiveStreamReader method getSliceKeysBlock.

private Block getSliceKeysBlock(List<DwrfSequenceEncoding> sequenceEncodings) {
    int bytes = 0;
    for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) {
        bytes += sequenceEncoding.getKey().getBytesKey().size();
    }
    VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes);
    for (int i = 0; i < keyCount; i++) {
        Slice key = Slices.wrappedBuffer(sequenceEncodings.get(keyIndices[i]).getKey().getBytesKey().toByteArray());
        builder.writeBytes(key, 0, key.length());
        builder.closeEntry();
    }
    return builder.build();
}