Examples with ArrayBlock com.facebook.presto.common.block.ArrayBlock used on opensource projects

Example 1 with ArrayBlock

use of com.facebook.presto.common.block.ArrayBlock in project presto by prestodb.

the class ListBatchStreamReader method readBlock.

@Override
public Block readBlock() throws IOException {
    if (!rowGroupOpen) {
        openRowGroup();
    }
    if (readOffset > 0) {
        if (presentStream != null) {
            // skip ahead the present bit reader, but count the set bits
            // and use this as the skip size for the data reader
            readOffset = presentStream.countBitsSet(readOffset);
        }
        if (readOffset > 0) {
            if (lengthStream == null) {
                throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
            }
            long elementSkipSize = lengthStream.sum(readOffset);
            elementStreamReader.prepareNextRead(toIntExact(elementSkipSize));
        }
    }
    // We will use the offsetVector as the buffer to read the length values from lengthStream,
    // and the length values will be converted in-place to an offset vector.
    int[] offsetVector = new int[nextBatchSize + 1];
    boolean[] nullVector = null;
    if (presentStream == null) {
        if (lengthStream == null) {
            throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
        }
        lengthStream.next(offsetVector, nextBatchSize);
    } else {
        nullVector = new boolean[nextBatchSize];
        int nullValues = presentStream.getUnsetBits(nextBatchSize, nullVector);
        if (nullValues != nextBatchSize) {
            if (lengthStream == null) {
                throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
            }
            lengthStream.next(offsetVector, nextBatchSize - nullValues);
            unpackLengthNulls(offsetVector, nullVector, nextBatchSize - nullValues);
        }
    }
    convertLengthVectorToOffsetVector(offsetVector);
    int elementCount = offsetVector[offsetVector.length - 1];
    Block elements;
    if (elementCount > 0) {
        elementStreamReader.prepareNextRead(elementCount);
        elements = elementStreamReader.readBlock();
    } else {
        elements = elementType.createBlockBuilder(null, 0).build();
    }
    Block arrayBlock = ArrayBlock.fromElementBlock(nextBatchSize, Optional.ofNullable(nullVector), offsetVector, elements);
    readOffset = 0;
    nextBatchSize = 0;
    return arrayBlock;
}

Example 2 with ArrayBlock

use of com.facebook.presto.common.block.ArrayBlock in project presto by prestodb.

the class OptimizedPartitionedOutputOperator method decodeBlock.

/**
 * Flatten the block and convert the nested-typed block into ColumnarArray/Map/Row.
 * For performance considerations we decode the block only once for each block instead of for each batch.
 *
 * @return A tree structure that contains the decoded block
 */
@VisibleForTesting
static DecodedBlockNode decodeBlock(BlockFlattener flattener, Closer blockLeaseCloser, Block block) {
    BlockLease lease = flattener.flatten(block);
    blockLeaseCloser.register(lease::close);
    Block decodedBlock = lease.get();
    long estimatedSizeInBytes = decodedBlock.getLogicalSizeInBytes();
    if (decodedBlock instanceof ArrayBlock) {
        ColumnarArray columnarArray = ColumnarArray.toColumnarArray(decodedBlock);
        Block childBlock = columnarArray.getElementsBlock();
        return new DecodedBlockNode(columnarArray, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), columnarArray.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof MapBlock) {
        ColumnarMap columnarMap = ColumnarMap.toColumnarMap(decodedBlock);
        Block keyBlock = columnarMap.getKeysBlock();
        Block valueBlock = columnarMap.getValuesBlock();
        return new DecodedBlockNode(columnarMap, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, keyBlock), decodeBlock(flattener, blockLeaseCloser, valueBlock)), columnarMap.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RowBlock) {
        ColumnarRow columnarRow = ColumnarRow.toColumnarRow(decodedBlock);
        ImmutableList.Builder<DecodedBlockNode> children = ImmutableList.builder();
        for (int i = 0; i < columnarRow.getFieldCount(); i++) {
            Block childBlock = columnarRow.getField(i);
            children.add(decodeBlock(flattener, blockLeaseCloser, childBlock));
        }
        return new DecodedBlockNode(columnarRow, children.build(), columnarRow.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof DictionaryBlock) {
        Block dictionary = ((DictionaryBlock) decodedBlock).getDictionary();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, dictionary)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RunLengthEncodedBlock) {
        Block childBlock = ((RunLengthEncodedBlock) decodedBlock).getValue();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    return new DecodedBlockNode(decodedBlock, ImmutableList.of(), block.getRetainedSizeInBytes(), estimatedSizeInBytes);
}

Example 3 with ArrayBlock

use of com.facebook.presto.common.block.ArrayBlock in project presto by prestodb.

the class ParquetReader method readArray.

private ColumnChunk readArray(GroupField field) throws IOException {
    List<Type> parameters = field.getType().getTypeParameters();
    checkArgument(parameters.size() == 1, "Arrays must have a single type parameter, found %d", parameters.size());
    Field elementField = field.getChildren().get(0).get();
    ColumnChunk columnChunk = readColumnChunk(elementField);
    IntList offsets = new IntArrayList();
    BooleanList valueIsNull = new BooleanArrayList();
    calculateCollectionOffsets(field, offsets, valueIsNull, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
    Block arrayBlock = ArrayBlock.fromElementBlock(valueIsNull.size(), Optional.of(valueIsNull.toBooleanArray()), offsets.toIntArray(), columnChunk.getBlock());
    return new ColumnChunk(arrayBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels());
}

Example 4 with ArrayBlock

use of com.facebook.presto.common.block.ArrayBlock in project presto by prestodb.

the class PrestoThriftBigintArray method fromBlock.

public static PrestoThriftBlock fromBlock(Block block) {
    checkArgument(block instanceof AbstractArrayBlock, "block is not of an array type");
    AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block;
    int positions = arrayBlock.getPositionCount();
    if (positions == 0) {
        return bigintArrayData(new PrestoThriftBigintArray(null, null, null));
    }
    boolean[] nulls = null;
    int[] sizes = null;
    for (int position = 0; position < positions; position++) {
        if (arrayBlock.isNull(position)) {
            if (nulls == null) {
                nulls = new boolean[positions];
            }
            nulls[position] = true;
        } else {
            if (sizes == null) {
                sizes = new int[positions];
            }
            sizes[position] = arrayBlock.apply((valuesBlock, startPosition, length) -> length, position);
        }
    }
    PrestoThriftBigint values = arrayBlock.apply((valuesBlock, startPosition, length) -> PrestoThriftBigint.fromBlock(valuesBlock), 0).getBigintData();
    checkState(values != null, "values must be present");
    checkState(totalSize(nulls, sizes) == values.numberOfRecords(), "unexpected number of values");
    return bigintArrayData(new PrestoThriftBigintArray(nulls, sizes, values));
}