Examples with BlockLease com.facebook.presto.common.block.BlockLease used on opensource projects

Example 1 with BlockLease

use of com.facebook.presto.common.block.BlockLease in project presto by prestodb.

the class OrcSelectiveRecordReader method applyFilterFunctions.

private int applyFilterFunctions(List<FilterFunctionWithStats> filterFunctions, Set<Integer> filterFunctionInputs, int[] positions, int positionCount) {
    BlockLease[] blockLeases = new BlockLease[hiveColumnIndices.length];
    Block[] blocks = new Block[hiveColumnIndices.length];
    for (int columnIndex : filterFunctionInputs) {
        if (constantValues[columnIndex] != null) {
            blocks[columnIndex] = RunLengthEncodedBlock.create(columnTypes.get(columnIndex), constantValues[columnIndex] == NULL_MARKER ? null : constantValues[columnIndex], positionCount);
        } else {
            blockLeases[columnIndex] = getStreamReader(columnIndex).getBlockView(positions, positionCount);
            Block block = blockLeases[columnIndex].get();
            if (coercers[columnIndex] != null) {
                block = coercers[columnIndex].apply(block);
            }
            blocks[columnIndex] = block;
        }
    }
    initializeTmpErrors(positionCount);
    for (int i = 0; i < positionCount; i++) {
        tmpErrors[i] = errors[positions[i]];
    }
    Arrays.fill(errors, null);
    try {
        initializeOutputPositions(positionCount);
        for (int i = 0; i < filterFunctions.size(); i++) {
            FilterFunctionWithStats functionWithStats = filterFunctions.get(i);
            FilterFunction function = functionWithStats.getFunction();
            int[] inputs = function.getInputChannels();
            Block[] inputBlocks = new Block[inputs.length];
            for (int j = 0; j < inputs.length; j++) {
                inputBlocks[j] = blocks[filterFunctionInputMapping.get(inputs[j])];
            }
            Page page = new Page(positionCount, inputBlocks);
            long startTime = System.nanoTime();
            int inputPositionCount = positionCount;
            positionCount = function.filter(page, outputPositions, positionCount, tmpErrors);
            functionWithStats.getStats().update(inputPositionCount, positionCount, System.nanoTime() - startTime);
            if (positionCount == 0) {
                break;
            }
        }
        // e.g. make outputPositions a subset of positions array
        for (int i = 0; i < positionCount; i++) {
            outputPositions[i] = positions[outputPositions[i]];
            errors[outputPositions[i]] = tmpErrors[i];
        }
        return positionCount;
    } finally {
        for (BlockLease blockLease : blockLeases) {
            if (blockLease != null) {
                blockLease.close();
            }
        }
    }
}

Example 2 with BlockLease

use of com.facebook.presto.common.block.BlockLease in project presto by prestodb.

the class OptimizedPartitionedOutputOperator method decodeBlock.

/**
 * Flatten the block and convert the nested-typed block into ColumnarArray/Map/Row.
 * For performance considerations we decode the block only once for each block instead of for each batch.
 *
 * @return A tree structure that contains the decoded block
 */
@VisibleForTesting
static DecodedBlockNode decodeBlock(BlockFlattener flattener, Closer blockLeaseCloser, Block block) {
    BlockLease lease = flattener.flatten(block);
    blockLeaseCloser.register(lease::close);
    Block decodedBlock = lease.get();
    long estimatedSizeInBytes = decodedBlock.getLogicalSizeInBytes();
    if (decodedBlock instanceof ArrayBlock) {
        ColumnarArray columnarArray = ColumnarArray.toColumnarArray(decodedBlock);
        Block childBlock = columnarArray.getElementsBlock();
        return new DecodedBlockNode(columnarArray, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), columnarArray.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof MapBlock) {
        ColumnarMap columnarMap = ColumnarMap.toColumnarMap(decodedBlock);
        Block keyBlock = columnarMap.getKeysBlock();
        Block valueBlock = columnarMap.getValuesBlock();
        return new DecodedBlockNode(columnarMap, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, keyBlock), decodeBlock(flattener, blockLeaseCloser, valueBlock)), columnarMap.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RowBlock) {
        ColumnarRow columnarRow = ColumnarRow.toColumnarRow(decodedBlock);
        ImmutableList.Builder<DecodedBlockNode> children = ImmutableList.builder();
        for (int i = 0; i < columnarRow.getFieldCount(); i++) {
            Block childBlock = columnarRow.getField(i);
            children.add(decodeBlock(flattener, blockLeaseCloser, childBlock));
        }
        return new DecodedBlockNode(columnarRow, children.build(), columnarRow.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof DictionaryBlock) {
        Block dictionary = ((DictionaryBlock) decodedBlock).getDictionary();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, dictionary)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RunLengthEncodedBlock) {
        Block childBlock = ((RunLengthEncodedBlock) decodedBlock).getValue();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    return new DecodedBlockNode(decodedBlock, ImmutableList.of(), block.getRetainedSizeInBytes(), estimatedSizeInBytes);
}

Example 3 with BlockLease

use of com.facebook.presto.common.block.BlockLease in project presto by prestodb.

the class MapFlatSelectiveStreamReader method assembleMapBlock.

private Block assembleMapBlock(boolean includeNulls) {
    offsets = ensureCapacity(offsets, outputPositionCount + 1);
    offsets[0] = 0;
    int offset = 0;
    int inMapIndex = 0;
    for (int i = 0; i < outputPositionCount; i++) {
        if (!includeNulls || !nulls[i]) {
            offset += nestedLengths[inMapIndex];
            inMapIndex++;
        }
        offsets[i + 1] = offset;
    }
    BlockLease[] valueBlockLeases = new BlockLease[keyCount];
    Block[] valueBlocks = new Block[keyCount];
    for (int i = 0; i < keyCount; i++) {
        if (nestedPositionCounts[i] > 0) {
            valueBlockLeases[i] = valueStreamReaders.get(i).getBlockView(nestedOutputPositions[i], nestedPositionCounts[i]);
            valueBlocks[i] = valueBlockLeases[i].get();
        } else {
            valueBlocks[i] = outputType.getKeyType().createBlockBuilder(null, 0).build();
        }
    }
    int[] keyIds = new int[offset];
    int count = 0;
    Type valueType = outputType.getValueType();
    BlockBuilder valueBlockBuilder;
    if (valueType instanceof FixedWidthType) {
        valueBlockBuilder = ((FixedWidthType) valueType).createFixedSizeBlockBuilder(offset);
    } else {
        valueBlockBuilder = valueType.createBlockBuilder(null, offset);
    }
    int[] valueBlockPositions = new int[keyCount];
    inMapIndex = 0;
    for (int i = 0; i < outputPositionCount; i++) {
        if (includeNulls && nulls[i]) {
            continue;
        }
        for (int keyIndex = 0; keyIndex < keyCount; keyIndex++) {
            if (inMap[keyIndex][inMapIndex]) {
                valueType.appendTo(valueBlocks[keyIndex], valueBlockPositions[keyIndex], valueBlockBuilder);
                valueBlockPositions[keyIndex]++;
                keyIds[count++] = keyIndex;
            }
        }
        inMapIndex++;
    }
    for (int i = 0; i < keyCount; i++) {
        if (valueBlockLeases[i] != null) {
            valueBlockLeases[i].close();
        }
    }
    return outputType.createBlockFromKeyValue(outputPositionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, new DictionaryBlock(keyBlock, keyIds), valueBlockBuilder.build());
}

Example 4 with BlockLease

use of com.facebook.presto.common.block.BlockLease in project presto by prestodb.

the class ListSelectiveStreamReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(new RunLengthEncodedBlock(outputType.createBlockBuilder(null, 1).appendNull().build(), positionCount));
    }
    boolean includeNulls = nullsAllowed && presentStream != null;
    if (positionCount != outputPositionCount) {
        compactValues(positions, positionCount, includeNulls);
    }
    BlockLease elementBlockLease;
    if (elementOutputPositionCount == 0) {
        elementBlockLease = newLease(outputType.getElementType().createBlockBuilder(null, 0).build());
    } else {
        elementBlockLease = elementStreamReader.getBlockView(elementOutputPositions, elementOutputPositionCount);
    }
    valuesInUse = true;
    Block block = ArrayBlock.fromElementBlock(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, elementBlockLease.get());
    return newLease(block, () -> closeBlockLease(elementBlockLease));
}

Example 5 with BlockLease

use of com.facebook.presto.common.block.BlockLease in project presto by prestodb.

the class MapDirectSelectiveStreamReader method getBlockView.

@Override
public BlockLease getBlockView(int[] positions, int positionCount) {
    checkArgument(outputPositionCount > 0, "outputPositionCount must be greater than zero");
    checkState(outputRequired, "This stream reader doesn't produce output");
    checkState(positionCount <= outputPositionCount, "Not enough values");
    checkState(!valuesInUse, "BlockLease hasn't been closed yet");
    if (allNulls) {
        return newLease(createNullBlock(outputType, positionCount));
    }
    boolean includeNulls = nullsAllowed && presentStream != null;
    if (positionCount != outputPositionCount) {
        compactValues(positions, positionCount, includeNulls);
    }
    if (nestedOutputPositionCount == 0) {
        return newLease(outputType.createBlockFromKeyValue(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, createEmptyBlock(outputType.getKeyType()), createEmptyBlock(outputType.getValueType())));
    }
    BlockLease keyBlockLease = keyReader.getBlockView(nestedOutputPositions, nestedOutputPositionCount);
    BlockLease valueBlockLease = valueReader.getBlockView(nestedOutputPositions, nestedOutputPositionCount);
    return newLease(outputType.createBlockFromKeyValue(positionCount, Optional.ofNullable(includeNulls ? nulls : null), offsets, keyBlockLease.get(), valueBlockLease.get()), keyBlockLease, valueBlockLease);
}