Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 56 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class Page method compactRelatedBlocks.

private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks) {
    DictionaryBlock firstDictionaryBlock = blocks.get(0);
    Block dictionary = firstDictionaryBlock.getDictionary();
    int positionCount = firstDictionaryBlock.getPositionCount();
    int dictionarySize = dictionary.getPositionCount();
    // determine which dictionary entries are referenced and build a reindex for them
    int[] dictionaryPositionsToCopy = new int[min(dictionarySize, positionCount)];
    int[] remapIndex = new int[dictionarySize];
    Arrays.fill(remapIndex, -1);
    int numberOfIndexes = 0;
    for (int i = 0; i < positionCount; i++) {
        int position = firstDictionaryBlock.getId(i);
        if (remapIndex[position] == -1) {
            dictionaryPositionsToCopy[numberOfIndexes] = position;
            remapIndex[position] = numberOfIndexes;
            numberOfIndexes++;
        }
    }
    // entire dictionary is referenced
    if (numberOfIndexes == dictionarySize) {
        return blocks;
    }
    // compact the dictionaries
    int[] newIds = getNewIds(positionCount, firstDictionaryBlock, remapIndex);
    List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
    DictionaryId newDictionaryId = randomDictionaryId();
    for (DictionaryBlock dictionaryBlock : blocks) {
        if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
            throw new IllegalArgumentException("dictionarySourceIds must be the same");
        }
        try {
            Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy, 0, numberOfIndexes);
            outputDictionaryBlocks.add(new DictionaryBlock(positionCount, compactDictionary, newIds, true, newDictionaryId));
        } catch (UnsupportedOperationException e) {
            // ignore if copy positions is not supported for the dictionary
            outputDictionaryBlocks.add(dictionaryBlock);
        }
    }
    return outputDictionaryBlocks;
}

Example 57 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class AbstractBlockEncodingBuffer method mapPositionsToNestedBlock.

/**
 * Map the positions for DictionaryBlock to its nested dictionaryBlock, and positions for RunLengthEncodedBlock
 * to its nested value block. For example, positions [0, 2, 5] on DictionaryBlock with dictionary block ['a', 'b', 'c']
 * and ids [1, 1, 2, 0, 2, 1] will be mapped to [1, 2, 1], and the copied data will be ['b', 'c', 'b'].
 */
protected DecodedBlockNode mapPositionsToNestedBlock(DecodedBlockNode decodedBlockNode) {
    Object decodedObject = decodedBlockNode.getDecodedBlock();
    if (decodedObject instanceof DictionaryBlock) {
        DictionaryBlock dictionaryBlock = (DictionaryBlock) decodedObject;
        mappedPositions = ensureCapacity(mappedPositions, positionCount, SMALL, NONE, bufferAllocator);
        for (int i = 0; i < positionCount; i++) {
            mappedPositions[i] = dictionaryBlock.getId(positions[i]);
        }
        positionsMapped = true;
        return decodedBlockNode.getChildren().get(0);
    }
    if (decodedObject instanceof RunLengthEncodedBlock) {
        mappedPositions = ensureCapacity(mappedPositions, positionCount, SMALL, INITIALIZE, bufferAllocator);
        positionsMapped = true;
        return decodedBlockNode.getChildren().get(0);
    }
    positionsMapped = false;
    return decodedBlockNode;
}

Example 58 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class OptimizedPartitionedOutputOperator method decodeBlock.

/**
 * Flatten the block and convert the nested-typed block into ColumnarArray/Map/Row.
 * For performance considerations we decode the block only once for each block instead of for each batch.
 *
 * @return A tree structure that contains the decoded block
 */
@VisibleForTesting
static DecodedBlockNode decodeBlock(BlockFlattener flattener, Closer blockLeaseCloser, Block block) {
    BlockLease lease = flattener.flatten(block);
    blockLeaseCloser.register(lease::close);
    Block decodedBlock = lease.get();
    long estimatedSizeInBytes = decodedBlock.getLogicalSizeInBytes();
    if (decodedBlock instanceof ArrayBlock) {
        ColumnarArray columnarArray = ColumnarArray.toColumnarArray(decodedBlock);
        Block childBlock = columnarArray.getElementsBlock();
        return new DecodedBlockNode(columnarArray, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), columnarArray.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof MapBlock) {
        ColumnarMap columnarMap = ColumnarMap.toColumnarMap(decodedBlock);
        Block keyBlock = columnarMap.getKeysBlock();
        Block valueBlock = columnarMap.getValuesBlock();
        return new DecodedBlockNode(columnarMap, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, keyBlock), decodeBlock(flattener, blockLeaseCloser, valueBlock)), columnarMap.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RowBlock) {
        ColumnarRow columnarRow = ColumnarRow.toColumnarRow(decodedBlock);
        ImmutableList.Builder<DecodedBlockNode> children = ImmutableList.builder();
        for (int i = 0; i < columnarRow.getFieldCount(); i++) {
            Block childBlock = columnarRow.getField(i);
            children.add(decodeBlock(flattener, blockLeaseCloser, childBlock));
        }
        return new DecodedBlockNode(columnarRow, children.build(), columnarRow.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof DictionaryBlock) {
        Block dictionary = ((DictionaryBlock) decodedBlock).getDictionary();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, dictionary)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RunLengthEncodedBlock) {
        Block childBlock = ((RunLengthEncodedBlock) decodedBlock).getValue();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    return new DecodedBlockNode(decodedBlock, ImmutableList.of(), block.getRetainedSizeInBytes(), estimatedSizeInBytes);
}

Example 59 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class DictionaryAwarePageFilter method filter.

@Override
public SelectedPositions filter(SqlFunctionProperties properties, Page page) {
    Block block = page.getBlock(0).getLoadedBlock();
    if (block instanceof RunLengthEncodedBlock) {
        Block value = ((RunLengthEncodedBlock) block).getValue();
        Optional<boolean[]> selectedPosition = processDictionary(properties, value);
        // in that case we fallback and process again so the correct error message sent
        if (selectedPosition.isPresent()) {
            return SelectedPositions.positionsRange(0, selectedPosition.get()[0] ? page.getPositionCount() : 0);
        }
    }
    if (block instanceof DictionaryBlock) {
        DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
        // Attempt to process the dictionary.  If dictionary is processing has not been considered effective, an empty response will be returned
        Optional<boolean[]> selectedDictionaryPositions = processDictionary(properties, dictionaryBlock.getDictionary());
        // record the usage count regardless of dictionary processing choice, so we have stats for next time
        lastDictionaryUsageCount += page.getPositionCount();
        // if dictionary was processed, produce a dictionary block; otherwise do normal processing
        if (selectedDictionaryPositions.isPresent()) {
            return selectDictionaryPositions(dictionaryBlock, selectedDictionaryPositions.get());
        }
    }
    return filter.filter(properties, new Page(block));
}

Example 60 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project urban-eureka by errir503.

the class MultiChannelGroupByHash method canProcessDictionary.

private boolean canProcessDictionary(Page page) {
    if (!this.processDictionary || channels.length > 1 || !(page.getBlock(channels[0]) instanceof DictionaryBlock)) {
        return false;
    }
    if (inputHashChannel.isPresent()) {
        Block inputHashBlock = page.getBlock(inputHashChannel.get());
        DictionaryBlock inputDataBlock = (DictionaryBlock) page.getBlock(channels[0]);
        if (!(inputHashBlock instanceof DictionaryBlock)) {
            // data channel is dictionary encoded but hash channel is not
            return false;
        }
        if (!((DictionaryBlock) inputHashBlock).getDictionarySourceId().equals(inputDataBlock.getDictionarySourceId())) {
            // dictionarySourceIds of data block and hash block do not match
            return false;
        }
    }
    return true;
}