Examples with DictionaryBlock com.facebook.presto.spi.block.DictionaryBlock used on opensource projects

Example 1 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class Page method compactRelatedBlocks.

private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks) {
    DictionaryBlock firstDictionaryBlock = blocks.get(0);
    Block dictionary = firstDictionaryBlock.getDictionary();
    int positionCount = firstDictionaryBlock.getPositionCount();
    int dictionarySize = dictionary.getPositionCount();
    // determine which dictionary entries are referenced and build a reindex for them
    List<Integer> dictionaryPositionsToCopy = new ArrayList<>(min(dictionarySize, positionCount));
    int[] remapIndex = new int[dictionarySize];
    Arrays.fill(remapIndex, -1);
    int newIndex = 0;
    for (int i = 0; i < positionCount; i++) {
        int position = firstDictionaryBlock.getId(i);
        if (remapIndex[position] == -1) {
            dictionaryPositionsToCopy.add(position);
            remapIndex[position] = newIndex;
            newIndex++;
        }
    }
    // entire dictionary is referenced
    if (dictionaryPositionsToCopy.size() == dictionarySize) {
        return blocks;
    }
    // compact the dictionaries
    int[] newIds = getNewIds(positionCount, firstDictionaryBlock, remapIndex);
    List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
    DictionaryId newDictionaryId = randomDictionaryId();
    for (DictionaryBlock dictionaryBlock : blocks) {
        if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
            throw new IllegalArgumentException("dictionarySourceIds must be the same");
        }
        try {
            Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy);
            outputDictionaryBlocks.add(new DictionaryBlock(positionCount, compactDictionary, newIds, true, newDictionaryId));
        } catch (UnsupportedOperationException e) {
            // ignore if copy positions is not supported for the dictionary
            outputDictionaryBlocks.add(dictionaryBlock);
        }
    }
    return outputDictionaryBlocks;
}

Example 2 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class Page method getRelatedDictionaryBlocks.

private Map<DictionaryId, DictionaryBlockIndexes> getRelatedDictionaryBlocks() {
    Map<DictionaryId, DictionaryBlockIndexes> relatedDictionaryBlocks = new HashMap<>();
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
            relatedDictionaryBlocks.computeIfAbsent(dictionaryBlock.getDictionarySourceId(), id -> new DictionaryBlockIndexes()).addBlock(dictionaryBlock, i);
        }
    }
    return relatedDictionaryBlocks;
}

Example 3 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class GenericPageProcessor method filterPage.

private int[] filterPage(Page page) {
    int[] selected = new int[page.getPositionCount()];
    int index = 0;
    if (filterFunction.getInputChannels().size() == 1) {
        int channel = getOnlyElement(filterFunction.getInputChannels());
        if (page.getBlock(channel) instanceof DictionaryBlock) {
            DictionaryBlock dictionaryBlock = (DictionaryBlock) page.getBlock(channel);
            Block dictionary = dictionaryBlock.getDictionary();
            Block[] blocks = new Block[page.getPositionCount()];
            blocks[channel] = dictionary;
            boolean[] selectedDictionaryPositions;
            if (inputFilterDictionary == dictionary) {
                selectedDictionaryPositions = filterResult;
            } else {
                selectedDictionaryPositions = new boolean[dictionary.getPositionCount()];
                for (int i = 0; i < dictionary.getPositionCount(); i++) {
                    selectedDictionaryPositions[i] = filterFunction.filter(i, blocks);
                }
                inputFilterDictionary = dictionary;
                filterResult = selectedDictionaryPositions;
            }
            for (int i = 0; i < page.getPositionCount(); i++) {
                if (selectedDictionaryPositions[dictionaryBlock.getId(i)]) {
                    selected[index] = i;
                    index++;
                }
            }
            return copyOf(selected, index);
        }
        if (page.getBlock(channel) instanceof RunLengthEncodedBlock) {
            RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) page.getBlock(channel);
            Block[] blocks = new Block[page.getPositionCount()];
            blocks[channel] = rleBlock.getValue();
            if (filterFunction.filter(0, blocks)) {
                return IntStream.range(0, page.getPositionCount()).toArray();
            }
            return new int[0];
        }
    }
    Block[] blocks = page.getBlocks();
    for (int position = 0; position < page.getPositionCount(); position++) {
        if (filterFunction.filter(position, blocks)) {
            selected[index] = position;
            index++;
        }
    }
    return copyOf(selected, index);
}

Example 4 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class MultiChannelGroupByHash method createPageWithExtractedDictionary.

// For a page that contains DictionaryBlocks, create a new page in which
// the dictionaries from the DictionaryBlocks are extracted into the corresponding channels
// From Page(DictionaryBlock1, DictionaryBlock2) create new page with Page(dictionary1, dictionary2)
private Page createPageWithExtractedDictionary(Page page) {
    Block[] blocks = new Block[page.getChannelCount()];
    Block dictionary = ((DictionaryBlock) page.getBlock(channels[0])).getDictionary();
    // extract data dictionary
    blocks[channels[0]] = dictionary;
    // extract hash dictionary
    if (inputHashChannel.isPresent()) {
        blocks[inputHashChannel.get()] = ((DictionaryBlock) page.getBlock(inputHashChannel.get())).getDictionary();
    }
    return new Page(dictionary.getPositionCount(), blocks);
}

Example 5 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class MultiChannelGroupByHash method processDictionary.

private Block processDictionary(Page page) {
    verify(canProcessDictionary(page), "invalid call to processDictionary");
    DictionaryBlock dictionaryBlock = (DictionaryBlock) page.getBlock(channels[0]);
    updateDictionaryLookBack(dictionaryBlock.getDictionary());
    Page dictionaryPage = createPageWithExtractedDictionary(page);
    BlockBuilder blockBuilder = BIGINT.createFixedSizeBlockBuilder(page.getPositionCount());
    for (int i = 0; i < page.getPositionCount(); i++) {
        int positionInDictionary = dictionaryBlock.getId(i);
        int groupId = getGroupId(hashGenerator, dictionaryPage, positionInDictionary);
        BIGINT.writeLong(blockBuilder, groupId);
    }
    verify(blockBuilder.getPositionCount() == page.getPositionCount(), "invalid position count");
    return blockBuilder.build();
}