Examples with DictionaryBlock com.facebook.presto.spi.block.DictionaryBlock used on opensource projects

Example 11 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class GenericPageProcessor method filterIds.

private static int[] filterIds(ProjectionFunction projection, Page page, int[] selectedPositions) {
    DictionaryBlock dictionaryBlock = (DictionaryBlock) page.getBlock(getOnlyElement(projection.getInputChannels()));
    int[] outputIds = new int[selectedPositions.length];
    for (int pos = 0; pos < selectedPositions.length; pos++) {
        outputIds[pos] = dictionaryBlock.getId(selectedPositions[pos]);
    }
    return outputIds;
}

Example 12 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class GenericPageProcessor method canDictionaryProcess.

private static boolean canDictionaryProcess(ProjectionFunction projection, Page inputPage) {
    if (!projection.isDeterministic()) {
        return false;
    }
    Set<Integer> inputChannels = projection.getInputChannels();
    if (inputChannels.size() != 1) {
        return false;
    }
    Block block = inputPage.getBlock(getOnlyElement(inputChannels));
    return block instanceof DictionaryBlock || block instanceof RunLengthEncodedBlock;
}

Example 13 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class MultiChannelGroupByHash method canProcessDictionary.

private boolean canProcessDictionary(Page page) {
    boolean processDictionary = this.processDictionary && channels.length == 1 && page.getBlock(channels[0]) instanceof DictionaryBlock;
    if (processDictionary && inputHashChannel.isPresent()) {
        Block inputHashBlock = page.getBlock(inputHashChannel.get());
        DictionaryBlock inputDataBlock = (DictionaryBlock) page.getBlock(channels[0]);
        verify(inputHashBlock instanceof DictionaryBlock, "data channel is dictionary encoded but hash channel is not");
        verify(((DictionaryBlock) inputHashBlock).getDictionarySourceId().equals(inputDataBlock.getDictionarySourceId()), "dictionarySourceIds of data block and hash block do not match");
    }
    return processDictionary;
}

Example 14 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class PageProcessorCompiler method getBytecodeFilterOnDictionary.

private static BytecodeBlock getBytecodeFilterOnDictionary(Parameter session, Scope scope, Variable blockVariable) {
    Variable position = scope.getVariable("position");
    Variable positionCount = scope.getVariable("positionCount");
    Variable selectedCount = scope.getVariable("selectedCount");
    Variable selectedPositions = scope.getVariable("selectedPositions");
    Variable thisVariable = scope.getThis();
    BytecodeExpression inputFilterDictionary = thisVariable.getField("inputFilterDictionary", Block.class);
    BytecodeExpression filterResult = thisVariable.getField("filterResult", boolean[].class);
    BytecodeBlock ifFilterOnDictionaryBlock = new BytecodeBlock();
    Variable dictionaryBlock = scope.declareVariable("dictionaryBlock", ifFilterOnDictionaryBlock, blockVariable.cast(DictionaryBlock.class));
    Variable dictionary = scope.declareVariable("dictionary", ifFilterOnDictionaryBlock, dictionaryBlock.invoke("getDictionary", Block.class));
    Variable dictionaryPositionCount = scope.declareVariable("dictionaryPositionCount", ifFilterOnDictionaryBlock, dictionary.invoke("getPositionCount", int.class));
    Variable selectedDictionaryPositions = scope.declareVariable("selectedDictionaryPositions", ifFilterOnDictionaryBlock, newArray(type(boolean[].class), dictionaryPositionCount));
    // if cached value is available use it else filter dictionary and cache it
    ifFilterOnDictionaryBlock.append(new IfStatement().condition(equal(dictionary, inputFilterDictionary)).ifTrue(selectedDictionaryPositions.set(filterResult)).ifFalse(new BytecodeBlock().append(new ForLoop().initialize(position.set(constantInt(0))).condition(lessThan(position, dictionaryPositionCount)).update(position.increment()).body(selectedDictionaryPositions.setElement(position, invokeFilter(thisVariable, session, singletonList(dictionary), position)))).append(thisVariable.setField("inputFilterDictionary", dictionary)).append(thisVariable.setField("filterResult", selectedDictionaryPositions))));
    // create selected positions
    ifFilterOnDictionaryBlock.append(new ForLoop().initialize(position.set(constantInt(0))).condition(lessThan(position, positionCount)).update(position.increment()).body(new IfStatement().condition(selectedDictionaryPositions.getElement(dictionaryBlock.invoke("getId", int.class, position))).ifTrue(new BytecodeBlock().append(selectedPositions.setElement(selectedCount, position)).append(selectedCount.increment()))));
    // return selectedPositions
    ifFilterOnDictionaryBlock.append(invokeStatic(Arrays.class, "copyOf", int[].class, selectedPositions, selectedCount).ret());
    return ifFilterOnDictionaryBlock;
}

Example 15 with DictionaryBlock

use of com.facebook.presto.spi.block.DictionaryBlock in project presto by prestodb.

the class HivePageSink method writePage.

private void writePage(Page page) {
    int[] writerIndexes = getWriterIndexes(page);
    // record which positions are used by which writer
    for (int position = 0; position < page.getPositionCount(); position++) {
        int writerIndex = writerIndexes[position];
        writerPositions.get(writerIndex).add(position);
    }
    // invoke the writers
    Page dataPage = getDataPage(page);
    IntSet writersUsed = new IntArraySet(writerIndexes);
    for (IntIterator iterator = writersUsed.iterator(); iterator.hasNext(); ) {
        int writerIndex = iterator.nextInt();
        WriterPositions currentWriterPositions = writerPositions.get(writerIndex);
        if (currentWriterPositions.isEmpty()) {
            continue;
        }
        // If write is partitioned across multiple writers, filter page using dictionary blocks
        Page pageForWriter = dataPage;
        if (currentWriterPositions.size() != dataPage.getPositionCount()) {
            Block[] blocks = new Block[dataPage.getChannelCount()];
            for (int channel = 0; channel < dataPage.getChannelCount(); channel++) {
                blocks[channel] = new DictionaryBlock(currentWriterPositions.size(), dataPage.getBlock(channel), currentWriterPositions.getPositionsArray());
            }
            pageForWriter = new Page(currentWriterPositions.size(), blocks);
        }
        HiveWriter writer = writers.get(writerIndex);
        long currentMemory = writer.getSystemMemoryUsage();
        writer.append(pageForWriter);
        systemMemoryUsage += (writer.getSystemMemoryUsage() - currentMemory);
        currentWriterPositions.clear();
    }
}