Examples with DictionaryId com.facebook.presto.spi.block.DictionaryId used on opensource projects

Example 1 with DictionaryId

use of com.facebook.presto.spi.block.DictionaryId in project presto by prestodb.

the class GenericPageProcessor method processColumnarDictionary.

@Override
public Page processColumnarDictionary(ConnectorSession session, Page page, List<? extends Type> types) {
    Page inputPage = getNonLazyPage(page);
    int[] selectedPositions = filterPage(inputPage);
    Map<DictionaryId, DictionaryId> dictionarySourceIds = new HashMap<>();
    if (selectedPositions.length == 0) {
        return null;
    }
    if (projections.isEmpty()) {
        return new Page(selectedPositions.length);
    }
    PageBuilder pageBuilder = new PageBuilder(types);
    Block[] inputBlocks = page.getBlocks();
    Block[] outputBlocks = new Block[projections.size()];
    for (int projectionIndex = 0; projectionIndex < projections.size(); projectionIndex++) {
        ProjectionFunction projection = projections.get(projectionIndex);
        if (canDictionaryProcess(projection, inputPage)) {
            outputBlocks[projectionIndex] = projectColumnarDictionary(inputPage, selectedPositions, projection, dictionarySourceIds);
        } else {
            outputBlocks[projectionIndex] = projectColumnar(selectedPositions, pageBuilder.getBlockBuilder(projectionIndex), inputBlocks, projection).build();
        }
    }
    for (Block block : outputBlocks) {
        verify(block.getPositionCount() == selectedPositions.length);
    }
    return new Page(selectedPositions.length, outputBlocks);
}

Example 2 with DictionaryId

use of com.facebook.presto.spi.block.DictionaryId in project presto by prestodb.

the class Page method compactRelatedBlocks.

private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks) {
    DictionaryBlock firstDictionaryBlock = blocks.get(0);
    Block dictionary = firstDictionaryBlock.getDictionary();
    int positionCount = firstDictionaryBlock.getPositionCount();
    int dictionarySize = dictionary.getPositionCount();
    // determine which dictionary entries are referenced and build a reindex for them
    List<Integer> dictionaryPositionsToCopy = new ArrayList<>(min(dictionarySize, positionCount));
    int[] remapIndex = new int[dictionarySize];
    Arrays.fill(remapIndex, -1);
    int newIndex = 0;
    for (int i = 0; i < positionCount; i++) {
        int position = firstDictionaryBlock.getId(i);
        if (remapIndex[position] == -1) {
            dictionaryPositionsToCopy.add(position);
            remapIndex[position] = newIndex;
            newIndex++;
        }
    }
    // entire dictionary is referenced
    if (dictionaryPositionsToCopy.size() == dictionarySize) {
        return blocks;
    }
    // compact the dictionaries
    int[] newIds = getNewIds(positionCount, firstDictionaryBlock, remapIndex);
    List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
    DictionaryId newDictionaryId = randomDictionaryId();
    for (DictionaryBlock dictionaryBlock : blocks) {
        if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
            throw new IllegalArgumentException("dictionarySourceIds must be the same");
        }
        try {
            Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy);
            outputDictionaryBlocks.add(new DictionaryBlock(positionCount, compactDictionary, newIds, true, newDictionaryId));
        } catch (UnsupportedOperationException e) {
            // ignore if copy positions is not supported for the dictionary
            outputDictionaryBlocks.add(dictionaryBlock);
        }
    }
    return outputDictionaryBlocks;
}

Example 3 with DictionaryId

use of com.facebook.presto.spi.block.DictionaryId in project presto by prestodb.

the class Page method getRelatedDictionaryBlocks.

private Map<DictionaryId, DictionaryBlockIndexes> getRelatedDictionaryBlocks() {
    Map<DictionaryId, DictionaryBlockIndexes> relatedDictionaryBlocks = new HashMap<>();
    for (int i = 0; i < blocks.length; i++) {
        Block block = blocks[i];
        if (block instanceof DictionaryBlock) {
            DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
            relatedDictionaryBlocks.computeIfAbsent(dictionaryBlock.getDictionarySourceId(), id -> new DictionaryBlockIndexes()).addBlock(dictionaryBlock, i);
        }
    }
    return relatedDictionaryBlocks;
}

Example 4 with DictionaryId

use of com.facebook.presto.spi.block.DictionaryId in project presto by prestodb.

the class GenericPageProcessor method projectColumnarDictionary.

private Block projectColumnarDictionary(Page inputPage, int[] selectedPositions, ProjectionFunction projection, Map<DictionaryId, DictionaryId> dictionarySourceIds) {
    int inputChannel = getOnlyElement(projection.getInputChannels());
    Block[] blocks = new Block[inputPage.getChannelCount()];
    if (inputPage.getBlock(inputChannel) instanceof RunLengthEncodedBlock) {
        RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) inputPage.getBlock(inputChannel);
        BlockBuilder builder = projection.getType().createBlockBuilder(new BlockBuilderStatus(), 1);
        blocks[inputChannel] = rleBlock.getValue();
        projection.project(0, blocks, builder);
        return new RunLengthEncodedBlock(builder.build(), selectedPositions.length);
    }
    Block outputDictionary = projectDictionary(projection, inputPage);
    int[] outputIds = filterIds(projection, inputPage, selectedPositions);
    DictionaryBlock dictionaryBlock = (DictionaryBlock) inputPage.getBlock(inputChannel);
    DictionaryId sourceId = dictionarySourceIds.get(dictionaryBlock.getDictionarySourceId());
    if (sourceId == null) {
        sourceId = randomDictionaryId();
        dictionarySourceIds.put(dictionaryBlock.getDictionarySourceId(), sourceId);
    }
    return new DictionaryBlock(selectedPositions.length, outputDictionary, outputIds, false, sourceId);
}

Example 5 with DictionaryId

use of com.facebook.presto.spi.block.DictionaryId in project presto by prestodb.

the class TestPage method testCompactDictionaryBlocks.

@Test
public void testCompactDictionaryBlocks() throws Exception {
    int positionCount = 100;
    // Create 2 dictionary blocks with the same source id
    DictionaryId commonSourceId = randomDictionaryId();
    int commonDictionaryUsedPositions = 20;
    int[] commonDictionaryIds = getDictionaryIds(positionCount, commonDictionaryUsedPositions);
    // first dictionary contains "varbinary" values
    Slice[] dictionaryValues1 = createExpectedValues(50);
    SliceArrayBlock dictionary1 = new SliceArrayBlock(dictionaryValues1.length, dictionaryValues1);
    DictionaryBlock commonSourceIdBlock1 = new DictionaryBlock(positionCount, dictionary1, commonDictionaryIds, commonSourceId);
    // second dictionary block is "length(firstColumn)"
    BlockBuilder dictionary2 = BIGINT.createBlockBuilder(new BlockBuilderStatus(), dictionary1.getPositionCount());
    for (Slice expectedValue : dictionaryValues1) {
        BIGINT.writeLong(dictionary2, expectedValue.length());
    }
    DictionaryBlock commonSourceIdBlock2 = new DictionaryBlock(positionCount, dictionary2.build(), commonDictionaryIds, commonSourceId);
    // Create block with a different source id, dictionary size, used
    int otherDictionaryUsedPositions = 30;
    int[] otherDictionaryIds = getDictionaryIds(positionCount, otherDictionaryUsedPositions);
    SliceArrayBlock dictionary3 = new SliceArrayBlock(70, createExpectedValues(70));
    DictionaryBlock randomSourceIdBlock = new DictionaryBlock(positionCount, dictionary3, otherDictionaryIds);
    Page page = new Page(commonSourceIdBlock1, randomSourceIdBlock, commonSourceIdBlock2);
    page.compact();
    // dictionary blocks should all be compact
    assertTrue(((DictionaryBlock) page.getBlock(0)).isCompact());
    assertTrue(((DictionaryBlock) page.getBlock(1)).isCompact());
    assertTrue(((DictionaryBlock) page.getBlock(2)).isCompact());
    assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionary().getPositionCount(), commonDictionaryUsedPositions);
    assertEquals(((DictionaryBlock) page.getBlock(1)).getDictionary().getPositionCount(), otherDictionaryUsedPositions);
    assertEquals(((DictionaryBlock) page.getBlock(2)).getDictionary().getPositionCount(), commonDictionaryUsedPositions);
    // Blocks that had the same source id before compacting page should have the same source id after compacting page
    assertNotEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(1)).getDictionarySourceId());
    assertEquals(((DictionaryBlock) page.getBlock(0)).getDictionarySourceId(), ((DictionaryBlock) page.getBlock(2)).getDictionarySourceId());
}