Examples with DictionaryBlock com.facebook.presto.common.block.DictionaryBlock used on opensource projects

Example 11 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class FilterFunction method filterWithDictionary.

private int filterWithDictionary(Page page, int[] positions, int positionCount, RuntimeException[] errors) {
    int outputCount = 0;
    DictionaryBlock block = (DictionaryBlock) page.getBlock(0);
    Block dictionary = block.getDictionary();
    if (dictionary != previousDictionary) {
        previousDictionary = dictionary;
        int numEntries = dictionary.getPositionCount();
        dictionaryPage = new Page(numEntries, dictionary);
        dictionaryResults = ensureCapacity(dictionaryResults, numEntries);
        fill(dictionaryResults, 0, numEntries, FILTER_NOT_EVALUATED);
    }
    for (int i = 0; i < positionCount; i++) {
        int position = positions[i];
        int dictionaryPosition = block.getId(position);
        byte result = dictionaryResults[dictionaryPosition];
        switch(result) {
            case FILTER_FAILED:
                continue;
            case FILTER_PASSED:
                positions[outputCount] = position;
                errors[outputCount] = errors[i];
                outputCount++;
                continue;
            case FILTER_NOT_EVALUATED:
                try {
                    if (predicate.evaluate(properties, dictionaryPage, dictionaryPosition)) {
                        positions[outputCount] = position;
                        errors[outputCount] = errors[i];
                        outputCount++;
                        dictionaryResults[dictionaryPosition] = FILTER_PASSED;
                    } else {
                        dictionaryResults[dictionaryPosition] = FILTER_FAILED;
                    }
                } catch (RuntimeException e) {
                    // We do not record errors in the dictionary results.
                    positions[outputCount] = position;
                    // keep last error
                    errors[outputCount] = e;
                    outputCount++;
                }
                break;
            default:
                verify(false, "Unexpected filter result: " + result);
        }
    }
    return outputCount;
}

Example 12 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class OptimizedPartitionedOutputOperator method decodeBlock.

/**
 * Flatten the block and convert the nested-typed block into ColumnarArray/Map/Row.
 * For performance considerations we decode the block only once for each block instead of for each batch.
 *
 * @return A tree structure that contains the decoded block
 */
@VisibleForTesting
static DecodedBlockNode decodeBlock(BlockFlattener flattener, Closer blockLeaseCloser, Block block) {
    BlockLease lease = flattener.flatten(block);
    blockLeaseCloser.register(lease::close);
    Block decodedBlock = lease.get();
    long estimatedSizeInBytes = decodedBlock.getLogicalSizeInBytes();
    if (decodedBlock instanceof ArrayBlock) {
        ColumnarArray columnarArray = ColumnarArray.toColumnarArray(decodedBlock);
        Block childBlock = columnarArray.getElementsBlock();
        return new DecodedBlockNode(columnarArray, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), columnarArray.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof MapBlock) {
        ColumnarMap columnarMap = ColumnarMap.toColumnarMap(decodedBlock);
        Block keyBlock = columnarMap.getKeysBlock();
        Block valueBlock = columnarMap.getValuesBlock();
        return new DecodedBlockNode(columnarMap, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, keyBlock), decodeBlock(flattener, blockLeaseCloser, valueBlock)), columnarMap.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RowBlock) {
        ColumnarRow columnarRow = ColumnarRow.toColumnarRow(decodedBlock);
        ImmutableList.Builder<DecodedBlockNode> children = ImmutableList.builder();
        for (int i = 0; i < columnarRow.getFieldCount(); i++) {
            Block childBlock = columnarRow.getField(i);
            children.add(decodeBlock(flattener, blockLeaseCloser, childBlock));
        }
        return new DecodedBlockNode(columnarRow, children.build(), columnarRow.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof DictionaryBlock) {
        Block dictionary = ((DictionaryBlock) decodedBlock).getDictionary();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, dictionary)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RunLengthEncodedBlock) {
        Block childBlock = ((RunLengthEncodedBlock) decodedBlock).getValue();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    return new DecodedBlockNode(decodedBlock, ImmutableList.of(), block.getRetainedSizeInBytes(), estimatedSizeInBytes);
}

Example 13 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class BlockAssertions method createStringDictionaryBlock.

public static Block createStringDictionaryBlock(int start, int length) {
    checkArgument(length > 5, "block must have more than 5 entries");
    int dictionarySize = length / 5;
    BlockBuilder builder = VARCHAR.createBlockBuilder(null, dictionarySize);
    for (int i = start; i < start + dictionarySize; i++) {
        VARCHAR.writeString(builder, String.valueOf(i));
    }
    int[] ids = new int[length];
    for (int i = 0; i < length; i++) {
        ids[i] = i % dictionarySize;
    }
    return new DictionaryBlock(builder.build(), ids);
}

Example 14 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestColumnarArray method assertDictionaryBlock.

private static <T> void assertDictionaryBlock(Block block, T[] expectedValues) {
    DictionaryBlock dictionaryBlock = createTestDictionaryBlock(block);
    T[] expectedDictionaryValues = createTestDictionaryExpectedValues(expectedValues);
    assertBlock(dictionaryBlock, expectedDictionaryValues);
    assertColumnarArray(dictionaryBlock, expectedDictionaryValues);
    assertRunLengthEncodedBlock(dictionaryBlock, expectedDictionaryValues);
}

Example 15 with DictionaryBlock

use of com.facebook.presto.common.block.DictionaryBlock in project presto by prestodb.

the class TestDictionaryBlock method testBasicGetPositions.

@Test
public void testBasicGetPositions() {
    Slice[] expectedValues = createExpectedValues(10);
    Block dictionaryBlock = new DictionaryBlock(createSlicesBlock(expectedValues), new int[] { 0, 1, 2, 3, 4, 5 });
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[0], expectedValues[1], expectedValues[2], expectedValues[3], expectedValues[4], expectedValues[5] });
    DictionaryId dictionaryId = ((DictionaryBlock) dictionaryBlock).getDictionarySourceId();
    // first getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 8, 1, 2, 4, 5, 7, 9 }, 2, 4);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[4], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // second getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 1, 3, 0, 0 }, 0, 3);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // third getPositions; we do not validate if -1 is an invalid position
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { -1, -1, 0, 1, 2 }, 2, 3);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[2], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // mixed getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 0, 2, 2 }, 0, 3);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[1], expectedValues[5], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // duplicated getPositions
    dictionaryBlock = dictionaryBlock.getPositions(new int[] { 1, 1, 1, 1, 1 }, 0, 5);
    assertBlock(dictionaryBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] { expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5], expectedValues[5] });
    assertEquals(((DictionaryBlock) dictionaryBlock).getDictionarySourceId(), dictionaryId);
    // out of range
    for (int position : ImmutableList.of(-1, 6)) {
        try {
            dictionaryBlock.getPositions(new int[] { position }, 0, 1);
            fail("Expected to fail");
        } catch (IllegalArgumentException e) {
            assertTrue(e.getMessage().startsWith("Invalid position"));
        }
    }
    for (int offset : ImmutableList.of(-1, 6)) {
        try {
            dictionaryBlock.getPositions(new int[] { 0 }, offset, 1);
            fail("Expected to fail");
        } catch (IndexOutOfBoundsException e) {
            assertTrue(e.getMessage().startsWith("Invalid offset"));
        }
    }
    for (int length : ImmutableList.of(-1, 6)) {
        try {
            dictionaryBlock.getPositions(new int[] { 0 }, 0, length);
            fail("Expected to fail");
        } catch (IndexOutOfBoundsException e) {
            assertTrue(e.getMessage().startsWith("Invalid offset"));
        }
    }
}