Examples with ColumnarMap com.facebook.presto.common.block.ColumnarMap used on opensource projects

Example 1 with ColumnarMap

use of com.facebook.presto.common.block.ColumnarMap in project presto by prestodb.

the class MapColumnWriter method writeBlock.

@Override
public long writeBlock(Block block) {
    checkState(!closed);
    checkArgument(block.getPositionCount() > 0, "Block is empty");
    ColumnarMap columnarMap = toColumnarMap(block);
    return writeColumnarMap(columnarMap);
}

Example 2 with ColumnarMap

use of com.facebook.presto.common.block.ColumnarMap in project presto by prestodb.

the class TestUnnesterUtil method calculateMaxCardinalities.

static int[] calculateMaxCardinalities(Page page, List<Type> replicatedTypes, List<Type> unnestTypes) {
    int positionCount = page.getPositionCount();
    int[] maxCardinalities = new int[positionCount];
    int replicatedChannelCount = replicatedTypes.size();
    int unnestChannelCount = unnestTypes.size();
    for (int i = 0; i < unnestChannelCount; i++) {
        Type type = unnestTypes.get(i);
        Block block = page.getBlock(replicatedChannelCount + i);
        assertTrue(type instanceof ArrayType || type instanceof MapType);
        if (type instanceof ArrayType) {
            ColumnarArray columnarArray = ColumnarArray.toColumnarArray(block);
            for (int j = 0; j < positionCount; j++) {
                maxCardinalities[j] = max(maxCardinalities[j], columnarArray.getLength(j));
            }
        } else if (type instanceof MapType) {
            ColumnarMap columnarMap = ColumnarMap.toColumnarMap(block);
            for (int j = 0; j < positionCount; j++) {
                maxCardinalities[j] = max(maxCardinalities[j], columnarMap.getEntryCount(j));
            }
        } else {
            fail("expected an ArrayType or MapType, but found " + type);
        }
    }
    return maxCardinalities;
}

Example 3 with ColumnarMap

use of com.facebook.presto.common.block.ColumnarMap in project presto by prestodb.

the class OptimizedPartitionedOutputOperator method decodeBlock.

/**
 * Flatten the block and convert the nested-typed block into ColumnarArray/Map/Row.
 * For performance considerations we decode the block only once for each block instead of for each batch.
 *
 * @return A tree structure that contains the decoded block
 */
@VisibleForTesting
static DecodedBlockNode decodeBlock(BlockFlattener flattener, Closer blockLeaseCloser, Block block) {
    BlockLease lease = flattener.flatten(block);
    blockLeaseCloser.register(lease::close);
    Block decodedBlock = lease.get();
    long estimatedSizeInBytes = decodedBlock.getLogicalSizeInBytes();
    if (decodedBlock instanceof ArrayBlock) {
        ColumnarArray columnarArray = ColumnarArray.toColumnarArray(decodedBlock);
        Block childBlock = columnarArray.getElementsBlock();
        return new DecodedBlockNode(columnarArray, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), columnarArray.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof MapBlock) {
        ColumnarMap columnarMap = ColumnarMap.toColumnarMap(decodedBlock);
        Block keyBlock = columnarMap.getKeysBlock();
        Block valueBlock = columnarMap.getValuesBlock();
        return new DecodedBlockNode(columnarMap, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, keyBlock), decodeBlock(flattener, blockLeaseCloser, valueBlock)), columnarMap.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RowBlock) {
        ColumnarRow columnarRow = ColumnarRow.toColumnarRow(decodedBlock);
        ImmutableList.Builder<DecodedBlockNode> children = ImmutableList.builder();
        for (int i = 0; i < columnarRow.getFieldCount(); i++) {
            Block childBlock = columnarRow.getField(i);
            children.add(decodeBlock(flattener, blockLeaseCloser, childBlock));
        }
        return new DecodedBlockNode(columnarRow, children.build(), columnarRow.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof DictionaryBlock) {
        Block dictionary = ((DictionaryBlock) decodedBlock).getDictionary();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, dictionary)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    if (decodedBlock instanceof RunLengthEncodedBlock) {
        Block childBlock = ((RunLengthEncodedBlock) decodedBlock).getValue();
        return new DecodedBlockNode(decodedBlock, ImmutableList.of(decodeBlock(flattener, blockLeaseCloser, childBlock)), decodedBlock.getRetainedSizeInBytes(), estimatedSizeInBytes);
    }
    return new DecodedBlockNode(decodedBlock, ImmutableList.of(), block.getRetainedSizeInBytes(), estimatedSizeInBytes);
}

Example 4 with ColumnarMap

use of com.facebook.presto.common.block.ColumnarMap in project presto by prestodb.

the class TestColumnarMap method assertColumnarMap.

private static void assertColumnarMap(Block block, Slice[][][] expectedValues) {
    ColumnarMap columnarMap = toColumnarMap(block);
    assertEquals(columnarMap.getPositionCount(), expectedValues.length);
    Block keysBlock = columnarMap.getKeysBlock();
    Block valuesBlock = columnarMap.getValuesBlock();
    int elementsPosition = 0;
    for (int position = 0; position < expectedValues.length; position++) {
        Slice[][] expectedMap = expectedValues[position];
        assertEquals(columnarMap.isNull(position), expectedMap == null);
        assertEquals(columnarMap.getOffset(position), elementsPosition);
        if (expectedMap == null) {
            assertEquals(columnarMap.getEntryCount(position), 0);
            continue;
        }
        assertEquals(columnarMap.getEntryCount(position), expectedMap.length);
        for (int i = 0; i < columnarMap.getEntryCount(position); i++) {
            Slice[] expectedEntry = expectedMap[i];
            Slice expectedKey = expectedEntry[0];
            assertBlockPosition(keysBlock, elementsPosition, expectedKey);
            Slice expectedValue = expectedEntry[1];
            assertBlockPosition(valuesBlock, elementsPosition, expectedValue);
            elementsPosition++;
        }
    }
}

Example 5 with ColumnarMap

use of com.facebook.presto.common.block.ColumnarMap in project presto by prestodb.

the class TestColumnarMap method testMapWithNullKeys.

@Test
public void testMapWithNullKeys() {
    // Presto Query Engine does not support Map with Null keys.
    // Presto ORC reader and Writer are used as library in some other
    // projects and it requires null keys to be supported in the Map.
    MapBlockBuilder mapBlockBuilder = createMapBuilder(100);
    mapBlockBuilder.appendNull();
    for (int i = 1; i < 100; i++) {
        Slice valueSlice = Slices.utf8Slice(Integer.toString(i));
        SingleMapBlockWriter blockWriter = mapBlockBuilder.beginBlockEntry();
        // Null Key
        blockWriter.appendNull();
        VARCHAR.writeSlice(blockWriter, valueSlice);
        for (int j = 1; j < i; j++) {
            Slice keySlice = Slices.utf8Slice(Integer.toString(j));
            VARCHAR.writeSlice(blockWriter, keySlice);
            VARCHAR.writeSlice(blockWriter, valueSlice);
        }
        mapBlockBuilder.closeEntry();
    }
    ColumnarMap columnarMap = toColumnarMap(mapBlockBuilder.build());
    assertEquals(columnarMap.getPositionCount(), 100);
    assertTrue(columnarMap.isNull(0));
    for (int i = 1; i < 100; i++) {
        assertFalse(columnarMap.isNull(i));
        Slice valueSlice = Slices.utf8Slice(Integer.toString(i));
        int offset = columnarMap.getOffset(i);
        assertTrue(columnarMap.getKeysBlock().isNull(offset));
        verifySlice(columnarMap.getValuesBlock(), offset, valueSlice);
        for (int j = 1; j < i; j++) {
            Slice keySlice = Slices.utf8Slice(Integer.toString(j));
            verifySlice(columnarMap.getKeysBlock(), offset + j, keySlice);
            verifySlice(columnarMap.getValuesBlock(), offset + j, valueSlice);
        }
    }
}