Examples with BlockBuilder com.facebook.presto.common.block.BlockBuilder used on opensource projects

Example 56 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class TestDictionaryColumnWriter method testDictionaryRetainedSizeWithDifferentSettings.

@Test
public void testDictionaryRetainedSizeWithDifferentSettings() {
    DictionaryColumnWriter ignoredRowGroupWriter = getStringDictionaryColumnWriter(true);
    DictionaryColumnWriter withRowGroupWriter = getStringDictionaryColumnWriter(false);
    int numEntries = 10_000;
    int numBlocks = 10;
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, numEntries);
    Slice slice = utf8Slice("SomeString");
    for (int i = 0; i < numEntries; i++) {
        VARCHAR.writeSlice(blockBuilder, slice);
    }
    Block block = blockBuilder.build();
    for (int i = 0; i < numBlocks; i++) {
        writeBlock(ignoredRowGroupWriter, block);
        writeBlock(withRowGroupWriter, block);
    }
    long ignoredRowGroupBytes = ignoredRowGroupWriter.getRowGroupRetainedSizeInBytes();
    long withRowGroupBytes = withRowGroupWriter.getRowGroupRetainedSizeInBytes();
    long expectedDictionaryIndexSize = (numBlocks * numEntries * SIZE_OF_BYTE);
    String message = String.format("Ignored bytes %s With bytes %s", ignoredRowGroupBytes, withRowGroupBytes);
    assertTrue(ignoredRowGroupBytes + expectedDictionaryIndexSize <= withRowGroupBytes, message);
}

Example 57 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class DeltaShardRewriter method writeDeltaDeleteFile.

@VisibleForTesting
Collection<Slice> writeDeltaDeleteFile(BitSet rowsToDelete) {
    if (rowsToDelete.isEmpty()) {
        return ImmutableList.of();
    }
    // TODO: Under current implementation, one block can only hold INT_MAX many rows
    // which theoretically may not be enough to hold all rows from an ORC file.
    // At this point, rowsToDelete couldn't be empty
    oldDeltaDeleteShardUuid.ifPresent(oldDeltaDeleteShardUuid -> mergeToRowsToDelete(rowsToDelete, oldDeltaDeleteShardUuid));
    if (rowsToDelete.cardinality() == oldShardRowCount) {
        // Delete original file
        return shardDeleteDelta(oldShardUuid, oldDeltaDeleteShardUuid, Optional.empty());
    }
    BlockBuilder blockBuilder = new LongArrayBlockBuilder(null, rowsToDelete.size());
    for (int i = rowsToDelete.nextSetBit(0); i >= 0; i = rowsToDelete.nextSetBit(i + 1)) {
        blockBuilder.writeLong(i);
    }
    // blockToDelete is LongArrayBlock
    StoragePageSink pageSink = orcStorageManager.createStoragePageSink(hdfsContext, transactionId, bucketNumber, ImmutableList.of(0L), ImmutableList.of(BIGINT), true);
    pageSink.appendPages(ImmutableList.of(new Page(blockBuilder.build())));
    List<ShardInfo> shardInfos = getFutureValue(pageSink.commit());
    // Guaranteed that shardInfos only has one element since we only call commit one time
    ShardInfo newDeltaDeleteShard = Iterables.getOnlyElement(shardInfos);
    return shardDeleteDelta(oldShardUuid, oldDeltaDeleteShardUuid, Optional.of(newDeltaDeleteShard));
}

Example 58 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class AvroColumnDecoder method serializeList.

private static Block serializeList(BlockBuilder blockBuilder, Object value, Type type, String columnName) {
    if (value == null) {
        requireNonNull(blockBuilder, "parent blockBuilder is null").appendNull();
        return blockBuilder.build();
    }
    List<?> list = (List) value;
    List<Type> typeParameters = type.getTypeParameters();
    Type elementType = typeParameters.get(0);
    BlockBuilder currentBlockBuilder;
    if (blockBuilder != null) {
        currentBlockBuilder = blockBuilder.beginBlockEntry();
    } else {
        currentBlockBuilder = elementType.createBlockBuilder(null, list.size());
    }
    for (Object element : list) {
        serializeObject(currentBlockBuilder, element, elementType, columnName);
    }
    if (blockBuilder != null) {
        blockBuilder.closeEntry();
        return null;
    }
    return currentBlockBuilder.build();
}

Example 59 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class AvroColumnDecoder method serializeMap.

private static Block serializeMap(BlockBuilder blockBuilder, Object value, Type type, String columnName) {
    if (value == null) {
        requireNonNull(blockBuilder, "parent blockBuilder is null").appendNull();
        return blockBuilder.build();
    }
    Map<?, ?> map = (Map) value;
    List<Type> typeParameters = type.getTypeParameters();
    Type keyType = typeParameters.get(0);
    Type valueType = typeParameters.get(1);
    BlockBuilder entryBuilder;
    boolean builderSynthesized = false;
    if (blockBuilder == null) {
        builderSynthesized = true;
        blockBuilder = type.createBlockBuilder(null, 1);
    }
    entryBuilder = blockBuilder.beginBlockEntry();
    for (Map.Entry<?, ?> entry : map.entrySet()) {
        if (entry.getKey() != null) {
            keyType.writeSlice(entryBuilder, truncateToLength(utf8Slice(entry.getKey().toString()), keyType));
            serializeObject(entryBuilder, entry.getValue(), valueType, columnName);
        }
    }
    blockBuilder.closeEntry();
    if (builderSynthesized) {
        return (Block) type.getObject(blockBuilder, 0);
    }
    return null;
}

Example 60 with BlockBuilder

use of com.facebook.presto.common.block.BlockBuilder in project presto by prestodb.

the class RecordPageSource method getNextPage.

@Override
public Page getNextPage() {
    if (!closed) {
        int i;
        for (i = 0; i < ROWS_PER_REQUEST; i++) {
            if (pageBuilder.isFull()) {
                break;
            }
            if (!cursor.advanceNextPosition()) {
                closed = true;
                break;
            }
            completedPositions++;
            pageBuilder.declarePosition();
            for (int column = 0; column < types.size(); column++) {
                BlockBuilder output = pageBuilder.getBlockBuilder(column);
                if (cursor.isNull(column)) {
                    output.appendNull();
                } else {
                    Type type = types.get(column);
                    Class<?> javaType = type.getJavaType();
                    if (javaType == boolean.class) {
                        type.writeBoolean(output, cursor.getBoolean(column));
                    } else if (javaType == long.class) {
                        type.writeLong(output, cursor.getLong(column));
                    } else if (javaType == double.class) {
                        type.writeDouble(output, cursor.getDouble(column));
                    } else if (javaType == Slice.class) {
                        Slice slice = cursor.getSlice(column);
                        type.writeSlice(output, slice, 0, slice.length());
                    } else {
                        type.writeObject(output, cursor.getObject(column));
                    }
                }
            }
        }
    }
    // only return a page if the buffer is full or we are finishing
    if (pageBuilder.isEmpty() || (!closed && !pageBuilder.isFull())) {
        return null;
    }
    Page page = pageBuilder.build();
    pageBuilder.reset();
    return page;
}