Examples with Block com.facebook.presto.spi.block.Block used on opensource projects

Example 1 with Block

use of com.facebook.presto.spi.block.Block in project presto by prestodb.

the class HivePageSink method getWriterIndexes.

private int[] getWriterIndexes(Page page) {
    Page partitionColumns = extractColumns(page, partitionColumnsInputIndex);
    Block bucketBlock = buildBucketBlock(page);
    int[] writerIndexes = pagePartitioner.partitionPage(partitionColumns, bucketBlock);
    if (pagePartitioner.getMaxIndex() >= maxOpenWriters) {
        throw new PrestoException(HIVE_TOO_MANY_OPEN_PARTITIONS, "Too many open partitions");
    }
    // expand writers list to new size
    while (writers.size() <= pagePartitioner.getMaxIndex()) {
        writers.add(null);
        WriterPositions newWriterPositions = new WriterPositions();
        systemMemoryUsage += sizeOf(newWriterPositions.getPositionsArray());
        writerPositions.add(newWriterPositions);
    }
    // create missing writers
    for (int position = 0; position < page.getPositionCount(); position++) {
        int writerIndex = writerIndexes[position];
        if (writers.get(writerIndex) != null) {
            continue;
        }
        OptionalInt bucketNumber = OptionalInt.empty();
        if (bucketBlock != null) {
            bucketNumber = OptionalInt.of(bucketBlock.getInt(position, 0));
        }
        HiveWriter writer = writerFactory.createWriter(partitionColumns, position, bucketNumber);
        writers.set(writerIndex, writer);
    }
    verify(writers.size() == pagePartitioner.getMaxIndex() + 1);
    verify(!writers.contains(null));
    return writerIndexes;
}

Example 2 with Block

use of com.facebook.presto.spi.block.Block in project presto by prestodb.

the class HivePageSource method getNextPage.

@Override
public Page getNextPage() {
    try {
        Block[] blocks = new Block[columnMappings.size()];
        Page dataPage = delegate.getNextPage();
        if (dataPage == null) {
            return null;
        }
        int batchSize = dataPage.getPositionCount();
        for (int fieldId = 0; fieldId < blocks.length; fieldId++) {
            ColumnMapping columnMapping = columnMappings.get(fieldId);
            if (columnMapping.isPrefilled()) {
                blocks[fieldId] = RunLengthEncodedBlock.create(types[fieldId], prefilledValues[fieldId], batchSize);
            } else {
                blocks[fieldId] = dataPage.getBlock(columnMapping.getIndex());
                if (coercers[fieldId] != null) {
                    blocks[fieldId] = new LazyBlock(batchSize, new CoercionLazyBlockLoader(blocks[fieldId], coercers[fieldId]));
                }
            }
        }
        return new Page(batchSize, blocks);
    } catch (PrestoException e) {
        closeWithSuppression(e);
        throw e;
    } catch (RuntimeException e) {
        closeWithSuppression(e);
        throw new PrestoException(HIVE_CURSOR_ERROR, e);
    }
}

Example 3 with Block

use of com.facebook.presto.spi.block.Block in project presto by prestodb.

the class ParquetReader method readMap.

private Block readMap(Type type, List<String> path, IntList elementOffsets) throws IOException {
    List<Type> parameters = type.getTypeParameters();
    checkArgument(parameters.size() == 2, "Maps must have two type parameters, found %d", parameters.size());
    Block[] blocks = new Block[parameters.size()];
    IntList keyOffsets = new IntArrayList();
    IntList valueOffsets = new IntArrayList();
    path.add(MAP_TYPE_NAME);
    blocks[0] = readBlock(MAP_KEY_NAME, parameters.get(0), path, keyOffsets);
    blocks[1] = readBlock(MAP_VALUE_NAME, parameters.get(1), path, valueOffsets);
    path.remove(MAP_TYPE_NAME);
    if (blocks[0].getPositionCount() == 0) {
        for (int i = 0; i < batchSize; i++) {
            elementOffsets.add(0);
        }
        return RunLengthEncodedBlock.create(parameters.get(0), null, batchSize);
    }
    InterleavedBlock interleavedBlock = new InterleavedBlock(new Block[] { blocks[0], blocks[1] });
    int[] offsets = new int[batchSize + 1];
    for (int i = 1; i < offsets.length; i++) {
        int elementPositionCount = keyOffsets.getInt(i - 1) * 2;
        elementOffsets.add(elementPositionCount);
        offsets[i] = offsets[i - 1] + elementPositionCount;
    }
    return new ArrayBlock(batchSize, new boolean[batchSize], offsets, interleavedBlock);
}

Example 4 with Block

use of com.facebook.presto.spi.block.Block in project presto by prestodb.

the class ParquetReader method readBlock.

private Block readBlock(String name, Type type, List<String> path, IntList offsets) throws IOException {
    path.add(name);
    Optional<RichColumnDescriptor> descriptor = getDescriptor(fileSchema, requestedSchema, path);
    if (!descriptor.isPresent()) {
        path.remove(name);
        return RunLengthEncodedBlock.create(type, null, batchSize);
    }
    Block block;
    if (ROW.equals(type.getTypeSignature().getBase())) {
        block = readStruct(type, path, offsets);
    } else if (MAP.equals(type.getTypeSignature().getBase())) {
        block = readMap(type, path, offsets);
    } else if (ARRAY.equals(type.getTypeSignature().getBase())) {
        block = readArray(type, path, offsets);
    } else {
        block = readPrimitive(descriptor.get(), type, offsets);
    }
    path.remove(name);
    return block;
}

Example 5 with Block

use of com.facebook.presto.spi.block.Block in project presto by prestodb.

the class TestSerDeUtils method testMapBlock.

@Test
public void testMapBlock() {
    MapHolder holder = new MapHolder();
    holder.map = new TreeMap<>();
    holder.map.put("twelve", new InnerStruct(13, 14L));
    holder.map.put("fifteen", new InnerStruct(16, 17L));
    com.facebook.presto.spi.type.Type rowType = new RowType(ImmutableList.of(INTEGER, BIGINT), Optional.empty());
    com.facebook.presto.spi.type.Type mapOfVarcharRowType = new RowType(ImmutableList.of(new MapType(createUnboundedVarcharType(), rowType)), Optional.empty());
    Block actual = toBinaryBlock(mapOfVarcharRowType, holder, getInspector(MapHolder.class));
    BlockBuilder blockBuilder = new InterleavedBlockBuilder(ImmutableList.of(createUnboundedVarcharType(), rowType), new BlockBuilderStatus(), 1024);
    createUnboundedVarcharType().writeString(blockBuilder, "fifteen");
    rowType.writeObject(blockBuilder, rowBlockOf(ImmutableList.of(INTEGER, BIGINT), 16, 17L));
    createUnboundedVarcharType().writeString(blockBuilder, "twelve");
    rowType.writeObject(blockBuilder, rowBlockOf(ImmutableList.of(INTEGER, BIGINT), 13, 14L));
    Block expected = rowBlockOf(ImmutableList.of(new MapType(createUnboundedVarcharType(), rowType)), blockBuilder);
    assertBlockEquals(actual, expected);
}