Examples with PageBuilder com.facebook.presto.spi.PageBuilder used on opensource projects

Example 31 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class HiveFileFormatBenchmark method createTpchDataSet.

private static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(HiveFileFormatBenchmark::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    return new TestData(columnNames, columnTypes, pages.build());
}

Example 32 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestOrcPageSourceMemoryTracking method testScanFilterAndProjectOperator.

@Test
public void testScanFilterAndProjectOperator() throws Exception {
    // Numbers used in assertions in this test may change when implementation is modified,
    // feel free to change them if they break in the future
    DriverContext driverContext = testPreparer.newDriverContext();
    SourceOperator operator = testPreparer.newScanFilterAndProjectOperator(driverContext);
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    for (int i = 0; i < 52; i++) {
        assertFalse(operator.isFinished());
        operator.getOutput();
        assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 550000L, 639999L);
    }
    for (int i = 52; i < 65; i++) {
        assertFalse(operator.isFinished());
        operator.getOutput();
        assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 450000L, 539999L);
    }
    // Page source is over, but data still exist in buffer of ScanFilterProjectOperator
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 100000L, 109999L);
    assertFalse(operator.isFinished());
    Page lastPage = operator.getOutput();
    assertNotNull(lastPage);
    // No data is left
    assertTrue(operator.isFinished());
    // an empty page builder of two variable width block builders is left in ScanFilterAndProjectOperator
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(createUnboundedVarcharType(), createUnboundedVarcharType()));
    for (int i = 0; i < lastPage.getPositionCount(); i++) {
        pageBuilder.declarePosition();
        createUnboundedVarcharType().appendTo(lastPage.getBlock(0), i, pageBuilder.getBlockBuilder(0));
        createUnboundedVarcharType().appendTo(lastPage.getBlock(1), i, pageBuilder.getBlockBuilder(1));
    }
    pageBuilder.reset();
    assertEquals(driverContext.getSystemMemoryUsage(), pageBuilder.getRetainedSizeInBytes());
}

Example 33 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class MapConcatFunction method mapConcat.

@UsedByGeneratedCode
public static Block mapConcat(Type keyType, Type valueType, Object state, Block[] maps) {
    int entries = 0;
    int lastMapIndex = maps.length - 1;
    int firstMapIndex = lastMapIndex;
    for (int i = 0; i < maps.length; i++) {
        entries += maps[i].getPositionCount();
        if (maps[i].getPositionCount() > 0) {
            lastMapIndex = i;
            firstMapIndex = min(firstMapIndex, i);
        }
    }
    if (lastMapIndex == firstMapIndex) {
        return maps[lastMapIndex];
    }
    PageBuilder pageBuilder = (PageBuilder) state;
    if (pageBuilder.isFull()) {
        pageBuilder.reset();
    }
    // TODO: we should move TypedSet into user state as well
    TypedSet typedSet = new TypedSet(keyType, entries / 2);
    BlockBuilder keyBlockBuilder = pageBuilder.getBlockBuilder(0);
    BlockBuilder valueBlockBuilder = pageBuilder.getBlockBuilder(1);
    // the last map
    Block map = maps[lastMapIndex];
    int total = 0;
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        typedSet.add(map, i);
        keyType.appendTo(map, i, keyBlockBuilder);
        valueType.appendTo(map, i + 1, valueBlockBuilder);
        total++;
    }
    // the map between the last and the first
    for (int idx = lastMapIndex - 1; idx > firstMapIndex; idx--) {
        map = maps[idx];
        for (int i = 0; i < map.getPositionCount(); i += 2) {
            if (!typedSet.contains(map, i)) {
                typedSet.add(map, i);
                keyType.appendTo(map, i, keyBlockBuilder);
                valueType.appendTo(map, i + 1, valueBlockBuilder);
                total++;
            }
        }
    }
    // the first map
    map = maps[firstMapIndex];
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        if (!typedSet.contains(map, i)) {
            keyType.appendTo(map, i, keyBlockBuilder);
            valueType.appendTo(map, i + 1, valueBlockBuilder);
            total++;
        }
    }
    pageBuilder.declarePositions(total);
    Block[] blocks = new Block[2];
    blocks[0] = keyBlockBuilder.getRegion(keyBlockBuilder.getPositionCount() - total, total);
    blocks[1] = valueBlockBuilder.getRegion(valueBlockBuilder.getPositionCount() - total, total);
    return new InterleavedBlock(blocks);
}

Example 34 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class BenchmarkPageProcessor method execute.

public static Page execute(Page inputPage, PageProcessor processor) {
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(DOUBLE));
    int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
    checkState(count == inputPage.getPositionCount());
    return pageBuilder.build();
}

Example 35 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class InCodeGeneratorBenchmark method benchmark.

@Benchmark
public Page benchmark() {
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(prestoType));
    int count = processor.process(null, inputPage, 0, inputPage.getPositionCount(), pageBuilder);
    checkState(count == inputPage.getPositionCount());
    return pageBuilder.build();
}