Examples with PageBuilder com.facebook.presto.spi.PageBuilder used on opensource projects

Example 41 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestOrcPageSourceMemoryTracking method testScanFilterAndProjectOperator.

@Test
public void testScanFilterAndProjectOperator() throws Exception {
    // Numbers used in assertions in this test may change when implementation is modified,
    // feel free to change them if they break in the future
    DriverContext driverContext = testPreparer.newDriverContext();
    SourceOperator operator = testPreparer.newScanFilterAndProjectOperator(driverContext);
    assertEquals(driverContext.getSystemMemoryUsage(), 0);
    for (int i = 0; i < 52; i++) {
        assertFalse(operator.isFinished());
        operator.getOutput();
        assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 550000L, 639999L);
    }
    for (int i = 52; i < 65; i++) {
        assertFalse(operator.isFinished());
        operator.getOutput();
        assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 450000L, 539999L);
    }
    // Page source is over, but data still exist in buffer of ScanFilterProjectOperator
    assertFalse(operator.isFinished());
    assertNull(operator.getOutput());
    assertBetweenInclusive(driverContext.getSystemMemoryUsage(), 100000L, 109999L);
    assertFalse(operator.isFinished());
    Page lastPage = operator.getOutput();
    assertNotNull(lastPage);
    // No data is left
    assertTrue(operator.isFinished());
    // an empty page builder of two variable width block builders is left in ScanFilterAndProjectOperator
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(createUnboundedVarcharType(), createUnboundedVarcharType()));
    for (int i = 0; i < lastPage.getPositionCount(); i++) {
        pageBuilder.declarePosition();
        createUnboundedVarcharType().appendTo(lastPage.getBlock(0), i, pageBuilder.getBlockBuilder(0));
        createUnboundedVarcharType().appendTo(lastPage.getBlock(1), i, pageBuilder.getBlockBuilder(1));
    }
    pageBuilder.reset();
    assertEquals(driverContext.getSystemMemoryUsage(), pageBuilder.getRetainedSizeInBytes());
}

Example 42 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class MapConcatFunction method mapConcat.

@UsedByGeneratedCode
public static Block mapConcat(Type keyType, Type valueType, Object state, Block[] maps) {
    int entries = 0;
    int lastMapIndex = maps.length - 1;
    int firstMapIndex = lastMapIndex;
    for (int i = 0; i < maps.length; i++) {
        entries += maps[i].getPositionCount();
        if (maps[i].getPositionCount() > 0) {
            lastMapIndex = i;
            firstMapIndex = min(firstMapIndex, i);
        }
    }
    if (lastMapIndex == firstMapIndex) {
        return maps[lastMapIndex];
    }
    PageBuilder pageBuilder = (PageBuilder) state;
    if (pageBuilder.isFull()) {
        pageBuilder.reset();
    }
    // TODO: we should move TypedSet into user state as well
    TypedSet typedSet = new TypedSet(keyType, entries / 2);
    BlockBuilder keyBlockBuilder = pageBuilder.getBlockBuilder(0);
    BlockBuilder valueBlockBuilder = pageBuilder.getBlockBuilder(1);
    // the last map
    Block map = maps[lastMapIndex];
    int total = 0;
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        typedSet.add(map, i);
        keyType.appendTo(map, i, keyBlockBuilder);
        valueType.appendTo(map, i + 1, valueBlockBuilder);
        total++;
    }
    // the map between the last and the first
    for (int idx = lastMapIndex - 1; idx > firstMapIndex; idx--) {
        map = maps[idx];
        for (int i = 0; i < map.getPositionCount(); i += 2) {
            if (!typedSet.contains(map, i)) {
                typedSet.add(map, i);
                keyType.appendTo(map, i, keyBlockBuilder);
                valueType.appendTo(map, i + 1, valueBlockBuilder);
                total++;
            }
        }
    }
    // the first map
    map = maps[firstMapIndex];
    for (int i = 0; i < map.getPositionCount(); i += 2) {
        if (!typedSet.contains(map, i)) {
            keyType.appendTo(map, i, keyBlockBuilder);
            valueType.appendTo(map, i + 1, valueBlockBuilder);
            total++;
        }
    }
    pageBuilder.declarePositions(total);
    Block[] blocks = new Block[2];
    blocks[0] = keyBlockBuilder.getRegion(keyBlockBuilder.getPositionCount() - total, total);
    blocks[1] = valueBlockBuilder.getRegion(valueBlockBuilder.getPositionCount() - total, total);
    return new InterleavedBlock(blocks);
}

Example 43 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class ArrayTransformFunction method generateTransform.

private static Class<?> generateTransform(Type inputType, Type outputType) {
    CallSiteBinder binder = new CallSiteBinder();
    Class<?> inputJavaType = Primitives.wrap(inputType.getJavaType());
    Class<?> outputJavaType = Primitives.wrap(outputType.getJavaType());
    ClassDefinition definition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName("ArrayTransform"), type(Object.class));
    definition.declareDefaultConstructor(a(PRIVATE));
    // define createPageBuilder
    MethodDefinition createPageBuilderMethod = definition.declareMethod(a(PUBLIC, STATIC), "createPageBuilder", type(PageBuilder.class));
    createPageBuilderMethod.getBody().append(newInstance(PageBuilder.class, constantType(binder, new ArrayType(outputType)).invoke("getTypeParameters", List.class)).ret());
    // define transform method
    Parameter pageBuilder = arg("pageBuilder", PageBuilder.class);
    Parameter block = arg("block", Block.class);
    Parameter function = arg("function", MethodHandle.class);
    MethodDefinition method = definition.declareMethod(a(PUBLIC, STATIC), "transform", type(Block.class), ImmutableList.of(pageBuilder, block, function));
    BytecodeBlock body = method.getBody();
    Scope scope = method.getScope();
    Variable positionCount = scope.declareVariable(int.class, "positionCount");
    Variable position = scope.declareVariable(int.class, "position");
    Variable blockBuilder = scope.declareVariable(BlockBuilder.class, "blockBuilder");
    Variable inputElement = scope.declareVariable(inputJavaType, "inputElement");
    Variable outputElement = scope.declareVariable(outputJavaType, "outputElement");
    // invoke block.getPositionCount()
    body.append(positionCount.set(block.invoke("getPositionCount", int.class)));
    // reset page builder if it is full
    body.append(new IfStatement().condition(pageBuilder.invoke("isFull", boolean.class)).ifTrue(pageBuilder.invoke("reset", void.class)));
    // get block builder
    body.append(blockBuilder.set(pageBuilder.invoke("getBlockBuilder", BlockBuilder.class, constantInt(0))));
    BytecodeNode loadInputElement;
    if (!inputType.equals(UNKNOWN)) {
        loadInputElement = new IfStatement().condition(block.invoke("isNull", boolean.class, position)).ifTrue(inputElement.set(constantNull(inputJavaType))).ifFalse(inputElement.set(constantType(binder, inputType).getValue(block, position).cast(inputJavaType)));
    } else {
        loadInputElement = new BytecodeBlock().append(inputElement.set(constantNull(inputJavaType)));
    }
    BytecodeNode writeOutputElement;
    if (!outputType.equals(UNKNOWN)) {
        writeOutputElement = new IfStatement().condition(equal(outputElement, constantNull(outputJavaType))).ifTrue(blockBuilder.invoke("appendNull", BlockBuilder.class).pop()).ifFalse(constantType(binder, outputType).writeValue(blockBuilder, outputElement.cast(outputType.getJavaType())));
    } else {
        writeOutputElement = new BytecodeBlock().append(blockBuilder.invoke("appendNull", BlockBuilder.class).pop());
    }
    body.append(new ForLoop().initialize(position.set(constantInt(0))).condition(lessThan(position, positionCount)).update(incrementVariable(position, (byte) 1)).body(new BytecodeBlock().append(loadInputElement).append(outputElement.set(function.invoke("invokeExact", outputJavaType, inputElement))).append(writeOutputElement)));
    body.append(pageBuilder.invoke("declarePositions", void.class, positionCount));
    body.append(blockBuilder.invoke("getRegion", Block.class, subtract(blockBuilder.invoke("getPositionCount", int.class), positionCount), positionCount).ret());
    return defineClass(definition, Object.class, binder.getBindings(), ArrayTransformFunction.class.getClassLoader());
}

Example 44 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestGroupByHash method testAppendTo.

@Test
public void testAppendTo() throws Exception {
    Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100);
    Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock);
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER);
    GroupByIdBlock groupIds = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock));
    for (int i = 0; i < groupIds.getPositionCount(); i++) {
        assertEquals(groupIds.getGroupId(i), i);
    }
    assertEquals(groupByHash.getGroupCount(), 100);
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int i = 0; i < groupByHash.getGroupCount(); i++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(i, pageBuilder, 0);
    }
    Page page = pageBuilder.build();
    // Ensure that all blocks have the same positionCount
    for (int i = 0; i < groupByHash.getTypes().size(); i++) {
        assertEquals(page.getBlock(i).getPositionCount(), 100);
    }
    assertEquals(page.getPositionCount(), 100);
    BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock);
    BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock);
}

Example 45 with PageBuilder

use of com.facebook.presto.spi.PageBuilder in project presto by prestodb.

the class TestShardCompactor method sortAndMaterialize.

private static MaterializedResult sortAndMaterialize(List<Page> pages, List<Type> columnTypes, List<Integer> sortIndexes, List<SortOrder> sortOrders, List<Type> sortTypes) {
    long[] orderedAddresses = PAGE_SORTER.sort(columnTypes, pages, sortIndexes, sortOrders, 10_000);
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    for (long orderedAddress : orderedAddresses) {
        int pageIndex = PAGE_SORTER.decodePageIndex(orderedAddress);
        int positionIndex = PAGE_SORTER.decodePositionIndex(orderedAddress);
        Page page = pages.get(pageIndex);
        pageBuilder.declarePosition();
        for (int i = 0; i < columnTypes.size(); i++) {
            columnTypes.get(i).appendTo(page.getBlock(i), positionIndex, pageBuilder.getBlockBuilder(i));
        }
    }
    // extract the sortIndexes and reorder the blocks by sort indexes (useful for debugging)
    Block[] blocks = pageBuilder.build().getBlocks();
    Block[] outputBlocks = new Block[blocks.length];
    for (int i = 0; i < sortIndexes.size(); i++) {
        outputBlocks[i] = blocks[sortIndexes.get(i)];
    }
    MaterializedResult.Builder resultBuilder = MaterializedResult.resultBuilder(SESSION, sortTypes);
    resultBuilder.page(new Page(outputBlocks));
    return resultBuilder.build();
}