Search in sources :

Example 36 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class HiveFileFormatBenchmark method createTpchDataSet.

private static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(HiveFileFormatBenchmark::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    return new TestData(columnNames, columnTypes, pages.build());
}
Also used : HdfsEnvironment(com.facebook.presto.hive.HdfsEnvironment) Page(com.facebook.presto.common.Page) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) MoreFiles.deleteRecursively(com.google.common.io.MoreFiles.deleteRecursively) RunResult(org.openjdk.jmh.results.RunResult) LINE_ITEM(io.airlift.tpch.TpchTable.LINE_ITEM) Random(java.util.Random) CacheConfig(com.facebook.presto.cache.CacheConfig) Warmup(org.openjdk.jmh.annotations.Warmup) PageBuilder(com.facebook.presto.common.PageBuilder) OutputTimeUnit(org.openjdk.jmh.annotations.OutputTimeUnit) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Slices(io.airlift.slice.Slices) HiveCompressionCodec(com.facebook.presto.hive.HiveCompressionCodec) TearDown(org.openjdk.jmh.annotations.TearDown) HiveClientConfig(com.facebook.presto.hive.HiveClientConfig) Setup(org.openjdk.jmh.annotations.Setup) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) Param(org.openjdk.jmh.annotations.Param) Collection(java.util.Collection) UUID(java.util.UUID) String.format(java.lang.String.format) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TpchTable(io.airlift.tpch.TpchTable) UncheckedIOException(java.io.UncheckedIOException) TpchEntity(io.airlift.tpch.TpchEntity) OptionsBuilder(org.openjdk.jmh.runner.options.OptionsBuilder) DataSize(io.airlift.units.DataSize) List(java.util.List) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) HiveSessionProperties(com.facebook.presto.hive.HiveSessionProperties) Options(org.openjdk.jmh.runner.options.Options) TpchColumn(io.airlift.tpch.TpchColumn) Measurement(org.openjdk.jmh.annotations.Measurement) ORDERS(io.airlift.tpch.TpchTable.ORDERS) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) Scope(org.openjdk.jmh.annotations.Scope) ParquetFileWriterConfig(com.facebook.presto.hive.ParquetFileWriterConfig) OrderColumn(io.airlift.tpch.OrderColumn) DATE(com.facebook.presto.common.type.DateType.DATE) ArrayList(java.util.ArrayList) Statistics(org.openjdk.jmh.util.Statistics) HiveTestUtils.createTestHdfsEnvironment(com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment) AuxCounters(org.openjdk.jmh.annotations.AuxCounters) ALLOW_INSECURE(com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE) ImmutableList(com.google.common.collect.ImmutableList) METASTORE_CLIENT_CONFIG(com.facebook.presto.hive.HiveTestUtils.METASTORE_CLIENT_CONFIG) ArrayType(com.facebook.presto.common.type.ArrayType) HiveTestUtils.mapType(com.facebook.presto.hive.HiveTestUtils.mapType) Runner(org.openjdk.jmh.runner.Runner) Type(com.facebook.presto.common.type.Type) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) OrcFileWriterConfig(com.facebook.presto.hive.OrcFileWriterConfig) HadoopNative(com.facebook.presto.hadoop.HadoopNative) IOException(java.io.IOException) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) State(org.openjdk.jmh.annotations.State) Benchmark(org.openjdk.jmh.annotations.Benchmark) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) Fork(org.openjdk.jmh.annotations.Fork) IntArrays(it.unimi.dsi.fastutil.ints.IntArrays) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) DATE(com.facebook.presto.common.type.DateType.DATE) ALLOW_INSECURE(com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE) ImmutableList(com.google.common.collect.ImmutableList) Page(com.facebook.presto.common.Page) PageBuilder(com.facebook.presto.common.PageBuilder) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) ArrayType(com.facebook.presto.common.type.ArrayType) HiveTestUtils.mapType(com.facebook.presto.hive.HiveTestUtils.mapType) Type(com.facebook.presto.common.type.Type) BlockBuilder(com.facebook.presto.common.block.BlockBuilder)

Example 37 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class ArrayTransformFunction method generateTransform.

private static Class<?> generateTransform(Type inputType, Type outputType) {
    CallSiteBinder binder = new CallSiteBinder();
    Class<?> inputJavaType = Primitives.wrap(inputType.getJavaType());
    Class<?> outputJavaType = Primitives.wrap(outputType.getJavaType());
    ClassDefinition definition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName("ArrayTransform"), type(Object.class));
    definition.declareDefaultConstructor(a(PRIVATE));
    // define createPageBuilder
    MethodDefinition createPageBuilderMethod = definition.declareMethod(a(PUBLIC, STATIC), "createPageBuilder", type(PageBuilder.class));
    createPageBuilderMethod.getBody().append(newInstance(PageBuilder.class, constantType(binder, new ArrayType(outputType)).invoke("getTypeParameters", List.class)).ret());
    // define transform method
    Parameter pageBuilder = arg("pageBuilder", PageBuilder.class);
    Parameter block = arg("block", Block.class);
    Parameter function = arg("function", UnaryFunctionInterface.class);
    MethodDefinition method = definition.declareMethod(a(PUBLIC, STATIC), "transform", type(Block.class), ImmutableList.of(pageBuilder, block, function));
    BytecodeBlock body = method.getBody();
    Scope scope = method.getScope();
    Variable positionCount = scope.declareVariable(int.class, "positionCount");
    Variable position = scope.declareVariable(int.class, "position");
    Variable blockBuilder = scope.declareVariable(BlockBuilder.class, "blockBuilder");
    Variable inputElement = scope.declareVariable(inputJavaType, "inputElement");
    Variable outputElement = scope.declareVariable(outputJavaType, "outputElement");
    // invoke block.getPositionCount()
    body.append(positionCount.set(block.invoke("getPositionCount", int.class)));
    // reset page builder if it is full
    body.append(new IfStatement().condition(pageBuilder.invoke("isFull", boolean.class)).ifTrue(pageBuilder.invoke("reset", void.class)));
    // get block builder
    body.append(blockBuilder.set(pageBuilder.invoke("getBlockBuilder", BlockBuilder.class, constantInt(0))));
    BytecodeNode loadInputElement;
    if (!inputType.equals(UNKNOWN)) {
        loadInputElement = new IfStatement().condition(block.invoke("isNull", boolean.class, position)).ifTrue(inputElement.set(constantNull(inputJavaType))).ifFalse(inputElement.set(constantType(binder, inputType).getValue(block, position).cast(inputJavaType)));
    } else {
        loadInputElement = new BytecodeBlock().append(inputElement.set(constantNull(inputJavaType)));
    }
    BytecodeNode writeOutputElement;
    if (!outputType.equals(UNKNOWN)) {
        writeOutputElement = new IfStatement().condition(equal(outputElement, constantNull(outputJavaType))).ifTrue(blockBuilder.invoke("appendNull", BlockBuilder.class).pop()).ifFalse(constantType(binder, outputType).writeValue(blockBuilder, outputElement.cast(outputType.getJavaType())));
    } else {
        writeOutputElement = new BytecodeBlock().append(blockBuilder.invoke("appendNull", BlockBuilder.class).pop());
    }
    body.append(new ForLoop().initialize(position.set(constantInt(0))).condition(lessThan(position, positionCount)).update(incrementVariable(position, (byte) 1)).body(new BytecodeBlock().append(loadInputElement).append(outputElement.set(function.invoke("apply", Object.class, inputElement.cast(Object.class)).cast(outputJavaType))).append(writeOutputElement)));
    body.append(pageBuilder.invoke("declarePositions", void.class, positionCount));
    body.append(blockBuilder.invoke("getRegion", Block.class, subtract(blockBuilder.invoke("getPositionCount", int.class), positionCount), positionCount).ret());
    return defineClass(definition, Object.class, binder.getBindings(), ArrayTransformFunction.class.getClassLoader());
}
Also used : Signature.typeVariable(com.facebook.presto.spi.function.Signature.typeVariable) Variable(com.facebook.presto.bytecode.Variable) VariableInstruction.incrementVariable(com.facebook.presto.bytecode.instruction.VariableInstruction.incrementVariable) ForLoop(com.facebook.presto.bytecode.control.ForLoop) BytecodeBlock(com.facebook.presto.bytecode.BytecodeBlock) PageBuilder(com.facebook.presto.common.PageBuilder) ClassDefinition(com.facebook.presto.bytecode.ClassDefinition) ArrayType(com.facebook.presto.common.type.ArrayType) IfStatement(com.facebook.presto.bytecode.control.IfStatement) Scope(com.facebook.presto.bytecode.Scope) MethodDefinition(com.facebook.presto.bytecode.MethodDefinition) CallSiteBinder(com.facebook.presto.bytecode.CallSiteBinder) Parameter(com.facebook.presto.bytecode.Parameter) BytecodeBlock(com.facebook.presto.bytecode.BytecodeBlock) Block(com.facebook.presto.common.block.Block) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) BytecodeNode(com.facebook.presto.bytecode.BytecodeNode) BlockBuilder(com.facebook.presto.common.block.BlockBuilder)

Example 38 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class TestGroupByHash method testAppendTo.

@Test
public void testAppendTo() {
    Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100);
    Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock);
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER);
    Work<GroupByIdBlock> work = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock));
    work.process();
    GroupByIdBlock groupIds = work.getResult();
    for (int i = 0; i < groupIds.getPositionCount(); i++) {
        assertEquals(groupIds.getGroupId(i), i);
    }
    assertEquals(groupByHash.getGroupCount(), 100);
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int i = 0; i < groupByHash.getGroupCount(); i++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(i, pageBuilder, 0);
    }
    Page page = pageBuilder.build();
    // Ensure that all blocks have the same positionCount
    for (int i = 0; i < groupByHash.getTypes().size(); i++) {
        assertEquals(page.getBlock(i).getPositionCount(), 100);
    }
    assertEquals(page.getPositionCount(), 100);
    BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock);
    BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock);
}
Also used : GroupByHash.createGroupByHash(com.facebook.presto.operator.GroupByHash.createGroupByHash) BlockAssertions.createLongsBlock(com.facebook.presto.block.BlockAssertions.createLongsBlock) BlockAssertions.createLongSequenceBlock(com.facebook.presto.block.BlockAssertions.createLongSequenceBlock) BlockAssertions.createStringSequenceBlock(com.facebook.presto.block.BlockAssertions.createStringSequenceBlock) DictionaryBlock(com.facebook.presto.common.block.DictionaryBlock) TypeUtils.getHashBlock(com.facebook.presto.type.TypeUtils.getHashBlock) Block(com.facebook.presto.common.block.Block) Page(com.facebook.presto.common.Page) PageBuilder(com.facebook.presto.common.PageBuilder) Test(org.testng.annotations.Test)

Example 39 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class PinotBrokerPageSourceBase method getNextPage.

@Override
public Page getNextPage() {
    if (finished) {
        return null;
    }
    long start = System.nanoTime();
    try {
        BlockAndTypeBuilder blockAndTypeBuilder = buildBlockAndTypeBuilder(columnHandles, getBrokerQuery());
        int counter = issueQueryAndPopulate(getBrokerQuery(), Collections.unmodifiableList(blockAndTypeBuilder.getColumnBlockBuilders()), Collections.unmodifiableList(blockAndTypeBuilder.getColumnTypes()));
        PageBuilder pageBuilder = blockAndTypeBuilder.getPageBuilder();
        pageBuilder.declarePositions(counter);
        Page page = pageBuilder.build();
        // TODO: Implement chunking if the result set is ginormous
        finished = true;
        return page;
    } finally {
        readTimeNanos += System.nanoTime() - start;
    }
}
Also used : Page(com.facebook.presto.common.Page) PageBuilder(com.facebook.presto.common.PageBuilder)

Example 40 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class JoinCompiler method generateAppendToMethod.

private static void generateAppendToMethod(ClassDefinition classDefinition, CallSiteBinder callSiteBinder, List<Type> types, List<Integer> outputChannels, List<FieldDefinition> channelFields) {
    Parameter blockIndex = arg("blockIndex", int.class);
    Parameter blockPosition = arg("blockPosition", int.class);
    Parameter pageBuilder = arg("pageBuilder", PageBuilder.class);
    Parameter outputChannelOffset = arg("outputChannelOffset", int.class);
    MethodDefinition appendToMethod = classDefinition.declareMethod(a(PUBLIC), "appendTo", type(void.class), blockIndex, blockPosition, pageBuilder, outputChannelOffset);
    Variable thisVariable = appendToMethod.getThis();
    BytecodeBlock appendToBody = appendToMethod.getBody();
    int pageBuilderOutputChannel = 0;
    for (int outputChannel : outputChannels) {
        Type type = types.get(outputChannel);
        BytecodeExpression typeExpression = constantType(callSiteBinder, type);
        BytecodeExpression block = thisVariable.getField(channelFields.get(outputChannel)).invoke("get", Object.class, blockIndex).cast(Block.class);
        appendToBody.comment("%s.appendTo(channel_%s.get(outputChannel), blockPosition, pageBuilder.getBlockBuilder(outputChannelOffset + %s));", type.getClass(), outputChannel, pageBuilderOutputChannel).append(typeExpression).append(block).append(blockPosition).append(pageBuilder).append(outputChannelOffset).push(pageBuilderOutputChannel++).append(OpCode.IADD).invokeVirtual(PageBuilder.class, "getBlockBuilder", BlockBuilder.class, int.class).invokeInterface(Type.class, "appendTo", void.class, Block.class, int.class, BlockBuilder.class);
    }
    appendToBody.ret();
}
Also used : BigintType(com.facebook.presto.common.type.BigintType) Type(com.facebook.presto.common.type.Type) SqlTypeBytecodeExpression.constantType(com.facebook.presto.sql.gen.SqlTypeBytecodeExpression.constantType) OperatorType(com.facebook.presto.common.function.OperatorType) Variable(com.facebook.presto.bytecode.Variable) MethodDefinition(com.facebook.presto.bytecode.MethodDefinition) BytecodeBlock(com.facebook.presto.bytecode.BytecodeBlock) Parameter(com.facebook.presto.bytecode.Parameter) PageBuilder(com.facebook.presto.common.PageBuilder) BytecodeExpression(com.facebook.presto.bytecode.expression.BytecodeExpression) BlockBuilder(com.facebook.presto.common.block.BlockBuilder)

Aggregations

PageBuilder (com.facebook.presto.common.PageBuilder)57 Page (com.facebook.presto.common.Page)27 Type (com.facebook.presto.common.type.Type)25 BlockBuilder (com.facebook.presto.common.block.BlockBuilder)23 ImmutableList (com.google.common.collect.ImmutableList)14 Block (com.facebook.presto.common.block.Block)11 Test (org.testng.annotations.Test)11 ArrayType (com.facebook.presto.common.type.ArrayType)8 ArrayList (java.util.ArrayList)8 List (java.util.List)8 Benchmark (org.openjdk.jmh.annotations.Benchmark)7 INTEGER (com.facebook.presto.common.type.IntegerType.INTEGER)4 BIGINT (com.facebook.presto.common.type.BigintType.BIGINT)3 DOUBLE (com.facebook.presto.common.type.DoubleType.DOUBLE)3 VarcharType.createUnboundedVarcharType (com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType)3 ConnectorPageSource (com.facebook.presto.spi.ConnectorPageSource)3 ConnectorSession (com.facebook.presto.spi.ConnectorSession)3 MaterializedResult (com.facebook.presto.testing.MaterializedResult)3 Slices (io.airlift.slice.Slices)3 File (java.io.File)3