Examples with PageBuilder com.facebook.presto.common.PageBuilder used on opensource projects

Example 31 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class HiveManifestUtils method createPartitionManifest.

public static Optional<Page> createPartitionManifest(PartitionUpdate partitionUpdate) {
    // Manifest Page layout:
    // fileName    fileSize
    // X           X
    // X           X
    // X           X
    // ....
    PageBuilder manifestBuilder = new PageBuilder(ImmutableList.of(VARCHAR, BIGINT));
    BlockBuilder fileNameBuilder = manifestBuilder.getBlockBuilder(0);
    BlockBuilder fileSizeBuilder = manifestBuilder.getBlockBuilder(1);
    for (FileWriteInfo fileWriteInfo : partitionUpdate.getFileWriteInfos()) {
        if (!fileWriteInfo.getFileSize().isPresent()) {
            return Optional.empty();
        }
        manifestBuilder.declarePosition();
        VARCHAR.writeSlice(fileNameBuilder, utf8Slice(fileWriteInfo.getWriteFileName()));
        BIGINT.writeLong(fileSizeBuilder, fileWriteInfo.getFileSize().get());
    }
    return Optional.of(manifestBuilder.build());
}

Example 32 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class TestDoubleHistogramAggregation method makeInput.

private static Page makeInput(int numberOfBuckets) {
    PageBuilder builder = new PageBuilder(ImmutableList.of(BIGINT, DOUBLE, DOUBLE));
    for (int i = 0; i < 100; i++) {
        builder.declarePosition();
        BIGINT.writeLong(builder.getBlockBuilder(0), numberOfBuckets);
        // value
        DOUBLE.writeDouble(builder.getBlockBuilder(1), i);
        // weight
        DOUBLE.writeDouble(builder.getBlockBuilder(2), 1);
    }
    return builder.build();
}

Example 33 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class HashAggregationOperator method getGlobalAggregationOutput.

private Page getGlobalAggregationOutput() {
    List<Accumulator> accumulators = accumulatorFactories.stream().map(accumulatorFactory -> accumulatorFactory.createAccumulator(UpdateMemory.NOOP)).collect(Collectors.toList());
    // global aggregation output page will only be constructed once,
    // so a new PageBuilder is constructed (instead of using PageBuilder.reset)
    PageBuilder output = new PageBuilder(globalAggregationGroupIds.size(), types);
    for (int groupId : globalAggregationGroupIds) {
        output.declarePosition();
        int channel = 0;
        for (; channel < groupByTypes.size(); channel++) {
            if (channel == groupIdChannel.get()) {
                output.getBlockBuilder(channel).writeLong(groupId);
            } else {
                output.getBlockBuilder(channel).appendNull();
            }
        }
        if (hashChannel.isPresent()) {
            long hashValue = calculateDefaultOutputHash(groupByTypes, groupIdChannel.get(), groupId);
            output.getBlockBuilder(channel++).writeLong(hashValue);
        }
        for (int j = 0; j < accumulators.size(); channel++, j++) {
            if (step.isOutputPartial()) {
                accumulators.get(j).evaluateIntermediate(output.getBlockBuilder(channel));
            } else {
                accumulators.get(j).evaluateFinal(output.getBlockBuilder(channel));
            }
        }
    }
    if (output.isEmpty()) {
        return null;
    }
    return output.build();
}

Example 34 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class AggregationOperator method getOutput.

@Override
public Page getOutput() {
    if (state != State.HAS_OUTPUT) {
        return null;
    }
    // project results into output blocks
    List<Type> types = aggregates.stream().map(Aggregator::getType).collect(toImmutableList());
    // output page will only be constructed once,
    // so a new PageBuilder is constructed (instead of using PageBuilder.reset)
    PageBuilder pageBuilder = new PageBuilder(1, types);
    pageBuilder.declarePosition();
    for (int i = 0; i < aggregates.size(); i++) {
        Aggregator aggregator = aggregates.get(i);
        BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
        aggregator.evaluate(blockBuilder);
    }
    state = State.FINISHED;
    return pageBuilder.build();
}

Example 35 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastoreClientConfig, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(FUNCTION_AND_TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, metastoreClientConfig, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(stats.getInputPageSizeInBytes().getAllTime().getMax(), page.getRetainedSizeInBytes());
    return length;
}