Examples with PageBuilder com.facebook.presto.common.PageBuilder used on opensource projects

Example 46 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class BenchmarkGroupByHash method groupByHashPreCompute.

@Benchmark
@OperationsPerInvocation(POSITIONS)
public Object groupByHashPreCompute(BenchmarkData data) {
    GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP);
    for (Page page : data.getPages()) {
        Work<?> work = groupByHash.addPage(page);
        boolean finished;
        do {
            finished = work.process();
        } while (!finished);
    }
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(groupId, pageBuilder, 0);
        if (pageBuilder.isFull()) {
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
    }
    pages.add(pageBuilder.build());
    return pageBuilder.build();
}

Example 47 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class BenchmarkGroupByHash method addPagePreCompute.

@Benchmark
@OperationsPerInvocation(POSITIONS)
public Object addPagePreCompute(BenchmarkData data) {
    GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP);
    for (Page page : data.getPages()) {
        Work<?> work = groupByHash.addPage(page);
        boolean finished;
        do {
            finished = work.process();
        } while (!finished);
    }
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(groupId, pageBuilder, 0);
        if (pageBuilder.isFull()) {
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
    }
    pages.add(pageBuilder.build());
    return pageBuilder.build();
}

Example 48 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class BenchmarkGroupByHash method createBigintPages.

private static List<Page> createBigintPages(int positionCount, int groupCount, int channelCount, boolean hashEnabled) {
    List<Type> types = Collections.nCopies(channelCount, BIGINT);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    if (hashEnabled) {
        types = ImmutableList.copyOf(Iterables.concat(types, ImmutableList.of(BIGINT)));
    }
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int position = 0; position < positionCount; position++) {
        int rand = ThreadLocalRandom.current().nextInt(groupCount);
        pageBuilder.declarePosition();
        for (int numChannel = 0; numChannel < channelCount; numChannel++) {
            BIGINT.writeLong(pageBuilder.getBlockBuilder(numChannel), rand);
        }
        if (hashEnabled) {
            BIGINT.writeLong(pageBuilder.getBlockBuilder(channelCount), BigintOperators.hashCode(rand));
        }
        if (pageBuilder.isFull()) {
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
    }
    pages.add(pageBuilder.build());
    return pages.build();
}

Example 49 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class AbstractTestHiveFileFormats method createTestFile.

public static FileSplit createTestFile(String filePath, HiveStorageFormat storageFormat, HiveCompressionCodec compressionCodec, List<TestColumn> testColumns, ConnectorSession session, int numRows, HiveFileWriterFactory fileWriterFactory) {
    // filter out partition keys, which are not written to the file
    testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));
    List<Type> types = testColumns.stream().map(TestColumn::getType).map(HiveType::valueOf).map(type -> type.getType(FUNCTION_AND_TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
        pageBuilder.declarePosition();
        for (int columnNumber = 0; columnNumber < testColumns.size(); columnNumber++) {
            serializeObject(types.get(columnNumber), pageBuilder.getBlockBuilder(columnNumber), testColumns.get(columnNumber).getWriteValue(), testColumns.get(columnNumber).getObjectInspector(), false);
        }
    }
    Page page = pageBuilder.build();
    JobConf jobConf = configureCompression(new JobConf(), compressionCodec);
    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
    tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
    Optional<HiveFileWriter> fileWriter = fileWriterFactory.createFileWriter(new Path(filePath), testColumns.stream().map(TestColumn::getName).collect(toList()), StorageFormat.fromHiveStorageFormat(storageFormat), tableProperties, jobConf, session, Optional.empty());
    HiveFileWriter hiveFileWriter = fileWriter.orElseThrow(() -> new IllegalArgumentException("fileWriterFactory"));
    hiveFileWriter.appendRows(page);
    Optional<Page> fileStatistics = hiveFileWriter.commit();
    assertFileStatistics(fileStatistics, hiveFileWriter.getFileSizeInBytes(), storageFormat);
    return new FileSplit(new Path(filePath), 0, new File(filePath).length(), new String[0]);
}

Example 50 with PageBuilder

use of com.facebook.presto.common.PageBuilder in project presto by prestodb.

the class TableWriterMergeOperator method createFinalPage.

private Page createFinalPage() {
    checkState(lastTableCommitContext.isLastPage(), "unexpected last table commit context: %s", lastTableCommitContext);
    PageBuilder pageBuilder = new PageBuilder(1, types);
    pageBuilder.declarePosition();
    for (int channel = 0; channel < types.size(); channel++) {
        if (channel == ROW_COUNT_CHANNEL) {
            types.get(channel).writeLong(pageBuilder.getBlockBuilder(channel), rowCount);
        } else if (channel == CONTEXT_CHANNEL) {
            types.get(channel).writeSlice(pageBuilder.getBlockBuilder(channel), createTableCommitContext(true));
        } else {
            pageBuilder.getBlockBuilder(channel).appendNull();
        }
    }
    return pageBuilder.build();
}