Examples with PageBuilder io.prestosql.spi.PageBuilder used on opensource projects

Example 26 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class BenchmarkGroupedTopNBuilder method createInputPage.

private static Page createInputPage(int positions, List<Type> types) {
    PageBuilder pageBuilder = new PageBuilder(types);
    LineItemGenerator lineItemGenerator = new LineItemGenerator(1, 1, 1);
    Iterator<LineItem> iterator = lineItemGenerator.iterator();
    for (int i = 0; i < positions; i++) {
        pageBuilder.declarePosition();
        LineItem lineItem = iterator.next();
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(EXTENDED_PRICE), lineItem.getExtendedPrice());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(DISCOUNT), lineItem.getDiscount());
        DATE.writeLong(pageBuilder.getBlockBuilder(SHIP_DATE), lineItem.getShipDate());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(QUANTITY), lineItem.getQuantity());
    }
    return pageBuilder.build();
}

Example 27 with PageBuilder

use of io.prestosql.spi.PageBuilder in project boostkit-bigdata by kunpengcompute.

the class HiveFileFormatBenchmark method createTpchDataSet.

private static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(HiveFileFormatBenchmark::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    return new TestData(columnNames, columnTypes, pages.build());
}

Example 28 with PageBuilder

use of io.prestosql.spi.PageBuilder in project boostkit-bigdata by kunpengcompute.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveConfig config, HiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(HiveTestUtils.TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(round(stats.getInputPageSizeInBytes().getAllTime().getMax()), page.getRetainedSizeInBytes());
    return length;
}

Example 29 with PageBuilder

use of io.prestosql.spi.PageBuilder in project boostkit-bigdata by kunpengcompute.

the class TestMergingPageIterator method testMerging.

@Test
public void testMerging() {
    List<Type> types = ImmutableList.of(INTEGER, INTEGER);
    List<Integer> sortIndexes = ImmutableList.of(1);
    List<SortOrder> sortOrders = ImmutableList.of(SortOrder.ASC_NULLS_FIRST);
    List<List<Page>> pageLists = new ArrayList<>();
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int i = 0; i < 10; i++) {
        Iterator<Integer> values = IntStream.range(0, 1000).map(ignored -> ThreadLocalRandom.current().nextInt(100_000)).mapToObj(n -> ((n % 100) == 0) ? null : n).sorted(nullsFirst(naturalOrder())).iterator();
        List<Page> pages = new ArrayList<>();
        for (int j = 0; j < 10; j++) {
            for (int k = 0; k < 100; k++) {
                Integer n = values.next();
                pageBuilder.declarePosition();
                if (n == null) {
                    pageBuilder.getBlockBuilder(0).appendNull();
                    pageBuilder.getBlockBuilder(1).appendNull();
                } else {
                    INTEGER.writeLong(pageBuilder.getBlockBuilder(0), n);
                    INTEGER.writeLong(pageBuilder.getBlockBuilder(1), n * 22L);
                }
            }
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
        pageLists.add(pages);
        assertFalse(values.hasNext());
    }
    List<Iterator<Page>> pages = pageLists.stream().map(List::iterator).collect(toList());
    Iterator<Page> iterator = new MergingPageIterator(pages, types, sortIndexes, sortOrders);
    List<Long> values = new ArrayList<>();
    while (iterator.hasNext()) {
        Page page = iterator.next();
        for (int i = 0; i < page.getPositionCount(); i++) {
            if (page.getBlock(0).isNull(i)) {
                assertTrue(page.getBlock(1).isNull(i));
                values.add(null);
            } else {
                long x = INTEGER.getLong(page.getBlock(0), i);
                long y = INTEGER.getLong(page.getBlock(1), i);
                assertEquals(y, x * 22);
                values.add(x);
            }
        }
    }
    assertThat(values).isSortedAccordingTo(nullsFirst(naturalOrder()));
}

Example 30 with PageBuilder

use of io.prestosql.spi.PageBuilder in project hetu-core by openlookeng.

the class InMemorySortAggregationBuilder method buildResult.

private WorkProcessor<Page> buildResult(IntIterator groupIds, AggregationNode.Step step, boolean isFinalizedValuePresent) {
    final PageBuilder pageBuilder = new PageBuilder(buildTypes(step, isFinalizedValuePresent));
    return WorkProcessor.create(() -> {
        if (!groupIds.hasNext()) {
            return WorkProcessor.ProcessState.finished();
        }
        pageBuilder.reset();
        List<Type> types = groupBy.getTypes();
        while (!pageBuilder.isFull() && groupIds.hasNext()) {
            int groupId = groupIds.nextInt();
            groupBy.appendValuesTo(groupId, pageBuilder, 0);
            pageBuilder.declarePosition();
            for (int i = 0; i < aggregators.size(); i++) {
                Aggregator aggregator = aggregators.get(i);
                BlockBuilder output = pageBuilder.getBlockBuilder(types.size() + i);
                aggregator.evaluate(groupId, output);
            }
            if (isFinalizedValuePresent && step == AggregationNode.Step.PARTIAL) {
                BlockBuilder output1 = pageBuilder.getBlockBuilder(types.size() + aggregators.size());
                if (groupId == 0 || !groupIds.hasNext()) {
                    BOOLEAN.writeBoolean(output1, false);
                } else {
                    // finalized values are set to true
                    BOOLEAN.writeBoolean(output1, true);
                }
            }
        }
        return WorkProcessor.ProcessState.ofResult(pageBuilder.build());
    });
}