Examples with PageBuilder io.trino.spi.PageBuilder used on opensource projects

Example 6 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class GenericPartitioningSpiller method flush.

private synchronized ListenableFuture<Void> flush(Predicate<PageBuilder> flushCondition) {
    requireNonNull(flushCondition, "flushCondition is null");
    ImmutableList.Builder<ListenableFuture<Void>> futures = ImmutableList.builder();
    for (int partition = 0; partition < spillers.size(); partition++) {
        PageBuilder pageBuilder = pageBuilders.get(partition);
        if (flushCondition.test(pageBuilder)) {
            futures.add(flush(partition));
        }
    }
    return asVoid(Futures.allAsList(futures.build()));
}

Example 7 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class AbstractTestHiveFileFormats method createTestFileTrino.

public static FileSplit createTestFileTrino(String filePath, HiveStorageFormat storageFormat, HiveCompressionCodec compressionCodec, List<TestColumn> testColumns, ConnectorSession session, int numRows, HiveFileWriterFactory fileWriterFactory) {
    // filter out partition keys, which are not written to the file
    testColumns = testColumns.stream().filter(column -> !column.isPartitionKey()).collect(toImmutableList());
    List<Type> types = testColumns.stream().map(TestColumn::getType).map(HiveType::valueOf).map(type -> type.getType(TESTING_TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(types);
    for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
        pageBuilder.declarePosition();
        for (int columnNumber = 0; columnNumber < testColumns.size(); columnNumber++) {
            serializeObject(types.get(columnNumber), pageBuilder.getBlockBuilder(columnNumber), testColumns.get(columnNumber).getWriteValue(), testColumns.get(columnNumber).getObjectInspector(), false);
        }
    }
    Page page = pageBuilder.build();
    JobConf jobConf = new JobConf();
    configureCompression(jobConf, compressionCodec);
    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", testColumns.stream().map(TestColumn::getName).collect(Collectors.joining(",")));
    tableProperties.setProperty("columns.types", testColumns.stream().map(TestColumn::getType).collect(Collectors.joining(",")));
    Optional<FileWriter> fileWriter = fileWriterFactory.createFileWriter(new Path(filePath), testColumns.stream().map(TestColumn::getName).collect(toList()), StorageFormat.fromHiveStorageFormat(storageFormat), tableProperties, jobConf, session, OptionalInt.empty(), NO_ACID_TRANSACTION, false, WriterKind.INSERT);
    FileWriter hiveFileWriter = fileWriter.orElseThrow(() -> new IllegalArgumentException("fileWriterFactory"));
    hiveFileWriter.appendRows(page);
    hiveFileWriter.commit();
    return new FileSplit(new Path(filePath), 0, new File(filePath).length(), new String[0]);
}

Example 8 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class BenchmarkFileFormatsUtils method createTpchDataSet.

public static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(BenchmarkFileFormatsUtils::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    if (!pageBuilder.isEmpty()) {
        pages.add(pageBuilder.build());
    }
    return new TestData(columnNames, columnTypes, pages.build());
}

Example 9 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class TestGroupByHash method testAppendTo.

@Test
public void testAppendTo() {
    Block valuesBlock = BlockAssertions.createStringSequenceBlock(0, 100);
    Block hashBlock = TypeTestUtils.getHashBlock(ImmutableList.of(VARCHAR), valuesBlock);
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(VARCHAR), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER, TYPE_OPERATOR_FACTORY, NOOP);
    Work<GroupByIdBlock> work = groupByHash.getGroupIds(new Page(valuesBlock, hashBlock));
    work.process();
    GroupByIdBlock groupIds = work.getResult();
    for (int i = 0; i < groupIds.getPositionCount(); i++) {
        assertEquals(groupIds.getGroupId(i), i);
    }
    assertEquals(groupByHash.getGroupCount(), 100);
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int i = 0; i < groupByHash.getGroupCount(); i++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(i, pageBuilder, 0);
    }
    Page page = pageBuilder.build();
    // Ensure that all blocks have the same positionCount
    for (int i = 0; i < groupByHash.getTypes().size(); i++) {
        assertEquals(page.getBlock(i).getPositionCount(), 100);
    }
    assertEquals(page.getPositionCount(), 100);
    BlockAssertions.assertBlockEquals(VARCHAR, page.getBlock(0), valuesBlock);
    BlockAssertions.assertBlockEquals(BIGINT, page.getBlock(1), hashBlock);
}

Example 10 with PageBuilder

use of io.trino.spi.PageBuilder in project trino by trinodb.

the class TestGroupByHash method testAppendToMultipleTuplesPerGroup.

@Test
public void testAppendToMultipleTuplesPerGroup() {
    List<Long> values = new ArrayList<>();
    for (long i = 0; i < 100; i++) {
        values.add(i % 50);
    }
    Block valuesBlock = BlockAssertions.createLongsBlock(values);
    Block hashBlock = TypeTestUtils.getHashBlock(ImmutableList.of(BIGINT), valuesBlock);
    GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] { 0 }, Optional.of(1), 100, JOIN_COMPILER, TYPE_OPERATOR_FACTORY, NOOP);
    groupByHash.getGroupIds(new Page(valuesBlock, hashBlock)).process();
    assertEquals(groupByHash.getGroupCount(), 50);
    PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes());
    for (int i = 0; i < groupByHash.getGroupCount(); i++) {
        pageBuilder.declarePosition();
        groupByHash.appendValuesTo(i, pageBuilder, 0);
    }
    Page outputPage = pageBuilder.build();
    assertEquals(outputPage.getPositionCount(), 50);
    BlockAssertions.assertBlockEquals(BIGINT, outputPage.getBlock(0), BlockAssertions.createLongSequenceBlock(0, 50));
}