Examples with LineItemColumn io.trino.tpch.LineItemColumn used on opensource projects

Example 1 with LineItemColumn

use of io.trino.tpch.LineItemColumn in project trino by trinodb.

the class TestHivePageSink method getColumnHandles.

private static List<HiveColumnHandle> getColumnHandles() {
    ImmutableList.Builder<HiveColumnHandle> handles = ImmutableList.builder();
    List<LineItemColumn> columns = getTestColumns();
    for (int i = 0; i < columns.size(); i++) {
        LineItemColumn column = columns.get(i);
        HiveType hiveType = getHiveType(column.getType());
        handles.add(createBaseColumn(column.getColumnName(), i, hiveType, hiveType.getType(TESTING_TYPE_MANAGER), REGULAR, Optional.empty()));
    }
    return handles.build();
}

Example 2 with LineItemColumn

use of io.trino.tpch.LineItemColumn in project trino by trinodb.

the class TestDeltaLakePageSink method getColumnHandles.

private static List<DeltaLakeColumnHandle> getColumnHandles() {
    ImmutableList.Builder<DeltaLakeColumnHandle> handles = ImmutableList.builder();
    LineItemColumn[] columns = LineItemColumn.values();
    for (LineItemColumn column : columns) {
        handles.add(new DeltaLakeColumnHandle(column.getColumnName(), getTrinoType(column.getType()), REGULAR));
    }
    return handles.build();
}

Example 3 with LineItemColumn

use of io.trino.tpch.LineItemColumn in project trino by trinodb.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveConfig config, HiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle(false);
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(TESTING_TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getHiveSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getHiveSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(round(stats.getInputPageSizeInBytes().getAllTime().getMax()), page.getRetainedSizeInBytes());
    return length;
}

Example 4 with LineItemColumn

use of io.trino.tpch.LineItemColumn in project trino by trinodb.

the class TestDeltaLakePageSink method testPageSinkStats.

@Test
public void testPageSinkStats() throws Exception {
    File tempDir = Files.createTempDir();
    try {
        DeltaLakeWriterStats stats = new DeltaLakeWriterStats();
        String tablePath = tempDir.getAbsolutePath() + "/test_table";
        ConnectorPageSink pageSink = createPageSink(new Path(tablePath), stats);
        List<LineItemColumn> columns = ImmutableList.copyOf(LineItemColumn.values());
        List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestDeltaLakePageSink::getTrinoType).collect(toList());
        PageBuilder pageBuilder = new PageBuilder(columnTypes);
        long rows = 0;
        for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
            if (rows >= NUM_ROWS) {
                break;
            }
            rows++;
            pageBuilder.declarePosition();
            for (int i = 0; i < columns.size(); i++) {
                LineItemColumn column = columns.get(i);
                BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
                writeToBlock(blockBuilder, column, lineItem);
            }
        }
        Page page = pageBuilder.build();
        pageSink.appendPage(page);
        JsonCodec<DataFileInfo> dataFileInfoCodec = new JsonCodecFactory().jsonCodec(DataFileInfo.class);
        Collection<Slice> fragments = getFutureValue(pageSink.finish());
        List<DataFileInfo> dataFileInfos = fragments.stream().map(Slice::getBytes).map(dataFileInfoCodec::fromJson).collect(toImmutableList());
        assertEquals(dataFileInfos.size(), 1);
        DataFileInfo dataFileInfo = dataFileInfos.get(0);
        List<File> files = ImmutableList.copyOf(new File(tablePath).listFiles((dir, name) -> !name.endsWith(".crc")));
        assertEquals(files.size(), 1);
        File outputFile = files.get(0);
        assertEquals(round(stats.getInputPageSizeInBytes().getAllTime().getMax()), page.getRetainedSizeInBytes());
        assertEquals(dataFileInfo.getStatistics().getNumRecords(), Optional.of(rows));
        assertEquals(dataFileInfo.getPartitionValues(), ImmutableList.of());
        assertEquals(dataFileInfo.getSize(), outputFile.length());
        assertEquals(dataFileInfo.getPath(), outputFile.getName());
        Instant now = Instant.now();
        assertTrue(dataFileInfo.getCreationTime() < now.toEpochMilli());
        assertTrue(dataFileInfo.getCreationTime() > now.minus(1, MINUTES).toEpochMilli());
    } finally {
        deleteRecursively(tempDir.toPath(), ALLOW_INSECURE);
    }
}