Examples with ConnectorPageSource io.trino.spi.connector.ConnectorPageSource used on opensource projects

Example 26 with ConnectorPageSource

use of io.trino.spi.connector.ConnectorPageSource in project trino by trinodb.

the class TestTimestampMicros method testTimestampMicros.

@Test(dataProvider = "testTimestampMicrosDataProvider")
public void testTimestampMicros(HiveTimestampPrecision timestampPrecision, LocalDateTime expected) throws Exception {
    ConnectorSession session = getHiveSession(new HiveConfig().setTimestampPrecision(timestampPrecision));
    File parquetFile = new File(Resources.getResource("issue-5483.parquet").toURI());
    Type columnType = createTimestampType(timestampPrecision.getPrecision());
    try (ConnectorPageSource pageSource = createPageSource(session, parquetFile, "created", HIVE_TIMESTAMP, columnType)) {
        MaterializedResult result = materializeSourceDataStream(session, pageSource, List.of(columnType)).toTestTypes();
        assertThat(result.getMaterializedRows()).containsOnly(new MaterializedRow(List.of(expected)));
    }
}

Example 27 with ConnectorPageSource

use of io.trino.spi.connector.ConnectorPageSource in project trino by trinodb.

the class TestTimestampMicros method testTimestampMicrosAsTimestampWithTimeZone.

@Test(dataProvider = "testTimestampMicrosDataProvider")
public void testTimestampMicrosAsTimestampWithTimeZone(HiveTimestampPrecision timestampPrecision, LocalDateTime expected) throws Exception {
    ConnectorSession session = getHiveSession(new HiveConfig().setTimestampPrecision(timestampPrecision));
    File parquetFile = new File(Resources.getResource("issue-5483.parquet").toURI());
    Type columnType = createTimestampWithTimeZoneType(timestampPrecision.getPrecision());
    try (ConnectorPageSource pageSource = createPageSource(session, parquetFile, "created", HIVE_TIMESTAMP, columnType)) {
        MaterializedResult result = materializeSourceDataStream(session, pageSource, List.of(columnType)).toTestTypes();
        assertThat(result.getMaterializedRows()).containsOnly(new MaterializedRow(List.of(expected.atZone(ZoneId.of("UTC")))));
    }
}

Example 28 with ConnectorPageSource

use of io.trino.spi.connector.ConnectorPageSource in project trino by trinodb.

the class ShardCompactor method compact.

private List<ShardInfo> compact(StoragePageSink storagePageSink, OptionalInt bucketNumber, Set<UUID> uuids, List<Long> columnIds, List<Type> columnTypes) throws IOException {
    for (UUID uuid : uuids) {
        try (ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), orcReaderOptions)) {
            while (!pageSource.isFinished()) {
                Page page = pageSource.getNextPage();
                if (isNullOrEmptyPage(page)) {
                    continue;
                }
                storagePageSink.appendPages(ImmutableList.of(page));
                if (storagePageSink.isFull()) {
                    storagePageSink.flush();
                }
            }
        }
    }
    return getFutureValue(storagePageSink.commit());
}

Example 29 with ConnectorPageSource

use of io.trino.spi.connector.ConnectorPageSource in project trino by trinodb.

the class ShardCompactor method compactSorted.

public List<ShardInfo> compactSorted(long transactionId, OptionalInt bucketNumber, Set<UUID> uuids, List<ColumnInfo> columns, List<Long> sortColumnIds, List<SortOrder> sortOrders) throws IOException {
    checkArgument(sortColumnIds.size() == sortOrders.size(), "sortColumnIds and sortOrders must be of the same size");
    long start = System.nanoTime();
    List<Long> columnIds = columns.stream().map(ColumnInfo::getColumnId).collect(toList());
    List<Type> columnTypes = columns.stream().map(ColumnInfo::getType).collect(toList());
    checkArgument(columnIds.containsAll(sortColumnIds), "sortColumnIds must be a subset of columnIds");
    List<Integer> sortIndexes = sortColumnIds.stream().map(columnIds::indexOf).collect(toList());
    Queue<SortedRowSource> rowSources = new PriorityQueue<>();
    StoragePageSink outputPageSink = storageManager.createStoragePageSink(transactionId, bucketNumber, columnIds, columnTypes, false);
    try {
        for (UUID uuid : uuids) {
            ConnectorPageSource pageSource = storageManager.getPageSource(uuid, bucketNumber, columnIds, columnTypes, TupleDomain.all(), orcReaderOptions);
            SortedRowSource rowSource = new SortedRowSource(pageSource, columnTypes, sortIndexes, sortOrders, typeOperators);
            rowSources.add(rowSource);
        }
        while (!rowSources.isEmpty()) {
            SortedRowSource rowSource = rowSources.poll();
            if (!rowSource.hasNext()) {
                // rowSource is empty, close it
                rowSource.close();
                continue;
            }
            outputPageSink.appendRow(rowSource.next());
            if (outputPageSink.isFull()) {
                outputPageSink.flush();
            }
            rowSources.add(rowSource);
        }
        outputPageSink.flush();
        List<ShardInfo> shardInfos = getFutureValue(outputPageSink.commit());
        updateStats(uuids.size(), shardInfos.size(), nanosSince(start).toMillis());
        return shardInfos;
    } catch (IOException | RuntimeException e) {
        outputPageSink.rollback();
        throw e;
    } finally {
        rowSources.forEach(SortedRowSource::closeQuietly);
    }
}

Example 30 with ConnectorPageSource

use of io.trino.spi.connector.ConnectorPageSource in project trino by trinodb.

the class TpchTables method getTablePages.

public static Iterator<Page> getTablePages(String tableName, double scaleFactor, DecimalTypeMapping decimalTypeMapping) {
    TpchTable<?> table = TpchTable.getTable(tableName);
    ConnectorPageSource pageSource = new RecordPageSource(createTpchRecordSet(table, decimalTypeMapping, scaleFactor, 1, 1, TupleDomain.all()));
    return new AbstractIterator<>() {

        @Override
        protected Page computeNext() {
            if (pageSource.isFinished()) {
                return endOfData();
            }
            Page page = pageSource.getNextPage();
            if (page == null) {
                return computeNext();
            }
            return page.getLoadedPage();
        }
    };
}