Search in sources :

Example 1 with ColumnAdaptation

use of io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation in project trino by trinodb.

the class RaptorStorageManager method getPageSource.

@Override
public ConnectorPageSource getPageSource(UUID shardUuid, OptionalInt bucketNumber, List<Long> columnIds, List<Type> columnTypes, TupleDomain<RaptorColumnHandle> effectivePredicate, OrcReaderOptions orcReaderOptions, OptionalLong transactionId) {
    orcReaderOptions = orcReaderOptions.withMaxReadBlockSize(HUGE_MAX_READ_BLOCK_SIZE);
    OrcDataSource dataSource = openShard(shardUuid, orcReaderOptions);
    AggregatedMemoryContext memoryUsage = newSimpleAggregatedMemoryContext();
    try {
        OrcReader reader = OrcReader.createOrcReader(dataSource, orcReaderOptions).orElseThrow(() -> new TrinoException(RAPTOR_ERROR, "Data file is empty for shard " + shardUuid));
        Map<Long, OrcColumn> indexMap = columnIdIndex(reader.getRootColumn().getNestedColumns());
        List<OrcColumn> fileReadColumn = new ArrayList<>(columnIds.size());
        List<Type> fileReadTypes = new ArrayList<>(columnIds.size());
        List<ColumnAdaptation> columnAdaptations = new ArrayList<>(columnIds.size());
        for (int i = 0; i < columnIds.size(); i++) {
            long columnId = columnIds.get(i);
            if (isHiddenColumn(columnId)) {
                columnAdaptations.add(specialColumnAdaptation(columnId, shardUuid, bucketNumber));
                continue;
            }
            Type type = toOrcFileType(columnTypes.get(i), typeManager);
            OrcColumn fileColumn = indexMap.get(columnId);
            if (fileColumn == null) {
                columnAdaptations.add(ColumnAdaptation.nullColumn(type));
            } else {
                int sourceIndex = fileReadColumn.size();
                columnAdaptations.add(ColumnAdaptation.sourceColumn(sourceIndex));
                fileReadColumn.add(fileColumn);
                fileReadTypes.add(type);
            }
        }
        OrcPredicate predicate = getPredicate(effectivePredicate, indexMap);
        OrcRecordReader recordReader = reader.createRecordReader(fileReadColumn, fileReadTypes, predicate, UTC, memoryUsage, INITIAL_BATCH_SIZE, RaptorPageSource::handleException);
        Optional<ShardRewriter> shardRewriter = Optional.empty();
        if (transactionId.isPresent()) {
            shardRewriter = Optional.of(createShardRewriter(transactionId.getAsLong(), bucketNumber, shardUuid));
        }
        return new RaptorPageSource(shardRewriter, recordReader, columnAdaptations, dataSource, memoryUsage);
    } catch (IOException | RuntimeException e) {
        closeQuietly(dataSource);
        throw new TrinoException(RAPTOR_ERROR, "Failed to create page source for shard " + shardUuid, e);
    } catch (Throwable t) {
        closeQuietly(dataSource);
        throw t;
    }
}
Also used : FileOrcDataSource(io.trino.orc.FileOrcDataSource) OrcDataSource(io.trino.orc.OrcDataSource) OrcColumn(io.trino.orc.OrcColumn) ArrayList(java.util.ArrayList) IOException(java.io.IOException) OrcRecordReader(io.trino.orc.OrcRecordReader) AggregatedMemoryContext(io.trino.memory.context.AggregatedMemoryContext) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) CharType.createCharType(io.trino.spi.type.CharType.createCharType) OrcType(io.trino.orc.metadata.OrcType) MapType(io.trino.spi.type.MapType) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) RowType(io.trino.spi.type.RowType) ArrayType(io.trino.spi.type.ArrayType) DecimalType(io.trino.spi.type.DecimalType) Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) OrcReader(io.trino.orc.OrcReader) OptionalLong(java.util.OptionalLong) TrinoException(io.trino.spi.TrinoException) ColumnAdaptation(io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation) OrcPredicate(io.trino.orc.OrcPredicate) TupleDomainOrcPredicate(io.trino.orc.TupleDomainOrcPredicate)

Aggregations

AggregatedMemoryContext (io.trino.memory.context.AggregatedMemoryContext)1 AggregatedMemoryContext.newSimpleAggregatedMemoryContext (io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext)1 FileOrcDataSource (io.trino.orc.FileOrcDataSource)1 OrcColumn (io.trino.orc.OrcColumn)1 OrcDataSource (io.trino.orc.OrcDataSource)1 OrcPredicate (io.trino.orc.OrcPredicate)1 OrcReader (io.trino.orc.OrcReader)1 OrcRecordReader (io.trino.orc.OrcRecordReader)1 TupleDomainOrcPredicate (io.trino.orc.TupleDomainOrcPredicate)1 OrcType (io.trino.orc.metadata.OrcType)1 ColumnAdaptation (io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation)1 TrinoException (io.trino.spi.TrinoException)1 ArrayType (io.trino.spi.type.ArrayType)1 CharType.createCharType (io.trino.spi.type.CharType.createCharType)1 DecimalType (io.trino.spi.type.DecimalType)1 MapType (io.trino.spi.type.MapType)1 RowType (io.trino.spi.type.RowType)1 Type (io.trino.spi.type.Type)1 VarcharType.createUnboundedVarcharType (io.trino.spi.type.VarcharType.createUnboundedVarcharType)1 VarcharType.createVarcharType (io.trino.spi.type.VarcharType.createVarcharType)1