Search in sources :

Example 6 with RowType.field

use of io.trino.spi.type.RowType.field in project trino by trinodb.

the class AvroColumnDecoder method serializeRow.

private static Block serializeRow(BlockBuilder parentBlockBuilder, Object value, Type type, String columnName) {
    if (value == null) {
        checkState(parentBlockBuilder != null, "parent block builder is null");
        parentBlockBuilder.appendNull();
        return null;
    }
    BlockBuilder blockBuilder;
    if (parentBlockBuilder != null) {
        blockBuilder = parentBlockBuilder;
    } else {
        blockBuilder = type.createBlockBuilder(null, 1);
    }
    BlockBuilder singleRowBuilder = blockBuilder.beginBlockEntry();
    GenericRecord record = (GenericRecord) value;
    List<Field> fields = ((RowType) type).getFields();
    for (Field field : fields) {
        checkState(field.getName().isPresent(), "field name not found");
        serializeObject(singleRowBuilder, record.get(field.getName().get()), field.getType(), columnName);
    }
    blockBuilder.closeEntry();
    if (parentBlockBuilder == null) {
        return blockBuilder.getObject(0, Block.class);
    }
    return null;
}
Also used : Field(io.trino.spi.type.RowType.Field) RowType(io.trino.spi.type.RowType) GenericRecord(org.apache.avro.generic.GenericRecord) BlockBuilder(io.trino.spi.block.BlockBuilder)

Example 7 with RowType.field

use of io.trino.spi.type.RowType.field in project trino by trinodb.

the class TestPushDownDereferencesRules method testMultiLevelPushdown.

@Test
public void testMultiLevelPushdown() {
    Type complexType = rowType(field("f1", rowType(field("f1", BIGINT), field("f2", BIGINT))), field("f2", BIGINT));
    tester().assertThat(new PushDownDereferenceThroughProject(tester().getTypeAnalyzer())).on(p -> p.project(Assignments.of(p.symbol("expr_1"), expression("a[1]"), p.symbol("expr_2"), expression("a[1][1] + 2 + b[1][1] + b[1][2]")), p.project(Assignments.identity(ImmutableList.of(p.symbol("a", complexType), p.symbol("b", complexType))), p.values(p.symbol("a", complexType), p.symbol("b", complexType))))).matches(strictProject(ImmutableMap.of("expr_1", PlanMatchPattern.expression("a_f1"), "expr_2", PlanMatchPattern.expression("a_f1[1] + 2 + b_f1_f1 + b_f1_f2")), strictProject(ImmutableMap.of("a", PlanMatchPattern.expression("a"), "b", PlanMatchPattern.expression("b"), "a_f1", PlanMatchPattern.expression("a[1]"), "b_f1_f1", PlanMatchPattern.expression("b[1][1]"), "b_f1_f2", PlanMatchPattern.expression("b[1][2]")), values("a", "b"))));
}
Also used : ROW_NUMBER(io.trino.sql.planner.plan.TopNRankingNode.RankingType.ROW_NUMBER) CATALOG_ID(io.trino.sql.planner.iterative.rule.test.RuleTester.CATALOG_ID) TypeSignatureProvider.fromTypes(io.trino.sql.analyzer.TypeSignatureProvider.fromTypes) PlanMatchPattern(io.trino.sql.planner.assertions.PlanMatchPattern) Test(org.testng.annotations.Test) Collections.singletonList(java.util.Collections.singletonList) PlanMatchPattern.assignUniqueId(io.trino.sql.planner.assertions.PlanMatchPattern.assignUniqueId) PlanMatchPattern.limit(io.trino.sql.planner.assertions.PlanMatchPattern.limit) CatalogName(io.trino.connector.CatalogName) PlanMatchPattern.markDistinct(io.trino.sql.planner.assertions.PlanMatchPattern.markDistinct) TpchTableHandle(io.trino.plugin.tpch.TpchTableHandle) TEST_SESSION(io.trino.SessionTestUtils.TEST_SESSION) ASCENDING(io.trino.sql.tree.SortItem.Ordering.ASCENDING) ExpressionMatcher(io.trino.sql.planner.assertions.ExpressionMatcher) TpchColumnHandle(io.trino.plugin.tpch.TpchColumnHandle) RowType(io.trino.spi.type.RowType) ImmutableMap(com.google.common.collect.ImmutableMap) PlanMatchPattern.topNRanking(io.trino.sql.planner.assertions.PlanMatchPattern.topNRanking) Assignments(io.trino.sql.planner.plan.Assignments) ArrayType(io.trino.spi.type.ArrayType) PlanMatchPattern.values(io.trino.sql.planner.assertions.PlanMatchPattern.values) SortItem(io.trino.sql.tree.SortItem) ASC_NULLS_FIRST(io.trino.spi.connector.SortOrder.ASC_NULLS_FIRST) FIRST(io.trino.sql.tree.SortItem.NullOrdering.FIRST) PlanMatchPattern.strictProject(io.trino.sql.planner.assertions.PlanMatchPattern.strictProject) BIGINT(io.trino.spi.type.BigintType.BIGINT) MetadataManager.createTestMetadataManager(io.trino.metadata.MetadataManager.createTestMetadataManager) WindowFrame(io.trino.sql.tree.WindowFrame) Optional(java.util.Optional) RowType.field(io.trino.spi.type.RowType.field) WindowNode(io.trino.sql.planner.plan.WindowNode) PlanMatchPattern.rowNumber(io.trino.sql.planner.assertions.PlanMatchPattern.rowNumber) PlanBuilder.expression(io.trino.sql.planner.iterative.rule.test.PlanBuilder.expression) PlanMatchPattern.semiJoin(io.trino.sql.planner.assertions.PlanMatchPattern.semiJoin) INNER(io.trino.sql.planner.plan.JoinNode.Type.INNER) PlanMatchPattern.window(io.trino.sql.planner.assertions.PlanMatchPattern.window) Type(io.trino.spi.type.Type) BOOLEAN(io.trino.spi.type.BooleanType.BOOLEAN) PlanMatchPattern.filter(io.trino.sql.planner.assertions.PlanMatchPattern.filter) ImmutableList(com.google.common.collect.ImmutableList) RowType.rowType(io.trino.spi.type.RowType.rowType) PlanMatchPattern.sort(io.trino.sql.planner.assertions.PlanMatchPattern.sort) PlanMatchPattern.join(io.trino.sql.planner.assertions.PlanMatchPattern.join) BaseRuleTest(io.trino.sql.planner.iterative.rule.test.BaseRuleTest) PlanMatchPattern.unnest(io.trino.sql.planner.assertions.PlanMatchPattern.unnest) TestingTransactionHandle(io.trino.testing.TestingTransactionHandle) PlanMatchPattern.topN(io.trino.sql.planner.assertions.PlanMatchPattern.topN) TupleDomain(io.trino.spi.predicate.TupleDomain) OrderingScheme(io.trino.sql.planner.OrderingScheme) UnnestNode(io.trino.sql.planner.plan.UnnestNode) SortOrder(io.trino.spi.connector.SortOrder) PlanMatchPattern.functionCall(io.trino.sql.planner.assertions.PlanMatchPattern.functionCall) QualifiedName(io.trino.sql.tree.QualifiedName) TableHandle(io.trino.metadata.TableHandle) PlanMatchPattern.project(io.trino.sql.planner.assertions.PlanMatchPattern.project) FrameBound(io.trino.sql.tree.FrameBound) UnnestMapping.unnestMapping(io.trino.sql.planner.assertions.PlanMatchPattern.UnnestMapping.unnestMapping) PlanMatchPattern.tableScan(io.trino.sql.planner.assertions.PlanMatchPattern.tableScan) RowType(io.trino.spi.type.RowType) ArrayType(io.trino.spi.type.ArrayType) Type(io.trino.spi.type.Type) RowType.rowType(io.trino.spi.type.RowType.rowType) Test(org.testng.annotations.Test) BaseRuleTest(io.trino.sql.planner.iterative.rule.test.BaseRuleTest)

Example 8 with RowType.field

use of io.trino.spi.type.RowType.field in project trino by trinodb.

the class RaptorStorageManager method toOrcFileType.

static Type toOrcFileType(Type raptorType, TypeManager typeManager) {
    // TIMESTAMPS are stored as BIGINT to void the poor encoding in ORC
    if (raptorType.equals(TIMESTAMP_MILLIS)) {
        return BIGINT;
    }
    if (raptorType instanceof ArrayType) {
        Type elementType = toOrcFileType(((ArrayType) raptorType).getElementType(), typeManager);
        return new ArrayType(elementType);
    }
    if (raptorType instanceof MapType) {
        TypeSignature keyType = toOrcFileType(((MapType) raptorType).getKeyType(), typeManager).getTypeSignature();
        TypeSignature valueType = toOrcFileType(((MapType) raptorType).getValueType(), typeManager).getTypeSignature();
        return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(keyType), TypeSignatureParameter.typeParameter(valueType)));
    }
    if (raptorType instanceof RowType) {
        List<Field> fields = ((RowType) raptorType).getFields().stream().map(field -> new Field(field.getName(), toOrcFileType(field.getType(), typeManager))).collect(toImmutableList());
        return RowType.from(fields);
    }
    return raptorType;
}
Also used : ArrayType(io.trino.spi.type.ArrayType) NamedTypeSignature(io.trino.spi.type.NamedTypeSignature) MoreFutures.allAsList(io.airlift.concurrent.MoreFutures.allAsList) OrcPredicate(io.trino.orc.OrcPredicate) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) CharType.createCharType(io.trino.spi.type.CharType.createCharType) RAPTOR_RECOVERY_TIMEOUT(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_TIMEOUT) Future(java.util.concurrent.Future) ShardDelta(io.trino.plugin.raptor.legacy.metadata.ShardDelta) Slices(io.airlift.slice.Slices) Map(java.util.Map) RaptorColumnHandle.isShardUuidColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardUuidColumn) FileOrcDataSource(io.trino.orc.FileOrcDataSource) RAPTOR_ERROR(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR) ConnectorPageSource(io.trino.spi.connector.ConnectorPageSource) BackupStore(io.trino.plugin.raptor.legacy.backup.BackupStore) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TypeSignatureParameter(io.trino.spi.type.TypeSignatureParameter) Slice(io.airlift.slice.Slice) OrcFileInfo(io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo) TIMESTAMP_MILLIS(io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS) Page(io.trino.spi.Page) BOOLEAN(io.trino.spi.type.BooleanType.BOOLEAN) ShardRecorder(io.trino.plugin.raptor.legacy.metadata.ShardRecorder) ArrayList(java.util.ArrayList) OptionalLong(java.util.OptionalLong) ShardStats.computeColumnStats(io.trino.plugin.raptor.legacy.storage.ShardStats.computeColumnStats) TupleDomainOrcPredicateBuilder(io.trino.orc.TupleDomainOrcPredicate.TupleDomainOrcPredicateBuilder) AggregatedMemoryContext(io.trino.memory.context.AggregatedMemoryContext) VARBINARY(io.trino.spi.type.VarbinaryType.VARBINARY) OrcType(io.trino.orc.metadata.OrcType) ATOMIC_MOVE(java.nio.file.StandardCopyOption.ATOMIC_MOVE) Files(java.nio.file.Files) MapType(io.trino.spi.type.MapType) StandardTypes(io.trino.spi.type.StandardTypes) IOException(java.io.IOException) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) CatalogName(io.trino.plugin.base.CatalogName) Throwables.throwIfInstanceOf(com.google.common.base.Throwables.throwIfInstanceOf) File(java.io.File) RAPTOR_RECOVERY_ERROR(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR) ExecutionException(java.util.concurrent.ExecutionException) JsonCodec.jsonCodec(io.airlift.json.JsonCodec.jsonCodec) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) ShardInfo(io.trino.plugin.raptor.legacy.metadata.ShardInfo) OrcColumn(io.trino.orc.OrcColumn) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) OrcColumnId(io.trino.orc.metadata.OrcColumnId) NodeManager(io.trino.spi.NodeManager) CompletableFuture.completedFuture(java.util.concurrent.CompletableFuture.completedFuture) TimeoutException(java.util.concurrent.TimeoutException) BackupManager(io.trino.plugin.raptor.legacy.backup.BackupManager) Duration(io.airlift.units.Duration) ColumnInfo(io.trino.plugin.raptor.legacy.metadata.ColumnInfo) PreDestroy(javax.annotation.PreDestroy) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OrcRecordReader(io.trino.orc.OrcRecordReader) OrcDataSource(io.trino.orc.OrcDataSource) ROOT_COLUMN(io.trino.orc.metadata.OrcColumnId.ROOT_COLUMN) TypeSignature(io.trino.spi.type.TypeSignature) RowType(io.trino.spi.type.RowType) ImmutableSet(com.google.common.collect.ImmutableSet) Field(io.trino.spi.type.RowType.Field) Collection(java.util.Collection) INITIAL_BATCH_SIZE(io.trino.orc.OrcReader.INITIAL_BATCH_SIZE) TrinoException(io.trino.spi.TrinoException) ArrayType(io.trino.spi.type.ArrayType) UUID(java.util.UUID) Math.min(java.lang.Math.min) RaptorColumnHandle(io.trino.plugin.raptor.legacy.RaptorColumnHandle) FileNotFoundException(java.io.FileNotFoundException) Preconditions.checkState(com.google.common.base.Preconditions.checkState) DataSize(io.airlift.units.DataSize) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) Optional(java.util.Optional) DecimalType(io.trino.spi.type.DecimalType) JsonCodec(io.airlift.json.JsonCodec) RaptorColumnHandle.isHiddenColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) CompletableFuture(java.util.concurrent.CompletableFuture) TupleDomainOrcPredicate(io.trino.orc.TupleDomainOrcPredicate) OptionalInt(java.util.OptionalInt) ColumnAdaptation(io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation) Inject(javax.inject.Inject) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) OrcReaderOptions(io.trino.orc.OrcReaderOptions) Objects.requireNonNull(java.util.Objects.requireNonNull) CompletableFuture.supplyAsync(java.util.concurrent.CompletableFuture.supplyAsync) RowFieldName(io.trino.spi.type.RowFieldName) ExecutorService(java.util.concurrent.ExecutorService) OrcReader(io.trino.orc.OrcReader) RAPTOR_LOCAL_DISK_FULL(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_LOCAL_DISK_FULL) PETABYTE(io.airlift.units.DataSize.Unit.PETABYTE) XxHash64(io.airlift.slice.XxHash64) ColumnMetadata(io.trino.orc.metadata.ColumnMetadata) FileInputStream(java.io.FileInputStream) TupleDomain(io.trino.spi.predicate.TupleDomain) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) TimeUnit(java.util.concurrent.TimeUnit) RaptorColumnHandle.isShardRowIdColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardRowIdColumn) RaptorColumnHandle.isBucketNumberColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isBucketNumberColumn) Collectors.toList(java.util.stream.Collectors.toList) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) ColumnStats(io.trino.plugin.raptor.legacy.metadata.ColumnStats) Closeable(java.io.Closeable) VisibleForTesting(com.google.common.annotations.VisibleForTesting) BitSet(java.util.BitSet) TypeManager(io.trino.spi.type.TypeManager) InputStream(java.io.InputStream) Field(io.trino.spi.type.RowType.Field) CharType.createCharType(io.trino.spi.type.CharType.createCharType) OrcType(io.trino.orc.metadata.OrcType) MapType(io.trino.spi.type.MapType) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) RowType(io.trino.spi.type.RowType) ArrayType(io.trino.spi.type.ArrayType) DecimalType(io.trino.spi.type.DecimalType) Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) NamedTypeSignature(io.trino.spi.type.NamedTypeSignature) TypeSignature(io.trino.spi.type.TypeSignature) RowType(io.trino.spi.type.RowType) MapType(io.trino.spi.type.MapType)

Aggregations

RowType (io.trino.spi.type.RowType)7 Field (io.trino.spi.type.RowType.Field)6 ImmutableList (com.google.common.collect.ImmutableList)5 ArrayType (io.trino.spi.type.ArrayType)5 Type (io.trino.spi.type.Type)5 TupleDomain (io.trino.spi.predicate.TupleDomain)3 BIGINT (io.trino.spi.type.BigintType.BIGINT)3 BOOLEAN (io.trino.spi.type.BooleanType.BOOLEAN)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 TEST_SESSION (io.trino.SessionTestUtils.TEST_SESSION)2 CatalogName (io.trino.connector.CatalogName)2 MetadataManager.createTestMetadataManager (io.trino.metadata.MetadataManager.createTestMetadataManager)2 TableHandle (io.trino.metadata.TableHandle)2 CharType (io.trino.spi.type.CharType)2 CharType.createCharType (io.trino.spi.type.CharType.createCharType)2 DecimalType (io.trino.spi.type.DecimalType)2 DecimalType.createDecimalType (io.trino.spi.type.DecimalType.createDecimalType)2 MapType (io.trino.spi.type.MapType)2 TimeType (io.trino.spi.type.TimeType)2 TimeType.createTimeType (io.trino.spi.type.TimeType.createTimeType)2