Search in sources :

Example 81 with MapType

use of io.trino.spi.type.MapType in project trino by trinodb.

the class RaptorStorageManager method toOrcFileType.

static Type toOrcFileType(Type raptorType, TypeManager typeManager) {
    // TIMESTAMPS are stored as BIGINT to void the poor encoding in ORC
    if (raptorType.equals(TIMESTAMP_MILLIS)) {
        return BIGINT;
    }
    if (raptorType instanceof ArrayType) {
        Type elementType = toOrcFileType(((ArrayType) raptorType).getElementType(), typeManager);
        return new ArrayType(elementType);
    }
    if (raptorType instanceof MapType) {
        TypeSignature keyType = toOrcFileType(((MapType) raptorType).getKeyType(), typeManager).getTypeSignature();
        TypeSignature valueType = toOrcFileType(((MapType) raptorType).getValueType(), typeManager).getTypeSignature();
        return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.typeParameter(keyType), TypeSignatureParameter.typeParameter(valueType)));
    }
    if (raptorType instanceof RowType) {
        List<Field> fields = ((RowType) raptorType).getFields().stream().map(field -> new Field(field.getName(), toOrcFileType(field.getType(), typeManager))).collect(toImmutableList());
        return RowType.from(fields);
    }
    return raptorType;
}
Also used : ArrayType(io.trino.spi.type.ArrayType) NamedTypeSignature(io.trino.spi.type.NamedTypeSignature) MoreFutures.allAsList(io.airlift.concurrent.MoreFutures.allAsList) OrcPredicate(io.trino.orc.OrcPredicate) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) CharType.createCharType(io.trino.spi.type.CharType.createCharType) RAPTOR_RECOVERY_TIMEOUT(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_TIMEOUT) Future(java.util.concurrent.Future) ShardDelta(io.trino.plugin.raptor.legacy.metadata.ShardDelta) Slices(io.airlift.slice.Slices) Map(java.util.Map) RaptorColumnHandle.isShardUuidColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardUuidColumn) FileOrcDataSource(io.trino.orc.FileOrcDataSource) RAPTOR_ERROR(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_ERROR) ConnectorPageSource(io.trino.spi.connector.ConnectorPageSource) BackupStore(io.trino.plugin.raptor.legacy.backup.BackupStore) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TypeSignatureParameter(io.trino.spi.type.TypeSignatureParameter) Slice(io.airlift.slice.Slice) OrcFileInfo(io.trino.plugin.raptor.legacy.storage.OrcFileRewriter.OrcFileInfo) TIMESTAMP_MILLIS(io.trino.spi.type.TimestampType.TIMESTAMP_MILLIS) Page(io.trino.spi.Page) BOOLEAN(io.trino.spi.type.BooleanType.BOOLEAN) ShardRecorder(io.trino.plugin.raptor.legacy.metadata.ShardRecorder) ArrayList(java.util.ArrayList) OptionalLong(java.util.OptionalLong) ShardStats.computeColumnStats(io.trino.plugin.raptor.legacy.storage.ShardStats.computeColumnStats) TupleDomainOrcPredicateBuilder(io.trino.orc.TupleDomainOrcPredicate.TupleDomainOrcPredicateBuilder) AggregatedMemoryContext(io.trino.memory.context.AggregatedMemoryContext) VARBINARY(io.trino.spi.type.VarbinaryType.VARBINARY) OrcType(io.trino.orc.metadata.OrcType) ATOMIC_MOVE(java.nio.file.StandardCopyOption.ATOMIC_MOVE) Files(java.nio.file.Files) MapType(io.trino.spi.type.MapType) StandardTypes(io.trino.spi.type.StandardTypes) IOException(java.io.IOException) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) UTC(org.joda.time.DateTimeZone.UTC) CatalogName(io.trino.plugin.base.CatalogName) Throwables.throwIfInstanceOf(com.google.common.base.Throwables.throwIfInstanceOf) File(java.io.File) RAPTOR_RECOVERY_ERROR(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_RECOVERY_ERROR) ExecutionException(java.util.concurrent.ExecutionException) JsonCodec.jsonCodec(io.airlift.json.JsonCodec.jsonCodec) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) ShardInfo(io.trino.plugin.raptor.legacy.metadata.ShardInfo) OrcColumn(io.trino.orc.OrcColumn) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) OrcColumnId(io.trino.orc.metadata.OrcColumnId) NodeManager(io.trino.spi.NodeManager) CompletableFuture.completedFuture(java.util.concurrent.CompletableFuture.completedFuture) TimeoutException(java.util.concurrent.TimeoutException) BackupManager(io.trino.plugin.raptor.legacy.backup.BackupManager) Duration(io.airlift.units.Duration) ColumnInfo(io.trino.plugin.raptor.legacy.metadata.ColumnInfo) PreDestroy(javax.annotation.PreDestroy) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) OrcRecordReader(io.trino.orc.OrcRecordReader) OrcDataSource(io.trino.orc.OrcDataSource) ROOT_COLUMN(io.trino.orc.metadata.OrcColumnId.ROOT_COLUMN) TypeSignature(io.trino.spi.type.TypeSignature) RowType(io.trino.spi.type.RowType) ImmutableSet(com.google.common.collect.ImmutableSet) Field(io.trino.spi.type.RowType.Field) Collection(java.util.Collection) INITIAL_BATCH_SIZE(io.trino.orc.OrcReader.INITIAL_BATCH_SIZE) TrinoException(io.trino.spi.TrinoException) ArrayType(io.trino.spi.type.ArrayType) UUID(java.util.UUID) Math.min(java.lang.Math.min) RaptorColumnHandle(io.trino.plugin.raptor.legacy.RaptorColumnHandle) FileNotFoundException(java.io.FileNotFoundException) Preconditions.checkState(com.google.common.base.Preconditions.checkState) DataSize(io.airlift.units.DataSize) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) Optional(java.util.Optional) DecimalType(io.trino.spi.type.DecimalType) JsonCodec(io.airlift.json.JsonCodec) RaptorColumnHandle.isHiddenColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isHiddenColumn) AggregatedMemoryContext.newSimpleAggregatedMemoryContext(io.trino.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext) Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) CompletableFuture(java.util.concurrent.CompletableFuture) TupleDomainOrcPredicate(io.trino.orc.TupleDomainOrcPredicate) OptionalInt(java.util.OptionalInt) ColumnAdaptation(io.trino.plugin.raptor.legacy.storage.RaptorPageSource.ColumnAdaptation) Inject(javax.inject.Inject) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) OrcReaderOptions(io.trino.orc.OrcReaderOptions) Objects.requireNonNull(java.util.Objects.requireNonNull) CompletableFuture.supplyAsync(java.util.concurrent.CompletableFuture.supplyAsync) RowFieldName(io.trino.spi.type.RowFieldName) ExecutorService(java.util.concurrent.ExecutorService) OrcReader(io.trino.orc.OrcReader) RAPTOR_LOCAL_DISK_FULL(io.trino.plugin.raptor.legacy.RaptorErrorCode.RAPTOR_LOCAL_DISK_FULL) PETABYTE(io.airlift.units.DataSize.Unit.PETABYTE) XxHash64(io.airlift.slice.XxHash64) ColumnMetadata(io.trino.orc.metadata.ColumnMetadata) FileInputStream(java.io.FileInputStream) TupleDomain(io.trino.spi.predicate.TupleDomain) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) TimeUnit(java.util.concurrent.TimeUnit) RaptorColumnHandle.isShardRowIdColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isShardRowIdColumn) RaptorColumnHandle.isBucketNumberColumn(io.trino.plugin.raptor.legacy.RaptorColumnHandle.isBucketNumberColumn) Collectors.toList(java.util.stream.Collectors.toList) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) ColumnStats(io.trino.plugin.raptor.legacy.metadata.ColumnStats) Closeable(java.io.Closeable) VisibleForTesting(com.google.common.annotations.VisibleForTesting) BitSet(java.util.BitSet) TypeManager(io.trino.spi.type.TypeManager) InputStream(java.io.InputStream) Field(io.trino.spi.type.RowType.Field) CharType.createCharType(io.trino.spi.type.CharType.createCharType) OrcType(io.trino.orc.metadata.OrcType) MapType(io.trino.spi.type.MapType) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) RowType(io.trino.spi.type.RowType) ArrayType(io.trino.spi.type.ArrayType) DecimalType(io.trino.spi.type.DecimalType) Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) NamedTypeSignature(io.trino.spi.type.NamedTypeSignature) TypeSignature(io.trino.spi.type.TypeSignature) RowType(io.trino.spi.type.RowType) MapType(io.trino.spi.type.MapType)

Example 82 with MapType

use of io.trino.spi.type.MapType in project trino by trinodb.

the class StructuralTestUtil method mapBlockOf.

public static Block mapBlockOf(Type keyType, Type valueType, Object key, Object value) {
    MapType mapType = mapType(keyType, valueType);
    BlockBuilder blockBuilder = mapType.createBlockBuilder(null, 10);
    BlockBuilder singleMapBlockWriter = blockBuilder.beginBlockEntry();
    appendToBlockBuilder(keyType, key, singleMapBlockWriter);
    appendToBlockBuilder(valueType, value, singleMapBlockWriter);
    blockBuilder.closeEntry();
    return mapType.getObject(blockBuilder, 0);
}
Also used : MapType(io.trino.spi.type.MapType) StructuralTestUtil.appendToBlockBuilder(io.trino.util.StructuralTestUtil.appendToBlockBuilder) BlockBuilder(io.trino.spi.block.BlockBuilder)

Example 83 with MapType

use of io.trino.spi.type.MapType in project trino by trinodb.

the class KafkaRecordSet method headerMapValueProvider.

public static FieldValueProvider headerMapValueProvider(MapType varcharMapType, Headers headers) {
    Type keyType = varcharMapType.getTypeParameters().get(0);
    Type valueArrayType = varcharMapType.getTypeParameters().get(1);
    Type valueType = valueArrayType.getTypeParameters().get(0);
    BlockBuilder mapBlockBuilder = varcharMapType.createBlockBuilder(null, 1);
    BlockBuilder builder = mapBlockBuilder.beginBlockEntry();
    // Group by keys and collect values as array.
    Multimap<String, byte[]> headerMap = ArrayListMultimap.create();
    for (Header header : headers) {
        headerMap.put(header.key(), header.value());
    }
    for (String headerKey : headerMap.keySet()) {
        writeNativeValue(keyType, builder, headerKey);
        BlockBuilder arrayBuilder = builder.beginBlockEntry();
        for (byte[] value : headerMap.get(headerKey)) {
            writeNativeValue(valueType, arrayBuilder, value);
        }
        builder.closeEntry();
    }
    mapBlockBuilder.closeEntry();
    return new FieldValueProvider() {

        @Override
        public boolean isNull() {
            return false;
        }

        @Override
        public Block getBlock() {
            return varcharMapType.getObject(mapBlockBuilder, 0);
        }
    };
}
Also used : Type(io.trino.spi.type.Type) MapType(io.trino.spi.type.MapType) Header(org.apache.kafka.common.header.Header) FieldValueProvider(io.trino.decoder.FieldValueProvider) BlockBuilder(io.trino.spi.block.BlockBuilder)

Example 84 with MapType

use of io.trino.spi.type.MapType in project trino by trinodb.

the class PrometheusRecordCursor method getBlockFromMap.

static Block getBlockFromMap(Type mapType, Map<?, ?> map) {
    // on functions like COUNT() the Type won't be a MapType
    if (!(mapType instanceof MapType)) {
        return null;
    }
    Type keyType = mapType.getTypeParameters().get(0);
    Type valueType = mapType.getTypeParameters().get(1);
    BlockBuilder mapBlockBuilder = mapType.createBlockBuilder(null, 1);
    BlockBuilder builder = mapBlockBuilder.beginBlockEntry();
    for (Map.Entry<?, ?> entry : map.entrySet()) {
        writeObject(builder, keyType, entry.getKey());
        writeObject(builder, valueType, entry.getValue());
    }
    mapBlockBuilder.closeEntry();
    return (Block) mapType.getObject(mapBlockBuilder, 0);
}
Also used : Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) VarcharType(io.trino.spi.type.VarcharType) MapType(io.trino.spi.type.MapType) ArrayType(io.trino.spi.type.ArrayType) Block(io.trino.spi.block.Block) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) MapType(io.trino.spi.type.MapType) BlockBuilder(io.trino.spi.block.BlockBuilder)

Example 85 with MapType

use of io.trino.spi.type.MapType in project trino by trinodb.

the class PrometheusRecordCursor method getMapFromBlock.

static Map<Object, Object> getMapFromBlock(Type type, Block block) {
    MapType mapType = (MapType) type;
    Type keyType = mapType.getKeyType();
    Type valueType = mapType.getValueType();
    Map<Object, Object> map = new HashMap<>(block.getPositionCount() / 2);
    for (int i = 0; i < block.getPositionCount(); i += 2) {
        map.put(readObject(keyType, block, i), readObject(valueType, block, i + 1));
    }
    return map;
}
Also used : Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) VarcharType(io.trino.spi.type.VarcharType) MapType(io.trino.spi.type.MapType) ArrayType(io.trino.spi.type.ArrayType) HashMap(java.util.HashMap) MapType(io.trino.spi.type.MapType)

Aggregations

MapType (io.trino.spi.type.MapType)85 Type (io.trino.spi.type.Type)45 ArrayType (io.trino.spi.type.ArrayType)42 RowType (io.trino.spi.type.RowType)38 BlockBuilder (io.trino.spi.block.BlockBuilder)28 VarcharType (io.trino.spi.type.VarcharType)26 Block (io.trino.spi.block.Block)17 DecimalType (io.trino.spi.type.DecimalType)17 Map (java.util.Map)17 Test (org.testng.annotations.Test)17 ImmutableList (com.google.common.collect.ImmutableList)16 List (java.util.List)14 TypeSignature.mapType (io.trino.spi.type.TypeSignature.mapType)13 CharType (io.trino.spi.type.CharType)12 ArrayList (java.util.ArrayList)12 HashMap (java.util.HashMap)11 TypeOperators (io.trino.spi.type.TypeOperators)10 ImmutableMap (com.google.common.collect.ImmutableMap)9 VarbinaryType (io.trino.spi.type.VarbinaryType)8 Collectors.toList (java.util.stream.Collectors.toList)8