Examples with MapType com.facebook.presto.common.type.MapType used on opensource projects

Example 61 with MapType

use of com.facebook.presto.common.type.MapType in project presto by prestodb.

the class TestOrcReaderMemoryUsage method testMapTypeWithNulls.

@Test
public void testMapTypeWithNulls() throws Exception {
    Type keyType = BIGINT;
    Type valueType = BIGINT;
    MethodHandle keyNativeEquals = getOperatorMethodHandle(OperatorType.EQUAL, keyType, keyType);
    MethodHandle keyBlockEquals = compose(keyNativeEquals, nativeValueGetter(keyType), nativeValueGetter(keyType));
    MethodHandle keyNativeHashCode = getOperatorMethodHandle(OperatorType.HASH_CODE, keyType);
    MethodHandle keyBlockHashCode = compose(keyNativeHashCode, nativeValueGetter(keyType));
    MapType mapType = new MapType(keyType, valueType, keyBlockEquals, keyBlockHashCode);
    int rows = 10000;
    OrcBatchRecordReader reader = null;
    try (TempFile tempFile = createSingleColumnMapFileWithNullValues(mapType, rows)) {
        reader = createCustomOrcRecordReader(tempFile, ORC, OrcPredicate.TRUE, mapType, INITIAL_BATCH_SIZE, false, false);
        assertInitialRetainedSizes(reader, rows);
        long stripeReaderRetainedSize = reader.getCurrentStripeRetainedSizeInBytes();
        long streamReaderRetainedSize = reader.getStreamReaderRetainedSizeInBytes();
        long readerRetainedSize = reader.getRetainedSizeInBytes();
        long readerSystemMemoryUsage = reader.getSystemMemoryUsage();
        while (true) {
            int batchSize = reader.nextBatch();
            if (batchSize == -1) {
                break;
            }
            Block block = reader.readBlock(0);
            assertEquals(block.getPositionCount(), batchSize);
            // increasing during the test, which will cause the StreamReader buffer sizes to increase too.
            if (batchSize < MAX_BATCH_SIZE) {
                continue;
            }
            // StripeReader memory should increase after reading a block.
            assertGreaterThan(reader.getCurrentStripeRetainedSizeInBytes(), stripeReaderRetainedSize);
            // There are no local buffers needed.
            assertEquals(reader.getStreamReaderRetainedSizeInBytes() - streamReaderRetainedSize, 0L);
            // The total retained size and system memory usage should be strictly larger than 0L because of the instance sizes.
            assertGreaterThan(reader.getRetainedSizeInBytes() - readerRetainedSize, 0L);
            assertGreaterThan(reader.getSystemMemoryUsage() - readerSystemMemoryUsage, 0L);
        }
    } finally {
        if (reader != null) {
            reader.close();
        }
    }
    assertClosedRetainedSizes(reader);
}

Example 62 with MapType

use of com.facebook.presto.common.type.MapType in project presto by prestodb.

the class TypeConverter method toPrestoType.

public static Type toPrestoType(org.apache.iceberg.types.Type type, TypeManager typeManager) {
    switch(type.typeId()) {
        case BOOLEAN:
            return BooleanType.BOOLEAN;
        case BINARY:
        case FIXED:
            return VarbinaryType.VARBINARY;
        case DATE:
            return DateType.DATE;
        case DECIMAL:
            Types.DecimalType decimalType = (Types.DecimalType) type;
            return DecimalType.createDecimalType(decimalType.precision(), decimalType.scale());
        case DOUBLE:
            return DoubleType.DOUBLE;
        case LONG:
            return BigintType.BIGINT;
        case FLOAT:
            return RealType.REAL;
        case INTEGER:
            return IntegerType.INTEGER;
        case TIME:
            return TimeType.TIME;
        case TIMESTAMP:
            return TimestampType.TIMESTAMP;
        case STRING:
            return VarcharType.createUnboundedVarcharType();
        case LIST:
            Types.ListType listType = (Types.ListType) type;
            return new ArrayType(toPrestoType(listType.elementType(), typeManager));
        case MAP:
            Types.MapType mapType = (Types.MapType) type;
            TypeSignature keyType = toPrestoType(mapType.keyType(), typeManager).getTypeSignature();
            TypeSignature valueType = toPrestoType(mapType.valueType(), typeManager).getTypeSignature();
            return typeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(keyType), TypeSignatureParameter.of(valueType)));
        case STRUCT:
            List<Types.NestedField> fields = ((Types.StructType) type).fields();
            return RowType.from(fields.stream().map(field -> new RowType.Field(Optional.of(field.name()), toPrestoType(field.type(), typeManager))).collect(toImmutableList()));
        default:
            throw new UnsupportedOperationException(format("Cannot convert from Iceberg type '%s' (%s) to Presto type", type, type.typeId()));
    }
}

Example 63 with MapType

use of com.facebook.presto.common.type.MapType in project presto by prestodb.

the class SessionPropertyManager method getJsonCodecForType.

private static <T> JsonCodec<T> getJsonCodecForType(Type type) {
    if (VarcharType.VARCHAR.equals(type)) {
        return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(String.class);
    }
    if (BooleanType.BOOLEAN.equals(type)) {
        return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Boolean.class);
    }
    if (BigintType.BIGINT.equals(type)) {
        return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Long.class);
    }
    if (IntegerType.INTEGER.equals(type)) {
        return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Integer.class);
    }
    if (DoubleType.DOUBLE.equals(type)) {
        return (JsonCodec<T>) JSON_CODEC_FACTORY.jsonCodec(Double.class);
    }
    if (type instanceof ArrayType) {
        Type elementType = ((ArrayType) type).getElementType();
        return (JsonCodec<T>) JSON_CODEC_FACTORY.listJsonCodec(getJsonCodecForType(elementType));
    }
    if (type instanceof MapType) {
        Type keyType = ((MapType) type).getKeyType();
        Type valueType = ((MapType) type).getValueType();
        return (JsonCodec<T>) JSON_CODEC_FACTORY.mapJsonCodec(getMapKeyType(keyType), getJsonCodecForType(valueType));
    }
    throw new PrestoException(INVALID_SESSION_PROPERTY, format("Session property type %s is not supported", type));
}

Example 64 with MapType

use of com.facebook.presto.common.type.MapType in project presto by prestodb.

the class MapFilterFunction method mapFilter.

@CodegenScalarFunction(value = "map_filter", deterministic = false)
@Description("return map containing entries that match the given predicate")
@TypeParameter("K")
@TypeParameter("V")
@SqlType("map(K,V)")
public static MethodHandle mapFilter(@SqlType("map(K,V)") Type inputType, @SqlType("function(K,V,boolean)") Type filter) {
    MapType mapType = (MapType) inputType;
    CallSiteBinder binder = new CallSiteBinder();
    Type keyType = mapType.getKeyType();
    Type valueType = mapType.getValueType();
    Class<?> keyJavaType = Primitives.wrap(keyType.getJavaType());
    Class<?> valueJavaType = Primitives.wrap(valueType.getJavaType());
    ClassDefinition definition = new ClassDefinition(a(PUBLIC, FINAL), makeClassName("MapFilter"), type(Object.class));
    definition.declareDefaultConstructor(a(PRIVATE));
    Parameter block = arg("block", Block.class);
    Parameter function = arg("function", BinaryFunctionInterface.class);
    MethodDefinition method = definition.declareMethod(a(PUBLIC, STATIC), "filter", type(Block.class), ImmutableList.of(block, function));
    BytecodeBlock body = method.getBody();
    Scope scope = method.getScope();
    Variable positionCount = scope.declareVariable(int.class, "positionCount");
    Variable position = scope.declareVariable(int.class, "position");
    Variable mapBlockBuilder = scope.declareVariable(BlockBuilder.class, "mapBlockBuilder");
    Variable singleMapBlockWriter = scope.declareVariable(BlockBuilder.class, "singleMapBlockWriter");
    Variable keyElement = scope.declareVariable(keyJavaType, "keyElement");
    Variable valueElement = scope.declareVariable(valueJavaType, "valueElement");
    Variable keep = scope.declareVariable(Boolean.class, "keep");
    // invoke block.getPositionCount()
    body.append(positionCount.set(block.invoke("getPositionCount", int.class)));
    // prepare the single map block builder
    body.append(mapBlockBuilder.set(constantType(binder, mapType).invoke("createBlockBuilder", BlockBuilder.class, constantNull(BlockBuilderStatus.class), constantInt(0))));
    body.append(singleMapBlockWriter.set(mapBlockBuilder.invoke("beginBlockEntry", BlockBuilder.class)));
    SqlTypeBytecodeExpression keySqlType = constantType(binder, keyType);
    BytecodeNode loadKeyElement;
    if (!keyType.equals(UNKNOWN)) {
        // key element must be non-null
        loadKeyElement = new BytecodeBlock().append(keyElement.set(keySqlType.getValue(block, position).cast(keyJavaType)));
    } else {
        loadKeyElement = new BytecodeBlock().append(keyElement.set(constantNull(keyJavaType)));
    }
    SqlTypeBytecodeExpression valueSqlType = constantType(binder, valueType);
    BytecodeNode loadValueElement;
    if (!valueType.equals(UNKNOWN)) {
        loadValueElement = new IfStatement().condition(block.invoke("isNull", boolean.class, add(position, constantInt(1)))).ifTrue(valueElement.set(constantNull(valueJavaType))).ifFalse(valueElement.set(valueSqlType.getValue(block, add(position, constantInt(1))).cast(valueJavaType)));
    } else {
        loadValueElement = new BytecodeBlock().append(valueElement.set(constantNull(valueJavaType)));
    }
    body.append(new ForLoop().initialize(position.set(constantInt(0))).condition(lessThan(position, positionCount)).update(incrementVariable(position, (byte) 2)).body(new BytecodeBlock().append(loadKeyElement).append(loadValueElement).append(keep.set(function.invoke("apply", Object.class, keyElement.cast(Object.class), valueElement.cast(Object.class)).cast(Boolean.class))).append(new IfStatement("if (keep != null && keep) ...").condition(and(notEqual(keep, constantNull(Boolean.class)), keep.cast(boolean.class))).ifTrue(new BytecodeBlock().append(keySqlType.invoke("appendTo", void.class, block, position, singleMapBlockWriter)).append(valueSqlType.invoke("appendTo", void.class, block, add(position, constantInt(1)), singleMapBlockWriter))))));
    body.append(mapBlockBuilder.invoke("closeEntry", BlockBuilder.class).pop());
    body.append(constantType(binder, mapType).invoke("getObject", Object.class, mapBlockBuilder.cast(Block.class), subtract(mapBlockBuilder.invoke("getPositionCount", int.class), constantInt(1))).ret());
    Class<?> generatedClass = defineClass(definition, Object.class, binder.getBindings(), MapFilterFunction.class.getClassLoader());
    return methodHandle(generatedClass, "filter", Block.class, BinaryFunctionInterface.class);
}

Example 65 with MapType

use of com.facebook.presto.common.type.MapType in project presto by prestodb.

the class MapTransformKeyFunction method specialize.

@Override
public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariables, int arity, FunctionAndTypeManager functionAndTypeManager) {
    Type keyType = boundVariables.getTypeVariable("K1");
    Type transformedKeyType = boundVariables.getTypeVariable("K2");
    Type valueType = boundVariables.getTypeVariable("V");
    MapType resultMapType = (MapType) functionAndTypeManager.getParameterizedType(StandardTypes.MAP, ImmutableList.of(TypeSignatureParameter.of(transformedKeyType.getTypeSignature()), TypeSignatureParameter.of(valueType.getTypeSignature())));
    return new BuiltInScalarFunctionImplementation(false, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL), functionTypeArgumentProperty(BinaryFunctionInterface.class)), generateTransformKey(keyType, transformedKeyType, valueType, resultMapType), Optional.of(STATE_FACTORY.bindTo(resultMapType)));
}