Examples with MapType org.apache.flink.table.types.logical.MapType used on opensource projects

Example 6 with MapType

use of org.apache.flink.table.types.logical.MapType in project flink by apache.

the class LogicalTypeJsonSerdeTest method testLogicalTypeSerde.

private static List<LogicalType> testLogicalTypeSerde() {
    final List<LogicalType> types = Arrays.asList(new BooleanType(), new TinyIntType(), new SmallIntType(), new IntType(), new BigIntType(), new FloatType(), new DoubleType(), new DecimalType(10), new DecimalType(15, 5), CharType.ofEmptyLiteral(), new CharType(), new CharType(5), VarCharType.ofEmptyLiteral(), new VarCharType(), new VarCharType(5), BinaryType.ofEmptyLiteral(), new BinaryType(), new BinaryType(100), VarBinaryType.ofEmptyLiteral(), new VarBinaryType(), new VarBinaryType(100), new DateType(), new TimeType(), new TimeType(3), new TimestampType(), new TimestampType(3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new TimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType(), new ZonedTimestampType(3), new ZonedTimestampType(false, TimestampKind.ROWTIME, 3), new LocalZonedTimestampType(), new LocalZonedTimestampType(3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3), new DayTimeIntervalType(DayTimeIntervalType.DayTimeResolution.DAY_TO_HOUR), new DayTimeIntervalType(false, DayTimeIntervalType.DayTimeResolution.DAY_TO_HOUR, 3, 6), new YearMonthIntervalType(YearMonthIntervalType.YearMonthResolution.YEAR_TO_MONTH), new YearMonthIntervalType(false, YearMonthIntervalType.YearMonthResolution.MONTH, 2), new ZonedTimestampType(), new LocalZonedTimestampType(), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new SymbolType<>(), new ArrayType(new IntType(false)), new ArrayType(new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3)), new ArrayType(new ZonedTimestampType(false, TimestampKind.ROWTIME, 3)), new ArrayType(new TimestampType()), new ArrayType(CharType.ofEmptyLiteral()), new ArrayType(VarCharType.ofEmptyLiteral()), new ArrayType(BinaryType.ofEmptyLiteral()), new ArrayType(VarBinaryType.ofEmptyLiteral()), new MapType(new BigIntType(), new IntType(false)), new MapType(new TimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType()), new MapType(CharType.ofEmptyLiteral(), CharType.ofEmptyLiteral()), new MapType(VarCharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral()), new MapType(BinaryType.ofEmptyLiteral(), BinaryType.ofEmptyLiteral()), new MapType(VarBinaryType.ofEmptyLiteral(), VarBinaryType.ofEmptyLiteral()), new MultisetType(new IntType(false)), new MultisetType(new TimestampType()), new MultisetType(new TimestampType(true, TimestampKind.ROWTIME, 3)), new MultisetType(CharType.ofEmptyLiteral()), new MultisetType(VarCharType.ofEmptyLiteral()), new MultisetType(BinaryType.ofEmptyLiteral()), new MultisetType(VarBinaryType.ofEmptyLiteral()), RowType.of(new BigIntType(), new IntType(false), new VarCharType(200)), RowType.of(new LogicalType[] { new BigIntType(), new IntType(false), new VarCharType(200) }, new String[] { "f1", "f2", "f3" }), RowType.of(new TimestampType(false, TimestampKind.ROWTIME, 3), new TimestampType(false, TimestampKind.REGULAR, 3), new ZonedTimestampType(false, TimestampKind.ROWTIME, 3), new ZonedTimestampType(false, TimestampKind.REGULAR, 3), new LocalZonedTimestampType(false, TimestampKind.ROWTIME, 3), new LocalZonedTimestampType(false, TimestampKind.PROCTIME, 3), new LocalZonedTimestampType(false, TimestampKind.REGULAR, 3)), RowType.of(CharType.ofEmptyLiteral(), VarCharType.ofEmptyLiteral(), BinaryType.ofEmptyLiteral(), VarBinaryType.ofEmptyLiteral()), // registered structured type
    StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", "structuredType"), PojoClass.class).attributes(Arrays.asList(new StructuredType.StructuredAttribute("f0", new IntType(true)), new StructuredType.StructuredAttribute("f1", new BigIntType(true)), new StructuredType.StructuredAttribute("f2", new VarCharType(200), "desc"))).comparison(StructuredType.StructuredComparison.FULL).setFinal(false).setInstantiable(false).superType(StructuredType.newBuilder(ObjectIdentifier.of("cat", "db", "structuredType2")).attributes(Collections.singletonList(new StructuredType.StructuredAttribute("f0", new BigIntType(false)))).build()).description("description for StructuredType").build(), // unregistered structured type
    StructuredType.newBuilder(PojoClass.class).attributes(Arrays.asList(new StructuredType.StructuredAttribute("f0", new IntType(true)), new StructuredType.StructuredAttribute("f1", new BigIntType(true)), new StructuredType.StructuredAttribute("f2", new VarCharType(200), "desc"))).build(), // registered distinct type
    DistinctType.newBuilder(ObjectIdentifier.of("cat", "db", "distinctType"), new VarCharType(5)).build(), DistinctType.newBuilder(ObjectIdentifier.of("cat", "db", "distinctType"), new VarCharType(false, 5)).build(), // custom RawType
    new RawType<>(LocalDateTime.class, LocalDateTimeSerializer.INSTANCE), // external RawType
    new RawType<>(Row.class, ExternalSerializer.of(DataTypes.ROW(DataTypes.INT(), DataTypes.STRING()))));
    final List<LogicalType> mutableTypes = new ArrayList<>(types);
    // RawType for MapView
    addRawTypesForMapView(mutableTypes, new VarCharType(100), new VarCharType(100));
    addRawTypesForMapView(mutableTypes, new VarCharType(100), new BigIntType());
    addRawTypesForMapView(mutableTypes, new BigIntType(), new VarCharType(100));
    addRawTypesForMapView(mutableTypes, new BigIntType(), new BigIntType());
    // RawType for ListView
    addRawTypesForListView(mutableTypes, new VarCharType(100));
    addRawTypesForListView(mutableTypes, new BigIntType());
    // RawType for custom MapView
    mutableTypes.add(DataViewUtils.adjustDataViews(MapView.newMapViewDataType(DataTypes.STRING().toInternal(), DataTypes.STRING().bridgedTo(byte[].class)), false).getLogicalType());
    final List<LogicalType> allTypes = new ArrayList<>();
    // consider nullable
    for (LogicalType type : mutableTypes) {
        allTypes.add(type.copy(true));
        allTypes.add(type.copy(false));
    }
    // ignore nullable for NullType
    allTypes.add(new NullType());
    return allTypes;
}

Example 7 with MapType

use of org.apache.flink.table.types.logical.MapType in project flink by apache.

the class DataTypePrecisionFixer method visit.

@Override
public DataType visit(KeyValueDataType keyValueDataType) {
    DataType keyType = keyValueDataType.getKeyDataType();
    DataType valueType = keyValueDataType.getValueDataType();
    if (logicalType.getTypeRoot() == LogicalTypeRoot.MAP) {
        MapType mapType = (MapType) logicalType;
        DataType newKeyType = keyType.accept(new DataTypePrecisionFixer(mapType.getKeyType()));
        DataType newValueType = valueType.accept(new DataTypePrecisionFixer(mapType.getValueType()));
        return DataTypes.MAP(newKeyType, newValueType).bridgedTo(keyValueDataType.getConversionClass());
    }
    throw new UnsupportedOperationException("Unsupported logical type : " + logicalType);
}

Example 8 with MapType

use of org.apache.flink.table.types.logical.MapType in project flink by apache.

the class RowDataVectorizer method setColumn.

private static void setColumn(int rowId, MapColumnVector mapColumnVector, LogicalType type, RowData row, int columnId) {
    MapData mapData = row.getMap(columnId);
    MapType mapType = (MapType) type;
    ArrayData keyArray = mapData.keyArray();
    ArrayData valueArray = mapData.valueArray();
    mapColumnVector.lengths[rowId] = mapData.size();
    mapColumnVector.offsets[rowId] = mapColumnVector.childCount;
    mapColumnVector.childCount += mapColumnVector.lengths[rowId];
    mapColumnVector.keys.ensureSize(mapColumnVector.childCount, mapColumnVector.offsets[rowId] != 0);
    mapColumnVector.values.ensureSize(mapColumnVector.childCount, mapColumnVector.offsets[rowId] != 0);
    RowData convertedKeyRowData = convert(keyArray, mapType.getKeyType());
    RowData convertedValueRowData = convert(valueArray, mapType.getValueType());
    for (int i = 0; i < keyArray.size(); i++) {
        setColumn((int) mapColumnVector.offsets[rowId] + i, mapColumnVector.keys, mapType.getKeyType(), convertedKeyRowData, i);
        setColumn((int) mapColumnVector.offsets[rowId] + i, mapColumnVector.values, mapType.getValueType(), convertedValueRowData, i);
    }
}

Example 9 with MapType

use of org.apache.flink.table.types.logical.MapType in project flink by apache.

the class HiveInspectors method getConversion.

/**
 * Get conversion for converting Flink object to Hive object from an ObjectInspector and the
 * corresponding Flink DataType.
 */
public static HiveObjectConversion getConversion(ObjectInspector inspector, LogicalType dataType, HiveShim hiveShim) {
    if (inspector instanceof PrimitiveObjectInspector) {
        HiveObjectConversion conversion;
        if (inspector instanceof BooleanObjectInspector || inspector instanceof StringObjectInspector || inspector instanceof ByteObjectInspector || inspector instanceof ShortObjectInspector || inspector instanceof IntObjectInspector || inspector instanceof LongObjectInspector || inspector instanceof FloatObjectInspector || inspector instanceof DoubleObjectInspector || inspector instanceof BinaryObjectInspector || inspector instanceof VoidObjectInspector) {
            conversion = IdentityConversion.INSTANCE;
        } else if (inspector instanceof DateObjectInspector) {
            conversion = hiveShim::toHiveDate;
        } else if (inspector instanceof TimestampObjectInspector) {
            conversion = hiveShim::toHiveTimestamp;
        } else if (inspector instanceof HiveCharObjectInspector) {
            conversion = o -> o == null ? null : new HiveChar((String) o, ((CharType) dataType).getLength());
        } else if (inspector instanceof HiveVarcharObjectInspector) {
            conversion = o -> o == null ? null : new HiveVarchar((String) o, ((VarCharType) dataType).getLength());
        } else if (inspector instanceof HiveDecimalObjectInspector) {
            conversion = o -> o == null ? null : HiveDecimal.create((BigDecimal) o);
        } else {
            throw new FlinkHiveUDFException("Unsupported primitive object inspector " + inspector.getClass().getName());
        }
        // currently this happens for constant arguments for UDFs
        if (((PrimitiveObjectInspector) inspector).preferWritable()) {
            conversion = new WritableHiveObjectConversion(conversion, hiveShim);
        }
        return conversion;
    }
    if (inspector instanceof ListObjectInspector) {
        HiveObjectConversion eleConvert = getConversion(((ListObjectInspector) inspector).getListElementObjectInspector(), ((ArrayType) dataType).getElementType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Object[] array = (Object[]) o;
            List<Object> result = new ArrayList<>();
            for (Object ele : array) {
                result.add(eleConvert.toHiveObject(ele));
            }
            return result;
        };
    }
    if (inspector instanceof MapObjectInspector) {
        MapObjectInspector mapInspector = (MapObjectInspector) inspector;
        MapType kvType = (MapType) dataType;
        HiveObjectConversion keyConversion = getConversion(mapInspector.getMapKeyObjectInspector(), kvType.getKeyType(), hiveShim);
        HiveObjectConversion valueConversion = getConversion(mapInspector.getMapValueObjectInspector(), kvType.getValueType(), hiveShim);
        return o -> {
            if (o == null) {
                return null;
            }
            Map<Object, Object> map = (Map) o;
            Map<Object, Object> result = new HashMap<>(map.size());
            for (Map.Entry<Object, Object> entry : map.entrySet()) {
                result.put(keyConversion.toHiveObject(entry.getKey()), valueConversion.toHiveObject(entry.getValue()));
            }
            return result;
        };
    }
    if (inspector instanceof StructObjectInspector) {
        StructObjectInspector structInspector = (StructObjectInspector) inspector;
        List<? extends StructField> structFields = structInspector.getAllStructFieldRefs();
        List<RowType.RowField> rowFields = ((RowType) dataType).getFields();
        HiveObjectConversion[] conversions = new HiveObjectConversion[structFields.size()];
        for (int i = 0; i < structFields.size(); i++) {
            conversions[i] = getConversion(structFields.get(i).getFieldObjectInspector(), rowFields.get(i).getType(), hiveShim);
        }
        return o -> {
            if (o == null) {
                return null;
            }
            Row row = (Row) o;
            List<Object> result = new ArrayList<>(row.getArity());
            for (int i = 0; i < row.getArity(); i++) {
                result.add(conversions[i].toHiveObject(row.getField(i)));
            }
            return result;
        };
    }
    throw new FlinkHiveUDFException(String.format("Flink doesn't support convert object conversion for %s yet", inspector));
}

Example 10 with MapType

use of org.apache.flink.table.types.logical.MapType in project flink by apache.

the class OrcBulkRowDataWriterTest method initInput.

@Before
public void initInput() {
    input = new ArrayList<>();
    fieldTypes = new LogicalType[4];
    fieldTypes[0] = new VarCharType();
    fieldTypes[1] = new IntType();
    List<RowType.RowField> arrayRowFieldList = Collections.singletonList(new RowType.RowField("_col2_col0", new VarCharType()));
    fieldTypes[2] = new ArrayType(new RowType(arrayRowFieldList));
    List<RowType.RowField> mapRowFieldList = Arrays.asList(new RowType.RowField("_col3_col0", new VarCharType()), new RowType.RowField("_col3_col1", new TimestampType()));
    fieldTypes[3] = new MapType(new VarCharType(), new RowType(mapRowFieldList));
    {
        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, new BinaryStringData("_col_0_string_1"));
        rowData.setField(1, 1);
        GenericRowData arrayValue1 = new GenericRowData(1);
        arrayValue1.setField(0, new BinaryStringData("_col_2_row_0_string_1"));
        GenericRowData arrayValue2 = new GenericRowData(1);
        arrayValue2.setField(0, new BinaryStringData("_col_2_row_1_string_1"));
        GenericArrayData arrayData = new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
        rowData.setField(2, arrayData);
        GenericRowData mapValue1 = new GenericRowData(2);
        mapValue1.setField(0, new BinaryStringData(("_col_3_map_value_string_1")));
        mapValue1.setField(1, TimestampData.fromTimestamp(new Timestamp(3600000)));
        Map<StringData, RowData> mapDataMap = new HashMap<>();
        mapDataMap.put(new BinaryStringData("_col_3_map_key_1"), mapValue1);
        GenericMapData mapData = new GenericMapData(mapDataMap);
        rowData.setField(3, mapData);
        input.add(rowData);
    }
    {
        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, new BinaryStringData("_col_0_string_2"));
        rowData.setField(1, 2);
        GenericRowData arrayValue1 = new GenericRowData(1);
        arrayValue1.setField(0, new BinaryStringData("_col_2_row_0_string_2"));
        GenericRowData arrayValue2 = new GenericRowData(1);
        arrayValue2.setField(0, new BinaryStringData("_col_2_row_1_string_2"));
        GenericArrayData arrayData = new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
        rowData.setField(2, arrayData);
        GenericRowData mapValue1 = new GenericRowData(2);
        mapValue1.setField(0, new BinaryStringData(("_col_3_map_value_string_2")));
        mapValue1.setField(1, TimestampData.fromTimestamp(new Timestamp(3600000)));
        Map<StringData, RowData> mapDataMap = new HashMap<>();
        mapDataMap.put(new BinaryStringData("_col_3_map_key_2"), mapValue1);
        GenericMapData mapData = new GenericMapData(mapDataMap);
        rowData.setField(3, mapData);
        input.add(rowData);
    }
}