Examples with ArrayType org.apache.flink.table.types.logical.ArrayType used on opensource projects

Example 16 with ArrayType

use of org.apache.flink.table.types.logical.ArrayType in project flink by apache.

the class JsonToRowDataConverters method createArrayConverter.

private JsonToRowDataConverter createArrayConverter(ArrayType arrayType) {
    JsonToRowDataConverter elementConverter = createConverter(arrayType.getElementType());
    final Class<?> elementClass = LogicalTypeUtils.toInternalConversionClass(arrayType.getElementType());
    return jsonNode -> {
        final ArrayNode node = (ArrayNode) jsonNode;
        final Object[] array = (Object[]) Array.newInstance(elementClass, node.size());
        for (int i = 0; i < node.size(); i++) {
            final JsonNode innerNode = node.get(i);
            array[i] = elementConverter.convert(innerNode);
        }
        return new GenericArrayData(array);
    };
}

Example 17 with ArrayType

use of org.apache.flink.table.types.logical.ArrayType in project flink by apache.

the class OrcBulkRowDataWriterTest method initInput.

@Before
public void initInput() {
    input = new ArrayList<>();
    fieldTypes = new LogicalType[4];
    fieldTypes[0] = new VarCharType();
    fieldTypes[1] = new IntType();
    List<RowType.RowField> arrayRowFieldList = Collections.singletonList(new RowType.RowField("_col2_col0", new VarCharType()));
    fieldTypes[2] = new ArrayType(new RowType(arrayRowFieldList));
    List<RowType.RowField> mapRowFieldList = Arrays.asList(new RowType.RowField("_col3_col0", new VarCharType()), new RowType.RowField("_col3_col1", new TimestampType()));
    fieldTypes[3] = new MapType(new VarCharType(), new RowType(mapRowFieldList));
    {
        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, new BinaryStringData("_col_0_string_1"));
        rowData.setField(1, 1);
        GenericRowData arrayValue1 = new GenericRowData(1);
        arrayValue1.setField(0, new BinaryStringData("_col_2_row_0_string_1"));
        GenericRowData arrayValue2 = new GenericRowData(1);
        arrayValue2.setField(0, new BinaryStringData("_col_2_row_1_string_1"));
        GenericArrayData arrayData = new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
        rowData.setField(2, arrayData);
        GenericRowData mapValue1 = new GenericRowData(2);
        mapValue1.setField(0, new BinaryStringData(("_col_3_map_value_string_1")));
        mapValue1.setField(1, TimestampData.fromTimestamp(new Timestamp(3600000)));
        Map<StringData, RowData> mapDataMap = new HashMap<>();
        mapDataMap.put(new BinaryStringData("_col_3_map_key_1"), mapValue1);
        GenericMapData mapData = new GenericMapData(mapDataMap);
        rowData.setField(3, mapData);
        input.add(rowData);
    }
    {
        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, new BinaryStringData("_col_0_string_2"));
        rowData.setField(1, 2);
        GenericRowData arrayValue1 = new GenericRowData(1);
        arrayValue1.setField(0, new BinaryStringData("_col_2_row_0_string_2"));
        GenericRowData arrayValue2 = new GenericRowData(1);
        arrayValue2.setField(0, new BinaryStringData("_col_2_row_1_string_2"));
        GenericArrayData arrayData = new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
        rowData.setField(2, arrayData);
        GenericRowData mapValue1 = new GenericRowData(2);
        mapValue1.setField(0, new BinaryStringData(("_col_3_map_value_string_2")));
        mapValue1.setField(1, TimestampData.fromTimestamp(new Timestamp(3600000)));
        Map<StringData, RowData> mapDataMap = new HashMap<>();
        mapDataMap.put(new BinaryStringData("_col_3_map_key_2"), mapValue1);
        GenericMapData mapData = new GenericMapData(mapDataMap);
        rowData.setField(3, mapData);
        input.add(rowData);
    }
}

Example 18 with ArrayType

use of org.apache.flink.table.types.logical.ArrayType in project flink by apache.

the class ParquetSchemaConverter method convertToParquetType.

private static Type convertToParquetType(String name, LogicalType type, Type.Repetition repetition) {
    switch(type.getTypeRoot()) {
        case CHAR:
        case VARCHAR:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition).as(OriginalType.UTF8).named(name);
        case BOOLEAN:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.BOOLEAN, repetition).named(name);
        case BINARY:
        case VARBINARY:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.BINARY, repetition).named(name);
        case DECIMAL:
            int precision = ((DecimalType) type).getPrecision();
            int scale = ((DecimalType) type).getScale();
            int numBytes = computeMinBytesForDecimalPrecision(precision);
            return Types.primitive(PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY, repetition).precision(precision).scale(scale).length(numBytes).as(OriginalType.DECIMAL).named(name);
        case TINYINT:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition).as(OriginalType.INT_8).named(name);
        case SMALLINT:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition).as(OriginalType.INT_16).named(name);
        case INTEGER:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition).named(name);
        case BIGINT:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT64, repetition).named(name);
        case FLOAT:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.FLOAT, repetition).named(name);
        case DOUBLE:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.DOUBLE, repetition).named(name);
        case DATE:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition).as(OriginalType.DATE).named(name);
        case TIME_WITHOUT_TIME_ZONE:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT32, repetition).as(OriginalType.TIME_MILLIS).named(name);
        case TIMESTAMP_WITHOUT_TIME_ZONE:
        case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
            return Types.primitive(PrimitiveType.PrimitiveTypeName.INT96, repetition).named(name);
        case ARRAY:
            ArrayType arrayType = (ArrayType) type;
            return ConversionPatterns.listOfElements(repetition, name, convertToParquetType(LIST_ELEMENT_NAME, arrayType.getElementType()));
        case MAP:
            MapType mapType = (MapType) type;
            return ConversionPatterns.mapType(repetition, name, MAP_REPEATED_NAME, convertToParquetType("key", mapType.getKeyType()), convertToParquetType("value", mapType.getValueType()));
        case ROW:
            RowType rowType = (RowType) type;
            return new GroupType(repetition, name, convertToParquetTypes(rowType));
        default:
            throw new UnsupportedOperationException("Unsupported type: " + type);
    }
}

Example 19 with ArrayType

use of org.apache.flink.table.types.logical.ArrayType in project flink by apache.

the class OrcFileSystemITCase method initNestedTypesFile.

private String initNestedTypesFile(List<RowData> data) throws Exception {
    LogicalType[] fieldTypes = new LogicalType[4];
    fieldTypes[0] = new VarCharType();
    fieldTypes[1] = new IntType();
    List<RowType.RowField> arrayRowFieldList = Collections.singletonList(new RowType.RowField("_col2_col0", new VarCharType()));
    fieldTypes[2] = new ArrayType(new RowType(arrayRowFieldList));
    List<RowType.RowField> mapRowFieldList = Arrays.asList(new RowType.RowField("_col3_col0", new VarCharType()), new RowType.RowField("_col3_col1", new TimestampType()));
    fieldTypes[3] = new MapType(new VarCharType(), new RowType(mapRowFieldList));
    String schema = "struct<_col0:string,_col1:int,_col2:array<struct<_col2_col0:string>>," + "_col3:map<string,struct<_col3_col0:string,_col3_col1:timestamp>>>";
    File outDir = TEMPORARY_FOLDER.newFolder();
    Properties writerProps = new Properties();
    writerProps.setProperty("orc.compress", "LZ4");
    final OrcBulkWriterFactory<RowData> writer = new OrcBulkWriterFactory<>(new RowDataVectorizer(schema, fieldTypes), writerProps, new Configuration());
    StreamingFileSink<RowData> sink = StreamingFileSink.forBulkFormat(new org.apache.flink.core.fs.Path(outDir.toURI()), writer).withBucketCheckInterval(10000).build();
    try (OneInputStreamOperatorTestHarness<RowData, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 1, 1, 0)) {
        testHarness.setup();
        testHarness.open();
        int time = 0;
        for (final RowData record : data) {
            testHarness.processElement(record, ++time);
        }
        testHarness.snapshot(1, ++time);
        testHarness.notifyOfCompletedCheckpoint(1);
    }
    return outDir.getAbsolutePath();
}

Example 20 with ArrayType

use of org.apache.flink.table.types.logical.ArrayType in project flink by apache.

the class ArrowReaderWriterTest method init.

@BeforeClass
public static void init() {
    fieldTypes.add(new TinyIntType());
    fieldTypes.add(new SmallIntType());
    fieldTypes.add(new IntType());
    fieldTypes.add(new BigIntType());
    fieldTypes.add(new BooleanType());
    fieldTypes.add(new FloatType());
    fieldTypes.add(new DoubleType());
    fieldTypes.add(new VarCharType());
    fieldTypes.add(new VarBinaryType());
    fieldTypes.add(new DecimalType(10, 3));
    fieldTypes.add(new DateType());
    fieldTypes.add(new TimeType(0));
    fieldTypes.add(new TimeType(2));
    fieldTypes.add(new TimeType(4));
    fieldTypes.add(new TimeType(8));
    fieldTypes.add(new LocalZonedTimestampType(0));
    fieldTypes.add(new LocalZonedTimestampType(2));
    fieldTypes.add(new LocalZonedTimestampType(4));
    fieldTypes.add(new LocalZonedTimestampType(8));
    fieldTypes.add(new TimestampType(0));
    fieldTypes.add(new TimestampType(2));
    fieldTypes.add(new TimestampType(4));
    fieldTypes.add(new TimestampType(8));
    fieldTypes.add(new ArrayType(new VarCharType()));
    rowFieldType = new RowType(Arrays.asList(new RowType.RowField("a", new IntType()), new RowType.RowField("b", new VarCharType()), new RowType.RowField("c", new ArrayType(new VarCharType())), new RowType.RowField("d", new TimestampType(2)), new RowType.RowField("e", new RowType(Arrays.asList(new RowType.RowField("e1", new IntType()), new RowType.RowField("e2", new VarCharType()))))));
    fieldTypes.add(rowFieldType);
    List<RowType.RowField> rowFields = new ArrayList<>();
    for (int i = 0; i < fieldTypes.size(); i++) {
        rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
    }
    rowType = new RowType(rowFields);
    allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}