Examples with RowData org.apache.flink.table.data.RowData used on opensource projects

Example 21 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class AvroRowDataDeSerializationSchemaTest method testSerializeDeserialize.

@Test
public void testSerializeDeserialize() throws Exception {
    final DataType dataType = ROW(FIELD("bool", BOOLEAN()), FIELD("tinyint", TINYINT()), FIELD("smallint", SMALLINT()), FIELD("int", INT()), FIELD("bigint", BIGINT()), FIELD("float", FLOAT()), FIELD("double", DOUBLE()), FIELD("name", STRING()), FIELD("bytes", BYTES()), FIELD("decimal", DECIMAL(19, 6)), FIELD("doubles", ARRAY(DOUBLE())), FIELD("time", TIME(0)), FIELD("date", DATE()), FIELD("timestamp3", TIMESTAMP(3)), FIELD("timestamp3_2", TIMESTAMP(3)), FIELD("map", MAP(STRING(), BIGINT())), FIELD("map2map", MAP(STRING(), MAP(STRING(), INT()))), FIELD("map2array", MAP(STRING(), ARRAY(INT()))), FIELD("nullEntryMap", MAP(STRING(), STRING()))).notNull();
    final RowType rowType = (RowType) dataType.getLogicalType();
    final Schema schema = AvroSchemaConverter.convertToSchema(rowType);
    final GenericRecord record = new GenericData.Record(schema);
    record.put(0, true);
    record.put(1, (int) Byte.MAX_VALUE);
    record.put(2, (int) Short.MAX_VALUE);
    record.put(3, 33);
    record.put(4, 44L);
    record.put(5, 12.34F);
    record.put(6, 23.45);
    record.put(7, "hello avro");
    record.put(8, ByteBuffer.wrap(new byte[] { 1, 2, 4, 5, 6, 7, 8, 12 }));
    record.put(9, ByteBuffer.wrap(BigDecimal.valueOf(123456789, 6).unscaledValue().toByteArray()));
    List<Double> doubles = new ArrayList<>();
    doubles.add(1.2);
    doubles.add(3.4);
    doubles.add(567.8901);
    record.put(10, doubles);
    record.put(11, 18397);
    record.put(12, 10087);
    record.put(13, 1589530213123L);
    record.put(14, 1589530213122L);
    Map<String, Long> map = new HashMap<>();
    map.put("flink", 12L);
    map.put("avro", 23L);
    record.put(15, map);
    Map<String, Map<String, Integer>> map2map = new HashMap<>();
    Map<String, Integer> innerMap = new HashMap<>();
    innerMap.put("inner_key1", 123);
    innerMap.put("inner_key2", 234);
    map2map.put("outer_key", innerMap);
    record.put(16, map2map);
    List<Integer> list1 = Arrays.asList(1, 2, 3, 4, 5, 6);
    List<Integer> list2 = Arrays.asList(11, 22, 33, 44, 55);
    Map<String, List<Integer>> map2list = new HashMap<>();
    map2list.put("list1", list1);
    map2list.put("list2", list2);
    record.put(17, map2list);
    Map<String, String> map2 = new HashMap<>();
    map2.put("key1", null);
    record.put(18, map2);
    AvroRowDataSerializationSchema serializationSchema = createSerializationSchema(dataType);
    AvroRowDataDeserializationSchema deserializationSchema = createDeserializationSchema(dataType);
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    GenericDatumWriter<IndexedRecord> datumWriter = new GenericDatumWriter<>(schema);
    Encoder encoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);
    datumWriter.write(record, encoder);
    encoder.flush();
    byte[] input = byteArrayOutputStream.toByteArray();
    RowData rowData = deserializationSchema.deserialize(input);
    byte[] output = serializationSchema.serialize(rowData);
    assertArrayEquals(input, output);
}

Example 22 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class CsvFileFormatFactory method createEncodingFormat.

@Override
public EncodingFormat<Factory<RowData>> createEncodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
    return new EncodingFormat<BulkWriter.Factory<RowData>>() {

        @Override
        public BulkWriter.Factory<RowData> createRuntimeEncoder(DynamicTableSink.Context context, DataType physicalDataType) {
            final RowType rowType = (RowType) physicalDataType.getLogicalType();
            final CsvSchema schema = buildCsvSchema(rowType, formatOptions);
            final RowDataToCsvConverter converter = RowDataToCsvConverters.createRowConverter(rowType);
            final CsvMapper mapper = new CsvMapper();
            final ObjectNode container = mapper.createObjectNode();
            final RowDataToCsvConverter.RowDataToCsvFormatConverterContext converterContext = new RowDataToCsvConverter.RowDataToCsvFormatConverterContext(mapper, container);
            return out -> CsvBulkWriter.forSchema(mapper, schema, converter, converterContext, out);
        }

        @Override
        public ChangelogMode getChangelogMode() {
            return ChangelogMode.insertOnly();
        }
    };
}

Example 23 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class CsvFormatFactory method createEncodingFormat.

@Override
public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
    FactoryUtil.validateFactoryOptions(this, formatOptions);
    CsvCommons.validateFormatOptions(formatOptions);
    return new EncodingFormat<SerializationSchema<RowData>>() {

        @Override
        public SerializationSchema<RowData> createRuntimeEncoder(DynamicTableSink.Context context, DataType consumedDataType) {
            final RowType rowType = (RowType) consumedDataType.getLogicalType();
            final CsvRowDataSerializationSchema.Builder schemaBuilder = new CsvRowDataSerializationSchema.Builder(rowType);
            configureSerializationSchema(formatOptions, schemaBuilder);
            return schemaBuilder.build();
        }

        @Override
        public ChangelogMode getChangelogMode() {
            return ChangelogMode.insertOnly();
        }
    };
}

Example 24 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class CsvFormatFactoryTest method testEscapedFieldDelimiter.

@Test
public void testEscapedFieldDelimiter() throws IOException {
    final CsvRowDataSerializationSchema expectedSer = new CsvRowDataSerializationSchema.Builder(PHYSICAL_TYPE).setFieldDelimiter('\t').setQuoteCharacter('\'').setArrayElementDelimiter("|").setEscapeCharacter('\\').setNullLiteral("n/a").build();
    final CsvRowDataDeserializationSchema expectedDeser = new CsvRowDataDeserializationSchema.Builder(PHYSICAL_TYPE, InternalTypeInfo.of(PHYSICAL_TYPE)).setFieldDelimiter('\t').setQuoteCharacter('\'').setAllowComments(true).setIgnoreParseErrors(true).setArrayElementDelimiter("|").setEscapeCharacter('\\').setNullLiteral("n/a").build();
    // test schema
    final Map<String, String> options1 = getModifiedOptions(opts -> opts.put("csv.field-delimiter", "\t"));
    SerializationSchema<RowData> serializationSchema1 = createSerializationSchema(options1);
    DeserializationSchema<RowData> deserializationSchema1 = createDeserializationSchema(options1);
    assertEquals(expectedSer, serializationSchema1);
    assertEquals(expectedDeser, deserializationSchema1);
    final Map<String, String> options2 = getModifiedOptions(opts -> opts.put("csv.field-delimiter", "\\t"));
    SerializationSchema<RowData> serializationSchema2 = createSerializationSchema(options2);
    DeserializationSchema<RowData> deserializationSchema2 = createDeserializationSchema(options2);
    assertEquals(expectedSer, serializationSchema2);
    assertEquals(expectedDeser, deserializationSchema2);
    // test (de)serialization
    RowData rowData = GenericRowData.of(fromString("abc"), 123, false);
    byte[] bytes = serializationSchema2.serialize(rowData);
    assertEquals("abc\t123\tfalse", new String(bytes));
    RowData actual = deserializationSchema2.deserialize("abc\t123\tfalse".getBytes());
    assertEquals(rowData, actual);
}

Example 25 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class CsvRowDataSerDeSchemaTest method testDeserialization.

@SuppressWarnings("unchecked")
private Row testDeserialization(boolean allowParsingErrors, boolean allowComments, String string) throws Exception {
    DataType dataType = ROW(FIELD("f0", STRING()), FIELD("f1", INT()), FIELD("f2", STRING()));
    RowType rowType = (RowType) dataType.getLogicalType();
    CsvRowDataDeserializationSchema.Builder deserSchemaBuilder = new CsvRowDataDeserializationSchema.Builder(rowType, InternalTypeInfo.of(rowType)).setIgnoreParseErrors(allowParsingErrors).setAllowComments(allowComments);
    RowData deserializedRow = deserialize(deserSchemaBuilder, string);
    return (Row) DataFormatConverters.getConverterForDataType(dataType).toExternal(deserializedRow);
}