Examples with RowData org.apache.flink.table.data.RowData used on opensource projects

Example 36 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class MaxwellJsonSerDerTest method testSerializationDeserialization.

@Test
public void testSerializationDeserialization() throws Exception {
    List<String> lines = readLines("maxwell-data.txt");
    MaxwellJsonDeserializationSchema deserializationSchema = new MaxwellJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_DATA_TYPE.getLogicalType()), false, TimestampFormat.ISO_8601);
    SimpleCollector collector = new SimpleCollector();
    for (String line : lines) {
        deserializationSchema.deserialize(line.getBytes(StandardCharsets.UTF_8), collector);
    }
    // Maxwell captures change data (`maxwell-data.txt`) on the `product` table:
    // 
    // CREATE TABLE product (
    // id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
    // name VARCHAR(255),
    // description VARCHAR(512),
    // weight FLOAT
    // );
    // ALTER TABLE product AUTO_INCREMENT = 101;
    // 
    // INSERT INTO product
    // VALUES (default,"scooter","Small 2-wheel scooter",3.14),
    // (default,"car battery","12V car battery",8.1),
    // (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40
    // to #3",0.8),
    // (default,"hammer","12oz carpenter's hammer",0.75),
    // (default,"hammer","14oz carpenter's hammer",0.875),
    // (default,"hammer","16oz carpenter's hammer",1.0),
    // (default,"rocks","box of assorted rocks",5.3),
    // (default,"jacket","water resistent black wind breaker",0.1),
    // (default,"spare tire","24 inch spare tire",22.2);
    // UPDATE product SET description='18oz carpenter hammer' WHERE id=106;
    // UPDATE product SET weight='5.1' WHERE id=107;
    // INSERT INTO product VALUES (default,"jacket","water resistent white wind breaker",0.2);
    // INSERT INTO product VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
    // UPDATE product SET description='new water resistent white wind breaker', weight='0.5'
    // WHERE id=110;
    // UPDATE product SET weight='5.17' WHERE id=111;
    // DELETE FROM product WHERE id=111;
    // UPDATE product SET weight='5.17' WHERE id=102 or id = 101;
    // DELETE FROM product WHERE id=102 or id = 103;
    List<String> expected = Arrays.asList("+I(101,scooter,Small 2-wheel scooter,3.14)", "+I(102,car battery,12V car battery,8.1)", "+I(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)", "+I(104,hammer,12oz carpenter's hammer,0.75)", "+I(105,hammer,14oz carpenter's hammer,0.875)", "+I(106,hammer,16oz carpenter's hammer,1.0)", "+I(107,rocks,box of assorted rocks,5.3)", "+I(108,jacket,water resistent black wind breaker,0.1)", "+I(109,spare tire,24 inch spare tire,22.2)", "-U(106,hammer,16oz carpenter's hammer,1.0)", "+U(106,hammer,18oz carpenter hammer,1.0)", "-U(107,rocks,box of assorted rocks,5.3)", "+U(107,rocks,box of assorted rocks,5.1)", "+I(110,jacket,water resistent white wind breaker,0.2)", "+I(111,scooter,Big 2-wheel scooter ,5.18)", "-U(110,jacket,water resistent white wind breaker,0.2)", "+U(110,jacket,new water resistent white wind breaker,0.5)", "-U(111,scooter,Big 2-wheel scooter ,5.18)", "+U(111,scooter,Big 2-wheel scooter ,5.17)", "-D(111,scooter,Big 2-wheel scooter ,5.17)", "-U(101,scooter,Small 2-wheel scooter,3.14)", "+U(101,scooter,Small 2-wheel scooter,5.17)", "-U(102,car battery,12V car battery,8.1)", "+U(102,car battery,12V car battery,5.17)", "-D(102,car battery,12V car battery,5.17)", "-D(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)");
    List<String> actual = collector.list.stream().map(Object::toString).collect(Collectors.toList());
    assertEquals(expected, actual);
    MaxwellJsonSerializationSchema serializationSchema = new MaxwellJsonSerializationSchema((RowType) PHYSICAL_DATA_TYPE.getLogicalType(), TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    serializationSchema.open(null);
    List<String> result = new ArrayList<>();
    for (RowData rowData : collector.list) {
        result.add(new String(serializationSchema.serialize(rowData), StandardCharsets.UTF_8));
    }
    List<String> expectedResult = Arrays.asList("{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"type\":\"insert\"}", "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"type\":\"insert\"}", "{\"data\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"type\":\"insert\"}", "{\"data\":{\"id\":104,\"name\":\"hammer\",\"description\":\"12oz carpenter's hammer\",\"weight\":0.75},\"type\":\"insert\"}", "{\"data\":{\"id\":105,\"name\":\"hammer\",\"description\":\"14oz carpenter's hammer\",\"weight\":0.875},\"type\":\"insert\"}", "{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"type\":\"insert\"}", "{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"type\":\"insert\"}", "{\"data\":{\"id\":108,\"name\":\"jacket\",\"description\":\"water resistent black wind breaker\",\"weight\":0.1},\"type\":\"insert\"}", "{\"data\":{\"id\":109,\"name\":\"spare tire\",\"description\":\"24 inch spare tire\",\"weight\":22.2},\"type\":\"insert\"}", "{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"type\":\"delete\"}", "{\"data\":{\"id\":106,\"name\":\"hammer\",\"description\":\"18oz carpenter hammer\",\"weight\":1.0},\"type\":\"insert\"}", "{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"type\":\"delete\"}", "{\"data\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.1},\"type\":\"insert\"}", "{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"type\":\"insert\"}", "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"type\":\"insert\"}", "{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"type\":\"delete\"}", "{\"data\":{\"id\":110,\"name\":\"jacket\",\"description\":\"new water resistent white wind breaker\",\"weight\":0.5},\"type\":\"insert\"}", "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"type\":\"delete\"}", "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"type\":\"insert\"}", "{\"data\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"type\":\"delete\"}", "{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"type\":\"delete\"}", "{\"data\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":5.17},\"type\":\"insert\"}", "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"type\":\"delete\"}", "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17},\"type\":\"insert\"}", "{\"data\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":5.17},\"type\":\"delete\"}", "{\"data\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"type\":\"delete\"}");
    assertEquals(expectedResult, result);
}

Example 37 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class DebeziumJsonDecodingFormat method createRuntimeDecoder.

@Override
public DeserializationSchema<RowData> createRuntimeDecoder(DynamicTableSource.Context context, DataType physicalDataType, int[][] projections) {
    physicalDataType = Projection.of(projections).project(physicalDataType);
    final List<ReadableMetadata> readableMetadata = metadataKeys.stream().map(k -> Stream.of(ReadableMetadata.values()).filter(rm -> rm.key.equals(k)).findFirst().orElseThrow(IllegalStateException::new)).collect(Collectors.toList());
    final List<DataTypes.Field> metadataFields = readableMetadata.stream().map(m -> DataTypes.FIELD(m.key, m.dataType)).collect(Collectors.toList());
    final DataType producedDataType = DataTypeUtils.appendRowFields(physicalDataType, metadataFields);
    final TypeInformation<RowData> producedTypeInfo = context.createTypeInformation(producedDataType);
    return new DebeziumJsonDeserializationSchema(physicalDataType, readableMetadata, producedTypeInfo, schemaInclude, ignoreParseErrors, timestampFormat);
}

Example 38 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class DebeziumJsonFormatFactory method createEncodingFormat.

@Override
public EncodingFormat<SerializationSchema<RowData>> createEncodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
    FactoryUtil.validateFactoryOptions(this, formatOptions);
    validateEncodingFormatOptions(formatOptions);
    TimestampFormat timestampFormat = JsonFormatOptionsUtil.getTimestampFormat(formatOptions);
    JsonFormatOptions.MapNullKeyMode mapNullKeyMode = JsonFormatOptionsUtil.getMapNullKeyMode(formatOptions);
    String mapNullKeyLiteral = formatOptions.get(JSON_MAP_NULL_KEY_LITERAL);
    final boolean encodeDecimalAsPlainNumber = formatOptions.get(ENCODE_DECIMAL_AS_PLAIN_NUMBER);
    return new EncodingFormat<SerializationSchema<RowData>>() {

        @Override
        public ChangelogMode getChangelogMode() {
            return ChangelogMode.newBuilder().addContainedKind(RowKind.INSERT).addContainedKind(RowKind.UPDATE_BEFORE).addContainedKind(RowKind.UPDATE_AFTER).addContainedKind(RowKind.DELETE).build();
        }

        @Override
        public SerializationSchema<RowData> createRuntimeEncoder(DynamicTableSink.Context context, DataType consumedDataType) {
            final RowType rowType = (RowType) consumedDataType.getLogicalType();
            return new DebeziumJsonSerializationSchema(rowType, timestampFormat, mapNullKeyMode, mapNullKeyLiteral, encodeDecimalAsPlainNumber);
        }
    };
}

Example 39 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class OggJsonSerDeSchemaTest method testSerializationDeserialization.

private void testSerializationDeserialization(String resourceFile) throws Exception {
    List<String> lines = readLines(resourceFile);
    OggJsonDeserializationSchema deserializationSchema = new OggJsonDeserializationSchema(PHYSICAL_DATA_TYPE, Collections.emptyList(), InternalTypeInfo.of(PHYSICAL_DATA_TYPE.getLogicalType()), false, TimestampFormat.ISO_8601);
    SimpleCollector collector = new SimpleCollector();
    for (String line : lines) {
        deserializationSchema.deserialize(line.getBytes(StandardCharsets.UTF_8), collector);
    }
    // Ogg captures change data (`ogg-data.txt`) on the `product`
    // table:
    // 
    // CREATE TABLE product (
    // id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY,
    // name VARCHAR(255),
    // description VARCHAR(512),
    // weight FLOAT
    // );
    // ALTER TABLE product AUTO_INCREMENT = 101;
    // 
    // INSERT INTO product
    // VALUES (default,"scooter","Small 2-wheel scooter",3.14),
    // (default,"car battery","12V car battery",8.1),
    // (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40
    // to #3",0.8),
    // (default,"hammer","12oz carpenter's hammer",0.75),
    // (default,"hammer","14oz carpenter's hammer",0.875),
    // (default,"hammer","16oz carpenter's hammer",1.0),
    // (default,"rocks","box of assorted rocks",5.3),
    // (default,"jacket","water resistent black wind breaker",0.1),
    // (default,"spare tire","24 inch spare tire",22.2);
    // UPDATE product SET description='18oz carpenter hammer' WHERE id=106;
    // UPDATE product SET weight='5.1' WHERE id=107;
    // INSERT INTO product VALUES (default,"jacket","water resistent white wind breaker",0.2);
    // INSERT INTO product VALUES (default,"scooter","Big 2-wheel scooter ",5.18);
    // UPDATE product SET description='new water resistent white wind breaker', weight='0.5'
    // WHERE id=110;
    // UPDATE product SET weight='5.17' WHERE id=111;
    // DELETE FROM product WHERE id=111;
    List<String> expected = Arrays.asList("+I(101,scooter,Small 2-wheel scooter,3.14)", "+I(102,car battery,12V car battery,8.1)", "+I(103,12-pack drill bits,12-pack of drill bits with sizes ranging from #40 to #3,0.8)", "+I(104,hammer,12oz carpenter's hammer,0.75)", "+I(105,hammer,14oz carpenter's hammer,0.875)", "+I(106,hammer,16oz carpenter's hammer,1.0)", "+I(107,rocks,box of assorted rocks,5.3)", "+I(108,jacket,water resistent black wind breaker,0.1)", "+I(109,spare tire,24 inch spare tire,22.2)", "-U(106,hammer,16oz carpenter's hammer,1.0)", "+U(106,hammer,18oz carpenter hammer,1.0)", "-U(107,rocks,box of assorted rocks,5.3)", "+U(107,rocks,box of assorted rocks,5.1)", "+I(110,jacket,water resistent white wind breaker,0.2)", "+I(111,scooter,Big 2-wheel scooter ,5.18)", "-U(110,jacket,water resistent white wind breaker,0.2)", "+U(110,jacket,new water resistent white wind breaker,0.5)", "-U(111,scooter,Big 2-wheel scooter ,5.18)", "+U(111,scooter,Big 2-wheel scooter ,5.17)", "-D(111,scooter,Big 2-wheel scooter ,5.17)");
    List<String> actual = collector.list.stream().map(Object::toString).collect(Collectors.toList());
    assertEquals(expected, actual);
    OggJsonSerializationSchema serializationSchema = new OggJsonSerializationSchema((RowType) PHYSICAL_DATA_TYPE.getLogicalType(), TimestampFormat.SQL, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    serializationSchema.open(null);
    actual = new ArrayList<>();
    for (RowData rowData : collector.list) {
        actual.add(new String(serializationSchema.serialize(rowData), StandardCharsets.UTF_8));
    }
    expected = Arrays.asList("{\"before\":null,\"after\":{\"id\":101,\"name\":\"scooter\",\"description\":\"Small 2-wheel scooter\",\"weight\":3.14},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":102,\"name\":\"car battery\",\"description\":\"12V car battery\",\"weight\":8.1},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":103,\"name\":\"12-pack drill bits\",\"description\":\"12-pack of drill bits with sizes ranging from #40 to #3\",\"weight\":0.8},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":104,\"name\":\"hammer\",\"description\":\"12oz carpenter's hammer\",\"weight\":0.75},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":105,\"name\":\"hammer\",\"description\":\"14oz carpenter's hammer\",\"weight\":0.875},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":108,\"name\":\"jacket\",\"description\":\"water resistent black wind breaker\",\"weight\":0.1},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":109,\"name\":\"spare tire\",\"description\":\"24 inch spare tire\",\"weight\":22.2},\"op_type\":\"I\"}", "{\"before\":{\"id\":106,\"name\":\"hammer\",\"description\":\"16oz carpenter's hammer\",\"weight\":1.0},\"after\":null,\"op_type\":\"D\"}", "{\"before\":null,\"after\":{\"id\":106,\"name\":\"hammer\",\"description\":\"18oz carpenter hammer\",\"weight\":1.0},\"op_type\":\"I\"}", "{\"before\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.3},\"after\":null,\"op_type\":\"D\"}", "{\"before\":null,\"after\":{\"id\":107,\"name\":\"rocks\",\"description\":\"box of assorted rocks\",\"weight\":5.1},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"op_type\":\"I\"}", "{\"before\":null,\"after\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"op_type\":\"I\"}", "{\"before\":{\"id\":110,\"name\":\"jacket\",\"description\":\"water resistent white wind breaker\",\"weight\":0.2},\"after\":null,\"op_type\":\"D\"}", "{\"before\":null,\"after\":{\"id\":110,\"name\":\"jacket\",\"description\":\"new water resistent white wind breaker\",\"weight\":0.5},\"op_type\":\"I\"}", "{\"before\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.18},\"after\":null,\"op_type\":\"D\"}", "{\"before\":null,\"after\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"op_type\":\"I\"}", "{\"before\":{\"id\":111,\"name\":\"scooter\",\"description\":\"Big 2-wheel scooter \",\"weight\":5.17},\"after\":null,\"op_type\":\"D\"}");
    assertEquals(expected, actual);
}

Example 40 with RowData

use of org.apache.flink.table.data.RowData in project flink by apache.

the class JsonRowDataSerDeSchemaTest method testSerializationDecimalEncode.

@Test
public void testSerializationDecimalEncode() throws Exception {
    RowType schema = (RowType) ROW(FIELD("decimal1", DECIMAL(9, 6)), FIELD("decimal2", DECIMAL(20, 0)), FIELD("decimal3", DECIMAL(11, 9))).getLogicalType();
    TypeInformation<RowData> resultTypeInfo = InternalTypeInfo.of(schema);
    JsonRowDataDeserializationSchema deserializer = new JsonRowDataDeserializationSchema(schema, resultTypeInfo, false, false, TimestampFormat.ISO_8601);
    JsonRowDataSerializationSchema plainDecimalSerializer = new JsonRowDataSerializationSchema(schema, TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", true);
    JsonRowDataSerializationSchema scientificDecimalSerializer = new JsonRowDataSerializationSchema(schema, TimestampFormat.ISO_8601, JsonFormatOptions.MapNullKeyMode.LITERAL, "null", false);
    String plainDecimalJson = "{\"decimal1\":123.456789,\"decimal2\":454621864049246170,\"decimal3\":0.000000027}";
    RowData rowData = deserializer.deserialize(plainDecimalJson.getBytes());
    String plainDecimalResult = new String(plainDecimalSerializer.serialize(rowData));
    assertEquals(plainDecimalJson, plainDecimalResult);
    String scientificDecimalJson = "{\"decimal1\":123.456789,\"decimal2\":4.5462186404924617E+17,\"decimal3\":2.7E-8}";
    String scientificDecimalResult = new String(scientificDecimalSerializer.serialize(rowData));
    assertEquals(scientificDecimalJson, scientificDecimalResult);
}