Examples with AvroRowDataDeserializationSchema org.apache.flink.formats.avro.AvroRowDataDeserializationSchema used on opensource projects

Example 1 with AvroRowDataDeserializationSchema

use of org.apache.flink.formats.avro.AvroRowDataDeserializationSchema in project flink by apache.

the class RegistryAvroRowDataSeDeSchemaTest method testRowDataWriteReadWithSchema.

private void testRowDataWriteReadWithSchema(Schema schema) throws Exception {
    DataType dataType = AvroSchemaConverter.convertToDataType(schema.toString());
    RowType rowType = (RowType) dataType.getLogicalType();
    AvroRowDataSerializationSchema serializer = getSerializationSchema(rowType, schema);
    Schema writeSchema = AvroSchemaConverter.convertToSchema(dataType.getLogicalType());
    AvroRowDataDeserializationSchema deserializer = getDeserializationSchema(rowType, writeSchema);
    serializer.open(null);
    deserializer.open(null);
    assertNull(deserializer.deserialize(null));
    RowData oriData = address2RowData(address);
    byte[] serialized = serializer.serialize(oriData);
    RowData rowData = deserializer.deserialize(serialized);
    assertThat(rowData.getArity(), equalTo(schema.getFields().size()));
    assertEquals(address.getNum(), rowData.getInt(0));
    assertEquals(address.getStreet(), rowData.getString(1).toString());
    if (schema != ADDRESS_SCHEMA_COMPATIBLE) {
        assertEquals(address.getCity(), rowData.getString(2).toString());
        assertEquals(address.getState(), rowData.getString(3).toString());
        assertEquals(address.getZip(), rowData.getString(4).toString());
    }
}

Example 2 with AvroRowDataDeserializationSchema

use of org.apache.flink.formats.avro.AvroRowDataDeserializationSchema in project flink by apache.

the class RegistryAvroRowDataSeDeSchemaTest method testRowDataReadWithNonRegistryAvro.

@Test
public void testRowDataReadWithNonRegistryAvro() throws Exception {
    DataType dataType = AvroSchemaConverter.convertToDataType(ADDRESS_SCHEMA.toString());
    RowType rowType = (RowType) dataType.getLogicalType();
    AvroRowDataDeserializationSchema deserializer = getDeserializationSchema(rowType, ADDRESS_SCHEMA);
    deserializer.open(null);
    client.register(SUBJECT, ADDRESS_SCHEMA);
    byte[] oriBytes = writeRecord(address, ADDRESS_SCHEMA);
    expectedEx.expect(IOException.class);
    expectedEx.expect(containsCause(new IOException("Unknown data format. Magic number does not match")));
    deserializer.deserialize(oriBytes);
}

Example 3 with AvroRowDataDeserializationSchema

use of org.apache.flink.formats.avro.AvroRowDataDeserializationSchema in project flink by apache.

the class RegistryAvroFormatFactoryTest method testDeserializationSchema.

@Test
public void testDeserializationSchema() {
    final AvroRowDataDeserializationSchema expectedDeser = new AvroRowDataDeserializationSchema(ConfluentRegistryAvroDeserializationSchema.forGeneric(AvroSchemaConverter.convertToSchema(ROW_TYPE), REGISTRY_URL), AvroToRowDataConverters.createRowConverter(ROW_TYPE), InternalTypeInfo.of(ROW_TYPE));
    final DynamicTableSource actualSource = createTableSource(SCHEMA, getDefaultOptions());
    assertThat(actualSource, instanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class));
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedDeser, actualDeser);
}

Example 4 with AvroRowDataDeserializationSchema

use of org.apache.flink.formats.avro.AvroRowDataDeserializationSchema in project flink by apache.

the class RegistryAvroFormatFactoryTest method testDeserializationSchemaWithOptionalProperties.

@Test
public void testDeserializationSchemaWithOptionalProperties() {
    final AvroRowDataDeserializationSchema expectedDeser = new AvroRowDataDeserializationSchema(ConfluentRegistryAvroDeserializationSchema.forGeneric(AvroSchemaConverter.convertToSchema(ROW_TYPE), REGISTRY_URL, EXPECTED_OPTIONAL_PROPERTIES), AvroToRowDataConverters.createRowConverter(ROW_TYPE), InternalTypeInfo.of(ROW_TYPE));
    final DynamicTableSource actualSource = createTableSource(SCHEMA, getOptionalProperties());
    assertThat(actualSource, instanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class));
    TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = (TestDynamicTableFactory.DynamicTableSourceMock) actualSource;
    DeserializationSchema<RowData> actualDeser = scanSourceMock.valueFormat.createRuntimeDecoder(ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType());
    assertEquals(expectedDeser, actualDeser);
}

Example 5 with AvroRowDataDeserializationSchema

use of org.apache.flink.formats.avro.AvroRowDataDeserializationSchema in project flink by apache.

the class RegistryAvroFormatFactory method createDecodingFormat.

@Override
public DecodingFormat<DeserializationSchema<RowData>> createDecodingFormat(DynamicTableFactory.Context context, ReadableConfig formatOptions) {
    FactoryUtil.validateFactoryOptions(this, formatOptions);
    String schemaRegistryURL = formatOptions.get(URL);
    Map<String, ?> optionalPropertiesMap = buildOptionalPropertiesMap(formatOptions);
    return new ProjectableDecodingFormat<DeserializationSchema<RowData>>() {

        @Override
        public DeserializationSchema<RowData> createRuntimeDecoder(DynamicTableSource.Context context, DataType producedDataType, int[][] projections) {
            producedDataType = Projection.of(projections).project(producedDataType);
            final RowType rowType = (RowType) producedDataType.getLogicalType();
            final TypeInformation<RowData> rowDataTypeInfo = context.createTypeInformation(producedDataType);
            return new AvroRowDataDeserializationSchema(ConfluentRegistryAvroDeserializationSchema.forGeneric(AvroSchemaConverter.convertToSchema(rowType), schemaRegistryURL, optionalPropertiesMap), AvroToRowDataConverters.createRowConverter(rowType), rowDataTypeInfo);
        }

        @Override
        public ChangelogMode getChangelogMode() {
            return ChangelogMode.insertOnly();
        }
    };
}