Examples with DefaultTableFormatDeserializer org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer used on opensource projects

Example 1 with DefaultTableFormatDeserializer

use of org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer in project incubator-inlong by apache.

the class KvFormatFactory method createFormatDeserializer.

@Override
public TableFormatDeserializer createFormatDeserializer(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    final KvDeserializationSchema deserializationSchema = buildDeserializationSchema(descriptorProperties, rowFormatInfo);
    boolean ignoreErrors = descriptorProperties.getOptionalBoolean(TableFormatConstants.FORMAT_IGNORE_ERRORS).orElse(TableFormatConstants.DEFAULT_IGNORE_ERRORS);
    return new DefaultTableFormatDeserializer(deserializationSchema, ignoreErrors);
}

Example 2 with DefaultTableFormatDeserializer

use of org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer in project incubator-inlong by apache.

the class CsvFormatFactory method createFormatDeserializer.

@Override
public TableFormatDeserializer createFormatDeserializer(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    final CsvDeserializationSchema deserializationSchema = buildDeserializationSchema(descriptorProperties, rowFormatInfo);
    boolean ignoreErrors = descriptorProperties.getOptionalBoolean(TableFormatConstants.FORMAT_IGNORE_ERRORS).orElse(TableFormatConstants.DEFAULT_IGNORE_ERRORS);
    return new DefaultTableFormatDeserializer(deserializationSchema, ignoreErrors);
}

Example 3 with DefaultTableFormatDeserializer

use of org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer in project incubator-inlong by apache.

the class CsvFormatFactoryTest method testCreateTableFormatDeserializer.

@Test
public void testCreateTableFormatDeserializer() throws Exception {
    final Map<String, String> properties = new Csv().schema(FormatUtils.marshall(TEST_FORMAT_SCHEMA)).delimiter(';').charset(StandardCharsets.ISO_8859_1).escapeCharacter('\\').quoteCharacter('\"').nullLiteral("null").toProperties();
    assertNotNull(properties);
    final CsvDeserializationSchema deserializationSchema = new CsvDeserializationSchema(TEST_FORMAT_SCHEMA, StandardCharsets.ISO_8859_1.name(), ';', '\\', '\"', "null");
    final DefaultTableFormatDeserializer expectedDeser = new DefaultTableFormatDeserializer(deserializationSchema);
    final TableFormatDeserializer actualDeser = TableFormatUtils.getTableFormatDeserializer(properties, getClass().getClassLoader());
    assertEquals(expectedDeser, actualDeser);
}

Example 4 with DefaultTableFormatDeserializer

use of org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer in project incubator-inlong by apache.

the class KvFormatFactoryTest method testCreateTableFormatDeserializerWithDerivation.

@Test
public void testCreateTableFormatDeserializerWithDerivation() {
    final Map<String, String> properties = new HashMap<>();
    properties.putAll(new Schema().schema(TableSchema.fromTypeInfo(SCHEMA)).toProperties());
    properties.putAll(new Kv().deriveSchema().toProperties());
    final KvDeserializationSchema deserializationSchema = new KvDeserializationSchema.Builder(TEST_FORMAT_SCHEMA).build();
    final DefaultTableFormatDeserializer expectedDeser = new DefaultTableFormatDeserializer(deserializationSchema);
    final TableFormatDeserializer actualDeser = TableFormatUtils.getTableFormatDeserializer(properties, getClass().getClassLoader());
    assertEquals(expectedDeser, actualDeser);
}

Example 5 with DefaultTableFormatDeserializer

use of org.apache.inlong.sort.formats.base.DefaultTableFormatDeserializer in project incubator-inlong by apache.

the class KvFormatFactoryTest method testCreateTableFormatDeserializer.

@Test
public void testCreateTableFormatDeserializer() throws Exception {
    final Map<String, String> properties = new Kv().schema(FormatUtils.marshall(TEST_FORMAT_SCHEMA)).entryDelimiter('&').kvDelimiter('=').charset(StandardCharsets.ISO_8859_1).escapeCharacter('\\').quoteCharacter('\"').nullLiteral("null").toProperties();
    assertNotNull(properties);
    final KvDeserializationSchema deserializationSchema = new KvDeserializationSchema.Builder(TEST_FORMAT_SCHEMA).setEntryDelimiter('&').setKvDelimiter('=').setCharset(StandardCharsets.ISO_8859_1.name()).setEscapeCharacter('\\').setQuoteCharacter('\"').setNullLiteral("null").build();
    final DefaultTableFormatDeserializer expectedDeser = new DefaultTableFormatDeserializer(deserializationSchema);
    final TableFormatDeserializer actualDeser = TableFormatUtils.getTableFormatDeserializer(properties, getClass().getClassLoader());
    assertEquals(expectedDeser, actualDeser);
}