Examples with RowFormatInfo org.apache.inlong.sort.formats.common.RowFormatInfo used on opensource projects

Example 6 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class CsvFormatFactory method buildSerializationSchema.

private static CsvSerializationSchema buildSerializationSchema(DescriptorProperties descriptorProperties, RowFormatInfo rowFormatInfo) {
    for (FormatInfo formatInfo : rowFormatInfo.getFieldFormatInfos()) {
        if (!(formatInfo instanceof BasicFormatInfo)) {
            throw new ValidationException("Currently only basic formats " + "are supported in csv formats.");
        }
    }
    CsvSerializationSchema.Builder builder = new CsvSerializationSchema.Builder(rowFormatInfo);
    descriptorProperties.getOptionalString(TableFormatConstants.FORMAT_CHARSET).ifPresent(builder::setCharset);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_DELIMITER).ifPresent(builder::setDelimiter);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_ESCAPE_CHARACTER).ifPresent(builder::setEscapeCharacter);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_QUOTE_CHARACTER).ifPresent(builder::setQuoteCharacter);
    descriptorProperties.getOptionalString(TableFormatConstants.FORMAT_NULL_LITERAL).ifPresent(builder::setNullLiteral);
    return builder.build();
}

Example 7 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class CsvFormatFactory method createDeserializationSchema.

@Override
public CsvDeserializationSchema createDeserializationSchema(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    return buildDeserializationSchema(descriptorProperties, rowFormatInfo);
}

Example 8 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class CsvFormatFactory method createFormatDeserializer.

@Override
public TableFormatDeserializer createFormatDeserializer(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    final CsvDeserializationSchema deserializationSchema = buildDeserializationSchema(descriptorProperties, rowFormatInfo);
    boolean ignoreErrors = descriptorProperties.getOptionalBoolean(TableFormatConstants.FORMAT_IGNORE_ERRORS).orElse(TableFormatConstants.DEFAULT_IGNORE_ERRORS);
    return new DefaultTableFormatDeserializer(deserializationSchema, ignoreErrors);
}

Example 9 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class KvSerializationSchemaTest method testBasicSerialization.

private static <T> void testBasicSerialization(Consumer<KvSerializationSchema.Builder> config, BasicFormatInfo<T> basicFormatInfo, T record, String expectedText) {
    RowFormatInfo rowFormatInfo = new RowFormatInfo(new String[] { "f" }, new FormatInfo[] { basicFormatInfo });
    KvSerializationSchema.Builder builder = new KvSerializationSchema.Builder(rowFormatInfo);
    config.accept(builder);
    KvSerializationSchema serializer = builder.build();
    String text = new String(serializer.serialize(Row.of(record)));
    assertEquals(expectedText, text);
}

Example 10 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class InLongMsgCsvFormatFactory method createFormatDeserializer.

@Override
public TableFormatDeserializer createFormatDeserializer(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
    descriptorProperties.putProperties(properties);
    final InLongMsgValidator validator = new InLongMsgValidator();
    validator.validate(descriptorProperties);
    RowFormatInfo rowFormatInfo = getDataFormatInfo(descriptorProperties);
    String timeFieldName = descriptorProperties.getOptionalString(FORMAT_TIME_FIELD_NAME).orElse(InLongMsgUtils.DEFAULT_TIME_FIELD_NAME);
    String attributesFieldName = descriptorProperties.getOptionalString(FORMAT_ATTRIBUTES_FIELD_NAME).orElse(InLongMsgUtils.DEFAULT_ATTRIBUTES_FIELD_NAME);
    validateFieldNames(timeFieldName, attributesFieldName, rowFormatInfo);
    String charset = descriptorProperties.getOptionalString(FORMAT_CHARSET).orElse(DEFAULT_CHARSET);
    Character delimiter = descriptorProperties.getOptionalCharacter(FORMAT_DELIMITER).orElse(DEFAULT_DELIMITER);
    Character escapeCharacter = descriptorProperties.getOptionalCharacter(FORMAT_ESCAPE_CHARACTER).orElse(null);
    Character quoteCharacter = descriptorProperties.getOptionalCharacter(FORMAT_QUOTE_CHARACTER).orElse(null);
    String nullLiteral = descriptorProperties.getOptionalString(FORMAT_NULL_LITERAL).orElse(null);
    Boolean deleteHeadDelimiter = descriptorProperties.getOptionalBoolean(FORMAT_DELETE_HEAD_DELIMITER).orElse(InLongMsgCsvUtils.DEFAULT_DELETE_HEAD_DELIMITER);
    boolean ignoreErrors = descriptorProperties.getOptionalBoolean(FORMAT_IGNORE_ERRORS).orElse(DEFAULT_IGNORE_ERRORS);
    return new InLongMsgCsvFormatDeserializer(rowFormatInfo, timeFieldName, attributesFieldName, charset, delimiter, escapeCharacter, quoteCharacter, nullLiteral, deleteHeadDelimiter, ignoreErrors);
}