Examples with RowFormatInfo org.apache.inlong.sort.formats.common.RowFormatInfo used on opensource projects

Example 16 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class KvFormatFactory method createDeserializationSchema.

@Override
public KvDeserializationSchema createDeserializationSchema(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    return buildDeserializationSchema(descriptorProperties, rowFormatInfo);
}

Example 17 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class KvFormatFactory method buildSerializationSchema.

private static KvSerializationSchema buildSerializationSchema(DescriptorProperties descriptorProperties, RowFormatInfo rowFormatInfo) {
    for (FormatInfo formatInfo : rowFormatInfo.getFieldFormatInfos()) {
        if (!(formatInfo instanceof BasicFormatInfo)) {
            throw new ValidationException("Currently only basic formats " + "are supported in kv formats.");
        }
    }
    KvSerializationSchema.Builder builder = new KvSerializationSchema.Builder(rowFormatInfo);
    descriptorProperties.getOptionalString(TableFormatConstants.FORMAT_CHARSET).ifPresent(builder::setCharset);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_ENTRY_DELIMITER).ifPresent(builder::setEntryDelimiter);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_KV_DELIMITER).ifPresent(builder::setKvDelimiter);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_ESCAPE_CHARACTER).ifPresent(builder::setEscapeCharacter);
    descriptorProperties.getOptionalCharacter(TableFormatConstants.FORMAT_QUOTE_CHARACTER).ifPresent(builder::setQuoteCharacter);
    descriptorProperties.getOptionalString(TableFormatConstants.FORMAT_NULL_LITERAL).ifPresent(builder::setNullLiteral);
    return builder.build();
}

Example 18 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class KvDeserializationSchemaTest method testBasicDeserialization.

private static <T> void testBasicDeserialization(Consumer<KvDeserializationSchema.Builder> config, BasicFormatInfo<T> basicFormatInfo, T expectedRecord, String text) throws IOException {
    RowFormatInfo rowFormatInfo = new RowFormatInfo(new String[] { "f" }, new FormatInfo[] { basicFormatInfo });
    KvDeserializationSchema.Builder builder = new KvDeserializationSchema.Builder(rowFormatInfo);
    config.accept(builder);
    KvDeserializationSchema deserializer = builder.build();
    Row row = deserializer.deserialize(text.getBytes());
    assertEquals(1, row.getArity());
    assertEquals(expectedRecord, row.getField(0));
}

Example 19 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class InLongMsgCsvUtils method buildRow.

public static Row buildRow(RowFormatInfo rowFormatInfo, String nullLiteral, Timestamp time, Map<String, String> attributes, List<String> predefinedFields, List<String> fields) {
    String[] fieldNames = rowFormatInfo.getFieldNames();
    FormatInfo[] fieldFormatInfos = rowFormatInfo.getFieldFormatInfos();
    int actualNumFields = predefinedFields.size() + fields.size();
    if (actualNumFields != fieldNames.length) {
        LOG.warn("The number of fields mismatches: " + fieldNames.length + " expected, but was " + actualNumFields + ".");
    }
    Row row = new Row(2 + fieldNames.length);
    row.setField(0, time);
    row.setField(1, attributes);
    for (int i = 0; i < predefinedFields.size(); ++i) {
        if (i >= fieldNames.length) {
            break;
        }
        String fieldName = fieldNames[i];
        FormatInfo fieldFormatInfo = fieldFormatInfos[i];
        String fieldText = predefinedFields.get(i);
        Object field = TableFormatUtils.deserializeBasicField(fieldName, fieldFormatInfo, fieldText, nullLiteral);
        row.setField(i + 2, field);
    }
    for (int i = 0; i < fields.size(); ++i) {
        if (i + predefinedFields.size() >= fieldNames.length) {
            break;
        }
        String fieldName = fieldNames[i + predefinedFields.size()];
        FormatInfo fieldFormatInfo = fieldFormatInfos[i + predefinedFields.size()];
        String fieldText = fields.get(i);
        Object field = TableFormatUtils.deserializeBasicField(fieldName, fieldFormatInfo, fieldText, nullLiteral);
        row.setField(i + predefinedFields.size() + 2, field);
    }
    for (int i = predefinedFields.size() + fields.size(); i < fieldNames.length; ++i) {
        row.setField(i + 2, null);
    }
    return row;
}

Example 20 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class CsvFormatFactory method createFormatSerializer.

@Override
public TableFormatSerializer createFormatSerializer(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    final CsvSerializationSchema serializationSchema = buildSerializationSchema(descriptorProperties, rowFormatInfo);
    boolean ignoreErrors = descriptorProperties.getOptionalBoolean(TableFormatConstants.FORMAT_IGNORE_ERRORS).orElse(TableFormatConstants.DEFAULT_IGNORE_ERRORS);
    return new DefaultTableFormatSerializer(serializationSchema, ignoreErrors);
}