Examples with RowFormatInfo org.apache.inlong.sort.formats.common.RowFormatInfo used on opensource projects

Example 1 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class TableFormatUtils method deriveFormatInfo.

/**
 * Derive the format information for the given type.
 *
 * @param logicalType The type whose format is derived.
 * @return The format information for the given type.
 */
public static FormatInfo deriveFormatInfo(LogicalType logicalType) {
    if (logicalType instanceof VarCharType) {
        return StringFormatInfo.INSTANCE;
    } else if (logicalType instanceof BooleanType) {
        return BooleanFormatInfo.INSTANCE;
    } else if (logicalType instanceof TinyIntType) {
        return ByteFormatInfo.INSTANCE;
    } else if (logicalType instanceof SmallIntType) {
        return ShortFormatInfo.INSTANCE;
    } else if (logicalType instanceof IntType) {
        return IntFormatInfo.INSTANCE;
    } else if (logicalType instanceof BigIntType) {
        return LongFormatInfo.INSTANCE;
    } else if (logicalType instanceof FloatType) {
        return FloatFormatInfo.INSTANCE;
    } else if (logicalType instanceof DoubleType) {
        return DoubleFormatInfo.INSTANCE;
    } else if (logicalType instanceof DecimalType) {
        return DecimalFormatInfo.INSTANCE;
    } else if (logicalType instanceof DateType) {
        return new DateFormatInfo();
    } else if (logicalType instanceof TimeType) {
        return new TimeFormatInfo();
    } else if (logicalType instanceof TimestampType) {
        return new TimestampFormatInfo();
    } else if (logicalType instanceof LocalZonedTimestampType) {
        return new LocalZonedTimestampFormatInfo();
    } else if (logicalType instanceof ArrayType) {
        ArrayType arrayType = (ArrayType) logicalType;
        LogicalType elementType = arrayType.getElementType();
        FormatInfo elementFormatInfo = deriveFormatInfo(elementType);
        return new ArrayFormatInfo(elementFormatInfo);
    } else if (logicalType instanceof MapType) {
        MapType mapType = (MapType) logicalType;
        LogicalType keyType = mapType.getKeyType();
        LogicalType valueType = mapType.getValueType();
        FormatInfo keyFormatInfo = deriveFormatInfo(keyType);
        FormatInfo valueFormatInfo = deriveFormatInfo(valueType);
        return new MapFormatInfo(keyFormatInfo, valueFormatInfo);
    } else if (logicalType instanceof RowType) {
        RowType rowType = (RowType) logicalType;
        List<RowType.RowField> rowFields = rowType.getFields();
        String[] fieldNames = new String[rowFields.size()];
        FormatInfo[] fieldFormatInfos = new FormatInfo[rowFields.size()];
        for (int i = 0; i < rowFields.size(); ++i) {
            RowType.RowField rowField = rowFields.get(i);
            fieldNames[i] = rowField.getName();
            fieldFormatInfos[i] = deriveFormatInfo(rowField.getType());
        }
        return new RowFormatInfo(fieldNames, fieldFormatInfos);
    } else if (logicalType instanceof BinaryType) {
        return BinaryFormatInfo.INSTANCE;
    } else if (logicalType instanceof NullType) {
        return NullFormatInfo.INSTANCE;
    } else {
        throw new UnsupportedOperationException();
    }
}

Example 2 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class TableFormatUtils method deserializeRowFormatInfo.

/**
 * Returns the format defined in the given property.
 *
 * @param descriptorProperties The properties of the descriptor.
 * @return The basic row format defined in the descriptor.
 */
public static RowFormatInfo deserializeRowFormatInfo(DescriptorProperties descriptorProperties) {
    try {
        String schema = descriptorProperties.getString(FORMAT_SCHEMA);
        FormatInfo formatInfo = FormatUtils.demarshall(schema);
        if (!(formatInfo instanceof RowFormatInfo)) {
            throw new IllegalStateException("Unexpected format type.");
        }
        return (RowFormatInfo) formatInfo;
    } catch (Exception e) {
        throw new ValidationException("The schema is invalid.", e);
    }
}

Example 3 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method generateDeserializer.

@VisibleForTesting
Pair<AbstractInLongMsgMixedFormatDeserializer, InLongMsgMixedFormatConverter> generateDeserializer(FieldInfo[] fields, InLongMsgDeserializationInfo inLongMsgDeserializationInfo) {
    final RowFormatInfo rowFormatInfo = CommonUtils.generateDeserializationRowFormatInfo(fields);
    final AbstractInLongMsgMixedFormatDeserializer preDeserializer;
    final InLongMsgMixedFormatConverter deserializer;
    if (inLongMsgDeserializationInfo instanceof InLongMsgCsvDeserializationInfo) {
        final InLongMsgCsvDeserializationInfo csvDeserializationInfo = (InLongMsgCsvDeserializationInfo) inLongMsgDeserializationInfo;
        preDeserializer = new InLongMsgCsvMixedFormatDeserializer(StandardCharsets.UTF_8.name(), csvDeserializationInfo.getDelimiter(), null, null, csvDeserializationInfo.isDeleteHeadDelimiter(), false);
        deserializer = new InLongMsgCsvMixedFormatConverter(rowFormatInfo, DEFAULT_TIME_FIELD_NAME, DEFAULT_ATTRIBUTES_FIELD_NAME, null, false);
    } else {
        throw new UnsupportedOperationException("Not supported yet " + inLongMsgDeserializationInfo.getClass().getSimpleName());
    }
    return Pair.of(preDeserializer, deserializer);
}

Example 4 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class CommonUtils method convertFieldInfosToLogicalType.

public static LogicalType convertFieldInfosToLogicalType(FieldInfo[] fieldInfos) {
    int fieldLength = fieldInfos.length;
    String[] fieldNames = new String[fieldLength];
    FormatInfo[] fieldFormatInfos = new FormatInfo[fieldLength];
    for (int i = 0; i < fieldLength; i++) {
        fieldNames[i] = fieldInfos[i].getName();
        fieldFormatInfos[i] = fieldInfos[i].getFormatInfo();
    }
    RowFormatInfo rowFormatInfo = new RowFormatInfo(fieldNames, fieldFormatInfos);
    return deriveLogicalType(rowFormatInfo);
}

Example 5 with RowFormatInfo

use of org.apache.inlong.sort.formats.common.RowFormatInfo in project incubator-inlong by apache.

the class KvFormatFactory method createFormatDeserializer.

@Override
public TableFormatDeserializer createFormatDeserializer(Map<String, String> properties) {
    final DescriptorProperties descriptorProperties = getValidatedProperties(properties);
    final RowFormatInfo rowFormatInfo = TableFormatUtils.getRowFormatInfo(descriptorProperties);
    final KvDeserializationSchema deserializationSchema = buildDeserializationSchema(descriptorProperties, rowFormatInfo);
    boolean ignoreErrors = descriptorProperties.getOptionalBoolean(TableFormatConstants.FORMAT_IGNORE_ERRORS).orElse(TableFormatConstants.DEFAULT_IGNORE_ERRORS);
    return new DefaultTableFormatDeserializer(deserializationSchema, ignoreErrors);
}