Examples with BuiltInFieldInfo org.apache.inlong.sort.protocol.BuiltInFieldInfo used on opensource projects

Example 1 with BuiltInFieldInfo

use of org.apache.inlong.sort.protocol.BuiltInFieldInfo in project incubator-inlong by apache.

the class FieldInfoUtils method setAllMigrationBuiltInField.

/**
 * Get all migration built-in field for binlog source.
 */
public static void setAllMigrationBuiltInField(List<FieldInfo> sourceFields, List<FieldInfo> sinkFields, List<FieldMappingUnit> fieldMappingUnitList) {
    BuiltInFieldInfo dataField = new BuiltInFieldInfo("data", StringFormatInfo.INSTANCE, BuiltInField.MYSQL_METADATA_DATA);
    sourceFields.add(dataField);
    sinkFields.add(dataField);
    fieldMappingUnitList.add(new FieldMappingUnit(dataField, dataField));
    for (Map.Entry<String, BuiltInField> entry : BUILT_IN_FIELD_MAP.entrySet()) {
        if (entry.getKey().equals("data_time")) {
            continue;
        }
        BuiltInFieldInfo fieldInfo = new BuiltInFieldInfo(entry.getKey(), StringFormatInfo.INSTANCE, entry.getValue());
        sourceFields.add(fieldInfo);
        sinkFields.add(fieldInfo);
        fieldMappingUnitList.add(new FieldMappingUnit(fieldInfo, fieldInfo));
    }
}

Example 2 with BuiltInFieldInfo

use of org.apache.inlong.sort.protocol.BuiltInFieldInfo in project incubator-inlong by apache.

the class FieldMappingTransformer method transform.

public Record transform(Record sourceRecord) throws Exception {
    final FieldInfo[] sinkFieldInfos = sinkFieldInfoMap.get(sourceRecord.getDataflowId());
    if (sinkFieldInfos == null) {
        throw new Exception("No field info found for data flow:" + sourceRecord.getDataflowId());
    }
    final Row sourceRow = sourceRecord.getRow();
    final Row sinkRow = new Row(sinkFieldInfos.length);
    int fieldIndex = SOURCE_FIELD_SKIP_STEP;
    for (int i = 0; i < sinkFieldInfos.length; i++) {
        Object fieldValue = null;
        if (sinkFieldInfos[i] instanceof BuiltInFieldInfo) {
            BuiltInFieldInfo builtInFieldInfo = (BuiltInFieldInfo) sinkFieldInfos[i];
            fieldValue = transformBuiltInField(builtInFieldInfo, sourceRecord.getTimestampMillis());
        } else if (fieldIndex < sourceRow.getArity()) {
            fieldValue = sourceRow.getField(fieldIndex);
            fieldIndex++;
        }
        if (fieldValue == null) {
            fieldValue = getDefaultValue(sourceRecord.getDataflowId(), sinkFieldInfos[i].getFormatInfo());
        }
        sinkRow.setField(i, fieldValue);
    }
    return new Record(sourceRecord.getDataflowId(), sourceRecord.getTimestampMillis(), sinkRow);
}

Example 3 with BuiltInFieldInfo

use of org.apache.inlong.sort.protocol.BuiltInFieldInfo in project incubator-inlong by apache.

the class DebeziumDeserializationSchemaBuilder method getMetadataFieldInfos.

public static FieldInfo[] getMetadataFieldInfos(FieldInfo[] fieldInfos) {
    List<FieldInfo> metadataFieldInfos = new ArrayList<>();
    Arrays.stream(fieldInfos).filter(fieldInfo -> fieldInfo instanceof BuiltInFieldInfo).forEach(fieldInfo -> {
        BuiltInFieldInfo builtInFieldInfo = (BuiltInFieldInfo) fieldInfo;
        BuiltInField builtInField = builtInFieldInfo.getBuiltInField();
        switch(builtInField) {
            case MYSQL_METADATA_DATABASE:
                metadataFieldInfos.add(new FieldInfo(ReadableMetadata.SOURCE_DATABASE.getKey(), StringFormatInfo.INSTANCE));
                break;
            case MYSQL_METADATA_TABLE:
                metadataFieldInfos.add(new FieldInfo(ReadableMetadata.SOURCE_TABLE.getKey(), StringFormatInfo.INSTANCE));
                break;
            case MYSQL_METADATA_EVENT_TIME:
                metadataFieldInfos.add(new FieldInfo(ReadableMetadata.INGESTION_TIMESTAMP.getKey(), LongFormatInfo.INSTANCE));
                break;
            case MYSQL_METADATA_IS_DDL:
            case MYSQL_METADATA_EVENT_TYPE:
            case MYSQL_METADATA_DATA:
                break;
            default:
                throw new IllegalArgumentException("Unsupported builtin field '" + builtInField + "' in debezium deserialization");
        }
    });
    return metadataFieldInfos.toArray(new FieldInfo[0]);
}

Example 4 with BuiltInFieldInfo

use of org.apache.inlong.sort.protocol.BuiltInFieldInfo in project incubator-inlong by apache.

the class FieldMappingTransformer method transform.

public Row transform(Row sourceRow, long dt) {
    final Row outputRow = new Row(outputFieldInfos.length);
    int sourceRowIndex = SOURCE_FIELD_SKIP_STEP;
    Map<String, String> attributes = (Map<String, String>) sourceRow.getField(0);
    for (int i = 0; i < outputFieldInfos.length; i++) {
        Object fieldValue = null;
        if (outputFieldInfos[i] instanceof BuiltInFieldInfo) {
            BuiltInFieldInfo builtInFieldInfo = (BuiltInFieldInfo) outputFieldInfos[i];
            fieldValue = transformBuiltInField(builtInFieldInfo, attributes, dt, sourceRow.getKind());
        } else if (sourceRowIndex < sourceRow.getArity()) {
            fieldValue = sourceRow.getField(sourceRowIndex);
            sourceRowIndex++;
        }
        if (fieldValue == null) {
            fieldValue = defaultValueStrategy.getDefaultValue(outputFieldInfos[i].getFormatInfo());
        }
        outputRow.setField(i, fieldValue);
    }
    outputRow.setKind(sourceRow.getKind());
    return outputRow;
}

Example 5 with BuiltInFieldInfo

use of org.apache.inlong.sort.protocol.BuiltInFieldInfo in project incubator-inlong by apache.

the class CanalSerializationSchemaBuilder method getFieldIndexToMetadata.

private static Map<Integer, ReadableMetadata> getFieldIndexToMetadata(FieldInfo[] fieldInfos) {
    Map<Integer, ReadableMetadata> fieldIndexToMetadata = new HashMap<>();
    for (int i = 0; i < fieldInfos.length; i++) {
        FieldInfo fieldInfo = fieldInfos[i];
        if (fieldInfo instanceof BuiltInFieldInfo) {
            BuiltInFieldInfo builtInFieldInfo = (BuiltInFieldInfo) fieldInfo;
            BuiltInField builtInField = builtInFieldInfo.getBuiltInField();
            switch(builtInField) {
                case MYSQL_METADATA_DATABASE:
                    fieldIndexToMetadata.put(i, ReadableMetadata.DATABASE);
                    break;
                case MYSQL_METADATA_TABLE:
                    fieldIndexToMetadata.put(i, ReadableMetadata.TABLE);
                    break;
                case MYSQL_METADATA_EVENT_TIME:
                    fieldIndexToMetadata.put(i, ReadableMetadata.EVENT_TIMESTAMP);
                    break;
                case MYSQL_METADATA_IS_DDL:
                    fieldIndexToMetadata.put(i, ReadableMetadata.IS_DDL);
                    break;
                case MYSQL_METADATA_DATA:
                case MYSQL_METADATA_EVENT_TYPE:
                    // We will always append `type` to the result
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported builtin field '" + builtInField + "' in debezium deserialization");
            }
        }
    }
    return fieldIndexToMetadata;
}