Examples with FieldMappingUnit org.apache.inlong.sort.protocol.transformation.FieldMappingRule.FieldMappingUnit used on opensource projects

Example 1 with FieldMappingUnit

use of org.apache.inlong.sort.protocol.transformation.FieldMappingRule.FieldMappingUnit in project incubator-inlong by apache.

the class FieldInfoUtils method setAllMigrationBuiltInField.

/**
 * Get all migration built-in field for binlog source.
 */
public static void setAllMigrationBuiltInField(List<FieldInfo> sourceFields, List<FieldInfo> sinkFields, List<FieldMappingUnit> fieldMappingUnitList) {
    BuiltInFieldInfo dataField = new BuiltInFieldInfo("data", StringFormatInfo.INSTANCE, BuiltInField.MYSQL_METADATA_DATA);
    sourceFields.add(dataField);
    sinkFields.add(dataField);
    fieldMappingUnitList.add(new FieldMappingUnit(dataField, dataField));
    for (Map.Entry<String, BuiltInField> entry : BUILT_IN_FIELD_MAP.entrySet()) {
        if (entry.getKey().equals("data_time")) {
            continue;
        }
        BuiltInFieldInfo fieldInfo = new BuiltInFieldInfo(entry.getKey(), StringFormatInfo.INSTANCE, entry.getValue());
        sourceFields.add(fieldInfo);
        sinkFields.add(fieldInfo);
        fieldMappingUnitList.add(new FieldMappingUnit(fieldInfo, fieldInfo));
    }
}

Example 2 with FieldMappingUnit

use of org.apache.inlong.sort.protocol.transformation.FieldMappingRule.FieldMappingUnit in project incubator-inlong by apache.

the class TransformerTest method testTransformWithNotExistSinkFieldName.

@Test
public void testTransformWithNotExistSinkFieldName() {
    TransformationInfo transformationInfo = new TransformationInfo(new FieldMappingRule(new FieldMappingUnit[] { new FieldMappingUnit(new FieldInfo("age", StringFormatInfo.INSTANCE), new FieldInfo("age_out", StringFormatInfo.INSTANCE)), new FieldMappingUnit(new FieldInfo("name", StringFormatInfo.INSTANCE), // not exist sink field name
    new FieldInfo("name_out_not_exist", StringFormatInfo.INSTANCE)) }));
    Transformer transformer = new Transformer(transformationInfo, sourceFieldInfos, sinkFieldInfos);
    transformer.open(new Configuration());
    Row input = Row.of("name", 29, 179);
    ListCollector<Row> collector = new ListCollector<>();
    transformer.processElement(input, null, collector);
    assertEquals(Row.of(29, null), collector.getInnerList().get(0));
}

Example 3 with FieldMappingUnit

use of org.apache.inlong.sort.protocol.transformation.FieldMappingRule.FieldMappingUnit in project incubator-inlong by apache.

the class Transformer method getRowTransformer.

private Function<Row, Row> getRowTransformer() {
    TransformationRule transformationRule = transformationInfo.getTransformRule();
    if (transformationRule instanceof FieldMappingRule) {
        // Get 'sink field name' => 'source field name' map
        FieldMappingRule fieldMappingRule = (FieldMappingRule) transformationRule;
        final Map<String, String> sinkFieldNameToSourceFieldName = new HashMap<>();
        for (FieldMappingUnit fieldMappingUnit : fieldMappingRule.getFieldMappingUnits()) {
            sinkFieldNameToSourceFieldName.put(fieldMappingUnit.getSinkFieldInfo().getName(), fieldMappingUnit.getSourceFieldInfo().getName());
        }
        // Get 'source field name' => 'index in source row' map
        final Map<String, Integer> sourceFieldNameToIndex = new HashMap<>();
        for (int i = 0; i < sourceFieldInfos.length; i++) {
            sourceFieldNameToIndex.put(sourceFieldInfos[i].getName(), i);
        }
        return row -> {
            final int sinkFieldsLength = sinkFieldInfos.length;
            Row sinkRow = new Row(sinkFieldsLength);
            for (int i = 0; i < sinkFieldsLength; i++) {
                String sinkFieldName = sinkFieldInfos[i].getName();
                String sourceFieldName = sinkFieldNameToSourceFieldName.get(sinkFieldName);
                if (sourceFieldName == null) {
                    LOGGER.warn("Mapping failed! Can't find correspond source field! sink field name `{}`", sinkFieldName);
                    sinkRow.setField(i, null);
                    continue;
                }
                Integer sourceFieldIndex = sourceFieldNameToIndex.get(sourceFieldName);
                if (sourceFieldIndex == null) {
                    LOGGER.warn("Mapping failed! Can't find correspond source field!" + " source field name `{}`, sink field name `{}`", sourceFieldName, sinkFieldName);
                    sinkRow.setField(i, null);
                    continue;
                }
                sinkRow.setField(i, row.getField(sourceFieldIndex));
            }
            return sinkRow;
        };
    } else {
        throw new IllegalArgumentException("Unsupported transformation rule " + transformationRule.getClass());
    }
}

Example 4 with FieldMappingUnit

use of org.apache.inlong.sort.protocol.transformation.FieldMappingRule.FieldMappingUnit in project incubator-inlong by apache.

the class TransformerTest method testTransform.

@Test
public void testTransform() {
    TransformationInfo transformationInfo = new TransformationInfo(new FieldMappingRule(new FieldMappingUnit[] { new FieldMappingUnit(new FieldInfo("age", StringFormatInfo.INSTANCE), new FieldInfo("age_out", StringFormatInfo.INSTANCE)), new FieldMappingUnit(new FieldInfo("name", StringFormatInfo.INSTANCE), new FieldInfo("name_out", StringFormatInfo.INSTANCE)) }));
    Transformer transformer = new Transformer(transformationInfo, sourceFieldInfos, sinkFieldInfos);
    transformer.open(new Configuration());
    Row input = Row.of("name", 29, 179);
    ListCollector<Row> collector = new ListCollector<>();
    transformer.processElement(input, null, collector);
    assertEquals(Row.of(29, "name"), collector.getInnerList().get(0));
}

Example 5 with FieldMappingUnit

use of org.apache.inlong.sort.protocol.transformation.FieldMappingRule.FieldMappingUnit in project incubator-inlong by apache.

the class TransformerTest method testTransformWithNotExistSourceFieldName.

@Test
public void testTransformWithNotExistSourceFieldName() {
    TransformationInfo transformationInfo = new TransformationInfo(new FieldMappingRule(new FieldMappingUnit[] { new FieldMappingUnit(new FieldInfo("age", StringFormatInfo.INSTANCE), new FieldInfo("age_out", StringFormatInfo.INSTANCE)), new FieldMappingUnit(// not exist source field name
    new FieldInfo("name_not_exist", StringFormatInfo.INSTANCE), new FieldInfo("name_out", StringFormatInfo.INSTANCE)) }));
    Transformer transformer = new Transformer(transformationInfo, sourceFieldInfos, sinkFieldInfos);
    transformer.open(new Configuration());
    Row input = Row.of("name", 29, 179);
    ListCollector<Row> collector = new ListCollector<>();
    transformer.processElement(input, null, collector);
    assertEquals(Row.of(29, null), collector.getInnerList().get(0));
}