Examples with TransformationInfo org.apache.inlong.sort.protocol.transformation.TransformationInfo used on opensource projects

Example 1 with TransformationInfo

use of org.apache.inlong.sort.protocol.transformation.TransformationInfo in project incubator-inlong by apache.

the class TransformerTest method testTransformWithNotExistSinkFieldName.

@Test
public void testTransformWithNotExistSinkFieldName() {
    TransformationInfo transformationInfo = new TransformationInfo(new FieldMappingRule(new FieldMappingUnit[] { new FieldMappingUnit(new FieldInfo("age", StringFormatInfo.INSTANCE), new FieldInfo("age_out", StringFormatInfo.INSTANCE)), new FieldMappingUnit(new FieldInfo("name", StringFormatInfo.INSTANCE), // not exist sink field name
    new FieldInfo("name_out_not_exist", StringFormatInfo.INSTANCE)) }));
    Transformer transformer = new Transformer(transformationInfo, sourceFieldInfos, sinkFieldInfos);
    transformer.open(new Configuration());
    Row input = Row.of("name", 29, 179);
    ListCollector<Row> collector = new ListCollector<>();
    transformer.processElement(input, null, collector);
    assertEquals(Row.of(29, null), collector.getInnerList().get(0));
}

Example 2 with TransformationInfo

use of org.apache.inlong.sort.protocol.transformation.TransformationInfo in project incubator-inlong by apache.

the class Entrance method buildTransformationStream.

private static DataStream<Row> buildTransformationStream(DataStream<Row> deserializationStream, DataFlowInfo dataFlowInfo, Configuration config) {
    final boolean orderlyOutput = config.getBoolean(Constants.JOB_ORDERLY_OUTPUT);
    TransformationInfo transformationInfo = dataFlowInfo.getTransformationInfo();
    if (transformationInfo == null) {
        return deserializationStream;
    }
    DataStream<Row> transformationStream = deserializationStream.process(new Transformer(transformationInfo, dataFlowInfo.getSourceInfo().getFields(), dataFlowInfo.getSinkInfo().getFields())).uid(Constants.TRANSFORMATION_UID).name("Transformation").setParallelism(config.getInteger(Constants.TRANSFORMATION_PARALLELISM));
    if (orderlyOutput) {
        return transformationStream.forward();
    }
    return transformationStream;
}

Example 3 with TransformationInfo

use of org.apache.inlong.sort.protocol.transformation.TransformationInfo in project incubator-inlong by apache.

the class TransformerTest method testTransform.

@Test
public void testTransform() {
    TransformationInfo transformationInfo = new TransformationInfo(new FieldMappingRule(new FieldMappingUnit[] { new FieldMappingUnit(new FieldInfo("age", StringFormatInfo.INSTANCE), new FieldInfo("age_out", StringFormatInfo.INSTANCE)), new FieldMappingUnit(new FieldInfo("name", StringFormatInfo.INSTANCE), new FieldInfo("name_out", StringFormatInfo.INSTANCE)) }));
    Transformer transformer = new Transformer(transformationInfo, sourceFieldInfos, sinkFieldInfos);
    transformer.open(new Configuration());
    Row input = Row.of("name", 29, 179);
    ListCollector<Row> collector = new ListCollector<>();
    transformer.processElement(input, null, collector);
    assertEquals(Row.of(29, "name"), collector.getInnerList().get(0));
}

Example 4 with TransformationInfo

use of org.apache.inlong.sort.protocol.transformation.TransformationInfo in project incubator-inlong by apache.

the class TransformerTest method testTransformWithNotExistSourceFieldName.

@Test
public void testTransformWithNotExistSourceFieldName() {
    TransformationInfo transformationInfo = new TransformationInfo(new FieldMappingRule(new FieldMappingUnit[] { new FieldMappingUnit(new FieldInfo("age", StringFormatInfo.INSTANCE), new FieldInfo("age_out", StringFormatInfo.INSTANCE)), new FieldMappingUnit(// not exist source field name
    new FieldInfo("name_not_exist", StringFormatInfo.INSTANCE), new FieldInfo("name_out", StringFormatInfo.INSTANCE)) }));
    Transformer transformer = new Transformer(transformationInfo, sourceFieldInfos, sinkFieldInfos);
    transformer.open(new Configuration());
    Row input = Row.of("name", 29, 179);
    ListCollector<Row> collector = new ListCollector<>();
    transformer.processElement(input, null, collector);
    assertEquals(Row.of(29, null), collector.getInnerList().get(0));
}

Example 5 with TransformationInfo

use of org.apache.inlong.sort.protocol.transformation.TransformationInfo in project incubator-inlong by apache.

the class CommonOperateService method createDataFlow.

/**
 * Create dataflow info for sort.
 */
public DataFlowInfo createDataFlow(InlongGroupInfo groupInfo, SinkResponse sinkResponse) {
    String groupId = sinkResponse.getInlongGroupId();
    String streamId = sinkResponse.getInlongStreamId();
    // TODO Support all source type, include AUTO_PUSH.
    List<SourceResponse> sourceList = streamSourceService.listSource(groupId, streamId);
    if (CollectionUtils.isEmpty(sourceList)) {
        throw new WorkflowListenerException(String.format("Source not found by groupId=%s and streamId=%s", groupId, streamId));
    }
    // Get all field info
    List<FieldInfo> sourceFields = new ArrayList<>();
    List<FieldInfo> sinkFields = new ArrayList<>();
    String partition = null;
    if (SinkType.forType(sinkResponse.getSinkType()) == SinkType.HIVE) {
        HiveSinkResponse hiveSink = (HiveSinkResponse) sinkResponse;
        partition = hiveSink.getPrimaryPartition();
    }
    // TODO Support more than one source and one sink
    final SourceResponse sourceResponse = sourceList.get(0);
    boolean isAllMigration = SourceInfoUtils.isBinlogAllMigration(sourceResponse);
    FieldMappingRule fieldMappingRule = FieldInfoUtils.createFieldInfo(isAllMigration, sinkResponse.getFieldList(), sourceFields, sinkFields, partition);
    // Get source info
    String masterAddress = getSpecifiedParam(Constant.TUBE_MASTER_URL);
    PulsarClusterInfo pulsarCluster = getPulsarClusterInfo(groupInfo.getMiddlewareType());
    InlongStreamInfo streamInfo = streamService.get(groupId, streamId);
    SourceInfo sourceInfo = SourceInfoUtils.createSourceInfo(pulsarCluster, masterAddress, clusterBean, groupInfo, streamInfo, sourceResponse, sourceFields);
    // Get sink info
    SinkInfo sinkInfo = SinkInfoUtils.createSinkInfo(sourceResponse, sinkResponse, sinkFields);
    // Get transformation info
    TransformationInfo transInfo = new TransformationInfo(fieldMappingRule);
    // Get properties
    Map<String, Object> properties = new HashMap<>();
    if (MapUtils.isNotEmpty(sinkResponse.getProperties())) {
        properties.putAll(sinkResponse.getProperties());
    }
    properties.put(Constant.DATA_FLOW_GROUP_ID_KEY, groupId);
    return new DataFlowInfo(sinkResponse.getId(), sourceInfo, transInfo, sinkInfo, properties);
}