Examples with DeserializationInfo org.apache.inlong.sort.protocol.deserialization.DeserializationInfo used on opensource projects

Example 1 with DeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.DeserializationInfo in project incubator-inlong by apache.

the class PushSortConfigListener method getSourceInfo.

/**
 * Get source info for DataFlowInfo
 */
@Deprecated
private SourceInfo getSourceInfo(InlongGroupInfo groupInfo, String streamId, List<StreamSinkFieldEntity> fieldList) {
    DeserializationInfo deserializationInfo = null;
    String groupId = groupInfo.getInlongGroupId();
    InlongStreamInfo streamInfo = streamService.get(groupId, streamId);
    boolean isDbType = Constant.DATA_SOURCE_DB.equals(streamInfo.getDataType());
    if (!isDbType) {
        // FILE and auto push source, the data format is TEXT or KEY-VALUE, temporarily use InLongMsgCsv
        String dataType = streamInfo.getDataType();
        if (Constant.DATA_TYPE_TEXT.equalsIgnoreCase(dataType) || Constant.DATA_TYPE_KEY_VALUE.equalsIgnoreCase(dataType)) {
            // Use the field separator from the inlong stream
            char separator = (char) Integer.parseInt(streamInfo.getDataSeparator());
            // TODO support escape
            /*Character escape = null;
                if (info.getDataEscapeChar() != null) {
                    escape = info.getDataEscapeChar().charAt(0);
                }*/
            // Whether to delete the first separator, the default is false for the time being
            deserializationInfo = new InLongMsgCsvDeserializationInfo(streamId, separator);
        }
    }
    // The number and order of the source fields must be the same as the target fields
    return null;
}

Example 2 with DeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.DeserializationInfo in project incubator-inlong by apache.

the class SourceInfoUtils method createSourceInfo.

/**
 * Create source info for DataFlowInfo.
 */
public static SourceInfo createSourceInfo(PulsarClusterInfo pulsarCluster, String masterAddress, ClusterBean clusterBean, InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, SourceResponse sourceResponse, List<FieldInfo> sourceFields) {
    String mqType = groupInfo.getMiddlewareType();
    DeserializationInfo deserializationInfo = SerializationUtils.createDeserialInfo(sourceResponse, streamInfo);
    SourceInfo sourceInfo;
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        sourceInfo = createPulsarSourceInfo(pulsarCluster, clusterBean, groupInfo, streamInfo, deserializationInfo, sourceFields);
    } else if (Constant.MIDDLEWARE_TUBE.equals(mqType)) {
        // InlongGroupInfo groupInfo, String masterAddress,
        sourceInfo = createTubeSourceInfo(groupInfo, masterAddress, clusterBean, deserializationInfo, sourceFields);
    } else {
        throw new WorkflowListenerException(String.format("Unsupported middleware {%s}", mqType));
    }
    return sourceInfo;
}

Example 3 with DeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.DeserializationInfo in project incubator-inlong by apache.

the class CommonUtils method getInLongGroupIdAndStreamId.

public static Pair<String, String> getInLongGroupIdAndStreamId(DataFlowInfo dataFlowInfo) {
    String groupId = "";
    String streamId = "";
    if (dataFlowInfo != null) {
        // Get group id
        Map<String, Object> properties = dataFlowInfo.getProperties();
        if (properties != null) {
            groupId = properties.getOrDefault(Constants.INLONG_GROUP_ID, "").toString();
        }
        // Get stream id
        final DeserializationInfo deserializationInfo = dataFlowInfo.getSourceInfo().getDeserializationInfo();
        if (deserializationInfo instanceof InLongMsgDeserializationInfo) {
            streamId = ((InLongMsgDeserializationInfo) deserializationInfo).getTid();
        }
    }
    return Pair.of(groupId, streamId);
}

Example 4 with DeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.DeserializationInfo in project incubator-inlong by apache.

the class MultiTenancyDeserializer method updateDataFlow.

@Override
public void updateDataFlow(DataFlowInfo dataFlowInfo) {
    final DeserializationInfo deserializationInfo = dataFlowInfo.getSourceInfo().getDeserializationInfo();
    final Deserializer<SerializedRecord, Record> deserializer = generateDeserializer(dataFlowInfo.getSourceInfo().getFields(), deserializationInfo);
    deserializers.put(dataFlowInfo.getId(), deserializer);
}

Example 5 with DeserializationInfo

use of org.apache.inlong.sort.protocol.deserialization.DeserializationInfo in project incubator-inlong by apache.

the class PulsarSourceInfoTest method testSerializeAndDeserialize.

@Test
public void testSerializeAndDeserialize() {
    FieldInfo[] pulsarFields = new FieldInfo[] { new FieldInfo("f1", StringFormatInfo.INSTANCE), new FieldInfo("f2", StringFormatInfo.INSTANCE) };
    DeserializationInfo deserializationInfo = new InLongMsgCsvDeserializationInfo("stream", ',');
    PulsarSourceInfo pulsarSourceInfo = new PulsarSourceInfo("http://127.0.0.1:8080", "pulsar://127.0.0.1:6650", "business", "consumer", deserializationInfo, pulsarFields, null);
    final ObjectMapper objectMapper = new ObjectMapper();
    try {
        byte[] values = objectMapper.writeValueAsBytes(pulsarSourceInfo);
        pulsarSourceInfo = objectMapper.readValue(values, PulsarSourceInfo.class);
        Assert.assertTrue(pulsarSourceInfo.getAdminUrl().equals("http://127.0.0.1:8080"));
        Assert.assertTrue(pulsarSourceInfo.getServiceUrl().equals("pulsar://127.0.0.1:6650"));
        Assert.assertTrue(pulsarSourceInfo.getTopic().equals("business"));
        Assert.assertTrue(pulsarSourceInfo.getSubscriptionName().equals("consumer"));
        Assert.assertTrue(pulsarSourceInfo.getDeserializationInfo() instanceof InLongMsgCsvDeserializationInfo);
        Assert.assertTrue(pulsarSourceInfo.getAuthentication() == null);
    } catch (JsonProcessingException e) {
        Assert.fail();
    } catch (IOException e) {
        Assert.fail();
    }
}