Examples with TDMQPulsarSourceInfo org.apache.inlong.sort.protocol.source.TDMQPulsarSourceInfo used on opensource projects

Example 1 with TDMQPulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.TDMQPulsarSourceInfo in project incubator-inlong by apache.

the class SourceInfoUtils method createPulsarSourceInfo.

/**
 * Create source info for Pulsar
 */
private static SourceInfo createPulsarSourceInfo(PulsarClusterInfo pulsarCluster, ClusterBean clusterBean, InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, DeserializationInfo deserializationInfo, List<FieldInfo> fieldInfos) {
    String topicName = streamInfo.getMqResourceObj();
    InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) groupInfo.getMqExtInfo();
    String tenant = clusterBean.getDefaultTenant();
    if (StringUtils.isNotEmpty(pulsarInfo.getTenant())) {
        tenant = pulsarInfo.getTenant();
    }
    final String namespace = groupInfo.getMqResourceObj();
    // Full name of topic in Pulsar
    final String fullTopicName = "persistent://" + tenant + "/" + namespace + "/" + topicName;
    final String consumerGroup = clusterBean.getAppName() + "_" + topicName + "_consumer_group";
    FieldInfo[] fieldInfosArr = fieldInfos.toArray(new FieldInfo[0]);
    String type = pulsarCluster.getType();
    if (StringUtils.isNotEmpty(type) && Constant.MIDDLEWARE_TDMQ_PULSAR.equals(type)) {
        return new TDMQPulsarSourceInfo(pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, pulsarCluster.getToken(), deserializationInfo, fieldInfosArr);
    } else {
        return new PulsarSourceInfo(pulsarCluster.getAdminUrl(), pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, deserializationInfo, fieldInfosArr, pulsarCluster.getToken());
    }
}

Example 2 with TDMQPulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.TDMQPulsarSourceInfo in project incubator-inlong by apache.

the class Entrance method buildSourceStream.

private static DataStream<SerializedRecord> buildSourceStream(StreamExecutionEnvironment env, Configuration config, SourceInfo sourceInfo, Map<String, Object> properties) {
    final String sourceType = checkNotNull(config.getString(Constants.SOURCE_TYPE));
    final int sourceParallelism = config.getInteger(Constants.SOURCE_PARALLELISM);
    final boolean orderlyOutput = config.getBoolean(Constants.JOB_ORDERLY_OUTPUT);
    DataStream<SerializedRecord> sourceStream;
    if (sourceType.equals(Constants.SOURCE_TYPE_PULSAR)) {
        checkState(sourceInfo instanceof PulsarSourceInfo);
        PulsarSourceInfo pulsarSourceInfo = (PulsarSourceInfo) sourceInfo;
        sourceStream = env.addSource(buildPulsarSource(pulsarSourceInfo, config, properties)).uid(Constants.SOURCE_UID).name("Pulsar source").setParallelism(sourceParallelism);
    } else if (sourceType.equals(Constants.SOURCE_TYPE_TDMQ_PULSAR)) {
        checkState(sourceInfo instanceof TDMQPulsarSourceInfo);
        TDMQPulsarSourceInfo tdmqPulsarSourceInfo = (TDMQPulsarSourceInfo) sourceInfo;
        sourceStream = env.addSource(buildTDMQPulsarSource(tdmqPulsarSourceInfo, config, properties)).uid(Constants.SOURCE_UID).name("TDMQ Pulsar source").setParallelism(sourceParallelism);
    } else {
        throw new IllegalArgumentException("Unsupported source type " + sourceType);
    }
    if (orderlyOutput) {
        return sourceStream.forward();
    } else {
        return sourceStream.rebalance();
    }
}