Examples with PulsarSourceInfo org.apache.inlong.sort.protocol.source.PulsarSourceInfo used on opensource projects

Example 1 with PulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.PulsarSourceInfo in project incubator-inlong by apache.

the class SourceInfoUtils method createPulsarSourceInfo.

/**
 * Create source info for Pulsar
 */
private static SourceInfo createPulsarSourceInfo(PulsarClusterInfo pulsarCluster, ClusterBean clusterBean, InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, DeserializationInfo deserializationInfo, List<FieldInfo> fieldInfos) {
    String topicName = streamInfo.getMqResourceObj();
    InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) groupInfo.getMqExtInfo();
    String tenant = clusterBean.getDefaultTenant();
    if (StringUtils.isNotEmpty(pulsarInfo.getTenant())) {
        tenant = pulsarInfo.getTenant();
    }
    final String namespace = groupInfo.getMqResourceObj();
    // Full name of topic in Pulsar
    final String fullTopicName = "persistent://" + tenant + "/" + namespace + "/" + topicName;
    final String consumerGroup = clusterBean.getAppName() + "_" + topicName + "_consumer_group";
    FieldInfo[] fieldInfosArr = fieldInfos.toArray(new FieldInfo[0]);
    String type = pulsarCluster.getType();
    if (StringUtils.isNotEmpty(type) && Constant.MIDDLEWARE_TDMQ_PULSAR.equals(type)) {
        return new TDMQPulsarSourceInfo(pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, pulsarCluster.getToken(), deserializationInfo, fieldInfosArr);
    } else {
        return new PulsarSourceInfo(pulsarCluster.getAdminUrl(), pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, deserializationInfo, fieldInfosArr, pulsarCluster.getToken());
    }
}

Example 2 with PulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.PulsarSourceInfo in project incubator-inlong by apache.

the class MultiTenancyInLongMsgMixedDeserializer method updateDataFlow.

@Override
public void updateDataFlow(DataFlowInfo dataFlowInfo) {
    if (!isInLongMsgDataFlow(dataFlowInfo)) {
        return;
    }
    final InLongMsgDeserializationInfo inLongMsgDeserializationInfo = (InLongMsgDeserializationInfo) dataFlowInfo.getSourceInfo().getDeserializationInfo();
    Pair<AbstractInLongMsgMixedFormatDeserializer, InLongMsgMixedFormatConverter> allDeserializer = generateDeserializer(dataFlowInfo.getSourceInfo().getFields(), inLongMsgDeserializationInfo);
    final AbstractInLongMsgMixedFormatDeserializer preDeserializer = allDeserializer.getLeft();
    final InLongMsgMixedFormatConverter deserializer = allDeserializer.getRight();
    final String topic;
    if (dataFlowInfo.getSourceInfo() instanceof TubeSourceInfo) {
        topic = ((TubeSourceInfo) dataFlowInfo.getSourceInfo()).getTopic();
    } else if (dataFlowInfo.getSourceInfo() instanceof PulsarSourceInfo) {
        topic = ((PulsarSourceInfo) dataFlowInfo.getSourceInfo()).getTopic();
    } else {
        throw new UnsupportedOperationException("Unknown source type " + dataFlowInfo.getSourceInfo());
    }
    final InLongMsgMixedDeserializer mixedDeserializer = mixedDeserializerMap.computeIfAbsent(topic, key -> new InLongMsgMixedDeserializer());
    mixedDeserializer.updateDataFlow(dataFlowInfo.getId(), inLongMsgDeserializationInfo.getTid(), preDeserializer, deserializer);
}

Example 3 with PulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.PulsarSourceInfo in project incubator-inlong by apache.

the class Entrance method buildSourceStream.

private static DataStream<SerializedRecord> buildSourceStream(StreamExecutionEnvironment env, Configuration config, SourceInfo sourceInfo, Map<String, Object> properties) {
    final String sourceType = checkNotNull(config.getString(Constants.SOURCE_TYPE));
    final int sourceParallelism = config.getInteger(Constants.SOURCE_PARALLELISM);
    final boolean orderlyOutput = config.getBoolean(Constants.JOB_ORDERLY_OUTPUT);
    DataStream<SerializedRecord> sourceStream;
    if (sourceType.equals(Constants.SOURCE_TYPE_PULSAR)) {
        checkState(sourceInfo instanceof PulsarSourceInfo);
        PulsarSourceInfo pulsarSourceInfo = (PulsarSourceInfo) sourceInfo;
        sourceStream = env.addSource(buildPulsarSource(pulsarSourceInfo, config, properties)).uid(Constants.SOURCE_UID).name("Pulsar source").setParallelism(sourceParallelism);
    } else if (sourceType.equals(Constants.SOURCE_TYPE_TDMQ_PULSAR)) {
        checkState(sourceInfo instanceof TDMQPulsarSourceInfo);
        TDMQPulsarSourceInfo tdmqPulsarSourceInfo = (TDMQPulsarSourceInfo) sourceInfo;
        sourceStream = env.addSource(buildTDMQPulsarSource(tdmqPulsarSourceInfo, config, properties)).uid(Constants.SOURCE_UID).name("TDMQ Pulsar source").setParallelism(sourceParallelism);
    } else {
        throw new IllegalArgumentException("Unsupported source type " + sourceType);
    }
    if (orderlyOutput) {
        return sourceStream.forward();
    } else {
        return sourceStream.rebalance();
    }
}

Example 4 with PulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.PulsarSourceInfo in project incubator-inlong by apache.

the class PulsarTestMetaManagerUtil method prepareDataFlowInfo.

@Override
public DataFlowInfo prepareDataFlowInfo(long dataFlowId, String... args) {
    FieldInfo[] pulsarFields = new FieldInfo[] { new FieldInfo("f1", StringFormatInfo.INSTANCE), new FieldInfo("f2", StringFormatInfo.INSTANCE) };
    Map<String, Object> config = new HashMap<>();
    config.put("consumer.bootstrap-mode", "earliest");
    return new DataFlowInfo(dataFlowId, new PulsarSourceInfo(args[0], args[1], args[2], args[3], new CsvDeserializationInfo(','), pulsarFields, null), new HiveSinkInfo(new FieldInfo[0], "testServerJdbcUrl", "testDatabaseName", "testTableName", "testUsername", "testPassword", "testDataPath", new HivePartitionInfo[0], new TextFileFormat(',')), config);
}

Example 5 with PulsarSourceInfo

use of org.apache.inlong.sort.protocol.source.PulsarSourceInfo in project incubator-inlong by apache.

the class MultiTopicPulsarSourceFunction method processEvent.

private void processEvent(SourceEvent sourceEvent) throws Exception {
    SourceEventType sourceEventType = sourceEvent.getSourceEventType();
    PulsarSourceInfo pulsarSourceInfo = (PulsarSourceInfo) sourceEvent.getSourceInfo();
    Map<String, Object> properties = sourceEvent.getProperties();
    long dataFlowId = sourceEvent.getDataFlowId();
    switch(sourceEventType) {
        case ADDED:
            PulsarSourceFunction<SerializedRecord> pulsarSourceFunction = generateSourceFunction(dataFlowId, properties, pulsarSourceInfo);
            pulsarConsumer.addPulsarSource(dataFlowId, pulsarSourceFunction);
            break;
        case UPDATE:
            PulsarSourceFunction<SerializedRecord> updateSourceFunction = generateSourceFunction(dataFlowId, properties, pulsarSourceInfo);
            pulsarConsumer.updatePulsarSource(dataFlowId, updateSourceFunction);
            break;
        case REMOVED:
            pulsarConsumer.removePulsarSource(dataFlowId);
            break;
        default:
            LOG.error("Unknown source event type {}", sourceEvent.getSourceEventType());
            throw new RuntimeException("Unknown source event type " + sourceEvent.getSourceEventType());
    }
}