Examples with PulsarClusterInfo org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo used on opensource projects

Example 6 with PulsarClusterInfo

use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.

the class ConsumptionCompleteProcessListener method createPulsarTopicMessage.

/**
 * Create Pulsar consumption information, including cross-regional cycle creation of consumption groups
 */
private void createPulsarTopicMessage(ConsumptionEntity entity) {
    String groupId = entity.getInlongGroupId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    Preconditions.checkNotNull(groupInfo, "inlong group not found for groupId=" + groupId);
    String mqResourceObj = groupInfo.getMqResourceObj();
    Preconditions.checkNotNull(mqResourceObj, "mq resource cannot empty for groupId=" + groupId);
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(entity.getMiddlewareType());
    try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        PulsarTopicBean topicMessage = new PulsarTopicBean();
        String tenant = clusterBean.getDefaultTenant();
        topicMessage.setTenant(tenant);
        topicMessage.setNamespace(mqResourceObj);
        // If cross-regional replication is started, each cluster needs to create consumer groups in cycles
        String consumerGroup = entity.getConsumerGroupId();
        List<String> clusters = PulsarUtils.getPulsarClusters(pulsarAdmin);
        List<String> topics = Arrays.asList(entity.getTopic().split(","));
        this.createPulsarSubscription(pulsarAdmin, consumerGroup, topicMessage, clusters, topics, globalCluster);
    } catch (Exception e) {
        log.error("create pulsar topic failed", e);
        throw new WorkflowListenerException("failed to create pulsar topic for groupId=" + groupId + ", reason: " + e.getMessage());
    }
}

Example 7 with PulsarClusterInfo

use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.

the class CommonOperateService method getPulsarClusterInfo.

/**
 * Get Pulsar cluster by the given type.
 *
 * @return Pulsar cluster info.
 */
public PulsarClusterInfo getPulsarClusterInfo(String type) {
    ThirdPartyClusterEntity clusterEntity = getMQCluster(type);
    if (clusterEntity == null || StringUtils.isBlank(clusterEntity.getExtParams())) {
        throw new BusinessException("pulsar cluster or pulsar ext params is empty");
    }
    Map<String, String> configParams = JsonUtils.parse(clusterEntity.getExtParams(), Map.class);
    PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().brokerServiceUrl(clusterEntity.getUrl()).token(clusterEntity.getToken()).build();
    String adminUrl = configParams.get(Constant.PULSAR_ADMINURL);
    Preconditions.checkNotNull(adminUrl, "adminUrl is empty, check third party cluster table");
    pulsarClusterInfo.setAdminUrl(adminUrl);
    pulsarClusterInfo.setType(clusterEntity.getType());
    return pulsarClusterInfo;
}

Example 8 with PulsarClusterInfo

use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.

the class CommonOperateService method createDataFlow.

/**
 * Create dataflow info for sort.
 */
public DataFlowInfo createDataFlow(InlongGroupInfo groupInfo, SinkResponse sinkResponse) {
    String groupId = sinkResponse.getInlongGroupId();
    String streamId = sinkResponse.getInlongStreamId();
    // TODO Support all source type, include AUTO_PUSH.
    List<SourceResponse> sourceList = streamSourceService.listSource(groupId, streamId);
    if (CollectionUtils.isEmpty(sourceList)) {
        throw new WorkflowListenerException(String.format("Source not found by groupId=%s and streamId=%s", groupId, streamId));
    }
    // Get all field info
    List<FieldInfo> sourceFields = new ArrayList<>();
    List<FieldInfo> sinkFields = new ArrayList<>();
    String partition = null;
    if (SinkType.forType(sinkResponse.getSinkType()) == SinkType.HIVE) {
        HiveSinkResponse hiveSink = (HiveSinkResponse) sinkResponse;
        partition = hiveSink.getPrimaryPartition();
    }
    // TODO Support more than one source and one sink
    final SourceResponse sourceResponse = sourceList.get(0);
    boolean isAllMigration = SourceInfoUtils.isBinlogAllMigration(sourceResponse);
    FieldMappingRule fieldMappingRule = FieldInfoUtils.createFieldInfo(isAllMigration, sinkResponse.getFieldList(), sourceFields, sinkFields, partition);
    // Get source info
    String masterAddress = getSpecifiedParam(Constant.TUBE_MASTER_URL);
    PulsarClusterInfo pulsarCluster = getPulsarClusterInfo(groupInfo.getMiddlewareType());
    InlongStreamInfo streamInfo = streamService.get(groupId, streamId);
    SourceInfo sourceInfo = SourceInfoUtils.createSourceInfo(pulsarCluster, masterAddress, clusterBean, groupInfo, streamInfo, sourceResponse, sourceFields);
    // Get sink info
    SinkInfo sinkInfo = SinkInfoUtils.createSinkInfo(sourceResponse, sinkResponse, sinkFields);
    // Get transformation info
    TransformationInfo transInfo = new TransformationInfo(fieldMappingRule);
    // Get properties
    Map<String, Object> properties = new HashMap<>();
    if (MapUtils.isNotEmpty(sinkResponse.getProperties())) {
        properties.putAll(sinkResponse.getProperties());
    }
    properties.put(Constant.DATA_FLOW_GROUP_ID_KEY, groupId);
    return new DataFlowInfo(sinkResponse.getId(), sourceInfo, transInfo, sinkInfo, properties);
}

Example 9 with PulsarClusterInfo

use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.

the class CreatePulsarGroupForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    if (groupInfo == null) {
        log.error("inlong group not found with groupId={}", groupId);
        throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
    }
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (streamEntity == null) {
        log.warn("inlong stream is empty for group={}, stream={}, skip to create pulsar group", groupId, streamId);
        return ListenerResult.success();
    }
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        // Query data sink info based on groupId and streamId
        List<String> sinkTypeList = sinkService.getSinkTypeList(groupId, streamId);
        if (sinkTypeList == null || sinkTypeList.size() == 0) {
            log.warn("sink info is empty for groupId={}, streamId={}, skip to create pulsar group", groupId, streamId);
            return ListenerResult.success();
        }
        PulsarTopicBean topicBean = new PulsarTopicBean();
        topicBean.setTenant(clusterBean.getDefaultTenant());
        topicBean.setNamespace(groupInfo.getMqResourceObj());
        String topic = streamEntity.getMqResourceObj();
        topicBean.setTopicName(topic);
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        // Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
        String tenant = clusterBean.getDefaultTenant();
        String namespace = groupInfo.getMqResourceObj();
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
                boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
                if (!exist) {
                    String fullTopic = tenant + "/" + namespace + "/" + topic;
                    log.error("topic={} not exists in {}", fullTopic, pulsarAdmin.getServiceUrl());
                    throw new BusinessException("topic=" + fullTopic + " not exists in " + serviceUrl);
                }
                // Consumer naming rules: sortAppName_topicName_consumer_group
                String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
                pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
                // Insert the consumption data into the consumption table
                consumptionService.saveSortConsumption(groupInfo, topic, subscription);
            }
        }
    } catch (Exception e) {
        log.error("create pulsar subscription error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar subscription error, reason: " + e.getMessage());
    }
    log.info("finish to create single pulsar subscription for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}