Examples with PulsarTopicBean org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean used on opensource projects

Example 1 with PulsarTopicBean

use of org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean in project incubator-inlong by apache.

the class CreatePulsarGroupTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    if (groupInfo == null) {
        log.error("inlong group not found with groupId={}", groupId);
        throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
    }
    // For Pulsar, each Stream corresponds to a Topic
    List<InlongStreamEntity> streamEntities = streamMapper.selectByGroupId(groupId);
    if (streamEntities == null || streamEntities.isEmpty()) {
        log.warn("inlong stream is empty for groupId={}, skip to create pulsar subscription", groupId);
        return ListenerResult.success();
    }
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        String tenant = clusterBean.getDefaultTenant();
        String namespace = groupInfo.getMqResourceObj();
        for (InlongStreamEntity streamEntity : streamEntities) {
            PulsarTopicBean topicBean = new PulsarTopicBean();
            topicBean.setTenant(tenant);
            topicBean.setNamespace(namespace);
            String topic = streamEntity.getMqResourceObj();
            topicBean.setTopicName(topic);
            List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
            // Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
            for (String cluster : pulsarClusters) {
                String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
                PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
                try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
                    boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
                    if (!exist) {
                        String topicFull = tenant + "/" + namespace + "/" + topic;
                        log.error("topic={} not exists in {}", topicFull, serviceUrl);
                        throw new WorkflowListenerException("topic=" + topicFull + " not exists in " + serviceUrl);
                    }
                    // Consumer naming rules: sortAppName_topicName_consumer_group
                    String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
                    pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
                    // Insert the consumption data into the consumption table
                    consumptionService.saveSortConsumption(groupInfo, topic, subscription);
                }
            }
        }
    } catch (Exception e) {
        log.error("create pulsar subscription error for groupId={}", groupId);
        throw new WorkflowListenerException("create pulsar subscription error: " + e.getMessage());
    }
    log.info("success to create pulsar subscription for groupId={}", groupId);
    return ListenerResult.success();
}

Example 2 with PulsarTopicBean

use of org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean in project incubator-inlong by apache.

the class CreatePulsarTopicForStreamTaskListener method createTopic.

private void createTopic(InlongGroupInfo bizInfo, String pulsarTopic, PulsarClusterInfo pulsarClusterInfo) throws Exception {
    Integer partitionNum = bizInfo.getTopicPartitionNum();
    int partition = 0;
    if (partitionNum != null && partitionNum > 0) {
        partition = partitionNum;
    }
    try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
        PulsarTopicBean topicBean = PulsarTopicBean.builder().tenant(clusterBean.getDefaultTenant()).namespace(bizInfo.getMqResourceObj()).topicName(pulsarTopic).numPartitions(partition).queueModule(bizInfo.getQueueModule()).build();
        pulsarOptService.createTopic(pulsarAdmin, topicBean);
    }
}

Example 3 with PulsarTopicBean

use of org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean in project incubator-inlong by apache.

the class ConsumptionCompleteProcessListener method createPulsarTopicMessage.

/**
 * Create Pulsar consumption information, including cross-regional cycle creation of consumption groups
 */
private void createPulsarTopicMessage(ConsumptionEntity entity) {
    String groupId = entity.getInlongGroupId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    Preconditions.checkNotNull(groupInfo, "inlong group not found for groupId=" + groupId);
    String mqResourceObj = groupInfo.getMqResourceObj();
    Preconditions.checkNotNull(mqResourceObj, "mq resource cannot empty for groupId=" + groupId);
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(entity.getMiddlewareType());
    try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        PulsarTopicBean topicMessage = new PulsarTopicBean();
        String tenant = clusterBean.getDefaultTenant();
        topicMessage.setTenant(tenant);
        topicMessage.setNamespace(mqResourceObj);
        // If cross-regional replication is started, each cluster needs to create consumer groups in cycles
        String consumerGroup = entity.getConsumerGroupId();
        List<String> clusters = PulsarUtils.getPulsarClusters(pulsarAdmin);
        List<String> topics = Arrays.asList(entity.getTopic().split(","));
        this.createPulsarSubscription(pulsarAdmin, consumerGroup, topicMessage, clusters, topics, globalCluster);
    } catch (Exception e) {
        log.error("create pulsar topic failed", e);
        throw new WorkflowListenerException("failed to create pulsar topic for groupId=" + groupId + ", reason: " + e.getMessage());
    }
}

Example 4 with PulsarTopicBean

use of org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean in project incubator-inlong by apache.

the class CreatePulsarResourceTaskListener method createPulsarProcess.

/**
 * Create Pulsar tenant, namespace and topic
 */
private void createPulsarProcess(InlongGroupInfo groupInfo, PulsarClusterInfo pulsarClusterInfo) throws Exception {
    String groupId = groupInfo.getInlongGroupId();
    log.info("begin to create pulsar resource for groupId={} in cluster={}", groupId, pulsarClusterInfo);
    String namespace = groupInfo.getMqResourceObj();
    Preconditions.checkNotNull(namespace, "pulsar namespace cannot be empty for groupId=" + groupId);
    String queueModule = groupInfo.getQueueModule();
    Preconditions.checkNotNull(queueModule, "queue module cannot be empty for groupId=" + groupId);
    String tenant = clusterBean.getDefaultTenant();
    try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
        // create pulsar tenant
        pulsarOptService.createTenant(pulsarAdmin, tenant);
        // create pulsar namespace
        InlongGroupPulsarEntity entity = groupPulsarMapper.selectByGroupId(groupId);
        pulsarOptService.createNamespace(pulsarAdmin, entity, tenant, namespace);
        // create pulsar topic
        Integer partitionNum = groupInfo.getTopicPartitionNum();
        List<InlongStreamTopicResponse> streamTopicList = streamMapper.selectTopicList(groupId);
        PulsarTopicBean topicBean = PulsarTopicBean.builder().tenant(tenant).namespace(namespace).numPartitions(partitionNum).queueModule(queueModule).build();
        for (InlongStreamTopicResponse topicVO : streamTopicList) {
            topicBean.setTopicName(topicVO.getMqResourceObj());
            pulsarOptService.createTopic(pulsarAdmin, topicBean);
        }
    }
    log.info("finish to create pulsar resource for groupId={}, service http url={}", groupId, pulsarClusterInfo.getAdminUrl());
}

Example 5 with PulsarTopicBean

use of org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean in project incubator-inlong by apache.

the class CreatePulsarGroupForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    if (groupInfo == null) {
        log.error("inlong group not found with groupId={}", groupId);
        throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
    }
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (streamEntity == null) {
        log.warn("inlong stream is empty for group={}, stream={}, skip to create pulsar group", groupId, streamId);
        return ListenerResult.success();
    }
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        // Query data sink info based on groupId and streamId
        List<String> sinkTypeList = sinkService.getSinkTypeList(groupId, streamId);
        if (sinkTypeList == null || sinkTypeList.size() == 0) {
            log.warn("sink info is empty for groupId={}, streamId={}, skip to create pulsar group", groupId, streamId);
            return ListenerResult.success();
        }
        PulsarTopicBean topicBean = new PulsarTopicBean();
        topicBean.setTenant(clusterBean.getDefaultTenant());
        topicBean.setNamespace(groupInfo.getMqResourceObj());
        String topic = streamEntity.getMqResourceObj();
        topicBean.setTopicName(topic);
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        // Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
        String tenant = clusterBean.getDefaultTenant();
        String namespace = groupInfo.getMqResourceObj();
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
                boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
                if (!exist) {
                    String fullTopic = tenant + "/" + namespace + "/" + topic;
                    log.error("topic={} not exists in {}", fullTopic, pulsarAdmin.getServiceUrl());
                    throw new BusinessException("topic=" + fullTopic + " not exists in " + serviceUrl);
                }
                // Consumer naming rules: sortAppName_topicName_consumer_group
                String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
                pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
                // Insert the consumption data into the consumption table
                consumptionService.saveSortConsumption(groupInfo, topic, subscription);
            }
        }
    } catch (Exception e) {
        log.error("create pulsar subscription error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar subscription error, reason: " + e.getMessage());
    }
    log.info("finish to create single pulsar subscription for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}