Examples with InlongStreamEntity org.apache.inlong.manager.dao.entity.InlongStreamEntity used on opensource projects

Example 1 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class CreateHiveTableEventSelector method accept.

@Override
public boolean accept(WorkflowContext context) {
    ProcessForm processForm = context.getProcessForm();
    if (!(processForm instanceof GroupResourceProcessForm)) {
        return false;
    }
    GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
    if (form.getGroupInfo() == null || StringUtils.isEmpty(form.getGroupInfo().getInlongGroupId())) {
        return false;
    }
    String groupId = form.getInlongGroupId();
    List<String> dsForHive = sinkService.getExistsStreamIdList(groupId, Constant.SINK_HIVE, streamMapper.selectByGroupId(groupId).stream().map(InlongStreamEntity::getInlongStreamId).collect(Collectors.toList()));
    if (CollectionUtils.isEmpty(dsForHive)) {
        log.warn("groupId={} streamId={} does not have sink, skip to create hive table ", groupId, form.getInlongStreamId());
        return true;
    }
    return false;
}

Example 2 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class CreatePulsarTopicForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (groupInfo == null || streamEntity == null) {
        throw new WorkflowListenerException("inlong group or inlong stream not found with groupId=" + groupId + ", streamId=" + streamId);
    }
    log.info("begin to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            String pulsarTopic = streamEntity.getMqResourceObj();
            this.createTopic(groupInfo, pulsarTopic, pulsarClusterInfo);
        }
    } catch (Exception e) {
        log.error("create pulsar topic error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar topic error for groupId=" + groupId + ", streamId=" + streamId);
    }
    log.info("success to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}

Example 3 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class CreatePulsarGroupTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    if (groupInfo == null) {
        log.error("inlong group not found with groupId={}", groupId);
        throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
    }
    // For Pulsar, each Stream corresponds to a Topic
    List<InlongStreamEntity> streamEntities = streamMapper.selectByGroupId(groupId);
    if (streamEntities == null || streamEntities.isEmpty()) {
        log.warn("inlong stream is empty for groupId={}, skip to create pulsar subscription", groupId);
        return ListenerResult.success();
    }
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        String tenant = clusterBean.getDefaultTenant();
        String namespace = groupInfo.getMqResourceObj();
        for (InlongStreamEntity streamEntity : streamEntities) {
            PulsarTopicBean topicBean = new PulsarTopicBean();
            topicBean.setTenant(tenant);
            topicBean.setNamespace(namespace);
            String topic = streamEntity.getMqResourceObj();
            topicBean.setTopicName(topic);
            List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
            // Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
            for (String cluster : pulsarClusters) {
                String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
                PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
                try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
                    boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
                    if (!exist) {
                        String topicFull = tenant + "/" + namespace + "/" + topic;
                        log.error("topic={} not exists in {}", topicFull, serviceUrl);
                        throw new WorkflowListenerException("topic=" + topicFull + " not exists in " + serviceUrl);
                    }
                    // Consumer naming rules: sortAppName_topicName_consumer_group
                    String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
                    pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
                    // Insert the consumption data into the consumption table
                    consumptionService.saveSortConsumption(groupInfo, topic, subscription);
                }
            }
        }
    } catch (Exception e) {
        log.error("create pulsar subscription error for groupId={}", groupId);
        throw new WorkflowListenerException("create pulsar subscription error: " + e.getMessage());
    }
    log.info("success to create pulsar subscription for groupId={}", groupId);
    return ListenerResult.success();
}

Example 4 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongGroupProcessOperation method genUpdateGroupProcessForm.

private UpdateGroupProcessForm genUpdateGroupProcessForm(String groupId, OperateType operateType) {
    InlongGroupInfo groupInfo = groupService.get(groupId);
    UpdateGroupProcessForm form = new UpdateGroupProcessForm();
    if (OperateType.RESTART == operateType) {
        List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupInfo.getInlongGroupId());
        List<InlongStreamInfo> inlongStreamInfoList = CommonBeanUtils.copyListProperties(inlongStreamEntityList, InlongStreamInfo::new);
        form.setInlongStreamInfoList(inlongStreamInfoList);
    }
    form.setGroupInfo(groupInfo);
    form.setOperateType(operateType);
    return form;
}

Example 5 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongStreamServiceImpl method getBriefList.

@Override
public List<StreamBriefResponse> getBriefList(String groupId) {
    LOGGER.debug("begin to get inlong stream brief list by groupId={}", groupId);
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    List<InlongStreamEntity> entityList = streamMapper.selectByGroupId(groupId);
    List<StreamBriefResponse> briefInfoList = CommonBeanUtils.copyListProperties(entityList, StreamBriefResponse::new);
    // Query stream sinks based on groupId and streamId
    for (StreamBriefResponse briefInfo : briefInfoList) {
        String streamId = briefInfo.getInlongStreamId();
        List<SinkBriefResponse> sinkList = sinkService.listBrief(groupId, streamId);
        briefInfo.setSinkList(sinkList);
    }
    LOGGER.info("success to get inlong stream brief list for groupId={}", groupId);
    return briefInfoList;
}