Search in sources :

Example 16 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreateHiveTableForStreamListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    log.info("begin create hive table for groupId={}, streamId={}", groupId, streamId);
    List<SinkForSortDTO> configList = sinkMapper.selectAllConfig(groupId, streamId);
    if (hiveTableOperator == null) {
        hiveTableOperator = (IHiveTableOperator) SpringContextUtils.getBean(IHiveTableOperator.BEAN_NAME, DefaultHiveTableOperator.class.getName());
    }
    hiveTableOperator.createHiveResource(groupId, configList);
    String result = "success to create hive table for group [" + groupId + "], stream [" + streamId + "]";
    log.info(result);
    return ListenerResult.success(result);
}
Also used : GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) SinkForSortDTO(org.apache.inlong.manager.common.pojo.sink.SinkForSortDTO)

Example 17 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreateTubeTopicTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    log.info("begin create tube topic for groupId={}", form.getInlongGroupId());
    String groupId = form.getInlongGroupId();
    try {
        InlongGroupInfo groupInfo = groupService.get(groupId);
        String topicName = groupInfo.getMqResourceObj();
        AddTubeMqTopicRequest request = new AddTubeMqTopicRequest();
        request.setUser("inlong-manager");
        AddTubeMqTopicRequest.AddTopicTasksBean tasksBean = new AddTubeMqTopicRequest.AddTopicTasksBean();
        tasksBean.setTopicName(topicName);
        request.setAddTopicTasks(Collections.singletonList(tasksBean));
        tubeMqOptService.createNewTopic(request);
        log.info("finish to create tube topic for groupId={}", groupId);
    } catch (Exception e) {
        log.error("create tube topic for groupId={} error, exception {} ", groupId, e.getMessage(), e);
    }
    return ListenerResult.success();
}
Also used : InlongGroupInfo(org.apache.inlong.manager.common.pojo.group.InlongGroupInfo) AddTubeMqTopicRequest(org.apache.inlong.manager.common.pojo.tubemq.AddTubeMqTopicRequest) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException)

Example 18 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreatePulsarGroupForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    if (groupInfo == null) {
        log.error("inlong group not found with groupId={}", groupId);
        throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
    }
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (streamEntity == null) {
        log.warn("inlong stream is empty for group={}, stream={}, skip to create pulsar group", groupId, streamId);
        return ListenerResult.success();
    }
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        // Query data sink info based on groupId and streamId
        List<String> sinkTypeList = sinkService.getSinkTypeList(groupId, streamId);
        if (sinkTypeList == null || sinkTypeList.size() == 0) {
            log.warn("sink info is empty for groupId={}, streamId={}, skip to create pulsar group", groupId, streamId);
            return ListenerResult.success();
        }
        PulsarTopicBean topicBean = new PulsarTopicBean();
        topicBean.setTenant(clusterBean.getDefaultTenant());
        topicBean.setNamespace(groupInfo.getMqResourceObj());
        String topic = streamEntity.getMqResourceObj();
        topicBean.setTopicName(topic);
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        // Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
        String tenant = clusterBean.getDefaultTenant();
        String namespace = groupInfo.getMqResourceObj();
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
                boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
                if (!exist) {
                    String fullTopic = tenant + "/" + namespace + "/" + topic;
                    log.error("topic={} not exists in {}", fullTopic, pulsarAdmin.getServiceUrl());
                    throw new BusinessException("topic=" + fullTopic + " not exists in " + serviceUrl);
                }
                // Consumer naming rules: sortAppName_topicName_consumer_group
                String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
                pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
                // Insert the consumption data into the consumption table
                consumptionService.saveSortConsumption(groupInfo, topic, subscription);
            }
        }
    } catch (Exception e) {
        log.error("create pulsar subscription error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar subscription error, reason: " + e.getMessage());
    }
    log.info("finish to create single pulsar subscription for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}
Also used : BusinessException(org.apache.inlong.manager.common.exceptions.BusinessException) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) InlongStreamEntity(org.apache.inlong.manager.dao.entity.InlongStreamEntity) PulsarClusterInfo(org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException) InlongGroupInfo(org.apache.inlong.manager.common.pojo.group.InlongGroupInfo) PulsarTopicBean(org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) BusinessException(org.apache.inlong.manager.common.exceptions.BusinessException) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException)

Example 19 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class StartCreateGroupProcessListener method listen.

/**
 * Initiate the process of creating inlong group resources after new inlong group access approved
 */
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    NewGroupProcessForm form = (NewGroupProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    GroupResourceProcessForm processForm = new GroupResourceProcessForm();
    processForm.setGroupInfo(groupService.get(groupId));
    String username = context.getApplicant();
    List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupId);
    List<InlongStreamInfo> inlongStreamInfoList = CommonBeanUtils.copyListProperties(inlongStreamEntityList, InlongStreamInfo::new);
    processForm.setInlongStreamInfoList(inlongStreamInfoList);
    workflowService.start(ProcessName.CREATE_GROUP_RESOURCE, username, processForm);
    return ListenerResult.success();
}
Also used : InlongStreamEntity(org.apache.inlong.manager.dao.entity.InlongStreamEntity) NewGroupProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.NewGroupProcessForm) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) InlongStreamInfo(org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo)

Example 20 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreateStreamWorkflowDefinition method defineProcess.

@Override
public WorkflowProcess defineProcess() {
    // Configuration process
    WorkflowProcess process = new WorkflowProcess();
    process.addListener(initGroupListener);
    process.addListener(streamFailedProcessListener);
    process.addListener(streamCompleteProcessListener);
    process.setType("Inlong stream access resource creation");
    process.setName(getProcessName().name());
    process.setDisplayName(getProcessName().getDisplayName());
    process.setFormClass(GroupResourceProcessForm.class);
    process.setVersion(1);
    process.setHidden(1);
    // Start node
    StartEvent startEvent = new StartEvent();
    process.setStartEvent(startEvent);
    // End node
    EndEvent endEvent = new EndEvent();
    process.setEndEvent(endEvent);
    ServiceTask createPulsarTopicTask = new ServiceTask();
    createPulsarTopicTask.setSkipResolver(c -> {
        GroupResourceProcessForm form = (GroupResourceProcessForm) c.getProcessForm();
        String mqType = form.getGroupInfo().getMiddlewareType();
        if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            return false;
        }
        log.warn("no need to create pulsar topic for groupId={}, streamId={}, as the middlewareType={}", form.getInlongGroupId(), form.getInlongStreamId(), mqType);
        return true;
    });
    createPulsarTopicTask.setName("createPulsarTopic");
    createPulsarTopicTask.setDisplayName("Stream-CreatePulsarTopic");
    createPulsarTopicTask.addListener(createPulsarTopicTaskListener);
    process.addTask(createPulsarTopicTask);
    ServiceTask createPulsarSubscriptionGroupTask = new ServiceTask();
    createPulsarSubscriptionGroupTask.setSkipResolver(c -> {
        GroupResourceProcessForm form = (GroupResourceProcessForm) c.getProcessForm();
        String mqType = form.getGroupInfo().getMiddlewareType();
        if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            return false;
        }
        log.warn("no need to create pulsar subscription for groupId={}, streamId={}, as the middlewareType={}", form.getInlongGroupId(), form.getInlongStreamId(), mqType);
        return true;
    });
    createPulsarSubscriptionGroupTask.setName("createPulsarSubscription");
    createPulsarSubscriptionGroupTask.setDisplayName("Stream-CreatePulsarSubscription");
    createPulsarSubscriptionGroupTask.addListener(createPulsarGroupTaskListener);
    process.addTask(createPulsarSubscriptionGroupTask);
    ServiceTask createHiveTableTask = new ServiceTask();
    createHiveTableTask.setSkipResolver(c -> {
        GroupResourceProcessForm form = (GroupResourceProcessForm) c.getProcessForm();
        String groupId = form.getInlongGroupId();
        String streamId = form.getInlongStreamId();
        List<String> dsForHive = sinkService.getExistsStreamIdList(groupId, Constant.SINK_HIVE, Collections.singletonList(streamId));
        if (CollectionUtils.isEmpty(dsForHive)) {
            log.warn("inlong group [{}] adn inlong stream [{}] does not have sink, skip create hive table", groupId, streamId);
            return true;
        }
        return false;
    });
    createHiveTableTask.setName("createHiveTable");
    createHiveTableTask.setDisplayName("Stream-CreateHiveTable");
    createHiveTableTask.addListener(createHiveTableListener);
    process.addTask(createHiveTableTask);
    ServiceTask pushSortConfig = new ServiceTask();
    pushSortConfig.setName("pushSortConfig");
    pushSortConfig.setDisplayName("Stream-PushSortConfig");
    pushSortConfig.addListener(pushSortConfigListener);
    process.addTask(pushSortConfig);
    startEvent.addNext(createPulsarTopicTask);
    createPulsarTopicTask.addNext(createPulsarSubscriptionGroupTask);
    createPulsarSubscriptionGroupTask.addNext(createHiveTableTask);
    createHiveTableTask.addNext(pushSortConfig);
    pushSortConfig.addNext(endEvent);
    return process;
}
Also used : ServiceTask(org.apache.inlong.manager.workflow.definition.ServiceTask) StartEvent(org.apache.inlong.manager.workflow.definition.StartEvent) EndEvent(org.apache.inlong.manager.workflow.definition.EndEvent) WorkflowProcess(org.apache.inlong.manager.workflow.definition.WorkflowProcess) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm)

Aggregations

GroupResourceProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm)26 InlongGroupInfo (org.apache.inlong.manager.common.pojo.group.InlongGroupInfo)16 WorkflowListenerException (org.apache.inlong.manager.common.exceptions.WorkflowListenerException)7 ProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.ProcessForm)6 InlongStreamEntity (org.apache.inlong.manager.dao.entity.InlongStreamEntity)5 PulsarClusterInfo (org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo)4 WorkflowContext (org.apache.inlong.manager.workflow.WorkflowContext)4 PulsarAdmin (org.apache.pulsar.client.admin.PulsarAdmin)4 BusinessException (org.apache.inlong.manager.common.exceptions.BusinessException)3 Test (org.junit.Test)3 InlongGroupPulsarInfo (org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo)2 PulsarTopicBean (org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean)2 SinkForSortDTO (org.apache.inlong.manager.common.pojo.sink.SinkForSortDTO)2 InlongStreamInfo (org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo)2 UpdateGroupProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.UpdateGroupProcessForm)2 ServiceBaseTest (org.apache.inlong.manager.service.ServiceBaseTest)2 ServiceTask (org.apache.inlong.manager.workflow.definition.ServiceTask)2 WorkflowProcess (org.apache.inlong.manager.workflow.definition.WorkflowProcess)2 List (java.util.List)1 Collectors (java.util.stream.Collectors)1