Search in sources :

Example 1 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreateHiveTableEventSelector method accept.

@Override
public boolean accept(WorkflowContext context) {
    ProcessForm processForm = context.getProcessForm();
    if (!(processForm instanceof GroupResourceProcessForm)) {
        return false;
    }
    GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
    if (form.getGroupInfo() == null || StringUtils.isEmpty(form.getGroupInfo().getInlongGroupId())) {
        return false;
    }
    String groupId = form.getInlongGroupId();
    List<String> dsForHive = sinkService.getExistsStreamIdList(groupId, Constant.SINK_HIVE, streamMapper.selectByGroupId(groupId).stream().map(InlongStreamEntity::getInlongStreamId).collect(Collectors.toList()));
    if (CollectionUtils.isEmpty(dsForHive)) {
        log.warn("groupId={} streamId={} does not have sink, skip to create hive table ", groupId, form.getInlongStreamId());
        return true;
    }
    return false;
}
Also used : InlongStreamEntity(org.apache.inlong.manager.dao.entity.InlongStreamEntity) ProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.ProcessForm) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm)

Example 2 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreateHiveTableListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    log.info("begin to create hive table for groupId={}", groupId);
    List<SinkForSortDTO> configList = sinkMapper.selectAllConfig(groupId, null);
    List<SinkForSortDTO> needCreateList = configList.stream().filter(sinkForSortDTO -> sinkForSortDTO.getEnableCreateResource() == 1).collect(Collectors.toList());
    if (hiveTableOperator == null) {
        hiveTableOperator = (IHiveTableOperator) SpringContextUtils.getBean(IHiveTableOperator.BEAN_NAME, DefaultHiveTableOperator.class.getName());
    }
    hiveTableOperator.createHiveResource(groupId, needCreateList);
    String result = "success to create hive table for group [" + groupId + "]";
    log.info(result);
    return ListenerResult.success(result);
}
Also used : StreamSinkEntityMapper(org.apache.inlong.manager.dao.mapper.StreamSinkEntityMapper) SinkForSortDTO(org.apache.inlong.manager.common.pojo.sink.SinkForSortDTO) Autowired(org.springframework.beans.factory.annotation.Autowired) Collectors(java.util.stream.Collectors) SpringContextUtils(org.apache.inlong.manager.service.utils.SpringContextUtils) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) List(java.util.List) ListenerResult(org.apache.inlong.manager.workflow.event.ListenerResult) Slf4j(lombok.extern.slf4j.Slf4j) SinkOperateListener(org.apache.inlong.manager.workflow.event.task.SinkOperateListener) TaskEvent(org.apache.inlong.manager.workflow.event.task.TaskEvent) Service(org.springframework.stereotype.Service) WorkflowContext(org.apache.inlong.manager.workflow.WorkflowContext) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) SinkForSortDTO(org.apache.inlong.manager.common.pojo.sink.SinkForSortDTO)

Example 3 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreatePulsarTopicForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (groupInfo == null || streamEntity == null) {
        throw new WorkflowListenerException("inlong group or inlong stream not found with groupId=" + groupId + ", streamId=" + streamId);
    }
    log.info("begin to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            String pulsarTopic = streamEntity.getMqResourceObj();
            this.createTopic(groupInfo, pulsarTopic, pulsarClusterInfo);
        }
    } catch (Exception e) {
        log.error("create pulsar topic error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar topic error for groupId=" + groupId + ", streamId=" + streamId);
    }
    log.info("success to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}
Also used : PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) InlongStreamEntity(org.apache.inlong.manager.dao.entity.InlongStreamEntity) PulsarClusterInfo(org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException) InlongGroupInfo(org.apache.inlong.manager.common.pojo.group.InlongGroupInfo) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException)

Example 4 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class CreateTubeGroupTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    log.info("try to create consumer group for groupId {}", groupId);
    InlongGroupInfo groupInfo = groupService.get(groupId);
    String topicName = groupInfo.getMqResourceObj();
    int clusterId = Integer.parseInt(commonOperateService.getSpecifiedParam(Constant.CLUSTER_TUBE_CLUSTER_ID));
    QueryTubeTopicRequest queryTubeTopicRequest = QueryTubeTopicRequest.builder().topicName(topicName).clusterId(clusterId).user(groupInfo.getCreator()).build();
    // Query whether the tube topic exists
    boolean topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
    Integer tryNumber = reTryConfigBean.getMaxAttempts();
    Long delay = reTryConfigBean.getDelay();
    while (!topicExist && --tryNumber > 0) {
        log.info("check whether the tube topic exists, try count={}", tryNumber);
        try {
            Thread.sleep(delay);
            delay *= reTryConfigBean.getMultiplier();
            topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
        } catch (InterruptedException e) {
            log.error("check the tube topic exists error", e);
        }
    }
    AddTubeConsumeGroupRequest addTubeConsumeGroupRequest = new AddTubeConsumeGroupRequest();
    addTubeConsumeGroupRequest.setClusterId(clusterId);
    addTubeConsumeGroupRequest.setCreateUser(groupInfo.getCreator());
    GroupNameJsonSetBean groupNameJsonSetBean = new GroupNameJsonSetBean();
    groupNameJsonSetBean.setTopicName(topicName);
    String consumeGroupName = "sort_" + topicName + "_group";
    groupNameJsonSetBean.setGroupName(consumeGroupName);
    addTubeConsumeGroupRequest.setGroupNameJsonSet(Collections.singletonList(groupNameJsonSetBean));
    try {
        tubeMqOptService.createNewConsumerGroup(addTubeConsumeGroupRequest);
    } catch (Exception e) {
        throw new WorkflowListenerException("create tube consumer group for groupId=" + groupId + " error", e);
    }
    log.info("finish to create consumer group for {}", groupId);
    return ListenerResult.success();
}
Also used : GroupNameJsonSetBean(org.apache.inlong.manager.common.pojo.tubemq.AddTubeConsumeGroupRequest.GroupNameJsonSetBean) InlongGroupInfo(org.apache.inlong.manager.common.pojo.group.InlongGroupInfo) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException) AddTubeConsumeGroupRequest(org.apache.inlong.manager.common.pojo.tubemq.AddTubeConsumeGroupRequest) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException) QueryTubeTopicRequest(org.apache.inlong.manager.common.pojo.tubemq.QueryTubeTopicRequest)

Example 5 with GroupResourceProcessForm

use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.

the class PulsarEventSelector method accept.

@Override
public boolean accept(WorkflowContext context) {
    ProcessForm processForm = context.getProcessForm();
    if (!(processForm instanceof GroupResourceProcessForm)) {
        return false;
    }
    GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
    String mqType = form.getGroupInfo().getMiddlewareType();
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) form.getGroupInfo().getMqExtInfo();
        return pulsarInfo.getEnableCreateResource() == 1;
    }
    log.warn("no need to create pulsar subscription group for groupId={}, as the middlewareType={}", form.getInlongGroupId(), mqType);
    return false;
}
Also used : InlongGroupPulsarInfo(org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm) ProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.ProcessForm) GroupResourceProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm)

Aggregations

GroupResourceProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm)26 InlongGroupInfo (org.apache.inlong.manager.common.pojo.group.InlongGroupInfo)16 WorkflowListenerException (org.apache.inlong.manager.common.exceptions.WorkflowListenerException)7 ProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.ProcessForm)6 InlongStreamEntity (org.apache.inlong.manager.dao.entity.InlongStreamEntity)5 PulsarClusterInfo (org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo)4 WorkflowContext (org.apache.inlong.manager.workflow.WorkflowContext)4 PulsarAdmin (org.apache.pulsar.client.admin.PulsarAdmin)4 BusinessException (org.apache.inlong.manager.common.exceptions.BusinessException)3 Test (org.junit.Test)3 InlongGroupPulsarInfo (org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo)2 PulsarTopicBean (org.apache.inlong.manager.common.pojo.pulsar.PulsarTopicBean)2 SinkForSortDTO (org.apache.inlong.manager.common.pojo.sink.SinkForSortDTO)2 InlongStreamInfo (org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo)2 UpdateGroupProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.UpdateGroupProcessForm)2 ServiceBaseTest (org.apache.inlong.manager.service.ServiceBaseTest)2 ServiceTask (org.apache.inlong.manager.workflow.definition.ServiceTask)2 WorkflowProcess (org.apache.inlong.manager.workflow.definition.WorkflowProcess)2 List (java.util.List)1 Collectors (java.util.stream.Collectors)1