Examples with InlongGroupInfo org.apache.inlong.manager.common.pojo.group.InlongGroupInfo used on opensource projects

Example 1 with InlongGroupInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupInfo in project incubator-inlong by apache.

the class InlongGroupTransfer method createGroupInfo.

public static InlongGroupInfo createGroupInfo(InlongGroupConf groupConf) {
    InlongGroupInfo groupInfo = new InlongGroupInfo();
    AssertUtil.hasLength(groupConf.getGroupName(), "GroupName should not be empty");
    groupInfo.setName(groupConf.getGroupName());
    groupInfo.setCnName(groupConf.getCnName());
    groupInfo.setDescription(groupConf.getDescription());
    groupInfo.setZookeeperEnabled(groupConf.isZookeeperEnabled() ? 1 : 0);
    groupInfo.setDailyRecords(groupConf.getDailyRecords().intValue());
    groupInfo.setPeakRecords(groupConf.getPeakRecords().intValue());
    groupInfo.setMaxLength(groupConf.getMaxLength());
    groupInfo.setProxyClusterId(groupConf.getProxyClusterId());
    MqBaseConf mqConf = groupConf.getMqBaseConf();
    MqType mqType = MqType.NONE;
    if (null != mqConf) {
        mqType = mqConf.getType();
        groupInfo.setMiddlewareType(mqType.name());
    }
    groupInfo.setInCharges(groupConf.getOperator());
    groupInfo.setExtList(Lists.newArrayList());
    groupInfo.setCreator(groupConf.getOperator());
    if (mqType == MqType.PULSAR || mqType == MqType.TDMQ_PULSAR) {
        PulsarBaseConf pulsarBaseConf = (PulsarBaseConf) mqConf;
        groupInfo.setMqResourceObj(pulsarBaseConf.getNamespace());
        InlongGroupPulsarInfo pulsarInfo = createPulsarInfo(pulsarBaseConf);
        groupInfo.setMqExtInfo(pulsarInfo);
        List<InlongGroupExtInfo> extInfos = createPulsarExtInfo(pulsarBaseConf);
        groupInfo.getExtList().addAll(extInfos);
        groupInfo.setTopicPartitionNum(pulsarBaseConf.getTopicPartitionNum());
    } else if (mqType == MqType.TUBE) {
        TubeBaseConf tubeBaseConf = (TubeBaseConf) mqConf;
        List<InlongGroupExtInfo> extInfos = createTubeExtInfo(tubeBaseConf);
        groupInfo.setMqResourceObj(tubeBaseConf.getGroupName());
        groupInfo.getExtList().addAll(extInfos);
        groupInfo.setTopicPartitionNum(tubeBaseConf.getTopicPartitionNum());
    }
    SortBaseConf sortBaseConf = groupConf.getSortBaseConf();
    SortType sortType = sortBaseConf.getType();
    if (sortType == SortType.FLINK) {
        FlinkSortBaseConf flinkSortBaseConf = (FlinkSortBaseConf) sortBaseConf;
        List<InlongGroupExtInfo> sortExtInfos = createFlinkExtInfo(flinkSortBaseConf);
        groupInfo.getExtList().addAll(sortExtInfos);
    } else if (sortType == SortType.USER_DEFINED) {
        UserDefinedSortConf udf = (UserDefinedSortConf) sortBaseConf;
        List<InlongGroupExtInfo> sortExtInfos = createUserDefinedSortExtInfo(udf);
        groupInfo.getExtList().addAll(sortExtInfos);
    } else {
    // todo local
    }
    return groupInfo;
}

Example 2 with InlongGroupInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupInfo in project incubator-inlong by apache.

the class AbstractSourceOperateListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws Exception {
    log.info("Delete data source for context={}", context);
    InlongGroupInfo groupInfo = getGroupInfo(context.getProcessForm());
    final String groupId = groupInfo.getInlongGroupId();
    List<StreamBriefResponse> streamBriefResponses = streamService.getBriefList(groupId);
    streamBriefResponses.forEach(streamBriefResponse -> operateStreamSources(groupId, streamBriefResponse.getInlongStreamId(), context.getApplicant()));
    return ListenerResult.success();
}

Example 3 with InlongGroupInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupInfo in project incubator-inlong by apache.

the class CreatePulsarTopicForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (groupInfo == null || streamEntity == null) {
        throw new WorkflowListenerException("inlong group or inlong stream not found with groupId=" + groupId + ", streamId=" + streamId);
    }
    log.info("begin to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            String pulsarTopic = streamEntity.getMqResourceObj();
            this.createTopic(groupInfo, pulsarTopic, pulsarClusterInfo);
        }
    } catch (Exception e) {
        log.error("create pulsar topic error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar topic error for groupId=" + groupId + ", streamId=" + streamId);
    }
    log.info("success to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}

Example 4 with InlongGroupInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupInfo in project incubator-inlong by apache.

the class CreateTubeGroupTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    log.info("try to create consumer group for groupId {}", groupId);
    InlongGroupInfo groupInfo = groupService.get(groupId);
    String topicName = groupInfo.getMqResourceObj();
    int clusterId = Integer.parseInt(commonOperateService.getSpecifiedParam(Constant.CLUSTER_TUBE_CLUSTER_ID));
    QueryTubeTopicRequest queryTubeTopicRequest = QueryTubeTopicRequest.builder().topicName(topicName).clusterId(clusterId).user(groupInfo.getCreator()).build();
    // Query whether the tube topic exists
    boolean topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
    Integer tryNumber = reTryConfigBean.getMaxAttempts();
    Long delay = reTryConfigBean.getDelay();
    while (!topicExist && --tryNumber > 0) {
        log.info("check whether the tube topic exists, try count={}", tryNumber);
        try {
            Thread.sleep(delay);
            delay *= reTryConfigBean.getMultiplier();
            topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
        } catch (InterruptedException e) {
            log.error("check the tube topic exists error", e);
        }
    }
    AddTubeConsumeGroupRequest addTubeConsumeGroupRequest = new AddTubeConsumeGroupRequest();
    addTubeConsumeGroupRequest.setClusterId(clusterId);
    addTubeConsumeGroupRequest.setCreateUser(groupInfo.getCreator());
    GroupNameJsonSetBean groupNameJsonSetBean = new GroupNameJsonSetBean();
    groupNameJsonSetBean.setTopicName(topicName);
    String consumeGroupName = "sort_" + topicName + "_group";
    groupNameJsonSetBean.setGroupName(consumeGroupName);
    addTubeConsumeGroupRequest.setGroupNameJsonSet(Collections.singletonList(groupNameJsonSetBean));
    try {
        tubeMqOptService.createNewConsumerGroup(addTubeConsumeGroupRequest);
    } catch (Exception e) {
        throw new WorkflowListenerException("create tube consumer group for groupId=" + groupId + " error", e);
    }
    log.info("finish to create consumer group for {}", groupId);
    return ListenerResult.success();
}

Example 5 with InlongGroupInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupInfo in project incubator-inlong by apache.

the class TubeEventSelector method accept.

@Override
public boolean accept(WorkflowContext context) {
    ProcessForm processForm = context.getProcessForm();
    if (!(processForm instanceof GroupResourceProcessForm)) {
        return false;
    }
    GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
    InlongGroupInfo groupInfo = form.getGroupInfo();
    if (Constant.MIDDLEWARE_TUBE.equalsIgnoreCase(groupInfo.getMiddlewareType())) {
        return true;
    }
    log.warn("not need to create tube resource for groupId={}, as the middleware type is {}", groupInfo.getMiddlewareType(), form.getInlongGroupId());
    return false;
}