Examples with InlongGroupPulsarEntity org.apache.inlong.manager.dao.entity.InlongGroupPulsarEntity used on opensource projects

Example 1 with InlongGroupPulsarEntity

use of org.apache.inlong.manager.dao.entity.InlongGroupPulsarEntity in project incubator-inlong by apache.

the class InlongGroupServiceImpl method get.

@Override
public InlongGroupInfo get(String groupId) {
    LOGGER.debug("begin to get inlong group info by groupId={}", groupId);
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    InlongGroupEntity entity = groupMapper.selectByGroupId(groupId);
    if (entity == null) {
        LOGGER.error("inlong group not found by groupId={}", groupId);
        throw new BusinessException(ErrorCodeEnum.GROUP_NOT_FOUND);
    }
    InlongGroupInfo groupInfo = CommonBeanUtils.copyProperties(entity, InlongGroupInfo::new);
    List<InlongGroupExtEntity> extEntityList = groupExtMapper.selectByGroupId(groupId);
    List<InlongGroupExtInfo> extInfoList = CommonBeanUtils.copyListProperties(extEntityList, InlongGroupExtInfo::new);
    groupInfo.setExtList(extInfoList);
    // If the middleware is Pulsar, we need to encapsulate Pulsar related data
    String mqType = entity.getMiddlewareType();
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        InlongGroupPulsarEntity pulsarEntity = groupPulsarMapper.selectByGroupId(groupId);
        Preconditions.checkNotNull(pulsarEntity, "Pulsar info not found by the groupId=" + groupId);
        InlongGroupPulsarInfo pulsarInfo = CommonBeanUtils.copyProperties(pulsarEntity, InlongGroupPulsarInfo::new);
        pulsarInfo.setMiddlewareType(mqType);
        groupInfo.setMqExtInfo(pulsarInfo);
    }
    // For approved inlong group, encapsulate the cluster address of the middleware
    if (GroupState.CONFIG_SUCCESSFUL == GroupState.forCode(groupInfo.getStatus())) {
        if (Constant.MIDDLEWARE_TUBE.equalsIgnoreCase(mqType)) {
            groupInfo.setTubeMaster(commonOperateService.getSpecifiedParam(Constant.TUBE_MASTER_URL));
        } else if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            PulsarClusterInfo pulsarCluster = commonOperateService.getPulsarClusterInfo(mqType);
            groupInfo.setPulsarAdminUrl(pulsarCluster.getAdminUrl());
            groupInfo.setPulsarServiceUrl(pulsarCluster.getBrokerServiceUrl());
        }
    }
    LOGGER.debug("success to get inlong group for groupId={}", groupId);
    return groupInfo;
}

Example 2 with InlongGroupPulsarEntity

use of org.apache.inlong.manager.dao.entity.InlongGroupPulsarEntity in project incubator-inlong by apache.

the class ThirdPartyClusterServiceImpl method getConfigV2.

/**
 * query data proxy config by cluster name, result includes pulsar/tube cluster configs and topic etc
 */
@Override
public ThirdPartyClusterDTO getConfigV2(String clusterName) {
    ThirdPartyClusterEntity clusterEntity = thirdPartyClusterMapper.selectByName(clusterName);
    if (clusterEntity == null) {
        throw new BusinessException("data proxy cluster not found by name=" + clusterName);
    }
    // TODO Optimize query conditions use dataProxyClusterId
    List<InlongGroupEntity> groupEntityList = groupMapper.selectAll(GroupState.CONFIG_SUCCESSFUL.getCode());
    if (CollectionUtils.isEmpty(groupEntityList)) {
        String msg = "not found any inlong group with success status for proxy cluster name = " + clusterName;
        LOGGER.warn(msg);
        throw new BusinessException(msg);
    }
    // third-party-cluster type
    String mqType = "";
    if (!groupEntityList.isEmpty()) {
        mqType = groupEntityList.get(0).getMiddlewareType();
    }
    // Get topic list by group id
    List<DataProxyConfig> topicList = new ArrayList<>();
    for (InlongGroupEntity groupEntity : groupEntityList) {
        final String groupId = groupEntity.getInlongGroupId();
        final String mqResource = groupEntity.getMqResourceObj();
        if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            List<InlongStreamEntity> streamList = streamMapper.selectByGroupId(groupId);
            for (InlongStreamEntity stream : streamList) {
                DataProxyConfig topicConfig = new DataProxyConfig();
                String streamId = stream.getInlongStreamId();
                String topic = stream.getMqResourceObj();
                String tenant = clusterBean.getDefaultTenant();
                InlongGroupPulsarEntity pulsarEntity = pulsarEntityMapper.selectByGroupId(groupId);
                if (pulsarEntity != null && StringUtils.isNotEmpty(pulsarEntity.getTenant())) {
                    tenant = pulsarEntity.getTenant();
                }
                topicConfig.setInlongGroupId(groupId + "/" + streamId);
                topicConfig.setTopic("persistent://" + tenant + "/" + mqResource + "/" + topic);
                topicList.add(topicConfig);
            }
        } else if (Constant.MIDDLEWARE_TUBE.equals(mqType)) {
            DataProxyConfig topicConfig = new DataProxyConfig();
            topicConfig.setInlongGroupId(groupId);
            topicConfig.setTopic(mqResource);
            topicList.add(topicConfig);
        }
    }
    // construct pulsarSet info
    List<ThirdPartyClusterInfo> mqSet = new ArrayList<>();
    List<String> clusterType = Arrays.asList(Constant.CLUSTER_TUBE, Constant.CLUSTER_PULSAR, Constant.CLUSTER_TDMQ_PULSAR);
    List<ThirdPartyClusterEntity> clusterList = thirdPartyClusterMapper.selectMQCluster(clusterEntity.getMqSetName(), clusterType);
    for (ThirdPartyClusterEntity cluster : clusterList) {
        ThirdPartyClusterInfo clusterInfo = new ThirdPartyClusterInfo();
        clusterInfo.setUrl(cluster.getUrl());
        clusterInfo.setToken(cluster.getToken());
        Map<String, String> configParams = GSON.fromJson(cluster.getExtParams(), Map.class);
        clusterInfo.setParams(configParams);
        mqSet.add(clusterInfo);
    }
    ThirdPartyClusterDTO object = new ThirdPartyClusterDTO();
    object.setMqSet(mqSet);
    object.setTopicList(topicList);
    return object;
}

Example 3 with InlongGroupPulsarEntity

use of org.apache.inlong.manager.dao.entity.InlongGroupPulsarEntity in project incubator-inlong by apache.

the class CreatePulsarResourceTaskListener method createPulsarProcess.

/**
 * Create Pulsar tenant, namespace and topic
 */
private void createPulsarProcess(InlongGroupInfo groupInfo, PulsarClusterInfo pulsarClusterInfo) throws Exception {
    String groupId = groupInfo.getInlongGroupId();
    log.info("begin to create pulsar resource for groupId={} in cluster={}", groupId, pulsarClusterInfo);
    String namespace = groupInfo.getMqResourceObj();
    Preconditions.checkNotNull(namespace, "pulsar namespace cannot be empty for groupId=" + groupId);
    String queueModule = groupInfo.getQueueModule();
    Preconditions.checkNotNull(queueModule, "queue module cannot be empty for groupId=" + groupId);
    String tenant = clusterBean.getDefaultTenant();
    try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
        // create pulsar tenant
        pulsarOptService.createTenant(pulsarAdmin, tenant);
        // create pulsar namespace
        InlongGroupPulsarEntity entity = groupPulsarMapper.selectByGroupId(groupId);
        pulsarOptService.createNamespace(pulsarAdmin, entity, tenant, namespace);
        // create pulsar topic
        Integer partitionNum = groupInfo.getTopicPartitionNum();
        List<InlongStreamTopicResponse> streamTopicList = streamMapper.selectTopicList(groupId);
        PulsarTopicBean topicBean = PulsarTopicBean.builder().tenant(tenant).namespace(namespace).numPartitions(partitionNum).queueModule(queueModule).build();
        for (InlongStreamTopicResponse topicVO : streamTopicList) {
            topicBean.setTopicName(topicVO.getMqResourceObj());
            pulsarOptService.createTopic(pulsarAdmin, topicBean);
        }
    }
    log.info("finish to create pulsar resource for groupId={}, service http url={}", groupId, pulsarClusterInfo.getAdminUrl());
}

Example 4 with InlongGroupPulsarEntity

use of org.apache.inlong.manager.dao.entity.InlongGroupPulsarEntity in project incubator-inlong by apache.

the class InlongGroupServiceImpl method update.

@Transactional(rollbackFor = Throwable.class)
@Override
public String update(InlongGroupRequest groupRequest, String operator) {
    LOGGER.debug("begin to update inlong group={}", groupRequest);
    Preconditions.checkNotNull(groupRequest, "inlong group is empty");
    String groupId = groupRequest.getInlongGroupId();
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    InlongGroupEntity entity = groupMapper.selectByGroupId(groupId);
    if (entity == null) {
        LOGGER.error("inlong group not found by groupId={}", groupId);
        throw new BusinessException(ErrorCodeEnum.GROUP_NOT_FOUND);
    }
    // Check whether the current status can be modified
    this.checkGroupCanUpdate(entity, groupRequest, operator);
    CommonBeanUtils.copyProperties(groupRequest, entity, true);
    if (GroupState.CONFIG_FAILED.getCode().equals(entity.getStatus())) {
        entity.setStatus(GroupState.TO_BE_SUBMIT.getCode());
    }
    entity.setModifier(operator);
    groupMapper.updateByIdentifierSelective(entity);
    // Save extended information
    this.saveOrUpdateExt(groupId, groupRequest.getExtList());
    // Update the Pulsar info
    String mqType = groupRequest.getMiddlewareType();
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) groupRequest.getMqExtInfo();
        Preconditions.checkNotNull(pulsarInfo, "Pulsar info cannot be empty, as the middleware is Pulsar");
        Integer writeQuorum = pulsarInfo.getWriteQuorum();
        Integer ackQuorum = pulsarInfo.getAckQuorum();
        if (!(ackQuorum <= writeQuorum)) {
            throw new BusinessException(ErrorCodeEnum.GROUP_SAVE_FAILED, "Pulsar params must meet: ackQuorum <= writeQuorum");
        }
        InlongGroupPulsarEntity pulsarEntity = CommonBeanUtils.copyProperties(pulsarInfo, InlongGroupPulsarEntity::new);
        pulsarEntity.setInlongGroupId(groupId);
        groupPulsarMapper.updateByIdentifierSelective(pulsarEntity);
    }
    LOGGER.debug("success to update inlong group for groupId={}", groupId);
    return groupId;
}

Example 5 with InlongGroupPulsarEntity

use of org.apache.inlong.manager.dao.entity.InlongGroupPulsarEntity in project incubator-inlong by apache.

the class InlongGroupServiceImpl method save.

@Transactional(rollbackFor = Throwable.class)
@Override
public String save(InlongGroupRequest groupInfo, String operator) {
    LOGGER.debug("begin to save inlong group info={}", groupInfo);
    Preconditions.checkNotNull(groupInfo, "inlong group info is empty");
    String groupName = groupInfo.getName();
    Preconditions.checkNotNull(groupName, "inlong group name is empty");
    // groupId=b_name, cannot update
    String groupId = "b_" + groupName.toLowerCase(Locale.ROOT);
    Integer count = groupMapper.selectIdentifierExist(groupId);
    if (count >= 1) {
        LOGGER.error("groupId [{}] has already exists", groupId);
        throw new BusinessException(ErrorCodeEnum.GROUP_DUPLICATE);
    }
    // Processing inlong group and extended information
    InlongGroupEntity entity = CommonBeanUtils.copyProperties(groupInfo, InlongGroupEntity::new);
    entity.setInlongGroupId(groupId);
    if (StringUtils.isEmpty(entity.getMqResourceObj())) {
        entity.setMqResourceObj(groupId);
    }
    // Only M0 is currently supported
    entity.setSchemaName(Constant.SCHEMA_M0_DAY);
    // After saving, the status is set to [GROUP_WAIT_SUBMIT]
    entity.setStatus(GroupState.TO_BE_SUBMIT.getCode());
    entity.setIsDeleted(EntityStatus.UN_DELETED.getCode());
    if (StringUtils.isEmpty(entity.getCreator())) {
        entity.setCreator(operator);
    }
    if (StringUtils.isEmpty(entity.getModifier())) {
        entity.setModifier(operator);
    }
    entity.setCreateTime(new Date());
    groupMapper.insertSelective(entity);
    this.saveOrUpdateExt(groupId, groupInfo.getExtList());
    String mqType = groupInfo.getMiddlewareType();
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) groupInfo.getMqExtInfo();
        Preconditions.checkNotNull(pulsarInfo, "Pulsar info cannot be empty, as the middleware is Pulsar");
        // Pulsar params must meet: ackQuorum <= writeQuorum <= ensemble
        Integer ackQuorum = pulsarInfo.getAckQuorum();
        Integer writeQuorum = pulsarInfo.getWriteQuorum();
        Preconditions.checkNotNull(ackQuorum, "Pulsar ackQuorum cannot be empty");
        Preconditions.checkNotNull(writeQuorum, "Pulsar writeQuorum cannot be empty");
        if (!(ackQuorum <= writeQuorum)) {
            throw new BusinessException(ErrorCodeEnum.GROUP_SAVE_FAILED, "Pulsar params must meet: ackQuorum <= writeQuorum");
        }
        // The default value of ensemble is writeQuorum
        pulsarInfo.setEnsemble(writeQuorum);
        // Pulsar entity may already exist, such as unsuccessfully deleted, or modify the MQ type to Tube,
        // need to delete and add the Pulsar entity with the same group id
        InlongGroupPulsarEntity pulsarEntity = groupPulsarMapper.selectByGroupId(groupId);
        if (pulsarEntity == null) {
            pulsarEntity = CommonBeanUtils.copyProperties(pulsarInfo, InlongGroupPulsarEntity::new);
            pulsarEntity.setIsDeleted(0);
            pulsarEntity.setInlongGroupId(groupId);
            groupPulsarMapper.insertSelective(pulsarEntity);
        } else {
            Integer id = pulsarEntity.getId();
            pulsarEntity = CommonBeanUtils.copyProperties(pulsarInfo, InlongGroupPulsarEntity::new);
            pulsarEntity.setId(id);
            groupPulsarMapper.updateByPrimaryKeySelective(pulsarEntity);
        }
    }
    LOGGER.debug("success to save inlong group info for groupId={}", groupId);
    return groupId;
}