Examples with InlongStreamEntity org.apache.inlong.manager.dao.entity.InlongStreamEntity used on opensource projects

Example 6 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class ThirdPartyClusterServiceImpl method getConfigV2.

/**
 * query data proxy config by cluster name, result includes pulsar/tube cluster configs and topic etc
 */
@Override
public ThirdPartyClusterDTO getConfigV2(String clusterName) {
    ThirdPartyClusterEntity clusterEntity = thirdPartyClusterMapper.selectByName(clusterName);
    if (clusterEntity == null) {
        throw new BusinessException("data proxy cluster not found by name=" + clusterName);
    }
    // TODO Optimize query conditions use dataProxyClusterId
    List<InlongGroupEntity> groupEntityList = groupMapper.selectAll(GroupState.CONFIG_SUCCESSFUL.getCode());
    if (CollectionUtils.isEmpty(groupEntityList)) {
        String msg = "not found any inlong group with success status for proxy cluster name = " + clusterName;
        LOGGER.warn(msg);
        throw new BusinessException(msg);
    }
    // third-party-cluster type
    String mqType = "";
    if (!groupEntityList.isEmpty()) {
        mqType = groupEntityList.get(0).getMiddlewareType();
    }
    // Get topic list by group id
    List<DataProxyConfig> topicList = new ArrayList<>();
    for (InlongGroupEntity groupEntity : groupEntityList) {
        final String groupId = groupEntity.getInlongGroupId();
        final String mqResource = groupEntity.getMqResourceObj();
        if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            List<InlongStreamEntity> streamList = streamMapper.selectByGroupId(groupId);
            for (InlongStreamEntity stream : streamList) {
                DataProxyConfig topicConfig = new DataProxyConfig();
                String streamId = stream.getInlongStreamId();
                String topic = stream.getMqResourceObj();
                String tenant = clusterBean.getDefaultTenant();
                InlongGroupPulsarEntity pulsarEntity = pulsarEntityMapper.selectByGroupId(groupId);
                if (pulsarEntity != null && StringUtils.isNotEmpty(pulsarEntity.getTenant())) {
                    tenant = pulsarEntity.getTenant();
                }
                topicConfig.setInlongGroupId(groupId + "/" + streamId);
                topicConfig.setTopic("persistent://" + tenant + "/" + mqResource + "/" + topic);
                topicList.add(topicConfig);
            }
        } else if (Constant.MIDDLEWARE_TUBE.equals(mqType)) {
            DataProxyConfig topicConfig = new DataProxyConfig();
            topicConfig.setInlongGroupId(groupId);
            topicConfig.setTopic(mqResource);
            topicList.add(topicConfig);
        }
    }
    // construct pulsarSet info
    List<ThirdPartyClusterInfo> mqSet = new ArrayList<>();
    List<String> clusterType = Arrays.asList(Constant.CLUSTER_TUBE, Constant.CLUSTER_PULSAR, Constant.CLUSTER_TDMQ_PULSAR);
    List<ThirdPartyClusterEntity> clusterList = thirdPartyClusterMapper.selectMQCluster(clusterEntity.getMqSetName(), clusterType);
    for (ThirdPartyClusterEntity cluster : clusterList) {
        ThirdPartyClusterInfo clusterInfo = new ThirdPartyClusterInfo();
        clusterInfo.setUrl(cluster.getUrl());
        clusterInfo.setToken(cluster.getToken());
        Map<String, String> configParams = GSON.fromJson(cluster.getExtParams(), Map.class);
        clusterInfo.setParams(configParams);
        mqSet.add(clusterInfo);
    }
    ThirdPartyClusterDTO object = new ThirdPartyClusterDTO();
    object.setMqSet(mqSet);
    object.setTopicList(topicList);
    return object;
}

Example 7 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class ThirdPartyClusterServiceImpl method getConfig.

@Override
public List<DataProxyConfig> getConfig() {
    // get all configs with inlong group status of 130, that is, config successful
    // TODO Optimize query conditions
    List<InlongGroupEntity> groupEntityList = groupMapper.selectAll(GroupState.CONFIG_SUCCESSFUL.getCode());
    List<DataProxyConfig> configList = new ArrayList<>();
    for (InlongGroupEntity groupEntity : groupEntityList) {
        String groupId = groupEntity.getInlongGroupId();
        String bizResource = groupEntity.getMqResourceObj();
        DataProxyConfig config = new DataProxyConfig();
        config.setM(groupEntity.getSchemaName());
        String mqType = groupEntity.getMiddlewareType();
        if (Constant.MIDDLEWARE_TUBE.equals(mqType)) {
            config.setInlongGroupId(groupId);
            config.setTopic(bizResource);
        } else if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
            List<InlongStreamEntity> streamList = streamMapper.selectByGroupId(groupId);
            for (InlongStreamEntity stream : streamList) {
                String topic = stream.getMqResourceObj();
                String streamId = stream.getInlongStreamId();
                config.setInlongGroupId(groupId + "/" + streamId);
                config.setTopic("persistent://" + clusterBean.getDefaultTenant() + "/" + bizResource + "/" + topic);
            }
        }
        configList.add(config);
    }
    return configList;
}

Example 8 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongStreamServiceImpl method insertDlqOrRlq.

@Override
public void insertDlqOrRlq(String groupId, String topicName, String operator) {
    Integer count = streamMapper.selectExistByIdentifier(groupId, topicName);
    if (count >= 1) {
        LOGGER.error("DLQ/RLQ topic already exists with name={}", topicName);
        throw new BusinessException(ErrorCodeEnum.STREAM_ID_DUPLICATE, "DLQ/RLQ topic already exists");
    }
    InlongStreamEntity streamEntity = new InlongStreamEntity();
    streamEntity.setInlongGroupId(groupId);
    streamEntity.setInlongStreamId(topicName);
    streamEntity.setMqResourceObj(topicName);
    streamEntity.setDescription("This is DLQ / RLQ topic created by SYSTEM");
    streamEntity.setDailyRecords(1000);
    streamEntity.setDailyStorage(1000);
    streamEntity.setPeakRecords(1000);
    streamEntity.setMaxLength(1000);
    streamEntity.setStatus(EntityStatus.STREAM_CONFIG_SUCCESSFUL.getCode());
    streamEntity.setIsDeleted(EntityStatus.UN_DELETED.getCode());
    streamEntity.setCreator(operator);
    streamEntity.setModifier(operator);
    Date now = new Date();
    streamEntity.setCreateTime(now);
    streamEntity.setModifyTime(now);
    streamMapper.insert(streamEntity);
}

Example 9 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongStreamServiceImpl method update.

@Transactional(rollbackFor = Throwable.class)
@Override
public boolean update(InlongStreamInfo streamInfo, String operator) {
    LOGGER.debug("begin to update inlong stream info={}", streamInfo);
    Preconditions.checkNotNull(streamInfo, "inlong stream info is empty");
    String groupId = streamInfo.getInlongGroupId();
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    String streamId = streamInfo.getInlongStreamId();
    Preconditions.checkNotNull(streamId, Constant.STREAM_ID_IS_EMPTY);
    // Check if it can be modified
    InlongGroupEntity inlongGroupEntity = this.checkBizIsTempStatus(groupId);
    // Add if it doesn't exist, modify if it exists
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (streamEntity == null) {
        this.save(streamInfo, operator);
    } else {
        // Check whether the current inlong group status supports modification
        this.checkCanUpdate(inlongGroupEntity.getStatus(), streamEntity, streamInfo);
        CommonBeanUtils.copyProperties(streamInfo, streamEntity, true);
        streamEntity.setModifier(operator);
        streamEntity.setStatus(EntityStatus.GROUP_CONFIG_ING.getCode());
        streamMapper.updateByIdentifierSelective(streamEntity);
        // Update extended information, field information
        this.updateExt(groupId, streamId, streamInfo.getExtList());
        this.updateField(groupId, streamId, streamInfo.getFieldList());
    }
    LOGGER.info("success to update inlong group for groupId={}", groupId);
    return true;
}

Example 10 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongStreamServiceImpl method logicDeleteAll.

@Transactional(rollbackFor = Throwable.class)
@Override
public boolean logicDeleteAll(String groupId, String operator) {
    LOGGER.debug("begin to delete all inlong stream by groupId={}", groupId);
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    // Check if it can be deleted
    this.checkBizIsTempStatus(groupId);
    List<InlongStreamEntity> entityList = streamMapper.selectByGroupId(groupId);
    if (CollectionUtils.isEmpty(entityList)) {
        LOGGER.info("inlong stream not found by groupId={}", groupId);
        return true;
    }
    for (InlongStreamEntity entity : entityList) {
        entity.setIsDeleted(1);
        entity.setModifier(operator);
        streamMapper.updateByIdentifierSelective(entity);
        String streamId = entity.getInlongStreamId();
        // To logically delete the associated extension table
        streamExtMapper.logicDeleteAllByIdentifier(groupId, streamId);
        // Logically delete the associated field table
        streamFieldMapper.logicDeleteAllByIdentifier(groupId, streamId);
        // Tombstone the associated data source
        sourceFileService.logicDeleteAllByIdentifier(groupId, streamId, operator);
        sourceDbService.logicDeleteAllByIdentifier(groupId, streamId, operator);
        // Logical deletion of associated data sink information
        sinkService.logicDeleteAll(groupId, streamId, operator);
    }
    LOGGER.info("success to delete all inlong stream, ext property and fields by groupId={}", groupId);
    return true;
}