Examples with InlongStreamEntity org.apache.inlong.manager.dao.entity.InlongStreamEntity used on opensource projects

Example 11 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class AgentServiceImpl method getDataConfig.

/**
 * Get the DataConfig from the stream source entity
 */
private DataConfig getDataConfig(StreamSourceEntity entity, int op) {
    DataConfig dataConfig = new DataConfig();
    dataConfig.setIp(entity.getAgentIp());
    dataConfig.setUuid(entity.getUuid());
    dataConfig.setOp(String.valueOf(op));
    dataConfig.setTaskId(entity.getId());
    dataConfig.setTaskType(getTaskType(entity));
    dataConfig.setTaskName(entity.getSourceName());
    dataConfig.setSnapshot(entity.getSnapshot());
    dataConfig.setExtParams(entity.getExtParams());
    LocalDateTime dateTime = LocalDateTime.ofInstant(entity.getModifyTime().toInstant(), ZoneId.systemDefault());
    dataConfig.setDeliveryTime(dateTime.format(TIME_FORMATTER));
    String groupId = entity.getInlongGroupId();
    String streamId = entity.getInlongStreamId();
    dataConfig.setInlongGroupId(groupId);
    dataConfig.setInlongStreamId(streamId);
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    dataConfig.setSyncSend(streamEntity.getSyncSend());
    return dataConfig;
}

Example 12 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class CreatePulsarGroupForStreamTaskListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    String streamId = form.getInlongStreamId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    if (groupInfo == null) {
        log.error("inlong group not found with groupId={}", groupId);
        throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
    }
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    if (streamEntity == null) {
        log.warn("inlong stream is empty for group={}, stream={}, skip to create pulsar group", groupId, streamId);
        return ListenerResult.success();
    }
    PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
    try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
        // Query data sink info based on groupId and streamId
        List<String> sinkTypeList = sinkService.getSinkTypeList(groupId, streamId);
        if (sinkTypeList == null || sinkTypeList.size() == 0) {
            log.warn("sink info is empty for groupId={}, streamId={}, skip to create pulsar group", groupId, streamId);
            return ListenerResult.success();
        }
        PulsarTopicBean topicBean = new PulsarTopicBean();
        topicBean.setTenant(clusterBean.getDefaultTenant());
        topicBean.setNamespace(groupInfo.getMqResourceObj());
        String topic = streamEntity.getMqResourceObj();
        topicBean.setTopicName(topic);
        List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
        // Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
        String tenant = clusterBean.getDefaultTenant();
        String namespace = groupInfo.getMqResourceObj();
        for (String cluster : pulsarClusters) {
            String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
            PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
            try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
                boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
                if (!exist) {
                    String fullTopic = tenant + "/" + namespace + "/" + topic;
                    log.error("topic={} not exists in {}", fullTopic, pulsarAdmin.getServiceUrl());
                    throw new BusinessException("topic=" + fullTopic + " not exists in " + serviceUrl);
                }
                // Consumer naming rules: sortAppName_topicName_consumer_group
                String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
                pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
                // Insert the consumption data into the consumption table
                consumptionService.saveSortConsumption(groupInfo, topic, subscription);
            }
        }
    } catch (Exception e) {
        log.error("create pulsar subscription error for groupId={}, streamId={}", groupId, streamId, e);
        throw new WorkflowListenerException("create pulsar subscription error, reason: " + e.getMessage());
    }
    log.info("finish to create single pulsar subscription for groupId={}, streamId={}", groupId, streamId);
    return ListenerResult.success();
}

Example 13 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class StartCreateGroupProcessListener method listen.

/**
 * Initiate the process of creating inlong group resources after new inlong group access approved
 */
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    NewGroupProcessForm form = (NewGroupProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    GroupResourceProcessForm processForm = new GroupResourceProcessForm();
    processForm.setGroupInfo(groupService.get(groupId));
    String username = context.getApplicant();
    List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupId);
    List<InlongStreamInfo> inlongStreamInfoList = CommonBeanUtils.copyListProperties(inlongStreamEntityList, InlongStreamInfo::new);
    processForm.setInlongStreamInfoList(inlongStreamInfoList);
    workflowService.start(ProcessName.CREATE_GROUP_RESOURCE, username, processForm);
    return ListenerResult.success();
}

Example 14 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongStreamServiceImpl method listAllWithGroupId.

@Override
public PageInfo<FullStreamResponse> listAllWithGroupId(InlongStreamPageRequest request) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin to list full inlong stream page by {}", request);
    }
    Preconditions.checkNotNull(request, "request is empty");
    Preconditions.checkNotNull(request.getInlongGroupId(), Constant.GROUP_ID_IS_EMPTY);
    // 1. Query all valid data sources under groupId
    String groupId = request.getInlongGroupId();
    // The person in charge of the inlong group has the authority of all inlong streams
    InlongGroupEntity inlongGroupEntity = groupMapper.selectByGroupId(groupId);
    Preconditions.checkNotNull(inlongGroupEntity, "inlong group not found by groupId=" + groupId);
    String inCharges = inlongGroupEntity.getInCharges();
    request.setInCharges(inCharges);
    PageHelper.startPage(request.getPageNum(), request.getPageSize());
    Page<InlongStreamEntity> page = (Page<InlongStreamEntity>) streamMapper.selectByCondition(request);
    List<InlongStreamInfo> streamInfoList = CommonBeanUtils.copyListProperties(page, InlongStreamInfo::new);
    // Convert and encapsulate the paged results
    List<FullStreamResponse> responseList = new ArrayList<>(streamInfoList.size());
    for (InlongStreamInfo streamInfo : streamInfoList) {
        // 2.1 Set the extended information and field information of the inlong stream
        String streamId = streamInfo.getInlongStreamId();
        setStreamExtAndField(groupId, streamId, streamInfo);
        // 2.3 Set the inlong stream to the result sub-object
        FullStreamResponse pageInfo = new FullStreamResponse();
        pageInfo.setStreamInfo(streamInfo);
        // 3. Query the basic and detailed information of the data source
        String dataSourceType = streamInfo.getDataSourceType();
        if (StringUtils.isEmpty(dataSourceType)) {
            continue;
        }
        switch(dataSourceType.toUpperCase(Locale.ROOT)) {
            case Constant.DATA_SOURCE_FILE:
                SourceFileBasicInfo fileBasicInfo = sourceFileService.getBasicByIdentifier(groupId, streamId);
                pageInfo.setFileBasicInfo(fileBasicInfo);
                List<SourceFileDetailInfo> fileDetailInfoList = sourceFileService.listDetailByIdentifier(groupId, streamId);
                pageInfo.setFileDetailInfoList(fileDetailInfoList);
                break;
            case Constant.DATA_SOURCE_DB:
                SourceDbBasicInfo dbBasicInfo = sourceDbService.getBasicByIdentifier(groupId, streamId);
                pageInfo.setDbBasicInfo(dbBasicInfo);
                List<SourceDbDetailInfo> dbDetailInfoList = sourceDbService.listDetailByIdentifier(groupId, streamId);
                pageInfo.setDbDetailInfoList(dbDetailInfoList);
                break;
            case Constant.DATA_SOURCE_AUTO_PUSH:
                break;
            default:
                throw new BusinessException(ErrorCodeEnum.SOURCE_TYPE_NOT_SUPPORTED);
        }
        // 4. Query stream sources information
        List<SourceResponse> sourceList = sourceService.listSource(groupId, streamId);
        pageInfo.setSourceInfo(sourceList);
        // 5. Query various stream sinks and its extended information, field information
        List<SinkResponse> sinkList = sinkService.listSink(groupId, streamId);
        pageInfo.setSinkInfo(sinkList);
        // 6. Add a single result to the paginated list
        responseList.add(pageInfo);
    }
    PageInfo<FullStreamResponse> pageInfo = new PageInfo<>(responseList);
    pageInfo.setTotal(pageInfo.getTotal());
    LOGGER.debug("success to list full inlong stream info");
    return pageInfo;
}

Example 15 with InlongStreamEntity

use of org.apache.inlong.manager.dao.entity.InlongStreamEntity in project incubator-inlong by apache.

the class InlongStreamServiceImpl method exist.

@Override
public Boolean exist(String groupId, String streamId) {
    Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
    InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
    return streamEntity != null;
}