Examples with InlongStreamInfo org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo used on opensource projects

Example 11 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class DefaultInlongStreamBuilder method initOrUpdate.

@Override
public InlongStream initOrUpdate() {
    InlongStreamInfo dataStreamInfo = streamContext.getStreamInfo();
    Pair<Boolean, InlongStreamInfo> existMsg = managerClient.isStreamExists(dataStreamInfo);
    if (existMsg.getKey()) {
        Pair<Boolean, String> updateMsg = managerClient.updateStreamInfo(dataStreamInfo);
        if (updateMsg.getKey()) {
            List<SourceRequest> sourceRequests = Lists.newArrayList(streamContext.getSourceRequests().values());
            for (SourceRequest sourceRequest : sourceRequests) {
                sourceRequest.setId(initOrUpdateSource(sourceRequest));
            }
            List<SinkRequest> sinkRequests = Lists.newArrayList(streamContext.getSinkRequests().values());
            for (SinkRequest sinkRequest : sinkRequests) {
                sinkRequest.setId(initOrUpdateSink(sinkRequest));
            }
        } else {
            throw new RuntimeException(String.format("Update data stream failed:%s", updateMsg.getValue()));
        }
        return inlongStream;
    } else {
        return init();
    }
}

Example 12 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class DefaultInlongStreamBuilder method init.

@Override
public InlongStream init() {
    InlongStreamInfo streamInfo = streamContext.getStreamInfo();
    String streamIndex = managerClient.createStreamInfo(streamInfo);
    streamInfo.setId(Double.valueOf(streamIndex).intValue());
    // Create source and update index
    List<SourceRequest> sourceRequests = Lists.newArrayList(streamContext.getSourceRequests().values());
    for (SourceRequest sourceRequest : sourceRequests) {
        String sourceIndex = managerClient.createSource(sourceRequest);
        sourceRequest.setId(Double.valueOf(sourceIndex).intValue());
    }
    // Create sink and update index
    List<SinkRequest> sinkRequests = Lists.newArrayList(streamContext.getSinkRequests().values());
    for (SinkRequest sinkRequest : sinkRequests) {
        String sinkIndex = managerClient.createSink(sinkRequest);
        sinkRequest.setId(Double.valueOf(sinkIndex).intValue());
    }
    return inlongStream;
}

Example 13 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class StartCreateGroupProcessListener method listen.

/**
 * Initiate the process of creating inlong group resources after new inlong group access approved
 */
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    NewGroupProcessForm form = (NewGroupProcessForm) context.getProcessForm();
    String groupId = form.getInlongGroupId();
    GroupResourceProcessForm processForm = new GroupResourceProcessForm();
    processForm.setGroupInfo(groupService.get(groupId));
    String username = context.getApplicant();
    List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupId);
    List<InlongStreamInfo> inlongStreamInfoList = CommonBeanUtils.copyListProperties(inlongStreamEntityList, InlongStreamInfo::new);
    processForm.setInlongStreamInfoList(inlongStreamInfoList);
    workflowService.start(ProcessName.CREATE_GROUP_RESOURCE, username, processForm);
    return ListenerResult.success();
}

Example 14 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class InlongStreamServiceTest method saveInlongStream.

/**
 * Test save inlong stream
 */
public Integer saveInlongStream(String groupId, String streamId, String operator) {
    InlongStreamInfo streamInfo;
    try {
        streamInfo = streamService.get(groupId, streamId);
        if (streamInfo != null) {
            return streamInfo.getId();
        }
    } catch (Exception e) {
    // ignore
    }
    groupServiceTest.saveGroup(globalGroupName, operator);
    streamInfo = new InlongStreamInfo();
    streamInfo.setInlongGroupId(groupId);
    streamInfo.setInlongStreamId(streamId);
    streamInfo.setDataEncoding("UTF-8");
    return streamService.save(streamInfo, operator);
}

Example 15 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class InlongStreamServiceImpl method listAllWithGroupId.

@Override
public PageInfo<FullStreamResponse> listAllWithGroupId(InlongStreamPageRequest request) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin to list full inlong stream page by {}", request);
    }
    Preconditions.checkNotNull(request, "request is empty");
    Preconditions.checkNotNull(request.getInlongGroupId(), Constant.GROUP_ID_IS_EMPTY);
    // 1. Query all valid data sources under groupId
    String groupId = request.getInlongGroupId();
    // The person in charge of the inlong group has the authority of all inlong streams
    InlongGroupEntity inlongGroupEntity = groupMapper.selectByGroupId(groupId);
    Preconditions.checkNotNull(inlongGroupEntity, "inlong group not found by groupId=" + groupId);
    String inCharges = inlongGroupEntity.getInCharges();
    request.setInCharges(inCharges);
    PageHelper.startPage(request.getPageNum(), request.getPageSize());
    Page<InlongStreamEntity> page = (Page<InlongStreamEntity>) streamMapper.selectByCondition(request);
    List<InlongStreamInfo> streamInfoList = CommonBeanUtils.copyListProperties(page, InlongStreamInfo::new);
    // Convert and encapsulate the paged results
    List<FullStreamResponse> responseList = new ArrayList<>(streamInfoList.size());
    for (InlongStreamInfo streamInfo : streamInfoList) {
        // 2.1 Set the extended information and field information of the inlong stream
        String streamId = streamInfo.getInlongStreamId();
        setStreamExtAndField(groupId, streamId, streamInfo);
        // 2.3 Set the inlong stream to the result sub-object
        FullStreamResponse pageInfo = new FullStreamResponse();
        pageInfo.setStreamInfo(streamInfo);
        // 3. Query the basic and detailed information of the data source
        String dataSourceType = streamInfo.getDataSourceType();
        if (StringUtils.isEmpty(dataSourceType)) {
            continue;
        }
        switch(dataSourceType.toUpperCase(Locale.ROOT)) {
            case Constant.DATA_SOURCE_FILE:
                SourceFileBasicInfo fileBasicInfo = sourceFileService.getBasicByIdentifier(groupId, streamId);
                pageInfo.setFileBasicInfo(fileBasicInfo);
                List<SourceFileDetailInfo> fileDetailInfoList = sourceFileService.listDetailByIdentifier(groupId, streamId);
                pageInfo.setFileDetailInfoList(fileDetailInfoList);
                break;
            case Constant.DATA_SOURCE_DB:
                SourceDbBasicInfo dbBasicInfo = sourceDbService.getBasicByIdentifier(groupId, streamId);
                pageInfo.setDbBasicInfo(dbBasicInfo);
                List<SourceDbDetailInfo> dbDetailInfoList = sourceDbService.listDetailByIdentifier(groupId, streamId);
                pageInfo.setDbDetailInfoList(dbDetailInfoList);
                break;
            case Constant.DATA_SOURCE_AUTO_PUSH:
                break;
            default:
                throw new BusinessException(ErrorCodeEnum.SOURCE_TYPE_NOT_SUPPORTED);
        }
        // 4. Query stream sources information
        List<SourceResponse> sourceList = sourceService.listSource(groupId, streamId);
        pageInfo.setSourceInfo(sourceList);
        // 5. Query various stream sinks and its extended information, field information
        List<SinkResponse> sinkList = sinkService.listSink(groupId, streamId);
        pageInfo.setSinkInfo(sinkList);
        // 6. Add a single result to the paginated list
        responseList.add(pageInfo);
    }
    PageInfo<FullStreamResponse> pageInfo = new PageInfo<>(responseList);
    pageInfo.setTotal(pageInfo.getTotal());
    LOGGER.debug("success to list full inlong stream info");
    return pageInfo;
}