Examples with SourceResponse org.apache.inlong.manager.common.pojo.source.SourceResponse used on opensource projects

Example 6 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class StreamSourceServiceTest method testGetAndUpdate.

@Test
public void testGetAndUpdate() {
    Integer id = this.saveSource();
    SourceResponse response = sourceService.get(id, Constant.SOURCE_BINLOG);
    Assert.assertEquals(globalGroupId, response.getInlongGroupId());
    BinlogSourceResponse binlogResponse = (BinlogSourceResponse) response;
    BinlogSourceRequest request = CommonBeanUtils.copyProperties(binlogResponse, BinlogSourceRequest::new);
    boolean result = sourceService.update(request, globalOperator);
    Assert.assertTrue(result);
    sourceService.delete(id, Constant.SOURCE_BINLOG, globalOperator);
}

Example 7 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class DataSourceListenerTest method testRestartSource.

@Test
public void testRestartSource() {
    // testFrozenSource();
    groupInfo = initGroupForm("PULSAR");
    groupInfo.setStatus(GroupState.CONFIG_SUCCESSFUL.getCode());
    groupService.update(groupInfo.genRequest(), OPERATOR);
    groupInfo.setStatus(GroupState.SUSPENDED.getCode());
    groupService.update(groupInfo.genRequest(), OPERATOR);
    final int sourceId = createBinlogSource(groupInfo);
    streamSourceService.updateStatus(groupInfo.getInlongGroupId(), null, SourceState.SOURCE_NORMAL.getCode(), OPERATOR);
    form = new UpdateGroupProcessForm();
    form.setGroupInfo(groupInfo);
    form.setOperateType(OperateType.RESTART);
    WorkflowContext context = workflowEngine.processService().start(ProcessName.RESTART_GROUP_PROCESS.name(), applicant, form);
    WorkflowResult result = WorkflowBeanUtils.result(context);
    ProcessResponse response = result.getProcessInfo();
    Assert.assertSame(response.getStatus(), ProcessStatus.COMPLETED);
    WorkflowProcess process = context.getProcess();
    WorkflowTask task = process.getTaskByName("restartSource");
    Assert.assertTrue(task instanceof ServiceTask);
    SourceResponse sourceResponse = streamSourceService.get(sourceId, SourceType.BINLOG.toString());
    Assert.assertSame(SourceState.forCode(sourceResponse.getStatus()), SourceState.TO_BE_ISSUED_ACTIVE);
}

Example 8 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class InlongStreamServiceImpl method listAllWithGroupId.

@Override
public PageInfo<FullStreamResponse> listAllWithGroupId(InlongStreamPageRequest request) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin to list full inlong stream page by {}", request);
    }
    Preconditions.checkNotNull(request, "request is empty");
    Preconditions.checkNotNull(request.getInlongGroupId(), Constant.GROUP_ID_IS_EMPTY);
    // 1. Query all valid data sources under groupId
    String groupId = request.getInlongGroupId();
    // The person in charge of the inlong group has the authority of all inlong streams
    InlongGroupEntity inlongGroupEntity = groupMapper.selectByGroupId(groupId);
    Preconditions.checkNotNull(inlongGroupEntity, "inlong group not found by groupId=" + groupId);
    String inCharges = inlongGroupEntity.getInCharges();
    request.setInCharges(inCharges);
    PageHelper.startPage(request.getPageNum(), request.getPageSize());
    Page<InlongStreamEntity> page = (Page<InlongStreamEntity>) streamMapper.selectByCondition(request);
    List<InlongStreamInfo> streamInfoList = CommonBeanUtils.copyListProperties(page, InlongStreamInfo::new);
    // Convert and encapsulate the paged results
    List<FullStreamResponse> responseList = new ArrayList<>(streamInfoList.size());
    for (InlongStreamInfo streamInfo : streamInfoList) {
        // 2.1 Set the extended information and field information of the inlong stream
        String streamId = streamInfo.getInlongStreamId();
        setStreamExtAndField(groupId, streamId, streamInfo);
        // 2.3 Set the inlong stream to the result sub-object
        FullStreamResponse pageInfo = new FullStreamResponse();
        pageInfo.setStreamInfo(streamInfo);
        // 3. Query the basic and detailed information of the data source
        String dataSourceType = streamInfo.getDataSourceType();
        if (StringUtils.isEmpty(dataSourceType)) {
            continue;
        }
        switch(dataSourceType.toUpperCase(Locale.ROOT)) {
            case Constant.DATA_SOURCE_FILE:
                SourceFileBasicInfo fileBasicInfo = sourceFileService.getBasicByIdentifier(groupId, streamId);
                pageInfo.setFileBasicInfo(fileBasicInfo);
                List<SourceFileDetailInfo> fileDetailInfoList = sourceFileService.listDetailByIdentifier(groupId, streamId);
                pageInfo.setFileDetailInfoList(fileDetailInfoList);
                break;
            case Constant.DATA_SOURCE_DB:
                SourceDbBasicInfo dbBasicInfo = sourceDbService.getBasicByIdentifier(groupId, streamId);
                pageInfo.setDbBasicInfo(dbBasicInfo);
                List<SourceDbDetailInfo> dbDetailInfoList = sourceDbService.listDetailByIdentifier(groupId, streamId);
                pageInfo.setDbDetailInfoList(dbDetailInfoList);
                break;
            case Constant.DATA_SOURCE_AUTO_PUSH:
                break;
            default:
                throw new BusinessException(ErrorCodeEnum.SOURCE_TYPE_NOT_SUPPORTED);
        }
        // 4. Query stream sources information
        List<SourceResponse> sourceList = sourceService.listSource(groupId, streamId);
        pageInfo.setSourceInfo(sourceList);
        // 5. Query various stream sinks and its extended information, field information
        List<SinkResponse> sinkList = sinkService.listSink(groupId, streamId);
        pageInfo.setSinkInfo(sinkList);
        // 6. Add a single result to the paginated list
        responseList.add(pageInfo);
    }
    PageInfo<FullStreamResponse> pageInfo = new PageInfo<>(responseList);
    pageInfo.setTotal(pageInfo.getTotal());
    LOGGER.debug("success to list full inlong stream info");
    return pageInfo;
}

Example 9 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class InlongParser method parseStreamList.

public static List<FullStreamResponse> parseStreamList(Response response) {
    Object data = response.getData();
    JsonObject pageInfoJson = GsonUtil.fromJson(GsonUtil.toJson(data), JsonObject.class);
    JsonArray fullStreamArray = pageInfoJson.getAsJsonArray("list");
    List<FullStreamResponse> list = Lists.newArrayList();
    for (int i = 0; i < fullStreamArray.size(); i++) {
        JsonObject fullStreamJson = (JsonObject) fullStreamArray.get(i);
        FullStreamResponse fullStreamResponse = GsonUtil.fromJson(fullStreamJson.toString(), FullStreamResponse.class);
        list.add(fullStreamResponse);
        // Parse sourceResponse in each stream
        JsonArray sourceJsonArr = fullStreamJson.getAsJsonArray(SOURCE_INFO);
        List<SourceResponse> sourceResponses = Lists.newArrayList();
        fullStreamResponse.setSourceInfo(sourceResponses);
        for (int j = 0; j < sourceJsonArr.size(); j++) {
            JsonObject sourceJson = (JsonObject) sourceJsonArr.get(i);
            String type = sourceJson.get(SOURCE_TYPE).getAsString();
            SourceType sourceType = SourceType.forType(type);
            switch(sourceType) {
                case BINLOG:
                    BinlogSourceResponse binlogSourceResponse = GsonUtil.fromJson(sourceJson.toString(), BinlogSourceResponse.class);
                    sourceResponses.add(binlogSourceResponse);
                    break;
                case KAFKA:
                    KafkaSourceResponse kafkaSourceResponse = GsonUtil.fromJson(sourceJson.toString(), KafkaSourceResponse.class);
                    sourceResponses.add(kafkaSourceResponse);
                    break;
                default:
                    throw new RuntimeException(String.format("Unsupport sourceType=%s for Inlong", sourceType));
            }
        }
        // Parse sinkResponse in each stream
        JsonArray sinkJsonArr = fullStreamJson.getAsJsonArray(SINK_INFO);
        List<SinkResponse> sinkResponses = Lists.newArrayList();
        fullStreamResponse.setSinkInfo(sinkResponses);
        for (int j = 0; j < sinkJsonArr.size(); j++) {
            JsonObject sinkJson = (JsonObject) sinkJsonArr.get(i);
            String type = sinkJson.get(SINK_TYPE).getAsString();
            SinkType sinkType = SinkType.forType(type);
            switch(sinkType) {
                case HIVE:
                    HiveSinkResponse hiveSinkResponse = GsonUtil.fromJson(sinkJson.toString(), HiveSinkResponse.class);
                    sinkResponses.add(hiveSinkResponse);
                    break;
                case KAFKA:
                    KafkaSinkResponse kafkaSinkResponse = GsonUtil.fromJson(sinkJson.toString(), KafkaSinkResponse.class);
                    sinkResponses.add(kafkaSinkResponse);
                    break;
                case ICEBERG:
                    IcebergSinkResponse icebergSinkResponse = GsonUtil.fromJson(sinkJson.toString(), IcebergSinkResponse.class);
                    sinkResponses.add(icebergSinkResponse);
                    break;
                case CLICKHOUSE:
                    ClickHouseSinkResponse clickHouseSinkResponse = GsonUtil.fromJson(sinkJson.toString(), ClickHouseSinkResponse.class);
                    sinkResponses.add(clickHouseSinkResponse);
                    break;
                default:
                    throw new RuntimeException(String.format("Unsupport sinkType=%s for Inlong", sinkType));
            }
        }
    }
    return list;
}