Examples with SourceType org.apache.inlong.manager.common.enums.SourceType used on opensource projects

Example 1 with SourceType

use of org.apache.inlong.manager.common.enums.SourceType in project incubator-inlong by apache.

the class InlongParser method parseSourceList.

public static PageInfo<SourceListResponse> parseSourceList(Response response) {
    Object data = response.getData();
    String pageInfoJson = GsonUtil.toJson(data);
    PageInfo<SourceListResponse> pageInfo = GsonUtil.fromJson(pageInfoJson, new TypeToken<PageInfo<SourceListResponse>>() {
    }.getType());
    if (pageInfo.getList() != null && !pageInfo.getList().isEmpty()) {
        SourceListResponse sourceListResponse = pageInfo.getList().get(0);
        SourceType sourceType = SourceType.forType(sourceListResponse.getSourceType());
        if (sourceType == BINLOG) {
            return GsonUtil.fromJson(pageInfoJson, new TypeToken<PageInfo<BinlogSourceListResponse>>() {
            }.getType());
        }
        if (sourceType == KAFKA) {
            return GsonUtil.fromJson(pageInfoJson, new TypeToken<PageInfo<KafkaSourceListResponse>>() {
            }.getType());
        }
        throw new IllegalArgumentException(String.format("Unsupported sourceType=%s for Inlong", sourceType));
    } else {
        return new PageInfo<>();
    }
}

Example 2 with SourceType

use of org.apache.inlong.manager.common.enums.SourceType in project incubator-inlong by apache.

the class InlongStreamSourceTransfer method parseStreamSource.

public static StreamSource parseStreamSource(SourceListResponse sourceListResponse) {
    String type = sourceListResponse.getSourceType();
    SourceType sourceType = SourceType.forType(type);
    if (sourceType == SourceType.KAFKA && sourceListResponse instanceof KafkaSourceListResponse) {
        return parseKafkaSource((KafkaSourceListResponse) sourceListResponse);
    }
    if (sourceType == SourceType.BINLOG && sourceListResponse instanceof BinlogSourceListResponse) {
        return parseMySQLBinlogSource((BinlogSourceListResponse) sourceListResponse);
    }
    throw new IllegalArgumentException(String.format("Unsupported source type : %s for Inlong", sourceType));
}

Example 3 with SourceType

use of org.apache.inlong.manager.common.enums.SourceType in project incubator-inlong by apache.

the class AbstractSourceOperateListener method createSourceRequest.

public SourceRequest createSourceRequest(SourceResponse sourceResponse) {
    String sourceType = sourceResponse.getSourceType();
    SourceType type = SourceType.valueOf(sourceType);
    switch(type) {
        case BINLOG:
            return CommonBeanUtils.copyProperties((BinlogSourceResponse) sourceResponse, BinlogSourceRequest::new);
        case KAFKA:
            return CommonBeanUtils.copyProperties((KafkaSourceResponse) sourceResponse, KafkaSourceRequest::new);
        default:
            throw new IllegalArgumentException(String.format("Unsupported type=%s for DataSourceOperateListener", type));
    }
}

Example 4 with SourceType

use of org.apache.inlong.manager.common.enums.SourceType in project incubator-inlong by apache.

the class StreamSourceServiceImpl method listByCondition.

@Override
public PageInfo<? extends SourceListResponse> listByCondition(SourcePageRequest request) {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin to list source page by " + request);
    }
    Preconditions.checkNotNull(request.getInlongGroupId(), Constant.GROUP_ID_IS_EMPTY);
    PageHelper.startPage(request.getPageNum(), request.getPageSize());
    List<StreamSourceEntity> entityPage = sourceMapper.selectByCondition(request);
    // Encapsulate the paging query results into the PageInfo object to obtain related paging information
    Map<SourceType, Page<StreamSourceEntity>> sourceMap = Maps.newHashMap();
    for (StreamSourceEntity streamSource : entityPage) {
        SourceType sourceType = SourceType.forType(streamSource.getSourceType());
        sourceMap.computeIfAbsent(sourceType, k -> new Page<>()).add(streamSource);
    }
    List<SourceListResponse> sourceListResponses = Lists.newArrayList();
    for (Map.Entry<SourceType, Page<StreamSourceEntity>> entry : sourceMap.entrySet()) {
        SourceType sourceType = entry.getKey();
        StreamSourceOperation operation = operationFactory.getInstance(sourceType);
        PageInfo<? extends SourceListResponse> pageInfo = operation.getPageInfo(entry.getValue());
        sourceListResponses.addAll(pageInfo.getList());
    }
    PageInfo<? extends SourceListResponse> pageInfo = PageInfo.of(sourceListResponses);
    LOGGER.debug("success to list source page");
    return pageInfo;
}

Example 5 with SourceType

use of org.apache.inlong.manager.common.enums.SourceType in project incubator-inlong by apache.

the class InlongParser method parseStreamList.

public static List<FullStreamResponse> parseStreamList(Response response) {
    Object data = response.getData();
    JsonObject pageInfoJson = GsonUtil.fromJson(GsonUtil.toJson(data), JsonObject.class);
    JsonArray fullStreamArray = pageInfoJson.getAsJsonArray("list");
    List<FullStreamResponse> list = Lists.newArrayList();
    for (int i = 0; i < fullStreamArray.size(); i++) {
        JsonObject fullStreamJson = (JsonObject) fullStreamArray.get(i);
        FullStreamResponse fullStreamResponse = GsonUtil.fromJson(fullStreamJson.toString(), FullStreamResponse.class);
        list.add(fullStreamResponse);
        // Parse sourceResponse in each stream
        JsonArray sourceJsonArr = fullStreamJson.getAsJsonArray(SOURCE_INFO);
        List<SourceResponse> sourceResponses = Lists.newArrayList();
        fullStreamResponse.setSourceInfo(sourceResponses);
        for (int j = 0; j < sourceJsonArr.size(); j++) {
            JsonObject sourceJson = (JsonObject) sourceJsonArr.get(i);
            String type = sourceJson.get(SOURCE_TYPE).getAsString();
            SourceType sourceType = SourceType.forType(type);
            switch(sourceType) {
                case BINLOG:
                    BinlogSourceResponse binlogSourceResponse = GsonUtil.fromJson(sourceJson.toString(), BinlogSourceResponse.class);
                    sourceResponses.add(binlogSourceResponse);
                    break;
                case KAFKA:
                    KafkaSourceResponse kafkaSourceResponse = GsonUtil.fromJson(sourceJson.toString(), KafkaSourceResponse.class);
                    sourceResponses.add(kafkaSourceResponse);
                    break;
                default:
                    throw new RuntimeException(String.format("Unsupport sourceType=%s for Inlong", sourceType));
            }
        }
        // Parse sinkResponse in each stream
        JsonArray sinkJsonArr = fullStreamJson.getAsJsonArray(SINK_INFO);
        List<SinkResponse> sinkResponses = Lists.newArrayList();
        fullStreamResponse.setSinkInfo(sinkResponses);
        for (int j = 0; j < sinkJsonArr.size(); j++) {
            JsonObject sinkJson = (JsonObject) sinkJsonArr.get(i);
            String type = sinkJson.get(SINK_TYPE).getAsString();
            SinkType sinkType = SinkType.forType(type);
            switch(sinkType) {
                case HIVE:
                    HiveSinkResponse hiveSinkResponse = GsonUtil.fromJson(sinkJson.toString(), HiveSinkResponse.class);
                    sinkResponses.add(hiveSinkResponse);
                    break;
                case KAFKA:
                    KafkaSinkResponse kafkaSinkResponse = GsonUtil.fromJson(sinkJson.toString(), KafkaSinkResponse.class);
                    sinkResponses.add(kafkaSinkResponse);
                    break;
                case ICEBERG:
                    IcebergSinkResponse icebergSinkResponse = GsonUtil.fromJson(sinkJson.toString(), IcebergSinkResponse.class);
                    sinkResponses.add(icebergSinkResponse);
                    break;
                case CLICKHOUSE:
                    ClickHouseSinkResponse clickHouseSinkResponse = GsonUtil.fromJson(sinkJson.toString(), ClickHouseSinkResponse.class);
                    sinkResponses.add(clickHouseSinkResponse);
                    break;
                default:
                    throw new RuntimeException(String.format("Unsupport sinkType=%s for Inlong", sinkType));
            }
        }
    }
    return list;
}