Examples with SinkResponse org.apache.inlong.manager.common.pojo.sink.SinkResponse used on opensource projects

Example 16 with SinkResponse

use of org.apache.inlong.manager.common.pojo.sink.SinkResponse in project incubator-inlong by apache.

the class PushSortConfigListener method listen.

@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("begin to push sort config by context={}", context);
    }
    GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
    String groupId = form.getGroupInfo().getInlongGroupId();
    InlongGroupInfo groupInfo = groupService.get(groupId);
    // if streamId not null, just push the config belongs to the groupId and the streamId
    String streamId = form.getInlongStreamId();
    List<SinkResponse> sinkResponseList = streamSinkService.listSink(groupId, streamId);
    if (CollectionUtils.isEmpty(sinkResponseList)) {
        LOGGER.warn("Sink not found by groupId={}", groupId);
        return ListenerResult.success();
    }
    for (SinkResponse sinkResponse : sinkResponseList) {
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("sink info: {}", sinkResponse);
        }
        DataFlowInfo dataFlowInfo = commonOperateService.createDataFlow(groupInfo, sinkResponse);
        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("try to push config to sort: {}", JsonUtils.toJson(dataFlowInfo));
        }
        Integer sinkId = sinkResponse.getId();
        try {
            String zkUrl = clusterBean.getZkUrl();
            String zkRoot = clusterBean.getZkRoot();
            // push data flow info to zk
            String sortClusterName = clusterBean.getAppName();
            ZkTools.updateDataFlowInfo(dataFlowInfo, sortClusterName, sinkId, zkUrl, zkRoot);
            // add sink id to zk
            ZkTools.addDataFlowToCluster(sortClusterName, sinkId, zkUrl, zkRoot);
        } catch (Exception e) {
            LOGGER.error("push sort config to zookeeper failed, sinkId={} ", sinkId, e);
            throw new WorkflowListenerException("push sort config to zookeeper failed: " + e.getMessage());
        }
    }
    return ListenerResult.success();
}

Example 17 with SinkResponse

use of org.apache.inlong.manager.common.pojo.sink.SinkResponse in project incubator-inlong by apache.

the class InlongParser method parseStreamList.

public static List<FullStreamResponse> parseStreamList(Response response) {
    Object data = response.getData();
    JsonObject pageInfoJson = GsonUtil.fromJson(GsonUtil.toJson(data), JsonObject.class);
    JsonArray fullStreamArray = pageInfoJson.getAsJsonArray("list");
    List<FullStreamResponse> list = Lists.newArrayList();
    for (int i = 0; i < fullStreamArray.size(); i++) {
        JsonObject fullStreamJson = (JsonObject) fullStreamArray.get(i);
        FullStreamResponse fullStreamResponse = GsonUtil.fromJson(fullStreamJson.toString(), FullStreamResponse.class);
        list.add(fullStreamResponse);
        // Parse sourceResponse in each stream
        JsonArray sourceJsonArr = fullStreamJson.getAsJsonArray(SOURCE_INFO);
        List<SourceResponse> sourceResponses = Lists.newArrayList();
        fullStreamResponse.setSourceInfo(sourceResponses);
        for (int j = 0; j < sourceJsonArr.size(); j++) {
            JsonObject sourceJson = (JsonObject) sourceJsonArr.get(i);
            String type = sourceJson.get(SOURCE_TYPE).getAsString();
            SourceType sourceType = SourceType.forType(type);
            switch(sourceType) {
                case BINLOG:
                    BinlogSourceResponse binlogSourceResponse = GsonUtil.fromJson(sourceJson.toString(), BinlogSourceResponse.class);
                    sourceResponses.add(binlogSourceResponse);
                    break;
                case KAFKA:
                    KafkaSourceResponse kafkaSourceResponse = GsonUtil.fromJson(sourceJson.toString(), KafkaSourceResponse.class);
                    sourceResponses.add(kafkaSourceResponse);
                    break;
                default:
                    throw new RuntimeException(String.format("Unsupport sourceType=%s for Inlong", sourceType));
            }
        }
        // Parse sinkResponse in each stream
        JsonArray sinkJsonArr = fullStreamJson.getAsJsonArray(SINK_INFO);
        List<SinkResponse> sinkResponses = Lists.newArrayList();
        fullStreamResponse.setSinkInfo(sinkResponses);
        for (int j = 0; j < sinkJsonArr.size(); j++) {
            JsonObject sinkJson = (JsonObject) sinkJsonArr.get(i);
            String type = sinkJson.get(SINK_TYPE).getAsString();
            SinkType sinkType = SinkType.forType(type);
            switch(sinkType) {
                case HIVE:
                    HiveSinkResponse hiveSinkResponse = GsonUtil.fromJson(sinkJson.toString(), HiveSinkResponse.class);
                    sinkResponses.add(hiveSinkResponse);
                    break;
                case KAFKA:
                    KafkaSinkResponse kafkaSinkResponse = GsonUtil.fromJson(sinkJson.toString(), KafkaSinkResponse.class);
                    sinkResponses.add(kafkaSinkResponse);
                    break;
                case ICEBERG:
                    IcebergSinkResponse icebergSinkResponse = GsonUtil.fromJson(sinkJson.toString(), IcebergSinkResponse.class);
                    sinkResponses.add(icebergSinkResponse);
                    break;
                case CLICKHOUSE:
                    ClickHouseSinkResponse clickHouseSinkResponse = GsonUtil.fromJson(sinkJson.toString(), ClickHouseSinkResponse.class);
                    sinkResponses.add(clickHouseSinkResponse);
                    break;
                default:
                    throw new RuntimeException(String.format("Unsupport sinkType=%s for Inlong", sinkType));
            }
        }
    }
    return list;
}