Examples with InlongGroupPulsarInfo org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo used on opensource projects

Example 1 with InlongGroupPulsarInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo in project incubator-inlong by apache.

the class InlongGroupTransfer method createGroupInfo.

public static InlongGroupInfo createGroupInfo(InlongGroupConf groupConf) {
    InlongGroupInfo groupInfo = new InlongGroupInfo();
    AssertUtil.hasLength(groupConf.getGroupName(), "GroupName should not be empty");
    groupInfo.setName(groupConf.getGroupName());
    groupInfo.setCnName(groupConf.getCnName());
    groupInfo.setDescription(groupConf.getDescription());
    groupInfo.setZookeeperEnabled(groupConf.isZookeeperEnabled() ? 1 : 0);
    groupInfo.setDailyRecords(groupConf.getDailyRecords().intValue());
    groupInfo.setPeakRecords(groupConf.getPeakRecords().intValue());
    groupInfo.setMaxLength(groupConf.getMaxLength());
    groupInfo.setProxyClusterId(groupConf.getProxyClusterId());
    MqBaseConf mqConf = groupConf.getMqBaseConf();
    MqType mqType = MqType.NONE;
    if (null != mqConf) {
        mqType = mqConf.getType();
        groupInfo.setMiddlewareType(mqType.name());
    }
    groupInfo.setInCharges(groupConf.getOperator());
    groupInfo.setExtList(Lists.newArrayList());
    groupInfo.setCreator(groupConf.getOperator());
    if (mqType == MqType.PULSAR || mqType == MqType.TDMQ_PULSAR) {
        PulsarBaseConf pulsarBaseConf = (PulsarBaseConf) mqConf;
        groupInfo.setMqResourceObj(pulsarBaseConf.getNamespace());
        InlongGroupPulsarInfo pulsarInfo = createPulsarInfo(pulsarBaseConf);
        groupInfo.setMqExtInfo(pulsarInfo);
        List<InlongGroupExtInfo> extInfos = createPulsarExtInfo(pulsarBaseConf);
        groupInfo.getExtList().addAll(extInfos);
        groupInfo.setTopicPartitionNum(pulsarBaseConf.getTopicPartitionNum());
    } else if (mqType == MqType.TUBE) {
        TubeBaseConf tubeBaseConf = (TubeBaseConf) mqConf;
        List<InlongGroupExtInfo> extInfos = createTubeExtInfo(tubeBaseConf);
        groupInfo.setMqResourceObj(tubeBaseConf.getGroupName());
        groupInfo.getExtList().addAll(extInfos);
        groupInfo.setTopicPartitionNum(tubeBaseConf.getTopicPartitionNum());
    }
    SortBaseConf sortBaseConf = groupConf.getSortBaseConf();
    SortType sortType = sortBaseConf.getType();
    if (sortType == SortType.FLINK) {
        FlinkSortBaseConf flinkSortBaseConf = (FlinkSortBaseConf) sortBaseConf;
        List<InlongGroupExtInfo> sortExtInfos = createFlinkExtInfo(flinkSortBaseConf);
        groupInfo.getExtList().addAll(sortExtInfos);
    } else if (sortType == SortType.USER_DEFINED) {
        UserDefinedSortConf udf = (UserDefinedSortConf) sortBaseConf;
        List<InlongGroupExtInfo> sortExtInfos = createUserDefinedSortExtInfo(udf);
        groupInfo.getExtList().addAll(sortExtInfos);
    } else {
    // todo local
    }
    return groupInfo;
}

Example 2 with InlongGroupPulsarInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo in project incubator-inlong by apache.

the class InlongParser method parseGroupInfo.

public static InlongGroupResponse parseGroupInfo(Response response) {
    Object data = response.getData();
    JsonObject groupJson = GsonUtil.fromJson(GsonUtil.toJson(data), JsonObject.class);
    InlongGroupResponse inlongGroupResponse = GsonUtil.fromJson(GsonUtil.toJson(data), InlongGroupResponse.class);
    JsonObject mqExtInfo = groupJson.getAsJsonObject(MQ_EXT_INFO);
    if (mqExtInfo != null && mqExtInfo.get(MIDDLEWARE_TYPE) != null) {
        String middlewareType = mqExtInfo.get(MIDDLEWARE_TYPE).getAsString();
        if (Constant.MIDDLEWARE_PULSAR.equals(middlewareType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(middlewareType)) {
            InlongGroupPulsarInfo pulsarInfo = GsonUtil.fromJson(mqExtInfo.toString(), InlongGroupPulsarInfo.class);
            inlongGroupResponse.setMqExtInfo(pulsarInfo);
        }
    }
    return inlongGroupResponse;
}

Example 3 with InlongGroupPulsarInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo in project incubator-inlong by apache.

the class InlongGroupTransfer method parsePulsarConf.

private static PulsarBaseConf parsePulsarConf(InlongGroupResponse groupResponse) {
    PulsarBaseConf pulsarBaseConf = new PulsarBaseConf();
    pulsarBaseConf.setNamespace(groupResponse.getMqResourceObj());
    InlongGroupPulsarInfo inlongGroupPulsarInfo = (InlongGroupPulsarInfo) groupResponse.getMqExtInfo();
    pulsarBaseConf.setAckQuorum(inlongGroupPulsarInfo.getAckQuorum());
    pulsarBaseConf.setWriteQuorum(inlongGroupPulsarInfo.getWriteQuorum());
    pulsarBaseConf.setEnsemble(inlongGroupPulsarInfo.getEnsemble());
    pulsarBaseConf.setTtl(inlongGroupPulsarInfo.getTtl());
    pulsarBaseConf.setTenant(inlongGroupPulsarInfo.getTenant());
    pulsarBaseConf.setRetentionTime(inlongGroupPulsarInfo.getRetentionTime());
    pulsarBaseConf.setRetentionSize(inlongGroupPulsarInfo.getRetentionSize());
    pulsarBaseConf.setRetentionSizeUnit(inlongGroupPulsarInfo.getRetentionSizeUnit());
    pulsarBaseConf.setRetentionTimeUnit(inlongGroupPulsarInfo.getRetentionTimeUnit());
    pulsarBaseConf.setEnableCreateResource(inlongGroupPulsarInfo.getEnableCreateResource() == 1 ? true : false);
    List<InlongGroupExtInfo> groupExtInfos = groupResponse.getExtList();
    for (InlongGroupExtInfo extInfo : groupExtInfos) {
        if (extInfo.getKeyName().equals(InlongGroupSettings.PULSAR_ADMIN_URL)) {
            pulsarBaseConf.setPulsarAdminUrl(extInfo.getKeyValue());
        }
        if (extInfo.getKeyName().equals(InlongGroupSettings.PULSAR_SERVICE_URL)) {
            pulsarBaseConf.setPulsarServiceUrl(extInfo.getKeyValue());
        }
    }
    return pulsarBaseConf;
}

Example 4 with InlongGroupPulsarInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo in project incubator-inlong by apache.

the class PulsarEventSelector method accept.

@Override
public boolean accept(WorkflowContext context) {
    ProcessForm processForm = context.getProcessForm();
    if (!(processForm instanceof GroupResourceProcessForm)) {
        return false;
    }
    GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
    String mqType = form.getGroupInfo().getMiddlewareType();
    if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
        InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) form.getGroupInfo().getMqExtInfo();
        return pulsarInfo.getEnableCreateResource() == 1;
    }
    log.warn("no need to create pulsar subscription group for groupId={}, as the middlewareType={}", form.getInlongGroupId(), mqType);
    return false;
}

Example 5 with InlongGroupPulsarInfo

use of org.apache.inlong.manager.common.pojo.group.InlongGroupPulsarInfo in project incubator-inlong by apache.

the class SourceInfoUtils method createPulsarSourceInfo.

/**
 * Create source info for Pulsar
 */
private static SourceInfo createPulsarSourceInfo(PulsarClusterInfo pulsarCluster, ClusterBean clusterBean, InlongGroupInfo groupInfo, InlongStreamInfo streamInfo, DeserializationInfo deserializationInfo, List<FieldInfo> fieldInfos) {
    String topicName = streamInfo.getMqResourceObj();
    InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) groupInfo.getMqExtInfo();
    String tenant = clusterBean.getDefaultTenant();
    if (StringUtils.isNotEmpty(pulsarInfo.getTenant())) {
        tenant = pulsarInfo.getTenant();
    }
    final String namespace = groupInfo.getMqResourceObj();
    // Full name of topic in Pulsar
    final String fullTopicName = "persistent://" + tenant + "/" + namespace + "/" + topicName;
    final String consumerGroup = clusterBean.getAppName() + "_" + topicName + "_consumer_group";
    FieldInfo[] fieldInfosArr = fieldInfos.toArray(new FieldInfo[0]);
    String type = pulsarCluster.getType();
    if (StringUtils.isNotEmpty(type) && Constant.MIDDLEWARE_TDMQ_PULSAR.equals(type)) {
        return new TDMQPulsarSourceInfo(pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, pulsarCluster.getToken(), deserializationInfo, fieldInfosArr);
    } else {
        return new PulsarSourceInfo(pulsarCluster.getAdminUrl(), pulsarCluster.getBrokerServiceUrl(), fullTopicName, consumerGroup, deserializationInfo, fieldInfosArr, pulsarCluster.getToken());
    }
}