use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.
the class CreatePulsarTopicForStreamTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
String streamId = form.getInlongStreamId();
InlongGroupInfo groupInfo = groupService.get(groupId);
InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
if (groupInfo == null || streamEntity == null) {
throw new WorkflowListenerException("inlong group or inlong stream not found with groupId=" + groupId + ", streamId=" + streamId);
}
log.info("begin to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
for (String cluster : pulsarClusters) {
String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
String pulsarTopic = streamEntity.getMqResourceObj();
this.createTopic(groupInfo, pulsarTopic, pulsarClusterInfo);
}
} catch (Exception e) {
log.error("create pulsar topic error for groupId={}, streamId={}", groupId, streamId, e);
throw new WorkflowListenerException("create pulsar topic error for groupId=" + groupId + ", streamId=" + streamId);
}
log.info("success to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
return ListenerResult.success();
}
use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.
the class CreatePulsarGroupTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
InlongGroupInfo groupInfo = groupService.get(groupId);
if (groupInfo == null) {
log.error("inlong group not found with groupId={}", groupId);
throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
}
// For Pulsar, each Stream corresponds to a Topic
List<InlongStreamEntity> streamEntities = streamMapper.selectByGroupId(groupId);
if (streamEntities == null || streamEntities.isEmpty()) {
log.warn("inlong stream is empty for groupId={}, skip to create pulsar subscription", groupId);
return ListenerResult.success();
}
PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
String tenant = clusterBean.getDefaultTenant();
String namespace = groupInfo.getMqResourceObj();
for (InlongStreamEntity streamEntity : streamEntities) {
PulsarTopicBean topicBean = new PulsarTopicBean();
topicBean.setTenant(tenant);
topicBean.setNamespace(namespace);
String topic = streamEntity.getMqResourceObj();
topicBean.setTopicName(topic);
List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
// Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
for (String cluster : pulsarClusters) {
String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
if (!exist) {
String topicFull = tenant + "/" + namespace + "/" + topic;
log.error("topic={} not exists in {}", topicFull, serviceUrl);
throw new WorkflowListenerException("topic=" + topicFull + " not exists in " + serviceUrl);
}
// Consumer naming rules: sortAppName_topicName_consumer_group
String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
// Insert the consumption data into the consumption table
consumptionService.saveSortConsumption(groupInfo, topic, subscription);
}
}
}
} catch (Exception e) {
log.error("create pulsar subscription error for groupId={}", groupId);
throw new WorkflowListenerException("create pulsar subscription error: " + e.getMessage());
}
log.info("success to create pulsar subscription for groupId={}", groupId);
return ListenerResult.success();
}
use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.
the class CreatePulsarResourceTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
log.info("begin to create pulsar resource for groupId={}", groupId);
InlongGroupInfo groupInfo = groupService.get(groupId);
if (groupInfo == null) {
throw new WorkflowListenerException("inlong group or pulsar cluster not found for groupId=" + groupId);
}
PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
for (String cluster : pulsarClusters) {
String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
this.createPulsarProcess(groupInfo, pulsarClusterInfo);
}
} catch (Exception e) {
log.error("create pulsar resource error for groupId={}", groupId, e);
throw new WorkflowListenerException("create pulsar resource error for groupId=" + groupId);
}
log.info("success to create pulsar resource for groupId={}", groupId);
return ListenerResult.success();
}
use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.
the class InlongGroupServiceImpl method get.
@Override
public InlongGroupInfo get(String groupId) {
LOGGER.debug("begin to get inlong group info by groupId={}", groupId);
Preconditions.checkNotNull(groupId, Constant.GROUP_ID_IS_EMPTY);
InlongGroupEntity entity = groupMapper.selectByGroupId(groupId);
if (entity == null) {
LOGGER.error("inlong group not found by groupId={}", groupId);
throw new BusinessException(ErrorCodeEnum.GROUP_NOT_FOUND);
}
InlongGroupInfo groupInfo = CommonBeanUtils.copyProperties(entity, InlongGroupInfo::new);
List<InlongGroupExtEntity> extEntityList = groupExtMapper.selectByGroupId(groupId);
List<InlongGroupExtInfo> extInfoList = CommonBeanUtils.copyListProperties(extEntityList, InlongGroupExtInfo::new);
groupInfo.setExtList(extInfoList);
// If the middleware is Pulsar, we need to encapsulate Pulsar related data
String mqType = entity.getMiddlewareType();
if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
InlongGroupPulsarEntity pulsarEntity = groupPulsarMapper.selectByGroupId(groupId);
Preconditions.checkNotNull(pulsarEntity, "Pulsar info not found by the groupId=" + groupId);
InlongGroupPulsarInfo pulsarInfo = CommonBeanUtils.copyProperties(pulsarEntity, InlongGroupPulsarInfo::new);
pulsarInfo.setMiddlewareType(mqType);
groupInfo.setMqExtInfo(pulsarInfo);
}
// For approved inlong group, encapsulate the cluster address of the middleware
if (GroupState.CONFIG_SUCCESSFUL == GroupState.forCode(groupInfo.getStatus())) {
if (Constant.MIDDLEWARE_TUBE.equalsIgnoreCase(mqType)) {
groupInfo.setTubeMaster(commonOperateService.getSpecifiedParam(Constant.TUBE_MASTER_URL));
} else if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
PulsarClusterInfo pulsarCluster = commonOperateService.getPulsarClusterInfo(mqType);
groupInfo.setPulsarAdminUrl(pulsarCluster.getAdminUrl());
groupInfo.setPulsarServiceUrl(pulsarCluster.getBrokerServiceUrl());
}
}
LOGGER.debug("success to get inlong group for groupId={}", groupId);
return groupInfo;
}
use of org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo in project incubator-inlong by apache.
the class ConsumptionCompleteProcessListener method createPulsarSubscription.
private void createPulsarSubscription(PulsarAdmin globalPulsarAdmin, String subscription, PulsarTopicBean topicBean, List<String> clusters, List<String> topics, PulsarClusterInfo globalCluster) {
try {
for (String cluster : clusters) {
String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
pulsarMqOptService.createSubscriptions(pulsarAdmin, subscription, topicBean, topics);
}
}
} catch (Exception e) {
log.error("create pulsar consumer group failed", e);
throw new WorkflowListenerException("failed to create pulsar consumer group");
}
}
Aggregations