use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreateHiveTableEventSelector method accept.
@Override
public boolean accept(WorkflowContext context) {
ProcessForm processForm = context.getProcessForm();
if (!(processForm instanceof GroupResourceProcessForm)) {
return false;
}
GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
if (form.getGroupInfo() == null || StringUtils.isEmpty(form.getGroupInfo().getInlongGroupId())) {
return false;
}
String groupId = form.getInlongGroupId();
List<String> dsForHive = sinkService.getExistsStreamIdList(groupId, Constant.SINK_HIVE, streamMapper.selectByGroupId(groupId).stream().map(InlongStreamEntity::getInlongStreamId).collect(Collectors.toList()));
if (CollectionUtils.isEmpty(dsForHive)) {
log.warn("groupId={} streamId={} does not have sink, skip to create hive table ", groupId, form.getInlongStreamId());
return true;
}
return false;
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreateHiveTableListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
log.info("begin to create hive table for groupId={}", groupId);
List<SinkForSortDTO> configList = sinkMapper.selectAllConfig(groupId, null);
List<SinkForSortDTO> needCreateList = configList.stream().filter(sinkForSortDTO -> sinkForSortDTO.getEnableCreateResource() == 1).collect(Collectors.toList());
if (hiveTableOperator == null) {
hiveTableOperator = (IHiveTableOperator) SpringContextUtils.getBean(IHiveTableOperator.BEAN_NAME, DefaultHiveTableOperator.class.getName());
}
hiveTableOperator.createHiveResource(groupId, needCreateList);
String result = "success to create hive table for group [" + groupId + "]";
log.info(result);
return ListenerResult.success(result);
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreatePulsarTopicForStreamTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
String streamId = form.getInlongStreamId();
InlongGroupInfo groupInfo = groupService.get(groupId);
InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
if (groupInfo == null || streamEntity == null) {
throw new WorkflowListenerException("inlong group or inlong stream not found with groupId=" + groupId + ", streamId=" + streamId);
}
log.info("begin to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
for (String cluster : pulsarClusters) {
String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
String pulsarTopic = streamEntity.getMqResourceObj();
this.createTopic(groupInfo, pulsarTopic, pulsarClusterInfo);
}
} catch (Exception e) {
log.error("create pulsar topic error for groupId={}, streamId={}", groupId, streamId, e);
throw new WorkflowListenerException("create pulsar topic error for groupId=" + groupId + ", streamId=" + streamId);
}
log.info("success to create pulsar topic for groupId={}, streamId={}", groupId, streamId);
return ListenerResult.success();
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreateTubeGroupTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
log.info("try to create consumer group for groupId {}", groupId);
InlongGroupInfo groupInfo = groupService.get(groupId);
String topicName = groupInfo.getMqResourceObj();
int clusterId = Integer.parseInt(commonOperateService.getSpecifiedParam(Constant.CLUSTER_TUBE_CLUSTER_ID));
QueryTubeTopicRequest queryTubeTopicRequest = QueryTubeTopicRequest.builder().topicName(topicName).clusterId(clusterId).user(groupInfo.getCreator()).build();
// Query whether the tube topic exists
boolean topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
Integer tryNumber = reTryConfigBean.getMaxAttempts();
Long delay = reTryConfigBean.getDelay();
while (!topicExist && --tryNumber > 0) {
log.info("check whether the tube topic exists, try count={}", tryNumber);
try {
Thread.sleep(delay);
delay *= reTryConfigBean.getMultiplier();
topicExist = tubeMqOptService.queryTopicIsExist(queryTubeTopicRequest);
} catch (InterruptedException e) {
log.error("check the tube topic exists error", e);
}
}
AddTubeConsumeGroupRequest addTubeConsumeGroupRequest = new AddTubeConsumeGroupRequest();
addTubeConsumeGroupRequest.setClusterId(clusterId);
addTubeConsumeGroupRequest.setCreateUser(groupInfo.getCreator());
GroupNameJsonSetBean groupNameJsonSetBean = new GroupNameJsonSetBean();
groupNameJsonSetBean.setTopicName(topicName);
String consumeGroupName = "sort_" + topicName + "_group";
groupNameJsonSetBean.setGroupName(consumeGroupName);
addTubeConsumeGroupRequest.setGroupNameJsonSet(Collections.singletonList(groupNameJsonSetBean));
try {
tubeMqOptService.createNewConsumerGroup(addTubeConsumeGroupRequest);
} catch (Exception e) {
throw new WorkflowListenerException("create tube consumer group for groupId=" + groupId + " error", e);
}
log.info("finish to create consumer group for {}", groupId);
return ListenerResult.success();
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class PulsarEventSelector method accept.
@Override
public boolean accept(WorkflowContext context) {
ProcessForm processForm = context.getProcessForm();
if (!(processForm instanceof GroupResourceProcessForm)) {
return false;
}
GroupResourceProcessForm form = (GroupResourceProcessForm) processForm;
String mqType = form.getGroupInfo().getMiddlewareType();
if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
InlongGroupPulsarInfo pulsarInfo = (InlongGroupPulsarInfo) form.getGroupInfo().getMqExtInfo();
return pulsarInfo.getEnableCreateResource() == 1;
}
log.warn("no need to create pulsar subscription group for groupId={}, as the middlewareType={}", form.getInlongGroupId(), mqType);
return false;
}
Aggregations