use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreateHiveTableForStreamListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
String streamId = form.getInlongStreamId();
log.info("begin create hive table for groupId={}, streamId={}", groupId, streamId);
List<SinkForSortDTO> configList = sinkMapper.selectAllConfig(groupId, streamId);
if (hiveTableOperator == null) {
hiveTableOperator = (IHiveTableOperator) SpringContextUtils.getBean(IHiveTableOperator.BEAN_NAME, DefaultHiveTableOperator.class.getName());
}
hiveTableOperator.createHiveResource(groupId, configList);
String result = "success to create hive table for group [" + groupId + "], stream [" + streamId + "]";
log.info(result);
return ListenerResult.success(result);
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreateTubeTopicTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
log.info("begin create tube topic for groupId={}", form.getInlongGroupId());
String groupId = form.getInlongGroupId();
try {
InlongGroupInfo groupInfo = groupService.get(groupId);
String topicName = groupInfo.getMqResourceObj();
AddTubeMqTopicRequest request = new AddTubeMqTopicRequest();
request.setUser("inlong-manager");
AddTubeMqTopicRequest.AddTopicTasksBean tasksBean = new AddTubeMqTopicRequest.AddTopicTasksBean();
tasksBean.setTopicName(topicName);
request.setAddTopicTasks(Collections.singletonList(tasksBean));
tubeMqOptService.createNewTopic(request);
log.info("finish to create tube topic for groupId={}", groupId);
} catch (Exception e) {
log.error("create tube topic for groupId={} error, exception {} ", groupId, e.getMessage(), e);
}
return ListenerResult.success();
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreatePulsarGroupForStreamTaskListener method listen.
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
GroupResourceProcessForm form = (GroupResourceProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
String streamId = form.getInlongStreamId();
InlongGroupInfo groupInfo = groupService.get(groupId);
if (groupInfo == null) {
log.error("inlong group not found with groupId={}", groupId);
throw new WorkflowListenerException("inlong group not found with groupId=" + groupId);
}
InlongStreamEntity streamEntity = streamMapper.selectByIdentifier(groupId, streamId);
if (streamEntity == null) {
log.warn("inlong stream is empty for group={}, stream={}, skip to create pulsar group", groupId, streamId);
return ListenerResult.success();
}
PulsarClusterInfo globalCluster = commonOperateService.getPulsarClusterInfo(groupInfo.getMiddlewareType());
try (PulsarAdmin globalPulsarAdmin = PulsarUtils.getPulsarAdmin(globalCluster)) {
// Query data sink info based on groupId and streamId
List<String> sinkTypeList = sinkService.getSinkTypeList(groupId, streamId);
if (sinkTypeList == null || sinkTypeList.size() == 0) {
log.warn("sink info is empty for groupId={}, streamId={}, skip to create pulsar group", groupId, streamId);
return ListenerResult.success();
}
PulsarTopicBean topicBean = new PulsarTopicBean();
topicBean.setTenant(clusterBean.getDefaultTenant());
topicBean.setNamespace(groupInfo.getMqResourceObj());
String topic = streamEntity.getMqResourceObj();
topicBean.setTopicName(topic);
List<String> pulsarClusters = PulsarUtils.getPulsarClusters(globalPulsarAdmin);
// Create a subscription in the Pulsar cluster (cross-region), you need to ensure that the Topic exists
String tenant = clusterBean.getDefaultTenant();
String namespace = groupInfo.getMqResourceObj();
for (String cluster : pulsarClusters) {
String serviceUrl = PulsarUtils.getServiceUrl(globalPulsarAdmin, cluster);
PulsarClusterInfo pulsarClusterInfo = PulsarClusterInfo.builder().token(globalCluster.getToken()).adminUrl(serviceUrl).build();
try (PulsarAdmin pulsarAdmin = PulsarUtils.getPulsarAdmin(pulsarClusterInfo)) {
boolean exist = pulsarOptService.topicIsExists(pulsarAdmin, tenant, namespace, topic);
if (!exist) {
String fullTopic = tenant + "/" + namespace + "/" + topic;
log.error("topic={} not exists in {}", fullTopic, pulsarAdmin.getServiceUrl());
throw new BusinessException("topic=" + fullTopic + " not exists in " + serviceUrl);
}
// Consumer naming rules: sortAppName_topicName_consumer_group
String subscription = clusterBean.getAppName() + "_" + topic + "_consumer_group";
pulsarOptService.createSubscription(pulsarAdmin, topicBean, subscription);
// Insert the consumption data into the consumption table
consumptionService.saveSortConsumption(groupInfo, topic, subscription);
}
}
} catch (Exception e) {
log.error("create pulsar subscription error for groupId={}, streamId={}", groupId, streamId, e);
throw new WorkflowListenerException("create pulsar subscription error, reason: " + e.getMessage());
}
log.info("finish to create single pulsar subscription for groupId={}, streamId={}", groupId, streamId);
return ListenerResult.success();
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class StartCreateGroupProcessListener method listen.
/**
* Initiate the process of creating inlong group resources after new inlong group access approved
*/
@Override
public ListenerResult listen(WorkflowContext context) throws WorkflowListenerException {
NewGroupProcessForm form = (NewGroupProcessForm) context.getProcessForm();
String groupId = form.getInlongGroupId();
GroupResourceProcessForm processForm = new GroupResourceProcessForm();
processForm.setGroupInfo(groupService.get(groupId));
String username = context.getApplicant();
List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupId);
List<InlongStreamInfo> inlongStreamInfoList = CommonBeanUtils.copyListProperties(inlongStreamEntityList, InlongStreamInfo::new);
processForm.setInlongStreamInfoList(inlongStreamInfoList);
workflowService.start(ProcessName.CREATE_GROUP_RESOURCE, username, processForm);
return ListenerResult.success();
}
use of org.apache.inlong.manager.common.pojo.workflow.form.GroupResourceProcessForm in project incubator-inlong by apache.
the class CreateStreamWorkflowDefinition method defineProcess.
@Override
public WorkflowProcess defineProcess() {
// Configuration process
WorkflowProcess process = new WorkflowProcess();
process.addListener(initGroupListener);
process.addListener(streamFailedProcessListener);
process.addListener(streamCompleteProcessListener);
process.setType("Inlong stream access resource creation");
process.setName(getProcessName().name());
process.setDisplayName(getProcessName().getDisplayName());
process.setFormClass(GroupResourceProcessForm.class);
process.setVersion(1);
process.setHidden(1);
// Start node
StartEvent startEvent = new StartEvent();
process.setStartEvent(startEvent);
// End node
EndEvent endEvent = new EndEvent();
process.setEndEvent(endEvent);
ServiceTask createPulsarTopicTask = new ServiceTask();
createPulsarTopicTask.setSkipResolver(c -> {
GroupResourceProcessForm form = (GroupResourceProcessForm) c.getProcessForm();
String mqType = form.getGroupInfo().getMiddlewareType();
if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
return false;
}
log.warn("no need to create pulsar topic for groupId={}, streamId={}, as the middlewareType={}", form.getInlongGroupId(), form.getInlongStreamId(), mqType);
return true;
});
createPulsarTopicTask.setName("createPulsarTopic");
createPulsarTopicTask.setDisplayName("Stream-CreatePulsarTopic");
createPulsarTopicTask.addListener(createPulsarTopicTaskListener);
process.addTask(createPulsarTopicTask);
ServiceTask createPulsarSubscriptionGroupTask = new ServiceTask();
createPulsarSubscriptionGroupTask.setSkipResolver(c -> {
GroupResourceProcessForm form = (GroupResourceProcessForm) c.getProcessForm();
String mqType = form.getGroupInfo().getMiddlewareType();
if (Constant.MIDDLEWARE_PULSAR.equals(mqType) || Constant.MIDDLEWARE_TDMQ_PULSAR.equals(mqType)) {
return false;
}
log.warn("no need to create pulsar subscription for groupId={}, streamId={}, as the middlewareType={}", form.getInlongGroupId(), form.getInlongStreamId(), mqType);
return true;
});
createPulsarSubscriptionGroupTask.setName("createPulsarSubscription");
createPulsarSubscriptionGroupTask.setDisplayName("Stream-CreatePulsarSubscription");
createPulsarSubscriptionGroupTask.addListener(createPulsarGroupTaskListener);
process.addTask(createPulsarSubscriptionGroupTask);
ServiceTask createHiveTableTask = new ServiceTask();
createHiveTableTask.setSkipResolver(c -> {
GroupResourceProcessForm form = (GroupResourceProcessForm) c.getProcessForm();
String groupId = form.getInlongGroupId();
String streamId = form.getInlongStreamId();
List<String> dsForHive = sinkService.getExistsStreamIdList(groupId, Constant.SINK_HIVE, Collections.singletonList(streamId));
if (CollectionUtils.isEmpty(dsForHive)) {
log.warn("inlong group [{}] adn inlong stream [{}] does not have sink, skip create hive table", groupId, streamId);
return true;
}
return false;
});
createHiveTableTask.setName("createHiveTable");
createHiveTableTask.setDisplayName("Stream-CreateHiveTable");
createHiveTableTask.addListener(createHiveTableListener);
process.addTask(createHiveTableTask);
ServiceTask pushSortConfig = new ServiceTask();
pushSortConfig.setName("pushSortConfig");
pushSortConfig.setDisplayName("Stream-PushSortConfig");
pushSortConfig.addListener(pushSortConfigListener);
process.addTask(pushSortConfig);
startEvent.addNext(createPulsarTopicTask);
createPulsarTopicTask.addNext(createPulsarSubscriptionGroupTask);
createPulsarSubscriptionGroupTask.addNext(createHiveTableTask);
createHiveTableTask.addNext(pushSortConfig);
pushSortConfig.addNext(endEvent);
return process;
}
Aggregations