Examples with InlongStreamInfo org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo used on opensource projects

Example 1 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class InlongStreamTransfer method createStreamInfo.

public static InlongStreamInfo createStreamInfo(InlongStreamConf streamConf, InlongGroupInfo groupInfo) {
    InlongStreamInfo dataStreamInfo = new InlongStreamInfo();
    dataStreamInfo.setInlongGroupId(groupInfo.getInlongGroupId());
    final String streamId = "b_" + streamConf.getName();
    dataStreamInfo.setInlongStreamId(streamId);
    dataStreamInfo.setName(streamConf.getName());
    dataStreamInfo.setDataEncoding(streamConf.getCharset().name());
    if (StringUtils.isEmpty(streamConf.getTopic())) {
        dataStreamInfo.setMqResourceObj(streamId);
    } else {
        dataStreamInfo.setMqResourceObj(streamConf.getTopic());
    }
    dataStreamInfo.setSyncSend(streamConf.isStrictlyOrdered() ? 1 : 0);
    dataStreamInfo.setDataSeparator(String.valueOf(streamConf.getDataSeparator().getAsciiCode()));
    dataStreamInfo.setDescription(streamConf.getDescription());
    dataStreamInfo.setCreator(groupInfo.getCreator());
    dataStreamInfo.setDailyRecords(streamConf.getDailyRecords());
    dataStreamInfo.setDailyStorage(streamConf.getDailyStorage());
    dataStreamInfo.setPeakRecords(streamConf.getPeakRecords());
    dataStreamInfo.setHavePredefinedFields(0);
    if (CollectionUtils.isNotEmpty(streamConf.getStreamFields())) {
        dataStreamInfo.setFieldList(createStreamFields(streamConf.getStreamFields(), dataStreamInfo));
    }
    return dataStreamInfo;
}

Example 2 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class PushSortConfigListener method getSourceInfo.

/**
 * Get source info for DataFlowInfo
 */
@Deprecated
private SourceInfo getSourceInfo(InlongGroupInfo groupInfo, String streamId, List<StreamSinkFieldEntity> fieldList) {
    DeserializationInfo deserializationInfo = null;
    String groupId = groupInfo.getInlongGroupId();
    InlongStreamInfo streamInfo = streamService.get(groupId, streamId);
    boolean isDbType = Constant.DATA_SOURCE_DB.equals(streamInfo.getDataType());
    if (!isDbType) {
        // FILE and auto push source, the data format is TEXT or KEY-VALUE, temporarily use InLongMsgCsv
        String dataType = streamInfo.getDataType();
        if (Constant.DATA_TYPE_TEXT.equalsIgnoreCase(dataType) || Constant.DATA_TYPE_KEY_VALUE.equalsIgnoreCase(dataType)) {
            // Use the field separator from the inlong stream
            char separator = (char) Integer.parseInt(streamInfo.getDataSeparator());
            // TODO support escape
            /*Character escape = null;
                if (info.getDataEscapeChar() != null) {
                    escape = info.getDataEscapeChar().charAt(0);
                }*/
            // Whether to delete the first separator, the default is false for the time being
            deserializationInfo = new InLongMsgCsvDeserializationInfo(streamId, separator);
        }
    }
    // The number and order of the source fields must be the same as the target fields
    return null;
}

Example 3 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class InlongGroupProcessOperation method genUpdateGroupProcessForm.

private UpdateGroupProcessForm genUpdateGroupProcessForm(String groupId, OperateType operateType) {
    InlongGroupInfo groupInfo = groupService.get(groupId);
    UpdateGroupProcessForm form = new UpdateGroupProcessForm();
    if (OperateType.RESTART == operateType) {
        List<InlongStreamEntity> inlongStreamEntityList = streamMapper.selectByGroupId(groupInfo.getInlongGroupId());
        List<InlongStreamInfo> inlongStreamInfoList = CommonBeanUtils.copyListProperties(inlongStreamEntityList, InlongStreamInfo::new);
        form.setInlongStreamInfoList(inlongStreamInfoList);
    }
    form.setGroupInfo(groupInfo);
    form.setOperateType(operateType);
    return form;
}

Example 4 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class InlongStreamServiceImpl method batchSaveAll.

@Transactional(rollbackFor = Throwable.class)
@Override
public boolean batchSaveAll(List<FullStreamRequest> fullStreamRequestList, String operator) {
    if (CollectionUtils.isEmpty(fullStreamRequestList)) {
        return true;
    }
    LOGGER.info("begin to batch save all stream page info, batch size={}", fullStreamRequestList.size());
    // Check if it can be added
    InlongStreamInfo firstStream = fullStreamRequestList.get(0).getStreamInfo();
    Preconditions.checkNotNull(firstStream, "inlong stream info is empty");
    String groupId = firstStream.getInlongGroupId();
    this.checkBizIsTempStatus(groupId);
    // This bulk save is only used when creating or editing inlong group after approval is rejected.
    // To ensure data consistency, you need to physically delete all associated data and then add
    // Note: There may be records with the same groupId and streamId in the historical data,
    // and the ones with is_deleted=0 should be deleted
    streamMapper.deleteAllByGroupId(groupId);
    for (FullStreamRequest pageInfo : fullStreamRequestList) {
        // 1.1 Delete the inlong stream extensions and fields corresponding to groupId and streamId
        InlongStreamInfo streamInfo = pageInfo.getStreamInfo();
        String streamId = streamInfo.getInlongStreamId();
        streamExtMapper.deleteAllByIdentifier(groupId, streamId);
        streamFieldMapper.deleteAllByIdentifier(groupId, streamId);
        // 2. Delete file data source, DB data source information
        sourceFileService.deleteAllByIdentifier(groupId, streamId);
        sourceDbService.deleteAllByIdentifier(groupId, streamId);
        // 3. Delete data sink information
        sinkService.deleteAll(groupId, streamId, operator);
        // 4. Save the inlong stream of this batch
        this.saveAll(pageInfo, operator);
    }
    LOGGER.info("success to batch save all stream page info");
    return true;
}

Example 5 with InlongStreamInfo

use of org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo in project incubator-inlong by apache.

the class DisableZkForSortTest method testCreateSortConfigInCreateWorkflow.

@Test
public void testCreateSortConfigInCreateWorkflow() {
    InlongGroupInfo groupInfo = initGroupForm("PULSAR");
    groupInfo.setStatus(GroupState.CONFIG_SUCCESSFUL.getCode());
    groupInfo.setZookeeperEnabled(0);
    groupService.update(groupInfo.genRequest(), OPERATOR);
    InlongStreamInfo streamInfo = createStreamInfo(groupInfo);
    createHiveSink(streamInfo);
    createKafkaSource(streamInfo);
    mockTaskListenerFactory();
    WorkflowContext context = workflowEngine.processService().start(processName.name(), applicant, form);
    WorkflowResult result = WorkflowBeanUtils.result(context);
    ProcessResponse response = result.getProcessInfo();
    Assert.assertSame(response.getStatus(), ProcessStatus.COMPLETED);
    WorkflowProcess process = context.getProcess();
    WorkflowTask task = process.getTaskByName("initSort");
    Assert.assertTrue(task instanceof ServiceTask);
    Assert.assertEquals(1, task.getNameToListenerMap().size());
    List<TaskEventListener> listeners = Lists.newArrayList(task.getNameToListenerMap().values());
    Assert.assertTrue(listeners.get(0) instanceof CreateSortConfigListener);
    ProcessForm form = context.getProcessForm();
    InlongGroupInfo curGroupRequest = ((GroupResourceProcessForm) form).getGroupInfo();
    Assert.assertTrue(curGroupRequest.getExtList().size() == 1);
}