Search in sources :

Example 1 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class StreamSourceServiceTest method testListByIdentifier.

@Test
public void testListByIdentifier() {
    Integer id = this.saveSource();
    SourceResponse source = sourceService.get(id, Constant.SOURCE_BINLOG);
    Assert.assertEquals(globalGroupId, source.getInlongGroupId());
    sourceService.delete(id, Constant.SOURCE_BINLOG, globalOperator);
}
Also used : BinlogSourceResponse(org.apache.inlong.manager.common.pojo.source.binlog.BinlogSourceResponse) SourceResponse(org.apache.inlong.manager.common.pojo.source.SourceResponse) ServiceBaseTest(org.apache.inlong.manager.service.ServiceBaseTest) InlongStreamServiceTest(org.apache.inlong.manager.service.core.impl.InlongStreamServiceTest) Test(org.junit.Test)

Example 2 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class DataSourceListenerTest method testFrozenSource.

@Test
public void testFrozenSource() {
    groupInfo = initGroupForm("PULSAR");
    groupInfo.setStatus(GroupState.CONFIG_SUCCESSFUL.getCode());
    groupService.update(groupInfo.genRequest(), OPERATOR);
    final int sourceId = createBinlogSource(groupInfo);
    streamSourceService.updateStatus(groupInfo.getInlongGroupId(), null, SourceState.SOURCE_NORMAL.getCode(), OPERATOR);
    form = new UpdateGroupProcessForm();
    form.setGroupInfo(groupInfo);
    form.setOperateType(OperateType.SUSPEND);
    WorkflowContext context = workflowEngine.processService().start(ProcessName.SUSPEND_GROUP_PROCESS.name(), applicant, form);
    WorkflowResult result = WorkflowBeanUtils.result(context);
    ProcessResponse response = result.getProcessInfo();
    Assert.assertSame(response.getStatus(), ProcessStatus.COMPLETED);
    WorkflowProcess process = context.getProcess();
    WorkflowTask task = process.getTaskByName("stopSource");
    Assert.assertTrue(task instanceof ServiceTask);
    SourceResponse sourceResponse = streamSourceService.get(sourceId, SourceType.BINLOG.toString());
    Assert.assertSame(SourceState.forCode(sourceResponse.getStatus()), SourceState.TO_BE_ISSUED_FROZEN);
}
Also used : UpdateGroupProcessForm(org.apache.inlong.manager.common.pojo.workflow.form.UpdateGroupProcessForm) WorkflowResult(org.apache.inlong.manager.common.pojo.workflow.WorkflowResult) ServiceTask(org.apache.inlong.manager.workflow.definition.ServiceTask) SourceResponse(org.apache.inlong.manager.common.pojo.source.SourceResponse) WorkflowContext(org.apache.inlong.manager.workflow.WorkflowContext) WorkflowTask(org.apache.inlong.manager.workflow.definition.WorkflowTask) WorkflowProcess(org.apache.inlong.manager.workflow.definition.WorkflowProcess) ProcessResponse(org.apache.inlong.manager.common.pojo.workflow.ProcessResponse) Test(org.junit.Test) WorkflowServiceImplTest(org.apache.inlong.manager.service.workflow.WorkflowServiceImplTest)

Example 3 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class CommonOperateService method createDataFlow.

/**
 * Create dataflow info for sort.
 */
public DataFlowInfo createDataFlow(InlongGroupInfo groupInfo, SinkResponse sinkResponse) {
    String groupId = sinkResponse.getInlongGroupId();
    String streamId = sinkResponse.getInlongStreamId();
    // TODO Support all source type, include AUTO_PUSH.
    List<SourceResponse> sourceList = streamSourceService.listSource(groupId, streamId);
    if (CollectionUtils.isEmpty(sourceList)) {
        throw new WorkflowListenerException(String.format("Source not found by groupId=%s and streamId=%s", groupId, streamId));
    }
    // Get all field info
    List<FieldInfo> sourceFields = new ArrayList<>();
    List<FieldInfo> sinkFields = new ArrayList<>();
    String partition = null;
    if (SinkType.forType(sinkResponse.getSinkType()) == SinkType.HIVE) {
        HiveSinkResponse hiveSink = (HiveSinkResponse) sinkResponse;
        partition = hiveSink.getPrimaryPartition();
    }
    // TODO Support more than one source and one sink
    final SourceResponse sourceResponse = sourceList.get(0);
    boolean isAllMigration = SourceInfoUtils.isBinlogAllMigration(sourceResponse);
    FieldMappingRule fieldMappingRule = FieldInfoUtils.createFieldInfo(isAllMigration, sinkResponse.getFieldList(), sourceFields, sinkFields, partition);
    // Get source info
    String masterAddress = getSpecifiedParam(Constant.TUBE_MASTER_URL);
    PulsarClusterInfo pulsarCluster = getPulsarClusterInfo(groupInfo.getMiddlewareType());
    InlongStreamInfo streamInfo = streamService.get(groupId, streamId);
    SourceInfo sourceInfo = SourceInfoUtils.createSourceInfo(pulsarCluster, masterAddress, clusterBean, groupInfo, streamInfo, sourceResponse, sourceFields);
    // Get sink info
    SinkInfo sinkInfo = SinkInfoUtils.createSinkInfo(sourceResponse, sinkResponse, sinkFields);
    // Get transformation info
    TransformationInfo transInfo = new TransformationInfo(fieldMappingRule);
    // Get properties
    Map<String, Object> properties = new HashMap<>();
    if (MapUtils.isNotEmpty(sinkResponse.getProperties())) {
        properties.putAll(sinkResponse.getProperties());
    }
    properties.put(Constant.DATA_FLOW_GROUP_ID_KEY, groupId);
    return new DataFlowInfo(sinkResponse.getId(), sourceInfo, transInfo, sinkInfo, properties);
}
Also used : SourceResponse(org.apache.inlong.manager.common.pojo.source.SourceResponse) FieldMappingRule(org.apache.inlong.sort.protocol.transformation.FieldMappingRule) SourceInfo(org.apache.inlong.sort.protocol.source.SourceInfo) HashMap(java.util.HashMap) PulsarClusterInfo(org.apache.inlong.common.pojo.dataproxy.PulsarClusterInfo) ArrayList(java.util.ArrayList) SinkInfo(org.apache.inlong.sort.protocol.sink.SinkInfo) HiveSinkResponse(org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkResponse) WorkflowListenerException(org.apache.inlong.manager.common.exceptions.WorkflowListenerException) FieldInfo(org.apache.inlong.sort.protocol.FieldInfo) InlongStreamInfo(org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo) TransformationInfo(org.apache.inlong.sort.protocol.transformation.TransformationInfo) DataFlowInfo(org.apache.inlong.sort.protocol.DataFlowInfo)

Example 4 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class StreamSourceServiceImpl method get.

@Override
public SourceResponse get(Integer id, String sourceType) {
    LOGGER.debug("begin to get source by id={}, sourceType={}", id, sourceType);
    StreamSourceOperation operation = operationFactory.getInstance(SourceType.forType(sourceType));
    SourceResponse sourceResponse = operation.getById(id);
    LOGGER.debug("success to get source info");
    return sourceResponse;
}
Also used : SourceResponse(org.apache.inlong.manager.common.pojo.source.SourceResponse)

Example 5 with SourceResponse

use of org.apache.inlong.manager.common.pojo.source.SourceResponse in project incubator-inlong by apache.

the class AbstractSourceOperateListener method operateStreamSources.

protected void operateStreamSources(String groupId, String streamId, String operator) {
    List<SourceResponse> sourceResponses = streamSourceService.listSource(groupId, streamId);
    sourceResponses.forEach(sourceResponse -> {
        SourceRequest sourceRequest = createSourceRequest(sourceResponse);
        operateStreamSource(sourceRequest, operator);
    });
}
Also used : BinlogSourceResponse(org.apache.inlong.manager.common.pojo.source.binlog.BinlogSourceResponse) SourceResponse(org.apache.inlong.manager.common.pojo.source.SourceResponse) KafkaSourceResponse(org.apache.inlong.manager.common.pojo.source.kafka.KafkaSourceResponse) SourceRequest(org.apache.inlong.manager.common.pojo.source.SourceRequest) BinlogSourceRequest(org.apache.inlong.manager.common.pojo.source.binlog.BinlogSourceRequest) KafkaSourceRequest(org.apache.inlong.manager.common.pojo.source.kafka.KafkaSourceRequest)

Aggregations

SourceResponse (org.apache.inlong.manager.common.pojo.source.SourceResponse)9 BinlogSourceResponse (org.apache.inlong.manager.common.pojo.source.binlog.BinlogSourceResponse)4 Test (org.junit.Test)4 ArrayList (java.util.ArrayList)2 SinkResponse (org.apache.inlong.manager.common.pojo.sink.SinkResponse)2 HiveSinkResponse (org.apache.inlong.manager.common.pojo.sink.hive.HiveSinkResponse)2 BinlogSourceRequest (org.apache.inlong.manager.common.pojo.source.binlog.BinlogSourceRequest)2 KafkaSourceResponse (org.apache.inlong.manager.common.pojo.source.kafka.KafkaSourceResponse)2 FullStreamResponse (org.apache.inlong.manager.common.pojo.stream.FullStreamResponse)2 InlongStreamInfo (org.apache.inlong.manager.common.pojo.stream.InlongStreamInfo)2 ProcessResponse (org.apache.inlong.manager.common.pojo.workflow.ProcessResponse)2 WorkflowResult (org.apache.inlong.manager.common.pojo.workflow.WorkflowResult)2 UpdateGroupProcessForm (org.apache.inlong.manager.common.pojo.workflow.form.UpdateGroupProcessForm)2 ServiceBaseTest (org.apache.inlong.manager.service.ServiceBaseTest)2 InlongStreamServiceTest (org.apache.inlong.manager.service.core.impl.InlongStreamServiceTest)2 WorkflowServiceImplTest (org.apache.inlong.manager.service.workflow.WorkflowServiceImplTest)2 WorkflowContext (org.apache.inlong.manager.workflow.WorkflowContext)2 ServiceTask (org.apache.inlong.manager.workflow.definition.ServiceTask)2 WorkflowProcess (org.apache.inlong.manager.workflow.definition.WorkflowProcess)2 Page (com.github.pagehelper.Page)1