Examples with PipelineParameter com.alibaba.otter.shared.common.model.config.pipeline.PipelineParameter used on opensource projects

Example 6 with PipelineParameter

use of com.alibaba.otter.shared.common.model.config.pipeline.PipelineParameter in project otter by alibaba.

the class PipelineAction method doEdit.

public void doEdit(@FormGroup("pipelineInfo") Group pipelineInfo, @FormGroup("pipelineParameterInfo") Group pipelineParameterInfo, @FormField(name = "formPipelineError", group = "pipelineInfo") CustomErrors err, HttpSession session, Navigator nav) {
    Pipeline pipeline = new Pipeline();
    PipelineParameter parameters = new PipelineParameter();
    pipelineInfo.setProperties(pipeline);
    pipelineParameterInfo.setProperties(parameters);
    // if (parameters.getLoadPoolSize() < 1) {
    // parameters.setLoadPoolSize(PipelineParameter.DEFAULT_LOAD_POOL_SIZE);
    // }
    List<Long> selectNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    List<Node> selectNodes = new ArrayList<Node>();
    for (Long selectNodeId : selectNodeIds) {
        Node node = new Node();
        node.setId(selectNodeId);
        selectNodes.add(node);
    }
    // select/extract节点普遍配置为同一个节点
    List<Long> extractNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    // List<Long> extractNodeIds =
    // Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("extractNodeIds").getLongValues()));
    List<Node> extractNodes = new ArrayList<Node>();
    for (Long extractNodeId : extractNodeIds) {
        Node node = new Node();
        node.setId(extractNodeId);
        extractNodes.add(node);
    }
    List<Long> loadNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("loadNodeIds").getLongValues()));
    List<Node> loadNodes = new ArrayList<Node>();
    for (Long loadNodeId : loadNodeIds) {
        Node node = new Node();
        node.setId(loadNodeId);
        loadNodes.add(node);
    }
    pipeline.setSelectNodes(selectNodes);
    pipeline.setExtractNodes(extractNodes);
    pipeline.setLoadNodes(loadNodes);
    pipeline.setParameters(parameters);
    List<Pipeline> values = pipelineService.listByDestinationWithoutOther(pipeline.getParameters().getDestinationName());
    if (!values.isEmpty()) {
        if (values.size() > 1 || !values.get(0).getId().equals(pipeline.getId())) {
            err.setMessage("invalidDestinationName");
            return;
        }
    }
    try {
        pipelineService.modify(pipeline);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidPipelineName");
        return;
    }
    nav.redirectToLocation("pipelineList.htm?channelId=" + pipeline.getChannelId());
}

Example 7 with PipelineParameter

use of com.alibaba.otter.shared.common.model.config.pipeline.PipelineParameter in project otter by alibaba.

the class ChannelServiceImpl method doToModelWithColumn.

private List<Channel> doToModelWithColumn(List<ChannelDO> channelDos) {
    List<Channel> channels = new ArrayList<Channel>();
    try {
        // 1.将ChannelID单独拿出来
        List<Long> channelIds = new ArrayList<Long>();
        for (ChannelDO channelDo : channelDos) {
            channelIds.add(channelDo.getId());
        }
        Long[] idArray = new Long[channelIds.size()];
        // 拿到所有的Pipeline进行ChannelID过滤，避免重复查询。
        List<Pipeline> pipelines = pipelineService.listByChannelIdsWithoutColumn(channelIds.toArray(idArray));
        SystemParameter systemParameter = systemParameterService.find();
        for (ChannelDO channelDo : channelDos) {
            Channel channel = new Channel();
            channel.setId(channelDo.getId());
            channel.setName(channelDo.getName());
            channel.setDescription(channelDo.getDescription());
            ChannelStatus channelStatus = arbitrateManageService.channelEvent().status(channelDo.getId());
            channel.setStatus(null == channelStatus ? ChannelStatus.STOP : channelStatus);
            channel.setParameters(channelDo.getParameters());
            channel.setGmtCreate(channelDo.getGmtCreate());
            channel.setGmtModified(channelDo.getGmtModified());
            // 遍历，将该Channel节点下的Pipeline提取出来。
            List<Pipeline> subPipelines = new ArrayList<Pipeline>();
            for (Pipeline pipeline : pipelines) {
                if (pipeline.getChannelId().equals(channelDo.getId())) {
                    // 合并PipelineParameter和ChannelParameter
                    PipelineParameter parameter = new PipelineParameter();
                    parameter.merge(systemParameter);
                    parameter.merge(channel.getParameters());
                    // 最后复制pipelineId参数
                    parameter.merge(pipeline.getParameters());
                    pipeline.setParameters(parameter);
                    subPipelines.add(pipeline);
                }
            }
            channel.setPipelines(subPipelines);
            channels.add(channel);
        }
    } catch (Exception e) {
        logger.error("ERROR ## change the channels DO to Model has an exception");
        throw new ManagerException(e);
    }
    return channels;
}

Example 8 with PipelineParameter

use of com.alibaba.otter.shared.common.model.config.pipeline.PipelineParameter in project otter by alibaba.

the class PipelineAction method doAdd.

public void doAdd(@FormGroup("pipelineInfo") Group pipelineInfo, @FormGroup("pipelineParameterInfo") Group pipelineParameterInfo, @FormField(name = "formPipelineError", group = "pipelineInfo") CustomErrors err, HttpSession session, Navigator nav) throws Exception {
    Pipeline pipeline = new Pipeline();
    PipelineParameter parameters = new PipelineParameter();
    pipelineInfo.setProperties(pipeline);
    pipelineParameterInfo.setProperties(parameters);
    // if (parameters.getLoadPoolSize() < 1) {
    // parameters.setLoadPoolSize(PipelineParameter.DEFAULT_LOAD_POOL_SIZE);
    // }
    List<Long> selectNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    List<Node> selectNodes = new ArrayList<Node>();
    for (Long selectNodeId : selectNodeIds) {
        Node node = new Node();
        node.setId(selectNodeId);
        selectNodes.add(node);
    }
    // select/extract节点普遍配置为同一个节点
    List<Long> extractNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    // List<Long> extractNodeIds =
    // Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("extractNodeIds").getLongValues()));
    List<Node> extractNodes = new ArrayList<Node>();
    for (Long extractNodeId : extractNodeIds) {
        Node node = new Node();
        node.setId(extractNodeId);
        extractNodes.add(node);
    }
    List<Long> loadNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("loadNodeIds").getLongValues()));
    List<Node> loadNodes = new ArrayList<Node>();
    for (Long loadNodeId : loadNodeIds) {
        Node node = new Node();
        node.setId(loadNodeId);
        loadNodes.add(node);
    }
    pipeline.setSelectNodes(selectNodes);
    pipeline.setExtractNodes(extractNodes);
    pipeline.setLoadNodes(loadNodes);
    pipeline.setParameters(parameters);
    List<Pipeline> values = pipelineService.listByDestinationWithoutOther(pipeline.getParameters().getDestinationName());
    if (!values.isEmpty()) {
        err.setMessage("invalidDestinationName");
        return;
    }
    try {
        pipelineService.create(pipeline);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidPipelineName");
        return;
    }
    nav.redirectToLocation("pipelineList.htm?channelId=" + pipeline.getChannelId());
}

Example 9 with PipelineParameter

use of com.alibaba.otter.shared.common.model.config.pipeline.PipelineParameter in project otter by alibaba.

the class OtterTransformerTest method test_rowData_oracle_mysql.

@Test
public void test_rowData_oracle_mysql() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia oracleMedia = getOracleMedia();
    oracleMedia.setId(1L);
    pair1.setSource(oracleMedia);
    DbDataMedia mysqlMedia = getMysqlMedia();
    pair1.setTarget(mysqlMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    PipelineParameter param = new PipelineParameter();
    param.setSyncMode(SyncMode.ROW);
    pipeline.setParameters(param);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("srf");
    eventData.setTableName("columns");
    eventData.setEventType(EventType.UPDATE);
    eventData.setExecuteTime(100L);
    eventData.getKeys().add(buildColumn("id", Types.NUMERIC, "1", true, false));
    eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh", true, false));
    eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello", false, false));
    eventData.getColumns().add(buildColumn("amount", Types.NUMERIC, "100.01", false, false));
    eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
    eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "text_c", false, false));
    eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
    eventData.getColumns().add(buildColumn("gmt_create", Types.DATE, "2011-01-01 11:11:11", false, false));
    eventData.getColumns().add(buildColumn("gmt_modify", Types.DATE, "2011-01-01 11:11:11", false, false));
    rowBatch.merge(eventData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(rowBatch);
    RowBatch result = (RowBatch) batchs.get(EventData.class);
    want.number(result.getDatas().size()).isEqualTo(1);
}