Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 36 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineServiceImpl method listByChannelIdsWithoutColumn.

public List<Pipeline> listByChannelIdsWithoutColumn(Long... channelIds) {
    Assert.assertNotNull(channelIds);
    List<Pipeline> pipelines = new ArrayList<Pipeline>();
    try {
        List<PipelineDO> pipelineDos = pipelineDao.listByChannelIds(channelIds);
        if (pipelineDos.isEmpty()) {
            logger.debug("DEBUG ## query pipeline by channelId:" + channelIds + " return null.");
            return pipelines;
        }
        pipelines = doToModelWithoutColumn(pipelineDos);
    } catch (Exception e) {
        logger.error("ERROR ## query pipelines by channelIds:" + channelIds.toString() + " has an exception!");
        throw new ManagerException(e);
    }
    return pipelines;
}

Example 37 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineServiceImpl method listByChannelIds.

/**
 * 更具channelId找到所属所有pipeline
 */
public List<Pipeline> listByChannelIds(Long... channelId) {
    Assert.assertNotNull(channelId);
    List<Pipeline> pipelines = new ArrayList<Pipeline>();
    try {
        List<PipelineDO> pipelineDos = pipelineDao.listByChannelIds(channelId);
        if (pipelineDos.isEmpty()) {
            logger.debug("DEBUG ## query pipeline by channelId:" + channelId + " return null.");
            return pipelines;
        }
        pipelines = doToModel(pipelineDos);
    } catch (Exception e) {
        logger.error("ERROR ## query pipelines by channelIds:" + channelId.toString() + " has an exception!");
        throw new ManagerException(e);
    }
    return pipelines;
}

Example 38 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineServiceImpl method doToModelWithoutColumn.

private Pipeline doToModelWithoutColumn(PipelineDO pipelineDo) {
    Pipeline pipeline = new Pipeline();
    try {
        pipeline.setId(pipelineDo.getId());
        pipeline.setName(pipelineDo.getName());
        pipeline.setParameters(pipelineDo.getParameters());
        pipeline.setDescription(pipelineDo.getDescription());
        pipeline.setGmtCreate(pipelineDo.getGmtCreate());
        pipeline.setGmtModified(pipelineDo.getGmtModified());
        pipeline.setChannelId(pipelineDo.getChannelId());
        pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());
        // 组装DatamediaPair
        List<DataMediaPair> pairs = dataMediaPairService.listByPipelineIdWithoutColumn(pipelineDo.getId());
        Collections.sort(pairs, new DataMediaPairComparable());
        pipeline.setPairs(pairs);
        // 组装Node
        List<PipelineNodeRelationDO> relations = pipelineNodeRelationDao.listByPipelineIds(pipelineDo.getId());
        List<Long> totalNodeIds = new ArrayList<Long>();
        for (PipelineNodeRelationDO relation : relations) {
            Long nodeId = relation.getNodeId();
            if (!totalNodeIds.contains(nodeId)) {
                totalNodeIds.add(nodeId);
            }
        }
        List<Node> totalNodes = nodeService.listByIds(totalNodeIds.toArray(new Long[totalNodeIds.size()]));
        List<Node> selectNodes = new ArrayList<Node>();
        List<Node> extractNodes = new ArrayList<Node>();
        List<Node> loadNodes = new ArrayList<Node>();
        for (Node node : totalNodes) {
            for (PipelineNodeRelationDO relation : relations) {
                if (node.getId().equals(relation.getNodeId())) {
                    if (relation.getLocation().isSelect()) {
                        selectNodes.add(node);
                    } else if (relation.getLocation().isExtract()) {
                        extractNodes.add(node);
                    } else if (relation.getLocation().isLoad()) {
                        loadNodes.add(node);
                    }
                }
            }
        }
        pipeline.setSelectNodes(selectNodes);
        pipeline.setExtractNodes(extractNodes);
        pipeline.setLoadNodes(loadNodes);
    } catch (Exception e) {
        logger.error("ERROR ## change the pipeline Do to Model has an exception");
        throw new ManagerException(e);
    }
    return pipeline;
}

Example 39 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineServiceImpl method doToModel.

/**
 * 用于DO对象转化为Model对象
 *
 * @param pipelineDO
 * @return Pipeline
 */
private Pipeline doToModel(PipelineDO pipelineDo) {
    Pipeline pipeline = new Pipeline();
    try {
        pipeline.setId(pipelineDo.getId());
        pipeline.setName(pipelineDo.getName());
        pipeline.setParameters(pipelineDo.getParameters());
        pipeline.setDescription(pipelineDo.getDescription());
        pipeline.setGmtCreate(pipelineDo.getGmtCreate());
        pipeline.setGmtModified(pipelineDo.getGmtModified());
        pipeline.setChannelId(pipelineDo.getChannelId());
        pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());
        // 组装DatamediaPair
        List<DataMediaPair> pairs = dataMediaPairService.listByPipelineId(pipelineDo.getId());
        Collections.sort(pairs, new DataMediaPairComparable());
        pipeline.setPairs(pairs);
        // 组装Node
        List<PipelineNodeRelationDO> relations = pipelineNodeRelationDao.listByPipelineIds(pipelineDo.getId());
        List<Long> totalNodeIds = new ArrayList<Long>();
        for (PipelineNodeRelationDO relation : relations) {
            Long nodeId = relation.getNodeId();
            if (!totalNodeIds.contains(nodeId)) {
                totalNodeIds.add(nodeId);
            }
        }
        List<Node> totalNodes = nodeService.listByIds(totalNodeIds.toArray(new Long[totalNodeIds.size()]));
        List<Node> selectNodes = new ArrayList<Node>();
        List<Node> extractNodes = new ArrayList<Node>();
        List<Node> loadNodes = new ArrayList<Node>();
        for (Node node : totalNodes) {
            for (PipelineNodeRelationDO relation : relations) {
                if (node.getId().equals(relation.getNodeId())) {
                    if (relation.getLocation().isSelect()) {
                        selectNodes.add(node);
                    } else if (relation.getLocation().isExtract()) {
                        extractNodes.add(node);
                    } else if (relation.getLocation().isLoad()) {
                        loadNodes.add(node);
                    }
                }
            }
        }
        pipeline.setSelectNodes(selectNodes);
        pipeline.setExtractNodes(extractNodes);
        pipeline.setLoadNodes(loadNodes);
    } catch (Exception e) {
        logger.error("ERROR ## change the pipeline Do to Model has an exception");
        throw new ManagerException(e);
    }
    return pipeline;
}

Example 40 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineServiceImpl method doToModelWithoutOther.

private Pipeline doToModelWithoutOther(PipelineDO pipelineDo) {
    Pipeline pipeline = new Pipeline();
    try {
        pipeline.setId(pipelineDo.getId());
        pipeline.setName(pipelineDo.getName());
        pipeline.setParameters(pipelineDo.getParameters());
        pipeline.setDescription(pipelineDo.getDescription());
        pipeline.setGmtCreate(pipelineDo.getGmtCreate());
        pipeline.setGmtModified(pipelineDo.getGmtModified());
        pipeline.setChannelId(pipelineDo.getChannelId());
        pipeline.getParameters().setMainstemClientId(pipeline.getId().shortValue());
    } catch (Exception e) {
        logger.error("ERROR ## change the pipeline Do to Model has an exception");
        throw new ManagerException(e);
    }
    return pipeline;
}