Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 21 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class PipelineAction method doEdit.

public void doEdit(@FormGroup("pipelineInfo") Group pipelineInfo, @FormGroup("pipelineParameterInfo") Group pipelineParameterInfo, @FormField(name = "formPipelineError", group = "pipelineInfo") CustomErrors err, HttpSession session, Navigator nav) {
    Pipeline pipeline = new Pipeline();
    PipelineParameter parameters = new PipelineParameter();
    pipelineInfo.setProperties(pipeline);
    pipelineParameterInfo.setProperties(parameters);
    // if (parameters.getLoadPoolSize() < 1) {
    // parameters.setLoadPoolSize(PipelineParameter.DEFAULT_LOAD_POOL_SIZE);
    // }
    List<Long> selectNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    List<Node> selectNodes = new ArrayList<Node>();
    for (Long selectNodeId : selectNodeIds) {
        Node node = new Node();
        node.setId(selectNodeId);
        selectNodes.add(node);
    }
    // select/extract节点普遍配置为同一个节点
    List<Long> extractNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("selectNodeIds").getLongValues()));
    // List<Long> extractNodeIds =
    // Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("extractNodeIds").getLongValues()));
    List<Node> extractNodes = new ArrayList<Node>();
    for (Long extractNodeId : extractNodeIds) {
        Node node = new Node();
        node.setId(extractNodeId);
        extractNodes.add(node);
    }
    List<Long> loadNodeIds = Arrays.asList(ArrayUtils.toObject(pipelineInfo.getField("loadNodeIds").getLongValues()));
    List<Node> loadNodes = new ArrayList<Node>();
    for (Long loadNodeId : loadNodeIds) {
        Node node = new Node();
        node.setId(loadNodeId);
        loadNodes.add(node);
    }
    pipeline.setSelectNodes(selectNodes);
    pipeline.setExtractNodes(extractNodes);
    pipeline.setLoadNodes(loadNodes);
    pipeline.setParameters(parameters);
    List<Pipeline> values = pipelineService.listByDestinationWithoutOther(pipeline.getParameters().getDestinationName());
    if (!values.isEmpty()) {
        if (values.size() > 1 || !values.get(0).getId().equals(pipeline.getId())) {
            err.setMessage("invalidDestinationName");
            return;
        }
    }
    try {
        pipelineService.modify(pipeline);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidPipelineName");
        return;
    }
    nav.redirectToLocation("pipelineList.htm?channelId=" + pipeline.getChannelId());
}

Example 22 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class MessageParser method parse.

/**
 * 将对应canal送出来的Entry对象解析为otter使用的内部对象
 *
 * <pre>
 * 需要处理数据过滤：
 * 1. Transaction Begin/End过滤
 * 2. retl.retl_client/retl.retl_mark 回环标记处理以及后续的回环数据过滤
 * 3. retl.xdual canal心跳表数据过滤
 * </pre>
 */
public List<EventData> parse(Long pipelineId, List<Entry> datas) throws SelectException {
    List<EventData> eventDatas = new ArrayList<EventData>();
    Pipeline pipeline = configClientService.findPipeline(pipelineId);
    List<Entry> transactionDataBuffer = new ArrayList<Entry>();
    // hz为主站点，us->hz的数据，需要回环同步会us。并且需要开启回环补救算法
    PipelineParameter pipelineParameter = pipeline.getParameters();
    boolean enableLoopbackRemedy = pipelineParameter.isEnableRemedy() && pipelineParameter.isHome() && pipelineParameter.getRemedyAlgorithm().isLoopback();
    boolean isLoopback = false;
    // 判断是否属于需要loopback处理的类型，只处理正常otter同步产生的回环数据，因为会有业务方手工屏蔽同步的接口，避免回环
    boolean needLoopback = false;
    long now = new Date().getTime();
    try {
        for (Entry entry : datas) {
            switch(entry.getEntryType()) {
                case TRANSACTIONBEGIN:
                    isLoopback = false;
                    break;
                case ROWDATA:
                    String tableName = entry.getHeader().getTableName();
                    // 判断是否是回环表retl_mark
                    boolean isMarkTable = tableName.equalsIgnoreCase(pipeline.getParameters().getSystemMarkTable());
                    if (isMarkTable) {
                        RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                        if (!rowChange.getIsDdl()) {
                            int loopback = 0;
                            if (rowChange.getRowDatasCount() > 0) {
                                loopback = checkLoopback(pipeline, rowChange.getRowDatas(0));
                            }
                            if (loopback == 2) {
                                // 只处理正常同步产生的回环数据
                                needLoopback |= true;
                            }
                            isLoopback |= loopback > 0;
                        }
                    }
                    // 检查下otter3.0的回环表，对应的schmea会比较随意，所以不做比较
                    boolean isCompatibleLoopback = tableName.equalsIgnoreCase(compatibleMarkTable);
                    if (isCompatibleLoopback) {
                        RowChange rowChange = RowChange.parseFrom(entry.getStoreValue());
                        if (!rowChange.getIsDdl()) {
                            int loopback = 0;
                            if (rowChange.getRowDatasCount() > 0) {
                                loopback = checkCompatibleLoopback(pipeline, rowChange.getRowDatas(0));
                            }
                            if (loopback == 2) {
                                // 只处理正常同步产生的回环数据
                                needLoopback |= true;
                            }
                            isLoopback |= loopback > 0;
                        }
                    }
                    if ((!isLoopback || (enableLoopbackRemedy && needLoopback)) && !isMarkTable && !isCompatibleLoopback) {
                        transactionDataBuffer.add(entry);
                    }
                    break;
                case TRANSACTIONEND:
                    if (!isLoopback || (enableLoopbackRemedy && needLoopback)) {
                        // 添加数据解析
                        for (Entry bufferEntry : transactionDataBuffer) {
                            List<EventData> parseDatas = internParse(pipeline, bufferEntry);
                            if (CollectionUtils.isEmpty(parseDatas)) {
                                // 可能为空，针对ddl返回时就为null
                                continue;
                            }
                            // 初步计算一下事件大小
                            long totalSize = bufferEntry.getHeader().getEventLength();
                            long eachSize = totalSize / parseDatas.size();
                            for (EventData eventData : parseDatas) {
                                if (eventData == null) {
                                    continue;
                                }
                                // 记录一下大小
                                eventData.setSize(eachSize);
                                if (needLoopback) {
                                    // 如果延迟超过指定的阀值，则设置为需要反查db
                                    if (now - eventData.getExecuteTime() > 1000 * pipeline.getParameters().getRemedyDelayThresoldForMedia()) {
                                        eventData.setSyncConsistency(SyncConsistency.MEDIA);
                                    } else {
                                        eventData.setSyncConsistency(SyncConsistency.BASE);
                                    }
                                    eventData.setRemedy(true);
                                }
                                eventDatas.add(eventData);
                            }
                        }
                    }
                    isLoopback = false;
                    needLoopback = false;
                    transactionDataBuffer.clear();
                    break;
                default:
                    break;
            }
        }
        // 添加最后一次的数据，可能没有TRANSACTIONEND
        if (!isLoopback || (enableLoopbackRemedy && needLoopback)) {
            // 添加数据解析
            for (Entry bufferEntry : transactionDataBuffer) {
                List<EventData> parseDatas = internParse(pipeline, bufferEntry);
                if (CollectionUtils.isEmpty(parseDatas)) {
                    // 可能为空，针对ddl返回时就为null
                    continue;
                }
                // 初步计算一下事件大小
                long totalSize = bufferEntry.getHeader().getEventLength();
                long eachSize = totalSize / parseDatas.size();
                for (EventData eventData : parseDatas) {
                    if (eventData == null) {
                        continue;
                    }
                    // 记录一下大小
                    eventData.setSize(eachSize);
                    if (needLoopback) {
                        // 如果延迟超过指定的阀值，则设置为需要反查db
                        if (now - eventData.getExecuteTime() > 1000 * pipeline.getParameters().getRemedyDelayThresoldForMedia()) {
                            eventData.setSyncConsistency(SyncConsistency.MEDIA);
                        } else {
                            eventData.setSyncConsistency(SyncConsistency.BASE);
                        }
                    }
                    eventDatas.add(eventData);
                }
            }
        }
    } catch (Exception e) {
        throw new SelectException(e);
    }
    return eventDatas;
}

Example 23 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class DatabaseExtractorTest method test_global_row.

@Test
public void test_global_row() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    pipeline.getParameters().setSyncMode(SyncMode.ROW);
    // 设置为全局
    pipeline.getParameters().setSyncConsistency(SyncConsistency.MEDIA);
    int start = RandomUtils.nextInt();
    int count = 10;
    List<DataMediaPair> pairs = getDataMediaPairForMysql(start, count);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(100L);
            returns(pipeline);
        }
    };
    // 构造数据
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    for (int tableId = start; tableId < start + count; tableId++) {
        for (int i = start; i < start + count; i++) {
            EventData eventData = getEventData(tableId, i);
            eventData.setSchemaName("srf");
            eventData.setTableName("columns");
            rowBatch.merge(eventData);
        }
    }
    databaseExtractor.extract(new DbBatch(rowBatch));
    want.number(rowBatch.getDatas().size()).isEqualTo(count);
}

Example 24 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class FileLoadAction method adjustPoolSize.

private void adjustPoolSize(FileLoadContext context) {
    Pipeline pipeline = context.getPipeline();
    int newPoolSize = pipeline.getParameters().getFileLoadPoolSize();
    if (newPoolSize != poolSize) {
        poolSize = newPoolSize;
        if (executor instanceof ThreadPoolExecutor) {
            ThreadPoolExecutor pool = (ThreadPoolExecutor) executor;
            pool.setCorePoolSize(newPoolSize);
            pool.setMaximumPoolSize(newPoolSize);
        }
    }
}

Example 25 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class FileLoadAction method buildContext.

private FileLoadContext buildContext(Identity identity) {
    FileLoadContext context = new FileLoadContext();
    context.setIdentity(identity);
    Channel channel = configClientService.findChannel(identity.getChannelId());
    Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
    context.setChannel(channel);
    context.setPipeline(pipeline);
    return context;
}